<commit_before>import time
import os
import json
import math
import urllib
import urllib2
import httplib

import filechunkio
from boto.s3.connection import S3Connection, OrdinaryCallingFormat
import boto
import jsonpatch

# Item class
#_________________________________________________________________________________________
class Item(object):
    """This class represents an archive.org item.
    You can use this class to access item metadata:
        >>> import internetarchive
        >>> item = internetarchive.Item('stairs')
        >>> print item.metadata

    This class also uses IA's S3-like interface to upload files to an item. You need to
    supply your IAS3 credentials in environment variables in order to upload. You can
    retrieve S3 keys from https://archive.org/account/s3.php
        >>> import os;
        >>> os.environ['AWS_ACCESS_KEY_ID']='x'; os.environ['AWS_SECRET_ACCESS_KEY']='y'
        >>> item.upload('myfile')
        True
    """

    # init()
    #_____________________________________________________________________________________
    def __init__(self, identifier):
        self.identifier = identifier
        self.details_url = 'https://archive.org/details/{0}'.format(identifier)
        self.download_url = 'https://archive.org/download/{0}'.format(identifier)
        self.metadata_url = 'https://archive.org/metadata/{0}'.format(identifier)
        self._s3_conn = None
        self.metadata = self._get_item_metadata()
        if self.metadata == {}:
            self.exists = False
        else:
            self.exists = True

    # _get_item_metadata()
    #_____________________________________________________________________________________
    def _get_item_metadata(self):
        f = urllib.urlopen(self.metadata_url)
        return json.loads(f.read())

    # files()
    #_____________________________________________________________________________________
    def files(self):
        """Generator for iterating over files in an item"""
        for file_dict in self.metadata['files']:
            file = File(self, file_dict)
            yield file

    # file()
    #_____________________________________________________________________________________
    def file(self, name):
        """Return an archive.org File object for the named file.
        If the specified file was not found in the item, return None
        """
        for file_dict in self.metadata['files']:
            if file_dict['name'] == name:
                return File(self, file_dict)
        return None

    # download()
    #_____________________________________________________________________________________
    def download(self, formats=None):
        """Download the entire item into the current working directory"""
        for f in self.files():
            if type(formats) == str:
                formats = [formats]
            if formats is not None and f.format not in formats:
                continue
            print '  downloading', f.name
            path = os.path.join(self.identifier, f.name)
            parent_dir = os.path.dirname(path)
            f.download(path)

    # modify_metadata()
    #_____________________________________________________________________________________
    def modify_metadata(self, metadata, target='metadata'):
        """function for modifying the metadata of an existing item on archive.org.
        Note: The Metadata Write API does not yet comply with the latest Json-Patch
        standard. It currently complies with version 02:

        https://tools.ietf.org/html/draft-ietf-appsawg-json-patch-02

        :param metadata: Dictionary. Metadata used to update the item.
        :param target: (optional) String. Metadata target to update.

        Usage:

        >>> import internetarchive
        >>> item = internetarchive.Item('mapi_test_item1')
        >>> md = dict(new_key='new_value', foo=['bar', 'bar2'])
        >>> item.modify_metadata(md)

        """
        access_key = os.environ['AWS_ACCESS_KEY_ID']
        secret_key = os.environ['AWS_SECRET_ACCESS_KEY']
        src = self.metadata.get(target, {})
        dest = dict((src.items() + metadata.items()))

        # Prepare patch to remove metadata elements with the value: "REMOVE_TAG".
        for k,v in metadata.items():
            if v == 'REMOVE_TAG' or not v:
                del dest[k]

        json_patch = jsonpatch.make_patch(src, dest).patch
        # Reformat patch to be compliant with version 02 of the Json-Patch standard.
        patch = []
        for p in json_patch:
            pd = {p['op']: p['path']}
            if p['op'] != 'remove':
                pd['value'] = p['value']
            patch.append(dict((k,v) for k,v in pd.items() if v))

        data = {
            '-patch': json.dumps(patch),
            '-target': target,
            'access': access_key,
            'secret': secret_key,
        }

        host = 'archive.org'
        path = '/metadata/{0}'.format(self.identifier)
        http = httplib.HTTP(host)
        http.putrequest("POST", path)
        http.putheader("Host", host)
        data = urllib.urlencode(data)
        http.putheader("Content-Type", 'application/x-www-form-urlencoded')
        http.putheader("Content-Length", str(len(data)))
        http.endheaders()
        http.send(data)
        status_code, error_message, headers = http.getreply()
        resp_file = http.getfile()
        self.metadata = self._get_item_metadata()
        return dict(
            status_code = status_code,
            content = json.loads(resp_file.read()),
        )

    # _get_s3_conn()
    #_____________________________________________________________________________________
    def _get_s3_conn(self):
        if self._s3_conn is None:
            self._s3_conn = S3Connection(host='s3.us.archive.org', is_secure=False,
                                         calling_format=OrdinaryCallingFormat())
        return self._s3_conn

    # _get_s3_bucket()
    #_____________________________________________________________________________________
    def _get_s3_bucket(self, conn, headers={}, ignore_bucket=False):
        if ignore_bucket is True:
            bucket = None
        else:
            bucket = conn.lookup(self.identifier)
        if bucket:
            return bucket
        headers['x-archive-queue-derive'] = 0
        bucket = conn.create_bucket(self.identifier, headers=headers)
        i=0
        while i<60:
            b = conn.lookup(self.identifier)
            if b:
                return bucket
            time.sleep(10)
            i+=1
        raise NameError('Could not create or lookup %s' % self.identifier)

    # upload()
    #_____________________________________________________________________________________
    def upload(self, files, meta_dict={}, headers={}, dry_run=False,
               derive=True, multipart=False, ignore_bucket=False):
        """Upload file(s) to an item. The item will be created if it does not
        exist.

        :param files: Either a list of filepaths, or a string pointing to a single file.
        :param meta_dict: Dictionary of metadata used to create a new item.
        :param dry_run: (optional) Boolean. Set to True to print headers to stdout -- don't upload anything.
        :param derive: (optional) Boolean. Set to False to prevent an item from being derived after upload.
        :param multipart: (optional) Boolean. Set to True to upload files in parts. Useful when uploading large files.
        :param ignore_bucket: (optional) Boolean. Set to True to ignore and clobber existing files and metadata.

        Usage::

            >>> import internetarchive
            >>> item = internetarchive.Item('identifier')
            >>> item.upload('/path/to/image.jpg', dict(mediatype='image', creator='Jake Johnson'))
        """
        # Convert metadata from :meta_dict: into S3 headers
        for key,v in meta_dict.iteritems():
            if type(v) == list:
                for i, value in enumerate(v):
                    s3_header_key = 'x-archive-meta{0:02d}-{1}'.format(i, key)
                    if type(value) == str:
                        headers[s3_header_key] = value.encode('utf-8')
                    else:
                        headers[s3_header_key] = value
            else:
                s3_header_key = 'x-archive-meta-%s' % key
                if type(v) == str:
                    headers[s3_header_key] = v.encode('utf-8')
                else:
                    headers[s3_header_key] = v
        headers = {k: v for k,v in headers.iteritems() if v}

        if dry_run:
            return headers

        if type(files) != list:
            files = [files]
        for file in files:
            filename = file.split('/')[-1]
            conn = self._get_s3_conn()
            bucket = self._get_s3_bucket(conn, headers, ignore_bucket=ignore_bucket)
            #headers['x-archive-ignore-preexisting-bucket'] = 1

            if bucket.get_key(filename):
                continue

            if not derive:
                headers = {'x-archive-queue-derive': 0}

            if not multipart:
                k = boto.s3.key.Key(bucket)
                k.name = filename
                k.set_contents_from_filename(file, headers=headers)
            else:
                mp = bucket.initiate_multipart_upload(filename, headers=headers)
                source_size = os.stat(file).st_size
                headers['x-archive-size-hint'] = source_size
                bytes_per_chunk = (max(int(math.sqrt(5242880) *
                                           math.sqrt(source_size)), 5242880))
                chunk_amount = (int(math.ceil(source_size/float(bytes_per_chunk))))
                for i in range(chunk_amount):
                    offset = i * bytes_per_chunk
                    remaining_bytes = source_size - offset
                    part_bytes = min([bytes_per_chunk, remaining_bytes])
                    part_num = i + 1
                    with filechunkio.FileChunkIO(file, 'r',
                                                 offset=offset,
                                                 bytes=part_bytes) as f:
                        mp.upload_part_from_file(fp=f, part_num=part_num)
                if len(mp.get_all_parts()) == chunk_amount:
                    mp.complete_upload()
                    key = bucket.get_key(filename)
                else:
                    mp.cancel_upload()
        return True

# File class
#_________________________________________________________________________________________
class File(object):

    # init()
    #_____________________________________________________________________________________
    def __init__(self, item, file_dict):
        def get(d, key):
            if key in d:
                return d[key]
            else:
                return None

        self.item = item
        self.name = file_dict['name']
        self.md5  = file_dict['md5']
        self.sha1 = get(file_dict, 'sha1')
        self.size = get(file_dict, 'size')
        if self.size is not None:
            self.size = int(self.size)
        self.format = file_dict['format']

    # download()
    #_____________________________________________________________________________________
    def download(self, file_path=None):
        if file_path is None:
            file_path = self.name

        if os.path.exists(file_path):
            raise IOError('File already exists: %s' % file_path)

        parent_dir = os.path.dirname(file_path)
        if parent_dir != '' and not os.path.exists(parent_dir):
            os.makedirs(parent_dir)

        url = 'https://archive.org/download/%s/%s' % (self.item.identifier, self.name)
        urllib.urlretrieve(url, file_path)

# Catalog class
#_________________________________________________________________________________________
class Catalog(object):

    # init()
    #_____________________________________________________________________________________
    def __init__(self, params=None):
        url = 'http://www.us.archive.org/catalog.php'

        if not params:
            params = {'justme': 1}

        # Add params required to retrieve JSONP from the IA catalog.
        params['json'] = 2
        params['output'] = 'json'
        params['callback'] = 'foo'
        params = urllib.urlencode(params)

        ia_cookies = ('logged-in-sig={LOGGED_IN_SIG}; '
                      'logged-in-user={LOGGED_IN_USER}; '
                      'verbose=1'.format(**os.environ))

        opener = urllib2.build_opener()
        opener.addheaders.append(('Cookie', ia_cookies))
        f = opener.open(url, params)

        # Hack to convert JSONP to JSON (then parse the JSON).
        jsonp_str = f.read()
        json_str = jsonp_str[(jsonp_str.index("(") + 1):jsonp_str.rindex(")")]

        self.tasks_json = json.loads(json_str)
        self.tasks = []
        for t in self.tasks_json:
            td = {}
            td['identifier'] = t[0]
            td['server'] = t[1]
            td['command'] = t[2]
            td['time'] = t[3]
            td['submitter'] = t[4]
            td['args'] = t[5]
            td['task_id'] = t[6]
            td['type'] = t[7]
            self.tasks.append(td)
        self.green_rows = [x for x in self.tasks if x['type'] == 0]
        self.blue_rows = [x for x in self.tasks if x['type'] == 1]
        self.red_rows = [x for x in self.tasks if x['type'] == 2]
        self.brown_rows = [x for x in self.tasks if x['type'] == 9]
<commit_msg>Use new style string formatting, rather than sprintf style<commit_after>import time
import os
import json
import math
import urllib
import urllib2
import httplib

import filechunkio
from boto.s3.connection import S3Connection, OrdinaryCallingFormat
import boto
import jsonpatch

# Item class
#_________________________________________________________________________________________
class Item(object):
    """This class represents an archive.org item.
    You can use this class to access item metadata:
        >>> import internetarchive
        >>> item = internetarchive.Item('stairs')
        >>> print item.metadata

    This class also uses IA's S3-like interface to upload files to an item. You need to
    supply your IAS3 credentials in environment variables in order to upload. You can
    retrieve S3 keys from https://archive.org/account/s3.php
        >>> import os;
        >>> os.environ['AWS_ACCESS_KEY_ID']='x'; os.environ['AWS_SECRET_ACCESS_KEY']='y'
        >>> item.upload('myfile')
        True
    """

    # init()
    #_____________________________________________________________________________________
    def __init__(self, identifier):
        self.identifier = identifier
        self.details_url = 'https://archive.org/details/{0}'.format(identifier)
        self.download_url = 'https://archive.org/download/{0}'.format(identifier)
        self.metadata_url = 'https://archive.org/metadata/{0}'.format(identifier)
        self._s3_conn = None
        self.metadata = self._get_item_metadata()
        if self.metadata == {}:
            self.exists = False
        else:
            self.exists = True

    # _get_item_metadata()
    #_____________________________________________________________________________________
    def _get_item_metadata(self):
        f = urllib.urlopen(self.metadata_url)
        return json.loads(f.read())

    # files()
    #_____________________________________________________________________________________
    def files(self):
        """Generator for iterating over files in an item"""
        for file_dict in self.metadata['files']:
            file = File(self, file_dict)
            yield file

    # file()
    #_____________________________________________________________________________________
    def file(self, name):
        """Return an archive.org File object for the named file.
        If the specified file was not found in the item, return None
        """
        for file_dict in self.metadata['files']:
            if file_dict['name'] == name:
                return File(self, file_dict)
        return None

    # download()
    #_____________________________________________________________________________________
    def download(self, formats=None):
        """Download the entire item into the current working directory"""
        for f in self.files():
            if type(formats) == str:
                formats = [formats]
            if formats is not None and f.format not in formats:
                continue
            print '  downloading', f.name
            path = os.path.join(self.identifier, f.name)
            parent_dir = os.path.dirname(path)
            f.download(path)

    # modify_metadata()
    #_____________________________________________________________________________________
    def modify_metadata(self, metadata, target='metadata'):
        """function for modifying the metadata of an existing item on archive.org.
        Note: The Metadata Write API does not yet comply with the latest Json-Patch
        standard. It currently complies with version 02:

        https://tools.ietf.org/html/draft-ietf-appsawg-json-patch-02

        :param metadata: Dictionary. Metadata used to update the item.
        :param target: (optional) String. Metadata target to update.

        Usage:

        >>> import internetarchive
        >>> item = internetarchive.Item('mapi_test_item1')
        >>> md = dict(new_key='new_value', foo=['bar', 'bar2'])
        >>> item.modify_metadata(md)

        """
        access_key = os.environ['AWS_ACCESS_KEY_ID']
        secret_key = os.environ['AWS_SECRET_ACCESS_KEY']
        src = self.metadata.get(target, {})
        dest = dict((src.items() + metadata.items()))

        # Prepare patch to remove metadata elements with the value: "REMOVE_TAG".
        for k,v in metadata.items():
            if v == 'REMOVE_TAG' or not v:
                del dest[k]

        json_patch = jsonpatch.make_patch(src, dest).patch
        # Reformat patch to be compliant with version 02 of the Json-Patch standard.
        patch = []
        for p in json_patch:
            pd = {p['op']: p['path']}
            if p['op'] != 'remove':
                pd['value'] = p['value']
            patch.append(dict((k,v) for k,v in pd.items() if v))

        data = {
            '-patch': json.dumps(patch),
            '-target': target,
            'access': access_key,
            'secret': secret_key,
        }

        host = 'archive.org'
        path = '/metadata/{0}'.format(self.identifier)
        http = httplib.HTTP(host)
        http.putrequest("POST", path)
        http.putheader("Host", host)
        data = urllib.urlencode(data)
        http.putheader("Content-Type", 'application/x-www-form-urlencoded')
        http.putheader("Content-Length", str(len(data)))
        http.endheaders()
        http.send(data)
        status_code, error_message, headers = http.getreply()
        resp_file = http.getfile()
        self.metadata = self._get_item_metadata()
        return dict(
            status_code = status_code,
            content = json.loads(resp_file.read()),
        )

    # _get_s3_conn()
    #_____________________________________________________________________________________
    def _get_s3_conn(self):
        if self._s3_conn is None:
            self._s3_conn = S3Connection(host='s3.us.archive.org', is_secure=False,
                                         calling_format=OrdinaryCallingFormat())
        return self._s3_conn

    # _get_s3_bucket()
    #_____________________________________________________________________________________
    def _get_s3_bucket(self, conn, headers={}, ignore_bucket=False):
        if ignore_bucket is True:
            bucket = None
        else:
            bucket = conn.lookup(self.identifier)
        if bucket:
            return bucket
        headers['x-archive-queue-derive'] = 0
        bucket = conn.create_bucket(self.identifier, headers=headers)
        i=0
        while i<60:
            b = conn.lookup(self.identifier)
            if b:
                return bucket
            time.sleep(10)
            i+=1
        raise NameError('Could not create or lookup {0}'.format(self.identifier))

    # upload()
    #_____________________________________________________________________________________
    def upload(self, files, meta_dict={}, headers={}, dry_run=False,
               derive=True, multipart=False, ignore_bucket=False):
        """Upload file(s) to an item. The item will be created if it does not
        exist.

        :param files: Either a list of filepaths, or a string pointing to a single file.
        :param meta_dict: Dictionary of metadata used to create a new item.
        :param dry_run: (optional) Boolean. Set to True to print headers to stdout -- don't upload anything.
        :param derive: (optional) Boolean. Set to False to prevent an item from being derived after upload.
        :param multipart: (optional) Boolean. Set to True to upload files in parts. Useful when uploading large files.
        :param ignore_bucket: (optional) Boolean. Set to True to ignore and clobber existing files and metadata.

        Usage::

            >>> import internetarchive
            >>> item = internetarchive.Item('identifier')
            >>> item.upload('/path/to/image.jpg', dict(mediatype='image', creator='Jake Johnson'))
        """
        # Convert metadata from :meta_dict: into S3 headers
        for key,v in meta_dict.iteritems():
            if type(v) == list:
                for i, value in enumerate(v):
                    s3_header_key = 'x-archive-meta{0:02d}-{1}'.format(i, key)
                    if type(value) == str:
                        headers[s3_header_key] = value.encode('utf-8')
                    else:
                        headers[s3_header_key] = value
            else:
                s3_header_key = 'x-archive-meta-{0}'.format(key)
                if type(v) == str:
                    headers[s3_header_key] = v.encode('utf-8')
                else:
                    headers[s3_header_key] = v
        headers = {k: v for k,v in headers.iteritems() if v}

        if dry_run:
            return headers

        if type(files) != list:
            files = [files]
        for file in files:
            filename = file.split('/')[-1]
            conn = self._get_s3_conn()
            bucket = self._get_s3_bucket(conn, headers, ignore_bucket=ignore_bucket)
            #headers['x-archive-ignore-preexisting-bucket'] = 1

            if bucket.get_key(filename):
                continue

            if not derive:
                headers = {'x-archive-queue-derive': 0}

            if not multipart:
                k = boto.s3.key.Key(bucket)
                k.name = filename
                k.set_contents_from_filename(file, headers=headers)
            else:
                mp = bucket.initiate_multipart_upload(filename, headers=headers)
                source_size = os.stat(file).st_size
                headers['x-archive-size-hint'] = source_size
                bytes_per_chunk = (max(int(math.sqrt(5242880) *
                                           math.sqrt(source_size)), 5242880))
                chunk_amount = (int(math.ceil(source_size/float(bytes_per_chunk))))
                for i in range(chunk_amount):
                    offset = i * bytes_per_chunk
                    remaining_bytes = source_size - offset
                    part_bytes = min([bytes_per_chunk, remaining_bytes])
                    part_num = i + 1
                    with filechunkio.FileChunkIO(file, 'r',
                                                 offset=offset,
                                                 bytes=part_bytes) as f:
                        mp.upload_part_from_file(fp=f, part_num=part_num)
                if len(mp.get_all_parts()) == chunk_amount:
                    mp.complete_upload()
                    key = bucket.get_key(filename)
                else:
                    mp.cancel_upload()
        return True

# File class
#_________________________________________________________________________________________
class File(object):

    # init()
    #_____________________________________________________________________________________
    def __init__(self, item, file_dict):
        def get(d, key):
            if key in d:
                return d[key]
            else:
                return None

        self.item = item
        self.name = file_dict['name']
        self.md5  = file_dict['md5']
        self.sha1 = get(file_dict, 'sha1')
        self.size = get(file_dict, 'size')
        if self.size is not None:
            self.size = int(self.size)
        self.format = file_dict['format']

    # download()
    #_____________________________________________________________________________________
    def download(self, file_path=None):
        if file_path is None:
            file_path = self.name

        if os.path.exists(file_path):
            raise IOError('File already exists: {0}'.format(file_path))

        parent_dir = os.path.dirname(file_path)
        if parent_dir != '' and not os.path.exists(parent_dir):
            os.makedirs(parent_dir)

        url = 'https://archive.org/download/{0}/{1}'.format(self.item.identifier,
                                                            self.name)
        urllib.urlretrieve(url, file_path)

# Catalog class
#_________________________________________________________________________________________
class Catalog(object):

    # init()
    #_____________________________________________________________________________________
    def __init__(self, params=None):
        url = 'http://www.us.archive.org/catalog.php'

        if not params:
            params = {'justme': 1}

        # Add params required to retrieve JSONP from the IA catalog.
        params['json'] = 2
        params['output'] = 'json'
        params['callback'] = 'foo'
        params = urllib.urlencode(params)

        ia_cookies = ('logged-in-sig={LOGGED_IN_SIG}; '
                      'logged-in-user={LOGGED_IN_USER}; '
                      'verbose=1'.format(**os.environ))

        opener = urllib2.build_opener()
        opener.addheaders.append(('Cookie', ia_cookies))
        f = opener.open(url, params)

        # Hack to convert JSONP to JSON (then parse the JSON).
        jsonp_str = f.read()
        json_str = jsonp_str[(jsonp_str.index("(") + 1):jsonp_str.rindex(")")]

        self.tasks_json = json.loads(json_str)
        self.tasks = []
        for t in self.tasks_json:
            td = {}
            td['identifier'] = t[0]
            td['server'] = t[1]
            td['command'] = t[2]
            td['time'] = t[3]
            td['submitter'] = t[4]
            td['args'] = t[5]
            td['task_id'] = t[6]
            td['type'] = t[7]
            self.tasks.append(td)
        self.green_rows = [x for x in self.tasks if x['type'] == 0]
        self.blue_rows = [x for x in self.tasks if x['type'] == 1]
        self.red_rows = [x for x in self.tasks if x['type'] == 2]
        self.brown_rows = [x for x in self.tasks if x['type'] == 9]
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
from django.conf import settings
from south.utils import datetime_utils as datetime
from south.db import db
from south.v2 import DataMigration

from ... import es

class Migration(DataMigration):
    depends_on = (
        ('feeds', '0016_auto__add_index_feed_url'),
    )

    def forwards(self, orm):
        "Write your forwards methods here."
        # Note: Don't use "from appname.models import ModelName".
        # Use orm.ModelName to refer to models in this application,
        # and orm['appname.ModelName'] for models in other applications.
        for user in orm.User.objects.exclude(es=True):
            name = es.user_alias(user.pk)
            es.client.indices.put_alias(
                index=settings.ES_INDEX,
                name=name,
                body={
                    'routing': user.pk,
                    'filter': {'term': {'user': user.pk}},
                },
            )
            for feed in user.feeds.all():
                entries = feed.entries.all()
                docs = [doc.serialize() for doc in entries]
                if not docs:
                    continue
                es.bulk(docs, index=name, timeout=60, raise_on_error=True)
            user.es = True
            user.save(update_fields=['es'])
        db.execute('alter sequence feeds_entry_id_seq owned by none')
        db.execute('truncate feeds_entry')

    def backwards(self, orm):
        "Write your backwards methods here."

    models = {
        u'auth.group': {
            'Meta': {'object_name': 'Group'},
            u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
            'name': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '80'}),
            'permissions': ('django.db.models.fields.related.ManyToManyField', [], {'to': u"orm['auth.Permission']", 'symmetrical': 'False', 'blank': 'True'})
        },
        u'auth.permission': {
            'Meta': {'ordering': "(u'content_type__app_label', u'content_type__model', u'codename')", 'unique_together': "((u'content_type', u'codename'),)", 'object_name': 'Permission'},
            'codename': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
            'content_type': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['contenttypes.ContentType']"}),
            u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
            'name': ('django.db.models.fields.CharField', [], {'max_length': '50'})
        },
        u'contenttypes.contenttype': {
            'Meta': {'ordering': "('name',)", 'unique_together': "(('app_label', 'model'),)", 'object_name': 'ContentType', 'db_table': "'django_content_type'"},
            'app_label': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
            u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
            'model': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
            'name': ('django.db.models.fields.CharField', [], {'max_length': '100'})
        },
        u'feeds.category': {
            'Meta': {'ordering': "('order', 'name', 'id')", 'unique_together': "(('user', 'slug'), ('user', 'name'))", 'object_name': 'Category'},
            'color': ('django.db.models.fields.CharField', [], {'default': "'blue'", 'max_length': '50'}),
            u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
            'name': ('django.db.models.fields.CharField', [], {'max_length': '1023', 'db_index': 'True'}),
            'order': ('django.db.models.fields.PositiveIntegerField', [], {'null': 'True', 'blank': 'True'}),
            'slug': ('django.db.models.fields.SlugField', [], {'max_length': '50'}),
            'user': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'categories'", 'to': u"orm['profiles.User']"})
        },
        u'feeds.entry': {
            'Meta': {'ordering': "('-date', '-id')", 'object_name': 'Entry', 'index_together': "(('user', 'date'), ('user', 'read'), ('user', 'starred'), ('user', 'broadcast'))"},
            'author': ('django.db.models.fields.CharField', [], {'max_length': '1023', 'blank': 'True'}),
            'broadcast': ('django.db.models.fields.BooleanField', [], {'default': 'False', 'db_index': 'True'}),
            'date': ('django.db.models.fields.DateTimeField', [], {'db_index': 'True'}),
            'feed': ('django.db.models.fields.related.ForeignKey', [], {'blank': 'True', 'related_name': "'entries'", 'null': 'True', 'to': u"orm['feeds.Feed']"}),
            'guid': ('feedhq.feeds.fields.URLField', [], {'db_index': 'True', 'blank': 'True'}),
            u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
            'link': ('feedhq.feeds.fields.URLField', [], {'db_index': 'True'}),
            'read': ('django.db.models.fields.BooleanField', [], {'default': 'False', 'db_index': 'True'}),
            'read_later_url': ('feedhq.feeds.fields.URLField', [], {'blank': 'True'}),
            'starred': ('django.db.models.fields.BooleanField', [], {'default': 'False', 'db_index': 'True'}),
            'subtitle': ('django.db.models.fields.TextField', [], {}),
            'title': ('django.db.models.fields.CharField', [], {'max_length': '255'}),
            'user': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'entries'", 'to': u"orm['profiles.User']"})
        },
        u'feeds.favicon': {
            'Meta': {'object_name': 'Favicon'},
            'favicon': ('django.db.models.fields.files.FileField', [], {'max_length': '100', 'blank': 'True'}),
            u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
            'url': ('feedhq.feeds.fields.URLField', [], {'unique': 'True', 'db_index': 'True'})
        },
        u'feeds.feed': {
            'Meta': {'ordering': "('name',)", 'object_name': 'Feed'},
            'category': ('django.db.models.fields.related.ForeignKey', [], {'blank': 'True', 'related_name': "'feeds'", 'null': 'True', 'to': u"orm['feeds.Category']"}),
            'favicon': ('django.db.models.fields.files.ImageField', [], {'max_length': '100', 'null': 'True', 'blank': 'True'}),
            u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
            'img_safe': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
            'name': ('django.db.models.fields.CharField', [], {'max_length': '1023'}),
            'unread_count': ('django.db.models.fields.PositiveIntegerField', [], {'default': '0'}),
            'url': ('feedhq.feeds.fields.URLField', [], {'db_index': 'True'}),
            'user': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'feeds'", 'to': u"orm['profiles.User']"})
        },
        u'feeds.uniquefeed': {
            'Meta': {'object_name': 'UniqueFeed'},
            'error': ('django.db.models.fields.CharField', [], {'max_length': '50', 'null': 'True', 'db_column': "'muted_reason'", 'blank': 'True'}),
            u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
            'muted': ('django.db.models.fields.BooleanField', [], {'default': 'False', 'db_index': 'True'}),
            'url': ('feedhq.feeds.fields.URLField', [], {'unique': 'True'})
        },
        u'profiles.user': {
            'Meta': {'object_name': 'User', 'db_table': "'auth_user'"},
            'allow_media': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
            'date_joined': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now'}),
            'email': ('django.db.models.fields.CharField', [], {'max_length': '75'}),
            'endless_pages': ('django.db.models.fields.BooleanField', [], {'default': 'True'}),
            'entries_per_page': ('django.db.models.fields.IntegerField', [], {'default': '50'}),
            'es': ('django.db.models.fields.NullBooleanField', [], {'default': 'True', 'null': 'True', 'blank': 'True'}),
            'first_name': ('django.db.models.fields.CharField', [], {'max_length': '30'}),
            'font': ('django.db.models.fields.CharField', [], {'default': "'pt-serif'", 'max_length': '75'}),
            'groups': ('django.db.models.fields.related.ManyToManyField', [], {'symmetrical': 'False', 'related_name': "u'user_set'", 'blank': 'True', 'to': u"orm['auth.Group']"}),
            u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
            'is_active': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
            'is_staff': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
            'is_superuser': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
            'is_suspended': ('django.db.models.fields.BooleanField', [], {'default': 'False', 'db_index': 'True'}),
            'last_login': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now'}),
            'last_name': ('django.db.models.fields.CharField', [], {'max_length': '30'}),
            'oldest_first': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
            'password': ('django.db.models.fields.CharField', [], {'max_length': '128'}),
            'read_later': ('django.db.models.fields.CharField', [], {'max_length': '50', 'blank': 'True'}),
            'read_later_credentials': ('django.db.models.fields.TextField', [], {'blank': 'True'}),
            'sharing_email': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
            'sharing_gplus': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
            'sharing_twitter': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
            'timezone': ('django.db.models.fields.CharField', [], {'default': "'UTC'", 'max_length': '75'}),
            'ttl': ('django.db.models.fields.PositiveIntegerField', [], {'default': '30'}),
            'user_permissions': ('django.db.models.fields.related.ManyToManyField', [], {'symmetrical': 'False', 'related_name': "u'user_set'", 'blank': 'True', 'to': u"orm['auth.Permission']"}),
            'username': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '75'})
        }
    }

    complete_apps = ['feeds']
    symmetrical = True
<commit_msg>profiles: fix migration using API from the Django model<commit_after># -*- coding: utf-8 -*-
from django.conf import settings
from south.utils import datetime_utils as datetime
from south.db import db
from south.v2 import DataMigration

from ... import es

def serialize(entry):
    data = {
        '_type': 'entries',
        '_id': entry.pk,
        'id': entry.pk,
        'timestamp': entry.date,
        'title': entry.title,
        'raw_title': entry.title,
        'author': entry.author,
        'content': entry.subtitle,
        'link': entry.link,
        'guid': entry.guid or entry.link or entry.title,
        'read': entry.read,
        'read_later_url': entry.read_later_url,
        'starred': entry.starred,
        'broadcast': entry.broadcast,
        'tags': [],
    }
    if entry.user_id:
        data['user'] = entry.user_id
    if entry.feed_id:
        data['feed'] = entry.feed_id
    if entry.feed and entry.feed.category_id:
        data['category'] = entry.feed.category_id
    return data

class Migration(DataMigration):
    depends_on = (
        ('feeds', '0016_auto__add_index_feed_url'),
    )

    def forwards(self, orm):
        "Write your forwards methods here."
        # Note: Don't use "from appname.models import ModelName".
        # Use orm.ModelName to refer to models in this application,
        # and orm['appname.ModelName'] for models in other applications.
        for user in orm.User.objects.exclude(es=True):
            name = es.user_alias(user.pk)
            es.client.indices.put_alias(
                index=settings.ES_INDEX,
                name=name,
                body={
                    'routing': user.pk,
                    'filter': {'term': {'user': user.pk}},
                },
            )
            for feed in user.feeds.all():
                entries = feed.entries.all()
                docs = [serialize(doc) for doc in entries]
                if not docs:
                    continue
                es.bulk(docs, index=name, timeout=60, raise_on_error=True)
            user.es = True
            user.save(update_fields=['es'])
        db.execute('alter sequence feeds_entry_id_seq owned by none')
        db.execute('truncate feeds_entry')

    def backwards(self, orm):
        "Write your backwards methods here."

    models = {
        u'auth.group': {
            'Meta': {'object_name': 'Group'},
            u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
            'name': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '80'}),
            'permissions': ('django.db.models.fields.related.ManyToManyField', [], {'to': u"orm['auth.Permission']", 'symmetrical': 'False', 'blank': 'True'})
        },
        u'auth.permission': {
            'Meta': {'ordering': "(u'content_type__app_label', u'content_type__model', u'codename')", 'unique_together': "((u'content_type', u'codename'),)", 'object_name': 'Permission'},
            'codename': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
            'content_type': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['contenttypes.ContentType']"}),
            u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
            'name': ('django.db.models.fields.CharField', [], {'max_length': '50'})
        },
        u'contenttypes.contenttype': {
            'Meta': {'ordering': "('name',)", 'unique_together': "(('app_label', 'model'),)", 'object_name': 'ContentType', 'db_table': "'django_content_type'"},
            'app_label': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
            u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
            'model': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
            'name': ('django.db.models.fields.CharField', [], {'max_length': '100'})
        },
        u'feeds.category': {
            'Meta': {'ordering': "('order', 'name', 'id')", 'unique_together': "(('user', 'slug'), ('user', 'name'))", 'object_name': 'Category'},
            'color': ('django.db.models.fields.CharField', [], {'default': "'blue'", 'max_length': '50'}),
            u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
            'name': ('django.db.models.fields.CharField', [], {'max_length': '1023', 'db_index': 'True'}),
            'order': ('django.db.models.fields.PositiveIntegerField', [], {'null': 'True', 'blank': 'True'}),
            'slug': ('django.db.models.fields.SlugField', [], {'max_length': '50'}),
            'user': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'categories'", 'to': u"orm['profiles.User']"})
        },
        u'feeds.entry': {
            'Meta': {'ordering': "('-date', '-id')", 'object_name': 'Entry', 'index_together': "(('user', 'date'), ('user', 'read'), ('user', 'starred'), ('user', 'broadcast'))"},
            'author': ('django.db.models.fields.CharField', [], {'max_length': '1023', 'blank': 'True'}),
            'broadcast': ('django.db.models.fields.BooleanField', [], {'default': 'False', 'db_index': 'True'}),
            'date': ('django.db.models.fields.DateTimeField', [], {'db_index': 'True'}),
            'feed': ('django.db.models.fields.related.ForeignKey', [], {'blank': 'True', 'related_name': "'entries'", 'null': 'True', 'to': u"orm['feeds.Feed']"}),
            'guid': ('feedhq.feeds.fields.URLField', [], {'db_index': 'True', 'blank': 'True'}),
            u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
            'link': ('feedhq.feeds.fields.URLField', [], {'db_index': 'True'}),
            'read': ('django.db.models.fields.BooleanField', [], {'default': 'False', 'db_index': 'True'}),
            'read_later_url': ('feedhq.feeds.fields.URLField', [], {'blank': 'True'}),
            'starred': ('django.db.models.fields.BooleanField', [], {'default': 'False', 'db_index': 'True'}),
            'subtitle': ('django.db.models.fields.TextField', [], {}),
            'title': ('django.db.models.fields.CharField', [], {'max_length': '255'}),
            'user': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'entries'", 'to': u"orm['profiles.User']"})
        },
        u'feeds.favicon': {
            'Meta': {'object_name': 'Favicon'},
            'favicon': ('django.db.models.fields.files.FileField', [], {'max_length': '100', 'blank': 'True'}),
            u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
            'url': ('feedhq.feeds.fields.URLField', [], {'unique': 'True', 'db_index': 'True'})
        },
        u'feeds.feed': {
            'Meta': {'ordering': "('name',)", 'object_name': 'Feed'},
            'category': ('django.db.models.fields.related.ForeignKey', [], {'blank': 'True', 'related_name': "'feeds'", 'null': 'True', 'to': u"orm['feeds.Category']"}),
            'favicon': ('django.db.models.fields.files.ImageField', [], {'max_length': '100', 'null': 'True', 'blank': 'True'}),
            u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
            'img_safe': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
            'name': ('django.db.models.fields.CharField', [], {'max_length': '1023'}),
            'unread_count': ('django.db.models.fields.PositiveIntegerField', [], {'default': '0'}),
            'url': ('feedhq.feeds.fields.URLField', [], {'db_index': 'True'}),
            'user': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'feeds'", 'to': u"orm['profiles.User']"})
        },
        u'feeds.uniquefeed': {
            'Meta': {'object_name': 'UniqueFeed'},
            'error': ('django.db.models.fields.CharField', [], {'max_length': '50', 'null': 'True', 'db_column': "'muted_reason'", 'blank': 'True'}),
            u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
            'muted': ('django.db.models.fields.BooleanField', [], {'default': 'False', 'db_index': 'True'}),
            'url': ('feedhq.feeds.fields.URLField', [], {'unique': 'True'})
        },
        u'profiles.user': {
            'Meta': {'object_name': 'User', 'db_table': "'auth_user'"},
            'allow_media': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
            'date_joined': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now'}),
            'email': ('django.db.models.fields.CharField', [], {'max_length': '75'}),
            'endless_pages': ('django.db.models.fields.BooleanField', [], {'default': 'True'}),
            'entries_per_page': ('django.db.models.fields.IntegerField', [], {'default': '50'}),
            'es': ('django.db.models.fields.NullBooleanField', [], {'default': 'True', 'null': 'True', 'blank': 'True'}),
            'first_name': ('django.db.models.fields.CharField', [], {'max_length': '30'}),
            'font': ('django.db.models.fields.CharField', [], {'default': "'pt-serif'", 'max_length': '75'}),
            'groups': ('django.db.models.fields.related.ManyToManyField', [], {'symmetrical': 'False', 'related_name': "u'user_set'", 'blank': 'True', 'to': u"orm['auth.Group']"}),
            u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
            'is_active': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
            'is_staff': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
            'is_superuser': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
            'is_suspended': ('django.db.models.fields.BooleanField', [], {'default': 'False', 'db_index': 'True'}),
            'last_login': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now'}),
            'last_name': ('django.db.models.fields.CharField', [], {'max_length': '30'}),
            'oldest_first': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
            'password': ('django.db.models.fields.CharField', [], {'max_length': '128'}),
            'read_later': ('django.db.models.fields.CharField', [], {'max_length': '50', 'blank': 'True'}),
            'read_later_credentials': ('django.db.models.fields.TextField', [], {'blank': 'True'}),
            'sharing_email': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
            'sharing_gplus': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
            'sharing_twitter': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
            'timezone': ('django.db.models.fields.CharField', [], {'default': "'UTC'", 'max_length': '75'}),
            'ttl': ('django.db.models.fields.PositiveIntegerField', [], {'default': '30'}),
            'user_permissions': ('django.db.models.fields.related.ManyToManyField', [], {'symmetrical': 'False', 'related_name': "u'user_set'", 'blank': 'True', 'to': u"orm['auth.Permission']"}),
            'username': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '75'})
        }
    }

    complete_apps = ['feeds']
    symmetrical = True
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
#
# This file is part of Invenio.
# Copyright (C) 2017 CERN.
#
# Invenio is free software; you can redistribute it
# and/or modify it under the terms of the GNU General Public License as
# published by the Free Software Foundation; either version 2 of the
# License, or (at your option) any later version.
#
# Invenio is distributed in the hope that it will be
# useful, but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
# General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Invenio; if not, write to the
# Free Software Foundation, Inc., 59 Temple Place, Suite 330, Boston,
# MA 02111-1307, USA.
#
# In applying this license, CERN does not
# waive the privileges and immunities granted to it by virtue of its status
# as an Intergovernmental Organization or submit itself to any jurisdiction.

"""Invenio-accounts views."""
<commit_msg>views: blueprint exposure fix<commit_after># -*- coding: utf-8 -*-
#
# This file is part of Invenio.
# Copyright (C) 2017 CERN.
#
# Invenio is free software; you can redistribute it
# and/or modify it under the terms of the GNU General Public License as
# published by the Free Software Foundation; either version 2 of the
# License, or (at your option) any later version.
#
# Invenio is distributed in the hope that it will be
# useful, but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
# General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Invenio; if not, write to the
# Free Software Foundation, Inc., 59 Temple Place, Suite 330, Boston,
# MA 02111-1307, USA.
#
# In applying this license, CERN does not
# waive the privileges and immunities granted to it by virtue of its status
# as an Intergovernmental Organization or submit itself to any jurisdiction.

"""Invenio-accounts views."""

from .settings import blueprint

__all__ = ('blueprint', )
<|endoftext|>


<commit_before>#!/usr/bin/python
import sys, argparse, StringIO, re, gzip
from multiprocessing import Pool, cpu_count, Queue
from Bio.Format.BGZF import is_bgzf, reader as BGZF_reader, get_block_bounds
from Bio.Format.Fastq import FastqEntry

# Create an index for bgzf zipped fastq files.
# Pre: A fastq file that has been compressed by bgzf
# Post: the Pre file, with the exension .bgi added.
#       the index is gzipped
#       <name> <blockStart> <innerStart> <dataSize> <read length>
#       Be cautious that name could contain spaces and tabs

# global
blocks = {}
ncount = 1
of = None
def main():
  global blocks
  parser = argparse.ArgumentParser(description="Take a bgzf compressed fastq file and make an index",formatter_class=argparse.ArgumentDefaultsHelpFormatter)
  parser.add_argument('input_file',help="BGZF compressed fastq file")
  parser.add_argument('--threads',type=int,default=1,help="number of threads")
  args = parser.parse_args()
  if not is_bgzf(args.input_file):
    sys.stderr.write("ERROR: not a proper BGZF compressed file\n")
    sys.exit()
  sys.stderr.write("scanning block starts\n")
  bs = get_block_bounds(args.input_file)
  blocks[bs[0][0]] = [[bs[0][1],-1]]
  sys.stderr.write("scanning for new lines\n")
  z = 0
  scan_for_newlines(bs,args)
  sys.stderr.write("Traverse blocks and writing index\n")
  z = 0
  traverse_and_write(blocks,args)

def traverse_and_write(blocks,args):
  global of
  n = 0
  z = 0
  of = gzip.open(args.input_file+'.bgi','w')
  bnames = sorted(blocks.keys())
  chunk_size = 100
  for block_set in [bnames[x:x+chunk_size] for x in range(0,len(bnames),chunk_size)]:
    ready_set = [[x,blocks[x][0][0],[y[1] for y in blocks[x]]] for x in block_set]
    #block set is a block,
    sys.stderr.write(str(n)+'/'+str(len(bnames))+"\r")
    n += chunk_size
    z += 1
    v = prepare_blocks(ready_set,args,1)
    write_results(v)

  sys.stderr.write("\n")
  of.close()

def write_results(v):
  global of
  for (i,line) in v:
    of.write(line)
  return

def prepare_blocks(blocks,args,z):
  results = []
  for block in blocks:
    #if len(blocks[block]) == 0: continue
    bstart = block[0]
    bend = block[1]
    starts = [x+1 for x in block[2]]
    #print starts
    with open(args.input_file,'rb') as inf:
      inf.seek(bstart)
      bytes = inf.read(bend-bstart)
      s = StringIO.StringIO(bytes)
      v = BGZF_reader(s)
      ubytes = v.read(70000)
      for i in range(len(starts)-1):
        if starts[i] >= len(ubytes): #problem
          sys.stderr.write("Problem start\n")
          sys.exit()
        m = re.match('([^\n]+)\n([^\n]+)(\n[^\n]+\n[^\n]+)',ubytes[starts[i]:])
        if not m:
          sys.stderr.write("Problem overlap\n")
          sys.exit()
        else:
          if m.group(1)[0] != '@':
            sys.stderr.write("failed to parse last\n")
            sys.exit()
          output = m.group(1)[1:]+"\t"+str(bstart)+"\t"+str(starts[i])+"\t"+str(len(m.group(1))+len(m.group(2))+len(m.group(3))+2)+"\t"+str(len(m.group(2)))+"\n"
          results.append([z,output])
    with open(args.input_file,'rb') as inf:
      v2 = BGZF_reader(inf,blockStart=bstart,innerStart=starts[-1]-1)
      spc = v2.read(1)
      if spc != "\n":
        sys.stderr.write("expected newline\n")
        sys.exit()
      cur = v2.get_block_start()
      inn = v2.get_inner_start()
      buffer = ''
      for i in range(0,4):
        while True:
          c = v2.read(1)
          if len(c) == 0: break
          buffer += c
          if c == "\n": break
      if buffer == "":
        break
      m = re.match('([^\n]+)\n([^\n]+)',buffer)
      if not m:
        sys.stderr.write("failed to parse last\n"+buffer+"\n")
        sys.exit()
      if m.group(1)[0] != '@':
        sys.stderr.write("failed to parse last\n"+buffer+"\n")
        sys.exit()
      output = m.group(1)[1:]+"\t"+str(cur)+"\t"+str(inn)+"\t"+str(len(buffer))+"\t"+str(len(m.group(2)))+"\n"
      results.append([z,output])
  return results

def scan_for_newlines(bs,args):
  z = 0
  # do them in chunks
  #for bounds in bs:
  chunk_size = 100
  for bounds_blocks in [bs[x:x+chunk_size] for x in range(0,len(bs),chunk_size)]:
      v = get_nls(bounds_blocks,args.input_file,z)
      do_nls_output(v)
      sys.stderr.write(str(z)+'/'+str(len(bs))+"\r")
      z += chunk_size
  sys.stderr.write("\n")

def get_nls(bounds_blocks,fname,i):
  breaks = []
  for bounds in bounds_blocks:
    with open(fname,'rb') as inf:
      inf.seek(bounds[0])
      bytes = inf.read(bounds[1]-bounds[0])
      s = StringIO.StringIO(bytes)
    #v = BGZF_reader(inf,blockStart=bound[0],innerStart=0)
    v = BGZF_reader(s)
    ubytes = v.read(70000) # always less than 65K by definition
    p = re.compile('\n')
    nls = [m.start() for m in p.finditer(ubytes)]
    for j in range(len(nls)):
      breaks.append([bounds[0],bounds[1],nls[j],i])
  return breaks

def do_nls_output(results):
  global blocks
  global ncount
  for e in results:
      useval = False
      if ncount%4 == 0: useval = True
      ncount += 1
      if not useval: continue
      if e[0] not in blocks: blocks[e[0]] = []
      blocks[e[0]].append([e[1],e[2]])

def readfastq(reader):
  buffer = ''
  for i in range(0,4):
    v = ''
    while v!="\n":
      v = reader.read(1)
      if len(v) == 0: return False
      buffer += v
  if len(buffer) == 0: return False
  return FastqEntry(buffer.rstrip().split("\n"))

if __name__=="__main__":
  main()
<commit_msg>first stage of multi processing<commit_after>#!/usr/bin/python
import sys, argparse, StringIO, re, gzip
from multiprocessing import Pool, cpu_count, Queue, Lock
from Bio.Format.BGZF import is_bgzf, reader as BGZF_reader, get_block_bounds
from Bio.Format.Fastq import FastqEntry

# Create an index for bgzf zipped fastq files.
# Pre: A fastq file that has been compressed by bgzf
# Post: the Pre file, with the exension .bgi added.
#       the index is gzipped
#       <name> <blockStart> <innerStart> <dataSize> <read length>
#       Be cautious that name could contain spaces and tabs

# global
blocks = {}
glock = Lock()
of = None
def main():
  global blocks
  parser = argparse.ArgumentParser(description="Take a bgzf compressed fastq file and make an index",formatter_class=argparse.ArgumentDefaultsHelpFormatter)
  parser.add_argument('input_file',help="BGZF compressed fastq file")
  parser.add_argument('--threads',type=int,default=1,help="number of threads")
  args = parser.parse_args()
  if not is_bgzf(args.input_file):
    sys.stderr.write("ERROR: not a proper BGZF compressed file\n")
    sys.exit()
  sys.stderr.write("scanning block starts\n")
  bs = get_block_bounds(args.input_file)
  blocks[bs[0][0]] = [[bs[0][1],-1]]
  sys.stderr.write("scanning for new lines\n")
  z = 0
  scan_for_newlines(bs,args)
  # now need to clean out non start lines
  ncount = 0
  bnames = sorted(blocks.keys())
  for bname in bnames:
     v = []
     #print blocks[bname]
     for i in range(len(blocks[bname])):
       if ncount%4 == 0: v.append(blocks[bname][i])
       ncount += 1
     blocks[bname] = v
  sys.stderr.write("Traverse blocks and writing index\n")
  z = 0
  traverse_and_write(blocks,args)

def traverse_and_write(blocks,args):
  global of
  n = 0
  z = 0
  of = gzip.open(args.input_file+'.bgi','w')
  bnames = sorted(blocks.keys())
  chunk_size = 1000
  for block_set in [bnames[x:x+chunk_size] for x in range(0,len(bnames),chunk_size)]:
    ready_set = [[x,blocks[x][0][0],[y[1] for y in blocks[x]]] for x in block_set if len(blocks[x]) > 0]
    #block set is a block,
    sys.stderr.write(str(n)+'/'+str(len(bnames))+"\r")
    n += chunk_size
    z += 1
    v = prepare_blocks(ready_set,args,z)
    write_results(v)

  sys.stderr.write("\n")
  of.close()

def write_results(v):
  global of
  for (i,line) in v:
    of.write(line)
  return

def prepare_blocks(blocks,args,z):
  results = []
  for block in blocks:
    #if len(blocks[block]) == 0: continue
    bstart = block[0]
    bend = block[1]
    starts = [x+1 for x in block[2]]
    #print starts
    with open(args.input_file,'rb') as inf:
      inf.seek(bstart)
      bytes = inf.read(bend-bstart)
      s = StringIO.StringIO(bytes)
      v = BGZF_reader(s)
      ubytes = v.read(70000)
      for i in range(len(starts)-1):
        if starts[i] >= len(ubytes): #problem
          sys.stderr.write("Problem start\n")
          sys.exit()
        m = re.match('([^\n]+)\n([^\n]+)(\n[^\n]+\n[^\n]+)',ubytes[starts[i]:])
        if not m:
          sys.stderr.write("Problem overlap\n")
          sys.exit()
        else:
          if m.group(1)[0] != '@':
            sys.stderr.write("failed to parse last\n")
            sys.exit()
          output = m.group(1)[1:]+"\t"+str(bstart)+"\t"+str(starts[i])+"\t"+str(len(m.group(1))+len(m.group(2))+len(m.group(3))+2)+"\t"+str(len(m.group(2)))+"\n"
          results.append([z,output])
    with open(args.input_file,'rb') as inf:
      v2 = BGZF_reader(inf,blockStart=bstart,innerStart=starts[-1]-1)
      spc = v2.read(1)
      if spc != "\n":
        sys.stderr.write("expected newline\n")
        sys.exit()
      cur = v2.get_block_start()
      inn = v2.get_inner_start()
      buffer = ''
      for i in range(0,4):
        while True:
          c = v2.read(1)
          if len(c) == 0: break
          buffer += c
          if c == "\n": break
      if buffer == "":
        break
      m = re.match('([^\n]+)\n([^\n]+)',buffer)
      if not m:
        sys.stderr.write("failed to parse last\n"+buffer+"\n")
        sys.exit()
      if m.group(1)[0] != '@':
        sys.stderr.write("failed to parse last\n"+buffer+"\n")
        sys.exit()
      output = m.group(1)[1:]+"\t"+str(cur)+"\t"+str(inn)+"\t"+str(len(buffer))+"\t"+str(len(m.group(2)))+"\n"
      results.append([z,output])
  return results

def scan_for_newlines(bs,args):
  z = 0
  # do them in chunks
  #for bounds in bs:
  chunk_size = 1000
  if args.threads > 1:
    p = Pool(processes=args.threads)
  for bounds_blocks in [bs[x:x+chunk_size] for x in range(0,len(bs),chunk_size)]:
    if args.threads > 1:
      p.apply_async(get_nls,args=(bounds_blocks,args.input_file,z),callback=do_nls_output)
      #do_nls_output(v)
    else:
      v = get_nls(bounds_blocks,args.input_file,z)
      do_nls_output(v)
    sys.stderr.write(str(z)+'/'+str(len(bs))+"\r")
    z += chunk_size
  sys.stderr.write("\n")
  if args.threads > 1:
    p.close()
    p.join()

def get_nls(bounds_blocks,fname,i):
  breaks = []
  for bounds in bounds_blocks:
    with open(fname,'rb') as inf:
      inf.seek(bounds[0])
      bytes = inf.read(bounds[1]-bounds[0])
      s = StringIO.StringIO(bytes)
    #v = BGZF_reader(inf,blockStart=bound[0],innerStart=0)
    v = BGZF_reader(s)
    ubytes = v.read(70000) # always less than 65K by definition
    p = re.compile('\n')
    nls = [m.start() for m in p.finditer(ubytes)]
    for j in range(len(nls)):
      breaks.append([bounds[0],bounds[1],nls[j],i])
    s.close()
  return breaks

def do_nls_output(results):
  global blocks
  #global ncount
  global glock
  glock.acquire()
  for e in results:
      #useval = False
      #if ncount%4 == 0: useval = True
      #ncount += 1
      #if not useval: continue
      if e[0] not in blocks: blocks[e[0]] = []
      blocks[e[0]].append([e[1],e[2]])
  glock.release()

def readfastq(reader):
  buffer = ''
  for i in range(0,4):
    v = ''
    while v!="\n":
      v = reader.read(1)
      if len(v) == 0: return False
      buffer += v
  if len(buffer) == 0: return False
  return FastqEntry(buffer.rstrip().split("\n"))

if __name__=="__main__":
  main()
<|endoftext|>


<commit_before>#!/usr/bin/python
import sys,argparse
from SequenceBasics import FastaHandleReader, FastqHandleReader

def main():
  parser = argparse.ArgumentParser()
  parser.add_argument('input',help="Use - for STDIN")
  group = parser.add_mutually_exclusive_group(required=True)
  group.add_argument('--fasta',action='store_true')
  group.add_argument('--fastq',action='store_true')
  parser.add_argument('--output_table',help='save coversion to file')
  args = parser.parse_args()

  if args.input=='-': args.input = sys.stdin
  else: args.input= open(args.input)

  if args.fasta:
    args.input = FastaHandleReader(args.input)
  elif args.fastq:
    args.input = FastqHandleReader(args.input)
  z = 0
  if args.output_table:  args.output_table= open(args.output_table,'w')
  while True:
    e = args.input.read_entry()
    if not e: break
    z+=1
    name = 'm150101_010101_11111_c111111111111111111_s1_p0/'+str(z)+'/ccs'
    if args.fastq:
      print '@'+name
      print e['seq']
      print '+'
      print e['qual']
    elif args.fasta:
      print '>'+name
      print e['seq']
    if args.output_table: args.output_table.write(e['name']+"\t"+name+"\n")
if __name__=="__main__":
  main()
<commit_msg>output option<commit_after>#!/usr/bin/python
import sys,argparse
from SequenceBasics import FastaHandleReader, FastqHandleReader

def main():
  parser = argparse.ArgumentParser()
  parser.add_argument('input',help="Use - for STDIN")
  group = parser.add_mutually_exclusive_group(required=True)
  group.add_argument('--fasta',action='store_true')
  group.add_argument('--fastq',action='store_true')
  parser.add_argument('--output_table',help='save coversion to file')
  parser.add_argument('-o','--output')
  args = parser.parse_args()

  if args.input=='-': args.input = sys.stdin
  else: args.input= open(args.input)

  if args.output: args.output = open(args.output,'w')
  else: args.output = sys.stdout

  if args.fasta:
    args.input = FastaHandleReader(args.input)
  elif args.fastq:
    args.input = FastqHandleReader(args.input)
  z = 0
  if args.output_table:  args.output_table= open(args.output_table,'w')
  while True:
    e = args.input.read_entry()
    if not e: break
    z+=1
    name = 'm150101_010101_11111_c111111111111111111_s1_p0/'+str(z)+'/ccs'
    if args.fastq:
      args.output.write( '@'+name+"\n"+ e['seq']+"\n"+ '+'+e['qual']+"\n")
    elif args.fasta:
      args.output.write('>'+name+"\n"+e['seq']+"\n")
    if args.output_table: args.output_table.write(e['name']+"\t"+name+"\n")
if __name__=="__main__":
  main()
<|endoftext|>


<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-
"""
nosetests for metadata.py
"""

from isaw.images import metadata, package
from nose.tools import assert_equals, assert_not_equal, assert_in, assert_is
import logging
import os
from pprint import pprint
import re
import shutil

logging.basicConfig(level=logging.DEBUG)

RWHITESPACE = re.compile(r"\s+", re.MULTILINE)
RDQUOTES = re.compile(r"\"")

def test_read_metadata():
    current = os.path.dirname(os.path.abspath(__file__))
    meta_path = os.path.join(current, 'data', 'kalabsha', '201107061813531', 'meta.xml')
    m = metadata.Metadata(meta_path)
    assert_equals(sorted(m.data.keys()), ['change-history', 'copyright-date', 'copyright-holder', 'date-photographed', 'description', 'flickr-url', 'geography', 'isaw-publish-cleared', 'license', 'license-release-verified', 'photographer', 'status', 'title', 'typology'])
    del m

def test_create_metadata():
    current = os.path.dirname(os.path.abspath(__file__))
    temp = os.path.join(current, 'temp')
    os.makedirs(temp)
    meta_path=os.path.join(temp, 'meta.xml')
    # by default, the file should *not* get created, and throws an error if not found
    try:
        m=metadata.Metadata(meta_path)
    except IOError:
        pass
    assert_equals(os.path.isfile(meta_path), False)
    # we can force creation of the file, if desired
    m=metadata.Metadata(meta_path, create=True)
    assert_equals(os.path.isfile(meta_path), True)
    assert_equals(m.data, {'status': 'draft', 'license': 'undetermined', 'isaw-publish-cleared': 'no', 'license-release-verified': 'no'})
    with open(meta_path, 'r') as f:
        guts = f.read()
    guts = RWHITESPACE.sub('', guts).strip()
    guts = RDQUOTES.sub('', guts)
    assert_equals(guts, "<?xmlversion=1.0encoding=UTF-8?><?xml-stylesheettype=text/csshref=meta-oxygen.csstitle=OxygenAuthoringCSSalternate=no?><?oxygenRNGSchema=meta-schema.rnctype=compact?><image-info><status>draft</status><license-release-verified>no</license-release-verified><isaw-publish-cleared>no</isaw-publish-cleared><review-notes></review-notes><image-files><imagetype=thumbnailhref=></image><imagetype=reviewhref=></image><imagetype=masterhref=></image><imagetype=originalhref=></image></image-files><infotype=original><original-file-name></original-file-name><origin>undetermined</origin><title></title><flickr-url></flickr-url><fda-handle></fda-handle><photographer><name></name></photographer><authority></authority><description></description><date-photographed></date-photographed><date-scanned></date-scanned><copyright-holder></copyright-holder><copyright-date></copyright-date><copyright-contact></copyright-contact><license>undetermined</license></info><infotype=isaw><title></title><flickr-url></flickr-url><fda-handle></fda-handle><photographer><given-name></given-name><family-name></family-name></photographer><authority></authority><description></description><date-photographed></date-photographed><date-scanned></date-scanned><copyright-holder></copyright-holder><copyright-date></copyright-date><copyright-contact></copyright-contact><license>undetermined</license><geography><photographed-place><ancient-name></ancient-name><modern-name></modern-name><uri></uri><institution></institution></photographed-place><find-place></find-place><original-place></original-place></geography><chronology></chronology><prosopography></prosopography><typology></typology><notes></notes></info><change-history></change-history></image-info>")
    del m
    shutil.rmtree(temp)
    # test with package creation and import of an image containing lots of metadata
    current = os.path.dirname(os.path.abspath(__file__))
    temp = os.path.join(current, 'temp')
    os.makedirs(temp)
    original_path = os.path.join(current, 'data', 'oracle.jpg')
    p = package.Package(temp, 'test_package', original_path)
    m = p.metadata
    d = m.data
    assert_equals(sorted(d.keys()), ['contributor', 'date-photographed', 'description', 'isaw-publish-cleared', 'license', 'license-release-verified', 'photographer', 'rights-statement', 'status', 'title', 'typology'])
    assert_equals(d['contributor']['name'], u'Tom Elliott')
    assert_equals(d['date-photographed'], '2014-11-15T17:39:19.457+00:00')
    assert_equals(d['description'], u'Interior of a Magic Hat #9 beer bottle cap bearing the slogan: "Banana costumes, by nature, are funny."')
    assert_equals(d['isaw-publish-cleared'], 'no')
    assert_equals(d['license'], u'https://creativecommons.org/licenses/by/3.0/us/')
    assert_equals(d['license-release-verified'], 'no')
    assert_equals(d['photographer']['email'], u'tom.elliott@nyu.edu')
    assert_equals(d['photographer']['name'], u'Tom Elliott')
    assert_equals(d['photographer']['title'], u'Associate Director for Digital Programs and Senior Research Scholar')
    assert_equals(d['photographer']['url'], u'http://www.paregorios.org/')
    assert_equals(d['rights-statement'], u'Copyright 2014. Tom Elliott')
    assert_equals(d['status'], 'draft')
    assert_equals(d['title'], u'The Oracle')
    assert_equals(sorted(d['typology']), [u'aphorisms', u'banana costumes', u'bananas', u'beer', u'bottle cap', u'costumes', u'humor', u'Magic Hat'])
    with open(m.path, 'r') as f:
        guts = f.read()
    guts = RWHITESPACE.sub('', guts).strip()
    guts = RDQUOTES.sub('', guts)
    assert_equals(guts, "<?xmlversion='1.0'encoding='UTF-8'?><image-info><status>draft</status><license-release-verified>no</license-release-verified><isaw-publish-cleared>no</isaw-publish-cleared><review-notes/><image-files><imagehref=type=thumbnail/><imagehref=type=review/><imagehref=type=master/><imagehref=type=original/></image-files><infotype=original><original-file-name/><origin>undetermined</origin><title>TheOracle</title><flickr-url/><fda-handle/><photographer><name>TomElliott</name><email>tom.elliott@nyu.edu</email><title>AssociateDirectorforDigitalPAssociateDirectorforDigitalProgramsandSeniorResearchScholar</title><url>http://www.paregorios.org/</url></photographer><authority/><description>InteriorofaMagicHat#9beerbottlecapbearingtheslogan:Bananacostumes,bynature,arefunny.</description><date-photographed>2014-11-15T17:39:19.457+00:00</date-photographed><date-scanned/><copyright-holder/><copyright-date/><copyright-contact/><license>https://creativecommons.org/licenses/by/3.0/us/</license><contributor><name>TomElliott</name></contributor><rights-statement>Copyright2014.TomElliott</rights-statement><typology/></info><infotype=isaw><title>TheOracle</title><flickr-url/><fda-handle/><photographer><given-name/><family-name/><email>tom.elliott@nyu.edu</email><name>TomElliott</name><title>AssociateDirectorforDigitalPAssociateDirectorforDigitalProgramsandSeniorResearchScholar</title><url>http://www.paregorios.org/</url></photographer><authority/><description>InteriorofaMagicHat#9beerbottlecapbearingtheslogan:Bananacostumes,bynature,arefunny.</description><date-photographed>2014-11-15T17:39:19.457+00:00</date-photographed><date-scanned/><copyright-holder/><copyright-date/><copyright-contact/><license>https://creativecommons.org/licenses/by/3.0/us/</license><geography><photographed-place><ancient-name/><modern-name/><uri/><institution/></photographed-place><find-place/><original-place/></geography><chronology></chronology><prosopography></prosopography><typology></typology><notes></notes><contributor><name>TomElliott</name></contributor><rights-statement>Copyright2014.TomElliott</rights-statement></info><change-history></change-history></image-info>")
    shutil.rmtree(temp)

<commit_msg>comprehensive value tests for metadata in xml file after import<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-
"""
nosetests for metadata.py
"""

from isaw.images import metadata, package
from nose.tools import assert_equals, assert_not_equal, assert_in, assert_is
import logging
import os
from pprint import pprint
import re
import shutil

logging.basicConfig(level=logging.DEBUG)

RWHITESPACE = re.compile(r"\s+", re.MULTILINE)
RDQUOTES = re.compile(r"\"")

def test_read_metadata():
    current = os.path.dirname(os.path.abspath(__file__))
    meta_path = os.path.join(current, 'data', 'kalabsha', '201107061813531', 'meta.xml')
    m = metadata.Metadata(meta_path)
    assert_equals(sorted(m.data.keys()), ['change-history', 'copyright-date', 'copyright-holder', 'date-photographed', 'description', 'flickr-url', 'geography', 'isaw-publish-cleared', 'license', 'license-release-verified', 'photographer', 'status', 'title', 'typology'])
    del m

def test_create_metadata():
    current = os.path.dirname(os.path.abspath(__file__))
    temp = os.path.join(current, 'temp')
    os.makedirs(temp)
    meta_path=os.path.join(temp, 'meta.xml')
    # by default, the file should *not* get created, and throws an error if not found
    try:
        m=metadata.Metadata(meta_path)
    except IOError:
        pass
    assert_equals(os.path.isfile(meta_path), False)
    # we can force creation of the file, if desired
    m=metadata.Metadata(meta_path, create=True)
    assert_equals(os.path.isfile(meta_path), True)
    assert_equals(m.data, {'status': 'draft', 'license': 'undetermined', 'isaw-publish-cleared': 'no', 'license-release-verified': 'no'})
    with open(meta_path, 'r') as f:
        guts = f.read()
    guts = RWHITESPACE.sub('', guts).strip()
    guts = RDQUOTES.sub('', guts)
    assert_equals(guts, "<?xmlversion=1.0encoding=UTF-8?><?xml-stylesheettype=text/csshref=meta-oxygen.csstitle=OxygenAuthoringCSSalternate=no?><?oxygenRNGSchema=meta-schema.rnctype=compact?><image-info><status>draft</status><license-release-verified>no</license-release-verified><isaw-publish-cleared>no</isaw-publish-cleared><review-notes></review-notes><image-files><imagetype=thumbnailhref=></image><imagetype=reviewhref=></image><imagetype=masterhref=></image><imagetype=originalhref=></image></image-files><infotype=original><original-file-name></original-file-name><origin>undetermined</origin><title></title><flickr-url></flickr-url><fda-handle></fda-handle><photographer><name></name></photographer><authority></authority><description></description><date-photographed></date-photographed><date-scanned></date-scanned><copyright-holder></copyright-holder><copyright-date></copyright-date><copyright-contact></copyright-contact><license>undetermined</license></info><infotype=isaw><title></title><flickr-url></flickr-url><fda-handle></fda-handle><photographer><given-name></given-name><family-name></family-name></photographer><authority></authority><description></description><date-photographed></date-photographed><date-scanned></date-scanned><copyright-holder></copyright-holder><copyright-date></copyright-date><copyright-contact></copyright-contact><license>undetermined</license><geography><photographed-place><ancient-name></ancient-name><modern-name></modern-name><uri></uri><institution></institution></photographed-place><find-place></find-place><original-place></original-place></geography><chronology></chronology><prosopography></prosopography><typology></typology><notes></notes></info><change-history></change-history></image-info>")
    del m
    shutil.rmtree(temp)
    # test with package creation and import of an image containing lots of metadata
    current = os.path.dirname(os.path.abspath(__file__))
    temp = os.path.join(current, 'temp')
    os.makedirs(temp)
    original_path = os.path.join(current, 'data', 'oracle.jpg')
    p = package.Package(temp, 'test_package', original_path)
    m = p.metadata
    d = m.data
    assert_equals(sorted(d.keys()), ['contributor', 'date-photographed', 'description', 'isaw-publish-cleared', 'license', 'license-release-verified', 'photographer', 'rights-statement', 'status', 'title', 'typology'])
    assert_equals(d['contributor']['name'], u'Tom Elliott')
    assert_equals(d['date-photographed'], '2014-11-15T17:39:19.457+00:00')
    assert_equals(d['description'], u'Interior of a Magic Hat #9 beer bottle cap bearing the slogan: "Banana costumes, by nature, are funny."')
    assert_equals(d['isaw-publish-cleared'], 'no')
    assert_equals(d['license'], u'https://creativecommons.org/licenses/by/3.0/us/')
    assert_equals(d['license-release-verified'], 'no')
    assert_equals(d['photographer']['email'], u'tom.elliott@nyu.edu')
    assert_equals(d['photographer']['name'], u'Tom Elliott')
    assert_equals(d['photographer']['title'], u'Associate Director for Digital Programs and Senior Research Scholar')
    assert_equals(d['photographer']['url'], u'http://www.paregorios.org/')
    assert_equals(d['rights-statement'], u'Copyright 2014. Tom Elliott')
    assert_equals(d['status'], 'draft')
    assert_equals(d['title'], u'The Oracle')
    assert_equals(sorted(d['typology'], key=lambda s: s.lower()), [u'aphorisms', u'banana costumes', u'bananas', u'beer', u'bottle cap', u'costumes', u'humor', u'Magic Hat'])
    with open(m.path, 'r') as f:
        guts = f.read()
    guts = RWHITESPACE.sub('', guts).strip()
    guts = RDQUOTES.sub('', guts)
    assert_equals(guts, "<?xmlversion='1.0'encoding='UTF-8'?><image-info><status>draft</status><license-release-verified>no</license-release-verified><isaw-publish-cleared>no</isaw-publish-cleared><review-notes/><image-files><imagehref=type=thumbnail/><imagehref=type=review/><imagehref=type=master/><imagehref=type=original/></image-files><infotype=original><original-file-name/><origin>undetermined</origin><title>TheOracle</title><flickr-url/><fda-handle/><photographer><name>TomElliott</name><email>tom.elliott@nyu.edu</email><title>AssociateDirectorforDigitalProgramsandSeniorResearchScholar</title><url>http://www.paregorios.org/</url></photographer><authority/><description>InteriorofaMagicHat#9beerbottlecapbearingtheslogan:Bananacostumes,bynature,arefunny.</description><date-photographed>2014-11-15T17:39:19.457+00:00</date-photographed><date-scanned/><copyright-holder/><copyright-date/><copyright-contact/><license>https://creativecommons.org/licenses/by/3.0/us/</license><contributor><name>TomElliott</name></contributor><rights-statement>Copyright2014.TomElliott</rights-statement><typology/></info><infotype=isaw><title>TheOracle</title><flickr-url/><fda-handle/><photographer><given-name/><family-name/><email>tom.elliott@nyu.edu</email><name>TomElliott</name><title>AssociateDirectorforDigitalProgramsandSeniorResearchScholar</title><url>http://www.paregorios.org/</url></photographer><authority/><description>InteriorofaMagicHat#9beerbottlecapbearingtheslogan:Bananacostumes,bynature,arefunny.</description><date-photographed>2014-11-15T17:39:19.457+00:00</date-photographed><date-scanned/><copyright-holder/><copyright-date/><copyright-contact/><license>https://creativecommons.org/licenses/by/3.0/us/</license><geography><photographed-place><ancient-name/><modern-name/><uri/><institution/></photographed-place><find-place/><original-place/></geography><chronology></chronology><prosopography></prosopography><typology></typology><notes></notes><contributor><name>TomElliott</name></contributor><rights-statement>Copyright2014.TomElliott</rights-statement></info><change-history></change-history></image-info>")
    shutil.rmtree(temp)

<|endoftext|>


<commit_before>from datetime import datetime
from decimal import Decimal
from flask.json import _json
from toolspy import dict_map
from .utils import is_list_like, is_dict_like
from collections import OrderedDict
from types import FunctionType

def json_encoder(obj):
    if isinstance(obj, datetime):
        return obj.isoformat()
    elif isinstance(obj, OrderedDict):
        return [[k, json_encoder(v)]
                for k, v in obj.items()
                if not (k == 'key' and isinstance(v, FunctionType))]
    elif isinstance(obj, Decimal):
        return str(obj)
    elif isinstance(obj, unicode):
        return obj
    elif hasattr(obj, 'todict'):
        return obj.todict()
    elif is_list_like(obj):
        return [json_encoder(i) for i in obj]
    elif is_dict_like(obj):
        return dict_map(obj, lambda v: json_encoder(v))
    else:
        try:
            return _json.JSONEncoder().default(obj)
        except:
            return unicode(obj)

class BoosterJSONEncoder(_json.JSONEncoder):
    def default(self, obj):
        return json_encoder(obj)

def booster_json_decoder(obj):
    if '__type__' in obj:
        if obj['__type__'] == '__datetime__':
            return datetime.fromtimestamp(obj['epoch'])
    return obj

# Encoder function
def booster_json_dumps(obj):
    return _json.dumps(obj, cls=BoosterJSONEncoder)

# Decoder function
def booster_json_loads(obj):
    return _json.loads(obj, object_hook=booster_json_decoder)
<commit_msg>fixing json encoder to render integer, long correctly<commit_after>from datetime import datetime
from decimal import Decimal
from flask.json import _json
from toolspy import dict_map
from .utils import is_list_like, is_dict_like
from collections import OrderedDict
from types import FunctionType

def json_encoder(obj):
    if isinstance(obj, datetime):
        return obj.isoformat()
    elif isinstance(obj, int) or isinstance(obj, long) or isinstance(obj, float):
        return obj
    elif isinstance(obj, Decimal):
        return float(obj)
    elif isinstance(obj, unicode):
        return obj
    elif isinstance(obj, OrderedDict):
        return [[k, json_encoder(v)]
                for k, v in obj.items()
                if not (k == 'key' and isinstance(v, FunctionType))]
    elif hasattr(obj, 'todict'):
        return obj.todict()
    elif is_list_like(obj):
        return [json_encoder(i) for i in obj]
    elif is_dict_like(obj):
        return dict_map(obj, lambda v: json_encoder(v))
    else:
        try:
            return _json.JSONEncoder().default(obj)
        except:
            return unicode(obj)

class BoosterJSONEncoder(_json.JSONEncoder):
    def default(self, obj):
        return json_encoder(obj)

def booster_json_decoder(obj):
    if '__type__' in obj:
        if obj['__type__'] == '__datetime__':
            return datetime.fromtimestamp(obj['epoch'])
    return obj

# Encoder function
def booster_json_dumps(obj):
    return _json.dumps(obj, cls=BoosterJSONEncoder)

# Decoder function
def booster_json_loads(obj):
    return _json.loads(obj, object_hook=booster_json_decoder)
<|endoftext|>