<commit_before>from string import Template
import os
from xml.sax.saxutils import quoteattr
from xml.sax.saxutils import escape
from django.db import models
from django.contrib.contenttypes import generic
from django.conf import settings

from ISO_639_2b import ISO_639_2b
from DublinCore.models import QualifiedDublinCoreElement

EAD_ROOT_DIRECTORY = settings.EAD_ROOT_DIRECTORY  if hasattr(settings, 'EAD_ROOT_DIRECTORY') else os.path.join(os.environ.get('HOME', '/dsc'), 'data/in/oac-ead/prime2002')

NOT_OAC = True
try:
    from oac.models import Institution
    NOT_OAC = False
except ImportError:
    pass

if NOT_OAC:
    class PublishingInstitution(models.Model):
        '''Publisher if you're not oac
        '''
        name = models.CharField(max_length=255)
        mainagency = models.CharField(max_length=255,)
        ark = models.CharField(max_length=255, unique=true)
        cdlpath = models.CharField(max_length=255, blank=True)

else:
    class PublishingInstitution(Institution):
        '''Proxy for the Institution, to make it look like a Publisher?
        '''
        class Meta:
            proxy = True

class CollectionRecord(models.Model):
    ark = models.CharField(max_length=255, primary_key=True) #mysql length limit
    publisher = models.ForeignKey(PublishingInstitution)
    title = models.CharField(max_length=512,)
    title_filing = models.SlugField(max_length=255)#, unique=True)
    date_dacs = models.CharField(max_length=128,)
    date_iso = models.CharField(max_length=128, blank=True)
    local_identifier = models.CharField(max_length=255, )
    extent=models.CharField(max_length=255)
    abstract=models.TextField()
    language = models.CharField(max_length=3, choices=(ISO_639_2b))
    accessrestrict = models.TextField()
    userestrict = models.TextField()
    acqinfo = models.TextField()
    scopecontent = models.TextField()
    bioghist = models.TextField(null=True, blank=True)
    online_items_url = models.URLField(null=True, blank=True, )
    updated_at = models.DateTimeField(auto_now=True)
    created_at = models.DateTimeField(auto_now_add=True)
    QDCElements = generic.GenericRelation(QualifiedDublinCoreElement)

    class Meta:
        unique_together = (("title_filing", "publisher"), ("local_identifier",
            "publisher"))

    @models.permalink
    def get_absolute_url(self):
        return ('collectionrecord_view', (), {'ark': self.ark, })

    def save_ead_file(self, directory_root=EAD_ROOT_DIRECTORY):
        '''Save the EAD file to it's DSC CDL specific location?
        '''
        fname = os.path.join(directory_root, self.publisher.cdlpath, self.ark.rsplit('/', 1)[1]+'.xml')
        print fname
        with open(fname, 'w') as foo:
            foo.write(self.ead_xml)

    def save(self, *args, **kwargs):
        '''On save if ark is not set, get a new one from EZID.
        Also, if someone is trying to change the ARK, don't let them
        '''
        if self.pk:# existing object
            try:
                db_self = CollectionRecord.objects.get(pk=self.pk)
                if self.ark != db_self.ark:
                    #NOTE: this only works if I have a hidden numeric pk
                    raise ValueError('Can not change ARK for an collection')
            except CollectionRecord.DoesNotExist:
                pass
        if not self.ark:
            raise ValueError('Collection Records must have an ARK')
        super(CollectionRecord, self).save(*args, **kwargs)
        #TODO: save the ead file?? can use a public bool to determine if saved
        # to disk
        self.save_ead_file()

    def has_extended_metadata(self):
        '''Tests if any extended metadata items (held in QDCElements)
        are associated with the instance.
        '''
        return True if self.QDCElements.count() > 0 else False
    has_extended_metadata.short_description = 'XMetadata'

    #or should I just make a nice dictionary of subsetted values?
    #Need to define corresponding accessors (& setters?) for the various
    #multi-valued terms stored in the QDCElements.
    @property
    def creator_person(self):
        return self.QDCElements.all().filter(term__exact='CR').filter(qualifier__exact='person')

    @property
    def creator_family(self):
        return self.QDCElements.all().filter(term__exact='CR').filter(qualifier__exact='family')

    @property
    def creator_organization(self):
        return self.QDCElements.all().filter(term__exact='CR').filter(qualifier__exact='organization')

    @property
    def subject_topic(self):
        return self.QDCElements.all().filter(term__exact='SUB').filter(qualifier__exact='topic')

    @property
    def subject_name_person(self):
        return self.QDCElements.all().filter(term__exact='SUB').filter(qualifier__exact='name_person')

    @property
    def subject_name_family(self):
        return self.QDCElements.all().filter(term__exact='SUB').filter(qualifier__exact='name_family')

    @property
    def subject_name_organization(self):
        return self.QDCElements.all().filter(term__exact='SUB').filter(qualifier__exact='name_organization')

    @property
    def subject_title(self):
        return self.QDCElements.all().filter(term__exact='SUB').filter(qualifier__exact='title')

    @property
    def subject_function(self):
        return self.QDCElements.all().filter(term__exact='SUB').filter(qualifier__exact='function')

    @property
    def subject_occupation(self):
        return self.QDCElements.all().filter(term__exact='SUB').filter(qualifier__exact='occupation')

    @property
    def coverage(self):
        return self.QDCElements.all().filter(term__exact='COV').filter(qualifier__exact='geo')

    @property
    def type_format(self):
        return self.QDCElements.all().filter(term__exact='TYP').filter(qualifier__exact='genre')

    @property
    def ead_xml(self):
        '''Return a unicode object that contains the EAD xml for the collection
        record
        '''
        #should I add an xml output for all the various fields?
        #could be a big map/template thingy?
        #xml_head='''<?xml version="1.0" encoding="UTF-8"?>''' #Don't need this.
        ead_head_xml = '''<!DOCTYPE ead PUBLIC "+//ISBN 1-931666-00-8//DTD ead.dtd (Encoded Archival Description (EAD) Version 2002)//EN" "ead.dtd">
<ead>
<eadheader langencoding="iso639-2b" scriptencoding="iso15924" repositoryencoding="iso15511" countryencoding="iso3166-1" dateencoding="iso8601">
'''
###        collection_mapping = dict()
###        for f in self._meta.fields:
###            collection_mapping[f.name] = unicode(getattr(self, f.name))
###        collection_mapping['publisher_marc'] = self.publisher.mainagency
###        collection_mapping['publisher_name'] = self.publisher.name
###        collection_mapping['publisher_ark'] = self.publisher.ark
        ead_id_template_str = '''<eadid xmlns:cdlpath="http://www.cdlib.org/path/" countrycode="us" identifier=$ark mainagencycode=${publisher_marc} publicid=$local_identifier cdlpath:parent=${publisher_ark}'''
        ead_id_template = Template(ead_id_template_str)
        ead_id_xml = ead_id_template.substitute(
                ark = quoteattr(self.ark),
                publisher_marc = quoteattr(self.publisher.mainagency),
                local_identifier = quoteattr(self.local_identifier),
                publisher_ark = quoteattr(self.publisher.ark)
        )
        if self.publisher.parent_institution:
            ead_id_xml = ''.join((ead_id_xml, ' cdlpath:grandparent=', quoteattr(self.publisher.parent_institution.ark)))
        ead_id_xml = ''.join((ead_id_xml, ">", escape(self.local_identifier), "</eadid>"))
        ead_filedesc_head_tmpl_str = '''
<filedesc>
<titlestmt>
<titleproper>$title</titleproper>
<titleproper type="filing">$title_filing</titleproper>
<author>${publisher_name}</author>
</titlestmt>
<publicationstmt>
<publisher>${publisher_name}</publisher>
<date>$date_dacs</date>
</publicationstmt>
</filedesc>
</eadheader>
<archdesc level="collection">
<did>
Descriptive Summary
<unittitle label="Title">$title</unittitle>
<unitdate normal=$date_iso label="Dates"></unitdate>
<unitid label="Collection Number" repositorycode=${publisher_marc} countrycode="US">$local_identifier</unitid>
'''
        ead_filedesc_head_template = Template(ead_filedesc_head_tmpl_str)
        ead_filedesc_head_xml = ead_filedesc_head_template.substitute(
                title = escape(self.title),
                title_filing = escape(self.title_filing),
                publisher_name = escape(self.publisher.name),
                date_dacs = escape(self.date_dacs),
                date_iso = quoteattr(self.date_iso),
                publisher_marc = quoteattr(self.publisher.mainagency),
                local_identifier = escape(self.local_identifier),
        )
        ead_xml = ''.join((ead_head_xml, ead_id_xml, ead_filedesc_head_xml))
        origination_xml = '<origination label="Creator/Collector">\n'
        for qdc in self.creator_person:
            origination_xml = ''.join((origination_xml, '<persname>', escape(qdc.content), '</persname>\n',))
        for qdc in self.creator_family:
            origination_xml = ''.join((origination_xml, '<famname>', escape(qdc.content), '</famname>\n',))
        for qdc in self.creator_organization:
            origination_xml = ''.join((origination_xml, '<corpname>', escape(qdc.content), '</corpname>\n',))
        origination_xml = ''.join((origination_xml, '</origination>\n',))
        ead_xml = ''.join((ead_xml, origination_xml,))
        ead_xml = ''.join((ead_xml, '<physdesc label="Extent"><extent>', self.extent, '</extent>\n',))
        if self.online_items_url:
            ead_xml = ''.join((ead_xml, '<extent type="dao">Online items available</extent>\n<dao role="http://oac.cdlib.org/arcrole/link/search/" href="', self.online_items_url, '" title="Online items"/>\n',))
        ead_xml = ''.join((ead_xml, '</physdesc>\n',))
        ead_xml = ''.join((ead_xml, '<repository label="Repository">\n<corpname>', self.publisher.name, '</corpname>\n</repository>\n'))
        ead_xml = ''.join((ead_xml, '<abstract label="Abstract">', self.abstract, '</abstract>\n',))
        ead_xml = ''.join((ead_xml, '<langmaterial><language langcode="', self.language, '"/></langmaterial>\n',))
        ead_xml = ''.join((ead_xml, '</did>\n', ))
        ead_xml = ''.join((ead_xml, '<accessrestrict>Access
', self.accessrestrict, '
</accessrestrict>\n',))
        ead_xml = ''.join((ead_xml, '<userestrict>Publication Rights
', self.userestrict, '
</userestrict>\n',))
        ead_xml = ''.join((ead_xml, '<prefercite>Preferred Citation
', self.title, ' , ',  self.local_identifier, '.  ', self.publisher.name, '.
</prefercite>\n',))
        ead_xml = ''.join((ead_xml, '<acqinfo>Acquisition Information
', self.acqinfo, '
</acqinfo>\n',))
        if self.bioghist:
            ead_xml = ''.join((ead_xml, '<bioghist>Biography/Administrative History
', self.bioghist, '
</bioghist>',))
        ead_xml = ''.join((ead_xml, '<scopecontent>Scope and Content of Collection
', self.scopecontent, '
</scopecontent>',))
        controlaccess_xml = '<controlaccess>\nIndexing Terms\n'
        for qdc in self.subject_topic:
            controlaccess_xml = ''.join((controlaccess_xml, '<subject>', escape(qdc.content), '</subject>\n',))
        for qdc in self.subject_name_person:
            controlaccess_xml = ''.join((controlaccess_xml, '<persname role="subject">', escape(qdc.content), '</persname>\n',))
        for qdc in self.subject_name_family:
            controlaccess_xml = ''.join((controlaccess_xml, '<famname role="subject">', escape(qdc.content), '</famname>\n',))
        for qdc in self.subject_name_organization:
            controlaccess_xml = ''.join((controlaccess_xml, '<corpname role="subject">', escape(qdc.content), '</corpname>\n',))
        for qdc in self.coverage:
            controlaccess_xml = ''.join((controlaccess_xml, '<geogname role="subject">', escape(qdc.content), '</geogname>\n',))
        for qdc in self.type_format:
            controlaccess_xml = ''.join((controlaccess_xml, '<genreform role="subject">', escape(qdc.content), '</genreform>\n',))
        for qdc in self.subject_title:
            controlaccess_xml = ''.join((controlaccess_xml, '<title role="subject">', escape(qdc.content), '</title>\n',))
        for qdc in self.subject_function:
            controlaccess_xml = ''.join((controlaccess_xml, '<function role="subject">', escape(qdc.content), '</function>\n',))
        for qdc in self.subject_occupation:
            controlaccess_xml = ''.join((controlaccess_xml, '<occupation role="subject">', escape(qdc.content), '</occupation>\n',))
        controlaccess_xml = ''.join((controlaccess_xml, '</controlaccess>\n',))
        ead_xml = ''.join((ead_xml, controlaccess_xml))
        ead_xml = ''.join((ead_xml, '</archdesc>\n</ead>\n',))
        return ead_xml
<commit_msg>filing title is not a slug<commit_after>from string import Template
import os
from xml.sax.saxutils import quoteattr
from xml.sax.saxutils import escape
from django.db import models
from django.contrib.contenttypes import generic
from django.conf import settings

from ISO_639_2b import ISO_639_2b
from DublinCore.models import QualifiedDublinCoreElement

EAD_ROOT_DIRECTORY = settings.EAD_ROOT_DIRECTORY  if hasattr(settings, 'EAD_ROOT_DIRECTORY') else os.path.join(os.environ.get('HOME', '/dsc'), 'data/in/oac-ead/prime2002')

NOT_OAC = True
try:
    from oac.models import Institution
    NOT_OAC = False
except ImportError:
    pass

if NOT_OAC:
    class PublishingInstitution(models.Model):
        '''Publisher if you're not oac
        '''
        name = models.CharField(max_length=255)
        mainagency = models.CharField(max_length=255,)
        ark = models.CharField(max_length=255, unique=true)
        cdlpath = models.CharField(max_length=255, blank=True)

else:
    class PublishingInstitution(Institution):
        '''Proxy for the Institution, to make it look like a Publisher?
        '''
        class Meta:
            proxy = True

class CollectionRecord(models.Model):
    ark = models.CharField(max_length=255, primary_key=True) #mysql length limit
    publisher = models.ForeignKey(PublishingInstitution)
    title = models.CharField(max_length=512,)
    title_filing = models.CharField(max_length=255)#, unique=True)
    date_dacs = models.CharField(max_length=128,)
    date_iso = models.CharField(max_length=128, blank=True)
    local_identifier = models.CharField(max_length=255, )
    extent=models.CharField(max_length=255)
    abstract=models.TextField()
    language = models.CharField(max_length=3, choices=(ISO_639_2b), help_text='Language of materials')
    accessrestrict = models.TextField()
    userestrict = models.TextField()
    acqinfo = models.TextField()
    scopecontent = models.TextField()
    bioghist = models.TextField(null=True, blank=True)
    online_items_url = models.URLField(null=True, blank=True, )
    updated_at = models.DateTimeField(auto_now=True)
    created_at = models.DateTimeField(auto_now_add=True)
    QDCElements = generic.GenericRelation(QualifiedDublinCoreElement)

    class Meta:
        unique_together = (("title_filing", "publisher"), ("local_identifier",
            "publisher"))

    @models.permalink
    def get_absolute_url(self):
        return ('collectionrecord_view', (), {'ark': self.ark, })

    def save_ead_file(self, directory_root=EAD_ROOT_DIRECTORY):
        '''Save the EAD file to it's DSC CDL specific location?
        '''
        fname = os.path.join(directory_root, self.publisher.cdlpath, self.ark.rsplit('/', 1)[1]+'.xml')
        print fname
        with open(fname, 'w') as foo:
            foo.write(self.ead_xml)

    def save(self, *args, **kwargs):
        '''On save if ark is not set, get a new one from EZID.
        Also, if someone is trying to change the ARK, don't let them
        '''
        if self.pk:# existing object
            try:
                db_self = CollectionRecord.objects.get(pk=self.pk)
                if self.ark != db_self.ark:
                    #NOTE: this only works if I have a hidden numeric pk
                    raise ValueError('Can not change ARK for an collection')
            except CollectionRecord.DoesNotExist:
                pass
        if not self.ark:
            raise ValueError('Collection Records must have an ARK')
        super(CollectionRecord, self).save(*args, **kwargs)
        #TODO: save the ead file?? can use a public bool to determine if saved
        # to disk
        self.save_ead_file()

    def has_extended_metadata(self):
        '''Tests if any extended metadata items (held in QDCElements)
        are associated with the instance.
        '''
        return True if self.QDCElements.count() > 0 else False
    has_extended_metadata.short_description = 'XMetadata'

    #or should I just make a nice dictionary of subsetted values?
    #Need to define corresponding accessors (& setters?) for the various
    #multi-valued terms stored in the QDCElements.
    @property
    def creator_person(self):
        return self.QDCElements.all().filter(term__exact='CR').filter(qualifier__exact='person')

    @property
    def creator_family(self):
        return self.QDCElements.all().filter(term__exact='CR').filter(qualifier__exact='family')

    @property
    def creator_organization(self):
        return self.QDCElements.all().filter(term__exact='CR').filter(qualifier__exact='organization')

    @property
    def subject_topic(self):
        return self.QDCElements.all().filter(term__exact='SUB').filter(qualifier__exact='topic')

    @property
    def subject_name_person(self):
        return self.QDCElements.all().filter(term__exact='SUB').filter(qualifier__exact='name_person')

    @property
    def subject_name_family(self):
        return self.QDCElements.all().filter(term__exact='SUB').filter(qualifier__exact='name_family')

    @property
    def subject_name_organization(self):
        return self.QDCElements.all().filter(term__exact='SUB').filter(qualifier__exact='name_organization')

    @property
    def subject_title(self):
        return self.QDCElements.all().filter(term__exact='SUB').filter(qualifier__exact='title')

    @property
    def subject_function(self):
        return self.QDCElements.all().filter(term__exact='SUB').filter(qualifier__exact='function')

    @property
    def subject_occupation(self):
        return self.QDCElements.all().filter(term__exact='SUB').filter(qualifier__exact='occupation')

    @property
    def coverage(self):
        return self.QDCElements.all().filter(term__exact='COV').filter(qualifier__exact='geo')

    @property
    def type_format(self):
        return self.QDCElements.all().filter(term__exact='TYP').filter(qualifier__exact='genre')

    @property
    def ead_xml(self):
        '''Return a unicode object that contains the EAD xml for the collection
        record
        '''
        #should I add an xml output for all the various fields?
        #could be a big map/template thingy?
        #xml_head='''<?xml version="1.0" encoding="UTF-8"?>''' #Don't need this.
        ead_head_xml = '''<!DOCTYPE ead PUBLIC "+//ISBN 1-931666-00-8//DTD ead.dtd (Encoded Archival Description (EAD) Version 2002)//EN" "ead.dtd">
<ead>
<eadheader langencoding="iso639-2b" scriptencoding="iso15924" repositoryencoding="iso15511" countryencoding="iso3166-1" dateencoding="iso8601">
'''
###        collection_mapping = dict()
###        for f in self._meta.fields:
###            collection_mapping[f.name] = unicode(getattr(self, f.name))
###        collection_mapping['publisher_marc'] = self.publisher.mainagency
###        collection_mapping['publisher_name'] = self.publisher.name
###        collection_mapping['publisher_ark'] = self.publisher.ark
        ead_id_template_str = '''<eadid xmlns:cdlpath="http://www.cdlib.org/path/" countrycode="us" identifier=$ark mainagencycode=${publisher_marc} publicid=$local_identifier cdlpath:parent=${publisher_ark}'''
        ead_id_template = Template(ead_id_template_str)
        ead_id_xml = ead_id_template.substitute(
                ark = quoteattr(self.ark),
                publisher_marc = quoteattr(self.publisher.mainagency),
                local_identifier = quoteattr(self.local_identifier),
                publisher_ark = quoteattr(self.publisher.ark)
        )
        if self.publisher.parent_institution:
            ead_id_xml = ''.join((ead_id_xml, ' cdlpath:grandparent=', quoteattr(self.publisher.parent_institution.ark)))
        ead_id_xml = ''.join((ead_id_xml, ">", escape(self.local_identifier), "</eadid>"))
        ead_filedesc_head_tmpl_str = '''
<filedesc>
<titlestmt>
<titleproper>$title</titleproper>
<titleproper type="filing">$title_filing</titleproper>
<author>${publisher_name}</author>
</titlestmt>
<publicationstmt>
<publisher>${publisher_name}</publisher>
<date>$date_dacs</date>
</publicationstmt>
</filedesc>
</eadheader>
<archdesc level="collection">
<did>
Descriptive Summary
<unittitle label="Title">$title</unittitle>
<unitdate normal=$date_iso label="Dates"></unitdate>
<unitid label="Collection Number" repositorycode=${publisher_marc} countrycode="US">$local_identifier</unitid>
'''
        ead_filedesc_head_template = Template(ead_filedesc_head_tmpl_str)
        ead_filedesc_head_xml = ead_filedesc_head_template.substitute(
                title = escape(self.title),
                title_filing = escape(self.title_filing),
                publisher_name = escape(self.publisher.name),
                date_dacs = escape(self.date_dacs),
                date_iso = quoteattr(self.date_iso),
                publisher_marc = quoteattr(self.publisher.mainagency),
                local_identifier = escape(self.local_identifier),
        )
        ead_xml = ''.join((ead_head_xml, ead_id_xml, ead_filedesc_head_xml))
        origination_xml = '<origination label="Creator/Collector">\n'
        for qdc in self.creator_person:
            origination_xml = ''.join((origination_xml, '<persname>', escape(qdc.content), '</persname>\n',))
        for qdc in self.creator_family:
            origination_xml = ''.join((origination_xml, '<famname>', escape(qdc.content), '</famname>\n',))
        for qdc in self.creator_organization:
            origination_xml = ''.join((origination_xml, '<corpname>', escape(qdc.content), '</corpname>\n',))
        origination_xml = ''.join((origination_xml, '</origination>\n',))
        ead_xml = ''.join((ead_xml, origination_xml,))
        ead_xml = ''.join((ead_xml, '<physdesc label="Extent"><extent>', self.extent, '</extent>\n',))
        if self.online_items_url:
            ead_xml = ''.join((ead_xml, '<extent type="dao">Online items available</extent>\n<dao role="http://oac.cdlib.org/arcrole/link/search/" href="', self.online_items_url, '" title="Online items"/>\n',))
        ead_xml = ''.join((ead_xml, '</physdesc>\n',))
        ead_xml = ''.join((ead_xml, '<repository label="Repository">\n<corpname>', self.publisher.name, '</corpname>\n</repository>\n'))
        ead_xml = ''.join((ead_xml, '<abstract label="Abstract">', self.abstract, '</abstract>\n',))
        ead_xml = ''.join((ead_xml, '<langmaterial><language langcode="', self.language, '"/></langmaterial>\n',))
        ead_xml = ''.join((ead_xml, '</did>\n', ))
        ead_xml = ''.join((ead_xml, '<accessrestrict>Access
', self.accessrestrict, '
</accessrestrict>\n',))
        ead_xml = ''.join((ead_xml, '<userestrict>Publication Rights
', self.userestrict, '
</userestrict>\n',))
        ead_xml = ''.join((ead_xml, '<prefercite>Preferred Citation
', self.title, ' , ',  self.local_identifier, '.  ', self.publisher.name, '.
</prefercite>\n',))
        ead_xml = ''.join((ead_xml, '<acqinfo>Acquisition Information
', self.acqinfo, '
</acqinfo>\n',))
        if self.bioghist:
            ead_xml = ''.join((ead_xml, '<bioghist>Biography/Administrative History
', self.bioghist, '
</bioghist>',))
        ead_xml = ''.join((ead_xml, '<scopecontent>Scope and Content of Collection
', self.scopecontent, '
</scopecontent>',))
        controlaccess_xml = '<controlaccess>\nIndexing Terms\n'
        for qdc in self.subject_topic:
            controlaccess_xml = ''.join((controlaccess_xml, '<subject>', escape(qdc.content), '</subject>\n',))
        for qdc in self.subject_name_person:
            controlaccess_xml = ''.join((controlaccess_xml, '<persname role="subject">', escape(qdc.content), '</persname>\n',))
        for qdc in self.subject_name_family:
            controlaccess_xml = ''.join((controlaccess_xml, '<famname role="subject">', escape(qdc.content), '</famname>\n',))
        for qdc in self.subject_name_organization:
            controlaccess_xml = ''.join((controlaccess_xml, '<corpname role="subject">', escape(qdc.content), '</corpname>\n',))
        for qdc in self.coverage:
            controlaccess_xml = ''.join((controlaccess_xml, '<geogname role="subject">', escape(qdc.content), '</geogname>\n',))
        for qdc in self.type_format:
            controlaccess_xml = ''.join((controlaccess_xml, '<genreform role="subject">', escape(qdc.content), '</genreform>\n',))
        for qdc in self.subject_title:
            controlaccess_xml = ''.join((controlaccess_xml, '<title role="subject">', escape(qdc.content), '</title>\n',))
        for qdc in self.subject_function:
            controlaccess_xml = ''.join((controlaccess_xml, '<function role="subject">', escape(qdc.content), '</function>\n',))
        for qdc in self.subject_occupation:
            controlaccess_xml = ''.join((controlaccess_xml, '<occupation role="subject">', escape(qdc.content), '</occupation>\n',))
        controlaccess_xml = ''.join((controlaccess_xml, '</controlaccess>\n',))
        ead_xml = ''.join((ead_xml, controlaccess_xml))
        ead_xml = ''.join((ead_xml, '</archdesc>\n</ead>\n',))
        return ead_xml
<|endoftext|>


<commit_before>from __future__ import division
import numpy as np
na = np.newaxis
from matplotlib import pyplot as plt
from matplotlib import cm
import scipy.special as special
import abc, copy
from warnings import warn

from abstractions import ModelGibbsSampling, ModelMeanField, ModelEM
from abstractions import Distribution, GibbsSampling, MeanField, Collapsed, MaxLikelihood
from distributions import Categorical, CategoricalAndConcentration
from internals.labels import Labels, CRPLabels
from pyhsmm.util.stats import getdatasize

class Mixture(ModelGibbsSampling, ModelMeanField, ModelEM):
    '''
    This class is for mixtures of other distributions.
    '''
    def __init__(self,components,alpha_0=None,a_0=None,b_0=None,weights=None):
        assert len(components) > 0
        assert (alpha_0 is not None) ^ (a_0 is not None and b_0 is not None)

        self.components = components

        if alpha_0 is not None:
            self.weights = Categorical(alpha_0=alpha_0,K=len(components),weights=weights)
        else:
            self.weights = CategoricalAndConcentration(a_0=a_0,b_0=b_0,K=len(components),weights=weights)

        self.labels_list = []

    def add_data(self,data,**kwargs):
        self.labels_list.append(Labels(data=np.asarray(data),
            components=self.components,weights=self.weights,
            **kwargs))

    def generate(self,N,keep=True):
        templabels = Labels(components=self.components,weights=self.weights,N=N) # this samples labels

        out = np.empty(self.components[0].rvs(N).shape)
        counts = np.bincount(templabels.z,minlength=len(self.components))
        for idx,(c,count) in enumerate(zip(self.components,counts)):
            out[templabels.z == idx,...] = c.rvs(count)

        perm = np.random.permutation(N)
        out = out[perm]
        templabels.z = templabels.z[perm]

        if keep:
            templabels.data = out
            self.labels_list.append(templabels)

        return out, templabels.z

    def _log_likelihoods(self,x):
        x = np.asarray(x)
        K = len(self.components)
        vals = np.empty((x.shape[0],K))
        for idx, c in enumerate(self.components):
            vals[:,idx] = c.log_likelihood(x)
        vals += self.weights.log_likelihood(np.arange(K))
        assert not np.isnan(vals).any()
        return np.logaddexp.reduce(vals,axis=1)

    def log_likelihood(self,x):
        return self._log_likelihoods(x).sum()

    ### Gibbs sampling

    def resample_model(self,temp=None):
        assert all(isinstance(c,GibbsSampling) for c in self.components), \
                'Components must implement GibbsSampling'

        for l in self.labels_list:
            l.resample(temp=temp)

        for idx, c in enumerate(self.components):
            c.resample(data=[(l.data[l.z == idx] if (l.z == idx).any() else []) for l in self.labels_list]) # numpy issue #2587, np.array([]).reshape((0,2))[[]]

        self.weights.resample([l.z for l in self.labels_list])

    def copy_sample(self):
        new = copy.copy(self)
        new.components = [c.copy_sample() for c in self.components]
        new.weights = self.weights.copy_sample()
        new.labels_list = [l.copy_sample() for l in self.labels_list]
        return new

    ### Mean Field

    def meanfield_coordinate_descent_step(self):
        assert all(isinstance(c,MeanField) for c in self.components), \
                'Components must implement MeanField'
        assert len(self.labels_list) > 0, 'Must have data to run MeanField'

        ### update sweep!
        # ask labels to get weights over z, stored as l.r
        for l in self.labels_list:
            l.meanfieldupdate()

        # pass the weights to pi
        K = len(self.components)
        self.weights.meanfieldupdate(np.arange(K),[l.r for l in self.labels_list])

        # pass the weights to the components
        for idx, c in enumerate(self.components):
            c.meanfieldupdate([l.data for l in self.labels_list],
                    [l.r[:,idx] for l in self.labels_list])

        ### get vlb!
        vlb = 0.

        # get labels terms
        vlb += sum(l.get_vlb() for l in self.labels_list)

        # get pi term
        vlb += self.weights.get_vlb()

        # get components terms
        vlb += sum(c.get_vlb() for c in self.components)

        # finally, need the evidence term in the vlb
        for l in self.labels_list:
            vlb += np.sum([r.dot(c.expected_log_likelihood(l.data))
                                for c,r in zip(self.components, l.r.T)])

        # add in symmetry factor (if we're actually symmetric)
        if len(set(self.weights.weights)) == 1 and \
                len(set(type(c) for c in self.components)) == 1:
            vlb += special.gammaln(len(self.components)+1)

        return vlb

    ### EM

    def EM_step(self):
        assert all(isinstance(c,MaxLikelihood) for c in self.components), \
                'Components must implement MaxLikelihood'
        assert len(self.labels_list) > 0, 'Must have data to run EM'

        ## E step
        for l in self.labels_list:
            l.E_step()

        ## M step
        # component parameters
        for idx, c in enumerate(self.components):
            c.max_likelihood([l.data for l in self.labels_list],
                    [l.expectations[:,idx] for l in self.labels_list])

        # mixture weights
        self.weights.max_likelihood(np.arange(len(self.components)),
                [l.expectations for l in self.labels_list])

    def num_parameters(self):
        # NOTE: scikit.learn's gmm.py doesn't count the weights in the number of
        # parameters, but I don't know why they wouldn't. Some convention?
        return sum(c.num_parameters() for c in self.components) + self.weights.num_parameters()

    def BIC(self,data=None):
        '''
        BIC on the passed data.
        If passed data is None (default), calculates BIC on the model's assigned data.
        '''
        # NOTE: in principle this method computes the BIC only after finding the
        # maximum likelihood parameters (or, of course, an EM fixed-point as an
        # approximation!)
        if data is None:
            assert len(self.labels_list) > 0, \
                    "If not passing in data, the class must already have it. Use the method add_data()"
            return -2*sum(self.log_likelihood(l.data) for l in self.labels_list) + \
                        self.num_parameters() * np.log(sum(l.data.shape[0] for l in self.labels_list))
        else:
            return -2*self.log_likelihood(data) + self.num_parameters() * np.log(data.shape[0])

    def AIC(self):
        # NOTE: in principle this method computes the AIC only after finding the
        # maximum likelihood parameters (or, of course, an EM fixed-point as an
        # approximation!)
        assert len(self.labels_list) > 0, 'Must have data to get AIC'
        return 2*self.num_parameters() - 2*sum(self.log_likelihood(l.data) for l in self.labels_list)

    ### Misc.

    def plot(self,color=None,legend=True):
        cmap = cm.get_cmap()

        if len(self.labels_list) > 0:
            label_colors = {}

            used_labels = reduce(set.union,[set(l.z) for l in self.labels_list],set([]))
            num_labels = len(used_labels)
            num_subfig_rows = len(self.labels_list)

            for idx,label in enumerate(used_labels):
                label_colors[label] = idx/(num_labels-1 if num_labels > 1 else 1) \
                        if color is None else color

            for subfigidx,l in enumerate(self.labels_list):
                # plot the current observation distributions (and obs. if given)
                plt.subplot(num_subfig_rows,1,1+subfigidx)
                for label, o in enumerate(self.components):
                    if label in l.z:
                        o.plot(color=cmap(label_colors[label]),
                                data=(l.data[l.z == label] if l.data is not None else None),
                                label='%d' % label)

            if legend:
                plt.legend(
                        [plt.Rectangle((0,0),1,1,fc=cmap(c))
                            for i,c in label_colors.iteritems() if i in used_labels],
                        [i for i in label_colors if i in used_labels],
                        loc='best'
                        )

        else:
            top10 = np.array(self.components)[np.argsort(self.weights.weights)][-1:-11:-1]
            colors = [cmap(x) for x in np.linspace(0,1,len(top10))] if color is None \
                    else [color]*len(top10)
            for i,(o,c) in enumerate(zip(top10,colors)):
                o.plot(color=c,label='%d' % i)

    def to_json_dict(self):
        assert len(self.labels_list) == 1
        data = self.labels_list[0].data
        z = self.labels_list[0].z
        assert data.ndim == 2 and data.shape[1] == 2

        return  {
                    'points':[{'x':x,'y':y,'label':int(label)} for x,y,label in zip(data[:,0],data[:,1],z)],
                    'ellipses':[dict(c.to_json_dict().items() + [('label',i)])
                        for i,c in enumerate(self.components) if i in z]
                }

    def predictive_likelihoods(self,test_data,forecast_horizons):
        likes = self._log_likelihoods(test_data)
        return [likes[k:] for k in forecast_horizons]

    def block_predictive_likelihoods(self,test_data,blocklens):
        csums = np.cumsum(self._log_likelihoods(test_data))
        outs = []
        for k in blocklens:
            outs.append(csums[k:] - csums[:-k])
        return outs

class MixtureDistribution(Mixture, GibbsSampling, Distribution):
    '''
    This makes a Mixture act like a Distribution for use in other compound models
    '''

    def log_likelihood(self,x):
        return self._log_likelihoods(x)

    def resample(self,data,niter=25,temp=None):
        # doesn't keep a reference to the data like a model would
        assert isinstance(data,list) or isinstance(data,np.ndarray)

        if getdatasize(data) > 0:
            if not isinstance(data,np.ndarray):
                data = np.concatenate(data)

            self.add_data(data)

            for itr in range(niter):
                self.resample_model(temp=temp)

            self.labels_list.pop()
        else:
            self.resample_model(temp=temp)

    def max_likelihood(self,data,weights=None):
        if weights is not None:
            raise NotImplementedError
        assert isinstance(data,list) or isinstance(data,np.ndarray)
        if isinstance(data,list):
            data = np.concatenate(data)

        if getdatasize(data) > 0:
            self.add_data(data)
            self.EM_fit()
            self.labels_list = []

    def plot(self,data=[],color='b',label='',plot_params=True):
        if not isinstance(data,list):
            data = [data]
        for d in data:
            self.add_data(d)

        for l in self.labels_list:
            l.E_step() # sets l.z to MAP estimates
            for label, o in enumerate(self.components):
                if label in l.z:
                    o.plot(color=color,label=label,
                            data=l.data[l.z == label] if l.data is not None else None)

        for d in data:
            self.labels_list.pop()

class CollapsedMixture(ModelGibbsSampling):
    __metaclass__ = abc.ABCMeta
    def _get_counts(self,k):
        return sum(l._get_counts(k) for l in self.labels_list)

    def _get_data_withlabel(self,k):
        return [l._get_data_withlabel(k) for l in self.labels_list]

    def _get_occupied(self):
        return reduce(set.union,(l._get_occupied() for l in self.labels_list),set([]))

    def plot(self):
        plt.figure()
        cmap = cm.get_cmap()
        used_labels = self._get_occupied()
        num_labels = len(used_labels)

        label_colors = {}
        for idx,label in enumerate(used_labels):
            label_colors[label] = idx/(num_labels-1. if num_labels > 1 else 1.)

        for subfigidx,l in enumerate(self.labels_list):
            plt.subplot(len(self.labels_list),1,1+subfigidx)
            # TODO assuming data is 2D
            for label in used_labels:
                if label in l.z:
                    plt.plot(l.data[l.z==label,0],l.data[l.z==label,1],
                            color=cmap(label_colors[label]),ls='None',marker='x')

class CRPMixture(CollapsedMixture):
    def __init__(self,alpha_0,obs_distn):
        assert isinstance(obs_distn,Collapsed)
        self.obs_distn = obs_distn
        self.alpha_0 = alpha_0

        self.labels_list = []

    def add_data(self,data):
        assert len(self.labels_list) == 0
        self.labels_list.append(CRPLabels(model=self,data=np.asarray(data),
            alpha_0=self.alpha_0,obs_distn=self.obs_distn))

    def resample_model(self):
        for l in self.labels_list:
            l.resample()

    def generate(self,N,keep=True):
        warn('not fully implemented')
        # TODO only works if there's no other data in the model; o/w need to add
        # existing data to obs resample. should be an easy update.
        # templabels needs to pay attention to its own counts as well as model
        # counts
        assert len(self.labels_list) == 0

        templabels = CRPLabels(model=self,alpha_0=self.alpha_0,obs_distn=self.obs_distn,N=N)

        counts = np.bincount(templabels.z)
        out = np.empty(self.obs_distn.rvs(N).shape)
        for idx, count in enumerate(counts):
            self.obs_distn.resample()
            out[templabels.z == idx,...] = self.obs_distn.rvs(count)

        perm = np.random.permutation(N)
        out = out[perm]
        templabels.z = templabels.z[perm]

        if keep:
            templabels.data = out
            self.labels_list.append(templabels)

        return out, templabels.z

<commit_msg>changed resampling order<commit_after>from __future__ import division
import numpy as np
na = np.newaxis
from matplotlib import pyplot as plt
from matplotlib import cm
import scipy.special as special
import abc, copy
from warnings import warn

from abstractions import ModelGibbsSampling, ModelMeanField, ModelEM
from abstractions import Distribution, GibbsSampling, MeanField, Collapsed, MaxLikelihood
from distributions import Categorical, CategoricalAndConcentration
from internals.labels import Labels, CRPLabels
from pyhsmm.util.stats import getdatasize

class Mixture(ModelGibbsSampling, ModelMeanField, ModelEM):
    '''
    This class is for mixtures of other distributions.
    '''
    def __init__(self,components,alpha_0=None,a_0=None,b_0=None,weights=None):
        assert len(components) > 0
        assert (alpha_0 is not None) ^ (a_0 is not None and b_0 is not None)

        self.components = components

        if alpha_0 is not None:
            self.weights = Categorical(alpha_0=alpha_0,K=len(components),weights=weights)
        else:
            self.weights = CategoricalAndConcentration(a_0=a_0,b_0=b_0,K=len(components),weights=weights)

        self.labels_list = []

    def add_data(self,data,**kwargs):
        self.labels_list.append(Labels(data=np.asarray(data),
            components=self.components,weights=self.weights,
            **kwargs))

    def generate(self,N,keep=True):
        templabels = Labels(components=self.components,weights=self.weights,N=N) # this samples labels

        out = np.empty(self.components[0].rvs(N).shape)
        counts = np.bincount(templabels.z,minlength=len(self.components))
        for idx,(c,count) in enumerate(zip(self.components,counts)):
            out[templabels.z == idx,...] = c.rvs(count)

        perm = np.random.permutation(N)
        out = out[perm]
        templabels.z = templabels.z[perm]

        if keep:
            templabels.data = out
            self.labels_list.append(templabels)

        return out, templabels.z

    def _log_likelihoods(self,x):
        x = np.asarray(x)
        K = len(self.components)
        vals = np.empty((x.shape[0],K))
        for idx, c in enumerate(self.components):
            vals[:,idx] = c.log_likelihood(x)
        vals += self.weights.log_likelihood(np.arange(K))
        assert not np.isnan(vals).any()
        return np.logaddexp.reduce(vals,axis=1)

    def log_likelihood(self,x):
        return self._log_likelihoods(x).sum()

    ### Gibbs sampling

    def resample_model(self,temp=None):
        assert all(isinstance(c,GibbsSampling) for c in self.components), \
                'Components must implement GibbsSampling'
        for idx, c in enumerate(self.components):
            c.resample(data=[(l.data[l.z == idx] if (l.z == idx).any() else []) for l in self.labels_list]) # numpy issue #2587, np.array([]).reshape((0,2))[[]]

        self.weights.resample([l.z for l in self.labels_list])

        for l in self.labels_list:
            l.resample(temp=temp)

    def copy_sample(self):
        new = copy.copy(self)
        new.components = [c.copy_sample() for c in self.components]
        new.weights = self.weights.copy_sample()
        new.labels_list = [l.copy_sample() for l in self.labels_list]
        return new

    ### Mean Field

    def meanfield_coordinate_descent_step(self):
        assert all(isinstance(c,MeanField) for c in self.components), \
                'Components must implement MeanField'
        assert len(self.labels_list) > 0, 'Must have data to run MeanField'

        ### update sweep!
        # ask labels to get weights over z, stored as l.r
        for l in self.labels_list:
            l.meanfieldupdate()

        # pass the weights to pi
        K = len(self.components)
        self.weights.meanfieldupdate(np.arange(K),[l.r for l in self.labels_list])

        # pass the weights to the components
        for idx, c in enumerate(self.components):
            c.meanfieldupdate([l.data for l in self.labels_list],
                    [l.r[:,idx] for l in self.labels_list])

        ### get vlb!
        vlb = 0.

        # get labels terms
        vlb += sum(l.get_vlb() for l in self.labels_list)

        # get pi term
        vlb += self.weights.get_vlb()

        # get components terms
        vlb += sum(c.get_vlb() for c in self.components)

        # finally, need the evidence term in the vlb
        for l in self.labels_list:
            vlb += np.sum([r.dot(c.expected_log_likelihood(l.data))
                                for c,r in zip(self.components, l.r.T)])

        # add in symmetry factor (if we're actually symmetric)
        if len(set(self.weights.weights)) == 1 and \
                len(set(type(c) for c in self.components)) == 1:
            vlb += special.gammaln(len(self.components)+1)

        return vlb

    ### EM

    def EM_step(self):
        assert all(isinstance(c,MaxLikelihood) for c in self.components), \
                'Components must implement MaxLikelihood'
        assert len(self.labels_list) > 0, 'Must have data to run EM'

        ## E step
        for l in self.labels_list:
            l.E_step()

        ## M step
        # component parameters
        for idx, c in enumerate(self.components):
            c.max_likelihood([l.data for l in self.labels_list],
                    [l.expectations[:,idx] for l in self.labels_list])

        # mixture weights
        self.weights.max_likelihood(np.arange(len(self.components)),
                [l.expectations for l in self.labels_list])

    def num_parameters(self):
        # NOTE: scikit.learn's gmm.py doesn't count the weights in the number of
        # parameters, but I don't know why they wouldn't. Some convention?
        return sum(c.num_parameters() for c in self.components) + self.weights.num_parameters()

    def BIC(self,data=None):
        '''
        BIC on the passed data.
        If passed data is None (default), calculates BIC on the model's assigned data.
        '''
        # NOTE: in principle this method computes the BIC only after finding the
        # maximum likelihood parameters (or, of course, an EM fixed-point as an
        # approximation!)
        if data is None:
            assert len(self.labels_list) > 0, \
                    "If not passing in data, the class must already have it. Use the method add_data()"
            return -2*sum(self.log_likelihood(l.data) for l in self.labels_list) + \
                        self.num_parameters() * np.log(sum(l.data.shape[0] for l in self.labels_list))
        else:
            return -2*self.log_likelihood(data) + self.num_parameters() * np.log(data.shape[0])

    def AIC(self):
        # NOTE: in principle this method computes the AIC only after finding the
        # maximum likelihood parameters (or, of course, an EM fixed-point as an
        # approximation!)
        assert len(self.labels_list) > 0, 'Must have data to get AIC'
        return 2*self.num_parameters() - 2*sum(self.log_likelihood(l.data) for l in self.labels_list)

    ### Misc.

    def plot(self,color=None,legend=True):
        cmap = cm.get_cmap()

        if len(self.labels_list) > 0:
            label_colors = {}

            used_labels = reduce(set.union,[set(l.z) for l in self.labels_list],set([]))
            num_labels = len(used_labels)
            num_subfig_rows = len(self.labels_list)

            for idx,label in enumerate(used_labels):
                label_colors[label] = idx/(num_labels-1 if num_labels > 1 else 1) \
                        if color is None else color

            for subfigidx,l in enumerate(self.labels_list):
                # plot the current observation distributions (and obs. if given)
                plt.subplot(num_subfig_rows,1,1+subfigidx)
                for label, o in enumerate(self.components):
                    if label in l.z:
                        o.plot(color=cmap(label_colors[label]),
                                data=(l.data[l.z == label] if l.data is not None else None),
                                label='%d' % label)

            if legend:
                plt.legend(
                        [plt.Rectangle((0,0),1,1,fc=cmap(c))
                            for i,c in label_colors.iteritems() if i in used_labels],
                        [i for i in label_colors if i in used_labels],
                        loc='best'
                        )

        else:
            top10 = np.array(self.components)[np.argsort(self.weights.weights)][-1:-11:-1]
            colors = [cmap(x) for x in np.linspace(0,1,len(top10))] if color is None \
                    else [color]*len(top10)
            for i,(o,c) in enumerate(zip(top10,colors)):
                o.plot(color=c,label='%d' % i)

    def to_json_dict(self):
        assert len(self.labels_list) == 1
        data = self.labels_list[0].data
        z = self.labels_list[0].z
        assert data.ndim == 2 and data.shape[1] == 2

        return  {
                    'points':[{'x':x,'y':y,'label':int(label)} for x,y,label in zip(data[:,0],data[:,1],z)],
                    'ellipses':[dict(c.to_json_dict().items() + [('label',i)])
                        for i,c in enumerate(self.components) if i in z]
                }

    def predictive_likelihoods(self,test_data,forecast_horizons):
        likes = self._log_likelihoods(test_data)
        return [likes[k:] for k in forecast_horizons]

    def block_predictive_likelihoods(self,test_data,blocklens):
        csums = np.cumsum(self._log_likelihoods(test_data))
        outs = []
        for k in blocklens:
            outs.append(csums[k:] - csums[:-k])
        return outs

class MixtureDistribution(Mixture, GibbsSampling, Distribution):
    '''
    This makes a Mixture act like a Distribution for use in other compound models
    '''

    def log_likelihood(self,x):
        return self._log_likelihoods(x)

    def resample(self,data,niter=25,temp=None):
        # doesn't keep a reference to the data like a model would
        assert isinstance(data,list) or isinstance(data,np.ndarray)

        if getdatasize(data) > 0:
            if not isinstance(data,np.ndarray):
                data = np.concatenate(data)

            self.add_data(data)

            for itr in range(niter):
                self.resample_model(temp=temp)

            self.labels_list.pop()
        else:
            self.resample_model(temp=temp)

    def max_likelihood(self,data,weights=None):
        if weights is not None:
            raise NotImplementedError
        assert isinstance(data,list) or isinstance(data,np.ndarray)
        if isinstance(data,list):
            data = np.concatenate(data)

        if getdatasize(data) > 0:
            self.add_data(data)
            self.EM_fit()
            self.labels_list = []

    def plot(self,data=[],color='b',label='',plot_params=True):
        if not isinstance(data,list):
            data = [data]
        for d in data:
            self.add_data(d)

        for l in self.labels_list:
            l.E_step() # sets l.z to MAP estimates
            for label, o in enumerate(self.components):
                if label in l.z:
                    o.plot(color=color,label=label,
                            data=l.data[l.z == label] if l.data is not None else None)

        for d in data:
            self.labels_list.pop()

class CollapsedMixture(ModelGibbsSampling):
    __metaclass__ = abc.ABCMeta
    def _get_counts(self,k):
        return sum(l._get_counts(k) for l in self.labels_list)

    def _get_data_withlabel(self,k):
        return [l._get_data_withlabel(k) for l in self.labels_list]

    def _get_occupied(self):
        return reduce(set.union,(l._get_occupied() for l in self.labels_list),set([]))

    def plot(self):
        plt.figure()
        cmap = cm.get_cmap()
        used_labels = self._get_occupied()
        num_labels = len(used_labels)

        label_colors = {}
        for idx,label in enumerate(used_labels):
            label_colors[label] = idx/(num_labels-1. if num_labels > 1 else 1.)

        for subfigidx,l in enumerate(self.labels_list):
            plt.subplot(len(self.labels_list),1,1+subfigidx)
            # TODO assuming data is 2D
            for label in used_labels:
                if label in l.z:
                    plt.plot(l.data[l.z==label,0],l.data[l.z==label,1],
                            color=cmap(label_colors[label]),ls='None',marker='x')

class CRPMixture(CollapsedMixture):
    def __init__(self,alpha_0,obs_distn):
        assert isinstance(obs_distn,Collapsed)
        self.obs_distn = obs_distn
        self.alpha_0 = alpha_0

        self.labels_list = []

    def add_data(self,data):
        assert len(self.labels_list) == 0
        self.labels_list.append(CRPLabels(model=self,data=np.asarray(data),
            alpha_0=self.alpha_0,obs_distn=self.obs_distn))

    def resample_model(self):
        for l in self.labels_list:
            l.resample()

    def generate(self,N,keep=True):
        warn('not fully implemented')
        # TODO only works if there's no other data in the model; o/w need to add
        # existing data to obs resample. should be an easy update.
        # templabels needs to pay attention to its own counts as well as model
        # counts
        assert len(self.labels_list) == 0

        templabels = CRPLabels(model=self,alpha_0=self.alpha_0,obs_distn=self.obs_distn,N=N)

        counts = np.bincount(templabels.z)
        out = np.empty(self.obs_distn.rvs(N).shape)
        for idx, count in enumerate(counts):
            self.obs_distn.resample()
            out[templabels.z == idx,...] = self.obs_distn.rvs(count)

        perm = np.random.permutation(N)
        out = out[perm]
        templabels.z = templabels.z[perm]

        if keep:
            templabels.data = out
            self.labels_list.append(templabels)

        return out, templabels.z

<|endoftext|>


<commit_before># -*- coding: utf-8 -*-

import time
from multiprocessing import Queue, Process
import weakref
from threading import Thread
# linux 默认就是epoll
from twisted.internet import reactor

from .log import logger
from .connection import ConnectionFactory
from .worker import Worker
from .callbacks_mixin import RoutesMixin
from .request import Request
from . import autoreload

class Melon(RoutesMixin):

    parent_input = None
    parent_output = None
    conn_dict = None

    server = None
    blueprints = None
    debug = False

    def __init__(self, box_class, conn_factory_class=None, request_class=None,
                 input_queue_maxsize=None, output_queue_maxsize=None):
        super(Melon, self).__init__()
        self.box_class = box_class
        self.conn_factory_class = conn_factory_class or ConnectionFactory

        self.blueprints = list()
        self.parent_input = Queue(input_queue_maxsize)
        self.parent_output = Queue(output_queue_maxsize)
        self.conn_dict = weakref.WeakValueDictionary()
        self.request_class = request_class or Request

    def register_blueprint(self, blueprint):
        blueprint.register2app(self)

    def run(self, host, port, debug=None, use_reloader=None, workers=1):
        if debug is not None:
            self.debug = debug
        use_reloader = use_reloader if use_reloader is not None else self.debug

        def run_wrapper():
            assert workers >= 1

            logger.info('Running server on %s:%s, debug: %s, use_reloader: %s',
                        host, port, self.debug, use_reloader)

            self._spawn_poll_worker_result_thread()
            self._spawn_fork_workers(workers)

            reactor.listenTCP(port, self.conn_factory_class(self, self.box_class), interface=host)

            # 否则会报exceptions.ValueError: signal only works in main thread
            installSignalHandlers = False if use_reloader else True

            try:
                reactor.run(installSignalHandlers=installSignalHandlers)
            except KeyboardInterrupt:
                pass
            except:
                logger.error('exc occur.', exc_info=True)

        if use_reloader:
            autoreload.main(run_wrapper)
        else:
            run_wrapper()

    def _spawn_poll_worker_result_thread(self):
        """
        启动获取worker数据的线程
        """
        thread = Thread(target=self._poll_worker_result)
        thread.daemon = True
        thread.start()

    def _spawn_fork_workers(self, workers):
        """
        通过线程启动多个worker
        """
        thread = Thread(target=self._fork_workers, args=(workers,))
        thread.daemon = True
        thread.start()

    def _fork_workers(self, workers):
        def make_worker_process():
            inner_p = Process(target=Worker(self, self.box_class, self.request_class).run)
            inner_p._daemonic = True
            return inner_p

        p_list = []

        for it in xrange(0, workers):
            p = make_worker_process()
            p.start()
            p_list.append(p)

        while True:
            for idx, p in enumerate(p_list):
                if not p.is_alive():
                    old_pid = p.pid
                    p = make_worker_process()
                    p.start()
                    p_list[idx] = p

                    logger.error('process[%s] dead. start new process[%s]', old_pid, p.pid)

            time.sleep(1)

    def _poll_worker_result(self):
        """
        从队列里面获取worker的返回
        """

        def handle_data(_conn, _data):
            if _conn and _conn.transport:
                try:
                    if _data:
                        _conn.transport.write(_data)
                    else:
                        # data 为NULL代表关闭链接的意思
                        _conn.transport.loseConnection()
                except:
                    logger.error('exc occur. msg: %r', msg, exc_info=True)

        while True:
            try:
                msg = self.parent_input.get()
            except KeyboardInterrupt:
                break
            except:
                logger.error('exc occur.', exc_info=True)
                break

            conn = self.conn_dict.get(msg.get('conn_id'))
            data = msg.get('data')

            # 参考 http://twistedsphinx.funsize.net/projects/core/howto/threading.html
            reactor.callFromThread(handle_data, conn, data)
<commit_msg>daemon<commit_after># -*- coding: utf-8 -*-

import time
from multiprocessing import Queue, Process
import weakref
from threading import Thread
# linux 默认就是epoll
from twisted.internet import reactor

from .log import logger
from .connection import ConnectionFactory
from .worker import Worker
from .callbacks_mixin import RoutesMixin
from .request import Request
from . import autoreload

class Melon(RoutesMixin):

    parent_input = None
    parent_output = None
    conn_dict = None

    server = None
    blueprints = None
    debug = False

    def __init__(self, box_class, conn_factory_class=None, request_class=None,
                 input_queue_maxsize=None, output_queue_maxsize=None):
        super(Melon, self).__init__()
        self.box_class = box_class
        self.conn_factory_class = conn_factory_class or ConnectionFactory

        self.blueprints = list()
        self.parent_input = Queue(input_queue_maxsize)
        self.parent_output = Queue(output_queue_maxsize)
        self.conn_dict = weakref.WeakValueDictionary()
        self.request_class = request_class or Request

    def register_blueprint(self, blueprint):
        blueprint.register2app(self)

    def run(self, host, port, debug=None, use_reloader=None, workers=1):
        if debug is not None:
            self.debug = debug
        use_reloader = use_reloader if use_reloader is not None else self.debug

        def run_wrapper():
            assert workers >= 1

            logger.info('Running server on %s:%s, debug: %s, use_reloader: %s',
                        host, port, self.debug, use_reloader)

            self._spawn_poll_worker_result_thread()
            self._spawn_fork_workers(workers)

            reactor.listenTCP(port, self.conn_factory_class(self, self.box_class), interface=host)

            # 否则会报exceptions.ValueError: signal only works in main thread
            installSignalHandlers = False if use_reloader else True

            try:
                reactor.run(installSignalHandlers=installSignalHandlers)
            except KeyboardInterrupt:
                pass
            except:
                logger.error('exc occur.', exc_info=True)

        if use_reloader:
            autoreload.main(run_wrapper)
        else:
            run_wrapper()

    def _spawn_poll_worker_result_thread(self):
        """
        启动获取worker数据的线程
        """
        thread = Thread(target=self._poll_worker_result)
        thread.daemon = True
        thread.start()

    def _spawn_fork_workers(self, workers):
        """
        通过线程启动多个worker
        """
        thread = Thread(target=self._fork_workers, args=(workers,))
        thread.daemon = True
        thread.start()

    def _fork_workers(self, workers):
        def make_worker_process():
            inner_p = Process(target=Worker(self, self.box_class, self.request_class).run)
            inner_p.daemon = True
            return inner_p

        p_list = []

        for it in xrange(0, workers):
            p = make_worker_process()
            p.start()
            p_list.append(p)

        while True:
            for idx, p in enumerate(p_list):
                if not p.is_alive():
                    old_pid = p.pid
                    p = make_worker_process()
                    p.start()
                    p_list[idx] = p

                    logger.error('process[%s] dead. start new process[%s]', old_pid, p.pid)

            time.sleep(1)

    def _poll_worker_result(self):
        """
        从队列里面获取worker的返回
        """

        def handle_data(_conn, _data):
            if _conn and _conn.transport:
                try:
                    if _data:
                        _conn.transport.write(_data)
                    else:
                        # data 为NULL代表关闭链接的意思
                        _conn.transport.loseConnection()
                except:
                    logger.error('exc occur. msg: %r', msg, exc_info=True)

        while True:
            try:
                msg = self.parent_input.get()
            except KeyboardInterrupt:
                break
            except:
                logger.error('exc occur.', exc_info=True)
                break

            conn = self.conn_dict.get(msg.get('conn_id'))
            data = msg.get('data')

            # 参考 http://twistedsphinx.funsize.net/projects/core/howto/threading.html
            reactor.callFromThread(handle_data, conn, data)
<|endoftext|>


<commit_before>"""Tools and helper functions for abinit calculations"""
from __future__ import print_function, division

import os.path
import collections
import shutil
import operator

from pymatgen.util.string_utils import list_strings, StringColorizer

import logging
logger = logging.getLogger(__name__)

class File(object):
    """
    Very simple class used to store file basenames, absolute paths and directory names.
    Provides wrappers for the most commonly used functions defined in os.path.
    """
    def __init__(self, path):
        self._path = os.path.abspath(path)

    def __repr__(self):
        return "<%s at %s, %s>" % (self.__class__.__name__, id(self), self.path)

    def __str__(self):
        return "<%s, %s>" % (self.__class__.__name__, self.path)

    def __eq__(self, other):
        if other is None: return False
        self.path == other.path

    def __ne__(self, other):
        return not self.__eq__(other)

    @property
    def path(self):
        """Absolute path of the file."""
        return self._path

    @property
    def basename(self):
        """File basename."""
        return os.path.basename(self.path)

    @property
    def dirname(self):
        """Absolute path of the directory where the file is located."""
        return os.path.dirname(self.path)

    @property
    def exists(self):
        "True if file exists."
        return os.path.exists(self.path)

    @property
    def isncfile(self):
        "True if self is a NetCDF file"
        return self.basename.endswith(".nc")

    def read(self):
        """Read data from file."""
        with open(self.path, "r") as f:
            return f.read()

    def readlines(self):
        """Read lines from files."""
        with open(self.path, "r") as f:
            return f.readlines()

    def write(self, string):
        """Write string to file."""
        self.make_dir()
        with open(self.path, "w") as f:
            return f.write(string)

    def writelines(self, lines):
        """Write a list of strings to file."""
        self.make_dir()
        with open(self.path, "w") as f:
            return f.writelines()

    def make_dir(self):
        """Make the directory where the file is located."""
        if not os.path.exists(self.dirname):
            os.makedirs(self.dirname)

class Directory(object):
    """
    Very simple class that provides helper functions
    wrapping the most commonly used functions defined in os.path.
    """
    def __init__(self, path):
        self._path = os.path.abspath(path)

    def __repr__(self):
        return "<%s at %s, %s>" % (self.__class__.__name__, id(self), self.path)

    def __str__(self):
        return "<%s, %s>" % (self.__class__.__name__, self.path)

    def __eq__(self, other):
        if other is None: return False
        self.path == other.path

    def __ne__(self, other):
        return not self.__eq__(other)

    @property
    def path(self):
        """Absolute path of the directory."""
        return self._path

    @property
    def basename(self):
        """Directory basename."""
        return os.path.basename(self.path)

    def path_join(self, *p):
        """
        Join two or more pathname components, inserting '/' as needed.
        If any component is an absolute path, all previous path components will be discarded.
        """
        return os.path.join(self.path, *p)

    @property
    def exists(self):
        "True if file exists."
        return os.path.exists(self.path)

    def makedirs(self):
        """
        Super-mkdir; create a leaf directory and all intermediate ones.
        Works like mkdir, except that any intermediate path segment (not
        just the rightmost) will be created if it does not exist.
        """
        if not self.exists:
            os.makedirs(self.path)

    def rmtree(self):
        """Recursively delete the directory tree"""
        shutil.rmtree(self.path, ignore_errors=True)

    def path_in(self, filename):
        """Return the absolute path of filename in the directory."""
        return os.path.join(self.path, filename)

    def list_filepaths(self):
        """Return the list of absolute filepaths in the top-level directory."""
        fnames = [f for f in os.listdir(self.path)]
        return [os.path.join(self.path, f) for f in fnames]

    def has_abiext(self, ext):
        """
        Returns the absolute path of the ABINIT file with extension ext.
        Support both Fortran files and netcdf files. In the later case,
        we check whether a file with extension ext + ".nc" is present
        in the directory. Returns empty string is file is not present.

        Raises:
            ValueError if multiple files with the given ext are found.
            This implies that this method is not compatible with multiple datasets.
        """
        files = []
        for f in self.list_filepaths():
            if f.endswith(ext) or f.endswith(ext + ".nc"):
                files.append(f)

        if not files:
            return ""

        if len(files) > 1:
            # ABINIT users must learn that multiple datasets are bad!
            err_msg = "Found multiple files with the same extensions\n Please avoid the use of mutiple datasets!"
            raise ValueError(err_msg)

        return files[0]

# This dictionary maps ABINIT file extensions to the
# variables that must be used to read the file in input.
#
# TODO: It would be nice to pass absolute paths to abinit with getden_path
# so that I can avoid creating symbolic links before running but
# the presence of the C-bindings complicates the implementation
# (gfortran SIGFAULTs if I add strings to dataset_type!
_EXT2VARS = {
    "DEN": {"irdden": 1},
    "WFK": {"irdwfk": 1},
    "SCR": {"irdscr": 1},
    "QPS": {"irdqps": 1},
    "1WF": {"ird1wf": 1},
    "1DEN": {"ird1den": 1},
    "BSR": {"irdbsreso": 1},
    "BSC": {"irdbscoup": 1},
    "HAYDR_SAVE": {"irdhaydock": 1},
    "DDK": {"irdddk": 1},
    "DDB": {},
    "GKK": {},
    "DKK": {},
}

def irdvars_for_ext(ext):
    """
    Returns a dictionary with the ABINIT variables
    that must be used to read the file with extension ext.
    """
    return _EXT2VARS[ext].copy()

def abi_extensions():
    """List with all the ABINIT extensions that are registered."""
    return list(_EXT2VARS.keys())[:]

def abi_splitext(filename):
    """
    Split the ABINIT extension from a filename.
    "Extension" are found by searching in an internal database.

    Returns "(root, ext)" where ext is the registered ABINIT extension
    The final ".nc" is included (if any)

    >>> abi_splitext("foo_WFK")
    ('foo_', 'WFK')

    >>> abi_splitext("/home/guido/foo_bar_WFK.nc")
    ('foo_bar_', 'WFK.nc')
    """
    filename = os.path.basename(filename)
    is_ncfile = False
    if filename.endswith(".nc"):
        is_ncfile = True
        filename = filename[:-3]

    known_extensions = abi_extensions()

    # This algorith fails if we have two files
    # e.g. HAYDR_SAVE, ANOTHER_HAYDR_SAVE
    for i in range(len(filename)-1, -1, -1):
        ext = filename[i:]
        if ext in known_extensions:
            break

    else:
        raise ValueError("Cannot find a registered extension in %s" % filename)

    root = filename[:i]
    if is_ncfile:
        ext = ext + ".nc"

    return root, ext

class FilepathFixer(object):
    """
    This object modifies the names of particular output files
    produced by ABINIT so that the file extension is preserved.
    Having a one-to-one mapping between file extension and data format
    is indeed fundamental for the correct behaviour of abinitio since:

        - We locate the output file by just inspecting the extension

        - We select the variables that must be added to the input file
          on the basis of the extension specified by the user during
          the initialization of the `AbinitFlow`.

    Unfortunately, ABINIT developers like to append extra stuff
    to the initial extension and therefore we have to call
    `FilepathFixer` to fix the output files produced by the run.

    Example:

    >>> fixer = FilepathFixer()

    >>> fixer.fix_paths('/foo/out_1WF17')
    {'/foo/out_1WF17': '/foo/out_1WF'}

    >>> fixer.fix_paths('/foo/out_1WF5.nc')
    {'/foo/out_1WF5.nc': '/foo/out_1WF.nc'}
    """
    def __init__(self):
        # dictionary mapping the *official* file extension to
        # the regular expression used to tokenize the basename of the file
        # To add a new fix it's sufficient to add a new regexp and
        # a static method _fix_EXTNAME
        self.regs = regs = {}
        import re
        regs["1WF"] = re.compile("(\w+_)1WF(\d+)(.nc)?$")
        regs["1DEN"] = re.compile("(\w+_)1DEN(\d+)(.nc)?$")

    @staticmethod
    def _fix_1WF(match):
        root, pert, ncext = match.groups()
        if ncext is None: ncext = ""
        return root + "1WF" + ncext

    @staticmethod
    def _fix_1DEN(match):
        root, pert, ncext = match.groups()
        if ncext is None: ncext = ""
        return root + "1DEN" + ncext

    def _fix_path(self, path):
        for ext, regex in self.regs.items():
            head, tail = os.path.split(path)

            match = regex.match(tail)
            if match:
                newtail = getattr(self, "_fix_" + ext)(match)
                newpath = os.path.join(head, newtail)
                return newpath, ext

        return None, None

    def fix_paths(self, paths):
        """
        Fix the filenames in the iterable paths

        Returns:
            old2new:
                Mapping old_path --> new_path
        """
        old2new, fixed_exts = {}, []

        for path in list_strings(paths):
            newpath, ext = self._fix_path(path)

            if newpath is not None:
                assert ext not in fixed_exts
                fixed_exts.append(ext)
                old2new[path] = newpath

        return old2new

def _bop_not(obj):
    """Boolean not."""
    return not bool(obj)

def _bop_and(obj1, obj2):
    """Boolean and."""
    return bool(obj1) and bool(obj2)

def _bop_or(obj1, obj2):
    """Boolean or."""
    return bool(obj1) or bool(obj2)

def _bop_divisible(num1, num2):
    """Return True if num1 is divisible by num2."""
    return (num1 % num2) == 0.0

# Mapping string --> operator.
_UNARY_OPS = {
    "$not": _bop_not,
}

_BIN_OPS = {
    "$eq": operator.eq,
    "$ne": operator.ne,
    "$gt": operator.gt,
    "$ge": operator.ge,
    "$lt": operator.lt,
    "$le": operator.le,
    "$divisible": _bop_divisible,
    "$and": _bop_and,
    "$or":  _bop_or,
    }

ops = _UNARY_OPS.keys() + _BIN_OPS.keys()

def map2rpn(map, obj):
    """
    Convert a Mongodb-like dictionary to a RPN list of operands and operators.

    Reverse Polish notation (RPN) is a mathematical notation in which every
    operator follows all of its operands, e.g.

    3 - 4 + 5 -->   3 4 - 5 +

    >>> d = {2.0: {'$eq': 1.0}}
    >>> map2rpn(d, None)
    [2.0, 1.0, '$eq']
    """
    rpn = []

    for k, v in map.items():

        if k in ops:
            if isinstance(v, collections.Mapping):
                # e.g "$not": {"$gt": "one"}
                # print("in op_vmap",k, v)
                values = map2rpn(v, obj)
                rpn.extend(values)
                rpn.append(k)

            elif isinstance(v, (list, tuple)):
                # e.g "$and": [{"$not": {"one": 1.0}}, {"two": {"$lt": 3}}]}
                # print("in_op_list",k, v)
                for d in v:
                    rpn.extend(map2rpn(d, obj))

                rpn.append(k)

            else:
                # Examples
                # 1) "$eq"": "attribute_name"
                # 2) "$eq"": 1.0
                try:
                    #print("in_otherv",k, v)
                    rpn.append(getattr(obj, v))
                    rpn.append(k)

                except TypeError:
                    #print("in_otherv, raised",k, v)
                    rpn.extend([v, k])
        else:
            try:
                k = getattr(obj, k)
            except TypeError:
                k = k

            if isinstance(v, collections.Mapping):
                # "one": {"$eq": 1.0}}
                values = map2rpn(v, obj)
                rpn.append(k)
                rpn.extend(values)
            else:
                #"one": 1.0
                rpn.extend([k, v, "$eq"])

    return rpn

def evaluate_rpn(rpn):
    """
    Evaluates the RPN form produced my map2rpn.

    Returns:
        bool
    """
    vals_stack = []

    for item in rpn:

        if item in ops:
            # Apply the operator and push to the task.
            v2 = vals_stack.pop()

            if item in _UNARY_OPS:
                res = _UNARY_OPS[item](v2)

            elif item in _BIN_OPS:
                v1 = vals_stack.pop()
                res = _BIN_OPS[item](v1, v2)
            else:
                raise ValuError("%s not in unary_ops or bin_ops" % str(item))

            vals_stack.append(res)

        else:
            # Push the operand
            vals_stack.append(item)

        #print(vals_stack)

    assert len(vals_stack) == 1
    assert isinstance(vals_stack[0], bool)

    return vals_stack[0]

class Condition(object):
    """
    This object receive a dictionary that defines a boolean condition whose syntax is similar
    to the one used in mongodb (albeit not all the operators available in mongodb are supported here).

    Example:

    $gt: {field: {$gt: value} }

    $gt selects those documents where the value of the field is greater than (i.e. >) the specified value.

    $and performs a logical AND operation on an array of two or more expressions (e.g. <expression1>, <expression2>, etc.)
    and selects the documents that satisfy all the expressions in the array.

    { $and: [ { <expression1> }, { <expression2> } , ... , { <expressionN> } ] }

    Consider the following example:

    db.inventory.find( { qty: { $gt: 20 } } )
    This query will select all documents in the inventory collection where the qty field value is greater than 20.
    Consider the following example:

    db.inventory.find( { qty: { $gt: 20 } } )
    db.inventory.find({ $and: [ { price: 1.99 }, { qty: { $lt: 20 } }, { sale: true } ] } )
    """
    def __init__(self, cmap):
        self.cmap = cmap

    def __str__(self):
        return str(self.cmap)

    def apply(self, obj):
        try:
            return evaluate_rpn(map2rpn(self.cmap, obj))
        except:
            logger.warning("Condition.apply() raise Exception")
            return False
<commit_msg>fix typo<commit_after>"""Tools and helper functions for abinit calculations"""
from __future__ import print_function, division

import os.path
import collections
import shutil
import operator

from pymatgen.util.string_utils import list_strings, StringColorizer

import logging
logger = logging.getLogger(__name__)

class File(object):
    """
    Very simple class used to store file basenames, absolute paths and directory names.
    Provides wrappers for the most commonly used functions defined in os.path.
    """
    def __init__(self, path):
        self._path = os.path.abspath(path)

    def __repr__(self):
        return "<%s at %s, %s>" % (self.__class__.__name__, id(self), self.path)

    def __str__(self):
        return "<%s, %s>" % (self.__class__.__name__, self.path)

    def __eq__(self, other):
        if other is None: return False
        self.path == other.path

    def __ne__(self, other):
        return not self.__eq__(other)

    @property
    def path(self):
        """Absolute path of the file."""
        return self._path

    @property
    def basename(self):
        """File basename."""
        return os.path.basename(self.path)

    @property
    def dirname(self):
        """Absolute path of the directory where the file is located."""
        return os.path.dirname(self.path)

    @property
    def exists(self):
        "True if file exists."
        return os.path.exists(self.path)

    @property
    def isncfile(self):
        "True if self is a NetCDF file"
        return self.basename.endswith(".nc")

    def read(self):
        """Read data from file."""
        with open(self.path, "r") as f:
            return f.read()

    def readlines(self):
        """Read lines from files."""
        with open(self.path, "r") as f:
            return f.readlines()

    def write(self, string):
        """Write string to file."""
        self.make_dir()
        with open(self.path, "w") as f:
            return f.write(string)

    def writelines(self, lines):
        """Write a list of strings to file."""
        self.make_dir()
        with open(self.path, "w") as f:
            return f.writelines()

    def make_dir(self):
        """Make the directory where the file is located."""
        if not os.path.exists(self.dirname):
            os.makedirs(self.dirname)

class Directory(object):
    """
    Very simple class that provides helper functions
    wrapping the most commonly used functions defined in os.path.
    """
    def __init__(self, path):
        self._path = os.path.abspath(path)

    def __repr__(self):
        return "<%s at %s, %s>" % (self.__class__.__name__, id(self), self.path)

    def __str__(self):
        return "<%s, %s>" % (self.__class__.__name__, self.path)

    def __eq__(self, other):
        if other is None: return False
        self.path == other.path

    def __ne__(self, other):
        return not self.__eq__(other)

    @property
    def path(self):
        """Absolute path of the directory."""
        return self._path

    @property
    def basename(self):
        """Directory basename."""
        return os.path.basename(self.path)

    def path_join(self, *p):
        """
        Join two or more pathname components, inserting '/' as needed.
        If any component is an absolute path, all previous path components will be discarded.
        """
        return os.path.join(self.path, *p)

    @property
    def exists(self):
        "True if file exists."
        return os.path.exists(self.path)

    def makedirs(self):
        """
        Super-mkdir; create a leaf directory and all intermediate ones.
        Works like mkdir, except that any intermediate path segment (not
        just the rightmost) will be created if it does not exist.
        """
        if not self.exists:
            os.makedirs(self.path)

    def rmtree(self):
        """Recursively delete the directory tree"""
        shutil.rmtree(self.path, ignore_errors=True)

    def path_in(self, filename):
        """Return the absolute path of filename in the directory."""
        return os.path.join(self.path, filename)

    def list_filepaths(self):
        """Return the list of absolute filepaths in the top-level directory."""
        fnames = [f for f in os.listdir(self.path)]
        return [os.path.join(self.path, f) for f in fnames]

    def has_abiext(self, ext):
        """
        Returns the absolute path of the ABINIT file with extension ext.
        Support both Fortran files and netcdf files. In the later case,
        we check whether a file with extension ext + ".nc" is present
        in the directory. Returns empty string is file is not present.

        Raises:
            ValueError if multiple files with the given ext are found.
            This implies that this method is not compatible with multiple datasets.
        """
        files = []
        for f in self.list_filepaths():
            if f.endswith(ext) or f.endswith(ext + ".nc"):
                files.append(f)

        if not files:
            return ""

        if len(files) > 1:
            # ABINIT users must learn that multiple datasets are bad!
            err_msg = "Found multiple files with the same extensions\n Please avoid the use of mutiple datasets!"
            raise ValueError(err_msg)

        return files[0]

# This dictionary maps ABINIT file extensions to the
# variables that must be used to read the file in input.
#
# TODO: It would be nice to pass absolute paths to abinit with getden_path
# so that I can avoid creating symbolic links before running but
# the presence of the C-bindings complicates the implementation
# (gfortran SIGFAULTs if I add strings to dataset_type!
_EXT2VARS = {
    "DEN": {"irdden": 1},
    "WFK": {"irdwfk": 1},
    "SCR": {"irdscr": 1},
    "QPS": {"irdqps": 1},
    "1WF": {"ird1wf": 1},
    "1DEN": {"ird1den": 1},
    "BSR": {"irdbsreso": 1},
    "BSC": {"irdbscoup": 1},
    "HAYDR_SAVE": {"irdhaydock": 1},
    "DDK": {"irdddk": 1},
    "DDB": {},
    "GKK": {},
    "DKK": {},
}

def irdvars_for_ext(ext):
    """
    Returns a dictionary with the ABINIT variables
    that must be used to read the file with extension ext.
    """
    return _EXT2VARS[ext].copy()

def abi_extensions():
    """List with all the ABINIT extensions that are registered."""
    return list(_EXT2VARS.keys())[:]

def abi_splitext(filename):
    """
    Split the ABINIT extension from a filename.
    "Extension" are found by searching in an internal database.

    Returns "(root, ext)" where ext is the registered ABINIT extension
    The final ".nc" is included (if any)

    >>> abi_splitext("foo_WFK")
    ('foo_', 'WFK')

    >>> abi_splitext("/home/guido/foo_bar_WFK.nc")
    ('foo_bar_', 'WFK.nc')
    """
    filename = os.path.basename(filename)
    is_ncfile = False
    if filename.endswith(".nc"):
        is_ncfile = True
        filename = filename[:-3]

    known_extensions = abi_extensions()

    # This algorith fails if we have two files
    # e.g. HAYDR_SAVE, ANOTHER_HAYDR_SAVE
    for i in range(len(filename)-1, -1, -1):
        ext = filename[i:]
        if ext in known_extensions:
            break

    else:
        raise ValueError("Cannot find a registered extension in %s" % filename)

    root = filename[:i]
    if is_ncfile:
        ext = ext + ".nc"

    return root, ext

class FilepathFixer(object):
    """
    This object modifies the names of particular output files
    produced by ABINIT so that the file extension is preserved.
    Having a one-to-one mapping between file extension and data format
    is indeed fundamental for the correct behaviour of abinitio since:

        - We locate the output file by just inspecting the extension

        - We select the variables that must be added to the input file
          on the basis of the extension specified by the user during
          the initialization of the `AbinitFlow`.

    Unfortunately, ABINIT developers like to append extra stuff
    to the initial extension and therefore we have to call
    `FilepathFixer` to fix the output files produced by the run.

    Example:

    >>> fixer = FilepathFixer()

    >>> fixer.fix_paths('/foo/out_1WF17')
    {'/foo/out_1WF17': '/foo/out_1WF'}

    >>> fixer.fix_paths('/foo/out_1WF5.nc')
    {'/foo/out_1WF5.nc': '/foo/out_1WF.nc'}
    """
    def __init__(self):
        # dictionary mapping the *official* file extension to
        # the regular expression used to tokenize the basename of the file
        # To add a new fix it's sufficient to add a new regexp and
        # a static method _fix_EXTNAME
        self.regs = regs = {}
        import re
        regs["1WF"] = re.compile("(\w+_)1WF(\d+)(.nc)?$")
        regs["1DEN"] = re.compile("(\w+_)1DEN(\d+)(.nc)?$")

    @staticmethod
    def _fix_1WF(match):
        root, pert, ncext = match.groups()
        if ncext is None: ncext = ""
        return root + "1WF" + ncext

    @staticmethod
    def _fix_1DEN(match):
        root, pert, ncext = match.groups()
        if ncext is None: ncext = ""
        return root + "1DEN" + ncext

    def _fix_path(self, path):
        for ext, regex in self.regs.items():
            head, tail = os.path.split(path)

            match = regex.match(tail)
            if match:
                newtail = getattr(self, "_fix_" + ext)(match)
                newpath = os.path.join(head, newtail)
                return newpath, ext

        return None, None

    def fix_paths(self, paths):
        """
        Fix the filenames in the iterable paths

        Returns:
            old2new:
                Mapping old_path --> new_path
        """
        old2new, fixed_exts = {}, []

        for path in list_strings(paths):
            newpath, ext = self._fix_path(path)

            if newpath is not None:
                assert ext not in fixed_exts
                fixed_exts.append(ext)
                old2new[path] = newpath

        return old2new

def _bop_not(obj):
    """Boolean not."""
    return not bool(obj)

def _bop_and(obj1, obj2):
    """Boolean and."""
    return bool(obj1) and bool(obj2)

def _bop_or(obj1, obj2):
    """Boolean or."""
    return bool(obj1) or bool(obj2)

def _bop_divisible(num1, num2):
    """Return True if num1 is divisible by num2."""
    return (num1 % num2) == 0.0

# Mapping string --> operator.
_UNARY_OPS = {
    "$not": _bop_not,
}

_BIN_OPS = {
    "$eq": operator.eq,
    "$ne": operator.ne,
    "$gt": operator.gt,
    "$ge": operator.ge,
    "$lt": operator.lt,
    "$le": operator.le,
    "$divisible": _bop_divisible,
    "$and": _bop_and,
    "$or":  _bop_or,
    }

ops = _UNARY_OPS.keys() + _BIN_OPS.keys()

def map2rpn(map, obj):
    """
    Convert a Mongodb-like dictionary to a RPN list of operands and operators.

    Reverse Polish notation (RPN) is a mathematical notation in which every
    operator follows all of its operands, e.g.

    3 - 4 + 5 -->   3 4 - 5 +

    >>> d = {2.0: {'$eq': 1.0}}
    >>> map2rpn(d, None)
    [2.0, 1.0, '$eq']
    """
    rpn = []

    for k, v in map.items():

        if k in ops:
            if isinstance(v, collections.Mapping):
                # e.g "$not": {"$gt": "one"}
                # print("in op_vmap",k, v)
                values = map2rpn(v, obj)
                rpn.extend(values)
                rpn.append(k)

            elif isinstance(v, (list, tuple)):
                # e.g "$and": [{"$not": {"one": 1.0}}, {"two": {"$lt": 3}}]}
                # print("in_op_list",k, v)
                for d in v:
                    rpn.extend(map2rpn(d, obj))

                rpn.append(k)

            else:
                # Examples
                # 1) "$eq"": "attribute_name"
                # 2) "$eq"": 1.0
                try:
                    #print("in_otherv",k, v)
                    rpn.append(getattr(obj, v))
                    rpn.append(k)

                except TypeError:
                    #print("in_otherv, raised",k, v)
                    rpn.extend([v, k])
        else:
            try:
                k = getattr(obj, k)
            except TypeError:
                k = k

            if isinstance(v, collections.Mapping):
                # "one": {"$eq": 1.0}}
                values = map2rpn(v, obj)
                rpn.append(k)
                rpn.extend(values)
            else:
                #"one": 1.0
                rpn.extend([k, v, "$eq"])

    return rpn

def evaluate_rpn(rpn):
    """
    Evaluates the RPN form produced my map2rpn.

    Returns:
        bool
    """
    vals_stack = []

    for item in rpn:

        if item in ops:
            # Apply the operator and push to the task.
            v2 = vals_stack.pop()

            if item in _UNARY_OPS:
                res = _UNARY_OPS[item](v2)

            elif item in _BIN_OPS:
                v1 = vals_stack.pop()
                res = _BIN_OPS[item](v1, v2)
            else:
                raise ValueError("%s not in unary_ops or bin_ops" % str(item))

            vals_stack.append(res)

        else:
            # Push the operand
            vals_stack.append(item)

        #print(vals_stack)

    assert len(vals_stack) == 1
    assert isinstance(vals_stack[0], bool)

    return vals_stack[0]

class Condition(object):
    """
    This object receive a dictionary that defines a boolean condition whose syntax is similar
    to the one used in mongodb (albeit not all the operators available in mongodb are supported here).

    Example:

    $gt: {field: {$gt: value} }

    $gt selects those documents where the value of the field is greater than (i.e. >) the specified value.

    $and performs a logical AND operation on an array of two or more expressions (e.g. <expression1>, <expression2>, etc.)
    and selects the documents that satisfy all the expressions in the array.

    { $and: [ { <expression1> }, { <expression2> } , ... , { <expressionN> } ] }

    Consider the following example:

    db.inventory.find( { qty: { $gt: 20 } } )
    This query will select all documents in the inventory collection where the qty field value is greater than 20.
    Consider the following example:

    db.inventory.find( { qty: { $gt: 20 } } )
    db.inventory.find({ $and: [ { price: 1.99 }, { qty: { $lt: 20 } }, { sale: true } ] } )
    """
    def __init__(self, cmap):
        self.cmap = cmap

    def __str__(self):
        return str(self.cmap)

    def apply(self, obj):
        try:
            return evaluate_rpn(map2rpn(self.cmap, obj))
        except:
            logger.warning("Condition.apply() raise Exception")
            return False
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
"""
Mercadopago API exceptions.
"""

class NoAccessTokenError(Exception):

    def __init__(self, value):
        self.value = value

    def __str__(self):
        return repr(self.value)

class UndefinedResponseError(Exception):

    def __init__(self, value):
        self.value = value

    def __str__(self):
        return repr(self.value)

class EmptyCredentialsError(Exception):

    def __init__(self, value):
        self.value = value

    def __str__(self):
        return repr(self.value)
<commit_msg>Adding an exception when the credentials expires<commit_after># -*- coding: utf-8 -*-
"""
Mercadopago API exceptions.
"""

class NoAccessTokenError(Exception):

    def __init__(self, value):
        self.value = value

    def __str__(self):
        return repr(self.value)

class UndefinedResponseError(Exception):

    def __init__(self, value):
        self.value = value

    def __str__(self):
        return repr(self.value)

class EmptyCredentialsError(Exception):

    def __init__(self, value):
        self.value = value

    def __str__(self):
        return repr(self.value)

class ExpiredAuthenticationError(Exception):

    def __init__(self, value):
        self.value = value

    def __str__(self):
        return repr(self.value)

<|endoftext|>


<commit_before>from django.db import models
from django import forms
from wheelcms_axle.models import Content, Node
from wheelcms_axle.models import type_registry

from two.ol.util import classproperty

class BaseForm(forms.ModelForm):
    class Meta:
        exclude = ["node", "meta_type"]

    slug = forms.Field(required=True)

    def __init__(self, parent=None, attach=False, *args, **kwargs):
        """
            Django will put the extra slug field at the bottom, below
            all model fields. I want it just after the title field
        """
        super(BaseForm, self).__init__(*args, **kwargs)
        slug = self.fields.pop('slug')
        titlepos = self.fields.keyOrder.index('title')
        self.fields.insert(titlepos+1, 'slug', slug)
        self.parent = parent
        self.attach = attach
        if attach:
            self.fields.pop('slug')

    def clean_slug(self):
        if self.attach:
            return

        slug = self.data.get('slug', '').strip().lower()
        if not Node.validpathre.match(slug):
            raise forms.ValidationError("Only numbers, letters, _-")
        try:
            existing = Node.objects.filter(path=self.parent.path + "/" + slug).get()
            if existing != self.instance.node:
                raise forms.ValidationError("Name in use")
        except Node.DoesNotExist:
            pass

        return slug

def formfactory(type):
    class Form(BaseForm):
        class Meta:
            model = type
            exclude = ["node", "meta_type", "created", "modified"]
    return Form

class Spoke(object):
    model = Content

    def __init__(self, o):
        self.o = o

    @classproperty
    def form(cls):
        return formfactory(cls.model)

    @classmethod
    def name(cls):
        """ This needs namespacing. But a model determines its name based
            on the classname and doesn't know about namespaces or packages """
        # import pytest; pytest.set_trace()
        # return cls.model.__class__.__name__.lower()
        return cls.model._meta.object_name.lower()  ## app_label

    def view_template(self):
        return "wheelcms_axle/content_view.html"

    def fields(self):
        ## move to spokes
        for i in self.o._meta.fields:
            yield (i.name, getattr(self.o, i.name))
    ## allowed subchildren, if any, can be dynamic

class Page(Content):
    """ A simple page object """
    body = models.TextField(blank=False)

class PageType(Spoke):
    model = Page

    title = "A simple HTML page"

class News(Content):
    """ A news object """
    intro = models.TextField(blank=False)
    body = models.TextField(blank=False)

class NewsType(Spoke):
    model = News

    title = "A simple News item"

type_registry.register(PageType)
type_registry.register(NewsType)
<commit_msg>handle /edit, test it<commit_after>from django.db import models
from django import forms
from wheelcms_axle.models import Content, Node
from wheelcms_axle.models import type_registry

from two.ol.util import classproperty

class BaseForm(forms.ModelForm):
    class Meta:
        exclude = ["node", "meta_type"]

    slug = forms.Field(required=True)

    def __init__(self, parent=None, attach=False, *args, **kwargs):
        """
            Django will put the extra slug field at the bottom, below
            all model fields. I want it just after the title field
        """
        super(BaseForm, self).__init__(*args, **kwargs)
        slug = self.fields.pop('slug')
        titlepos = self.fields.keyOrder.index('title')
        self.fields.insert(titlepos+1, 'slug', slug)
        self.parent = parent
        self.attach = attach
        if attach:
            self.fields.pop('slug')

    def clean_slug(self):
        if self.attach:
            return

        slug = self.data.get('slug', '').strip().lower()
        if not Node.validpathre.match(slug):
            raise forms.ValidationError("Only numbers, letters, _-")
        try:
            existing = Node.objects.filter(path=self.parent.path + "/" + slug).get()
            if existing != self.instance.node:
                raise forms.ValidationError("Name in use")
        except Node.DoesNotExist:
            pass

        return slug

def formfactory(type):
    class Form(BaseForm):
        class Meta:
            model = type
            exclude = ["node", "meta_type", "created", "modified"]
    return Form

class Spoke(object):
    model = Content

    def __init__(self, o):
        self.o = o

    @classproperty
    def form(cls):
        return formfactory(cls.model)

    @classmethod
    def name(cls):
        """ This needs namespacing. But a model determines its name based
            on the classname and doesn't know about namespaces or packages """
        # import pytest; pytest.set_trace()
        # return cls.model.__class__.__name__.lower()
        return cls.model._meta.object_name.lower()  ## app_label

    @classmethod
    def title(cls):
        """ a default title """
        return cls.model._meta.object_name + " content"

    def view_template(self):
        return "wheelcms_axle/content_view.html"

    def fields(self):
        ## move to spokes
        for i in self.o._meta.fields:
            yield (i.name, getattr(self.o, i.name))
    ## allowed subchildren, if any, can be dynamic

class Page(Content):
    """ A simple page object """
    body = models.TextField(blank=False)

class PageType(Spoke):
    model = Page

    title = "A simple HTML page"

class News(Content):
    """ A news object """
    intro = models.TextField(blank=False)
    body = models.TextField(blank=False)

class NewsType(Spoke):
    model = News

    title = "A simple News item"

type_registry.register(PageType)
type_registry.register(NewsType)
<|endoftext|>