<commit_before>from synapse.tests.common import *

class PersonTest(SynTest, ModelSeenMixin):

    def test_model_ps_has(self):
        with self.getRamCore() as core:
            person_guid = 32 * '0'
            person_tufo = core.formTufoByProp('ps:person', person_guid, name='Kenshoto,Invisigoth')
            personval = person_tufo[1].get('ps:person')
            fqdn_tufo = core.formTufoByProp('inet:fqdn', 'vertex.link')

            node = core.formTufoByProp('ps:has', (personval, ('inet:fqdn', 'vertex.link')))
            self.ge(node[1].get('node:created'), 1519852535218)
            self.eq(node[1].get('ps:has'), '03870dc800bc21c7c594a900ae65f5cb')
            self.eq(node[1].get('ps:has:person'), personval)
            self.eq(node[1].get('ps:has:xref'), 'inet:fqdn=vertex.link')
            self.eq(node[1].get('ps:has:xref:prop'), 'inet:fqdn')
            self.eq(node[1].get('ps:has:xref:strval'), 'vertex.link')

    def test_model_person(self):

        with self.getRamCore() as core:
            dob = core.getTypeParse('time', '19700101000000001')
            node = core.formTufoByProp('ps:person', guid(), dob=dob[0],
                                       name='Kenshoto,Invisigoth')
            self.eq(node[1].get('ps:person:dob'), 1)
            self.eq(node[1].get('ps:person:name'), 'kenshoto,invisigoth')
            self.eq(node[1].get('ps:person:name:sur'), 'kenshoto')
            self.eq(node[1].get('ps:person:name:given'), 'invisigoth')
            self.none(node[1].get('ps:person:name:middle'))

            node = core.formTufoByProp('ps:person', guid(), dob=dob[0],
                                       name='Kenshoto,Invisigoth,Ninja')
            self.eq(node[1].get('ps:person:name'), 'kenshoto,invisigoth,ninja')
            self.eq(node[1].get('ps:person:name:sur'), 'kenshoto')
            self.eq(node[1].get('ps:person:name:given'), 'invisigoth')
            self.eq(node[1].get('ps:person:name:middle'), 'ninja')

            node = core.formTufoByProp('ps:person', guid(), dob=dob[0],
                                       name='Kenshoto,Invisigoth,Ninja,Gray')
            self.eq(node[1].get('ps:person:name'), 'kenshoto,invisigoth,ninja,gray')
            self.eq(node[1].get('ps:person:name:sur'), 'kenshoto')
            self.eq(node[1].get('ps:person:name:given'), 'invisigoth')
            self.eq(node[1].get('ps:person:name:middle'), 'ninja')

            node = core.formTufoByProp('ps:person', guid(),
                                       **{'name': 'Гольдштейн, Эммануэль, брат',
                                          'name:en': 'Goldstein, Emmanuel, Brother',
                                          })
            self.eq(node[1].get('ps:person:name'), 'гольдштейн,эммануэль,брат')
            self.eq(node[1].get('ps:person:name:sur'), 'гольдштейн')
            self.eq(node[1].get('ps:person:name:given'), 'эммануэль')
            self.eq(node[1].get('ps:person:name:middle'), 'брат')
            self.eq(node[1].get('ps:person:name:en'), 'goldstein,emmanuel,brother')
            self.eq(node[1].get('ps:person:name:en:sur'), 'goldstein')
            self.eq(node[1].get('ps:person:name:en:given'), 'emmanuel')
            self.eq(node[1].get('ps:person:name:en:middle'), 'brother')

    def test_model_person_tokn(self):
        with self.getRamCore() as core:
            node = core.formTufoByProp('ps:tokn', 'Invisigoth')
            self.eq(node[1].get('ps:tokn'), 'invisigoth')

    def test_model_person_name(self):
        with self.getRamCore() as core:
            node = core.formTufoByProp('ps:name', 'Kenshoto,Invisigoth')

            self.eq(node[1].get('ps:name:sur'), 'kenshoto')
            self.eq(node[1].get('ps:name:given'), 'invisigoth')

            self.nn(core.getTufoByProp('ps:tokn', 'kenshoto'))
            self.nn(core.getTufoByProp('ps:tokn', 'invisigoth'))

    def test_model_person_2(self):

        with self.getRamCore() as core:
            node = core.formTufoByProp('ps:name', 'Kenshoto,Invisigoth')

            self.eq(node[1].get('ps:name'), 'kenshoto,invisigoth')
            self.eq(node[1].get('ps:name:sur'), 'kenshoto')
            self.eq(node[1].get('ps:name:given'), 'invisigoth')

            self.nn(core.getTufoByProp('ps:tokn', 'kenshoto'))
            self.nn(core.getTufoByProp('ps:tokn', 'invisigoth'))

    def test_model_person_has_alias(self):
        with self.getRamCore() as core:
            iden = guid()
            node = core.formTufoByProp('ps:hasalias', '%s/Kenshoto,Invisigoth' % iden)

            self.eq(node[1].get('ps:hasalias:alias'), 'kenshoto,invisigoth')
            self.eq(node[1].get('ps:hasalias:person'), iden)

            self.nn(core.getTufoByProp('ps:person', iden))
            self.nn(core.getTufoByProp('ps:name', 'kenshoto,invisigoth'))

            self.check_seen(core, node)

    def test_model_person_has_host(self):
        with self.getRamCore() as core:
            pval = 32 * 'a'
            hval = 32 * 'b'

            node = core.formTufoByProp('ps:hashost', (pval, hval))

            self.eq(node[1]['ps:hashost'], '20ffcd864aeb7f4f6e23d95680eeed47')
            self.eq(node[1].get('ps:hashost:host'), hval)
            self.eq(node[1].get('ps:hashost:person'), pval)

            self.nn(core.getTufoByProp('ps:person', pval))
            self.nn(core.getTufoByProp('it:host', hval))

            self.check_seen(core, node)

    def test_model_person_has_email(self):
        with self.getRamCore() as core:
            iden = guid()
            node = core.formTufoByProp('ps:hasemail', '%s/visi@VERTEX.link' % iden)

            self.eq(node[1].get('ps:hasemail:email'), 'visi@vertex.link')
            self.eq(node[1].get('ps:hasemail:person'), iden)

            self.nn(core.getTufoByProp('ps:person', iden))
            self.nn(core.getTufoByProp('inet:email', 'visi@vertex.link'))

            self.check_seen(core, node)

    def test_model_person_has_phone(self):
        with self.getRamCore() as core:
            iden = guid()
            node = core.formTufoByProp('ps:hasphone', '%s/17035551212' % iden)

            self.eq(node[1].get('ps:hasphone:phone'), 17035551212)
            self.eq(node[1].get('ps:hasphone:person'), iden)

            self.nn(core.getTufoByProp('ps:person', iden))
            self.nn(core.getTufoByProp('tel:phone', 17035551212))

            self.check_seen(core, node)

    def test_model_person_has_webacct(self):
        with self.getRamCore() as core:
            iden = guid()
            node = core.formTufoByProp('ps:haswebacct', '%s/ROOTKIT.com/visi' % iden)

            self.eq(node[1].get('ps:haswebacct:web:acct'), 'rootkit.com/visi')
            self.eq(node[1].get('ps:haswebacct:person'), iden)

            self.nn(core.getTufoByProp('ps:person', iden))
            self.nn(core.getTufoByProp('inet:user', 'visi'))
            self.nn(core.getTufoByProp('inet:web:acct', 'rootkit.com/visi'))

            self.check_seen(core, node)

    def test_model_person_guidname(self):

        with self.getRamCore() as core:

            node = core.formTufoByProp('ps:person:guidname', 'visi')
            self.eq(node[1].get('ps:person:guidname'), 'visi')

            iden = node[1].get('ps:person')

            node = core.formTufoByProp('ps:haswebacct', '$visi/rootkit.com/visi')

            self.eq(node[1].get('ps:haswebacct:web:acct'), 'rootkit.com/visi')
            self.eq(node[1].get('ps:haswebacct:person'), iden)

            self.nn(core.getTufoByProp('ps:person', iden))

            self.eq(len(core.eval('ps:person=$visi')), 1)

    def test_model_person_contact(self):

        with self.getRamCore() as core:

            info = {
                'org': '*',
                'person': '*',

                'name': 'Stark,Tony',

                'title': 'CEO',
                'orgname': 'Stark Industries, INC',

                'user': 'ironman',
                'web:acct': 'twitter.com/ironman',

                'dob': '1976-12-17',
                'url': 'https://starkindustries.com/',

                'email': 'tony.stark@gmail.com',
                'email:work': 'tstark@starkindustries.com',

                'phone': '12345678910',
                'phone:fax': '12345678910',
                'phone:work': '12345678910',

                'address': '1 Iron Suit Drive, San Francisco, CA, 22222, USA',
            }

            node = core.formTufoByProp('ps:contact', info)

            self.nn(core.getTufoByProp('ou:org', node[1].get('ps:contact:org')))
            self.nn(core.getTufoByProp('ps:person', node[1].get('ps:contact:person')))
            self.nn(core.getTufoByProp('inet:url', node[1].get('ps:contact:url')))
            self.nn(core.getTufoByProp('inet:user', node[1].get('ps:contact:user')))
            self.nn(core.getTufoByProp('inet:email', node[1].get('ps:contact:email')))
            self.nn(core.getTufoByProp('inet:email', node[1].get('ps:contact:email:work')))
            self.nn(core.getTufoByProp('inet:web:acct', node[1].get('ps:contact:web:acct')))
            self.nn(core.getTufoByProp('tel:phone', node[1].get('ps:contact:phone')))
            self.nn(core.getTufoByProp('tel:phone', node[1].get('ps:contact:phone:fax')))
            self.nn(core.getTufoByProp('tel:phone', node[1].get('ps:contact:phone:work')))

            self.eq(node[1].get('ps:contact:address'), '1 iron suit drive, san francisco, ca, 22222, usa')
<commit_msg>Adds migration test for 1 of 6 forms<commit_after>import synapse.lib.tufo as s_tufo
import synapse.lib.types as s_types

from synapse.tests.common import *

class PersonTest(SynTest, ModelSeenMixin):

    def test_model_ps_has(self):
        with self.getRamCore() as core:
            person_guid = 32 * '0'
            person_tufo = core.formTufoByProp('ps:person', person_guid, name='Kenshoto,Invisigoth')
            personval = person_tufo[1].get('ps:person')
            fqdn_tufo = core.formTufoByProp('inet:fqdn', 'vertex.link')

            node = core.formTufoByProp('ps:has', (personval, ('inet:fqdn', 'vertex.link')))
            self.ge(node[1].get('node:created'), 1519852535218)
            self.eq(node[1].get('ps:has'), '03870dc800bc21c7c594a900ae65f5cb')
            self.eq(node[1].get('ps:has:person'), personval)
            self.eq(node[1].get('ps:has:xref'), 'inet:fqdn=vertex.link')
            self.eq(node[1].get('ps:has:xref:prop'), 'inet:fqdn')
            self.eq(node[1].get('ps:has:xref:strval'), 'vertex.link')

    def test_model_person(self):

        with self.getRamCore() as core:
            dob = core.getTypeParse('time', '19700101000000001')
            node = core.formTufoByProp('ps:person', guid(), dob=dob[0],
                                       name='Kenshoto,Invisigoth')
            self.eq(node[1].get('ps:person:dob'), 1)
            self.eq(node[1].get('ps:person:name'), 'kenshoto,invisigoth')
            self.eq(node[1].get('ps:person:name:sur'), 'kenshoto')
            self.eq(node[1].get('ps:person:name:given'), 'invisigoth')
            self.none(node[1].get('ps:person:name:middle'))

            node = core.formTufoByProp('ps:person', guid(), dob=dob[0],
                                       name='Kenshoto,Invisigoth,Ninja')
            self.eq(node[1].get('ps:person:name'), 'kenshoto,invisigoth,ninja')
            self.eq(node[1].get('ps:person:name:sur'), 'kenshoto')
            self.eq(node[1].get('ps:person:name:given'), 'invisigoth')
            self.eq(node[1].get('ps:person:name:middle'), 'ninja')

            node = core.formTufoByProp('ps:person', guid(), dob=dob[0],
                                       name='Kenshoto,Invisigoth,Ninja,Gray')
            self.eq(node[1].get('ps:person:name'), 'kenshoto,invisigoth,ninja,gray')
            self.eq(node[1].get('ps:person:name:sur'), 'kenshoto')
            self.eq(node[1].get('ps:person:name:given'), 'invisigoth')
            self.eq(node[1].get('ps:person:name:middle'), 'ninja')

            node = core.formTufoByProp('ps:person', guid(),
                                       **{'name': 'Гольдштейн, Эммануэль, брат',
                                          'name:en': 'Goldstein, Emmanuel, Brother',
                                          })
            self.eq(node[1].get('ps:person:name'), 'гольдштейн,эммануэль,брат')
            self.eq(node[1].get('ps:person:name:sur'), 'гольдштейн')
            self.eq(node[1].get('ps:person:name:given'), 'эммануэль')
            self.eq(node[1].get('ps:person:name:middle'), 'брат')
            self.eq(node[1].get('ps:person:name:en'), 'goldstein,emmanuel,brother')
            self.eq(node[1].get('ps:person:name:en:sur'), 'goldstein')
            self.eq(node[1].get('ps:person:name:en:given'), 'emmanuel')
            self.eq(node[1].get('ps:person:name:en:middle'), 'brother')

    def test_model_person_tokn(self):
        with self.getRamCore() as core:
            node = core.formTufoByProp('ps:tokn', 'Invisigoth')
            self.eq(node[1].get('ps:tokn'), 'invisigoth')

    def test_model_person_name(self):
        with self.getRamCore() as core:
            node = core.formTufoByProp('ps:name', 'Kenshoto,Invisigoth')

            self.eq(node[1].get('ps:name:sur'), 'kenshoto')
            self.eq(node[1].get('ps:name:given'), 'invisigoth')

            self.nn(core.getTufoByProp('ps:tokn', 'kenshoto'))
            self.nn(core.getTufoByProp('ps:tokn', 'invisigoth'))

    def test_model_person_2(self):

        with self.getRamCore() as core:
            node = core.formTufoByProp('ps:name', 'Kenshoto,Invisigoth')

            self.eq(node[1].get('ps:name'), 'kenshoto,invisigoth')
            self.eq(node[1].get('ps:name:sur'), 'kenshoto')
            self.eq(node[1].get('ps:name:given'), 'invisigoth')

            self.nn(core.getTufoByProp('ps:tokn', 'kenshoto'))
            self.nn(core.getTufoByProp('ps:tokn', 'invisigoth'))

    def test_model_person_has_alias(self):
        with self.getRamCore() as core:
            iden = guid()
            node = core.formTufoByProp('ps:hasalias', '%s/Kenshoto,Invisigoth' % iden)

            self.eq(node[1].get('ps:hasalias:alias'), 'kenshoto,invisigoth')
            self.eq(node[1].get('ps:hasalias:person'), iden)

            self.nn(core.getTufoByProp('ps:person', iden))
            self.nn(core.getTufoByProp('ps:name', 'kenshoto,invisigoth'))

            self.check_seen(core, node)

    def test_model_person_has_host(self):
        with self.getRamCore() as core:
            pval = 32 * 'a'
            hval = 32 * 'b'

            node = core.formTufoByProp('ps:hashost', (pval, hval))

            self.eq(node[1]['ps:hashost'], '20ffcd864aeb7f4f6e23d95680eeed47')
            self.eq(node[1].get('ps:hashost:host'), hval)
            self.eq(node[1].get('ps:hashost:person'), pval)

            self.nn(core.getTufoByProp('ps:person', pval))
            self.nn(core.getTufoByProp('it:host', hval))

            self.check_seen(core, node)

    def test_model_person_has_email(self):
        with self.getRamCore() as core:
            iden = guid()
            node = core.formTufoByProp('ps:hasemail', '%s/visi@VERTEX.link' % iden)

            self.eq(node[1].get('ps:hasemail:email'), 'visi@vertex.link')
            self.eq(node[1].get('ps:hasemail:person'), iden)

            self.nn(core.getTufoByProp('ps:person', iden))
            self.nn(core.getTufoByProp('inet:email', 'visi@vertex.link'))

            self.check_seen(core, node)

    def test_model_person_has_phone(self):
        with self.getRamCore() as core:
            iden = guid()
            node = core.formTufoByProp('ps:hasphone', '%s/17035551212' % iden)

            self.eq(node[1].get('ps:hasphone:phone'), 17035551212)
            self.eq(node[1].get('ps:hasphone:person'), iden)

            self.nn(core.getTufoByProp('ps:person', iden))
            self.nn(core.getTufoByProp('tel:phone', 17035551212))

            self.check_seen(core, node)

    def test_model_person_has_webacct(self):
        with self.getRamCore() as core:
            iden = guid()
            node = core.formTufoByProp('ps:haswebacct', '%s/ROOTKIT.com/visi' % iden)

            self.eq(node[1].get('ps:haswebacct:web:acct'), 'rootkit.com/visi')
            self.eq(node[1].get('ps:haswebacct:person'), iden)

            self.nn(core.getTufoByProp('ps:person', iden))
            self.nn(core.getTufoByProp('inet:user', 'visi'))
            self.nn(core.getTufoByProp('inet:web:acct', 'rootkit.com/visi'))

            self.check_seen(core, node)

    def test_model_person_guidname(self):

        with self.getRamCore() as core:

            node = core.formTufoByProp('ps:person:guidname', 'visi')
            self.eq(node[1].get('ps:person:guidname'), 'visi')

            iden = node[1].get('ps:person')

            node = core.formTufoByProp('ps:haswebacct', '$visi/rootkit.com/visi')

            self.eq(node[1].get('ps:haswebacct:web:acct'), 'rootkit.com/visi')
            self.eq(node[1].get('ps:haswebacct:person'), iden)

            self.nn(core.getTufoByProp('ps:person', iden))

            self.eq(len(core.eval('ps:person=$visi')), 1)

    def test_model_person_contact(self):

        with self.getRamCore() as core:

            info = {
                'org': '*',
                'person': '*',

                'name': 'Stark,Tony',

                'title': 'CEO',
                'orgname': 'Stark Industries, INC',

                'user': 'ironman',
                'web:acct': 'twitter.com/ironman',

                'dob': '1976-12-17',
                'url': 'https://starkindustries.com/',

                'email': 'tony.stark@gmail.com',
                'email:work': 'tstark@starkindustries.com',

                'phone': '12345678910',
                'phone:fax': '12345678910',
                'phone:work': '12345678910',

                'address': '1 Iron Suit Drive, San Francisco, CA, 22222, USA',
            }

            node = core.formTufoByProp('ps:contact', info)

            self.nn(core.getTufoByProp('ou:org', node[1].get('ps:contact:org')))
            self.nn(core.getTufoByProp('ps:person', node[1].get('ps:contact:person')))
            self.nn(core.getTufoByProp('inet:url', node[1].get('ps:contact:url')))
            self.nn(core.getTufoByProp('inet:user', node[1].get('ps:contact:user')))
            self.nn(core.getTufoByProp('inet:email', node[1].get('ps:contact:email')))
            self.nn(core.getTufoByProp('inet:email', node[1].get('ps:contact:email:work')))
            self.nn(core.getTufoByProp('inet:web:acct', node[1].get('ps:contact:web:acct')))
            self.nn(core.getTufoByProp('tel:phone', node[1].get('ps:contact:phone')))
            self.nn(core.getTufoByProp('tel:phone', node[1].get('ps:contact:phone:fax')))
            self.nn(core.getTufoByProp('tel:phone', node[1].get('ps:contact:phone:work')))

            self.eq(node[1].get('ps:contact:address'), '1 iron suit drive, san francisco, ca, 22222, usa')

    def test_model_ps_201802281621(self):
        N = 2
        FORMS = ('ps:hasuser', )
        NODECOUNT = N * len(FORMS)
        TAGS = ['hehe', 'hehe.hoho']

        def _check_no_old_data_remains(core, oldname):
            tufos = core.getTufosByProp(oldname)
            self.len(0, tufos)
            rows = core.getJoinByProp(oldname)
            self.len(0, rows)

        def _check_tags(core, tufo, tags):
            self.eq(sorted(tags), sorted(s_tufo.tags(tufo)))

        def _check_tagforms(core, oldname, newname, count):
            self.len(0, core.getRowsByProp('syn:tagform:form', oldname))
            self.len(0, core.getJoinByProp(oldname))
            self.len(count, core.getRowsByProp('syn:tagform:form', newname))

        def _check_tagdarks(core, oldname, newname, count):
            self.len(0, core.getRowsByProp('_:*' + oldname + '#hehe.hoho'))
            self.len(0, core.getRowsByProp('_:*' + oldname + '#hehe'))
            self.len(count, core.getRowsByProp('_:*' + newname + '#hehe.hoho'))
            self.len(count, core.getRowsByProp('_:*' + newname + '#hehe'))

        def run_assertions(core, oldname, reftype, tufo_check):
            # assert that the correct number of items was migrated
            tufos = core.getTufosByProp('ps:has:xref:prop', 'inet:user')
            self.len(N, tufos)

            # check that properties were correctly migrated and tags were not damaged
            tufo = tufo_check(core)

            # check that tags were correctly migrated
            _check_tags(core, tufo, TAGS)
            _check_tagforms(core, oldname, 'ps:has', NODECOUNT)
            _check_tagdarks(core, oldname, 'ps:has', NODECOUNT)

            # assert that no old data remains
            _check_no_old_data_remains(core, oldname)

        def _addTag(tag, form):
            tick = now()
            iden = guid()
            tlib = s_types.TypeLib()
            form_valu, _ = tlib.getTypeNorm('syn:tagform', (tag, form))
            return [
                (iden, 'syn:tagform:title', '??', tick),
                (iden, 'syn:tagform', form_valu, tick),
                (iden, 'tufo:form', 'syn:tagform', tick),
                (iden, 'syn:tagform:tag', tag, tick),
                (iden, 'syn:tagform:form', form, tick),
                (iden, 'syn:tagform:doc', '??', tick),
            ]

        # Add rows to the storage layer so we have something to migrate
        adds = []
        for form in FORMS:
            adds.extend(_addTag('hehe.hoho', form))
            adds.extend(_addTag('hehe', form))

        for i in range(N):
            user = 'pennywise%d' % i
            iden = guid()
            dark_iden = iden[::-1]
            tick = now()

            adds.extend([
                (iden, 'tufo:form', 'ps:hasuser', tick),
                (iden, 'ps:hasuser:user', user, tick),
                (iden, 'ps:hasuser:person', '2f6d1248de48f451e1f349cff33f336c', tick),
                (iden, 'ps:hasuser', '2f6d1248de48f451e1f349cff33f336c/' + user, tick),
                (iden, '#hehe.hoho', tick, tick),
                (iden, '#hehe', tick, tick),
                (dark_iden, '_:*ps:hasuser#hehe.hoho', tick, tick),
                (dark_iden, '_:*ps:hasuser#hehe', tick, tick),
            ])

        # Spin up a core with the old rows, then run the migration and check the results
        with s_cortex.openstore('ram:///') as stor:
            stor.setModlVers('ps', 0)
            def addrows(mesg):
                stor.addRows(adds)
            stor.on('modl:vers:rev', addrows, name='ps', vers=201802281621)

            with s_cortex.fromstore(stor) as core:

                # ps:hasuser ======================================================================
                oldname = 'ps:hasuser'
                reftype = 'inet:user'
                def tufo_check(core):
                    tufo = core.getTufoByProp('ps:has:xref', 'inet:user=pennywise0')
                    self.eq(tufo[1]['tufo:form'], 'ps:has')
                    self.eq(tufo[1]['ps:has'], 'f8cb39f7e4d8f1b82a5263c14655df68')
                    self.eq(tufo[1]['ps:has:xref'], 'inet:user=pennywise0')
                    self.eq(tufo[1]['ps:has:xref:prop'], 'inet:user')
                    self.eq(tufo[1]['ps:has:xref:strval'], 'pennywise0')
                    self.eq(tufo[1]['ps:has:person'], '2f6d1248de48f451e1f349cff33f336c')
                    return tufo
                run_assertions(core, oldname, reftype, tufo_check)
                return

                # ps:hashost ======================================================================
                oldname = 'ps:hashost'
                reftype = 'it:host'
                tufo_check = None  # FIXME
                run_assertions(core, oldname, reftype, tufo_check)
                return

                # ps:hasalias =====================================================================
                oldname = 'ps:hasalias'
                reftype = 'ps:name'
                tufo_check = None  # FIXME
                run_assertions(core, oldname, reftype, tufo_check)
                return

                # ps:hasphone =====================================================================
                oldname = 'ps:hasphone'
                reftype = 'tel:phone'
                tufo_check = None  # FIXME
                run_assertions(core, oldname, reftype, tufo_check)
                return

                # ps:hasemail =====================================================================
                oldname = 'ps:hasemail'
                reftype = 'inet:email'
                tufo_check = None  # FIXME
                run_assertions(core, oldname, reftype, tufo_check)
                return

                # ps:haswebacct ===================================================================
                oldname = 'ps:haswebacct'
                reftype = 'inet:web:acct'
                tufo_check = None  # FIXME
                run_assertions(core, oldname, reftype, tufo_check)
                return
<|endoftext|>


<commit_before>#!/usr/bin/env python

__author__ = 'Thomas R. Lennan, Michael Meisinger'
__license__ = 'Apache 2.0'

from pyon.datastore.datastore import DataStore

COUCHDB_CONFIGS = {
    DataStore.DS_PROFILE.OBJECTS:{
        'views': ['object','association','attachment']
    },
    DataStore.DS_PROFILE.RESOURCES:{
        'views': ['resource','association','attachment']
    },
    DataStore.DS_PROFILE.DIRECTORY:{
        'views': ['directory','association']
    },
    DataStore.DS_PROFILE.EVENTS:{
        'views': ['event']
    },
    DataStore.DS_PROFILE.STATE:{
        'views': []
    },
    DataStore.DS_PROFILE.SCIDATA:{
        'views': ['datasets']
    },
    DataStore.DS_PROFILE.EXAMPLES:{
        'views':['posts']
    },
    DataStore.DS_PROFILE.BASIC:{
        'views': []
    },
}

COUCHDB_VIEWS = {
    # Association (triple) related views
    'association':{
        'by_sub':{
            'map':"""
function(doc) {
  if (doc.type_ == "Association") {
    emit([doc.s, doc.p, doc.ot, doc.o], doc);
  }
}""",
        },
        'by_obj':{
            'map':"""
function(doc) {
  if (doc.type_ == "Association") {
    emit([doc.o, doc.p, doc.st, doc.s], doc);
  }
}""",
        },
        # For directed association lookup
        'by_ids':{
            'map':"""
function(doc) {
  if (doc.type_ == "Association") {
    emit([doc.s, doc.o, doc.p, doc.at, doc.srv, doc.orv], doc);
  }
}""",
        },
        # For undirected association lookup
        'by_id':{
            'map':"""
function(doc) {
  if (doc.type_ == "Association") {
    emit([doc.s, doc.p, doc.at, doc.srv, doc.orv], doc);
    emit([doc.o, doc.p, doc.at, doc.srv, doc.orv], doc);
  }
}""",
        },
        'by_pred':{
            'map':"""
function(doc) {
  if (doc.type_ == "Association") {
    emit([doc.p, doc.s, doc.o, doc.at, doc.srv, doc.orv], doc);
  }
}""",
        }
    },

    # Pure ION object related views
    # Every object has a type and ID
    'object':{
        'by_type':{
            'map':"""
function(doc) {
  emit([doc.type_], null);
}""",
        },
    },

    # Attachment objects
    'attachment':{
        'by_resource':{
            'map':"""
function(doc) {
  if (doc.type_ && doc.type_=="Attachment") {
    emit([doc.object_id, doc.ts_created], null);
  }
}""",
        }
    },

    # Resource ION object related views
    # Resources have a type, life cycle state and name
    # Note: the name in the indexes leads to a sort by name
    'resource':{
        'by_type':{
            'map':"""
function(doc) {
  if (doc.type_ && doc.type_!="Association") {
    emit([doc.type_, doc.lcstate, doc.name], null);
  }
}""",
        },
        # The following is a more sophisticated index. It does two things for each Resource object:
        # 1: It emits an index value prefixed by 0 for the actual lcstate
        # 2: It emits an index value prefixed by 1,parent_state for all parent states
        # Thereby it is possible to search for resources by hierarchical state and still be able
        # to return result sets that objects once only.
        # Note: the order of the type_ in the key is important for case 2, so that range queries are possible
        # with both type_ without.
        'by_lcstate':{
            'map':"""
function(doc) {
  if (doc.type_ && doc.type_!="Association") {
    emit([0, doc.lcstate, doc.type_, doc.name], null);
    if (doc.lcstate != undefined && doc.lcstate != "") {
      if (doc.lcstate.lastIndexOf("DRAFT",0)!=0 && doc.lcstate != "RETIRED") {
        emit([1, "REGISTERED", doc.type_, doc.lcstate, doc.name], null);
      }
      comps = doc.lcstate.split("_")
      if (comps.length == 2) {
        emit([1, comps[0], doc.type_, doc.lcstate, doc.name], null);
        emit([1, comps[1], doc.type_, doc.lcstate, doc.name], null);
      }
    }
  }
}""",
        },
        'by_name':{
            'map':"""
function(doc) {
  if (doc.type_ && doc.type_!="Association") {
    emit([doc.name, doc.type_, doc.lcstate], null);
  }
}""",
        },
    },

    # Directory related objects
    # DirEntry objects are the elements of the directory tree
    'directory':{
        'by_path':{
            'map':"""
function(doc) {
  if (doc.type_ == "DirEntry") {
    levels = doc.parent.split('/');
    if (doc.parent == "") levels.pop();
    levels.push(doc.key);
    emit(levels, doc);
  }
}""",
        },
        'by_key':{
            'map':"""
function(doc) {
  if (doc.type_ == "DirEntry") {
    emit([doc.key, doc.parent], doc);
  }
}""",
        },
        'by_parent':{
            'map':"""
function(doc) {
  if (doc.type_ == "DirEntry") {
    emit([doc.parent, doc.key], doc);
  }
}""",
        },
        'by_attribute':{
            'map':"""
function(doc) {
  if (doc.type_ == "DirEntry") {
    for (var attr in doc.attributes) {
      emit([attr, doc.attributes[attr], doc.parent], doc);
    }
  }
}""",
        },
    },

    # Event related objects
    'event':{
        'by_time':{
            'map':"""
function(doc) {
  if (doc.origin) {
    emit([doc.ts_created]);
  }
}""",
            },
        'by_type':{
            'map':"""
function(doc) {
  if (doc.origin) {
    emit([doc.type_, doc.ts_created]);
  }
}""",
        },
        'by_origin':{
            'map':"""
function(doc) {
  if (doc.origin) {
    emit([doc.origin, doc.ts_created]);
  }
}""",
        },
        'by_origintype':{
            'map':"""
function(doc) {
  if (doc.origin) {
    emit([doc.origin, doc.type_, doc.ts_created]);
  }
}""",
        },
    },
    'posts' : {
        "dataset_by_id": {
            "map": "function(doc)\n{\tif(doc.type_==\"BlogPost\") { emit([doc.post_id,0],doc._id);}\n\telse if(doc.type_==\"BlogComment\") { emit([doc.ref_id,1],doc._id);}\n}"
        },
        "posts_by_id": {
            "map": "function(doc)\n{\tif(doc.type_==\"BlogPost\") { emit(doc.post_id,doc._id);}}"
        },
        "posts_by_title": {
            "map": "function(doc)\n{\tif(doc.type_==\"BlogPost\") { emit(doc.title,doc._id);}}"
        },
        "posts_by_updated": {
            "map": "function(doc)\n{\tif(doc.type_==\"BlogPost\") { emit(doc.updated,doc._id);}}"
        },
        "posts_by_author": {
            "map": "function(doc)\n{\tif(doc.type_==\"BlogPost\") { emit(doc.author.name,doc._id);}}"
        },
        "comments_by_post_id": {
            "map": "function(doc)\n{\tif(doc.type_==\"BlogComment\") { emit(doc.ref_id,doc._id);}}"
        },
        "comments_by_author": {
            "map": "function(doc)\n{\tif(doc.type_==\"BlogComment\") { emit(doc.author.name,doc._id);}}"
        },
        "comments_by_updated": {
            "map": "function(doc)\n{\tif(doc.type_==\"BlogComment\") { emit(doc.updated,doc._id);}}"
        },
        "posts_join_comments": {
            "map": "function(doc)\n{\tif(doc.type_==\"BlogPost\") { emit([doc.post_id,0],doc._id);}\n\telse if(doc.type_==\"BlogComment\") { emit([doc.ref_id,1],doc._id);}\n}"
        },
        "posts_by_author_date": {
            "map": "function(doc) {\n  if(doc.type_==\"BlogPost\")\n    emit([doc.author.name,doc.updated,doc.post_id], doc.post_id);\n  else if(doc.type==\"BlogComment\")\n    emit([doc.author.name,doc.updated,doc.ref_id], doc.ref_id);\n}"
        }
    },
    'datasets': {
        # Bounds
        # Map: https://gist.github.com/1781675#file_maps.js
        # Reduce: https://gist.github.com/1781675#file_reduce_bounds.js
        "bounds": {
            "map": "\n/*\n * Author: Luke Campbell <lcampbell@asascience.com>\n * Description: Utility and map functions to map the DataContainer's bounding elements.\n */\n\n/*\n * Traverses an \"identifiable\" in a document to see if it contains a CoordinateAxis\n */\nfunction traverse(identifiable) {\n    if(identifiable.type_==\"CoordinateAxis\")\n    {\n        return identifiable.bounds_id;\n    }\n    else return null;\n}\n/*\n * Gets the CoordinateAxis objects and their bounds_ids\n */\nfunction get_bounds(doc)\n{\n    identifiables = doc.identifiables;\n    var bounds = [];\n    for(var i in identifiables)\n    {\n        var bounds_id = traverse(identifiables[i]);\n        if(bounds_id)\n            bounds.push(bounds_id);\n    }\n    return bounds;\n}\n\n/* Data map */\nfunction (doc) {\n    if(doc.type_ == \"StreamGranuleContainer\"){\n        var bounds = get_bounds(doc);\n        for(var b in bounds)\n        {\n            var key = bounds[b];\n            var s = String(key)\n            var pack = {};\n            pack[key] = doc.identifiables[key].value_pair;\n\n            emit([doc.stream_resource_id,1], pack);\n        }\n    }\n}\n\n",
            "reduce": "\n\nfunction value_in(value,ary) {\n    for(var i in ary)\n    {\n        if(value == ary[i])\n            return i;\n    }\n}\nfunction get_keys(obj){\n    var keys = [];\n    for(var key in obj)\n        keys.push(key);\n    return keys;\n}\nfunction print_dic(obj) {\n    for(var key in obj)\n    {\n        debug(key);\n        debug(obj[key]);\n    }\n}\n\nfunction(keys,values,rereduce) {\n    var keys = [];\n    var results = values[0];\n        \n    /* Not a rereduce so populate the results dictionary */\n\n\n    for(var i in values)\n    {\n        var key = get_keys(values[i])[0];\n        // The k,v pair is new, put it in results.\n        if(! value_in(key,keys))\n        {\n            keys.push(key);\n            results[key] = values[i][key];\n            continue;\n        }\n        var value = values[i][key];\n        /* minimum between the result and the new value */\n        if(value[0] < results[key][0])\n            results[key][0] = value[0];\n        if(value[1] > results[key][1])\n            results[key][1] = value[1];\n\n    }\n    return results;\n\n    \n    \n}"
        },
        # Stream join granule
        # Map: https://gist.github.com/1781675#file_map_stream_join_granule.js
        "stream_join_granule": {
            "map": "function (doc) {\n    if(doc.type_ == \"StreamDefinitionContainer\")\n        emit([doc.stream_resource_id,0],doc._id);\n    else if(doc.type_ == \"StreamGranuleContainer\")\n        emit([doc.stream_resource_id,1], doc._id);\n}\n"
        },
        # Cannonical reference to Stream join granule
        "dataset_by_id": {
            "map": "function (doc) {\n    if(doc.type_ == \"StreamDefinitionContainer\")\n        emit([doc.stream_resource_id,0],doc._id);\n    else if(doc.type_ == \"StreamGranuleContainer\")\n        emit([doc.stream_resource_id,1], doc._id);\n}\n"
        },
        # https://gist.github.com/1781675#file_by_time.js
        # This view only works for granules built using the constructor API or one that uses 'time_bounds' specifically.
        "dataset_by_latest": {
            "map": "/********************************\n * Author: Luke Campbell\n * Description: simple map to order by time\n ********************************/\n \n// If the doc has a time_bounds display it\nfunction get_time(doc) {\n    if(doc.identifiables.time_bounds) {\n        emit([doc.stream_resource_id,doc.identifiables.time_bounds.value_pair[0]],doc._id);\n    }   \n}\nfunction(doc) {\n  get_time(doc);\n}"
        }

    }
}

def get_couchdb_views(config):
    store_config = COUCHDB_CONFIGS[config]
    views = store_config['views']
    res_views = {}
    for view in views:
        res_views[view] = COUCHDB_VIEWS[view]
    return res_views
<commit_msg>Luke: Fixed bug in bounds (large datasets)<commit_after>#!/usr/bin/env python

__author__ = 'Thomas R. Lennan, Michael Meisinger'
__license__ = 'Apache 2.0'

from pyon.datastore.datastore import DataStore

COUCHDB_CONFIGS = {
    DataStore.DS_PROFILE.OBJECTS:{
        'views': ['object','association','attachment']
    },
    DataStore.DS_PROFILE.RESOURCES:{
        'views': ['resource','association','attachment']
    },
    DataStore.DS_PROFILE.DIRECTORY:{
        'views': ['directory','association']
    },
    DataStore.DS_PROFILE.EVENTS:{
        'views': ['event']
    },
    DataStore.DS_PROFILE.STATE:{
        'views': []
    },
    DataStore.DS_PROFILE.SCIDATA:{
        'views': ['datasets']
    },
    DataStore.DS_PROFILE.EXAMPLES:{
        'views':['posts']
    },
    DataStore.DS_PROFILE.BASIC:{
        'views': []
    },
}

COUCHDB_VIEWS = {
    # Association (triple) related views
    'association':{
        'by_sub':{
            'map':"""
function(doc) {
  if (doc.type_ == "Association") {
    emit([doc.s, doc.p, doc.ot, doc.o], doc);
  }
}""",
        },
        'by_obj':{
            'map':"""
function(doc) {
  if (doc.type_ == "Association") {
    emit([doc.o, doc.p, doc.st, doc.s], doc);
  }
}""",
        },
        # For directed association lookup
        'by_ids':{
            'map':"""
function(doc) {
  if (doc.type_ == "Association") {
    emit([doc.s, doc.o, doc.p, doc.at, doc.srv, doc.orv], doc);
  }
}""",
        },
        # For undirected association lookup
        'by_id':{
            'map':"""
function(doc) {
  if (doc.type_ == "Association") {
    emit([doc.s, doc.p, doc.at, doc.srv, doc.orv], doc);
    emit([doc.o, doc.p, doc.at, doc.srv, doc.orv], doc);
  }
}""",
        },
        'by_pred':{
            'map':"""
function(doc) {
  if (doc.type_ == "Association") {
    emit([doc.p, doc.s, doc.o, doc.at, doc.srv, doc.orv], doc);
  }
}""",
        }
    },

    # Pure ION object related views
    # Every object has a type and ID
    'object':{
        'by_type':{
            'map':"""
function(doc) {
  emit([doc.type_], null);
}""",
        },
    },

    # Attachment objects
    'attachment':{
        'by_resource':{
            'map':"""
function(doc) {
  if (doc.type_ && doc.type_=="Attachment") {
    emit([doc.object_id, doc.ts_created], null);
  }
}""",
        }
    },

    # Resource ION object related views
    # Resources have a type, life cycle state and name
    # Note: the name in the indexes leads to a sort by name
    'resource':{
        'by_type':{
            'map':"""
function(doc) {
  if (doc.type_ && doc.type_!="Association") {
    emit([doc.type_, doc.lcstate, doc.name], null);
  }
}""",
        },
        # The following is a more sophisticated index. It does two things for each Resource object:
        # 1: It emits an index value prefixed by 0 for the actual lcstate
        # 2: It emits an index value prefixed by 1,parent_state for all parent states
        # Thereby it is possible to search for resources by hierarchical state and still be able
        # to return result sets that objects once only.
        # Note: the order of the type_ in the key is important for case 2, so that range queries are possible
        # with both type_ without.
        'by_lcstate':{
            'map':"""
function(doc) {
  if (doc.type_ && doc.type_!="Association") {
    emit([0, doc.lcstate, doc.type_, doc.name], null);
    if (doc.lcstate != undefined && doc.lcstate != "") {
      if (doc.lcstate.lastIndexOf("DRAFT",0)!=0 && doc.lcstate != "RETIRED") {
        emit([1, "REGISTERED", doc.type_, doc.lcstate, doc.name], null);
      }
      comps = doc.lcstate.split("_")
      if (comps.length == 2) {
        emit([1, comps[0], doc.type_, doc.lcstate, doc.name], null);
        emit([1, comps[1], doc.type_, doc.lcstate, doc.name], null);
      }
    }
  }
}""",
        },
        'by_name':{
            'map':"""
function(doc) {
  if (doc.type_ && doc.type_!="Association") {
    emit([doc.name, doc.type_, doc.lcstate], null);
  }
}""",
        },
    },

    # Directory related objects
    # DirEntry objects are the elements of the directory tree
    'directory':{
        'by_path':{
            'map':"""
function(doc) {
  if (doc.type_ == "DirEntry") {
    levels = doc.parent.split('/');
    if (doc.parent == "") levels.pop();
    levels.push(doc.key);
    emit(levels, doc);
  }
}""",
        },
        'by_key':{
            'map':"""
function(doc) {
  if (doc.type_ == "DirEntry") {
    emit([doc.key, doc.parent], doc);
  }
}""",
        },
        'by_parent':{
            'map':"""
function(doc) {
  if (doc.type_ == "DirEntry") {
    emit([doc.parent, doc.key], doc);
  }
}""",
        },
        'by_attribute':{
            'map':"""
function(doc) {
  if (doc.type_ == "DirEntry") {
    for (var attr in doc.attributes) {
      emit([attr, doc.attributes[attr], doc.parent], doc);
    }
  }
}""",
        },
    },

    # Event related objects
    'event':{
        'by_time':{
            'map':"""
function(doc) {
  if (doc.origin) {
    emit([doc.ts_created]);
  }
}""",
            },
        'by_type':{
            'map':"""
function(doc) {
  if (doc.origin) {
    emit([doc.type_, doc.ts_created]);
  }
}""",
        },
        'by_origin':{
            'map':"""
function(doc) {
  if (doc.origin) {
    emit([doc.origin, doc.ts_created]);
  }
}""",
        },
        'by_origintype':{
            'map':"""
function(doc) {
  if (doc.origin) {
    emit([doc.origin, doc.type_, doc.ts_created]);
  }
}""",
        },
    },
    'posts' : {
        "dataset_by_id": {
            "map": "function(doc)\n{\tif(doc.type_==\"BlogPost\") { emit([doc.post_id,0],doc._id);}\n\telse if(doc.type_==\"BlogComment\") { emit([doc.ref_id,1],doc._id);}\n}"
        },
        "posts_by_id": {
            "map": "function(doc)\n{\tif(doc.type_==\"BlogPost\") { emit(doc.post_id,doc._id);}}"
        },
        "posts_by_title": {
            "map": "function(doc)\n{\tif(doc.type_==\"BlogPost\") { emit(doc.title,doc._id);}}"
        },
        "posts_by_updated": {
            "map": "function(doc)\n{\tif(doc.type_==\"BlogPost\") { emit(doc.updated,doc._id);}}"
        },
        "posts_by_author": {
            "map": "function(doc)\n{\tif(doc.type_==\"BlogPost\") { emit(doc.author.name,doc._id);}}"
        },
        "comments_by_post_id": {
            "map": "function(doc)\n{\tif(doc.type_==\"BlogComment\") { emit(doc.ref_id,doc._id);}}"
        },
        "comments_by_author": {
            "map": "function(doc)\n{\tif(doc.type_==\"BlogComment\") { emit(doc.author.name,doc._id);}}"
        },
        "comments_by_updated": {
            "map": "function(doc)\n{\tif(doc.type_==\"BlogComment\") { emit(doc.updated,doc._id);}}"
        },
        "posts_join_comments": {
            "map": "function(doc)\n{\tif(doc.type_==\"BlogPost\") { emit([doc.post_id,0],doc._id);}\n\telse if(doc.type_==\"BlogComment\") { emit([doc.ref_id,1],doc._id);}\n}"
        },
        "posts_by_author_date": {
            "map": "function(doc) {\n  if(doc.type_==\"BlogPost\")\n    emit([doc.author.name,doc.updated,doc.post_id], doc.post_id);\n  else if(doc.type==\"BlogComment\")\n    emit([doc.author.name,doc.updated,doc.ref_id], doc.ref_id);\n}"
        }
    },
    'datasets': {
        # Bounds
        # Map: https://gist.github.com/1781675#file_maps.js
        # Reduce: https://gist.github.com/1781675#file_reduce_bounds.js
        "bounds": {
            "map": "\n/*\n * Author: Luke Campbell <lcampbell@asascience.com>\n * Description: Utility and map functions to map the DataContainer's bounding elements.\n */\n\n/*\n * Traverses an \"identifiable\" in a document to see if it contains a CoordinateAxis\n */\nfunction traverse(identifiable) {\n    if(identifiable.type_==\"CoordinateAxis\")\n    {\n        return identifiable.bounds_id;\n    }\n    else return null;\n}\n/*\n * Gets the CoordinateAxis objects and their bounds_ids\n */\nfunction get_bounds(doc)\n{\n    identifiables = doc.identifiables;\n    var bounds = [];\n    for(var i in identifiables)\n    {\n        var bounds_id = traverse(identifiables[i]);\n        if(bounds_id)\n            bounds.push(bounds_id);\n    }\n    return bounds;\n}\n\n/* Data map */\nfunction (doc) {\n    if(doc.type_ == \"StreamGranuleContainer\"){\n        var bounds = get_bounds(doc);\n        for(var b in bounds)\n        {\n            var key = bounds[b];\n            var s = String(key)\n            var pack = {};\n            pack[key] = doc.identifiables[key].value_pair;\n\n            emit([doc.stream_resource_id,1], pack);\n        }\n    }\n}\n\n",
            "reduce": "\n\nfunction value_in(value,ary) {\n    for(var i in ary)\n    {\n        if(value == ary[i])\n            return i;\n    }\n}\nfunction get_keys(obj){\n    var keys = [];\n    for(var key in obj)\n        keys.push(key);\n    return keys;\n}\nfunction print_dic(obj) {\n    for(var key in obj)\n    {\n        debug(key);\n        debug(obj[key]);\n    }\n}\n\nfunction(keys,values,rereduce) {\n    var res_keys = [];\n    var results = values[0];\n        \n    /* Not a rereduce so populate the results dictionary */\n\n\n    for(var i in values)\n    {\n\n        var keys = get_keys(values[i]);\n        for(var j in keys)\n        {\n            var key = keys[j];\n\n            if(! value_in(key, res_keys))\n            {\n                res_keys.push(key);\n                results[key] = values[i][key];\n                continue;\n            }\n            var value = values[i][key];\n            results[key][0] = Math.min(value[0], results[key][0]);\n            results[key][1] = Math.max(value[1], results[key][1]);\n        }\n    }\n    return results;\n\n    \n    \n}"
        },
        # Stream join granule
        # Map: https://gist.github.com/1781675#file_map_stream_join_granule.js
        "stream_join_granule": {
            "map": "function (doc) {\n    if(doc.type_ == \"StreamDefinitionContainer\")\n        emit([doc.stream_resource_id,0],doc._id);\n    else if(doc.type_ == \"StreamGranuleContainer\")\n        emit([doc.stream_resource_id,1], doc._id);\n}\n"
        },
        # Cannonical reference to Stream join granule
        "dataset_by_id": {
            "map": "function (doc) {\n    if(doc.type_ == \"StreamDefinitionContainer\")\n        emit([doc.stream_resource_id,0],doc._id);\n    else if(doc.type_ == \"StreamGranuleContainer\")\n        emit([doc.stream_resource_id,1], doc._id);\n}\n"
        },
        # https://gist.github.com/1781675#file_by_time.js
        # This view only works for granules built using the constructor API or one that uses 'time_bounds' specifically.
        "dataset_by_latest": {
            "map": "/********************************\n * Author: Luke Campbell\n * Description: simple map to order by time\n ********************************/\n \n// If the doc has a time_bounds display it\nfunction get_time(doc) {\n    if(doc.identifiables.time_bounds) {\n        emit([doc.stream_resource_id,doc.identifiables.time_bounds.value_pair[0]],doc._id);\n    }   \n}\nfunction(doc) {\n  get_time(doc);\n}"
        }

    }
}

def get_couchdb_views(config):
    store_config = COUCHDB_CONFIGS[config]
    views = store_config['views']
    res_views = {}
    for view in views:
        res_views[view] = COUCHDB_VIEWS[view]
    return res_views
<|endoftext|>


<commit_before>import warnings
import pandas as pd
import time
import logging as lg

from urbanaccess.utils import log
from urbanaccess.gtfs.network import _time_selector

warnings.simplefilter(action = "ignore", category = FutureWarning)

def _calc_headways_by_route_stop(df):
    """
    Calculate headways by route stop

    Parameters
    ----------
    df : pandas.DataFrame
        interpolated stop times dataframe for stop times within the time
        range with appended trip and route information

    Returns
    -------
    dataframe : pandas.DataFrame
        dataframe of statistics of route stop headways in units of minutes
    """

    #TODO: Optimize for speed

    start_time = time.time()

    df['unique_stop_route'] = (
        df['unique_stop_id'].str.cat(
            df['unique_route_id'].astype('str'), sep=','))

    stop_route_groups = df.groupby('unique_stop_route')
    log('Starting route stop headway calculation for {:,} route stops...'.format(len(stop_route_groups)))
    results = {}
    for unique_stop_route, stop_route_group in stop_route_groups:
        stop_route_group.sort(['departure_time_sec_interpolate'],ascending = True, inplace=True)
        next_bus_time = stop_route_group['departure_time_sec_interpolate'].iloc[1:].values
        prev_bus_time = stop_route_group['departure_time_sec_interpolate'].iloc[:-1].values
        stop_route_group_headways = (next_bus_time - prev_bus_time)/60
        results[unique_stop_route] = pd.Series(stop_route_group_headways).describe()

    log('Route stop headway calculation complete. Took {:,.2f} seconds'.format(time.time()-start_time))

    return pd.DataFrame(results).T

def _headway_handler(interpolated_stop_times_df, trips_df,
                     routes_df, headway_timerange):
    """
    route stop headway calculator handler

    Parameters
    ----------
    interpolated_stop_times_df : pandas.DataFrame
        interpolated stop times dataframe for stop times within the time range
    trips_df : pandas.DataFrame
        trips dataframe
    routes_df : pandas.DataFrame
        routes dataframe
    headway_timerange : list
        time range for which to calculate headways between as a
        list of time 1 and time 2
        where times are a 24 hour clock strings such as:
        ['07:00:00','10:00:00']

    Returns
    -------
    headway_by_routestop_df : pandas.DataFrame
        dataframe of statistics of route stop headways in units of minutes
        with relevant route and stop information
    """
    start_time = time.time()

    # add unique trip and route id
    trips_df['unique_trip_id'] = (
        trips_df['trip_id'].str.cat(
            trips_df['unique_agency_id'].astype('str'), sep='_'))
    trips_df['unique_route_id'] = (
        trips_df['route_id'].str.cat(
            trips_df['unique_agency_id'].astype('str'), sep='_'))

    columns = ['unique_route_id','service_id','unique_trip_id','unique_agency_id']
    # if these optional cols exist then keep those that do
    optional_cols = ['direction_id', 'shape_id']
    for item in optional_cols:
       if item in trips_df.columns:
            columns.append(item)

    trips_df = trips_df[columns]

    # add unique route id
    routes_df['unique_route_id'] = (
        routes_df['route_id'].str.cat(
            routes_df['unique_agency_id'].astype('str'), sep='_'))

    columns = ['unique_route_id','route_long_name','route_type','unique_agency_id']
    routes_df = routes_df[columns]

    selected_interpolated_stop_times_df = _time_selector(df=interpolated_stop_times_df, starttime=headway_timerange[0], endtime=headway_timerange[1])

    tmp1 = pd.merge(trips_df, routes_df, how='left', left_on='unique_route_id', right_on='unique_route_id', sort=False)
    merge_df = pd.merge(selected_interpolated_stop_times_df, tmp1, how='left',
                        left_on ='unique_trip_id' , right_on='unique_trip_id', sort=False)
    cols_to_drop = ['unique_agency_id_y','unique_agency_id_x']
    merge_df.drop(cols_to_drop, axis=1, inplace=True)

    headway_by_routestop_df = _calc_headways_by_route_stop(df=merge_df)

    #add unique route stop node_id
    headway_by_routestop_df = pd.merge(headway_by_routestop_df, merge_df[['unique_stop_route','unique_stop_id','unique_route_id']], how='left', left_index = True, right_on='unique_stop_route', sort=False)
    headway_by_routestop_df.drop('unique_stop_route', axis=1, inplace=True)
    headway_by_routestop_df['node_id_route'] = (
        headway_by_routestop_df['unique_stop_id'].str.cat(
            headway_by_routestop_df['unique_route_id'].astype('str'), sep='_'))

    log('headway calculation complete. Took {:,.2f} seconds'.format(time.time()-start_time))

    return headway_by_routestop_df

def headways(gtfsfeeds_df,headway_timerange):
    """
    Calculate headways by route stop for a specific time range

    Parameters
    ----------
    gtfsfeeds_df : object
        gtfsfeeds_dfs object with all processed GTFS data tables
    headway_timerange : list
        time range for which to calculate headways between as a list of
        time 1 and time 2
        where times are 24 hour clock strings such as:
        ['07:00:00','10:00:00']

    Returns
    -------
    gtfsfeeds_dfs.headways : pandas.DataFrame
        gtfsfeeds_dfs object for the headways dataframe with statistics of
        route stop headways in units of minutes
        with relevant route and stop information
    """

    # TODO: Change Assertion to errors/exceptions
    time_error_statement = ('{} starttime and endtime are not in the correct format. '
                       'Format should be 24 hour clock in following format: 08:00:00 or 17:00:00'.format(headway_timerange))
    assert isinstance(headway_timerange,list) and len(headway_timerange) == 2, time_error_statement
    assert headway_timerange[0] < headway_timerange[1], time_error_statement

    for t in headway_timerange:
        assert isinstance(t,str), time_error_statement
        assert len(t) == 8, time_error_statement
    if int(str(headway_timerange[1][0:2])) - int(str(headway_timerange[0][0:2])) > 3:
        log('WARNING: Time range passed: {} is a {} hour period. Long periods over 3 hours may take a '
            'significant amount of time to process.'.format(headway_timerange,
                                                            int(str(headway_timerange[1][0:2])) - int(str(headway_timerange[0][0:2]))),level=lg.WARNING)

    assert gtfsfeeds_df is not None
    if gtfsfeeds_df.stop_times_int.empty or gtfsfeeds_df.trips.empty or gtfsfeeds_df.routes.empty:
        raise ValueError('one of the gtfsfeeds_dfs objects: stop_times_int, trips, or routes were found to be empty.')

    headways_df = _headway_handler(interpolated_stop_times_df=gtfsfeeds_df.stop_times_int,
                                   trips_df=gtfsfeeds_df.trips,
                                   routes_df=gtfsfeeds_df.routes,
                                   headway_timerange=headway_timerange)

    gtfsfeeds_df.headways = headways_df

    return gtfsfeeds_df
<commit_msg>replaced depreciated pandas sort method<commit_after>import warnings
import pandas as pd
import time
import logging as lg

from urbanaccess.utils import log
from urbanaccess.gtfs.network import _time_selector

warnings.simplefilter(action = "ignore", category = FutureWarning)

def _calc_headways_by_route_stop(df):
    """
    Calculate headways by route stop

    Parameters
    ----------
    df : pandas.DataFrame
        interpolated stop times dataframe for stop times within the time
        range with appended trip and route information

    Returns
    -------
    dataframe : pandas.DataFrame
        dataframe of statistics of route stop headways in units of minutes
    """

    #TODO: Optimize for speed

    start_time = time.time()

    df['unique_stop_route'] = (
        df['unique_stop_id'].str.cat(
            df['unique_route_id'].astype('str'), sep=','))

    stop_route_groups = df.groupby('unique_stop_route')
    log('Starting route stop headway calculation for {:,} route stops...'.format(len(stop_route_groups)))
    results = {}
    for unique_stop_route, stop_route_group in stop_route_groups:
        stop_route_group.sort_value(['departure_time_sec_interpolate'],
                                    ascending=True, inplace=True)
        next_bus_time = stop_route_group['departure_time_sec_interpolate'].iloc[1:].values
        prev_bus_time = stop_route_group['departure_time_sec_interpolate'].iloc[:-1].values
        stop_route_group_headways = (next_bus_time - prev_bus_time)/60
        results[unique_stop_route] = pd.Series(stop_route_group_headways).describe()

    log('Route stop headway calculation complete. Took {:,.2f} seconds'.format(time.time()-start_time))

    return pd.DataFrame(results).T

def _headway_handler(interpolated_stop_times_df, trips_df,
                     routes_df, headway_timerange):
    """
    route stop headway calculator handler

    Parameters
    ----------
    interpolated_stop_times_df : pandas.DataFrame
        interpolated stop times dataframe for stop times within the time range
    trips_df : pandas.DataFrame
        trips dataframe
    routes_df : pandas.DataFrame
        routes dataframe
    headway_timerange : list
        time range for which to calculate headways between as a
        list of time 1 and time 2
        where times are a 24 hour clock strings such as:
        ['07:00:00','10:00:00']

    Returns
    -------
    headway_by_routestop_df : pandas.DataFrame
        dataframe of statistics of route stop headways in units of minutes
        with relevant route and stop information
    """
    start_time = time.time()

    # add unique trip and route id
    trips_df['unique_trip_id'] = (
        trips_df['trip_id'].str.cat(
            trips_df['unique_agency_id'].astype('str'), sep='_'))
    trips_df['unique_route_id'] = (
        trips_df['route_id'].str.cat(
            trips_df['unique_agency_id'].astype('str'), sep='_'))

    columns = ['unique_route_id','service_id','unique_trip_id','unique_agency_id']
    # if these optional cols exist then keep those that do
    optional_cols = ['direction_id', 'shape_id']
    for item in optional_cols:
       if item in trips_df.columns:
            columns.append(item)

    trips_df = trips_df[columns]

    # add unique route id
    routes_df['unique_route_id'] = (
        routes_df['route_id'].str.cat(
            routes_df['unique_agency_id'].astype('str'), sep='_'))

    columns = ['unique_route_id','route_long_name','route_type','unique_agency_id']
    routes_df = routes_df[columns]

    selected_interpolated_stop_times_df = _time_selector(df=interpolated_stop_times_df, starttime=headway_timerange[0], endtime=headway_timerange[1])

    tmp1 = pd.merge(trips_df, routes_df, how='left', left_on='unique_route_id', right_on='unique_route_id', sort=False)
    merge_df = pd.merge(selected_interpolated_stop_times_df, tmp1, how='left',
                        left_on ='unique_trip_id' , right_on='unique_trip_id', sort=False)
    cols_to_drop = ['unique_agency_id_y','unique_agency_id_x']
    merge_df.drop(cols_to_drop, axis=1, inplace=True)

    headway_by_routestop_df = _calc_headways_by_route_stop(df=merge_df)

    #add unique route stop node_id
    headway_by_routestop_df = pd.merge(headway_by_routestop_df, merge_df[['unique_stop_route','unique_stop_id','unique_route_id']], how='left', left_index = True, right_on='unique_stop_route', sort=False)
    headway_by_routestop_df.drop('unique_stop_route', axis=1, inplace=True)
    headway_by_routestop_df['node_id_route'] = (
        headway_by_routestop_df['unique_stop_id'].str.cat(
            headway_by_routestop_df['unique_route_id'].astype('str'), sep='_'))

    log('headway calculation complete. Took {:,.2f} seconds'.format(time.time()-start_time))

    return headway_by_routestop_df

def headways(gtfsfeeds_df,headway_timerange):
    """
    Calculate headways by route stop for a specific time range

    Parameters
    ----------
    gtfsfeeds_df : object
        gtfsfeeds_dfs object with all processed GTFS data tables
    headway_timerange : list
        time range for which to calculate headways between as a list of
        time 1 and time 2
        where times are 24 hour clock strings such as:
        ['07:00:00','10:00:00']

    Returns
    -------
    gtfsfeeds_dfs.headways : pandas.DataFrame
        gtfsfeeds_dfs object for the headways dataframe with statistics of
        route stop headways in units of minutes
        with relevant route and stop information
    """

    # TODO: Change Assertion to errors/exceptions
    time_error_statement = ('{} starttime and endtime are not in the correct format. '
                       'Format should be 24 hour clock in following format: 08:00:00 or 17:00:00'.format(headway_timerange))
    assert isinstance(headway_timerange,list) and len(headway_timerange) == 2, time_error_statement
    assert headway_timerange[0] < headway_timerange[1], time_error_statement

    for t in headway_timerange:
        assert isinstance(t,str), time_error_statement
        assert len(t) == 8, time_error_statement
    if int(str(headway_timerange[1][0:2])) - int(str(headway_timerange[0][0:2])) > 3:
        log('WARNING: Time range passed: {} is a {} hour period. Long periods over 3 hours may take a '
            'significant amount of time to process.'.format(headway_timerange,
                                                            int(str(headway_timerange[1][0:2])) - int(str(headway_timerange[0][0:2]))),level=lg.WARNING)

    assert gtfsfeeds_df is not None
    if gtfsfeeds_df.stop_times_int.empty or gtfsfeeds_df.trips.empty or gtfsfeeds_df.routes.empty:
        raise ValueError('one of the gtfsfeeds_dfs objects: stop_times_int, trips, or routes were found to be empty.')

    headways_df = _headway_handler(interpolated_stop_times_df=gtfsfeeds_df.stop_times_int,
                                   trips_df=gtfsfeeds_df.trips,
                                   routes_df=gtfsfeeds_df.routes,
                                   headway_timerange=headway_timerange)

    gtfsfeeds_df.headways = headways_df

    return gtfsfeeds_df
<|endoftext|>


<commit_before>"""Library of command line tools for Squadron Automation."""

import sys
import argparse

VERBOSE = False
DEBUG = False

class PreprocessUI(object):
    """UI Interface for the Preprocessor Command Line tool."""

    description = """Squadron Preprocessor Tool."""

    def __init__(self, version, verbose=False, debug=False):
        self.parser = argparse.ArgumentParser(description=__doc__)
        self.version = version
        self.built = False
        global VERBOSE
        VERBOSE = verbose
        global DEBUG
        DEBUG = debug

    def build(self):
        """Build the argument parser."""
        self.parser.add_argument(
            '--version', help='Print the version and exit.', action='version',
            version='%(prog)s {}'.format(self.version))
        DebugAction.add_parser_argument(self.parser)
        VerboseAction.add_parser_argument(self.parser)
        self.built = True

    def parse_cmd_line(self):
        if not self.built:
            self.build()
        return self.parser.parse_args()

def dprint(msg):
    """Conditionally print a debug message."""
    if DEBUG:
        print(msg)

def vprint(msg):
    """Conditionally print a verbose message."""
    if VERBOSE:
        print(msg)

class DebugAction(argparse.Action):
    """Enable the debugging output mechanism."""

    flag = '--debug'
    help = 'Enable debugging output.'

    @classmethod
    def add_parser_argument(cls, parser):
        parser.add_argument(cls.flag, help=cls.help, action=cls)

    def __init__(self, option_strings, dest, **kwargs):
        super(DebugAction, self).__init__(option_strings, dest, nargs=0,
                                          default=False, **kwargs)

    def __call__(self, parser, namespace, values, option_string=None):
        print('Enabling debugging output.')
        global DEBUG
        DEBUG = True
        setattr(namespace, self.dest, True)

class VerboseAction(DebugAction):
    """Enable the verbose output mechanism."""

    flag = '--verbose'
    help = 'Enable verbose output.'

    def __call__(self, parser, namespace, values, option_string=None):
        print('Enabling verbose output.')
        global VERBOSE
        VERBOSE = True
        setattr(namespace, self.dest, True)
<commit_msg>cmdline.py: add short flags for debug and verbose<commit_after>"""Library of command line tools for Squadron Automation."""

import sys
import argparse

VERBOSE = False
DEBUG = False

class PreprocessUI(object):
    """UI Interface for the Preprocessor Command Line tool."""

    description = "A description of the UI tool."

    def __init__(self, version, verbose=False, debug=False):
        self.parser = argparse.ArgumentParser(description=__doc__)
        self.version = version
        self.built = False
        global VERBOSE
        VERBOSE = verbose
        global DEBUG
        DEBUG = debug

    def build(self):
        """Build the argument parser."""
        self.parser.add_argument(
            '--version', help='Print the version and exit.', action='version',
            version='%(prog)s {}'.format(self.version))
        DebugAction.add_parser_argument(self.parser)
        VerboseAction.add_parser_argument(self.parser)
        self.built = True

    def parse_cmd_line(self):
        if not self.built:
            self.build()
        return self.parser.parse_args()

def dprint(msg):
    """Conditionally print a debug message."""
    if DEBUG:
        print('dbg:', msg)

def vprint(msg):
    """Conditionally print a verbose message."""
    if VERBOSE:
        print(msg)

class DebugAction(argparse.Action):
    """Enable the debugging output mechanism."""

    shortflag = '-d'
    flag = '--debug'
    help = 'Enable debugging output.'

    @classmethod
    def add_parser_argument(cls, parser):
        parser.add_argument(cls.flag, help=cls.help, action=cls)

    def __init__(self, option_strings, dest, **kwargs):
        super().__init__(option_strings, dest, nargs=0,
                         default=False, **kwargs)

    def __call__(self, parser, namespace, values, option_string=None):
        print('Enabling debugging output.')
        global DEBUG
        DEBUG = True
        setattr(namespace, self.dest, True)

class VerboseAction(DebugAction):
    """Enable the verbose output mechanism."""

    shortflag = '-v'
    flag = '--verbose'
    help = 'Enable verbose output.'

    def __call__(self, parser, namespace, values, option_string=None):
        print('Enabling verbose output.')
        global VERBOSE
        VERBOSE = True
        setattr(namespace, self.dest, True)
<|endoftext|>


<commit_before>#!/usr/bin/env python

from __future__ import print_function

import sys
import json
import os
import os.path
import operator
from pymining import itemmining
from os import listdir
from os.path import isfile, join
from libbin import *
from pyspark import SparkContext
import numpy as np

MIN_SUPPORT = 0
MIN_DEPENDENT_V = 2
START_WINDOW = -1
STOP_WINDOW = -1

def validDependent(input):
    validNumber = 0
    for item in input:
        if item in DEPENDENT_VARIABLE:
            validNumber += 1

    if validNumber >= MIN_DEPENDENT_V:
        return True
    # Python interpretter is not smart engouht to make a default return without else
    else:
        return False

def frequentSet(input):
    relim_input = itemmining.get_relim_input(input)
    report = itemmining.relim(relim_input, min_support=MIN_SUPPORT)

    return report
    # by pass filtering parameters
    filteredResult = []
    for item in report:
        for key in item:
            if validDependent(list(key)):
                filteredResult.append(item)

    return (0,filteredResult)

'''
# This function is used when we want to indicate the specific databand of the output
def filterDependentVariable(input):
    resultSet = ()
    for key, value in input:
        if validDependent(key):
            resultSet.append((key,value))

    return resultSet
'''

def mergeResult(input1,input2):
    return input1.entend(input2)

def filterResult(input):
    resultSet = ()
    for item in input:
        print(item)
        for key, value in item:
            if validDependent(key):
                resultSet.append((key,value))

    return resultSet

if __name__ == "__main__":
    if len(sys.argv) != 5:
        print("Usage: urdmeMiningPartialWindow <winSize> <minSupport> <startRange:inclusion> <stopRange:exclusion>\nThe number of range can be 0 to 99 (Subject to metadata).", file=sys.stderr)
        exit(-1)

    print("NOTE: The division of \"(stopWindowPosition - startWindowPosition) / windowSize\" must contain no fraction.\nOtherwise, it will be ignore!
")
    MIN_SUPPORT = int(sys.argv[2])
    START_WINDOW = int(sys.argv[3])
    STOP_WINDOW = int(sys.argv[4])
    WIN_SIZE = int(sys.argv[1])

    if STOP_WINDOW < START_WINDOW:
        print("Invalid wondow range!")
        quit

    windowGroup = []

    '''
Efficiency Note: The developer intend to program it easy which is trade off with the efficiency of the algorithm that need to go throught every file when the createTransaction() is called. This can be optimize, but the developer would like to point out and note to clarify why this step take certain amount of time.
    '''
    for i in range( (STOP_WINDOW -  START_WINDOW) / WIN_SIZE ):

        windowGroup.append(createTransaction(range(i * WIN_SIZE + START_WINDOW, i * WIN_SIZE + START_WINDOW + WIN_SIZE)))
        print("Window Frame: %s" % range(i * WIN_SIZE + START_WINDOW, i * WIN_SIZE + START_WINDOW + WIN_SIZE))

    sc = SparkContext( appName="urdmeMining",  master=os.environ['MASTER'])

    rddWindows = sc.parallelize(windowGroup)
    '''
    Note: Cannot do reduceByKey. Might be something here. Will check later
    result = rddWindows.map(frequentSet).reduceByKey(mergeResult).collect()
    '''
    result = rddWindows.map(frequentSet).collect()

    print(result)

    '''
    outputFile = "urdmeMiningPartialWindow_" + WIN_SIZE + "_" + MIN_SOPPORT + "_" + STOP_WINDOW + "_" + START_WINDOW + ".json"
    f = open(outpuFile,'w')
    f.write(json.dumps(outputData))
    f.close()
    '''
    sc.stop()
<commit_msg>Include output expoert to file<commit_after>#!/usr/bin/env python

from __future__ import print_function

import sys
import json
import os
import os.path
import operator
from pymining import itemmining
from os import listdir
from os.path import isfile, join
from libbin import *
from pyspark import SparkContext
import numpy as np

MIN_SUPPORT = 0
MIN_DEPENDENT_V = 2
START_WINDOW = -1
STOP_WINDOW = -1

def validDependent(input):
    validNumber = 0
    for item in input:
        if item in DEPENDENT_VARIABLE:
            validNumber += 1

    if validNumber >= MIN_DEPENDENT_V:
        return True
    # Python interpretter is not smart engouht to make a default return without else
    else:
        return False

def frequentSet(input):
    relim_input = itemmining.get_relim_input(input)
    report = itemmining.relim(relim_input, min_support=MIN_SUPPORT)

    return report
    # by pass filtering parameters
    filteredResult = []
    for item in report:
        for key in item:
            if validDependent(list(key)):
                filteredResult.append(item)

    return (0,filteredResult)

'''
# This function is used when we want to indicate the specific databand of the output
def filterDependentVariable(input):
    resultSet = ()
    for key, value in input:
        if validDependent(key):
            resultSet.append((key,value))

    return resultSet
'''

def mergeResult(input1,input2):
    return input1.entend(input2)

def filterResult(input):
    resultSet = ()
    for item in input:
        print(item)
        for key, value in item:
            if validDependent(key):
                resultSet.append((key,value))

    return resultSet

def convertFrozenSetToJson(input):
    dataSingleObject = "["
    for item in input:
        resultKeys = item.keys()
        resultValues = item.values()
        resultData = "["
        for i in range(len(resultKeys)):
            resultData += "{ \"_key\" : \"" +  str(list(resultKeys[i])) + "\", \"_value\": " + str(resultValues[i]) + "}"
            if i < len(resultKeys) - 1:
                resultData += ","
        dataSingleObject += resultData + "],"
    return dataSingleObject[:-1] + "]"

if __name__ == "__main__":
    if len(sys.argv) != 5:
        print("Usage: urdmeMiningPartialWindow <winSize> <minSupport> <startRange:inclusion> <stopRange:exclusion>\nThe number of range can be 0 to 99 (Subject to metadata).", file=sys.stderr)
        exit(-1)

    print("\033[93m" + "NOTE: The division of \"(stopWindowPosition - startWindowPosition) / windowSize\" must contain no fraction. Otherwise, it will be ignore!" + "\033[0m")
    MIN_SUPPORT = int(sys.argv[2])
    START_WINDOW = int(sys.argv[3])
    STOP_WINDOW = int(sys.argv[4])
    WIN_SIZE = int(sys.argv[1])

    if STOP_WINDOW < START_WINDOW:
        print("Invalid wondow range!", file=sys.stderr)
        quit()

    windowGroup = []

    '''
Efficiency Note: The developer intend to program it easy which is trade off with the efficiency of the algorithm that need to go throught every file when the createTransaction() is called. This can be optimize, but the developer would like to point out and note to clarify why this step take certain amount of time.
    '''
    for i in range( (STOP_WINDOW -  START_WINDOW) / WIN_SIZE ):

        windowGroup.append(createTransaction(range(i * WIN_SIZE + START_WINDOW, i * WIN_SIZE + START_WINDOW + WIN_SIZE)))
        print("Window Frame: %s" % range(i * WIN_SIZE + START_WINDOW, i * WIN_SIZE + START_WINDOW + WIN_SIZE))

    sc = SparkContext( appName="urdmeMining",  master=os.environ['MASTER'])

    rddWindows = sc.parallelize(windowGroup)
    '''
    Note: Cannot do reduceByKey. Might be something here. Will check later
    result = rddWindows.map(frequentSet).reduceByKey(mergeResult).collect()
    '''
    result = rddWindows.map(frequentSet).collect()

    outputData = convertFrozenSetToJson(result)

    print("\033[92m" + outputData + "\033[0m")

    outputFile = "output/urdmeMiningPartialWindow_" + str(WIN_SIZE) + "_" + str(MIN_SUPPORT) + "_" + str(STOP_WINDOW) + "_" + str(START_WINDOW) + ".json"
    f = open(outputFile,'w')
    f.write(json.dumps(outputData))
    f.close()
    print("Complete!!!")
    sc.stop()
<|endoftext|>


<commit_before>#!/usr/bin/env python
from decimal import *

from synctools import command

__all__ = ['AdjustOffset']

class AdjustOffset(command.SynctoolsCommand):

    title = 'Adjust offset'
    description = 'tweak offsets for hardware delay and/or personal preference'
    fields = [
        {
            'name': 'amount',
            'title': 'Seconds to add',
            'input': command.FieldInputs.text,
            'default': '0.000',
            'type': Decimal,
        },
        command.common_fields['backup'],
    ]

    def run(self, simfile):
        super(AdjustOffset, self).run(simfile)
        old_offset = simfile['OFFSET']
        new_offset = Decimal(old_offset) + Decimal(self.options['amount'])
        self.log.info('%s -> %s' % (old_offset, new_offset))
        simfile['OFFSET'] = new_offset
        simfile.save()<commit_msg>Don't re-cast self.options['amount']<commit_after>#!/usr/bin/env python
from decimal import *

from synctools import command

__all__ = ['AdjustOffset']

class AdjustOffset(command.SynctoolsCommand):

    title = 'Adjust offset'
    description = 'tweak offsets for hardware delay and/or personal preference'
    fields = [
        {
            'name': 'amount',
            'title': 'Seconds to add',
            'input': command.FieldInputs.text,
            'default': '0.000',
            'type': Decimal,
        },
        command.common_fields['backup'],
    ]

    def run(self, simfile):
        super(AdjustOffset, self).run(simfile)
        old_offset = simfile['OFFSET']
        new_offset = Decimal(old_offset) + self.options['amount']
        self.log.info('%s -> %s' % (old_offset, new_offset))
        simfile['OFFSET'] = new_offset
        simfile.save()<|endoftext|>


<commit_before>"""
==========================
PySpecKit ASCII Reader
==========================

Routines for reading in ASCII format spectra.  If atpy is not installed,
will use a very simple routine for reading in the data.

.. moduleauthor:: Adam Ginsburg <adam.g.ginsburg@gmail.com>
.. moduleauthor:: Jordan Mirocha <mirochaj@gmail.com>
"""
try:
    import atpy
    atpyOK = True
except ImportError:
    atpyOK = False
from .. import units
import numpy as np
from pyspeckit.specwarnings import warn

def open_1d_txt(filename, xaxcol=0, datacol=1, errorcol=2,
        text_reader='simple', atpytype='ascii', **kwargs):
    """
    Attempt to read a 1D spectrum from a text file assuming wavelength as the
    first column, data as the second, and (optionally) error as the third.

    Reading can be done either with atpy or a 'simple' reader.  If you have an
    IPAC, CDS, or formally formatted table, you'll want to use atpy.

    If you have a simply formatted file of the form, e.g.
    # name name
    # unit unit
    data data
    data data

    kwargs are passed to atpy.Table
    """
    if text_reader=='simple' or not atpyOK:
        warn("WARNING: atpy not installed; will use simple reader instead.")

        data, error, XAxis, T = simple_txt(filename, xaxcol = xaxcol,
            datacol = datacol, errorcol = errorcol, **kwargs)

    elif text_reader in ('atpy','asciitable'):
        T = atpy.Table(filename, type=atpytype, masked=True, **kwargs)

        xarr = T.data[T.data.dtype.names[xaxcol]]
        data = T.data[T.data.dtype.names[datacol]]
        if len(T.columns) > errorcol:
            error = T.data[T.data.dtype.names[errorcol]]
        else:
            # assume uniform, zero error
            error = data*0

        if 'xunits' in T.keywords:
            xunits = T.keywords['xunits']
        else:
            xunits = 'unknown'

        XAxis = units.SpectroscopicAxis(xarr,xunits)

    # Need this in Spectrum class to correctly parse header
    T.xaxcol = xaxcol
    T.datacol = datacol

    return data, error, XAxis, T

def simple_txt(filename, xaxcol=0, datacol=1, errorcol=2, **kwargs):
    """
    Very simple method for reading columns from ASCII file.
    """

    f = open(filename, 'r')

    colunits = []
    coldata = []
    for i, line in enumerate(f):

        # Ignore blank lines
        if not line.strip(): continue

        # Possibly read in header
        if line.split()[0][0] == '#':
            if i == 0: hdr = line[1:].split()
            if i == 1: colunits = line[1:].split()

            continue

        coldata.append(line.split())

        for j, element in enumerate(coldata[-1]):
            try: coldata[-1][j] = float(element)
            except ValueError: coldata[-1][j] = str(element)

    f.close()

    if not colunits:
        colunits = ['unknown', 'unknown', 'unknown']
    if not hdr:
        hdr = ['unknown', 'unknown', 'unknown']

    N = len(hdr)
    coldata = zip(*coldata)

    # Prepare to return data
    data = coldata[datacol]
    xarr = coldata[xaxcol]
    if errorcol > N - 1:
        error = np.array(data)*0
    else:
        error = coldata[errorcol]

    if len(error) != len(data):
        raise ValueError("Data and Error lengths do not match.")

    XAxis = units.SpectroscopicAxis(xarr, colunits[xaxcol])

    # Create atPy style Table instance
    T = dummy_class()
    T.data = dummy_class()
    T.data.dtype = dummy_class()
    T.data.dtype.names = hdr
    T.columns = {}
    T.columns[T.data.dtype.names[xaxcol]] = dummy_class()
    T.columns[T.data.dtype.names[xaxcol]].unit = colunits[xaxcol]
    T.columns[T.data.dtype.names[datacol]] = dummy_class()
    T.columns[T.data.dtype.names[datacol]].unit = colunits[datacol]

    return np.array(data), np.array(error), XAxis, T

class dummy_class:
    def __init__(self):
        pass

<commit_msg>minor change - when 'simple' text reader is used, should not warn user that atpy is not available (it may be!)<commit_after>"""
==========================
PySpecKit ASCII Reader
==========================

Routines for reading in ASCII format spectra.  If atpy is not installed,
will use a very simple routine for reading in the data.

.. moduleauthor:: Adam Ginsburg <adam.g.ginsburg@gmail.com>
.. moduleauthor:: Jordan Mirocha <mirochaj@gmail.com>
"""
try:
    import atpy
    atpyOK = True
except ImportError:
    atpyOK = False
from .. import units
import numpy as np
from pyspeckit.specwarnings import warn

def open_1d_txt(filename, xaxcol=0, datacol=1, errorcol=2,
        text_reader='simple', atpytype='ascii', **kwargs):
    """
    Attempt to read a 1D spectrum from a text file assuming wavelength as the
    first column, data as the second, and (optionally) error as the third.

    Reading can be done either with atpy or a 'simple' reader.  If you have an
    IPAC, CDS, or formally formatted table, you'll want to use atpy.

    If you have a simply formatted file of the form, e.g.
    # name name
    # unit unit
    data data
    data data

    kwargs are passed to atpy.Table
    """
    if text_reader=='simple' or not atpyOK:
        if not atpyOK:
            warn("WARNING: atpy not installed; will use simple reader instead.")

        data, error, XAxis, T = simple_txt(filename, xaxcol = xaxcol,
            datacol = datacol, errorcol = errorcol, **kwargs)

    elif text_reader in ('atpy','asciitable'):
        T = atpy.Table(filename, type=atpytype, masked=True, **kwargs)

        xarr = T.data[T.data.dtype.names[xaxcol]]
        data = T.data[T.data.dtype.names[datacol]]
        if len(T.columns) > errorcol:
            error = T.data[T.data.dtype.names[errorcol]]
        else:
            # assume uniform, zero error
            error = data*0

        if 'xunits' in T.keywords:
            xunits = T.keywords['xunits']
        else:
            xunits = 'unknown'

        XAxis = units.SpectroscopicAxis(xarr,xunits)

    # Need this in Spectrum class to correctly parse header
    T.xaxcol = xaxcol
    T.datacol = datacol

    return data, error, XAxis, T

def simple_txt(filename, xaxcol=0, datacol=1, errorcol=2, **kwargs):
    """
    Very simple method for reading columns from ASCII file.
    """

    f = open(filename, 'r')

    colunits = []
    coldata = []
    for i, line in enumerate(f):

        # Ignore blank lines
        if not line.strip(): continue

        # Possibly read in header
        if line.split()[0][0] == '#':
            if i == 0: hdr = line[1:].split()
            if i == 1: colunits = line[1:].split()

            continue

        coldata.append(line.split())

        for j, element in enumerate(coldata[-1]):
            try: coldata[-1][j] = float(element)
            except ValueError: coldata[-1][j] = str(element)

    f.close()

    if not colunits:
        colunits = ['unknown', 'unknown', 'unknown']
    if not hdr:
        hdr = ['unknown', 'unknown', 'unknown']

    N = len(hdr)
    coldata = zip(*coldata)

    # Prepare to return data
    data = coldata[datacol]
    xarr = coldata[xaxcol]
    if errorcol > N - 1:
        error = np.array(data)*0
    else:
        error = coldata[errorcol]

    if len(error) != len(data):
        raise ValueError("Data and Error lengths do not match.")

    XAxis = units.SpectroscopicAxis(xarr, colunits[xaxcol])

    # Create atPy style Table instance
    T = dummy_class()
    T.data = dummy_class()
    T.data.dtype = dummy_class()
    T.data.dtype.names = hdr
    T.columns = {}
    T.columns[T.data.dtype.names[xaxcol]] = dummy_class()
    T.columns[T.data.dtype.names[xaxcol]].unit = colunits[xaxcol]
    T.columns[T.data.dtype.names[datacol]] = dummy_class()
    T.columns[T.data.dtype.names[datacol]].unit = colunits[datacol]

    return np.array(data), np.array(error), XAxis, T

class dummy_class:
    def __init__(self):
        pass

<|endoftext|>