<commit_before>import synapse.common as s_common

from synapse.lib.types import DataType
from synapse.common import addpref, guid
from synapse.lib.module import CoreModule, modelrev

class FileBaseType(DataType):
    def norm(self, valu, oldval=None):
        if not (isinstance(valu, str) and not valu.find('/') > -1):
            self._raiseBadValu(valu)

        return valu.lower(), {}

    def repr(self, valu):
        return valu

class FilePathType(DataType):
    def norm(self, valu, oldval=None):

        if not isinstance(valu, str):
            self._raiseBadValu(valu)

        lead = ''

        valu = valu.replace('\\', '/').lower()
        if valu and valu[0] == '/':
            lead = '/'

        valu = valu.strip('/')

        vals = [v for v in valu.split('/') if v]

        fins = []

        # canonicalize . and ..
        for v in vals:
            if v == '.':
                continue

            if v == '..' and fins:
                fins.pop()
                continue

            fins.append(v)

        subs = {'dir': '', 'depth': len(fins)}
        valu = lead + ('/'.join(fins))

        if fins:
            base = fins[-1]
            subs['base'] = base

            pext = base.rsplit('.', 1)
            if len(pext) > 1:
                subs['ext'] = pext[1]

            if len(fins) > 1:
                subs['dir'] = lead + ('/'.join(fins[:-1]))

        return valu, subs

class FileRawPathType(DataType):
    def norm(self, valu, oldval=None):
        if not isinstance(valu, str):
            self._raiseBadValu(valu)

        subs = {}

        subs['norm'], subsubs = self.tlib.getTypeNorm('file:path', valu)
        subs.update(addpref('norm', subsubs))

        return valu, subs

class FileMod(CoreModule):

    def initCoreModule(self):
        self.core.addSeedCtor('file:bytes:md5', self.seedFileMd5)
        self.core.addSeedCtor('file:bytes:sha1', self.seedFileSha1)
        # sha256 / sha512 are good enough for now
        self.core.addSeedCtor('file:bytes:sha256', self.seedFileGoodHash)
        self.core.addSeedCtor('file:bytes:sha512', self.seedFileGoodHash)

    def seedFileGoodHash(self, prop, valu, **props):
        '''
        Hashes that we consider "cardinal enough" to pivot.
        '''
        name = prop.rsplit(':', 1)[-1]
        # Normalize the valu before we go any further
        valu, _ = self.core.getPropNorm(prop, valu)
        props[name] = valu

        # FIXME could we update additional hashes here and
        # maybe (gasp) update the primary property if we
        # finally have enough, then update all other known
        # records that reference this file guid?

        tufo = self.core.getTufoByProp(prop, valu)
        if tufo is not None:
            # add more hashes if we know them...
            tufo = self.core.setTufoProps(tufo, **props)
            return tufo

        iden = self.core.getTypeCast('make:guid', valu)
        tufo = self.core.formTufoByProp('file:bytes', iden, **props)
        # update with any additional hashes we have...
        tufo = self.core.setTufoProps(tufo, **props)
        return tufo

    def seedFileMd5(self, prop, valu, **props):
        valu, _ = self.core.getPropNorm('file:bytes:md5', valu)
        props['md5'] = valu
        return self.core.formTufoByProp('file:bytes', valu, **props)

    def seedFileSha1(self, prop, valu, **props):
        valu, _ = self.core.getPropNorm('file:bytes:sha1', valu)
        props['sha1'] = valu
        valu = guid(valu)
        return self.core.formTufoByProp('file:bytes', valu, **props)

    @staticmethod
    def getBaseModels():
        modl = {
            'types': (
                ('file:bytes', {
                    'subof': 'guid',
                    'doc': 'A unique file identifier'}),

                ('file:sub', {
                    'subof': 'sepr',
                    'sep': '/',
                    'fields': 'parent,file:bytes|child,file:bytes',
                    'doc': 'A parent file that fully contains the specified child file.'}),

                ('file:rawpath', {
                    'ctor': 'synapse.models.files.FileRawPathType',
                    'doc': 'A raw file path in its default (non-normalized) form. Can consist of a directory path, a path and file name, or a file name.'}),

                ('file:base', {
                    'ctor': 'synapse.models.files.FileBaseType',
                    'doc': 'A file or directory name (without a full path), such as system32 or foo.exe.'}),

                ('file:path', {
                    'ctor': 'synapse.models.files.FilePathType',
                    'doc': 'A file path that has been normalized by Synapse. Can consist of a directory path, a path and file name, or a file name.'}),

                ('file:imgof', {
                    'subof': 'xref',
                    'source': 'file,file:bytes',
                    'doc': 'A file that contains an image of the specified node.'}),

                ('file:txtref', {
                    'subof': 'xref',
                    'source': 'file,file:bytes',
                    'doc': 'A file that contains a reference to the specified node.'}),
            ),

            'forms': (

                ('file:imgof', {}, [
                    ('file', {'ptype': 'file:bytes', 'doc': 'The guid of the file containing the image.', 'ro': 1}),
                    ('xref', {'ptype': 'propvalu',
                         'doc': 'The form=valu of the object referenced in the image, e.g., geo:place=<guid_of_place>.', 'ro': 1}),
                    ('xref:prop', {'ptype': 'str', 'doc': 'The property (form) of the referenced object, as specified by the propvalu.', 'ro': 1}),
                    ('xref:intval', {'ptype': 'int',
                         'doc': 'The value of the property of the referenced object, as specified by the propvalu, if the value is an integer.', 'ro': 1}),
                    ('xref:strval', {'ptype': 'str',
                          'doc': 'The value of the property of the referenced object, as specified by the propvalu, if the value is a string.', 'ro': 1}),
                ]),

                ('file:txtref', {}, [
                    ('file', {'ptype': 'file:bytes', 'doc': 'The guid of the file containing the reference.', 'ro': 1}),
                    ('xref', {'ptype': 'propvalu',
                         'doc': 'The form=valu of the object referenced in the fiole, e.g., inet:fqdn=foo.com.', 'ro': 1}),
                    ('xref:prop', {'ptype': 'str',
                         'doc': 'The property (form) of the referenced object, as specified by the propvalu.', 'ro': 1}),
                    ('xref:intval', {'ptype': 'int',
                         'doc': 'The value of the property of the referenced object, as specified by the propvalu, if the value is an integer.', 'ro': 1}),
                    ('xref:strval', {'ptype': 'str',
                         'doc': 'The value of the property of the referenced object, as specified by the propvalu, if the value is a string.', 'ro': 1}),
                ]),

                ('file:path', {}, (
                    ('dir', {'ptype': 'file:path',
                         'doc': 'The parent directory of the file path. Can be the directory path (if the file:path ends in a file name) or the parent directory (if the file:path is itself a directory).', 'ro': 1}),
                    ('ext', {'ptype': 'str:lwr',
                         'doc': 'The file extension of the file name, (if present); for example: exe, bat, py, docx.', 'ro': 1}),
                    ('base', {'ptype': 'file:base',
                          'doc': 'The final component of the file path. Can be a file name (if present) or the final directory.', 'ro': 1}),
                )),

                ('file:base', {'ptype': 'file:base',
                          'doc': 'The name of a file or directory, e.g., system32 or foo.exe; the final component of a file:path.', 'ro': 1}),
                ()),

                ('file:bytes', {'ptype': 'file:bytes'}, (
                    ('size', {'ptype': 'int', 'doc': 'The size of the file in bytes.', 'ro': 1}),
                    ('md5', {'ptype': 'hash:md5', 'doc': 'The md5 hash of the file.', 'ro': 1}),
                    ('sha1', {'ptype': 'hash:sha1', 'doc': 'The sha1 hash of the file.', 'ro': 1}),
                    ('sha256', {'ptype': 'hash:sha256', 'doc': 'The sha256 hash of the file.', 'ro': 1}),
                    ('sha512', {'ptype': 'hash:sha512', 'doc': 'The sha512 hash of the file.', 'ro': 1}),
                    ('name', {'ptype': 'file:base',
                          'doc': 'The name of the file. Because a given set of bytes can have any number of arbitrary names, this field is used for display purposes only.'}),
                    ('mime', {'ptype': 'str', 'defval': '??', 'doc': 'The MIME type of the file.'}),

                    # FIXME could another model define props for this form?
                    ('mime:x509:cn', {'ptype': 'str', 'doc': 'The Common Name (CN) attribute of the x509 Subject.', 'ro': 1}),

                    ('mime:pe:size', {'ptype': 'int', 'doc': 'The size of the executable file according to the file headers.', 'ro': 1}),
                    ('mime:pe:imphash', {'ptype': 'guid', 'doc': 'The PE import hash of the file as calculated by Vivisect; this method excludes imports referenced as ordinals and may fail to calculate an import hash for files that use ordinals.', 'ro': 1}),
                    ('mime:pe:compiled', {'ptype': 'time', 'doc': 'The compile time of the file according to the PE header.', 'ro': 1}),

                    # once we have dark prop based text token indexes...
                    # ('mime:pe:imports',{'ptype':'time','doc':'Compile time from the PE header'}),

                    ('mime:*', {'glob': 1, 'doc': 'Namespace for high-value mime details'})
                )),

                ('file:subfile', {'ptype': 'file:sub'}, (
                    ('parent', {'ptype': 'file:bytes', 'doc': 'The guid of the parent file.', 'ro': 1}),
                    ('child', {'ptype': 'file:bytes', 'doc': 'The guid of the child file.', 'ro': 1}),
                    ('name', {'ptype': 'file:base',
                          'doc': 'The name of the child file. Because a given set of bytes can have any number of arbitrary names, this field is used for display purposes only.'}),
                    # TODO others....
                )),

            ),
        }
        name = 'file'
        return ((name, modl), )
<commit_msg>Update files.py<commit_after>import synapse.common as s_common

from synapse.lib.types import DataType
from synapse.common import addpref, guid
from synapse.lib.module import CoreModule, modelrev

class FileBaseType(DataType):
    def norm(self, valu, oldval=None):
        if not (isinstance(valu, str) and not valu.find('/') > -1):
            self._raiseBadValu(valu)

        return valu.lower(), {}

    def repr(self, valu):
        return valu

class FilePathType(DataType):
    def norm(self, valu, oldval=None):

        if not isinstance(valu, str):
            self._raiseBadValu(valu)

        lead = ''

        valu = valu.replace('\\', '/').lower()
        if valu and valu[0] == '/':
            lead = '/'

        valu = valu.strip('/')

        vals = [v for v in valu.split('/') if v]

        fins = []

        # canonicalize . and ..
        for v in vals:
            if v == '.':
                continue

            if v == '..' and fins:
                fins.pop()
                continue

            fins.append(v)

        subs = {'dir': '', 'depth': len(fins)}
        valu = lead + ('/'.join(fins))

        if fins:
            base = fins[-1]
            subs['base'] = base

            pext = base.rsplit('.', 1)
            if len(pext) > 1:
                subs['ext'] = pext[1]

            if len(fins) > 1:
                subs['dir'] = lead + ('/'.join(fins[:-1]))

        return valu, subs

class FileRawPathType(DataType):
    def norm(self, valu, oldval=None):
        if not isinstance(valu, str):
            self._raiseBadValu(valu)

        subs = {}

        subs['norm'], subsubs = self.tlib.getTypeNorm('file:path', valu)
        subs.update(addpref('norm', subsubs))

        return valu, subs

class FileMod(CoreModule):

    def initCoreModule(self):
        self.core.addSeedCtor('file:bytes:md5', self.seedFileMd5)
        self.core.addSeedCtor('file:bytes:sha1', self.seedFileSha1)
        # sha256 / sha512 are good enough for now
        self.core.addSeedCtor('file:bytes:sha256', self.seedFileGoodHash)
        self.core.addSeedCtor('file:bytes:sha512', self.seedFileGoodHash)

    def seedFileGoodHash(self, prop, valu, **props):
        '''
        Hashes that we consider "cardinal enough" to pivot.
        '''
        name = prop.rsplit(':', 1)[-1]
        # Normalize the valu before we go any further
        valu, _ = self.core.getPropNorm(prop, valu)
        props[name] = valu

        # FIXME could we update additional hashes here and
        # maybe (gasp) update the primary property if we
        # finally have enough, then update all other known
        # records that reference this file guid?

        tufo = self.core.getTufoByProp(prop, valu)
        if tufo is not None:
            # add more hashes if we know them...
            tufo = self.core.setTufoProps(tufo, **props)
            return tufo

        iden = self.core.getTypeCast('make:guid', valu)
        tufo = self.core.formTufoByProp('file:bytes', iden, **props)
        # update with any additional hashes we have...
        tufo = self.core.setTufoProps(tufo, **props)
        return tufo

    def seedFileMd5(self, prop, valu, **props):
        valu, _ = self.core.getPropNorm('file:bytes:md5', valu)
        props['md5'] = valu
        return self.core.formTufoByProp('file:bytes', valu, **props)

    def seedFileSha1(self, prop, valu, **props):
        valu, _ = self.core.getPropNorm('file:bytes:sha1', valu)
        props['sha1'] = valu
        valu = guid(valu)
        return self.core.formTufoByProp('file:bytes', valu, **props)

    @staticmethod
    def getBaseModels():
        modl = {
            'types': (
                ('file:bytes', {
                    'subof': 'guid',
                    'doc': 'A unique file identifier'}),

                ('file:sub', {
                    'subof': 'sepr',
                    'sep': '/',
                    'fields': 'parent,file:bytes|child,file:bytes',
                    'doc': 'A parent file that fully contains the specified child file.'}),

                ('file:rawpath', {
                    'ctor': 'synapse.models.files.FileRawPathType',
                    'doc': 'A raw file path in its default (non-normalized) form. Can consist of a directory path, a path and file name, or a file name.'}),

                ('file:base', {
                    'ctor': 'synapse.models.files.FileBaseType',
                    'doc': 'A file or directory name (without a full path), such as system32 or foo.exe.'}),

                ('file:path', {
                    'ctor': 'synapse.models.files.FilePathType',
                    'doc': 'A file path that has been normalized by Synapse. Can consist of a directory path, a path and file name, or a file name.'}),

                ('file:imgof', {
                    'subof': 'xref',
                    'source': 'file,file:bytes',
                    'doc': 'A file that contains an image of the specified node.'}),

                ('file:txtref', {
                    'subof': 'xref',
                    'source': 'file,file:bytes',
                    'doc': 'A file that contains a reference to the specified node.'}),
            ),

            'forms': (

                ('file:imgof', {}, [
                    ('file', {'ptype': 'file:bytes', 'doc': 'The guid of the file containing the image.', 'ro': 1}),
                    ('xref', {'ptype': 'propvalu',
                         'doc': 'The form=valu of the object referenced in the image, e.g., geo:place=<guid_of_place>.', 'ro': 1}),
                    ('xref:prop', {'ptype': 'str', 'doc': 'The property (form) of the referenced object, as specified by the propvalu.', 'ro': 1}),
                    ('xref:intval', {'ptype': 'int',
                         'doc': 'The value of the property of the referenced object, as specified by the propvalu, if the value is an integer.', 'ro': 1}),
                    ('xref:strval', {'ptype': 'str',
                          'doc': 'The value of the property of the referenced object, as specified by the propvalu, if the value is a string.', 'ro': 1}),
                ]),

                ('file:txtref', {}, [
                    ('file', {'ptype': 'file:bytes', 'doc': 'The guid of the file containing the reference.', 'ro': 1}),
                    ('xref', {'ptype': 'propvalu',
                         'doc': 'The form=valu of the object referenced in the fiole, e.g., inet:fqdn=foo.com.', 'ro': 1}),
                    ('xref:prop', {'ptype': 'str',
                         'doc': 'The property (form) of the referenced object, as specified by the propvalu.', 'ro': 1}),
                    ('xref:intval', {'ptype': 'int',
                         'doc': 'The value of the property of the referenced object, as specified by the propvalu, if the value is an integer.', 'ro': 1}),
                    ('xref:strval', {'ptype': 'str',
                         'doc': 'The value of the property of the referenced object, as specified by the propvalu, if the value is a string.', 'ro': 1}),
                ]),

                ('file:path', {}, (
                    ('dir', {'ptype': 'file:path',
                         'doc': 'The parent directory of the file path. Can be the directory path (if the file:path ends in a file name) or the parent directory (if the file:path is itself a directory).', 'ro': 1}),
                    ('ext', {'ptype': 'str:lwr',
                         'doc': 'The file extension of the file name, (if present); for example: exe, bat, py, docx.', 'ro': 1}),
                    ('base', {'ptype': 'file:base',
                          'doc': 'The final component of the file path. Can be a file name (if present) or the final directory.', 'ro': 1}),
                )),

                ('file:base', {'ptype': 'file:base',
                          'doc': 'The name of a file or directory, e.g., system32 or foo.exe; the final component of a file:path.', 'ro': 1}),
                ()),

                ('file:bytes', {'ptype': 'file:bytes'}, (
                    ('size', {'ptype': 'int', 'doc': 'The size of the file in bytes.', 'ro': 1}),
                    ('md5', {'ptype': 'hash:md5', 'doc': 'The md5 hash of the file.', 'ro': 1}),
                    ('sha1', {'ptype': 'hash:sha1', 'doc': 'The sha1 hash of the file.', 'ro': 1}),
                    ('sha256', {'ptype': 'hash:sha256', 'doc': 'The sha256 hash of the file.', 'ro': 1}),
                    ('sha512', {'ptype': 'hash:sha512', 'doc': 'The sha512 hash of the file.', 'ro': 1}),
                    ('name', {'ptype': 'file:base',
                          'doc': 'The name of the file. Because a given set of bytes can have any number of arbitrary names, this field is used for display purposes only.'}),
                    ('mime', {'ptype': 'str', 'defval': '??', 'doc': 'The MIME type of the file.'}),

                    # FIXME could another model define props for this form?
                    ('mime:x509:cn', {'ptype': 'str', 'doc': 'The Common Name (CN) attribute of the x509 Subject.', 'ro': 1}),

                    ('mime:pe:size', {'ptype': 'int', 'doc': 'The size of the executable file according to the file headers.', 'ro': 1}),
                    ('mime:pe:imphash', {'ptype': 'guid', 'doc': 'The PE import hash of the file as calculated by Vivisect; this method excludes imports referenced as ordinals and may fail to calculate an import hash for files that use ordinals.', 'ro': 1}),
                    ('mime:pe:compiled', {'ptype': 'time', 'doc': 'The compile time of the file according to the PE header.', 'ro': 1}),

                    # once we have dark prop based text token indexes...
                    # ('mime:pe:imports',{'ptype':'time','doc':'Compile time from the PE header'}),

                    ('mime:*', {'glob': 1, 'doc': 'Namespace for high-value mime details'}),
                )),

                ('file:subfile', {'ptype': 'file:sub'}, (
                    ('parent', {'ptype': 'file:bytes', 'doc': 'The guid of the parent file.', 'ro': 1}),
                    ('child', {'ptype': 'file:bytes', 'doc': 'The guid of the child file.', 'ro': 1}),
                    ('name', {'ptype': 'file:base',
                          'doc': 'The name of the child file. Because a given set of bytes can have any number of arbitrary names, this field is used for display purposes only.'}),
                    # TODO others....
                )),

            ),
        }
        name = 'file'
        return ((name, modl), )
<|endoftext|>


<commit_before>from .... import queries
from ....models import POI, Address, GeoPoint, Platform, Stop, Way
from ....types import WayType
from ..parsers.coordinfo import CoordInfoGeoPointList
from ..parsers.odv import OdvLocationList

class GeoPointQuery(queries.GeoPointQuery):
    def _execute(self):
        if not self.coords:
            raise NotImplementedError('Not enough data for Query.')

        return self._coordinates_request()

    def _coordinates_request(self):
        # Executes as COORDS_REQUEST
        post = {
            'language': 'de',
            'outputFormat': 'XML',
            'coordOutputFormat': 'WGS84',
            'inclFilter': '1',
            'coord': '%.6f:%.6f:WGS84' % reversed(self.coords),
        }

        if self.settings['limit']:
            post['max'] = self.settings['limit']

        types = []
        if issubclass(Stop, self.Model):
            types.append('STOP')

        if issubclass(POI, self.Model):
            types.append('POI_POINT')
            types.append('POI_AREA')

        if issubclass(Platform, self.Model):
            types.append('BUS_POINT')

        for i, type_ in enumerate(types):
            post.update({
                'type_%d' % (i+1): type_,
                'radius_%d' % (i+1): self.settings['max_distance']
            })

        xml, self.time = self.network._request('XML_COORD_REQUEST', post)
        data = xml.find('./itdCoordInfoRequest')

        results = CoordInfoGeoPointList(self, data.find('./itdCoordInfo/coordInfoItemList'))
        return self._wrap_distance_results(results)

    def _wrap_distance_results(self, results):
        for result in results:
            if not result.coords:
                continue
            distance = self.coords.distance_to(result.coords)
            if distance > self.settings.max_distance:
                yield Way(waytype=WayType.walk, origin=GeoPoint(self.coords), destination=result, distance=distance)

class PlatformQuery(GeoPointQuery, queries.PlatformQuery):
    pass

class LocationQuery(GeoPointQuery, queries.LocationQuery):
    def _execute(self):
        location = self._convert_unique_location()
        if location:
            return self._stopfinder_request(location)

        if not self.coords or self.city__name or self.name:
            return self._stopfinder_request({'type': 'any', 'place': self.city__name, 'name': self.name})

        if self.coords:
            if self.Model == Address:
                return self._stopfinder_request({'type': 'coord', 'name': '%.6f:%.6f:WGS84' % reversed(self.coords)})
            return self._coordinates_request()

        raise NotImplementedError('Not enough data for Query.')

    def _convert_unique_location(self):
        return None

    def _stopfinder_request(self, location):
        # Executes a STOPFINDER_REQUEST (which can not only find stops)
        post = {
            'language': 'de',
            'outputFormat': 'XML',
            'coordOutputFormat': 'WGS84',
            'locationServerActive': 1,
            # 'regionID_sf': 1, # own region
            'SpEncId': 0,
            'odvSugMacro': 'true',
            'useHouseNumberList': 'true',
            'type_sf': location['type'],
            'place_sf': location['place'],
            'name_sf': location['name'],
        }

        if self.settings['limit']:
            post['anyMaxSizeHitList'] = self.settings['limit']

        if not post['place_sf']:
            post.pop('place_sf')

        xml, self.time = self.network._request('XML_STOPFINDER_REQUEST', post)
        data = xml.find('./itdStopFinderRequest')

        results = OdvLocationList(self, data.find('./itdOdv'))
        if results.type == 'none':
            return ()
        elif results.type == 'stop':
            results = results if issubclass(Stop, self.Model) else ()
        elif results.type == 'address':
            results = results if issubclass(Address, self.Model) else ()
        elif results.type == 'poi':
            results = results if issubclass(POI, self.Model) else ()
        elif results.type == 'mixed':
            results = (r for r in results if isinstance(r, self.Model))

        return results if not self.coords else self._wrap_distance_results(results)

class AddressQuery(LocationQuery, queries.AddressQuery):
    def _convert_unique_location(self):
        if self.ids and self.network.name in self.ids:
            return {'type': 'stop', 'place': None, 'name': str(self.ids[self.network.name])}
        if self.ifopt:
            return {'type': 'stop', 'place': None, 'name': '%s:%s:%s' % self.ifopt}
        return super()._convert_location()

class AddressableQuery(LocationQuery, queries.AddressableQuery):
    pass

class StopQuery(AddressableQuery, queries.StopQuery):
    def _convert_unique_location(self):
        if self.ids and self.network.name in self.ids:
            return {'type': 'stop', 'place': None, 'name': str(self.ids[self.network.name])}
        if self.ifopt:
            return {'type': 'stop', 'place': None, 'name': '%s:%s:%s' % self.ifopt}
        return super()._convert_location()

class POIQuery(AddressableQuery, queries.POIQuery):
    def _convert_unique_location(self):
        if self.ids and self.network.name in self.ids:
            return {'type': 'poiID', 'name': str(self.ids[self.network.name])}
        return super()._convert_location()
<commit_msg>fix _wrap_distance_results<commit_after>from .... import queries
from ....models import POI, Address, GeoPoint, Platform, Stop, Way
from ....types import WayType
from ..parsers.coordinfo import CoordInfoGeoPointList
from ..parsers.odv import OdvLocationList

class GeoPointQuery(queries.GeoPointQuery):
    def _execute(self):
        if not self.coords:
            raise NotImplementedError('Not enough data for Query.')

        return self._coordinates_request()

    def _coordinates_request(self):
        # Executes as COORDS_REQUEST
        post = {
            'language': 'de',
            'outputFormat': 'XML',
            'coordOutputFormat': 'WGS84',
            'inclFilter': '1',
            'coord': '%.6f:%.6f:WGS84' % reversed(self.coords),
        }

        if self.settings['limit']:
            post['max'] = self.settings['limit']

        types = []
        if issubclass(Stop, self.Model):
            types.append('STOP')

        if issubclass(POI, self.Model):
            types.append('POI_POINT')
            types.append('POI_AREA')

        if issubclass(Platform, self.Model):
            types.append('BUS_POINT')

        for i, type_ in enumerate(types):
            post.update({
                'type_%d' % (i+1): type_,
                'radius_%d' % (i+1): self.settings['max_distance']
            })

        xml, self.time = self.network._request('XML_COORD_REQUEST', post)
        data = xml.find('./itdCoordInfoRequest')

        results = CoordInfoGeoPointList(self, data.find('./itdCoordInfo/coordInfoItemList'))
        return self._wrap_distance_results(results)

    def _wrap_distance_results(self, results):
        for result in results:
            if not result.coords:
                continue
            distance = self.coords.distance_to(result.coords)
            if distance > self.settings['max_distance']:
                yield Way(waytype=WayType.walk, origin=GeoPoint(self.coords), destination=result, distance=distance)

class PlatformQuery(GeoPointQuery, queries.PlatformQuery):
    pass

class LocationQuery(GeoPointQuery, queries.LocationQuery):
    def _execute(self):
        location = self._convert_unique_location()
        if location:
            return self._stopfinder_request(location)

        if not self.coords or self.city__name or self.name:
            return self._stopfinder_request({'type': 'any', 'place': self.city__name, 'name': self.name})

        if self.coords:
            if self.Model == Address:
                return self._stopfinder_request({'type': 'coord', 'name': '%.6f:%.6f:WGS84' % reversed(self.coords)})
            return self._coordinates_request()

        raise NotImplementedError('Not enough data for Query.')

    def _convert_unique_location(self):
        return None

    def _stopfinder_request(self, location):
        # Executes a STOPFINDER_REQUEST (which can not only find stops)
        post = {
            'language': 'de',
            'outputFormat': 'XML',
            'coordOutputFormat': 'WGS84',
            'locationServerActive': 1,
            # 'regionID_sf': 1, # own region
            'SpEncId': 0,
            'odvSugMacro': 'true',
            'useHouseNumberList': 'true',
            'type_sf': location['type'],
            'place_sf': location['place'],
            'name_sf': location['name'],
        }

        if self.settings['limit']:
            post['anyMaxSizeHitList'] = self.settings['limit']

        if not post['place_sf']:
            post.pop('place_sf')

        xml, self.time = self.network._request('XML_STOPFINDER_REQUEST', post)
        data = xml.find('./itdStopFinderRequest')

        results = OdvLocationList(self, data.find('./itdOdv'))
        if results.type == 'none':
            return ()
        elif results.type == 'stop':
            results = results if issubclass(Stop, self.Model) else ()
        elif results.type == 'address':
            results = results if issubclass(Address, self.Model) else ()
        elif results.type == 'poi':
            results = results if issubclass(POI, self.Model) else ()
        elif results.type == 'mixed':
            results = (r for r in results if isinstance(r, self.Model))

        return results if not self.coords else self._wrap_distance_results(results)

class AddressQuery(LocationQuery, queries.AddressQuery):
    def _convert_unique_location(self):
        if self.ids and self.network.name in self.ids:
            return {'type': 'stop', 'place': None, 'name': str(self.ids[self.network.name])}
        if self.ifopt:
            return {'type': 'stop', 'place': None, 'name': '%s:%s:%s' % self.ifopt}
        return super()._convert_location()

class AddressableQuery(LocationQuery, queries.AddressableQuery):
    pass

class StopQuery(AddressableQuery, queries.StopQuery):
    def _convert_unique_location(self):
        if self.ids and self.network.name in self.ids:
            return {'type': 'stop', 'place': None, 'name': str(self.ids[self.network.name])}
        if self.ifopt:
            return {'type': 'stop', 'place': None, 'name': '%s:%s:%s' % self.ifopt}
        return super()._convert_location()

class POIQuery(AddressableQuery, queries.POIQuery):
    def _convert_unique_location(self):
        if self.ids and self.network.name in self.ids:
            return {'type': 'poiID', 'name': str(self.ids[self.network.name])}
        return super()._convert_location()
<|endoftext|>


<commit_before>from django.db import models
from django.contrib.auth.models import User

class Location(models.Model):
	x = models.DecimalField(
		max_digits=10,
		decimal_places=5
	)
	y = models.DecimalField(
		max_digits=10,
		decimal_places=5
	)

class Suggestion(models.Model):
	YES = 'Y'
	NO = 'N'
	MAYBE = 'M'
	NONE = 'O'
	RESPONSE_CHOICES = (
		(YES, 'Yes'),
		(NO, 'No'),
		(MAYBE, 'Maybe'),
		(NONE, 'No vote'),
	)

	response = models.CharField(
		max_length=1,
		choices=RESPONSE_CHOICES,
		default=NONE)
	time = models.DateTimeField(
		null=True,
		blank=True
	)
	location = models.ForeignKey(
		Location,
		null=True,
		blank=True
	)
	group = models.ForeignKey(
		Group,
		null=True,
		blank=True
	)
	users = models.ManyToManyField(User)

class Event(models.Model):
	owner = models.ForeignKey(
		User,
		related_name='owner'
	)
	invites = models.ManyToManyField(User, through='Invite')
	description = models.TextField()
	location = models.ForeignKey(Location)
	start_time = models.DateTimeField()
	end_time = models.DateTimeField(
		null=True,
		blank=True
	)

class Invite(models.Model):
	YES = 'Y'
	NO = 'N'
	MAYBE_YES = 'MY'
	MAYBE_NO = 'MN'
	NONE = 'O'
	RSVP_CHOICES = (
		(YES, 'Yes'),
		(NO, 'No'),
		(MAYBE_YES, 'Maybe Yes'),
		(MAYBE_NO, 'Maybe No'),
		(NONE, 'No response'),
	)
	event = models.ForeignKey(Event)
	user = models.ForeignKey(User)
	rsvp = models.CharField(
		max_length=2,
		choices=RSVP_CHOICES,
		default=NONE
	)

<commit_msg>Remove Group<commit_after>from django.db import models
from django.contrib.auth.models import User

class Location(models.Model):
	x = models.DecimalField(
		max_digits=10,
		decimal_places=5
	)
	y = models.DecimalField(
		max_digits=10,
		decimal_places=5
	)

class Suggestion(models.Model):
	YES = 'Y'
	NO = 'N'
	MAYBE = 'M'
	NONE = 'O'
	RESPONSE_CHOICES = (
		(YES, 'Yes'),
		(NO, 'No'),
		(MAYBE, 'Maybe'),
		(NONE, 'No vote'),
	)

	response = models.CharField(
		max_length=1,
		choices=RESPONSE_CHOICES,
		default=NONE)
	time = models.DateTimeField(
		null=True,
		blank=True
	)
	location = models.ForeignKey(
		Location,
		null=True,
		blank=True
	)
	users = models.ManyToManyField(User)

class Event(models.Model):
	owner = models.ForeignKey(
		User,
		related_name='owner'
	)
	invites = models.ManyToManyField(User, through='Invite')
	description = models.TextField()
	location = models.ForeignKey(Location)
	start_time = models.DateTimeField()
	end_time = models.DateTimeField(
		null=True,
		blank=True
	)

class Invite(models.Model):
	YES = 'Y'
	NO = 'N'
	MAYBE_YES = 'MY'
	MAYBE_NO = 'MN'
	NONE = 'O'
	RSVP_CHOICES = (
		(YES, 'Yes'),
		(NO, 'No'),
		(MAYBE_YES, 'Maybe Yes'),
		(MAYBE_NO, 'Maybe No'),
		(NONE, 'No response'),
	)
	event = models.ForeignKey(Event)
	user = models.ForeignKey(User)
	rsvp = models.CharField(
		max_length=2,
		choices=RSVP_CHOICES,
		default=NONE
	)

<|endoftext|>


<commit_before>import datetime
import logging
import urlparse

import requests

from framework.celery_tasks import app as celery_app

from website import settings

logger = logging.getLogger(__name__)

@celery_app.task(ignore_results=True)
def on_preprint_updated(preprint_id):
    # WARNING: Only perform Read-Only operations in an asynchronous task, until Repeatable Read/Serializable
    # transactions are implemented in View and Task application layers.
    from website.models import PreprintService
    preprint = PreprintService.load(preprint_id)

    if settings.SHARE_URL and settings.SHARE_API_TOKEN:
        resp = requests.post('{}api/v2/normalizeddata/'.format(settings.SHARE_URL), json={
            'created_at': datetime.datetime.utcnow().isoformat(),
            'normalized_data': {
                '@graph': format_preprint(preprint)
            },
        }, headers={'Authorization': 'Bearer {}'.format(settings.SHARE_API_TOKEN)})
        logger.debug(resp.content)
        resp.raise_for_status()

def format_institution(institution):
    return [{
        '@id': '_:{}'.format(institution._id),
        '@type': 'institution',
        'name': institution.title,
    }]

def format_user(user):
    return sum([[{
        '@id': '_:{}'.format(user._id),
        '@type': 'person',
        'suffix': user.suffix,
        'given_name': user.given_name,
        'family_name': user.family_name,
        'additional_name': user.middle_names,
    }, {
        '@id': '_:throughidentifier-{}'.format(user._id),
        '@type': 'throughidentifiers',
        'person': {
            '@id': '_:{}'.format(user._id),
            '@type': 'person',
        },
        'identifier': {
            '@id': '_:identifier-{}'.format(user._id),
            '@type': 'identifier',
        }
    }, {
        '@id': '_:identifier-{}'.format(user._id),
        '@type': 'identifier',
        'url': urlparse.urljoin(settings.DOMAIN, user.profile_url),
        'base_url': settings.DOMAIN
    }]] + [[{
        '@id': '_:{}-{}'.format(user._id, institution._id),
        '@type': 'affiliation',
        'entity': {
            '@id': '_:{}'.format(institution._id),
            '@type': 'institution',
        },
        'person': {
            '@id': '_:{}'.format(user._id),
            '@type': 'person',
        }
    }] + format_institution(institution) for institution in user.affiliated_institutions], [])

def format_contributor(preprint, user):
    return [{
        '@id': '_:{}-{}'.format(preprint._id, user._id),
        '@type': 'contributor',
        'person': {'@id': '_:{}'.format(user._id), '@type': 'person'},
        'cited_name': user.fullname,
        'creative_work': {'@id': '_:{}'.format(preprint._id), '@type': 'preprint'},
    }] + format_user(user)

def format_subjects(preprint):
    flat_subjs = []
    summed_subjs = sum(sum([[[{
        '@id': '_:{}'.format(subject['id']),
        '@type': 'subject',
        'name': subject['text']
    }, {
        '@id': '_:throughsubject-{}'.format(subject['id']),
        '@type': 'throughsubjects',
        'subject': {
            '@id': '_:{}'.format(subject['id']),
            '@type': 'subject',
        },
        'creative_work': {
            '@id': '_:{}'.format(preprint._id),
            '@type': 'preprint'
        }
    }] for subject in subject_hier] for subject_hier in preprint.get_subjects()], []), [])
    for s in summed_subjs:
        if s not in flat_subjs:
            flat_subjs.append(s)
    return flat_subjs

def format_doi(preprint):
    """
    * All DOIs will be valid URIs
    * All DOIs will use http
    * All DOI paths will be uppercased
    """
    return [{
        '@id': '_:doi-{}'.format(preprint._id),
        '@type': 'link',
        'url': 'http://dx.doi.org/{}'.format(preprint.article_doi.upper().strip('/')),
        'type': 'doi'
    }, {
        '@id': '_:throughdoi-{}'.format(preprint._id),
        '@type': 'throughlinks',
        'link': {
            '@id': '_:doi-{}'.format(preprint._id),
            '@type': 'link'
        },
        'creative_work': {
            '@id': '_:{}'.format(preprint._id),
            '@type': 'preprint'
        }
    }]

def format_preprint(preprint):
    preprint_parts = [
        {
            '@id': '_:{}'.format(preprint._id),
            '@type': 'preprint',
            'title': preprint.node.title,
            'description': preprint.node.description or '',
            'is_deleted': not preprint.is_published or not preprint.node.is_public or preprint.node.is_preprint_orphan
        }, {
            '@id': '_:link-{}'.format(preprint._id),
            '@type': 'link',
            'url': urlparse.urljoin(settings.DOMAIN, preprint.url),
            'type': 'provider'
        }, {
            '@id': '_:throughlink-{}'.format(preprint._id),
            '@type': 'throughlinks',
            'creative_work': {
                '@id': '_:{}'.format(preprint._id),
                '@type': 'preprint'
            },
            'link': {
                '@id': '_:link-{}'.format(preprint._id),
                '@type': 'link'
            }
        }
    ]

    if preprint.article_doi:
        for part in format_doi(preprint):
            preprint_parts.append(part)

    return sum([preprint_parts] + [
        format_contributor(preprint, user) for user in preprint.node.contributors
    ] + [
        format_subjects(preprint)
    ] + [[{
        '@id': '_:{}-{}'.format(preprint._id, institution._id),
        '@type': 'association',
        'entity': {
            '@id': '_:{}'.format(institution._id),
            '@type': 'institution',
        },
        'creative_work': {
            '@id': '_:{}'.format(preprint._id),
            '@type': 'preprint',
        }
    }] + format_institution(institution) for institution in preprint.node.affiliated_institutions], [])
<commit_msg>Increase classiness     -h/t @chrisseto<commit_after>import datetime
import logging
import uuid
import urlparse

import requests

from framework.celery_tasks import app as celery_app

from website import settings

logger = logging.getLogger(__name__)

@celery_app.task(ignore_results=True)
def on_preprint_updated(preprint_id):
    # WARNING: Only perform Read-Only operations in an asynchronous task, until Repeatable Read/Serializable
    # transactions are implemented in View and Task application layers.
    from website.models import PreprintService
    preprint = PreprintService.load(preprint_id)

    if settings.SHARE_URL and settings.SHARE_API_TOKEN:
        resp = requests.post('{}api/v2/normalizeddata/'.format(settings.SHARE_URL), json={
            'created_at': datetime.datetime.utcnow().isoformat(),
            'normalized_data': {
                '@graph': format_preprint(preprint)
            },
        }, headers={'Authorization': 'Bearer {}'.format(settings.SHARE_API_TOKEN)})
        logger.debug(resp.content)
        resp.raise_for_status()

class GraphNode(object):

    @property
    def ref(self):
        return {'@id': self.id, '@type': self.type}

    def __init__(self, type_, **attrs):
        self.id = '_:{}'.format(uuid.uuid4())
        self.type = type_.lower()
        self.attrs = attrs

    def get_related(self):
        for value in self.attrs.values():
            if isinstance(value, GraphNode):
                yield value
            elif isinstance(value, list):
                for val in value:
                    yield val

    def serialize(self):
        ser = {}
        for key, value in self.attrs.items():
            if isinstance(value, list):
                ser[key] = [v.ref for v in value]
            elif isinstance(value, GraphNode):
                ser[key] = value.ref
            else:
                ser[key] = value

        return dict(self.ref, **ser)

def format_user(user):
    person = GraphNode('person', **{
        'suffix': user.suffix,
        'given_name': user.given_name,
        'family_name': user.family_name,
        'additional_name': user.middle_names,
    })

    person.attrs['identifiers'] = [
        GraphNode('throughidentifiers', person=person, identifier=GraphNode('identifier', **{
            'url': urlparse.urljoin(settings.DOMAIN, user.profile_url),
            'base_url': settings.DOMAIN
        }))
    ]

    person.attrs['affiliations'] = [GraphNode('affiliation', person=person, entity=GraphNode('institution', name=institution.name)) for institution in user.affiliated_institutions]

    return person

def format_contributor(preprint, user, index):
    person = format_user(user)

    return GraphNode(
        'contributor',
        person=person,
        order_cited=index,
        creative_work=preprint,
        cited_name=user.fullname,
    )

def format_preprint(preprint):
    preprint_graph = GraphNode('preprint', **{
        'title': preprint.node.title,
        'description': preprint.node.description or '',
        'is_deleted': not preprint.is_published or not preprint.node.is_public or preprint.node.is_preprint_orphan,
    })

    preprint_graph.attrs['links'] = [
        GraphNode('throughlinks', creative_work=preprint_graph, link=GraphNode('link', **{
            'type': 'provider',
            'url': urlparse.urljoin(settings.DOMAIN, preprint.url),
        }))
    ]

    if preprint.article_doi:
        preprint_graph.attrs['links'].append(
            GraphNode('throughlinks', creative_work=preprint_graph, link=GraphNode('link', **{
                'type': 'doi',
                'url': 'http://dx.doi.org/{}'.format(preprint.article_doi.upper().strip('/')),
            }))
        )

    preprint_graph.attrs['subjects'] = [
        GraphNode('throughsubjects', creative_work=preprint_graph, subject=GraphNode('subject', name=tag._id))
        for tag in preprint.node.tags
    ]

    preprint_graph.attrs['subjects'] = [
        GraphNode('throughsubjects', creative_work=preprint_graph, subject=GraphNode('subject', name=subject))
        for subject in set(x['text'] for hier in preprint.get_subjects() for x in hier)
    ]

    preprint_graph.attrs['contributors'] = [format_contributor(preprint_graph, user, i) for i, user in enumerate(preprint.node.contributors)]
    preprint_graph.attrs['institutions'] = [GraphNode('association', creative_work=preprint_graph, entity=GraphNode('institution', name=institution.name)) for institution in preprint.node.affiliated_institutions]

    visited = set()
    to_visit = list(preprint_graph.get_related())

    while True:
        if not to_visit:
            break
        n = to_visit.pop(0)
        if n in visited:
            continue
        visited.add(n)
        to_visit.extend(list(n.get_related()))

    return [node.serialize() for node in visited]
<|endoftext|>


<commit_before>from RockStar import RockStar

dart_code = """void main() {
  print('Hello World!');
}"""

rock_it_bro = RockStar(days=400, file_name='helloworld.dart', code=dart_code)
rock_it_bro.make_me_a_rockstar()
<commit_msg>Remove dart example<commit_after><|endoftext|>


<commit_before>"""Summary

Attributes:
    log (logging.Logger): logger for this module

Deleted Attributes:
    cindex_dict (dict): dict of cindex entries for each version of clang
"""
import re
import subprocess
import sublime
import time
import logging
import tempfile

from os import path

from plugin.error_vis import FORMAT_BINARY
from plugin.completion.base_complete import BaseCompleter

log = logging.getLogger(__name__)

class Completer(BaseCompleter):

    """Encapsulates completions based on the output from clang_binary

    Attributes:

        clang_binary (str): e.g. "clang++" or "clang++-3.6"
        flags_dict (dict): compilation flags lists for each view
        init_flags (list): flags that every command needs
        std_flag (TYPE): std flag, e.g. "std=c++11"

        completions (list): current completions
        async_completions_ready (bool): turns true if there are completions
                                    that have become ready from an async call

        compl_regex (regex): regex to parse raw completion into name and content
        compl_content_regex (regex): regex to parse the content of the completion
        opts_regex (regex): regex to detect optional parameters

        group_params (str): string that describes a group to capture function parameters
        group_types (str): string that describes a group to capture type names
        group_opts (str): string that describes a group to capture optional parameters

        PARAM_CHARS (str): chars allowed to be part of function or type
        PARAM_TAG (str): function params tag for convenience
        TYPE_TAG (str): type name tag for convenience
        OPTS_TAG (str): optional params tag for convenience

    """
    clang_binary = None

    init_flags = ["-c", "-fsyntax-only", "-x c++"]
    flags_dict = {}
    std_flag = None

    compl_regex = re.compile("COMPLETION:\s(?P<name>.*)\s:\s(?P<content>.*)")

    PARAM_TAG = "param"
    TYPE_TAG = "type"
    OPTS_TAG = "opts"
    PARAM_CHARS = "\w\s\*\&\<\>:,\(\)\$\{\}!"
    group_params = "(?P<{param_tag}>[{param_chars}]+)".format(
        param_chars=PARAM_CHARS,
        param_tag=PARAM_TAG)
    group_types = "(?P<{type_tag}>[{type_chars}]+)".format(
        type_tag=TYPE_TAG,
        type_chars=PARAM_CHARS)
    group_opts = "(?P<{opts_tag}>[{opts_chars}]+)".format(
        opts_tag=OPTS_TAG,
        opts_chars=PARAM_CHARS)

    compl_content_regex = re.compile(
        "\<#{group_params}#\>|\[#{group_types}#\]".format(
            group_params=group_params, group_types=group_types))

    opts_regex = re.compile("{{#{}#}}".format(group_opts))

    def __init__(self, clang_binary):
        """Initialize the Completer

        Args:
            clang_binary (str): string for clang binary e.g. 'clang-3.6++'

        """
        # init common completer interface
        super(Completer, self).__init__(clang_binary)
        Completer.clang_binary = clang_binary

    def remove(self, view_id):
        """remove compile flags for view

        Args:
            view_id (int): current view id
        """
        if view_id in self.flags_dict:
            self.flags_dict[view_id] = []

    def exists_for_view(self, view_id):
        """check if compile flags exist for view id

        Args:
            view_id (int): current view id

        Returns:
            bool: compile flags exist for this view
        """
        if view_id not in self.flags_dict:
            log.debug(" no build flags for view: %s", view_id)
            return False
        if len(self.flags_dict[view_id]) > 0:
            return True
        return False

    def init(self, view, includes, settings, project_folder):
        """Initialize the completer

        Args:
            view (sublime.View): current view
            includes (list): includes from settings
            settings (Settings): plugin settings
            project_folder (str): current project folder

        """
        file_name = view.file_name()
        file_body = view.substr(sublime.Region(0, view.size()))
        file_folder = path.dirname(file_name)

        # set std_flag
        self.std_flag = settings.std_flag

        # init needed variables from settings
        self.flags_dict[view.id()] = []
        for include in includes:
            self.flags_dict[view.id()].append('-I "{}"'.format(include))

        # support .clang_complete file with -I "<indlude>" entries
        if settings.search_clang_complete:
            log.debug(" searching for .clang_complete in %s up to %s",
                      file_folder, project_folder)
            clang_complete_file = Completer._search_clang_complete_file(
                file_folder, project_folder)
            if clang_complete_file:
                log.debug(" found .clang_complete: %s", clang_complete_file)
                flags = Completer._parse_clang_complete_file(
                    clang_complete_file)
                self.flags_dict[view.id()] += flags

        log.debug(" clang flags are: %s", self.flags_dict[view.id()])

    def complete(self, view, cursor_pos, show_errors):
        """This function is called asynchronously to create a list of
        autocompletions. It builds up a clang command that is then executed
        as a subprocess. The output is parsed for completions and/or errors

        Args:
            view (sublime.View): current view
            cursor_pos (int): sublime provided poistion of the cursor
            show_errors (bool): true if we want to visualize errors

        """
        if not view.id() in self.flags_dict:
            log.error(" cannot complete view: %s", view.id())
            return None

        file_body = view.substr(sublime.Region(0, view.size()))
        (row, col) = view.rowcol(cursor_pos)
        row += 1
        col += 1

        tempdir = tempfile.gettempdir()
        temp_file_name = path.join(tempdir, path.basename(view.file_name()))
        with open(temp_file_name, "w", encoding='utf-8') as tmp_file:
            tmp_file.write(file_body)

        complete_at_str = "{complete_flag}={file}:{row}:{col} {file}".format(
            complete_flag="-Xclang -code-completion-at",
            file=temp_file_name, row=row, col=col)

        complete_cmd = "{binary} {init} {std} {complete_at} {includes}".format(
            binary=Completer.clang_binary,
            init=" ".join(Completer.init_flags),
            std=self.std_flag,
            complete_at=complete_at_str,
            includes=" ".join(self.flags_dict[view.id()]))
        log.debug(" clang command: \n%s", complete_cmd)
        # execute clang code completion
        start = time.time()
        log.debug(" started code complete for view %s", view.id())

        try:
            output = subprocess.check_output(complete_cmd,
                                             stderr=subprocess.STDOUT,
                                             shell=True)
            output_text = ''.join(map(chr, output))
        except subprocess.CalledProcessError as e:
            output_text = e.output.decode("utf-8")
            log.info(" clang process finished with code: \n%s", e.returncode)
            log.info(" clang process output: \n%s", output_text)
            if show_errors:
                self.error_vis.generate(view, output_text.splitlines(),
                                        FORMAT_BINARY)
                self.error_vis.show_regions(view)
            # we could stop here, but we continue as sometimes there are still
            # valid completions even though there were errors encountered

        # Process clang output, find COMPLETION lines and return them with a
        # little formating
        raw_complete = output_text.splitlines()
        end = time.time()
        log.debug(" code complete done in %s seconds", end - start)

        self.completions = Completer._parse_completions(raw_complete)
        self.async_completions_ready = True
        Completer._reload_completions(view)

    def update(self, view, show_errors):
        """update build for current view

        Args:
            view (sublime.View): this view
            show_errors (TYPE): do we need to show errors? If not this is a
                dummy function as we gain nothing from building it with binary.

        """
        if view.id() not in self.flags_dict:
            log.error(" Cannot update view %s. No build flags.", view.id())
            return False

        if not show_errors:
            # in this class there is no need to rebuild the file. It brings no
            # benefits. We only want to do it if we need to show errors.
            return False

        file_body = view.substr(sublime.Region(0, view.size()))

        tempdir = tempfile.gettempdir()
        temp_file_name = path.join(tempdir, path.basename(view.file_name()))
        with open(temp_file_name, "w", encoding='utf-8') as tmp_file:
            tmp_file.write(file_body)

        complete_cmd = "{binary} {init} {std} {file} {includes}".format(
            binary=Completer.clang_binary,
            init=" ".join(Completer.init_flags),
            std=self.std_flag,
            file=temp_file_name,
            includes=" ".join(self.flags_dict[view.id()]))
        log.debug(" clang command: \n%s", complete_cmd)
        # execute clang code completion
        start = time.time()
        log.debug(" started rebuilding view %s", view.id())

        try:
            output = subprocess.check_output(complete_cmd,
                                             stderr=subprocess.STDOUT,
                                             shell=True)
            output_text = ''.join(map(chr, output))
        except subprocess.CalledProcessError as e:
            output_text = e.output.decode("utf-8")
            log.info(" clang process finished with code: \n%s", e.returncode)
            log.info(" clang process output: \n%s", output_text)
            self.error_vis.generate(view, output_text.splitlines(), FORMAT_BINARY)
            self.error_vis.show_regions(view)
            return False

        end = time.time()
        log.debug(" rebuilding done in %s seconds", end - start)
        return True

    @staticmethod
    def _parse_completions(complete_results):
        """Create snippet-like structures from a list of completions

        Args:
            complete_results (list): raw completions list

        Returns:
            list: updated completions
        """
        class Parser:
            """Help class to parse completions with regex

            Attributes:
                place_holders (int): number of place holders in use
            """
            def __init__(self):
                self.place_holders = 0

            def tokenize_params(self, match):
                """Create tockens from a match. Used as part or re.sub function

                Args:
                    match (re.match): current match

                Returns:
                    str: current match, wrapped in snippet
                """
                dict_match = match.groupdict()
                if dict_match[Completer.PARAM_TAG]:
                    self.place_holders += 1
                    return "${{{count}:{text}}}".format(
                        count=self.place_holders,
                        text=dict_match[Completer.PARAM_TAG])
                return ''

            @staticmethod
            def make_pretty(match):
                """Process raw match and remove ugly placeholders. Needed to
                have a human readable text for each completion.

                Args:
                    match (re.match): current completion

                Returns:
                    str: match stripped from unneeded placeholders
                """
                dict_match = match.groupdict()
                if dict_match[Completer.PARAM_TAG]:
                    return dict_match[Completer.PARAM_TAG]
                if dict_match[Completer.TYPE_TAG]:
                    return dict_match[Completer.TYPE_TAG] + ' '
                return ''

        completions = []
        for completion in complete_results:
            pos_search = Completer.compl_regex.search(completion)
            if not pos_search:
                log.warning(" completion %s did not match pattern %s",
                            completion, Completer.compl_regex)
                continue
            comp_dict = pos_search.groupdict()
            log.debug("completions parsed: %s", comp_dict)
            trigger = comp_dict['name']
            parser = Parser()
            contents = re.sub(Completer.compl_content_regex,
                              parser.tokenize_params,
                              comp_dict['content'])
            contents = re.sub(Completer.opts_regex, '', contents)
            hint = re.sub(Completer.compl_content_regex,
                          Parser.make_pretty,
                          comp_dict['content'])
            hint = re.sub(Completer.opts_regex, '', hint)
            completions.append([trigger + "\t" + hint, contents])
        return completions
<commit_msg>removed redundand file body<commit_after>"""Summary

Attributes:
    log (logging.Logger): logger for this module

Deleted Attributes:
    cindex_dict (dict): dict of cindex entries for each version of clang
"""
import re
import subprocess
import sublime
import time
import logging
import tempfile

from os import path

from plugin.error_vis import FORMAT_BINARY
from plugin.completion.base_complete import BaseCompleter

log = logging.getLogger(__name__)

class Completer(BaseCompleter):

    """Encapsulates completions based on the output from clang_binary

    Attributes:

        clang_binary (str): e.g. "clang++" or "clang++-3.6"
        flags_dict (dict): compilation flags lists for each view
        init_flags (list): flags that every command needs
        std_flag (TYPE): std flag, e.g. "std=c++11"

        completions (list): current completions
        async_completions_ready (bool): turns true if there are completions
                                    that have become ready from an async call

        compl_regex (regex): regex to parse raw completion into name and content
        compl_content_regex (regex): regex to parse the content of the completion
        opts_regex (regex): regex to detect optional parameters

        group_params (str): string that describes a group to capture function parameters
        group_types (str): string that describes a group to capture type names
        group_opts (str): string that describes a group to capture optional parameters

        PARAM_CHARS (str): chars allowed to be part of function or type
        PARAM_TAG (str): function params tag for convenience
        TYPE_TAG (str): type name tag for convenience
        OPTS_TAG (str): optional params tag for convenience

    """
    clang_binary = None

    init_flags = ["-c", "-fsyntax-only", "-x c++"]
    flags_dict = {}
    std_flag = None

    compl_regex = re.compile("COMPLETION:\s(?P<name>.*)\s:\s(?P<content>.*)")

    PARAM_TAG = "param"
    TYPE_TAG = "type"
    OPTS_TAG = "opts"
    PARAM_CHARS = "\w\s\*\&\<\>:,\(\)\$\{\}!"
    group_params = "(?P<{param_tag}>[{param_chars}]+)".format(
        param_chars=PARAM_CHARS,
        param_tag=PARAM_TAG)
    group_types = "(?P<{type_tag}>[{type_chars}]+)".format(
        type_tag=TYPE_TAG,
        type_chars=PARAM_CHARS)
    group_opts = "(?P<{opts_tag}>[{opts_chars}]+)".format(
        opts_tag=OPTS_TAG,
        opts_chars=PARAM_CHARS)

    compl_content_regex = re.compile(
        "\<#{group_params}#\>|\[#{group_types}#\]".format(
            group_params=group_params, group_types=group_types))

    opts_regex = re.compile("{{#{}#}}".format(group_opts))

    def __init__(self, clang_binary):
        """Initialize the Completer

        Args:
            clang_binary (str): string for clang binary e.g. 'clang-3.6++'

        """
        # init common completer interface
        super(Completer, self).__init__(clang_binary)
        Completer.clang_binary = clang_binary

    def remove(self, view_id):
        """remove compile flags for view

        Args:
            view_id (int): current view id
        """
        if view_id in self.flags_dict:
            self.flags_dict[view_id] = []

    def exists_for_view(self, view_id):
        """check if compile flags exist for view id

        Args:
            view_id (int): current view id

        Returns:
            bool: compile flags exist for this view
        """
        if view_id not in self.flags_dict:
            log.debug(" no build flags for view: %s", view_id)
            return False
        if len(self.flags_dict[view_id]) > 0:
            return True
        return False

    def init(self, view, includes, settings, project_folder):
        """Initialize the completer

        Args:
            view (sublime.View): current view
            includes (list): includes from settings
            settings (Settings): plugin settings
            project_folder (str): current project folder

        """
        file_name = view.file_name()
        file_folder = path.dirname(file_name)

        # set std_flag
        self.std_flag = settings.std_flag

        # init needed variables from settings
        self.flags_dict[view.id()] = []
        for include in includes:
            self.flags_dict[view.id()].append('-I "{}"'.format(include))

        # support .clang_complete file with -I "<indlude>" entries
        if settings.search_clang_complete:
            log.debug(" searching for .clang_complete in %s up to %s",
                      file_folder, project_folder)
            clang_complete_file = Completer._search_clang_complete_file(
                file_folder, project_folder)
            if clang_complete_file:
                log.debug(" found .clang_complete: %s", clang_complete_file)
                flags = Completer._parse_clang_complete_file(
                    clang_complete_file)
                self.flags_dict[view.id()] += flags

        log.debug(" clang flags are: %s", self.flags_dict[view.id()])

    def complete(self, view, cursor_pos, show_errors):
        """This function is called asynchronously to create a list of
        autocompletions. It builds up a clang command that is then executed
        as a subprocess. The output is parsed for completions and/or errors

        Args:
            view (sublime.View): current view
            cursor_pos (int): sublime provided poistion of the cursor
            show_errors (bool): true if we want to visualize errors

        """
        if not view.id() in self.flags_dict:
            log.error(" cannot complete view: %s", view.id())
            return None

        file_body = view.substr(sublime.Region(0, view.size()))
        (row, col) = view.rowcol(cursor_pos)
        row += 1
        col += 1

        tempdir = tempfile.gettempdir()
        temp_file_name = path.join(tempdir, path.basename(view.file_name()))
        with open(temp_file_name, "w", encoding='utf-8') as tmp_file:
            tmp_file.write(file_body)

        complete_at_str = "{complete_flag}={file}:{row}:{col} {file}".format(
            complete_flag="-Xclang -code-completion-at",
            file=temp_file_name, row=row, col=col)

        complete_cmd = "{binary} {init} {std} {complete_at} {includes}".format(
            binary=Completer.clang_binary,
            init=" ".join(Completer.init_flags),
            std=self.std_flag,
            complete_at=complete_at_str,
            includes=" ".join(self.flags_dict[view.id()]))
        log.debug(" clang command: \n%s", complete_cmd)
        # execute clang code completion
        start = time.time()
        log.debug(" started code complete for view %s", view.id())

        try:
            output = subprocess.check_output(complete_cmd,
                                             stderr=subprocess.STDOUT,
                                             shell=True)
            output_text = ''.join(map(chr, output))
        except subprocess.CalledProcessError as e:
            output_text = e.output.decode("utf-8")
            log.info(" clang process finished with code: \n%s", e.returncode)
            log.info(" clang process output: \n%s", output_text)
            if show_errors:
                self.error_vis.generate(view, output_text.splitlines(),
                                        FORMAT_BINARY)
                self.error_vis.show_regions(view)
            # we could stop here, but we continue as sometimes there are still
            # valid completions even though there were errors encountered

        # Process clang output, find COMPLETION lines and return them with a
        # little formating
        raw_complete = output_text.splitlines()
        end = time.time()
        log.debug(" code complete done in %s seconds", end - start)

        self.completions = Completer._parse_completions(raw_complete)
        self.async_completions_ready = True
        Completer._reload_completions(view)

    def update(self, view, show_errors):
        """update build for current view

        Args:
            view (sublime.View): this view
            show_errors (TYPE): do we need to show errors? If not this is a
                dummy function as we gain nothing from building it with binary.

        """
        if view.id() not in self.flags_dict:
            log.error(" Cannot update view %s. No build flags.", view.id())
            return False

        if not show_errors:
            # in this class there is no need to rebuild the file. It brings no
            # benefits. We only want to do it if we need to show errors.
            return False

        file_body = view.substr(sublime.Region(0, view.size()))

        tempdir = tempfile.gettempdir()
        temp_file_name = path.join(tempdir, path.basename(view.file_name()))
        with open(temp_file_name, "w", encoding='utf-8') as tmp_file:
            tmp_file.write(file_body)

        complete_cmd = "{binary} {init} {std} {file} {includes}".format(
            binary=Completer.clang_binary,
            init=" ".join(Completer.init_flags),
            std=self.std_flag,
            file=temp_file_name,
            includes=" ".join(self.flags_dict[view.id()]))
        log.debug(" clang command: \n%s", complete_cmd)
        # execute clang code completion
        start = time.time()
        log.debug(" started rebuilding view %s", view.id())

        try:
            output = subprocess.check_output(complete_cmd,
                                             stderr=subprocess.STDOUT,
                                             shell=True)
            output_text = ''.join(map(chr, output))
        except subprocess.CalledProcessError as e:
            output_text = e.output.decode("utf-8")
            log.info(" clang process finished with code: \n%s", e.returncode)
            log.info(" clang process output: \n%s", output_text)
            self.error_vis.generate(view, output_text.splitlines(), FORMAT_BINARY)
            self.error_vis.show_regions(view)
            return False

        end = time.time()
        log.debug(" rebuilding done in %s seconds", end - start)
        return True

    @staticmethod
    def _parse_completions(complete_results):
        """Create snippet-like structures from a list of completions

        Args:
            complete_results (list): raw completions list

        Returns:
            list: updated completions
        """
        class Parser:
            """Help class to parse completions with regex

            Attributes:
                place_holders (int): number of place holders in use
            """
            def __init__(self):
                self.place_holders = 0

            def tokenize_params(self, match):
                """Create tockens from a match. Used as part or re.sub function

                Args:
                    match (re.match): current match

                Returns:
                    str: current match, wrapped in snippet
                """
                dict_match = match.groupdict()
                if dict_match[Completer.PARAM_TAG]:
                    self.place_holders += 1
                    return "${{{count}:{text}}}".format(
                        count=self.place_holders,
                        text=dict_match[Completer.PARAM_TAG])
                return ''

            @staticmethod
            def make_pretty(match):
                """Process raw match and remove ugly placeholders. Needed to
                have a human readable text for each completion.

                Args:
                    match (re.match): current completion

                Returns:
                    str: match stripped from unneeded placeholders
                """
                dict_match = match.groupdict()
                if dict_match[Completer.PARAM_TAG]:
                    return dict_match[Completer.PARAM_TAG]
                if dict_match[Completer.TYPE_TAG]:
                    return dict_match[Completer.TYPE_TAG] + ' '
                return ''

        completions = []
        for completion in complete_results:
            pos_search = Completer.compl_regex.search(completion)
            if not pos_search:
                log.warning(" completion %s did not match pattern %s",
                            completion, Completer.compl_regex)
                continue
            comp_dict = pos_search.groupdict()
            log.debug("completions parsed: %s", comp_dict)
            trigger = comp_dict['name']
            parser = Parser()
            contents = re.sub(Completer.compl_content_regex,
                              parser.tokenize_params,
                              comp_dict['content'])
            contents = re.sub(Completer.opts_regex, '', contents)
            hint = re.sub(Completer.compl_content_regex,
                          Parser.make_pretty,
                          comp_dict['content'])
            hint = re.sub(Completer.opts_regex, '', hint)
            completions.append([trigger + "\t" + hint, contents])
        return completions
<|endoftext|>


<commit_before>from peewee import *
from pprint import pprint
from time import time
import simplejson
import binascii

import sys, os
sys.path.append( os.path.join( os.path.dirname(__file__), '..' ) )
sys.path.append( os.path.join( os.path.dirname(__file__), '..' , 'lib' ) )

import config
import misc

# our mixin
from queue_gov_object import QueueGovObject

env = os.environ.get('SENTINEL_ENV') or 'production'
db_cfg = config.db[env].copy()
dbname = db_cfg.pop('database')

db = MySQLDatabase(dbname, **db_cfg)

# === models ===

class BaseModel(Model):
    def get_dict(self):
      dikt = {}
      for field_name in self._meta.columns.keys():

        # don't include DB id
        if "id" == field_name:
            continue

        dikt[ field_name ] = getattr( self, field_name )
      return dikt

    class Meta:
        database = db

    def is_valid(self):
        raise NotImplementedError("Method be over-ridden in subclasses")

class GovernanceObject(BaseModel):
    #id = IntegerField(primary_key = True)
    parent_id = IntegerField(default=0)
    object_creation_time = IntegerField(default=int(time()))
    object_hash = CharField(default='0')
    object_parent_hash = CharField(default='0')
    object_name = CharField(default='')
    object_type = IntegerField(default=0)
    object_revision = IntegerField(default=1)
    object_fee_tx = CharField(default='')

    class Meta:
        db_table = 'governance_objects'

    subclasses = ['proposals', 'superblocks']

    @classmethod
    def root(self):
        root_properties = {
            "object_name" : "root",
            "object_type" : 0,
            "object_creation_time" : 0,
        }
        return self(**root_properties)

    @classmethod
    def object_with_name_exists(self, name):
        count = self.select().where(self.object_name == name).count()
        return count > 0

    @property
    def object_data(self):
        return self.serialize_subclasses()

    def serialize_subclasses(self):
        objects = []

        for obj_type in self._meta.reverse_rel.keys():
            if obj_type in self.subclasses:
                res = getattr( self, obj_type )
                if res:
                    # should only return one row, but for completeness...
                    for row in res:
                        objects.append((obj_type, row.get_dict()))

        the_json = simplejson.dumps(objects, sort_keys = True)
        # print "the_json = %s" % the_json

        the_hex = binascii.hexlify( the_json )
        # print "the_hex = %s" % the_hex

        return the_hex

    def get_prepare_command(self):
        cmd = "gobject prepare %s %s %s %s %s" % (
            self.object_parent_hash,
            self.object_revision,
            self.object_creation_time,
            self.object_name,
            self.object_data
        )
        return cmd

    def get_submit_command(self):
        cmd = "gobject submit %s %s %s %s %s %s" % (
            self.object_fee_tx,
            self.object_parent_hash,
            self.object_revision,
            self.object_creation_time,
            self.object_name,
            self.object_data
        )
        return cmd

    def vote(self):
        # TODO
        pass

    def is_valid(self):
        raise NotImplementedError("Method be over-ridden in subclasses")
        # -- might be possible to do base checks here and then ...
        # govobj.is_valid() in sub-classes (as an alternative "super" since
        # they're not true Python sub-classes)
        """
            - check tree position validity
            - check signatures of owners
            - check validity of revision (must be n+1)
            - check validity of field data (address format, etc)
        """

class Action(BaseModel):
    #id = IntegerField(primary_key = True)
    #governance_object_id = IntegerField(unique=True)
    governance_object = ForeignKeyField(GovernanceObject, related_name = 'actions')
    absolute_yes_count = IntegerField()
    yes_count = IntegerField()
    no_count = IntegerField()
    abstain_count = IntegerField()
    class Meta:
        db_table = 'actions'

class Event(BaseModel):
    #id = IntegerField(primary_key = True)
    #governance_object_id = IntegerField(unique=True)
    governance_object = ForeignKeyField(GovernanceObject, related_name = 'events')
    start_time = IntegerField(default=int(time()))
    prepare_time = IntegerField()
    submit_time = IntegerField()
    error_time = IntegerField()
    error_message = CharField()

    class Meta:
        db_table = 'events'

    @classmethod
    def new(self):
        return self.select().where(
            (self.start_time <= misc.get_epoch() ) &
            (self.error_time == 0) &
            (self.prepare_time == 0)
        )

    @classmethod
    def prepared(self):
        now = misc.get_epoch()

        return self.select().where(
            (self.start_time <= now ) &
            (self.prepare_time <= now ) &
            (self.prepare_time > 0 ) &
            (self.submit_time == 0)
        )

    @classmethod
    def submitted(self):
        now = misc.get_epoch()

        return self.select().where(
            (self.submit_time > 0 )
        )

class Setting(BaseModel):
    #id = IntegerField(primary_key = True)
    datetime = IntegerField()
    setting  = CharField()
    name     = CharField()
    value    = CharField()
    class Meta:
        db_table = 'settings'

class Proposal(BaseModel, QueueGovObject):
    #id = IntegerField(primary_key = True)
    #governance_object_id = IntegerField(unique=True)
    governance_object = ForeignKeyField(GovernanceObject, related_name = 'proposals')
    proposal_name = CharField(unique=True)
    start_epoch = IntegerField()
    end_epoch = IntegerField()
    payment_address = CharField()
    payment_amount = DecimalField(max_digits=16, decimal_places=8)

    govobj_type = 1

    class Meta:
        db_table = 'proposals'

    # TODO: rename column 'proposal_name' to 'name' and remove this
    @property
    def name(self):
        return self.proposal_name

    def is_valid(self):
        # max value < budget allocation
        # end date > start date
        # end date > current date
        # payment addresss is valid base58 dash addr, non-multisig
        # proposal name is normalized (something like "[a-zA-Z0-9-_]+")
        pass

    def is_deletable(self):
        # end_date < (current_date - 30 days)
        # TBD (item moved to external storage/DashDrive, etc.)
        pass

class Superblock(BaseModel, QueueGovObject):
    #id = IntegerField(primary_key = True)
    #governance_object_id = IntegerField(unique=True)
    governance_object = ForeignKeyField(GovernanceObject, related_name = 'superblocks')
    superblock_name      = CharField() # unique?
    event_block_height   = IntegerField()
    payment_addresses    = TextField()
    payment_amounts      = TextField()

    govobj_type = 2

    class Meta:
        db_table = 'superblocks'

    # TODO: rename column 'superblock_name' to 'name' and remove this
    @property
    def name(self):
        return self.superblock_name

    def is_valid(self):
        # vout != generated vout
        # blockheight != generated blockheight
        pass

    def is_deletable(self):
        # end_date < (current_date - 30 days)
        # TBD (item moved to external storage/DashDrive, etc.)
        pass

# === /models ===

db.connect()
<commit_msg>add Proposal validation rules (from Tim)<commit_after>import pdb
from peewee import *
from pprint import pprint
from time import time
import simplejson
import binascii
import re

import sys, os
sys.path.append( os.path.join( os.path.dirname(__file__), '..' ) )
sys.path.append( os.path.join( os.path.dirname(__file__), '..' , 'lib' ) )

import config
import misc
import dashd

# our mixin
from queue_gov_object import QueueGovObject
from dashd import is_valid_dash_address

env = os.environ.get('SENTINEL_ENV') or 'production'
db_cfg = config.db[env].copy()
dbname = db_cfg.pop('database')

db = MySQLDatabase(dbname, **db_cfg)

# === models ===

class BaseModel(Model):
    def get_dict(self):
      dikt = {}
      for field_name in self._meta.columns.keys():

        # don't include DB id
        if "id" == field_name:
            continue

        dikt[ field_name ] = getattr( self, field_name )
      return dikt

    class Meta:
        database = db

    def is_valid(self):
        raise NotImplementedError("Method be over-ridden in subclasses")

class GovernanceObject(BaseModel):
    #id = IntegerField(primary_key = True)
    parent_id = IntegerField(default=0)
    object_creation_time = IntegerField(default=int(time()))
    object_hash = CharField(default='0')
    object_parent_hash = CharField(default='0')
    object_name = CharField(default='')
    object_type = IntegerField(default=0)
    object_revision = IntegerField(default=1)
    object_fee_tx = CharField(default='')

    class Meta:
        db_table = 'governance_objects'

    subclasses = ['proposals', 'superblocks']

    @classmethod
    def root(self):
        root_properties = {
            "object_name" : "root",
            "object_type" : 0,
            "object_creation_time" : 0,
        }
        return self(**root_properties)

    @classmethod
    def object_with_name_exists(self, name):
        count = self.select().where(self.object_name == name).count()
        return count > 0

    @property
    def object_data(self):
        return self.serialize_subclasses()

    def serialize_subclasses(self):
        objects = []

        for obj_type in self._meta.reverse_rel.keys():
            if obj_type in self.subclasses:
                res = getattr( self, obj_type )
                if res:
                    # should only return one row, but for completeness...
                    for row in res:
                        objects.append((obj_type, row.get_dict()))

        the_json = simplejson.dumps(objects, sort_keys = True)
        # print "the_json = %s" % the_json

        the_hex = binascii.hexlify( the_json )
        # print "the_hex = %s" % the_hex

        return the_hex

    def get_prepare_command(self):
        cmd = "gobject prepare %s %s %s %s %s" % (
            self.object_parent_hash,
            self.object_revision,
            self.object_creation_time,
            self.object_name,
            self.object_data
        )
        return cmd

    def get_submit_command(self):
        cmd = "gobject submit %s %s %s %s %s %s" % (
            self.object_fee_tx,
            self.object_parent_hash,
            self.object_revision,
            self.object_creation_time,
            self.object_name,
            self.object_data
        )
        return cmd

    def vote(self):
        # TODO
        pass

    def is_valid(self):
        raise NotImplementedError("Method be over-ridden in subclasses")
        # -- might be possible to do base checks here and then ...
        # govobj.is_valid() in sub-classes (as an alternative "super" since
        # they're not true Python sub-classes)
        """
            - check tree position validity
            - check signatures of owners
            - check validity of revision (must be n+1)
            - check validity of field data (address format, etc)
        """

class Action(BaseModel):
    #id = IntegerField(primary_key = True)
    #governance_object_id = IntegerField(unique=True)
    governance_object = ForeignKeyField(GovernanceObject, related_name = 'actions')
    absolute_yes_count = IntegerField()
    yes_count = IntegerField()
    no_count = IntegerField()
    abstain_count = IntegerField()
    class Meta:
        db_table = 'actions'

class Event(BaseModel):
    #id = IntegerField(primary_key = True)
    #governance_object_id = IntegerField(unique=True)
    governance_object = ForeignKeyField(GovernanceObject, related_name = 'events')
    start_time = IntegerField(default=int(time()))
    prepare_time = IntegerField()
    submit_time = IntegerField()
    error_time = IntegerField()
    error_message = CharField()

    class Meta:
        db_table = 'events'

    @classmethod
    def new(self):
        return self.select().where(
            (self.start_time <= misc.get_epoch() ) &
            (self.error_time == 0) &
            (self.prepare_time == 0)
        )

    @classmethod
    def prepared(self):
        now = misc.get_epoch()

        return self.select().where(
            (self.start_time <= now ) &
            (self.prepare_time <= now ) &
            (self.prepare_time > 0 ) &
            (self.submit_time == 0)
        )

    @classmethod
    def submitted(self):
        now = misc.get_epoch()

        return self.select().where(
            (self.submit_time > 0 )
        )

class Setting(BaseModel):
    #id = IntegerField(primary_key = True)
    datetime = IntegerField()
    setting  = CharField()
    name     = CharField()
    value    = CharField()
    class Meta:
        db_table = 'settings'

class Proposal(BaseModel, QueueGovObject):
    #id = IntegerField(primary_key = True)
    #governance_object_id = IntegerField(unique=True)
    governance_object = ForeignKeyField(GovernanceObject, related_name = 'proposals')
    proposal_name = CharField(unique=True)
    start_epoch = IntegerField()
    end_epoch = IntegerField()
    payment_address = CharField()
    payment_amount = DecimalField(max_digits=16, decimal_places=8)

    govobj_type = 1

    class Meta:
        db_table = 'proposals'

    # TODO: rename column 'proposal_name' to 'name' and remove this
    @property
    def name(self):
        return self.proposal_name

    # TODO: unit tests for all these items, both individually and some grouped
    # **This can be easily mocked.**
    def is_valid(self):
        now = misc.get_epoch()

        # proposal name is normalized (something like "[a-zA-Z0-9-_]+")
        if not re.match( '^[-_a-zA-Z0-9]+$', self.name ):
            return False

        # end date < start date
        if ( self.end_epoch <= self.start_epoch ):
            return False

        # end date < current date
        if ( self.end_epoch <= now ):
            return False

        # TODO: ask Tim about this, how to get block height
        #
        # TODO: consider a mixin for this class's dashd calls -- that or a
        # global... need to find an elegant way to handle this...
        #
        # max_budget_allocation = dashd.rpccommand('getsuperblockbudget', block_height)
        # block_height = 62000
        # max value > budget allocation
        max_budget_allocation = 2000
        if ( self.payment_amount > max_budget_allocation ):
            return False

        # payment addresss is valid base58 dash addr, non-multisig
        if not is_valid_dash_address( self.payment_address, config.network ):
            return False

        return True

    def is_deletable(self):
        # end_date < (current_date - 30 days)
        # TBD (item moved to external storage/DashDrive, etc.)
        pass

class Superblock(BaseModel, QueueGovObject):
    #id = IntegerField(primary_key = True)
    #governance_object_id = IntegerField(unique=True)
    governance_object = ForeignKeyField(GovernanceObject, related_name = 'superblocks')
    superblock_name      = CharField() # unique?
    event_block_height   = IntegerField()
    payment_addresses    = TextField()
    payment_amounts      = TextField()

    govobj_type = 2

    class Meta:
        db_table = 'superblocks'

    # TODO: rename column 'superblock_name' to 'name' and remove this
    @property
    def name(self):
        return self.superblock_name

    def is_valid(self):
        # vout != generated vout
        # blockheight != generated blockheight
        pass

    def is_deletable(self):
        # end_date < (current_date - 30 days)
        # TBD (item moved to external storage/DashDrive, etc.)
        pass

# === /models ===

db.connect()
<|endoftext|>


<commit_before>#!/usr/bin/env python
import matplotlib.pyplot as plt
import numpy as np
import csv

# Python 2.7 does not have str.isnumeric()?
def isInt(string):
  try:
    int(string)
    return True
  except ValueError:
    return False

# Format of the CSV file is:
# time,dofs,integral
# 1,221,2.3592493758695,
# 2,841,0.30939803328432,
# 3,3281,0.088619511656913,
# 4,12961,0.022979021365857,
# 5,51521,0.0057978748995635,
# 6,205441,0.0014528130907967,
reader = csv.reader(file('out.csv'))

dofs = []
errs = []
for row in reader:
  if row and isInt(row[0]): # Skip rows that don't start with numbers.
    dofs.append(int(row[1]))
    errs.append(float(row[2]))

# Construct data to be plotted
xdata = np.log10(np.sqrt(dofs))
ydata = np.log10(errs)

fig = plt.figure()
ax1 = fig.add_subplot(111)
ax1.plot(xdata, ydata, 'bo-')
ax1.set_xlabel('log (1/h)')
ax1.set_ylabel('log (L2-error)')

# Create linear curve fits of the data, but just the last couple data
# point when we are in the asymptotic regime.
fit = np.polyfit(xdata[2:-1], ydata[2:-1], 1)
fit_msg = 'Slope ~ ' + '%.2f' % fit[0]
ax1.text(2.0, -1.0, fit_msg)

plt.savefig('plot.pdf', format='pdf')

# Local Variables:
# python-indent: 2
# End:
<commit_msg>The name of the CSV output file changed at some point.<commit_after>#!/usr/bin/env python
import matplotlib.pyplot as plt
import numpy as np
import csv

# Python 2.7 does not have str.isnumeric()?
def isInt(string):
  try:
    int(string)
    return True
  except ValueError:
    return False

# Format of the CSV file is:
# time,dofs,integral
# 1,221,2.3592493758695,
# 2,841,0.30939803328432,
# 3,3281,0.088619511656913,
# 4,12961,0.022979021365857,
# 5,51521,0.0057978748995635,
# 6,205441,0.0014528130907967,
reader = csv.reader(file('ex14_out.csv'))

dofs = []
errs = []
for row in reader:
  if row and isInt(row[0]): # Skip rows that don't start with numbers.
    dofs.append(int(row[1]))
    errs.append(float(row[2]))

# Construct data to be plotted
xdata = np.log10(np.sqrt(dofs))
ydata = np.log10(errs)

fig = plt.figure()
ax1 = fig.add_subplot(111)
ax1.plot(xdata, ydata, 'bo-')
ax1.set_xlabel('log (1/h)')
ax1.set_ylabel('log (L2-error)')

# Create linear curve fits of the data, but just the last couple data
# point when we are in the asymptotic regime.
fit = np.polyfit(xdata[2:-1], ydata[2:-1], 1)
fit_msg = 'Slope ~ ' + '%.2f' % fit[0]
ax1.text(2.0, -1.0, fit_msg)

plt.savefig('plot.pdf', format='pdf')

# Local Variables:
# python-indent: 2
# End:
<|endoftext|>


<commit_before># Copyright (c) 2019-2020 Manfred Moitzi
# License: MIT License
from typing import TYPE_CHECKING
from ezdxf.lldxf.attributes import DXFAttr, DXFAttributes, DefSubclass
from ezdxf.lldxf.const import (
    DXF2000, STRUCTURE_MARKER, OWNER_CODE, SUBCLASS_MARKER,
)
from .dxfentity import SubclassProcessor, DXFEntity
from .factory import register_entity

if TYPE_CHECKING:
    from ezdxf.eztypes import TagWriter, DXFNamespace

__all__ = ['TableHead']

base_class = DefSubclass(None, {
    'name': DXFAttr(2),
    'handle': DXFAttr(5),
    'owner': DXFAttr(330),
})

acdb_symbol_table = DefSubclass('AcDbSymbolTable', {
    'count': DXFAttr(70, default=0),
})

@register_entity
class TableHead(DXFEntity):
    """ The table head structure is only maintained for export and not for
    internal usage, ezdxf ignores an inconsistent table head at runtime.

    """
    DXFTYPE = 'TABLE'
    DXFATTRIBS = DXFAttributes(base_class, acdb_symbol_table)

    def load_dxf_attribs(
            self, processor: SubclassProcessor = None) -> 'DXFNamespace':
        dxf = super().load_dxf_attribs(processor)
        if processor:
            dxf.name = processor.base_class.get_first_value(2)
            # Stored max table count is not required:
            dxf.count = 0
        return dxf

    def export_dxf(self, tagwriter: 'TagWriter') -> None:
        assert self.dxf.handle, 'TABLE needs a handle, maybe loaded from ' \
                                'DXF R12 without handle!'
        tagwriter.write_tag2(STRUCTURE_MARKER, self.DXFTYPE)
        tagwriter.write_tag2(2, self.dxf.name)
        if tagwriter.dxfversion >= DXF2000:
            tagwriter.write_tag2(5, self.dxf.handle)
            if self.has_extension_dict:
                self.extension_dict.export_dxf(tagwriter)
            tagwriter.write_tag2(OWNER_CODE, self.dxf.owner)
            tagwriter.write_tag2(SUBCLASS_MARKER, acdb_symbol_table.name)
            tagwriter.write_tag2(70, self.dxf.count)
            # There is always one exception:
            if self.dxf.name == 'DIMSTYLE':
                tagwriter.write_tag2(SUBCLASS_MARKER, 'AcDbDimStyleTable')
        else:  # DXF R12
            # TABLE does not need a handle at all
            tagwriter.write_tag2(70, self.dxf.count)
<commit_msg>reformat code by black<commit_after># Copyright (c) 2019-2021 Manfred Moitzi
# License: MIT License
from typing import TYPE_CHECKING
from ezdxf.lldxf.attributes import DXFAttr, DXFAttributes, DefSubclass
from ezdxf.lldxf import const
from .dxfentity import SubclassProcessor, DXFEntity
from .factory import register_entity

if TYPE_CHECKING:
    from ezdxf.eztypes import TagWriter, DXFNamespace

__all__ = ["TableHead"]

base_class = DefSubclass(
    None,
    {
        "name": DXFAttr(2),
        "handle": DXFAttr(5),
        "owner": DXFAttr(330),
    },
)

acdb_symbol_table = DefSubclass(
    "AcDbSymbolTable",
    {
        "count": DXFAttr(70, default=0),
    },
)

@register_entity
class TableHead(DXFEntity):
    """The table head structure is only maintained for export and not for
    internal usage, ezdxf ignores an inconsistent table head at runtime.

    """

    DXFTYPE = "TABLE"
    DXFATTRIBS = DXFAttributes(base_class, acdb_symbol_table)

    def load_dxf_attribs(
        self, processor: SubclassProcessor = None
    ) -> "DXFNamespace":
        dxf = super().load_dxf_attribs(processor)
        if processor:
            dxf.name = processor.base_class.get_first_value(2)
            # Stored max table count is not required:
            dxf.count = 0
        return dxf

    def export_dxf(self, tagwriter: "TagWriter") -> None:
        assert self.dxf.handle, (
            "TABLE needs a handle, maybe loaded from " "DXF R12 without handle!"
        )
        tagwriter.write_tag2(const.STRUCTURE_MARKER, self.DXFTYPE)
        tagwriter.write_tag2(2, self.dxf.name)
        if tagwriter.dxfversion >= const.DXF2000:
            tagwriter.write_tag2(5, self.dxf.handle)
            if self.has_extension_dict:
                self.extension_dict.export_dxf(tagwriter)  # type: ignore
            tagwriter.write_tag2(const.OWNER_CODE, self.dxf.owner)
            tagwriter.write_tag2(const.SUBCLASS_MARKER, acdb_symbol_table.name)
            tagwriter.write_tag2(70, self.dxf.count)
            # There is always one exception:
            if self.dxf.name == "DIMSTYLE":
                tagwriter.write_tag2(const.SUBCLASS_MARKER, "AcDbDimStyleTable")
        else:  # DXF R12
            # TABLE does not need a handle at all
            tagwriter.write_tag2(70, self.dxf.count)
<|endoftext|>