<commit_before>#__BEGIN_LICENSE__
# Copyright (c) 2015, United States Government, as represented by the
# Administrator of the National Aeronautics and Space Administration.
# All rights reserved.
#
# The xGDS platform is licensed under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
# http://www.apache.org/licenses/LICENSE-2.0.
#
# Unless required by applicable law or agreed to in writing, software distributed
# under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
# CONDITIONS OF ANY KIND, either express or implied. See the License for the
# specific language governing permissions and limitations under the License.
#__END_LICENSE__

import re
import datetime
import copy
import logging
import os

import iso8601
from django.db import models
from django.conf import settings
from django.contrib.auth.models import User
from django.core.urlresolvers import reverse

from geocamUtil.models.UuidField import UuidField, makeUuid
from geocamUtil.models.ExtrasDotField import ExtrasDotField
from geocamUtil.modelJson import modelToDict
from geocamUtil.dotDict import DotDict

from xgds_planner2 import xpjson, statsPlanExporter
from geocamUtil.loader import LazyGetModelByName

# pylint: disable=C1001,E1101

# SCHEMA = xpjson.loadDocument(settings.XGDS_PLANNER_SCHEMA_PATH)
# LIBRARY = xpjson.loadDocument(settings.XGDS_PLANNER_LIBRARY_PATH, schema=SCHEMA)
#
# _schema = 'xgds_planner2/schema.json'
# _library = 'xgds_planner2/library.json'
# SIMPLIFIED_SCHEMA_PATH = settings.STATIC_ROOT + _schema
# SIMPLIFIED_LIBRARY_PATH = settings.STATIC_ROOT + _library
# SIMPLIFIED_SCHEMA_URL = settings.STATIC_URL + _schema
# SIMPLIFIED_LIBRARY_URL = settings.STATIC_URL + _library

PLAN_SCHEMA_CACHE = {}

class AbstractVehicle(models.Model):
    name = models.CharField(max_length=192, blank=True)
    notes = models.TextField(blank=True)
    type = models.CharField(max_length=16)

    class Meta:
        abstract = True

    def __unicode__(self):
        return self.name

    def getDict(self):
        return {"name": self.name, "notes": self.notes, "type": self.type}

class Vehicle(AbstractVehicle):
    pass

class PlanExecution(models.Model):
    """
    Relationship table for managing
    flight to plan's many to many relationship.
    """
    start_time = models.DateTimeField(null=True, blank=True)
    planned_start_time = models.DateTimeField(null=True, blank=True)
    end_time = models.DateTimeField(null=True, blank=True)

    flight = models.ForeignKey(settings.XGDS_PLANNER2_FLIGHT_MODEL) #, related_name="plans")
    plan = models.ForeignKey(settings.XGDS_PLANNER2_PLAN_MODEL) # , related_name="executions")

    def toSimpleDict(self):
        result = {}
        result['start_time'] = self.start_time
        result['planned_start_time'] = self.planned_start_time
        result['end_time'] = self.end_time
        if self.plan:
            result['plan'] = self.plan.pk
        else:
            result['plan'] = None
        if self.flight:
            result['flight'] = self.flight.name
        else:
            result['flight'] = None
        return result

    def __unicode__(self):
        return str(self.pk)

    class Meta:
        ordering = ['planned_start_time']

# class PlanExecution(AbstractPlanExecution):
#     pass

class AbstractFlight(models.Model):
    uuid = UuidField(unique=True, db_index=True)
    name = models.CharField(max_length=255, blank=True, unique=True, help_text='it is episode name + asset role. i.e. 20130925A_ROV')
    locked = models.BooleanField(blank=True, default=False)
    start_time = models.DateTimeField(null=True, blank=True)
    end_time = models.DateTimeField(null=True, blank=True)
    vehicle = models.ForeignKey(settings.XGDS_PLANNER2_VEHICLE_MODEL, null=True, blank=True)
    notes = models.TextField(blank=True)
    group = models.ForeignKey(settings.XGDS_PLANNER2_GROUP_FLIGHT_MODEL, null=True, blank=True)

    def startFlightExtras(self, request):
        pass

    def stopFlightExtras(self, request):
        pass

    def __unicode__(self):
        return self.name

    def getTreeJson(self):
        result = {"title": self.name,
                  "key": self.uuid,
                  "tooltip": self.notes,
                  "data": {"type": self.__class__.__name__,
                           "vehicle": self.vehicle.name
                           }
                  }
        return result

    @property
    def plans(self):
        return LazyGetModelByName(settings.XGDS_PLANNER2_PLAN_EXECUTION_MODEL).get().objects.filter(flight=self)

    class Meta:
        abstract = True
        ordering = ['-name']

class Flight(AbstractFlight):
    pass

class AbstractActiveFlight(models.Model):
    flight = models.OneToOneField(settings.XGDS_PLANNER2_FLIGHT_MODEL, related_name="active")

    def __unicode__(self):
        return (u'ActiveFlight(%s, %s)' %
                (self.pk, repr(self.flight.name)))

    class Meta:
        abstract = True

class ActiveFlight(AbstractActiveFlight):
    pass

class AbstractGroupFlight(models.Model):
    """
    This GroupFlight model represents the overall coordinated
    operation.
    """
    name = models.CharField(max_length=255, blank=True, unique=True, help_text='Usually same as episode name. I.e. 201340925A')
    notes = models.TextField(blank=True)

    class Meta:
        abstract = True

    def __unicode__(self):
        return self.name

class GroupFlight(AbstractGroupFlight):
    pass

class AbstractPlan(models.Model):
    uuid = UuidField(unique=True, db_index=True)
    name = models.CharField(max_length=256)
    dateModified = models.DateTimeField()
    creator = models.ForeignKey(User, null=True, blank=True)

    # the canonical serialization of the plan exchanged with javascript clients
    jsonPlan = ExtrasDotField()

    # a place to put an auto-generated summary of the plan
    summary = models.CharField(max_length=256)

    # allow users to mark plans as deleted.  remember to use this field!
    deleted = models.BooleanField(blank=True, default=False)

    # allow users to mark plans as read only, so when they are opened they cannot be edited
    readOnly = models.BooleanField(blank=True, default=False)

    # cache commonly used stats derived from the plan (relatively expensive to calculate)
    numStations = models.PositiveIntegerField(default=0)
    numSegments = models.PositiveIntegerField(default=0)
    numCommands = models.PositiveIntegerField(default=0)
    lengthMeters = models.FloatField(null=True, blank=True)
    estimatedDurationSeconds = models.FloatField(null=True, blank=True)
    stats = ExtrasDotField()  # a place for richer stats such as numCommandsByType

    class Meta:
        ordering = ['-dateModified']
        abstract = True

    def get_absolute_url(self):
        return reverse('planner2_planREST', args=[self.pk, self.name])

    def extractFromJson(self, overWriteDateModified=True, overWriteUuid=True):
        if overWriteUuid:
            if not self.uuid:
                self.uuid = makeUuid()
                self.jsonPlan.uuid = self.uuid
            self.jsonPlan.serverId = self.pk
        if overWriteDateModified:
            self.jsonPlan.dateModified = (datetime.datetime
                                          .utcnow()
                                          .replace(microsecond=0)
                                          .isoformat()
                                          + 'Z')

        self.name = self.jsonPlan.name
        self.jsonPlan.url = self.get_absolute_url()
#         print 'name is now ' + self.name
        self.jsonPlan.serverId = self.pk
        self.dateModified = (iso8601.parse_date(self.jsonPlan.dateModified)
                             .replace(tzinfo=None))
        plannerUsers = User.objects.filter(username=self.jsonPlan.creator)
        if plannerUsers:
            self.creator = plannerUsers[0]
        else:
            self.creator = None

        # fill in stats
        try:
            exporter = statsPlanExporter.StatsPlanExporter()
#             print ' about to do stats'
            stats = exporter.exportDbPlan(self)
            for f in ('numStations', 'numSegments', 'numCommands', 'lengthMeters'):
                setattr(self, f, stats[f])
            self.stats.numCommandsByType = stats["numCommandsByType"]
            self.summary = statsPlanExporter.getSummary(stats)
        except:
            logging.warning('extractFromJson: could not extract stats from plan %s',
                            self.uuid)
            raise  # FIX
        return self

    def getSummaryOfCommandsByType(self):
        return statsPlanExporter.getSummaryOfCommandsByType(self.stats)

    # TODO test
    def toXpjson(self):
        platform = self.jsonPlan['platform']
        if platform:
            planSchema = getPlanSchema(platform[u'name'])
            return xpjson.loadDocumentFromDict(self.jsonPlan,
                                               schema=planSchema.getSchema())
        logging.warning('toXpjson: could not convert to xpjson, probably no schema %s',
                        self.uuid)
        raise  # FIX

    def escapedName(self):
        name = re.sub(r'[^\w]', '', self.name)
        if name == '':
            return 'plan'
        else:
            if self.jsonPlan and self.jsonPlan.planVersion:
                return name + "_" + self.jsonPlan.planVersion
            return name

    def getExportUrl(self, extension):
        return reverse('planner2_planExport',
                       kwargs={'uuid': self.uuid, 'name': self.escapedName() + extension})

    def getExporters(self):
        import choosePlanExporter  # delayed import avoids import loop
        result = []
        for exporterInfo in choosePlanExporter.PLAN_EXPORTERS:
            info = copy.deepcopy(exporterInfo)
            info.url = self.getExportUrl(info.extension)
            result.append(info)
        return result

    def getLinks(self):
        """
        The links tab wil be populated with the name, value contents of this dictionary as links,
        name is the string displayed and link is what will be opened
        """
        result = {}
        result["KML"] = reverse('planner2_planExport', kwargs={'uuid': self.uuid, 'name': self.name + '.kml'})
        for exporter in self.getExporters():
            result[exporter.label] = exporter.url
        return result

    def getEscapedId(self):
        if self.jsonPlan and self.jsonPlan.id:
            result = re.sub(r'[^\w]', '', self.jsonPlan.id)
            result = re.sub('_PLAN$', '', result)
            return result
        else:
            return None

    def toMapDict(self):
        """
        Return a reduced dictionary that will be turned to JSON for rendering in a map
        Here we are just interested in the route plan and not in activities
        We just include stations
        """
        result = {}
        result['id'] = self.uuid
        result['author'] = self.jsonPlan.creator
        result['name'] = self.jsonPlan.name
        result['type'] = 'AbstractPlan'
        if self.jsonPlan.notes:
            result['notes'] = self.jsonPlan.notes
        else:
            result['notes'] = ''

        stations = []
        seq = self.jsonPlan.sequence
        for el in seq:
            if el.type == "Station":
                sta = {}
                sta['id'] = el.id
                sta['coords'] = el.geometry.coordinates
                if el.notes:
                    sta['notes'] = el.notes
                stations.append(sta)
        result['stations'] = stations
        return result

    def getTreeJson(self):
        result = {"title": self.name,
                  "key": self.uuid,
                  "tooltip": self.summary,
                  "data": {"type": "MapLink",  # we cheat so this will be 'live'
                           "json": reverse('planner2_mapJsonPlan', kwargs={'uuid': str(self.uuid)}),
                           "url": reverse('planner2_edit', kwargs={'plan_id': str(self.pk)})
                           }
                  }
        return result

    @property
    def executions(self):
        return LazyGetModelByName(settings.XGDS_PLANNER2_PLAN_EXECUTION_MODEL).get().objects.filter(plan=self)

    def __unicode__(self):
        if self.name:
            return self.name
        else:
            return 'Unnamed plan ' + self.uuid

class Plan(AbstractPlan):
    pass

# PlanSchema used to be a database model, but is now a normal Python
# class built from the Django settings.  This will not change during
# runtime so we should cache these in PLAN_SCHEMA_CACHE.
class PlanSchema:
    def __init__(self, platform, schemaDict):
        self.platform = platform
        self.schemaSource = settings.PROJ_ROOT + schemaDict['schemaSource']
        self.librarySource = settings.PROJ_ROOT + schemaDict['librarySource']
        self.simulatorUrl = settings.STATIC_URL + schemaDict['simulatorUrl']
        self.simulator = schemaDict['simulator']

        schemaSuffix = os.path.join('xgds_planner2', os.path.basename(self.schemaSource))
        librarySuffix = os.path.join('xgds_planner2', os.path.basename(self.librarySource))

        self.simplifiedSchemaPath = os.path.join(settings.STATIC_ROOT, schemaSuffix)
        self.simplifiedLibraryPath = os.path.join(settings.STATIC_ROOT, librarySuffix)

        self.schemaUrl = os.path.join(settings.STATIC_URL, schemaSuffix)
        self.libraryUrl = os.path.join(settings.STATIC_URL, librarySuffix)

        self.schema = None
        self.library = None
        self.jsonSchema = None
        self.jsonLibrary = None

    def getJsonSchema(self):
        if not self.jsonSchema:
            try:
                with open(self.simplifiedSchemaPath) as schemaFile:
                    self.jsonSchema = schemaFile.read()
            except:  # pylint: disable=W0702
                logging.warning('could not load XPJSON schema from ' + self.simplifiedSchemaPath)
                raise
        return self.jsonSchema

    def getSchema(self):
        if not self.schema:
            try:
                self.schema = xpjson.loadDocument(self.simplifiedSchemaPath)
            except:  # pylint: disable=W0702
                logging.warning('could not load XPJSON schema from ' + self.simplifiedSchemaPath)
                raise
        return self.schema

    def getJsonLibrary(self):
        if not self.jsonLibrary:
            try:
                with open(self.simplifiedLibraryPath) as libraryFile:
                    self.jsonLibrary = libraryFile.read()
            except:  # pylint: disable=W0702
                logging.warning('could not load XPJSON library from ' + self.simplifiedLibraryPath)
                raise
        return self.jsonLibrary

    def getLibrary(self):
        if not self.library:
            try:
                self.library = xpjson.loadDocument(self.simplifiedLibraryPath,
                                                   schema=self.getSchema(),
                                                   fillInDefaults=True)
            except:  # pylint: disable=W0702
                logging.warning('could not load XPJSON library from ' + self.simplifiedLibraryPath)
                raise
        return self.library

def loadSchema(platform):
    schemaDict = settings.XGDS_PLANNER_SCHEMAS[platform]
    schema = PlanSchema(platform, schemaDict)
    schema.getSchema()
    schema.getJsonSchema()
    schema.getLibrary()
    schema.getJsonLibrary()
    return schema

# get the cached plan schema, building it if need be.
def getPlanSchema(platform):
    result = PLAN_SCHEMA_CACHE.get(platform)
    if not result:
        try:
            result = loadSchema(platform)
            PLAN_SCHEMA_CACHE[platform] = result
        except:
            logging.warning('could not find plan schema for platform %s', platform)
            raise
    return result
<commit_msg>code cleanup<commit_after>#__BEGIN_LICENSE__
# Copyright (c) 2015, United States Government, as represented by the
# Administrator of the National Aeronautics and Space Administration.
# All rights reserved.
#
# The xGDS platform is licensed under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
# http://www.apache.org/licenses/LICENSE-2.0.
#
# Unless required by applicable law or agreed to in writing, software distributed
# under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
# CONDITIONS OF ANY KIND, either express or implied. See the License for the
# specific language governing permissions and limitations under the License.
#__END_LICENSE__

import re
import datetime
import copy
import logging
import os

import iso8601
from django.db import models
from django.conf import settings
from django.contrib.auth.models import User
from django.core.urlresolvers import reverse

from geocamUtil.models.UuidField import UuidField, makeUuid
from geocamUtil.models.ExtrasDotField import ExtrasDotField
from geocamUtil.modelJson import modelToDict
from geocamUtil.dotDict import DotDict

from xgds_planner2 import xpjson, statsPlanExporter
from geocamUtil.loader import LazyGetModelByName

# pylint: disable=C1001,E1101

# SCHEMA = xpjson.loadDocument(settings.XGDS_PLANNER_SCHEMA_PATH)
# LIBRARY = xpjson.loadDocument(settings.XGDS_PLANNER_LIBRARY_PATH, schema=SCHEMA)
#
# _schema = 'xgds_planner2/schema.json'
# _library = 'xgds_planner2/library.json'
# SIMPLIFIED_SCHEMA_PATH = settings.STATIC_ROOT + _schema
# SIMPLIFIED_LIBRARY_PATH = settings.STATIC_ROOT + _library
# SIMPLIFIED_SCHEMA_URL = settings.STATIC_URL + _schema
# SIMPLIFIED_LIBRARY_URL = settings.STATIC_URL + _library

PLAN_SCHEMA_CACHE = {}

class AbstractVehicle(models.Model):
    name = models.CharField(max_length=192, blank=True)
    notes = models.TextField(blank=True)
    type = models.CharField(max_length=16)

    class Meta:
        abstract = True

    def __unicode__(self):
        return self.name

    def getDict(self):
        return {"name": self.name, "notes": self.notes, "type": self.type}

class Vehicle(AbstractVehicle):
    pass

class PlanExecution(models.Model):
    """
    Relationship table for managing
    flight to plan's many to many relationship.
    """
    start_time = models.DateTimeField(null=True, blank=True)
    planned_start_time = models.DateTimeField(null=True, blank=True)
    end_time = models.DateTimeField(null=True, blank=True)

    flight = models.ForeignKey(settings.XGDS_PLANNER2_FLIGHT_MODEL) #, related_name="plans")
    plan = models.ForeignKey(settings.XGDS_PLANNER2_PLAN_MODEL) # , related_name="executions")

    def toSimpleDict(self):
        result = {}
        result['start_time'] = self.start_time
        result['planned_start_time'] = self.planned_start_time
        result['end_time'] = self.end_time
        if self.plan:
            result['plan'] = self.plan.pk
        else:
            result['plan'] = None
        if self.flight:
            result['flight'] = self.flight.name
        else:
            result['flight'] = None
        return result

    def __unicode__(self):
        return str(self.pk)

    class Meta:
        ordering = ['planned_start_time']

class AbstractFlight(models.Model):
    uuid = UuidField(unique=True, db_index=True)
    name = models.CharField(max_length=255, blank=True, unique=True, help_text='it is episode name + asset role. i.e. 20130925A_ROV')
    locked = models.BooleanField(blank=True, default=False)
    start_time = models.DateTimeField(null=True, blank=True)
    end_time = models.DateTimeField(null=True, blank=True)
    vehicle = models.ForeignKey(settings.XGDS_PLANNER2_VEHICLE_MODEL, null=True, blank=True)
    notes = models.TextField(blank=True)
    group = models.ForeignKey(settings.XGDS_PLANNER2_GROUP_FLIGHT_MODEL, null=True, blank=True)

    def startFlightExtras(self, request):
        pass

    def stopFlightExtras(self, request):
        pass

    def __unicode__(self):
        return self.name

    def getTreeJson(self):
        result = {"title": self.name,
                  "key": self.uuid,
                  "tooltip": self.notes,
                  "data": {"type": self.__class__.__name__,
                           "vehicle": self.vehicle.name
                           }
                  }
        return result

    @property
    def plans(self):
        return LazyGetModelByName(settings.XGDS_PLANNER2_PLAN_EXECUTION_MODEL).get().objects.filter(flight=self)

    class Meta:
        abstract = True
        ordering = ['-name']

class Flight(AbstractFlight):
    pass

class AbstractActiveFlight(models.Model):
    flight = models.OneToOneField(settings.XGDS_PLANNER2_FLIGHT_MODEL, related_name="active")

    def __unicode__(self):
        return (u'ActiveFlight(%s, %s)' %
                (self.pk, repr(self.flight.name)))

    class Meta:
        abstract = True

class ActiveFlight(AbstractActiveFlight):
    pass

class AbstractGroupFlight(models.Model):
    """
    This GroupFlight model represents the overall coordinated
    operation.
    """
    name = models.CharField(max_length=255, blank=True, unique=True, help_text='Usually same as episode name. I.e. 201340925A')
    notes = models.TextField(blank=True)

    class Meta:
        abstract = True

    def __unicode__(self):
        return self.name

class GroupFlight(AbstractGroupFlight):
    pass

class AbstractPlan(models.Model):
    uuid = UuidField(unique=True, db_index=True)
    name = models.CharField(max_length=256)
    dateModified = models.DateTimeField()
    creator = models.ForeignKey(User, null=True, blank=True)

    # the canonical serialization of the plan exchanged with javascript clients
    jsonPlan = ExtrasDotField()

    # a place to put an auto-generated summary of the plan
    summary = models.CharField(max_length=256)

    # allow users to mark plans as deleted.  remember to use this field!
    deleted = models.BooleanField(blank=True, default=False)

    # allow users to mark plans as read only, so when they are opened they cannot be edited
    readOnly = models.BooleanField(blank=True, default=False)

    # cache commonly used stats derived from the plan (relatively expensive to calculate)
    numStations = models.PositiveIntegerField(default=0)
    numSegments = models.PositiveIntegerField(default=0)
    numCommands = models.PositiveIntegerField(default=0)
    lengthMeters = models.FloatField(null=True, blank=True)
    estimatedDurationSeconds = models.FloatField(null=True, blank=True)
    stats = ExtrasDotField()  # a place for richer stats such as numCommandsByType

    class Meta:
        ordering = ['-dateModified']
        abstract = True

    def get_absolute_url(self):
        return reverse('planner2_planREST', args=[self.pk, self.name])

    def extractFromJson(self, overWriteDateModified=True, overWriteUuid=True):
        if overWriteUuid:
            if not self.uuid:
                self.uuid = makeUuid()
                self.jsonPlan.uuid = self.uuid
            self.jsonPlan.serverId = self.pk
        if overWriteDateModified:
            self.jsonPlan.dateModified = (datetime.datetime
                                          .utcnow()
                                          .replace(microsecond=0)
                                          .isoformat()
                                          + 'Z')

        self.name = self.jsonPlan.name
        self.jsonPlan.url = self.get_absolute_url()
#         print 'name is now ' + self.name
        self.jsonPlan.serverId = self.pk
        self.dateModified = (iso8601.parse_date(self.jsonPlan.dateModified)
                             .replace(tzinfo=None))
        plannerUsers = User.objects.filter(username=self.jsonPlan.creator)
        if plannerUsers:
            self.creator = plannerUsers[0]
        else:
            self.creator = None

        # fill in stats
        try:
            exporter = statsPlanExporter.StatsPlanExporter()
#             print ' about to do stats'
            stats = exporter.exportDbPlan(self)
            for f in ('numStations', 'numSegments', 'numCommands', 'lengthMeters'):
                setattr(self, f, stats[f])
            self.stats.numCommandsByType = stats["numCommandsByType"]
            self.summary = statsPlanExporter.getSummary(stats)
        except:
            logging.warning('extractFromJson: could not extract stats from plan %s',
                            self.uuid)
            raise  # FIX
        return self

    def getSummaryOfCommandsByType(self):
        return statsPlanExporter.getSummaryOfCommandsByType(self.stats)

    # TODO test
    def toXpjson(self):
        platform = self.jsonPlan['platform']
        if platform:
            planSchema = getPlanSchema(platform[u'name'])
            return xpjson.loadDocumentFromDict(self.jsonPlan,
                                               schema=planSchema.getSchema())
        logging.warning('toXpjson: could not convert to xpjson, probably no schema %s',
                        self.uuid)
        raise  # FIX

    def escapedName(self):
        name = re.sub(r'[^\w]', '', self.name)
        if name == '':
            return 'plan'
        else:
            if self.jsonPlan and self.jsonPlan.planVersion:
                return name + "_" + self.jsonPlan.planVersion
            return name

    def getExportUrl(self, extension):
        return reverse('planner2_planExport',
                       kwargs={'uuid': self.uuid, 'name': self.escapedName() + extension})

    def getExporters(self):
        import choosePlanExporter  # delayed import avoids import loop
        result = []
        for exporterInfo in choosePlanExporter.PLAN_EXPORTERS:
            info = copy.deepcopy(exporterInfo)
            info.url = self.getExportUrl(info.extension)
            result.append(info)
        return result

    def getLinks(self):
        """
        The links tab wil be populated with the name, value contents of this dictionary as links,
        name is the string displayed and link is what will be opened
        """
        result = {}
        result["KML"] = reverse('planner2_planExport', kwargs={'uuid': self.uuid, 'name': self.name + '.kml'})
        for exporter in self.getExporters():
            result[exporter.label] = exporter.url
        return result

    def getEscapedId(self):
        if self.jsonPlan and self.jsonPlan.id:
            result = re.sub(r'[^\w]', '', self.jsonPlan.id)
            result = re.sub('_PLAN$', '', result)
            return result
        else:
            return None

    def toMapDict(self):
        """
        Return a reduced dictionary that will be turned to JSON for rendering in a map
        Here we are just interested in the route plan and not in activities
        We just include stations
        """
        result = {}
        result['id'] = self.uuid
        result['author'] = self.jsonPlan.creator
        result['name'] = self.jsonPlan.name
        result['type'] = 'AbstractPlan'
        if self.jsonPlan.notes:
            result['notes'] = self.jsonPlan.notes
        else:
            result['notes'] = ''

        stations = []
        seq = self.jsonPlan.sequence
        for el in seq:
            if el.type == "Station":
                sta = {}
                sta['id'] = el.id
                sta['coords'] = el.geometry.coordinates
                if el.notes:
                    sta['notes'] = el.notes
                stations.append(sta)
        result['stations'] = stations
        return result

    def getTreeJson(self):
        result = {"title": self.name,
                  "key": self.uuid,
                  "tooltip": self.summary,
                  "data": {"type": "MapLink",  # we cheat so this will be 'live'
                           "json": reverse('planner2_mapJsonPlan', kwargs={'uuid': str(self.uuid)}),
                           "url": reverse('planner2_edit', kwargs={'plan_id': str(self.pk)})
                           }
                  }
        return result

    @property
    def executions(self):
        return LazyGetModelByName(settings.XGDS_PLANNER2_PLAN_EXECUTION_MODEL).get().objects.filter(plan=self)

    def __unicode__(self):
        if self.name:
            return self.name
        else:
            return 'Unnamed plan ' + self.uuid

class Plan(AbstractPlan):
    pass

# PlanSchema used to be a database model, but is now a normal Python
# class built from the Django settings.  This will not change during
# runtime so we should cache these in PLAN_SCHEMA_CACHE.
class PlanSchema:
    def __init__(self, platform, schemaDict):
        self.platform = platform
        self.schemaSource = settings.PROJ_ROOT + schemaDict['schemaSource']
        self.librarySource = settings.PROJ_ROOT + schemaDict['librarySource']
        self.simulatorUrl = settings.STATIC_URL + schemaDict['simulatorUrl']
        self.simulator = schemaDict['simulator']

        schemaSuffix = os.path.join('xgds_planner2', os.path.basename(self.schemaSource))
        librarySuffix = os.path.join('xgds_planner2', os.path.basename(self.librarySource))

        self.simplifiedSchemaPath = os.path.join(settings.STATIC_ROOT, schemaSuffix)
        self.simplifiedLibraryPath = os.path.join(settings.STATIC_ROOT, librarySuffix)

        self.schemaUrl = os.path.join(settings.STATIC_URL, schemaSuffix)
        self.libraryUrl = os.path.join(settings.STATIC_URL, librarySuffix)

        self.schema = None
        self.library = None
        self.jsonSchema = None
        self.jsonLibrary = None

    def getJsonSchema(self):
        if not self.jsonSchema:
            try:
                with open(self.simplifiedSchemaPath) as schemaFile:
                    self.jsonSchema = schemaFile.read()
            except:  # pylint: disable=W0702
                logging.warning('could not load XPJSON schema from ' + self.simplifiedSchemaPath)
                raise
        return self.jsonSchema

    def getSchema(self):
        if not self.schema:
            try:
                self.schema = xpjson.loadDocument(self.simplifiedSchemaPath)
            except:  # pylint: disable=W0702
                logging.warning('could not load XPJSON schema from ' + self.simplifiedSchemaPath)
                raise
        return self.schema

    def getJsonLibrary(self):
        if not self.jsonLibrary:
            try:
                with open(self.simplifiedLibraryPath) as libraryFile:
                    self.jsonLibrary = libraryFile.read()
            except:  # pylint: disable=W0702
                logging.warning('could not load XPJSON library from ' + self.simplifiedLibraryPath)
                raise
        return self.jsonLibrary

    def getLibrary(self):
        if not self.library:
            try:
                self.library = xpjson.loadDocument(self.simplifiedLibraryPath,
                                                   schema=self.getSchema(),
                                                   fillInDefaults=True)
            except:  # pylint: disable=W0702
                logging.warning('could not load XPJSON library from ' + self.simplifiedLibraryPath)
                raise
        return self.library

def loadSchema(platform):
    schemaDict = settings.XGDS_PLANNER_SCHEMAS[platform]
    schema = PlanSchema(platform, schemaDict)
    schema.getSchema()
    schema.getJsonSchema()
    schema.getLibrary()
    schema.getJsonLibrary()
    return schema

# get the cached plan schema, building it if need be.
def getPlanSchema(platform):
    result = PLAN_SCHEMA_CACHE.get(platform)
    if not result:
        try:
            result = loadSchema(platform)
            PLAN_SCHEMA_CACHE[platform] = result
        except:
            logging.warning('could not find plan schema for platform %s', platform)
            raise
    return result
<|endoftext|>


<commit_before>from ConfigParser import RawConfigParser as ConfigParser
import json
import os.path

import nltk
import numpy as np
from scipy.stats import itemfreq

from vsm import *
from codecs import open
from unidecode import unidecode

def stop_language(c, language):
    words = nltk.corpus.stopwords.words(language)
    words = [word for word in words if word in c.words]
    return c.apply_stoplist(words)

def main(args):
    config = ConfigParser({"htrc": False})
    config.read(args.config_file)
    args.corpus_path = config.get("main", "corpus_file")

    c = Corpus.load(args.corpus_path)

    # NLTK Langauges
    langs = dict(da='danish', nl='dutch', en='english', fi='finnish', fr='french',
                 de='german', hu='hungarian', it='italian', no='norwegian',
                 pt='portuguese', ru='russian', es='spanish', sv='swedish',
                 tr='turkish')

    langs_rev = dict((v, k) for k, v in langs.items())

    if args.lang is None:
        args.lang = []

    # check for htrc metadata
    if args.htrc or config.get("main","htrc"):
        metadata_path = os.path.dirname(args.corpus_path)
        metadata_path = os.path.join(metadata_path, '../metadata.json')
        if os.path.exists(metadata_path):
            print "HTRC metadata file found!"
            with open(metadata_path) as jsonfile:
                data = json.load(jsonfile)

            md_langs = set([lang for d in data.values() for lang in d.get('language', list())
                if lang.lower() in langs.values()])

            print "Stoplist the following languages?"
            for lang in md_langs:
                accept = None
                while accept not in ['y','n']:
                    accept = raw_input(lang + "? [y/n] ")
                if accept == 'y':
                    code = langs_rev[lang.lower()]
                    if code not in args.lang:
                        args.lang.append(code)

    if args.lang is None:
        args.lang = []
    for lang in args.lang:
        print "Applying", langs[lang], "stopwords"
        c = stop_language(c, langs[lang])

    if args.stopword_file:
        print "Applying custom stopword file"
        with open(args.stopword_file, encoding='utf8') as swf:
            c = c.apply_stoplist([unidecode(word.strip()) for word in swf])

    print "\n\n*** FILTER HIGH FREQUENCY WORDS ***"
    items=itemfreq(c.corpus)
    counts = items[:,1]
    high_filter = False
    while not high_filter:
        bin_counts, bins = np.histogram(items[:,1][items[:,1].argsort()[::-1]], range=(0,len(c.words)/4.))
        print "Freq   ",
        for bin in bins:
            print "%10.0f" % bin,
        print "\nWords   ",
        for count in bin_counts:
            print "%10.0f" % count,
        print "\n"
        print counts.sum(), "total words"

        input_filter = 0
        while not input_filter:
            try:
                input_filter = int(raw_input("Enter a top filter: "))
                candidates = c.words[items[:,0][counts > input_filter][counts[counts > input_filter].argsort()[::-1]]]

                print counts[counts > input_filter].sum(), "Words"
                print ' '.join(candidates)
                print counts[counts > input_filter].sum(), "Words"

                accept = None
                while accept not in ['y', 'n', 'c']:
                    accept = raw_input("\nAccept filter? [y/n/c] ")
                    if accept == 'y':
                        high_filter = input_filter
                    if accept == 'c':
                        high_filter = -1
            except ValueError:
                input_filter = 0

    if high_filter > 0:
        print "Applying frequency filter > ", high_filter
        c = c.apply_stoplist(candidates)

    dirname = os.path.basename(args.corpus_path).split('-nltk-')[0]
    lowfreq = os.path.basename(args.corpus_path).split('-freq')[1].split('-')[0].split('.npz')[0]
    def name_corpus(dirname, languages, lowfreq=5, highfreq=None):
        corpus_name = [dirname]
        if args.lang:
            corpus_name.append('nltk')
            corpus_name.append(''.join(args.lang))
        if lowfreq:
            corpus_name.append('freq%s'%lowfreq)
        if highfreq > 0:
            corpus_name.append('N%s'%highfreq)
        corpus_name = '-'.join(corpus_name)
        corpus_name += '.npz'
        return corpus_name

    corpus_name = name_corpus(dirname, ['en'], lowfreq, high_filter)
    model_path = os.path.dirname(args.corpus_path)

    args.corpus_path = os.path.join(model_path, corpus_name)
    c.save(args.corpus_path)

    config.set("main", "corpus_file", args.corpus_path)
    with open(args.config_file, 'wb') as configfh:
        config.write(configfh)

if __name__ == '__main__':
    from argparse import ArgumentParser
    parser = ArgumentParser()
    parser_prep.add_argument("config_file", help="Path to Config",
        type=lambda x: is_valid_filepath(parser_prep, x))
    parser.add_argument("--htrc", action="store_true")
    parser.add_argument("--stopword-file", dest="stopword_file",
        help="File with custom stopwords")
    parser.add_argument("--lang", nargs='+', help="Languages to stoplist")
    args = parser.parse_args()

    main(args)
<commit_msg>adding better frequency table<commit_after>from ConfigParser import RawConfigParser as ConfigParser
import json
import os.path

import nltk
import numpy as np
from scipy.stats import itemfreq

from vsm import *
from codecs import open
from unidecode import unidecode

def stop_language(c, language):
    words = nltk.corpus.stopwords.words(language)
    words = [word for word in words if word in c.words]
    return c.apply_stoplist(words)

def main(args):
    config = ConfigParser({"htrc": False})
    config.read(args.config_file)
    args.corpus_path = config.get("main", "corpus_file")

    c = Corpus.load(args.corpus_path)

    # NLTK Langauges
    langs = dict(da='danish', nl='dutch', en='english', fi='finnish', fr='french',
                 de='german', hu='hungarian', it='italian', no='norwegian',
                 pt='portuguese', ru='russian', es='spanish', sv='swedish',
                 tr='turkish')

    langs_rev = dict((v, k) for k, v in langs.items())

    if args.lang is None:
        args.lang = []

    # check for htrc metadata
    if args.htrc or config.get("main","htrc"):
        metadata_path = os.path.dirname(args.corpus_path)
        metadata_path = os.path.join(metadata_path, '../metadata.json')
        if os.path.exists(metadata_path):
            print "HTRC metadata file found!"
            with open(metadata_path) as jsonfile:
                data = json.load(jsonfile)

            md_langs = set([lang for d in data.values() for lang in d.get('language', list())
                if lang.lower() in langs.values()])

            print "Stoplist the following languages?"
            for lang in md_langs:
                accept = None
                while accept not in ['y','n']:
                    accept = raw_input(lang + "? [y/n] ")
                if accept == 'y':
                    code = langs_rev[lang.lower()]
                    if code not in args.lang:
                        args.lang.append(code)

    if args.lang is None:
        args.lang = []
    for lang in args.lang:
        print "Applying", langs[lang], "stopwords"
        c = stop_language(c, langs[lang])

    if args.stopword_file:
        print "Applying custom stopword file"
        with open(args.stopword_file, encoding='utf8') as swf:
            c = c.apply_stoplist([unidecode(word.strip()) for word in swf])

    print "\n\n*** FILTER HIGH FREQUENCY WORDS ***"
    items=itemfreq(c.corpus)
    counts = items[:,1]
    high_filter = False
    while not high_filter:
        bin_counts, bins = np.histogram(counts[counts.argsort()[::-1]], range=(0,len(c.words)/4.))
	print "{0:>10s} {1:>10s}".format("# Tokens", "# Words")
	for bin, count in zip(bins[1:], bin_counts):
	    print "{0:10.0f} {1:10.0f}".format(bin, count)
        print counts.sum(), "total tokens"
	print len(c.words), "total words"

        input_filter = 0
        while not input_filter:
            try:
                input_filter = int(raw_input("Enter a top filter: "))
                candidates = c.words[items[:,0][counts > input_filter][counts[counts > input_filter].argsort()[::-1]]]

                print "filtering", counts[counts > input_filter].sum(), "tokens", "of these", len(counts[counts > input_filter]), "words:"
                print ' '.join(candidates)

                accept = None
                while accept not in ['y', 'n', 'c']:
                    accept = raw_input("\nAccept filter? [y/n/c] ")
                    if accept == 'y':
                        high_filter = input_filter
                    if accept == 'c':
                        high_filter = -1
            except ValueError:
                input_filter = 0

    if high_filter > 0:
        print "Applying frequency filter > ", high_filter
        c = c.apply_stoplist(candidates)

    dirname = os.path.basename(args.corpus_path).split('-nltk-')[0]
    lowfreq = os.path.basename(args.corpus_path).split('-freq')[1].split('-')[0].split('.npz')[0]
    def name_corpus(dirname, languages, lowfreq=5, highfreq=None):
        corpus_name = [dirname]
        if args.lang:
            corpus_name.append('nltk')
            corpus_name.append(''.join(args.lang))
        if lowfreq:
            corpus_name.append('freq%s'%lowfreq)
        if highfreq > 0:
            corpus_name.append('N%s'%highfreq)
        corpus_name = '-'.join(corpus_name)
        corpus_name += '.npz'
        return corpus_name

    corpus_name = name_corpus(dirname, ['en'], lowfreq, high_filter)
    model_path = os.path.dirname(args.corpus_path)

    args.corpus_path = os.path.join(model_path, corpus_name)
    c.save(args.corpus_path)

    config.set("main", "corpus_file", args.corpus_path)
    with open(args.config_file, 'wb') as configfh:
        config.write(configfh)

if __name__ == '__main__':
    from argparse import ArgumentParser
    parser = ArgumentParser()
    parser_prep.add_argument("config_file", help="Path to Config",
        type=lambda x: is_valid_filepath(parser_prep, x))
    parser.add_argument("--htrc", action="store_true")
    parser.add_argument("--stopword-file", dest="stopword_file",
        help="File with custom stopwords")
    parser.add_argument("--lang", nargs='+', help="Languages to stoplist")
    args = parser.parse_args()

    main(args)
<|endoftext|>


<commit_before># Copyright 2014 The Chromium Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

PACKAGES = [
  {
    'name' : 'core-elements',
    'package_dependencies' : [],
  },
  {
    'name' : 'paper-elements',
    'package_dependencies' : ['core-elements'],
  },
  {
    'name' : 'discovery_api_dart_client_generator',
    'package_dependencies' : [],
  },
]
<commit_msg>Add protobuf package<commit_after># Copyright 2014 The Chromium Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

PACKAGES = [
  {
    'name' : 'core-elements',
    'package_dependencies' : [],
  },
  {
    'name' : 'paper-elements',
    'package_dependencies' : ['core-elements'],
  },
  {
    'name' : 'dart-protobuf',
    'package_dependencies' : [],
  },
]
<|endoftext|>


<commit_before>import math
from bigfloat import *

import _norm
import _angular_dist
import Quadtree_Utilities as utils

MAX = 70
class Quadtree(object):
    """
    Quadtree base class. Only functions that are agnostic to
    the type of coordinate system or source object used. Must
    use a subclass.
    """
    def __init__(self, xmin, ymin, xmax, ymax):
        self.top = Node(xmin, ymin, xmax, ymax)
        self.num_subdivides = 0
        self.num_insert = 0
        self.num_inserttonodes = 0
        self.num_matched = 0
        self.num_inserttoquads = 0
        self.num_nearersources = 0

    def debug(self):
        print "Number of subdivides: %d" % self.num_subdivides
        print "Inserttonode was called %d times" % self.num_inserttonodes
        print "Matched was called %d times" % self.num_matched
        print "Inserttoquad was called %d times" % self.num_inserttoquads
        print "Nearer sources was called %d times" % self.num_nearersources
        print "Insert was called %d times" % self.num_insert

    def inserttonode(self, node, source):
        self.num_inserttonodes+=1
        if len(node.contents) == MAX:
            self.subdivide(node)
        if node.q1:
            self.inserttoquad(node, source)
        else:
            # If no subquads exist add source to the list in CONTENTS element
            node.contents.append(source)

    def inserttoquad(self, node, source):
        self.num_inserttoquads+=1
        if source.x >= node.xmid:
            if source.y >= node.ymid:
                quadrant = node.q1
            else:
                quadrant = node.q4
        else:
            if source.y >= node.ymid:
                quadrant = node.q2
            else:
                quadrant = node.q3
        self.inserttonode(quadrant, source)

    def subdivide(self, node):
        self.num_subdivides+=1
        node.q1 = Node(node.xmid, node.ymid, node.xmax, node.ymax)
        node.q2 = Node(node.xmin, node.ymid, node.xmid, node.ymax)
        node.q3 = Node(node.xmin, node.ymin, node.xmid, node.ymid)
        node.q4 = Node(node.xmid, node.ymin, node.xmax, node.ymid)
        # Pop the list and insert the sources as they come off
        while node.contents:
            self.inserttoquad(node, node.contents.pop())

    def match(self, x, y):
        self.num_matched+=1
        dist = self.initial_dist(self.top.xmax, self.top.xmin,
                                 self.top.ymax, self.top.ymin)
        nearest = utils.Nearest(dist*dist)

        interest = utils.Interest(x, y, dist, self.top)

        self.nearersource(self.top, interest, nearest)
        return nearest.source

    def nearersource(self, node, interest, nearest):
        self.num_nearersources+=1
        if interest.intersect(node):
            if node.q1 == None:
               for s in node.contents:
                    dist2 = self.norm2(s.x, s.y, interest.tx, interest.ty)
                    if dist2 < nearest.dist2:
                        nearest.source = s.source
                        nearest.dist2 = dist2
                        interest.update(sqrt(dist2))

            else:
                self.nearersource(node.q1, interest, nearest)
                self.nearersource(node.q2, interest, nearest)
                self.nearersource(node.q3, interest, nearest)
                self.nearersource(node.q4, interest, nearest)

class Node(object):
    def __init__(self, xmin, ymin, xmax, ymax):
        self.xmin = BigFloat(xmin)
        self.ymin = BigFloat(ymin)
        self.xmax = BigFloat(xmax)
        self.ymax = BigFloat(ymax)
        self.xmid = BigFloat((self.xmin + self.xmax)/2.0)
        self.ymid = BigFloat((self.ymin + self.ymax)/2.0)
        self.q1 = self.q2 = self.q3 = self.q4 = None
        self.contents = []

class Point(object):
    """
    The point of Point (heh.) is to have a uniform object that
    can be passed around the Quadtree. This makes for
    easy switching between equatorial and pixel coordinate
    systems or different objects.
    """
    def __init__(self, source, x, y):
        self.source = source
        self.x = BigFloat(x)
        self.y = BigFloat(y)

class ScamPixelQuadtree(Quadtree):
    def __init__(self, xmin, ymin, xmax, ymax):
        super(ScamPixelQuadtree, self).__init__(xmin, ymin, xmax, ymax)

    def insert(self, source):
        self.num_insert+=1
        self.inserttonode(self.top, Point(source, source.ximg, source.yimg))

    def norm2(self, x1, y1, x2, y2):
        return _norm.norm2(x1, y1, x2, y2)

    def initial_dist(self, x2, x1, y2, y1):
        return  (BigFloat(x2) - BigFloat(x1), BigFloat(y2) - BigFloat(y1))/1000.0

class ScamEquatorialQuadtree(Quadtree):
    def __init__(self, xmin, ymin, xmax, ymax):
        super(ScamEquatorialQuadtree, self).__init__(xmin, ymin, xmax, ymax)

    def insert(self, source):
        self.num_insert+=1
        self.inserttonode(self.top, Point(source, source.ra, source.dec))

    def norm2(self, x1, y1, x2, y2):
        return _angular_dist.angular_dist2(x1, y1, x2, y2)

    def initial_dist(self, x2, x1, y2, y1):
        return  min(BigFloat(x2) - BigFloat(x1), BigFloat(y2) - BigFloat(y1))/100.0

<commit_msg>Pixel matching seems to be working, something still wrong with the equatorial coordinates<commit_after>import math
#from bigfloat import *

import _norm
import _angular_dist
import Quadtree_Utilities as utils

MAX = 70
class Quadtree(object):
    """
    Quadtree base class. Only functions that are agnostic to
    the type of coordinate system or source object used. Must
    use a subclass.
    """
    def __init__(self, xmin, ymin, xmax, ymax):
        self.top = Node(xmin, ymin, xmax, ymax)
        self.num_subdivides = 0
        self.num_insert = 0
        self.num_inserttonodes = 0
        self.num_matched = 0
        self.num_inserttoquads = 0
        self.num_nearersources = 0

    def debug(self):
        print "Number of subdivides: %d" % self.num_subdivides
        print "Inserttonode was called %d times" % self.num_inserttonodes
        print "Matched was called %d times" % self.num_matched
        print "Inserttoquad was called %d times" % self.num_inserttoquads
        print "Nearer sources was called %d times" % self.num_nearersources
        print "Insert was called %d times" % self.num_insert

    def inserttonode(self, node, source):
        self.num_inserttonodes+=1
        if len(node.contents) == MAX:
            self.subdivide(node)
        if node.q1:
            self.inserttoquad(node, source)
        else:
            # If no subquads exist add source to the list in CONTENTS element
            node.contents.append(source)

    def inserttoquad(self, node, source):
        self.num_inserttoquads+=1
        if source.x >= node.xmid:
            if source.y >= node.ymid:
                quadrant = node.q1
            else:
                quadrant = node.q4
        else:
            if source.y >= node.ymid:
                quadrant = node.q2
            else:
                quadrant = node.q3
        self.inserttonode(quadrant, source)

    def subdivide(self, node):
        self.num_subdivides+=1
        node.q1 = Node(node.xmid, node.ymid, node.xmax, node.ymax)
        node.q2 = Node(node.xmin, node.ymid, node.xmid, node.ymax)
        node.q3 = Node(node.xmin, node.ymin, node.xmid, node.ymid)
        node.q4 = Node(node.xmid, node.ymin, node.xmax, node.ymid)
        # Pop the list and insert the sources as they come off
        while node.contents:
            self.inserttoquad(node, node.contents.pop())

    def match(self, x, y):
        self.num_matched+=1
        dist = self.initial_dist(self.top.xmax, self.top.xmin,
                                 self.top.ymax, self.top.ymin)
        nearest = utils.Nearest(dist*dist)

        interest = utils.Interest(x, y, dist, self.top)

        self.nearersource(self.top, interest, nearest)
        return nearest.source

    def nearersource(self, node, interest, nearest):
        self.num_nearersources+=1
        if interest.intersect(node):
            print "Intersecting Quadtrant: "
            if node.q1 == None:
               for s in node.contents:
                    dist2 = self.norm2(s.x, s.y, interest.tx, interest.ty)
                    if dist2 < nearest.dist2:
                        print "     Searching"
                        print "     ", dist2
                        nearest.source = s.source
                        nearest.dist2 = dist2
                        interest.update(math.sqrt(dist2))
            else:
                self.nearersource(node.q1, interest, nearest)
                self.nearersource(node.q2, interest, nearest)
                self.nearersource(node.q3, interest, nearest)
                self.nearersource(node.q4, interest, nearest)

class Node(object):
    def __init__(self, xmin, ymin, xmax, ymax):
        self.xmin = (xmin)
        self.ymin = (ymin)
        self.xmax = (xmax)
        self.ymax = (ymax)
        self.xmid = ((self.xmin + self.xmax)/2.0)
        self.ymid = ((self.ymin + self.ymax)/2.0)
        self.q1 = self.q2 = self.q3 = self.q4 = None
        self.contents = []

class Point(object):
    """
    The point of Point (heh.) is to have a uniform object that
    can be passed around the Quadtree. This makes for
    easy switching between equatorial and pixel coordinate
    systems or different objects.
    """
    def __init__(self, source, x, y):
        self.source = source
        self.x = (x)
        self.y = (y)

class ScamPixelQuadtree(Quadtree):
    def __init__(self, xmin, ymin, xmax, ymax):
        super(ScamPixelQuadtree, self).__init__(xmin, ymin, xmax, ymax)

    def insert(self, source):
        self.num_insert+=1
        self.inserttonode(self.top, Point(source, source.ximg, source.yimg))

    def norm2(self, x1, y1, x2, y2):
        return _norm.norm2(x1, y1, x2, y2)

    def initial_dist(self, x2, x1, y2, y1):
        return  min((x2) - (x1), (y2) - (y1))

class ScamEquatorialQuadtree(Quadtree):
    def __init__(self, xmin, ymin, xmax, ymax):
        super(ScamEquatorialQuadtree, self).__init__(xmin, ymin, xmax, ymax)

    def insert(self, source):
        self.num_insert+=1
        self.inserttonode(self.top, Point(source, source.ra, source.dec))

    def norm2(self, x1, y1, x2, y2):
        return _angular_dist.angular_dist2(x1, y1, x2, y2)

    def initial_dist(self, x2, x1, y2, y1):
        return  min((x2) - (x1), (y2) - (y1))

<|endoftext|>


<commit_before>from __future__ import print_function, absolute_import, division

import sys
import click

CONTEXT_SETTINGS = dict(help_option_names=['-h', '--help'])

def main():
    try:
        cli(obj={})
    except Exception as e:
        import traceback
        click.echo(traceback.format_exc(), err=True)
        sys.exit(1)

@click.group(context_settings=CONTEXT_SETTINGS)
@click.option('--namenode', '-n', default='localhost', required=False, help='Namenode host', show_default=True)
@click.option('--port', '-p', default=8020, required=False, help='Namenode port', show_default=True)
@click.pass_context
def cli(ctx, namenode, port):
    # import pyximport; pyximport.install()
    import os
    import cyhdfs3
    os.environ["LIBHDFS3_CONF"] = "/etc/hadoop/conf/hdfs-site.xml"
    ctx.obj = {}
    ctx.obj['client'] = cyhdfs3.HDFSClient()

@cli.command(short_help='Change mode of a file or directory.')
@click.argument('mode', required=True)
@click.argument('path', required=True)
@click.pass_context
def chmod(ctx, mode, path):
    client = ctx.obj['client']
    client.chmod_s(path.encode('UTF-8'), mode.encode('UTF-8'))

@cli.command(short_help='Change owner of a file or directory.')
@click.argument('owner', required=True)
@click.argument('path', required=True)
@click.pass_context
def chown(ctx, owner, path):
    client = ctx.obj['client']
    client.chown(path.encode('UTF-8'), owner=owner.encode('UTF-8'))

@cli.command(short_help='Change group of a file or directory.')
@click.argument('group', required=True)
@click.argument('path', required=True)
@click.pass_context
def chgrp(ctx, group, path):
    client = ctx.obj['client']
    client.chown(path.encode('UTF-8'), group=group.encode('UTF-8'))

@cli.command(short_help='Display the first (n) bytes of a file')
@click.argument('path', required=False, default='/')
@click.option('--bytes', '-b', 'nbytes', default=1*2**10, required=False, help='Number of bytes', show_default=True)
@click.pass_context
def head(ctx, path, nbytes):
    client = ctx.obj['client']
    try:
        with client.open(path, 'r') as f:
            click.echo(f.read(length=nbytes))
    except IOError:
        if client.path_info(path).kind == 'd':
            click.echo("head: error reading '{}': Is a directory".format(path), err=True)
            sys.exit(1)

@cli.command(short_help='Display the last (n) bytes of a file')
@click.argument('path', required=False, default='/')
@click.option('--bytes', '-b', 'nbytes', default=1*2**10, required=False, help='Number of bytes', show_default=True)
@click.pass_context
def tail(ctx, path, nbytes):
    client = ctx.obj['client']
    try:
        with client.open(path, 'r') as f:
            start = max(0, f.info.size - nbytes)
            f.seek(start)
            click.echo(f.read(length=nbytes))
    except IOError as e:
        if client.path_info(path).kind == 'd':
            click.echo("tail: error reading '{}': Is a directory".format(path), err=True)

@cli.command(short_help='Create directory and all non-existent parents')
@click.argument('path', required=False, default='/')
@click.pass_context
def mkdir(ctx, path):
    client = ctx.obj['client']
    client.create_dir(path)

@cli.command(short_help='List a path')
@click.argument('path', required=False, default='/')
@click.option('--recurse', '-R', is_flag=True, default=False, required=False, help='Recurse into subdirectories', show_default=True)
@click.pass_context
def ls(ctx, path, recurse):
    import datetime
    client = ctx.obj['client']

    if not client.exists(path):
        click.echo("ls: cannot access {}: No such file or directory".format(path))
        sys.exit(2)

    files = client.list_dir(path, recurse=recurse)
    click.echo("Found {} files".format(len(files)))
    for f in files:
        row = []
        perm = octal_to_perm(f.permissions)
        t_perm = ('d' if f.kind == 'd' else '-') + perm
        row.append(t_perm)

        s = max([len(str(j.replication)) for j in files]) + 1
        r = '-' if f.kind == 'd' else str(f.replication)
        row.append(r.rjust(s))
        s = max([len(j.owner) for j in files]) + 1
        row.append(f.owner.ljust(s))
        s = max([len(j.group) for j in files]) + 1
        row.append(f.group.ljust(s))
        s = max([len(str(j.size)) for j in files]) + 1
        row.append(str(f.size).ljust(s))
        d = lambda x: datetime.datetime.fromtimestamp(x).strftime("%Y-%m-%d %H:%M")
        s = max([len(d(j.lastMod)) for j in files]) + 1
        row.append(d(f.lastMod).ljust(s))
        row.append(f.name)
        click.echo(" ".join(row))

def octal_to_perm(octal):
    import stat
    perms = list("-" * 9)
    if octal & stat.S_IRUSR:
        perms[0] = "r"
    if octal & stat.S_IWUSR:
        perms[1] = "w"
    if octal & stat.S_IXUSR:
        perms[2] = "x"
    if octal & stat.S_IRGRP:
        perms[3] = "r"
    if octal & stat.S_IWGRP:
        perms[4] = "w"
    if octal & stat.S_IXGRP:
        perms[5] = "x"
    if octal & stat.S_IROTH:
        perms[6] = "r"
    if octal & stat.S_IWOTH:
        perms[7] = "w"
    if octal & stat.S_IXOTH:
        perms[8] = "x"
    return "".join(perms)

@cli.command(short_help='Display fs stats')
@click.pass_context
def df(ctx):
    client = ctx.obj['client']

    fs = "hdfs://{}:{}".format(client.host, client.port)
    used = client.get_used()
    capacity = client.get_capacity()
    avalable = capacity - used
    block_size = client.get_default_block_size()
    use_p = "%.2f" % (used / capacity)

    headers = ["Filesystem", "Block Size", "Size", "Used", "Available", "Use%"]
    row1 = [fs, block_size, capacity, used, avalable, use_p]
    cols_lenghts = []

    for header, row in zip(headers, row1):
        cols_lenghts.append(max([len(str(_)) for _ in [header, row]]) + 1)
    for row in [headers, row1]:
        lrow = []
        for i, col in enumerate(row):
            if i == 0:
                lrow.append(str(col).ljust(cols_lenghts[i]))
            else:
                lrow.append(str(col).rjust(cols_lenghts[i]))
        click.echo(" ".join(lrow))

if __name__ == '__main__':
    main()
<commit_msg>Add put and get<commit_after>from __future__ import print_function, absolute_import, division

import sys
import click

CONTEXT_SETTINGS = dict(help_option_names=['-h', '--help'])

def main():
    try:
        cli(obj={})
    except Exception as e:
        import traceback
        click.echo(traceback.format_exc(), err=True)
        sys.exit(1)

@click.group(context_settings=CONTEXT_SETTINGS)
@click.option('--namenode', '-n', default='localhost', required=False, help='Namenode host', show_default=True)
@click.option('--port', '-p', default=8020, required=False, help='Namenode port', show_default=True)
@click.pass_context
def cli(ctx, namenode, port):
    import os
    import cyhdfs3
    os.environ["LIBHDFS3_CONF"] = "/etc/hadoop/conf/hdfs-site.xml"
    ctx.obj = {}
    ctx.obj['client'] = cyhdfs3.HDFSClient()

@cli.command(short_help='Copy files to local file system destination')
@click.argument('path', required=True)
@click.argument('localpath', required=True)
@click.option('--buffer', '-b', 'buffersize', default=1*2**10, required=False, help='Buffer size', show_default=True)
@click.pass_context
def get(ctx, path, localpath, buffersize):
    client = ctx.obj['client']
    with open(localpath, 'w') as lf:
        with client.open(path, 'r') as hdfsf:
            data = ''
            while data is not None:
                data = hdfsf.read(length=buffersize)
                if data is not None:
                    lf.write(data.decode('utf-8'))

@cli.command(short_help='Copy files to local file system destination')
@click.argument('localpath', required=True)
@click.argument('path', required=True)
@click.option('--buffer', '-b', 'buffersize', default=1*2**10, required=False, help='Buffer size', show_default=True)
@click.pass_context
def put(ctx, localpath, path, buffersize):
    client = ctx.obj['client']
    with client.open(path, 'w') as hdfsf:
        with open(localpath, 'rb') as lf:
            while True:
                data = lf.read(buffersize)
                if not data:
                    break
                hdfsf.write(data)

@cli.command(short_help='Change mode of a file or directory.')
@click.argument('mode', required=True)
@click.argument('path', required=True)
@click.pass_context
def chmod(ctx, mode, path):
    client = ctx.obj['client']
    client.chmod_s(path.encode('UTF-8'), mode.encode('UTF-8'))

@cli.command(short_help='Change owner of a file or directory.')
@click.argument('owner', required=True)
@click.argument('path', required=True)
@click.pass_context
def chown(ctx, owner, path):
    client = ctx.obj['client']
    client.chown(path.encode('UTF-8'), owner=owner.encode('UTF-8'))

@cli.command(short_help='Change group of a file or directory.')
@click.argument('group', required=True)
@click.argument('path', required=True)
@click.pass_context
def chgrp(ctx, group, path):
    client = ctx.obj['client']
    client.chown(path.encode('UTF-8'), group=group.encode('UTF-8'))

@cli.command(short_help='Display the first (n) bytes of a file')
@click.argument('path', required=False, default='/')
@click.option('--bytes', '-b', 'nbytes', default=1*2**10, required=False, help='Number of bytes', show_default=True)
@click.pass_context
def head(ctx, path, nbytes):
    client = ctx.obj['client']
    try:
        with client.open(path, 'r') as f:
            click.echo(f.read(length=nbytes))
    except IOError:
        if client.path_info(path).kind == 'd':
            click.echo("head: error reading '{}': Is a directory".format(path), err=True)
            sys.exit(1)

@cli.command(short_help='Display the last (n) bytes of a file')
@click.argument('path', required=False, default='/')
@click.option('--bytes', '-b', 'nbytes', default=1*2**10, required=False, help='Number of bytes', show_default=True)
@click.pass_context
def tail(ctx, path, nbytes):
    client = ctx.obj['client']
    try:
        with client.open(path, 'r') as f:
            start = max(0, f.info.size - nbytes)
            f.seek(start)
            click.echo(f.read(length=nbytes))
    except IOError as e:
        if client.path_info(path).kind == 'd':
            click.echo("tail: error reading '{}': Is a directory".format(path), err=True)

@cli.command(short_help='Create directory and all non-existent parents')
@click.argument('path', required=False, default='/')
@click.pass_context
def mkdir(ctx, path):
    client = ctx.obj['client']
    client.create_dir(path)

@cli.command(short_help='List a path')
@click.argument('path', required=False, default='/')
@click.option('--recurse', '-R', is_flag=True, default=False, required=False, help='Recurse into subdirectories', show_default=True)
@click.pass_context
def ls(ctx, path, recurse):
    import datetime
    client = ctx.obj['client']

    if not client.exists(path):
        click.echo("ls: cannot access {}: No such file or directory".format(path))
        sys.exit(2)

    files = client.list_dir(path, recurse=recurse)
    click.echo("Found {} files".format(len(files)))
    for f in files:
        row = []
        perm = octal_to_perm(f.permissions)
        t_perm = ('d' if f.kind == 'd' else '-') + perm
        row.append(t_perm)

        s = max([len(str(j.replication)) for j in files]) + 1
        r = '-' if f.kind == 'd' else str(f.replication)
        row.append(r.rjust(s))
        s = max([len(j.owner) for j in files]) + 1
        row.append(f.owner.ljust(s))
        s = max([len(j.group) for j in files]) + 1
        row.append(f.group.ljust(s))
        s = max([len(str(j.size)) for j in files]) + 1
        row.append(str(f.size).ljust(s))
        d = lambda x: datetime.datetime.fromtimestamp(x).strftime("%Y-%m-%d %H:%M")
        s = max([len(d(j.lastMod)) for j in files]) + 1
        row.append(d(f.lastMod).ljust(s))
        row.append(f.name)
        click.echo(" ".join(row))

def octal_to_perm(octal):
    import stat
    perms = list("-" * 9)
    if octal & stat.S_IRUSR:
        perms[0] = "r"
    if octal & stat.S_IWUSR:
        perms[1] = "w"
    if octal & stat.S_IXUSR:
        perms[2] = "x"
    if octal & stat.S_IRGRP:
        perms[3] = "r"
    if octal & stat.S_IWGRP:
        perms[4] = "w"
    if octal & stat.S_IXGRP:
        perms[5] = "x"
    if octal & stat.S_IROTH:
        perms[6] = "r"
    if octal & stat.S_IWOTH:
        perms[7] = "w"
    if octal & stat.S_IXOTH:
        perms[8] = "x"
    return "".join(perms)

@cli.command(short_help='Display fs stats')
@click.pass_context
def df(ctx):
    client = ctx.obj['client']

    fs = "hdfs://{}:{}".format(client.host, client.port)
    used = client.get_used()
    capacity = client.get_capacity()
    avalable = capacity - used
    block_size = client.get_default_block_size()
    use_p = "%.2f" % (used / capacity)

    headers = ["Filesystem", "Block Size", "Size", "Used", "Available", "Use%"]
    row1 = [fs, block_size, capacity, used, avalable, use_p]
    cols_lenghts = []

    for header, row in zip(headers, row1):
        cols_lenghts.append(max([len(str(_)) for _ in [header, row]]) + 1)
    for row in [headers, row1]:
        lrow = []
        for i, col in enumerate(row):
            if i == 0:
                lrow.append(str(col).ljust(cols_lenghts[i]))
            else:
                lrow.append(str(col).rjust(cols_lenghts[i]))
        click.echo(" ".join(lrow))

if __name__ == '__main__':
    main()
<|endoftext|>


<commit_before># stdlib
from io import BytesIO
from typing import Any
from typing import Generator
from typing import List

# third party
import boto3
from botocore.client import Config
from pydantic import BaseSettings
import requests
from tqdm import tqdm

# relative
from ....grid import GridURL
from ....util import get_fully_qualified_name
from ....util import size_mb
from ...common.serde.serialize import _serialize as serialize
from ...common.uid import UID
from ...store.proxy_dataset import ProxyDataset
from .exceptions import DatasetUploadError

MIN_BLOB_UPLOAD_SIZE_MB = 1

def read_chunks(
    fp: BytesIO, chunk_size: int = 1024**3
) -> Generator[bytes, None, None]:
    """Read data in chunks from the file."""
    while True:
        data = fp.read(chunk_size)
        if not data:
            break
        yield data

def listify(x: Any) -> List[Any]:
    """turns x into a list.
    If x is a list or tuple, return as list.
    if x is not a list: return [x]
    if x is None: return []

    Args:
        x (Any): some object

    Returns:
        List[Any]: x, as a list
    """
    return list(x) if isinstance(x, (list, tuple)) else ([] if x is None else [x])

def upload_result_to_s3(
    asset_name: str,
    dataset_name: str,
    domain_id: UID,
    data: Any,
    settings: BaseSettings,
) -> ProxyDataset:
    """Upload data to Seaweed using boto3 client.

    - Serialize data to binary
    - Upload data to Seaweed using boto3 client
    - Create a ProxyDataset to store the metadata of the data uploaded to Seaweed

    Args:
        asset_name (str): name of the data being uploaded to Seaweed
        dataset_name (str): name of the dataset to which the data belongs
        domain_id (UID): unique id of the domain node
        data (Any): data to be uploaded to Seaweed
        settings (BaseSettings): base settings of the PyGrid server

    Returns:
        ProxyDataset: Class to store the metadata of the data being uploaded
    """

    s3_client = get_s3_client(settings=settings)

    binary_dataset: bytes = serialize(data, to_bytes=True)  # type: ignore

    # 1- Serialize the data to be uploaded to bytes.
    binary_buffer = BytesIO(binary_dataset)
    filename = f"{dataset_name}/{asset_name}"

    # 2 - Start to upload binary data into Seaweed.
    upload_response = s3_client.put_object(
        Bucket=domain_id.no_dash,
        Body=binary_buffer,
        Key=filename,
        ContentType="application/octet-stream",
    )

    # TODO: Throw an exception if the response is not valid
    print("Upload Result")
    print(upload_response)

    # 3 - Create a ProxyDataset for the given data
    # Retrieve fully qualified name to  use for pointer creation.
    obj_public_kwargs = getattr(data, "proxy_public_kwargs", {})
    data_fqn = str(get_fully_qualified_name(data))
    data_dtype = str(type(data))

    # relative
    from ...tensor import Tensor
    from ...tensor.autodp.gamma_tensor import GammaTensor as GT
    from ...tensor.autodp.phi_tensor import PhiTensor as PT

    if isinstance(data, Tensor) and (
        isinstance(data.child, PT) or isinstance(data.child, GT)
    ):
        data_fqn = str(get_fully_qualified_name(data.child))
        child_kwargs = getattr(data.child, "proxy_public_kwargs", {})
        obj_public_kwargs.update(child_kwargs)

    proxy_obj = ProxyDataset(
        asset_name=asset_name,
        dataset_name=dataset_name,
        node_id=domain_id,
        dtype=data_dtype,
        fqn=data_fqn,
        shape=data.shape,
        obj_public_kwargs=obj_public_kwargs,
    )
    return proxy_obj

def abort_s3_object_upload(
    client: boto3.client, upload_id: str, asset_name: str
) -> None:
    """Abort upload to s3 for the given asset.

    Args:
        client (boto3.client): boto3 client.
        upload_id (str): upload id generated for mutlipart upload.
        asset_name (str): name of the data/asset.
    """
    # relative
    from .node_service.upload_service.upload_service_messages import (
        AbortDataUploadMessage,
    )

    # Send a message to PyGrid to abort the data being uploaded to s3.
    # TODO: make this call async -> could use celery .delay method to do
    _ = client.datasets.perform_api_request_generic(
        syft_msg=AbortDataUploadMessage,
        content={
            "upload_id": upload_id,
            "asset_name": asset_name,
        },
    )

def upload_to_s3_using_presigned(
    client: Any,
    data: Any,
    chunk_size: int,
    asset_name: str,
    dataset_name: str = "",
) -> ProxyDataset:
    """Perform a multipart upload of data to Seaweed using boto3 presigned urls.
    The main steps involve:
    - Converting the data to binary data
    - Chunking the binary into smaller chunks
    - Create presigned urls for each chunk
    - Upload data to Seaweed via PUT request
    - Send a acknowledge to Seaweed via PyGrid when all chunks are successfully uploaded
    - Create a ProxyDataset to store metadata of the uploaded data
    Args:
        client (Any): Client to send object to
        data (Any): Data to be uploaded to Seaweed
        chunk_size (int): smallest size of the data to be uploaded in bytes
        asset_name (str): name of the data being uploaded
        dataset_name Optional[(str)]: name of the dataset to which the data belongs
    Raises:
        Exception: If upload of data chunks to Seaweed fails.
    Returns:
        ProxyDataset: Class to store metadata about the data that is uploaded to Seaweed.
    """
    data_upload_description = f"Uploading `{asset_name}`"
    # relative
    from .node_service.upload_service.upload_service_messages import (
        UploadDataCompleteMessage,
    )
    from .node_service.upload_service.upload_service_messages import UploadDataMessage

    dataset_name = str(dataset_name)

    # Step 1 - Convert data to be uploaded to binary
    binary_dataset: bytes = serialize(data, to_bytes=True)  # type: ignore
    file_size = len(binary_dataset)

    # Step 2 - Send a message to PyGrid to inform of the data being uploaded,
    # and get presigned url for each chunk of data being uploaded.
    upload_response = client.datasets.perform_api_request_generic(
        syft_msg=UploadDataMessage,
        content={
            "filename": f"{dataset_name}/{asset_name}",
            "file_size": file_size,
            "chunk_size": chunk_size,
            "address": client.address,
            "reply_to": client.address,
        },
    )

    try:
        # Step 3 - Starts to upload binary data into Seaweed.
        binary_buffer = BytesIO(binary_dataset)
        parts = sorted(upload_response.payload.parts, key=lambda x: x["part_no"])
        etag_chunk_no_pairs = list()
        data_chunks = zip(read_chunks(binary_buffer, chunk_size), parts)
        for _ in tqdm(
            parts, desc=data_upload_description, colour="green", ncols=100, leave=True
        ):
            data_chunk, part = next(data_chunks)
            presigned_url = part["url"]
            part_no = part["part_no"]
            client_url = client.url_from_path(presigned_url)
            part["client_url"] = client_url

            res = requests.put(client_url, data=data_chunk)

            if not res.ok:  # raise an error if upload fails
                error_message = (
                    f"\n\nFailed to upload `{asset_name}` to store\n"
                    + f"Status code: {res.status_code} {res.reason}\n"
                    + f"Error: {str(res.text)}"
                )
                raise DatasetUploadError(message=error_message)
            etag = res.headers["ETag"]
            etag_chunk_no_pairs.append(
                {"ETag": etag, "PartNumber": part_no}
            )  # maintain list of part no and ETag

        # Step 4 - Send a message to PyGrid informing about dataset upload complete!
        upload_response = client.datasets.perform_request(
            syft_msg=UploadDataCompleteMessage,
            content={
                "upload_id": upload_response.payload.upload_id,
                "filename": f"{dataset_name}/{asset_name}",
                "parts": etag_chunk_no_pairs,
            },
        )

        # Step 5 - Create a proxy dataset for the uploaded data.
        # Retrieve fully qualified name to  use for pointer creation.
        obj_public_kwargs = getattr(data, "proxy_public_kwargs", {})
        data_fqn = str(get_fully_qualified_name(data))
        data_dtype = str(type(data))

        # relative
        from ...tensor import Tensor
        from ...tensor.autodp.gamma_tensor import GammaTensor as GT
        from ...tensor.autodp.phi_tensor import PhiTensor as PT

        if isinstance(data, Tensor) and (
            isinstance(data.child, PT) or isinstance(data.child, GT)
        ):
            data_fqn = str(get_fully_qualified_name(data.child))
            child_kwargs = getattr(data.child, "proxy_public_kwargs", {})
            obj_public_kwargs.update(child_kwargs)

        proxy_data = ProxyDataset(
            asset_name=asset_name,
            dataset_name=dataset_name,
            node_id=client.id,
            dtype=data_dtype,
            fqn=data_fqn,
            shape=data.shape,
            obj_public_kwargs=obj_public_kwargs,
        )
    except (Exception, KeyboardInterrupt) as e:
        upload_id = upload_response.payload.upload_id
        abort_s3_object_upload(
            client=client, upload_id=upload_id, asset_name=asset_name
        )
        raise e

    return proxy_data

def get_s3_client(settings: BaseSettings) -> "boto3.client.S3":
    try:
        s3_endpoint = settings.S3_ENDPOINT
        s3_port = settings.S3_PORT
        s3_grid_url = GridURL(host_or_ip=s3_endpoint, port=s3_port)
        return boto3.client(
            "s3",
            endpoint_url=s3_grid_url.url,
            aws_access_key_id=settings.S3_ROOT_USER,
            aws_secret_access_key=settings.S3_ROOT_PWD,
            config=Config(signature_version="s3v4"),
            region_name=settings.S3_REGION,
        )
    except Exception as e:
        print(f"Failed to create S3 Client with {s3_endpoint} {s3_port} {s3_grid_url}")
        raise e

def check_send_to_blob_storage(obj: Any, use_blob_storage: bool = False) -> bool:
    """Check if the data needs to be send to Seaweed storage depending upon its size and type.
    Args:
        obj (Any): Data to be stored to Seaweed.
        use_blob_storage (bool, optional): Explicit flag to send the data to blob storage. Defaults to False.

    Returns:
        bool: Returns True if obj can be stored in blob store else returns False.
    """
    # relative
    from ...tensor import Tensor
    from ...tensor.autodp.phi_tensor import PhiTensor as PT

    if use_blob_storage and (
        isinstance(obj, (PT, Tensor)) or size_mb(obj) > MIN_BLOB_UPLOAD_SIZE_MB
    ):
        return True
    return False
<commit_msg>Fix dtype issue<commit_after># stdlib
from io import BytesIO
from typing import Any
from typing import Generator
from typing import List

# third party
import boto3
from botocore.client import Config
from pydantic import BaseSettings
import requests
from tqdm import tqdm

# relative
from ....grid import GridURL
from ....util import get_fully_qualified_name
from ....util import size_mb
from ...common.serde.serialize import _serialize as serialize
from ...common.uid import UID
from ...store.proxy_dataset import ProxyDataset
from .exceptions import DatasetUploadError

MIN_BLOB_UPLOAD_SIZE_MB = 1

def read_chunks(
    fp: BytesIO, chunk_size: int = 1024**3
) -> Generator[bytes, None, None]:
    """Read data in chunks from the file."""
    while True:
        data = fp.read(chunk_size)
        if not data:
            break
        yield data

def listify(x: Any) -> List[Any]:
    """turns x into a list.
    If x is a list or tuple, return as list.
    if x is not a list: return [x]
    if x is None: return []

    Args:
        x (Any): some object

    Returns:
        List[Any]: x, as a list
    """
    return list(x) if isinstance(x, (list, tuple)) else ([] if x is None else [x])

def upload_result_to_s3(
    asset_name: str,
    dataset_name: str,
    domain_id: UID,
    data: Any,
    settings: BaseSettings,
) -> ProxyDataset:
    """Upload data to Seaweed using boto3 client.

    - Serialize data to binary
    - Upload data to Seaweed using boto3 client
    - Create a ProxyDataset to store the metadata of the data uploaded to Seaweed

    Args:
        asset_name (str): name of the data being uploaded to Seaweed
        dataset_name (str): name of the dataset to which the data belongs
        domain_id (UID): unique id of the domain node
        data (Any): data to be uploaded to Seaweed
        settings (BaseSettings): base settings of the PyGrid server

    Returns:
        ProxyDataset: Class to store the metadata of the data being uploaded
    """

    s3_client = get_s3_client(settings=settings)

    binary_dataset: bytes = serialize(data, to_bytes=True)  # type: ignore

    # 1- Serialize the data to be uploaded to bytes.
    binary_buffer = BytesIO(binary_dataset)
    filename = f"{dataset_name}/{asset_name}"

    # 2 - Start to upload binary data into Seaweed.
    upload_response = s3_client.put_object(
        Bucket=domain_id.no_dash,
        Body=binary_buffer,
        Key=filename,
        ContentType="application/octet-stream",
    )

    # TODO: Throw an exception if the response is not valid
    print("Upload Result")
    print(upload_response)

    # 3 - Create a ProxyDataset for the given data
    # Retrieve fully qualified name to  use for pointer creation.
    obj_public_kwargs = getattr(data, "proxy_public_kwargs", {})
    data_fqn = str(get_fully_qualified_name(data))
    data_dtype = str(type(data))

    # relative
    from ...tensor import Tensor
    from ...tensor.autodp.gamma_tensor import GammaTensor as GT
    from ...tensor.autodp.phi_tensor import PhiTensor as PT

    if isinstance(data, Tensor) and (
        isinstance(data.child, PT) or isinstance(data.child, GT)
    ):
        data_fqn = str(get_fully_qualified_name(data.child))
        child_kwargs = getattr(data.child, "proxy_public_kwargs", {})
        obj_public_kwargs.update(child_kwargs)

    proxy_obj = ProxyDataset(
        asset_name=asset_name,
        dataset_name=dataset_name,
        node_id=domain_id,
        dtype=".".join([data.__module__, data.__class__.__name__]),
        fqn=data_fqn,
        shape=data.shape,
        obj_public_kwargs=obj_public_kwargs,
    )
    return proxy_obj

def abort_s3_object_upload(
    client: boto3.client, upload_id: str, asset_name: str
) -> None:
    """Abort upload to s3 for the given asset.

    Args:
        client (boto3.client): boto3 client.
        upload_id (str): upload id generated for mutlipart upload.
        asset_name (str): name of the data/asset.
    """
    # relative
    from .node_service.upload_service.upload_service_messages import (
        AbortDataUploadMessage,
    )

    # Send a message to PyGrid to abort the data being uploaded to s3.
    # TODO: make this call async -> could use celery .delay method to do
    _ = client.datasets.perform_api_request_generic(
        syft_msg=AbortDataUploadMessage,
        content={
            "upload_id": upload_id,
            "asset_name": asset_name,
        },
    )

def upload_to_s3_using_presigned(
    client: Any,
    data: Any,
    chunk_size: int,
    asset_name: str,
    dataset_name: str = "",
) -> ProxyDataset:
    """Perform a multipart upload of data to Seaweed using boto3 presigned urls.
    The main steps involve:
    - Converting the data to binary data
    - Chunking the binary into smaller chunks
    - Create presigned urls for each chunk
    - Upload data to Seaweed via PUT request
    - Send a acknowledge to Seaweed via PyGrid when all chunks are successfully uploaded
    - Create a ProxyDataset to store metadata of the uploaded data
    Args:
        client (Any): Client to send object to
        data (Any): Data to be uploaded to Seaweed
        chunk_size (int): smallest size of the data to be uploaded in bytes
        asset_name (str): name of the data being uploaded
        dataset_name Optional[(str)]: name of the dataset to which the data belongs
    Raises:
        Exception: If upload of data chunks to Seaweed fails.
    Returns:
        ProxyDataset: Class to store metadata about the data that is uploaded to Seaweed.
    """
    data_upload_description = f"Uploading `{asset_name}`"
    # relative
    from .node_service.upload_service.upload_service_messages import (
        UploadDataCompleteMessage,
    )
    from .node_service.upload_service.upload_service_messages import UploadDataMessage

    dataset_name = str(dataset_name)

    # Step 1 - Convert data to be uploaded to binary
    binary_dataset: bytes = serialize(data, to_bytes=True)  # type: ignore
    file_size = len(binary_dataset)

    # Step 2 - Send a message to PyGrid to inform of the data being uploaded,
    # and get presigned url for each chunk of data being uploaded.
    upload_response = client.datasets.perform_api_request_generic(
        syft_msg=UploadDataMessage,
        content={
            "filename": f"{dataset_name}/{asset_name}",
            "file_size": file_size,
            "chunk_size": chunk_size,
            "address": client.address,
            "reply_to": client.address,
        },
    )

    try:
        # Step 3 - Starts to upload binary data into Seaweed.
        binary_buffer = BytesIO(binary_dataset)
        parts = sorted(upload_response.payload.parts, key=lambda x: x["part_no"])
        etag_chunk_no_pairs = list()
        data_chunks = zip(read_chunks(binary_buffer, chunk_size), parts)
        for _ in tqdm(
            parts, desc=data_upload_description, colour="green", ncols=100, leave=True
        ):
            data_chunk, part = next(data_chunks)
            presigned_url = part["url"]
            part_no = part["part_no"]
            client_url = client.url_from_path(presigned_url)
            part["client_url"] = client_url

            res = requests.put(client_url, data=data_chunk)

            if not res.ok:  # raise an error if upload fails
                error_message = (
                    f"\n\nFailed to upload `{asset_name}` to store\n"
                    + f"Status code: {res.status_code} {res.reason}\n"
                    + f"Error: {str(res.text)}"
                )
                raise DatasetUploadError(message=error_message)
            etag = res.headers["ETag"]
            etag_chunk_no_pairs.append(
                {"ETag": etag, "PartNumber": part_no}
            )  # maintain list of part no and ETag

        # Step 4 - Send a message to PyGrid informing about dataset upload complete!
        upload_response = client.datasets.perform_request(
            syft_msg=UploadDataCompleteMessage,
            content={
                "upload_id": upload_response.payload.upload_id,
                "filename": f"{dataset_name}/{asset_name}",
                "parts": etag_chunk_no_pairs,
            },
        )

        # Step 5 - Create a proxy dataset for the uploaded data.
        # Retrieve fully qualified name to  use for pointer creation.
        obj_public_kwargs = getattr(data, "proxy_public_kwargs", {})
        data_fqn = str(get_fully_qualified_name(data))
        data_dtype = str(type(data))

        # relative
        from ...tensor import Tensor
        from ...tensor.autodp.gamma_tensor import GammaTensor as GT
        from ...tensor.autodp.phi_tensor import PhiTensor as PT

        if isinstance(data, Tensor) and (
            isinstance(data.child, PT) or isinstance(data.child, GT)
        ):
            data_fqn = str(get_fully_qualified_name(data.child))
            child_kwargs = getattr(data.child, "proxy_public_kwargs", {})
            obj_public_kwargs.update(child_kwargs)

        proxy_data = ProxyDataset(
            asset_name=asset_name,
            dataset_name=dataset_name,
            node_id=client.id,
            dtype=".".join([data.__module__, data.__class__.__name__]),
            fqn=data_fqn,
            shape=data.shape,
            obj_public_kwargs=obj_public_kwargs,
        )
    except (Exception, KeyboardInterrupt) as e:
        upload_id = upload_response.payload.upload_id
        abort_s3_object_upload(
            client=client, upload_id=upload_id, asset_name=asset_name
        )
        raise e

    return proxy_data

def get_s3_client(settings: BaseSettings) -> "boto3.client.S3":
    try:
        s3_endpoint = settings.S3_ENDPOINT
        s3_port = settings.S3_PORT
        s3_grid_url = GridURL(host_or_ip=s3_endpoint, port=s3_port)
        return boto3.client(
            "s3",
            endpoint_url=s3_grid_url.url,
            aws_access_key_id=settings.S3_ROOT_USER,
            aws_secret_access_key=settings.S3_ROOT_PWD,
            config=Config(signature_version="s3v4"),
            region_name=settings.S3_REGION,
        )
    except Exception as e:
        print(f"Failed to create S3 Client with {s3_endpoint} {s3_port} {s3_grid_url}")
        raise e

def check_send_to_blob_storage(obj: Any, use_blob_storage: bool = False) -> bool:
    """Check if the data needs to be send to Seaweed storage depending upon its size and type.
    Args:
        obj (Any): Data to be stored to Seaweed.
        use_blob_storage (bool, optional): Explicit flag to send the data to blob storage. Defaults to False.

    Returns:
        bool: Returns True if obj can be stored in blob store else returns False.
    """
    # relative
    from ...tensor import Tensor
    from ...tensor.autodp.phi_tensor import PhiTensor as PT

    if use_blob_storage and (
        isinstance(obj, (PT, Tensor)) or size_mb(obj) > MIN_BLOB_UPLOAD_SIZE_MB
    ):
        return True
    return False
<|endoftext|>


<commit_before>#!/usr/bin/python

import socket
import struct
import threading

class Discovery(threading.Thread):

  def __init__(self):
    print "initiating discovery"

    self.known_services = {}
    self.notify_received = {}

    self.on_timeout()

    threading.Thread.__init__(self)
    self.start();

  def on_timeout(self):
    self.timer = threading.Timer(10, self.on_timeout)
    self.timer.start()

    for s, r in self.notify_received.copy().iteritems():
      if r == False:
        self.erase_service(s)

    for s in self.notify_received.iterkeys():
      self.notify_received[s] = False

  def run(self):
    print "thread started"

    self.cont = True;

    MCAST_GRP = '239.255.255.255'
    MCAST_PORT = 10001

    sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM, socket.IPPROTO_UDP)
    sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
    sock.bind(('', MCAST_PORT))
    mreq = struct.pack("4sl", socket.inet_aton(MCAST_GRP), socket.INADDR_ANY)

    sock.setsockopt(socket.IPPROTO_IP, socket.IP_ADD_MEMBERSHIP, mreq)
    sock.settimeout(0.1)

    while self.cont:
      try:
        msg = sock.recv(10240).split('\n')
        if (msg[0] == "notify"): self.handle_notify(msg)
        elif (msg[0] == "hello"): self.handle_hello(msg)
        elif (msg[0] == "bye"): self.handle_bye(msg)
        else: print "Unknown message: " + msg
      except socket.timeout:
        pass

    print "thread exiting"

  def store_service(self, service, endpoint):
    print "Storing service: " + service + " (" + endpoint + ")"
    self.known_services[service] = endpoint

  def erase_service(self, service):
    print "Erasing service: " + service
    del self.known_services[service]
    del self.notify_received[service]

  def check_service(self, service, endpoint):
    if service not in self.known_services:
      self.store_service(service, endpoint)
    elif self.known_services[service] != endpoint:
      self.erase_service(service)
      self.store_service(service, endpoint)
    self.notify_received[service] = True

  def handle_notify(self, msg):
    if (len(msg) >= 3):
      self.check_service(msg[1], msg[2])

  def handle_bye(self, msg):
    if msg[1] in self.known_services:
      self.erase_service(msg[1])

  def handle_hello(self, msg):
    if (len(msg) >= 3):
      self.check_service(msg[1], msg[2])

  def exit(self):
    self.cont = False
    self.join()
    self.timer.cancel()

discovery = Discovery()

while 1:
  if raw_input() == "q":
    break

discovery.exit()
<commit_msg>small work<commit_after>#!/usr/bin/python

import socket
import struct
import threading
import logging
import daemon

class Discovery(threading.Thread):

  def __init__(self):
    logging.debug("initiating discovery")

    self.known_services = {}
    self.notify_received = {}

    self.on_timeout()

    threading.Thread.__init__(self)
    self.start();

  def on_timeout(self):
    self.timer = threading.Timer(10, self.on_timeout)
    self.timer.start()

    for s, r in self.notify_received.copy().iteritems():
      if r == False:
        self.erase_service(s)

    for s in self.notify_received.iterkeys():
      self.notify_received[s] = False

  def run(self):
    self.cont = True;

    MCAST_GRP = '239.255.255.255'
    MCAST_PORT = 10001

    sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM, socket.IPPROTO_UDP)
    sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
    sock.bind(('', MCAST_PORT))
    mreq = struct.pack("4sl", socket.inet_aton(MCAST_GRP), socket.INADDR_ANY)

    sock.setsockopt(socket.IPPROTO_IP, socket.IP_ADD_MEMBERSHIP, mreq)
    sock.settimeout(0.1)

    while self.cont:
      try:
        msg = sock.recv(10240).split('\n')
        if (msg[0] == "notify"): self.handle_notify(msg)
        elif (msg[0] == "hello"): self.handle_hello(msg)
        elif (msg[0] == "bye"): self.handle_bye(msg)
        else: logging.warning("Unknown message: " + msg)
      except socket.timeout:
        pass

  def store_service(self, service, endpoint):
    logging.debug("Storing service: " + service + " (" + endpoint + ")")
    self.known_services[service] = endpoint

  def erase_service(self, service):
    logging.debug("Erasing service: " + service)
    del self.known_services[service]
    del self.notify_received[service]

  def check_service(self, service, endpoint):
    if service not in self.known_services:
      self.store_service(service, endpoint)
    elif self.known_services[service] != endpoint:
      self.erase_service(service)
      self.store_service(service, endpoint)
    self.notify_received[service] = True

  def handle_notify(self, msg):
    if (len(msg) >= 3):
      self.check_service(msg[1], msg[2])

  def handle_bye(self, msg):
    if msg[1] in self.known_services:
      self.erase_service(msg[1])

  def handle_hello(self, msg):
    if (len(msg) >= 3):
      self.check_service(msg[1], msg[2])

  def exit(self):
    self.cont = False
    self.join()
    self.timer.cancel()
    logging.debug("Discovery exit")

logging.basicConfig(level=logging.DEBUG)

discovery = Discovery()

with daemon.DaemonContext():
  while 1:
    threading.sleep(1)
 # if raw_input() == "q":
  #  break

discovery.exit()
<|endoftext|>