<commit_before>import paasmaker
import json
import tornado
import logging

logger = logging.getLogger(__name__)
logger.addHandler(logging.NullHandler())

class APIResponse(object):
	def __init__(self, response):
		self.errors = []
		self.warnings = []
		self.data = {}
		self.success = False

		logger.debug("Raw API response body: %s", response.body)
		if response.body[0] == '{' and response.body[-1] == '}':
			try:
				parsed = json.loads(response.body)
				self.errors.extend(parsed['errors'])
				self.warnings.extend(parsed['warnings'])
				self.data = parsed['data']
				self.raw = response

			except ValueError, ex:
				self.errors.append("Unable to parse JSON: " + str(ex))
				self.errors.append("Supplied JSON: " + response.body)
			except KeyError, ex:
				self.errors.append("Response was malformed: " + str(ex))
		else:
			self.errors.append("Returned response was not JSON.")

		if response.error:
			logger.error("API request failed with error: %s", response.error)
			self.errors.append(str(response.error))

		if len(self.errors) == 0:
			self.success = True
		else:
			logger.error("API request failed with the following errors:")
			for error in self.errors:
				logger.error("- %s", error)

class APIRequest(object):
	def __init__(self, configuration):
		self.configuration = configuration
		self.target = None
		self.method = 'POST'
		if configuration:
			self.authmethod = 'node'
			self.authvalue = self.configuration.get_flat('node_token')
			self.io_loop = configuration.io_loop
		else:
			self.authmethod = None
			self.authvalue = None
			self.io_loop = tornado.ioloop.IOLoop.instance()

	def method_get(self):
		self.method = 'GET'

	def duplicate_auth(self, request):
		self.authmethod = request.authmethod
		self.authvalue = request.authvalue

	def set_apikey_auth(self, key):
		self.authmethod = 'tokenheader'
		self.authvalue = key

	def set_nodekey_auth(self, key):
		self.authmethod = 'node'
		self.authvalue = key

	def set_superkey_auth(self, key=None):
		self.authmethod = 'superheader'
		if key:
			self.authvalue = key
		else:
			# TODO: More error checking here.
			self.authvalue = self.configuration.get_flat('pacemaker.super_token')

	def build_payload(self):
		# Override in your subclass.
		return {}

	def get_master(self):
		# TODO: SSL ?
		return "http://%s:%d" % (self.configuration.get_flat('master.host'), self.configuration.get_flat('master.port'))

	def get_endpoint(self):
		# Override in your subclass to change the URI.
		raise NotImplementedError("You must implement get_endpoint().")

	def set_target(self, target):
		if isinstance(target, basestring):
			self.target = target
		elif isinstance(target, paasmaker.model.Node):
			self.target = "http://%s:%d" % (target.route, target.apiport)
		else:
			raise ValueError("Target is not a string or Node object.")

	def process_response(self, response):
		# For overriding in your subclasses, if it's all self contained.
		# Note that the supplied callback, if provided, is called after this.
		pass

	def send(self, callback=None, **kwargs):
		if self.method == 'GET' and self.authmethod in ['node', 'super', 'token']:
			raise ValueError("Can't do a GET request with node, super, or token authentication methods.")

		logger.debug("In send() of API request of type %s", type(self))

		if self.method == 'POST':
			# Build what we're sending.
			data = {}
			data['data'] = self.build_payload()
			data['auth'] = { 'method': self.authmethod, 'value': self.authvalue }

			encoded = json.dumps(data, cls=paasmaker.util.jsonencoder.JsonEncoder)

			kwargs['body'] = encoded
			# Always a POST, because we're sending a body.
			kwargs['method'] = 'POST'

			logger.debug("Payload sending to server: %s", encoded)
		else:
			kwargs['method'] = 'GET'

		# Don't follow redirects - this is an API request.
		kwargs['follow_redirects'] = False

		if self.authmethod == 'userheader':
			if not kwargs.has_key('headers'):
				kwargs['headers'] = {}
			kwargs['headers']['User-Token'] = self.authvalue
		if self.authmethod == 'superheader':
			if not kwargs.has_key('headers'):
				kwargs['headers'] = {}
			kwargs['headers']['Super-Token'] = self.authvalue

		# The function called when it returns.
		# It's a closure to preserve the callback provided.
		def our_callback(response):
			# Parse and handle what came back.
			our_response = APIResponse(response)
			# Use the built in response processor.
			self.process_response(our_response)
			# And call the user defined callback back.
			if callback:
				callback(our_response)

		if not self.target:
			self.target = self.get_master()

		# Build and make the request.
		endpoint = self.target + self.get_endpoint()
		if endpoint.find('?') == -1:
			endpoint += '?format=json'
		else:
			endpoint += '&format=json'
		logger.debug("Endpoint for request: %s", endpoint)
		request = tornado.httpclient.HTTPRequest(endpoint, **kwargs)
		client = tornado.httpclient.AsyncHTTPClient(io_loop=self.io_loop)
		client.fetch(request, our_callback)

# TODO: Add unit tests!
<commit_msg>Fixed a minor bug with the API request not sending the API key when requested.<commit_after>import paasmaker
import json
import tornado
import logging

logger = logging.getLogger(__name__)
logger.addHandler(logging.NullHandler())

class APIResponse(object):
	def __init__(self, response):
		self.errors = []
		self.warnings = []
		self.data = {}
		self.success = False

		logger.debug("Raw API response body: %s", response.body)
		if response.body[0] == '{' and response.body[-1] == '}':
			try:
				parsed = json.loads(response.body)
				self.errors.extend(parsed['errors'])
				self.warnings.extend(parsed['warnings'])
				self.data = parsed['data']
				self.raw = response

			except ValueError, ex:
				self.errors.append("Unable to parse JSON: " + str(ex))
				self.errors.append("Supplied JSON: " + response.body)
			except KeyError, ex:
				self.errors.append("Response was malformed: " + str(ex))
		else:
			self.errors.append("Returned response was not JSON.")

		if response.error:
			logger.error("API request failed with error: %s", response.error)
			self.errors.append(str(response.error))

		if len(self.errors) == 0:
			self.success = True
		else:
			logger.error("API request failed with the following errors:")
			for error in self.errors:
				logger.error("- %s", error)

class APIRequest(object):
	def __init__(self, configuration):
		self.configuration = configuration
		self.target = None
		self.method = 'POST'
		if configuration:
			self.authmethod = 'node'
			self.authvalue = self.configuration.get_flat('node_token')
			self.io_loop = configuration.io_loop
		else:
			self.authmethod = None
			self.authvalue = None
			self.io_loop = tornado.ioloop.IOLoop.instance()

	def method_get(self):
		self.method = 'GET'

	def duplicate_auth(self, request):
		self.authmethod = request.authmethod
		self.authvalue = request.authvalue

	def set_apikey_auth(self, key):
		self.authmethod = 'tokenheader'
		self.authvalue = key

	def set_nodekey_auth(self, key):
		self.authmethod = 'node'
		self.authvalue = key

	def set_superkey_auth(self, key=None):
		self.authmethod = 'superheader'
		if key:
			self.authvalue = key
		else:
			# TODO: More error checking here.
			self.authvalue = self.configuration.get_flat('pacemaker.super_token')

	def build_payload(self):
		# Override in your subclass.
		return {}

	def get_master(self):
		# TODO: SSL ?
		return "http://%s:%d" % (self.configuration.get_flat('master.host'), self.configuration.get_flat('master.port'))

	def get_endpoint(self):
		# Override in your subclass to change the URI.
		raise NotImplementedError("You must implement get_endpoint().")

	def set_target(self, target):
		if isinstance(target, basestring):
			self.target = target
		elif isinstance(target, paasmaker.model.Node):
			self.target = "http://%s:%d" % (target.route, target.apiport)
		else:
			raise ValueError("Target is not a string or Node object.")

	def process_response(self, response):
		# For overriding in your subclasses, if it's all self contained.
		# Note that the supplied callback, if provided, is called after this.
		pass

	def send(self, callback=None, **kwargs):
		if self.method == 'GET' and self.authmethod in ['node', 'super', 'token']:
			raise ValueError("Can't do a GET request with node, super, or token authentication methods.")

		logger.debug("In send() of API request of type %s", type(self))

		if self.method == 'POST':
			# Build what we're sending.
			data = {}
			data['data'] = self.build_payload()
			data['auth'] = { 'method': self.authmethod, 'value': self.authvalue }

			encoded = json.dumps(data, cls=paasmaker.util.jsonencoder.JsonEncoder)

			kwargs['body'] = encoded
			# Always a POST, because we're sending a body.
			kwargs['method'] = 'POST'

			logger.debug("Payload sending to server: %s", encoded)
		else:
			kwargs['method'] = 'GET'

		# Don't follow redirects - this is an API request.
		kwargs['follow_redirects'] = False

		if self.authmethod == 'tokenheader':
			if not kwargs.has_key('headers'):
				kwargs['headers'] = {}
			kwargs['headers']['User-Token'] = self.authvalue
		if self.authmethod == 'superheader':
			if not kwargs.has_key('headers'):
				kwargs['headers'] = {}
			kwargs['headers']['Super-Token'] = self.authvalue

		# The function called when it returns.
		# It's a closure to preserve the callback provided.
		def our_callback(response):
			# Parse and handle what came back.
			our_response = APIResponse(response)
			# Use the built in response processor.
			self.process_response(our_response)
			# And call the user defined callback back.
			if callback:
				callback(our_response)

		if not self.target:
			self.target = self.get_master()

		# Build and make the request.
		endpoint = self.target + self.get_endpoint()
		if endpoint.find('?') == -1:
			endpoint += '?format=json'
		else:
			endpoint += '&format=json'
		logger.debug("Endpoint for request: %s", endpoint)
		request = tornado.httpclient.HTTPRequest(endpoint, **kwargs)
		client = tornado.httpclient.AsyncHTTPClient(io_loop=self.io_loop)
		client.fetch(request, our_callback)

# TODO: Add unit tests!
<|endoftext|>


<commit_before>
from tastypie import fields
from tastypie.constants import ALL, ALL_WITH_RELATIONS
from django.utils.text import Truncator
from tastypie.cache import SimpleCache
from tastypie.throttle import BaseThrottle
from django.utils.html import strip_tags
from django.conf.urls import url

from tastypie.authentication import ApiKeyAuthentication
from datea_api.apps.api.base_resources import DateaBaseGeoResource
from datea_api.apps.api.authorization import DateaBaseAuthorization
from api.authentication import ApiKeyPlusWebAuthentication
from tastypie.utils import trailing_slash

from .models import Dateo
from image.models import Image
from image.resources import ImageResource
from tag.models import Tag
from tag.resources import TagResource
from comment.models import Comment
from follow.models import Follow

from haystack.utils.geo import Point
from haystack.utils.geo import Distance
from haystack.query import SearchQuerySet
from haystack.inputs import AutoQuery
from django.core.paginator import Paginator, InvalidPage, EmptyPage

from pprint import pprint as pp

class DateoResource(DateaBaseGeoResource):

    user = fields.ToOneField('datea_api.apps.account.resources.UserResource',
            attribute="user", null=False, full=True, readonly=True)
    category = fields.ToOneField('datea_api.apps.category.resources.CategoryResource',
            attribute= 'category', null=True, full=False, readonly=False)
    tags = fields.ToManyField('datea_api.apps.tag.resources.TagResource',
            attribute='tags', related_name='tags', null=True, full=True, readonly=True)
    images = fields.ToManyField('datea_api.apps.image.resources.ImageResource',
            attribute='images', null=True, full=True, readonly=True)
    comments = fields.ToManyField('datea_api.apps.comment.resources.CommentResource',
            attribute=lambda bundle: Comment.objects.filter(object_id=bundle.obj.id, content_type__model='dateo'),
            null=True, full=True, readonly=True)

    def dehydrate(self, bundle):

        user_data = {
                     'username': bundle.data['user'].data['username'],
                     'image_small': bundle.data['user'].data['image_small'],
                     'url': bundle.data['user'].data['url'],
                     'resource_uri': bundle.data['user'].data['resource_uri']
                     }
        bundle.data['user'] = user_data
        bundle.data['extract'] = Truncator( strip_tags(bundle.obj.content) ).chars(140).replace("\n",' ')
        bundle.data['url'] = bundle.obj.get_absolute_url()
        return bundle

    def hydrate(self, bundle):

        # Some security measures in regards to an object's owner
        if bundle.request.method == 'POST':
            # use request user
            if 'user' not in bundle.data:
                bundle.data['user'] = '/api/v2/user/'+str(bundle.request.user.id)
            bundle.obj.user = bundle.request.user

        elif bundle.request.method in 'PATCH':
            # preserve original owner
            orig_object = Dateo.objects.get(pk=bundle.data['id'])
            bundle.obj.user = orig_object.user
            # don't touch 'created'
            if 'created' in bundle.data:
                del bundle.data['created']

        return bundle

    # do our own saving of related m2m fields
    def hydrate_m2m(self, bundle):
        #print bundle.data
        if 'images' in bundle.data and bundle.data['images']:
            imgs = []
            for imgdata in bundle.data['images']:
                if 'id' in imgdata:
                    imgs.append(imgdata['id'])
                else:
                    imgrsc = ImageResource()
                    imgbundle = imgrsc.build_bundle(data=imgdata, request=bundle.request)
                    imgbundle = imgrsc.full_hydrate(imgbundle)
                    imgbundle.obj.save()
                    imgs.append(imgbundle.obj.pk)
            bundle.obj.images = Image.objects.filter(pk__in=imgs)

        if 'tags' in bundle.data and bundle.data['tags']:
            tags = []
            for tagdata in bundle.data['tags']:
                if 'id' in tagdata:
                    tags.append(tagdata['id'])
                else:
                    found = Tag.objects.filter(tag=tagdata['tag'])
                    if found.count() > 0:
                        tags.append(found[0].pk)
                    else:
                        tagrsc = TagResource()
                        tagbundle = tagrsc.build_bundle(data=tagdata, request=bundle.request)
                        tagbundle = tagrsc.full_hydrate(tagbundle)
                        tagbundle.obj.save()
                        tags.append(tagbundle.obj.pk)

            bundle.obj.tags = Tag.objects.filter(pk__in=tags)

        return bundle

    class Meta:
        queryset = Dateo.objects.all()
        resource_name = 'dateo'
        list_allowed_methods = ['get', 'post']
        detail_allowed_methods = ['get', 'post', 'patch', 'delete']
        authentication = ApiKeyPlusWebAuthentication()
        authorization = DateaBaseAuthorization()
        filtering = {
            'action': ALL_WITH_RELATIONS,
            'id': ['exact'],
            'created': ['range', 'gt', 'gte', 'lt', 'lte'],
            'position': ['distance', 'contained','latitude', 'longitude'],
            'published': ['exact'],
            'user': ALL_WITH_RELATIONS
        }
        ordering = ['name', 'created', 'distance']
        limit = 200
        cache = SimpleCache(timeout=10)
        always_return_data = True

    # Replace GET dispatch_list with HAYSTACK SEARCH
    def dispatch_list(self, request, **kwargs):
        if request.method == "GET":
            return self.get_search(request, **kwargs)
        else:
            return self.dispatch('list', request, **kwargs)

    # HAYSTACK SEARCH
    def get_search(self, request, **kwargs):

        # tests
        self.method_check(request, allowed=['get'])
        self.is_authenticated(request)
        self.throttle_check(request)

        # pagination
        limit = int(request.GET.get('limit', 20))
        offset = int(request.GET.get('offset', 0))
        page = (offset / limit) + 1

        # Do the query
        q_args = {'published': request.GET.get('published', True)}

        # add search query
        if 'q' in request.GET and request.GET['q'] != '':
            q_args['content'] = AutoQuery(request.GET['q'])

        # check for more params
        params = ['category_id', 'category', 'user', 'user_id',
                  'published', 'status', 'created__year', 'created__month', 'created__day']
        for p in params:
            if p in request.GET:
                q_args[p] = request.GET.get(p)

        # check for additional date filters (with datetime objects)
        date_params = ['created__gt', 'created__lt']
        for p in date_params:
            if p in request.GET:
                q_args[p] = models.DateTimeField().to_python(request.get(p))

        # GET ONLY DATEOS I FOLLOW INDIVIDUALLY
        if 'followed' in request.GET:
            uid = int(request.GET['followed'])
            dateo_ids = [f.object_id for f in Follow.objects.filter(content_type__model='dateo', user__id=uid)]
            q_args['id__in'] = dateo_ids

        # GET DATEOS BY TAGS I FOLLOW
        if 'followed_by_tags' in request.GET:
            uid = int(request.GET['followed_by_tags'])
            tag_ids = [f.object_id for f in Follow.objects.filter(content_type__model='tag', user__id=uid)]
            q_args['tag__in'] = tag_ids

        # show also one's own unpublished actions
        user_id = int(request.GET.get('user_id', -1))
        show_unpublished = int(request.GET.get('show_unpublished', 0))
        if request.user.is_authenticated() and user_id == request.user.id and show_unpublished ==1:
            del q_args['published']

        # INIT THE QUERY
        sqs = SearchQuerySet().models(Dateo).load_all().filter(**q_args)

        # SPATIAL QUERY ADDONS
        # WITHIN QUERY
        if all(k in request.GET and request.GET.get(k) != '' for k in ('bottom_left', 'top_right')):
            bleft = [float(c) for c in request.GET.get('bottom_left').split(',')]
            bottom_left = Point(bleft[0], bleft[1])
            tright = [float(c) for c in request.GET.get('top_right').split(',')]
            top_right = Point(tright[0], tright[1])

            sqs = sqs.within('position', bottom_left, top_right)

        # DWITHIN QUERY
        if all(k in request.GET and request.GET.get(k) != '' for k in ('max_distance', 'position')):
            dist = Distance( m = int(request.GET.get('max_distance')))
            pos = [float(c) for c in request.GET.get('position').split(',')]
            position = Point(pos[0], pos[1])

            sqs = sqs.dwithin('position', position, dist)

        # ORDER BY
        order_by = request.GET.get('order_by', '-created').split(',')
        order_by = [o if 'score' not in o else o.replace('score', '_score') for o in order_by]

        if 'q' in request.GET:
            if order_by == ['-created'] and '-created' not in request.GET:
                order_by = ['_score']

        # if q is set, then order will be relevance first
        # if not, then do normal order by
        if 'distance' in order_by and 'position' in request.GET and request.GET['position'] != '':
            pos = [float(c) for c in request.GET.get('position').split(',')]
            position = Point(pos[0], pos[1])
            sqs = sqs.distance('position', position).order_by(*order_by)
        elif len(order_by) > 0:
            sqs = sqs.order_by(*order_by)

        paginator = Paginator(sqs, limit)

        try:
            page = paginator.page(page)
        except InvalidPage:
            raise Http404("Sorry, no results on that page.")

        objects = []

        for result in page.object_list:
            pp(vars(result))
            bundle = self.build_bundle(obj=result.object, request=request)
            bundle = self.full_dehydrate(bundle)
            objects.append(bundle)

        object_list = {
            'meta': {
                'limit': limit,
                'next': page.has_next(),
                'previous': page.has_previous(),
                'total_count': sqs.count(),
                'offset': offset
            },
            'objects': objects,
        }

        self.log_throttled_access(request)
        return self.create_response(request, object_list)

<commit_msg>debug search for solr<commit_after>
from tastypie import fields
from tastypie.constants import ALL, ALL_WITH_RELATIONS
from django.utils.text import Truncator
from tastypie.cache import SimpleCache
from tastypie.throttle import BaseThrottle
from django.utils.html import strip_tags
from django.conf.urls import url

from tastypie.authentication import ApiKeyAuthentication
from datea_api.apps.api.base_resources import DateaBaseGeoResource
from datea_api.apps.api.authorization import DateaBaseAuthorization
from api.authentication import ApiKeyPlusWebAuthentication
from tastypie.utils import trailing_slash

from .models import Dateo
from image.models import Image
from image.resources import ImageResource
from tag.models import Tag
from tag.resources import TagResource
from comment.models import Comment
from follow.models import Follow

from haystack.utils.geo import Point
from haystack.utils.geo import Distance
from haystack.query import SearchQuerySet
from haystack.inputs import AutoQuery
from django.core.paginator import Paginator, InvalidPage, EmptyPage

from pprint import pprint as pp

class DateoResource(DateaBaseGeoResource):

    user = fields.ToOneField('datea_api.apps.account.resources.UserResource',
            attribute="user", null=False, full=True, readonly=True)
    category = fields.ToOneField('datea_api.apps.category.resources.CategoryResource',
            attribute= 'category', null=True, full=False, readonly=False)
    tags = fields.ToManyField('datea_api.apps.tag.resources.TagResource',
            attribute='tags', related_name='tags', null=True, full=True, readonly=True)
    images = fields.ToManyField('datea_api.apps.image.resources.ImageResource',
            attribute='images', null=True, full=True, readonly=True)
    comments = fields.ToManyField('datea_api.apps.comment.resources.CommentResource',
            attribute=lambda bundle: Comment.objects.filter(object_id=bundle.obj.id, content_type__model='dateo'),
            null=True, full=True, readonly=True)

    def dehydrate(self, bundle):

        user_data = {
                     'username': bundle.data['user'].data['username'],
                     'image_small': bundle.data['user'].data['image_small'],
                     'url': bundle.data['user'].data['url'],
                     'resource_uri': bundle.data['user'].data['resource_uri']
                     }
        bundle.data['user'] = user_data
        bundle.data['extract'] = Truncator( strip_tags(bundle.obj.content) ).chars(140).replace("\n",' ')
        bundle.data['url'] = bundle.obj.get_absolute_url()
        return bundle

    def hydrate(self, bundle):

        # Some security measures in regards to an object's owner
        if bundle.request.method == 'POST':
            # use request user
            if 'user' not in bundle.data:
                bundle.data['user'] = '/api/v2/user/'+str(bundle.request.user.id)
            bundle.obj.user = bundle.request.user

        elif bundle.request.method in 'PATCH':
            # preserve original owner
            orig_object = Dateo.objects.get(pk=bundle.data['id'])
            bundle.obj.user = orig_object.user
            # don't touch 'created'
            if 'created' in bundle.data:
                del bundle.data['created']

        return bundle

    # do our own saving of related m2m fields
    def hydrate_m2m(self, bundle):
        #print bundle.data
        if 'images' in bundle.data and bundle.data['images']:
            imgs = []
            for imgdata in bundle.data['images']:
                if 'id' in imgdata:
                    imgs.append(imgdata['id'])
                else:
                    imgrsc = ImageResource()
                    imgbundle = imgrsc.build_bundle(data=imgdata, request=bundle.request)
                    imgbundle = imgrsc.full_hydrate(imgbundle)
                    imgbundle.obj.save()
                    imgs.append(imgbundle.obj.pk)
            bundle.obj.images = Image.objects.filter(pk__in=imgs)

        if 'tags' in bundle.data and bundle.data['tags']:
            tags = []
            for tagdata in bundle.data['tags']:
                if 'id' in tagdata:
                    tags.append(tagdata['id'])
                else:
                    found = Tag.objects.filter(tag=tagdata['tag'])
                    if found.count() > 0:
                        tags.append(found[0].pk)
                    else:
                        tagrsc = TagResource()
                        tagbundle = tagrsc.build_bundle(data=tagdata, request=bundle.request)
                        tagbundle = tagrsc.full_hydrate(tagbundle)
                        tagbundle.obj.save()
                        tags.append(tagbundle.obj.pk)

            bundle.obj.tags = Tag.objects.filter(pk__in=tags)

        return bundle

    class Meta:
        queryset = Dateo.objects.all()
        resource_name = 'dateo'
        list_allowed_methods = ['get', 'post']
        detail_allowed_methods = ['get', 'post', 'patch', 'delete']
        authentication = ApiKeyPlusWebAuthentication()
        authorization = DateaBaseAuthorization()
        filtering = {
            'action': ALL_WITH_RELATIONS,
            'id': ['exact'],
            'created': ['range', 'gt', 'gte', 'lt', 'lte'],
            'position': ['distance', 'contained','latitude', 'longitude'],
            'published': ['exact'],
            'user': ALL_WITH_RELATIONS
        }
        ordering = ['name', 'created', 'distance']
        limit = 200
        cache = SimpleCache(timeout=10)
        always_return_data = True

    # Replace GET dispatch_list with HAYSTACK SEARCH
    def dispatch_list(self, request, **kwargs):
        if request.method == "GET":
            return self.get_search(request, **kwargs)
        else:
            return self.dispatch('list', request, **kwargs)

    # HAYSTACK SEARCH
    def get_search(self, request, **kwargs):

        # tests
        self.method_check(request, allowed=['get'])
        self.is_authenticated(request)
        self.throttle_check(request)

        # pagination
        limit = int(request.GET.get('limit', 100))
        offset = int(request.GET.get('offset', 0))
        page = (offset / limit) + 1

        # Do the query
        q_args = {'published': request.GET.get('published', True)}

        # add search query
        if 'q' in request.GET and request.GET['q'] != '':
            q_args['content'] = AutoQuery(request.GET['q'])

        # check for more params
        params = ['category_id', 'category', 'user', 'user_id',
                  'published', 'status', 'created__year', 'created__month', 'created__day']
        for p in params:
            if p in request.GET:
                q_args[p] = request.GET.get(p)

        # check for additional date filters (with datetime objects)
        date_params = ['created__gt', 'created__lt']
        for p in date_params:
            if p in request.GET:
                q_args[p] = models.DateTimeField().to_python(request.get(p))

        # GET ONLY DATEOS I FOLLOW INDIVIDUALLY
        if 'followed' in request.GET:
            uid = int(request.GET['followed'])
            dateo_ids = [f.object_id for f in Follow.objects.filter(content_type__model='dateo', user__id=uid)]
            q_args['id__in'] = dateo_ids

        # GET DATEOS BY TAGS I FOLLOW
        if 'followed_by_tags' in request.GET:
            uid = int(request.GET['followed_by_tags'])
            tag_ids = [f.object_id for f in Follow.objects.filter(content_type__model='tag', user__id=uid)]
            q_args['tag__in'] = tag_ids

        # show also one's own unpublished actions
        user_id = int(request.GET.get('user_id', -1))
        show_unpublished = int(request.GET.get('show_unpublished', 0))
        if request.user.is_authenticated() and user_id == request.user.id and show_unpublished ==1:
            del q_args['published']

        # INIT THE QUERY
        sqs = SearchQuerySet().models(Dateo).load_all().filter(**q_args)

        # SPATIAL QUERY ADDONS
        # WITHIN QUERY
        if all(k in request.GET and request.GET.get(k) != '' for k in ('bottom_left', 'top_right')):
            bleft = [float(c) for c in request.GET.get('bottom_left').split(',')]
            bottom_left = Point(bleft[0], bleft[1])
            tright = [float(c) for c in request.GET.get('top_right').split(',')]
            top_right = Point(tright[0], tright[1])

            sqs = sqs.within('position', bottom_left, top_right)

        # DWITHIN QUERY
        if all(k in request.GET and request.GET.get(k) != '' for k in ('max_distance', 'position')):
            dist = Distance( m = int(request.GET.get('max_distance')))
            pos = [float(c) for c in request.GET.get('position').split(',')]
            position = Point(pos[0], pos[1])

            sqs = sqs.dwithin('position', position, dist)

        # ORDER BY
        order_by = request.GET.get('order_by', '-created').split(',')

        # in elastic search 'score' is '_score'
        #order_by = [o if 'score' not in o else o.replace('score', '_score') for o in order_by]

        if 'q' in request.GET:
            if order_by == ['-created'] and '-created' not in request.GET:
                #order_by = ['_score']
                order_by = ['score']

        # if q is set, then order will be relevance first
        # if not, then do normal order by
        if 'distance' in order_by and 'position' in request.GET and request.GET['position'] != '':
            pos = [float(c) for c in request.GET.get('position').split(',')]
            position = Point(pos[0], pos[1])
            sqs = sqs.distance('position', position).order_by(*order_by)
        elif len(order_by) > 0:
            sqs = sqs.order_by(*order_by)

        paginator = Paginator(sqs, limit)

        try:
            page = paginator.page(page)
        except InvalidPage:
            raise Http404("Sorry, no results on that page.")

        objects = []

        for result in page.object_list:
            pp(vars(result))
            bundle = self.build_bundle(obj=result.object, request=request)
            bundle = self.full_dehydrate(bundle)
            objects.append(bundle)

        object_list = {
            'meta': {
                'limit': limit,
                'next': page.has_next(),
                'previous': page.has_previous(),
                'total_count': sqs.count(),
                'offset': offset
            },
            'objects': objects,
        }

        self.log_throttled_access(request)
        return self.create_response(request, object_list)

<|endoftext|>


<commit_before># Copyright 2013-2021 Lawrence Livermore National Security, LLC and other
# Spack Project Developers. See the top-level COPYRIGHT file for details.
#
# SPDX-License-Identifier: (Apache-2.0 OR MIT)

from spack import *

class PyJupyterConsole(PythonPackage):
    """Jupyter Terminal Console"""

    homepage = "https://github.com/jupyter/jupyter_console"
    url      = "https://github.com/jupyter/jupyter_console/archive/5.0.0.tar.gz"

    version('6.1.0', sha256='838c95c99ce52978e1660e7a30dd933dede158e2f4da1bc5fad1a8fad44570b7')
    version('5.2.0', sha256='371d03aeefcf48967f2f00af4c1709f52d2a688deee33f395c6330e4e8aa171c')
    version('5.0.0', sha256='e966b2b5bf9a1e8c5bd11a6335bb11f68ec585ea39b801721b2ed9dd964468fa')
    version('4.1.1', sha256='0bb06a1f878d0c44c2f6f66406a80f949bcd86f3508035500af7dceffb9cc7dc')
    version('4.1.0', sha256='9c72097721676ba13d036e68d82ef9ef76772254c8a995a474339a8bd48aaf91')
    version('4.0.3', sha256='b1867a89b693f247e9089a8f367fa4f27af6eac27930cad2966054adfa7b9aa1')
    version('4.0.2', sha256='116a56763899bbb12c762f865372eb52c08619ef070c237c7f1387e192bfd3df')

    depends_on('python@2.7:2.8,3.3:', type=('build', 'run'))
    depends_on('python@3.5:', type=('build', 'run'), when='@6:')
    depends_on('py-jupyter-client', type=('build', 'run'))
    depends_on('py-ipython@:5.8.0', type=('build', 'run'), when='@:5')
    depends_on('py-ipython', type=('build', 'run'))
    depends_on('py-ipykernel', type=('build', 'run'))
    depends_on('py-pygments', type=('build', 'run'))
    depends_on('py-prompt-toolkit@1.0.0:1', type=('build', 'run'), when='@:5')
    depends_on('py-prompt-toolkit@2.0.0:2,3.0.2:3.0', type=('build', 'run'), when='@6:')
<commit_msg>New version: py-jupyter-console 6.4.0; download sources from pip (#27150)<commit_after># Copyright 2013-2021 Lawrence Livermore National Security, LLC and other
# Spack Project Developers. See the top-level COPYRIGHT file for details.
#
# SPDX-License-Identifier: (Apache-2.0 OR MIT)

from spack import *

class PyJupyterConsole(PythonPackage):
    """Jupyter Terminal Console"""

    homepage = "https://github.com/jupyter/jupyter_console"
    pypi     = "jupyter_console/jupyter_console-6.4.0.tar.gz"

    version('6.4.0', sha256='242248e1685039cd8bff2c2ecb7ce6c1546eb50ee3b08519729e6e881aec19c7')
    version('6.1.0', sha256='6f6ead433b0534909df789ea64f0a14cdf9b6b2360757756f08182be4b9e431b')
    version('5.2.0', sha256='545dedd3aaaa355148093c5609f0229aeb121b4852995c2accfa64fe3e0e55cd')
    version('5.0.0', sha256='7ddfc8cc49921b0ed852500928922e637f9188358c94b5c76339a5a8f9ac4c11')
    version('4.1.1', sha256='d754cfd18d258fa9e7dde39a36e589c4a7241075b5d0f420691fa3d50e4c4ae3')
    version('4.1.0', sha256='3f9703b632e38d68713fc2ea1f546edc4db2a8f925c94b6dd91a8d0c13816ce9')
    version('4.0.3', sha256='555be6963a8f6431fbe1d424c7ffefee90824758058e4c9a2ab3aa045948eb85')
    version('4.0.2', sha256='97e27e1c27a6dd04d166b7a4c81d717becdd979a0879a628e08f295a43a2bc58')

    depends_on('python@2.7:2.8,3.3:', type=('build', 'run'))
    depends_on('python@3.5:', type=('build', 'run'), when='@6:')
    depends_on('python@3.6:', type=('build', 'run'), when='@6.2:')
    depends_on('py-setuptools@40.8.0:', type='build', when='@6.2:')
    depends_on('py-jupyter-client', type=('build', 'run'))
    depends_on('py-ipython@:5.8.0', type=('build', 'run'), when='@:5')
    depends_on('py-ipython', type=('build', 'run'))
    depends_on('py-ipykernel', type=('build', 'run'))
    depends_on('py-pygments', type=('build', 'run'))
    depends_on('py-prompt-toolkit@1.0.0:1', type=('build', 'run'), when='@:5')
    depends_on('py-prompt-toolkit@2.0.0:2,3.0.2:3.0', type=('build', 'run'), when='@6:')
<|endoftext|>


<commit_before>import os,sys,string
import math
import optparse
from fred import FRED,FRED_RUN,FRED_Infection_Set,FRED_Locations_Set,FRED_People_Set
import apollo
import time
#from threading import Thread
from multiprocessing import Process,Queue

def fillInfection(q,j,fred_run):
        InfByBG = []
	numDays = fred_run.get_param("days")
	for i in range(0,int(numDays)):
		InfByBG.append({})

        infection_file = fred_run.infection_file_names[j]
        run_number = infection_file.split("infections")[1].split(".")[0]
        print "Run = " + run_number
        infSet = fred_run.get_infection_set(int(run_number))
        #print "Length = %d"%len(infSet.infectionList)
        for i in range(0,len(infSet.infectionList)):
		day = int(infSet.get('day',i))
		infected_id = int(infSet.get('host',i))
            #personRec = fred_population.people[int(infRec.infected_id)]
		personFips = fred_population.get("stcotrbg",infected_id)
            #if personFips == "420034754012":
            #    print "There is someone here on Day %d"%(day)
            #print personFips
	#	if personFips not in fipsList:
	#		fipsList.append(personFips)
            #print "%d %d"%(j,day)
		if InfByBG[day].has_key(personFips) is False:
			InfByBG[day][personFips] = 0.0
		InfByBG[day][personFips] += 1
	q.put(InfByBG)

if __name__ == '__main__':
    parser = optparse.OptionParser(usage="""
    %prog [--help][-k key][-r run_number][-t vis_type]
    -k or --key   Fred Key
    -r or --run   The run number of vizualize
                  ave = produce an average result
                  all = produce an input for all runs

    -w or --time  Turn on profiling
    -d or --debug Turn on debug printing
    """)

    parser.add_option("-k","--key",type="string",
                      help="The FRED key for the run you wish to visualize")
    parser.add_option("-r","--run",type="string",
                      help="The number of the run you would like to visualize (number,ave, or all)",
                      default=1)
    parser.add_option("-w","--time",action="store_true",
                      default=False)
    parser.add_option("-d","--debug",action="store_true",
                      default=False)

    opts,args=parser.parse_args()

    DBHosts = ["gaia.psc.edu","warhol-fred.psc.edu"]
    ### This only currently works for allegheny county

    #### Initialize the FRED SIMS
    fred = FRED()

    key = opts.key

    run = opts.run

    try:
        int(run)
    except:
        if run != "ave" and run != "all":
            print "Invalid parameter for run, needs to be an integer, all or ave"
            sys.exit(1)

    fred_run = FRED_RUN(fred,key)
    numberDays = int(fred_run.get_param("days"))
    outputsAve = fred_run.outputsAve

    apolloDBs = [ apollo.ApolloDB(host_=x) for x in DBHosts ]
    for apolloDB in apolloDBs:
	apolloDB.connect()

    ## Fill in the run table
    	SQLString = 'INSERT INTO run set label = "' + key + '"'
    	apolloDB.query(SQLString)
    	runInsertID = apolloDB.insertID()

    	stateList = {'S':'susceptible','E':'exposed','I':'infectious','R':'recovered'}
    	locationList = ['42003'] # need to replace with an automatic way of getting this info
    	stateInsertIDDict = {}

    ### setup simulated_population_axis

    	axisInsertIDDict = apolloDB._populationAxis()

    ### setup simulated_populations table
    	for state in stateList.keys():
	    for location in locationList:
		stateInsertIDDict[(state,location)] =\
		  apolloDB.checkAndAddSimulatedPopulation(stateList[state],location)

    	for state in stateList.keys():
	    for location in locationList:
		for day in outputsAve:
		    populationID = stateInsertIDDict[(state,location)]
		    SQLString = 'INSERT INTO time_series set '\
				+'run_id = "' + str(runInsertID) + '", '\
				+'population_id = "' + str(populationID) + '", '\
				+'time_step = "' + str(day['Day']) +'", '\
				+'pop_count = "' + str(day[state]) + '"'
		    apolloDB.query(SQLString)

    synth_pop_dir = fred_run.get_param("synthetic_population_directory")
    synth_pop_id = fred_run.get_param("synthetic_population_id")
    synth_pop_prefix = os.path.expandvars(synth_pop_dir + "/" + synth_pop_id + "/" + synth_pop_id)
    synth_pop_filename = synth_pop_prefix + "_synth_people.txt"
    synth_household_filename = synth_pop_prefix + "_synth_households.txt"

    try:
        open(synth_pop_filename,"r")
    except IOError:
        print "Problem opening FRED population file " + synth_pop_filename
        sys.exit(1)

    fred_population = FRED_People_Set(synth_pop_filename)

    numDays = fred_run.get_param("days")
    aveInfByBG = []
    fipsList = []
    time1 = time.time()
    for i in range(0,int(numDays)):
	aveInfByBG.append({})
    InfByBG = []

    #for infection_file in fred_run.infection_file_names:
    queues = []
    threads = []
    for j in range(0,len(fred_run.infection_file_names)):
	queues.append(Queue())
	threads.append(Process(target=fillInfection,args=(queues[j],j,fred_run)))

    for t in threads:
	t.start()

    for q in queues:
	InfByBG.append(q.get())

    for t in threads:
	t.join()

    ### finish off averages
    numRuns = fred_run.number_of_runs
    for infFile in InfByBG:
	sum = 0
        for day in range(0,len(infFile)):
	    for fips in infFile[day].keys():
		if fips not in fipsList:
		   fipsList.append(fips)
		if fips not in aveInfByBG[day].keys():
		   aveInfByBG[day][fips]=0.0
	        aveInfByBG[day][fips] += infFile[day][fips]
		sum += infFile[day][fips]
	print "Sum = " + str(sum)

    for day in range(0,len(aveInfByBG)):
            for fips in aveInfByBG[day].keys():
                aveInfByBG[day][fips] /= float(numRuns)

    time2 = time.time()
    print "Time to calculate ave = %10.5f seconds"%(time2-time1)
    for apolloDB in apolloDBs:
    ### update simulated_population
	fipsInsertIDDict = {}
	for fips in fipsList:
	    insertID = apolloDB.checkAndAddSimulatedPopulation('newly exposed',fips)
	    fipsInsertIDDict[fips] = insertID

	time1 = time.time()
	#    for day in range(0,len(aveInfByBG)):
	#    SQLString = 'INSERT INTO time_series (run_id,population_id,time_step,pop_count)'\
	#                        +' VALUES'
	for day in range(0,numberDays):
	    if False: #day > len(aveInfByBG):
		for fips in fipsList:
		    SQLString = 'INSERT INTO time_series set '\
				+'run_id = "' + str(runInsertID) + '", '\
				+'population_id = "' + str(fipsInsertIDDict[fips]) + '", '\
				+'time_step = "' + str(day) + '", '\
				+'pop_count = "0"'
		    apolloDB.query(SQLString)
	    else:
		if(len(aveInfByBG[day].keys())>0):
		    SQLString = 'INSERT INTO time_series (run_id,population_id,time_step,pop_count)'\
				+' VALUES'

		    for fips in aveInfByBG[day].keys():
			SQLString += '('\
				     +'"'+str(runInsertID) + '", '\
				     +'"' + str(fipsInsertIDDict[fips]) + '",'\
				     +'"' + str(day) + '", '\
				     +'"' + str(int(aveInfByBG[day][fips])) + '"),'
		    SQLString = SQLString[:-1]

	            apolloDB.query(SQLString)

	time2 = time.time()
	print "Time To populate TimeSeries = %10.5f"%(time2-time1)
	apolloDB.close()
<commit_msg>Altered this script to be able to handle any state or country fips code.<commit_after>import os,sys,string
import math
import optparse
from fred import FRED,FRED_RUN,FRED_Infection_Set,FRED_Household_Set,FRED_People_Set
import apollo
import time
#from threading import Thread
from multiprocessing import Process,Queue

def fillInfection(q,j,fred_run):
        InfByBG = []
	numDays = fred_run.get_param("days")
	for i in range(0,int(numDays)):
		InfByBG.append({})

        infection_file = fred_run.infection_file_names[j]
        run_number = infection_file.split("infections")[1].split(".")[0]
        print "Run = " + run_number
        infSet = fred_run.get_infection_set(int(run_number))
        #print "Length = %d"%len(infSet.infectionList)
        for i in range(0,len(infSet.infectionList)):
		day = int(infSet.get('day',i))
		infected_id = int(infSet.get('host',i))
            #personRec = fred_population.people[int(infRec.infected_id)]
		personFips = fred_population.get("stcotrbg",infected_id)
            #if personFips == "420034754012":
            #    print "There is someone here on Day %d"%(day)
            #print personFips
	#	if personFips not in fipsList:
	#		fipsList.append(personFips)
            #print "%d %d"%(j,day)
		if InfByBG[day].has_key(personFips) is False:
			InfByBG[day][personFips] = 0.0
		InfByBG[day][personFips] += 1
	q.put(InfByBG)

if __name__ == '__main__':
    parser = optparse.OptionParser(usage="""
    %prog [--help][-k key][-r run_number][-t vis_type]
    -k or --key   Fred Key
    -r or --run   The run number of vizualize
                  ave = produce an average result
                  all = produce an input for all runs

    -w or --time  Turn on profiling
    -d or --debug Turn on debug printing
    """)

    parser.add_option("-k","--key",type="string",
                      help="The FRED key for the run you wish to visualize")
    parser.add_option("-r","--run",type="string",
                      help="The number of the run you would like to visualize (number,ave, or all)",
                      default=1)
    parser.add_option("-w","--time",action="store_true",
                      default=False)
    parser.add_option("-d","--debug",action="store_true",
                      default=False)

    opts,args=parser.parse_args()

    DBHosts = ["gaia.psc.edu","warhol-fred.psc.edu"]
    ### This only currently works for allegheny county

    #### Initialize the FRED SIMS
    fred = FRED()

    key = opts.key

    run = opts.run

    try:
        int(run)
    except:
        if run != "ave" and run != "all":
            print "Invalid parameter for run, needs to be an integer, all or ave"
            sys.exit(1)

    fred_run = FRED_RUN(fred,key)
    numberDays = int(fred_run.get_param("days"))
    outputsAve = fred_run.outputsAve

    apolloDBs = [ apollo.ApolloDB(host_=x) for x in DBHosts ]
    for apolloDB in apolloDBs:
	apolloDB.connect()

    ## Fill in the run table
    	SQLString = 'INSERT INTO run set label = "' + key + '"'
    	apolloDB.query(SQLString)
    	runInsertID = apolloDB.insertID()

    	stateList = {'S':'susceptible','E':'exposed','I':'infectious','R':'recovered'}
    	#locationList = ['42003'] # need to replace with an automatic way of getting this info
	locationList = [fred_run.get_param('fips')]
    	stateInsertIDDict = {}

    ### setup simulated_population_axis

    	axisInsertIDDict = apolloDB._populationAxis()

    ### setup simulated_populations table
    	for state in stateList.keys():
	    for location in locationList:
		stateInsertIDDict[(state,location)] =\
		  apolloDB.checkAndAddSimulatedPopulation(stateList[state],location)

    	for state in stateList.keys():
	    for location in locationList:
		for day in outputsAve:
		    populationID = stateInsertIDDict[(state,location)]
		    SQLString = 'INSERT INTO time_series set '\
				+'run_id = "' + str(runInsertID) + '", '\
				+'population_id = "' + str(populationID) + '", '\
				+'time_step = "' + str(day['Day']) +'", '\
				+'pop_count = "' + str(day[state]) + '"'
		    apolloDB.query(SQLString)

    synth_pops_dir = fred_run.get_param("synthetic_population_directory")
    synth_pop_dir = os.path.expandvars(synth_pops_dir + "/2005_2009_ver2_"+location)
    #synth_pop_id = fred_run.get_param("synthetic_population_id")
    synth_pop_prefix = os.path.expandvars(synth_pop_dir +"/2005_2009_ver2_"+location)
    synth_pop_filename = synth_pop_prefix + "_synth_people.txt.fsz"
    synth_household_filename = synth_pop_prefix + "_synth_households.txt"

    try:
        open(synth_pop_filename,"r")
    except IOError:
        print "Problem opening FRED population file " + synth_pop_filename
        sys.exit(1)

    tmpFilename = synth_pop_dir + "/tmp.txt"
    os.system("fsz -u " + synth_pop_filename + " > " + tmpFilename)
    fred_population = FRED_People_Set(tmpFilename)

    os.remove(tmpFilename)
    numDays = fred_run.get_param("days")
    aveInfByBG = []
    fipsList = []
    time1 = time.time()
    for i in range(0,int(numDays)):
	aveInfByBG.append({})
    InfByBG = []

    #for infection_file in fred_run.infection_file_names:
    queues = []
    threads = []
    for j in range(0,len(fred_run.infection_file_names)):
	queues.append(Queue())
	threads.append(Process(target=fillInfection,args=(queues[j],j,fred_run)))

    for t in threads:
	t.start()

    for q in queues:
	InfByBG.append(q.get())

    for t in threads:
	t.join()

    ### finish off averages
    numRuns = fred_run.number_of_runs
    for infFile in InfByBG:
	sum = 0
        for day in range(0,len(infFile)):
	    for fips in infFile[day].keys():
		if fips not in fipsList:
		   fipsList.append(fips)
		if fips not in aveInfByBG[day].keys():
		   aveInfByBG[day][fips]=0.0
	        aveInfByBG[day][fips] += infFile[day][fips]
		sum += infFile[day][fips]
	print "Sum = " + str(sum)

    for day in range(0,len(aveInfByBG)):
            for fips in aveInfByBG[day].keys():
                aveInfByBG[day][fips] /= float(numRuns)

    time2 = time.time()
    print "Time to calculate ave = %10.5f seconds"%(time2-time1)
    for apolloDB in apolloDBs:
    ### update simulated_population
	fipsInsertIDDict = {}
	for fips in fipsList:
	    insertID = apolloDB.checkAndAddSimulatedPopulation('newly exposed',fips)
	    fipsInsertIDDict[fips] = insertID

	time1 = time.time()
	#    for day in range(0,len(aveInfByBG)):
	#    SQLString = 'INSERT INTO time_series (run_id,population_id,time_step,pop_count)'\
	#                        +' VALUES'
	for day in range(0,numberDays):
	    if False: #day > len(aveInfByBG):
		for fips in fipsList:
		    SQLString = 'INSERT INTO time_series set '\
				+'run_id = "' + str(runInsertID) + '", '\
				+'population_id = "' + str(fipsInsertIDDict[fips]) + '", '\
				+'time_step = "' + str(day) + '", '\
				+'pop_count = "0"'
		    apolloDB.query(SQLString)
	    else:
		if(len(aveInfByBG[day].keys())>0):
		    SQLString = 'INSERT INTO time_series (run_id,population_id,time_step,pop_count)'\
				+' VALUES'

		    for fips in aveInfByBG[day].keys():
			SQLString += '('\
				     +'"'+str(runInsertID) + '", '\
				     +'"' + str(fipsInsertIDDict[fips]) + '",'\
				     +'"' + str(day) + '", '\
				     +'"' + str(int(aveInfByBG[day][fips])) + '"),'
		    SQLString = SQLString[:-1]

	            apolloDB.query(SQLString)

	time2 = time.time()
	print "Time To populate TimeSeries = %10.5f"%(time2-time1)
	apolloDB.close()
<|endoftext|>


<commit_before>import os,sys,string
import math
import optparse
from fred import FRED,FRED_RUN,FRED_Infection_Set,FRED_Locations_Set,FRED_People_Set
import apollo
import time
#from threading import Thread
from multiprocessing import Process,Queue

def fillInfection(q,j,fred_run):
        InfByBG = []
	numDays = fred_run.get_param("days")
	for i in range(0,int(numDays)):
		InfByBG.append({})

        infection_file = fred_run.infection_file_names[j]
        run_number = infection_file.split("infections")[1].split(".")[0]
        print "Run = " + run_number
        infSet = fred_run.get_infection_set(int(run_number))
        #print "Length = %d"%len(infSet.infectionList)
        for i in range(0,len(infSet.infectionList)):
		day = int(infSet.get('day',i))
		infected_id = int(infSet.get('host',i))
            #personRec = fred_population.people[int(infRec.infected_id)]
		personFips = fred_population.get("stcotrbg",infected_id)
            #if personFips == "420034754012":
            #    print "There is someone here on Day %d"%(day)
            #print personFips
	#	if personFips not in fipsList:
	#		fipsList.append(personFips)
            #print "%d %d"%(j,day)
		if InfByBG[day].has_key(personFips) is False:
			InfByBG[day][personFips] = 0.0
		InfByBG[day][personFips] += 1
	q.put(InfByBG)

if __name__ == '__main__':
    parser = optparse.OptionParser(usage="""
    %prog [--help][-k key][-r run_number][-t vis_type]
    -k or --key   Fred Key
    -r or --run   The run number of vizualize
                  ave = produce an average result
                  all = produce an input for all runs

    -w or --time  Turn on profiling
    -d or --debug Turn on debug printing
    """)

    parser.add_option("-k","--key",type="string",
                      help="The FRED key for the run you wish to visualize")
    parser.add_option("-r","--run",type="string",
                      help="The number of the run you would like to visualize (number,ave, or all)",
                      default=1)
    parser.add_option("-w","--time",action="store_true",
                      default=False)
    parser.add_option("-d","--debug",action="store_true",
                      default=False)

    opts,args=parser.parse_args()

    ### This only currently works for allegheny county

    #### Initialize the FRED SIMS
    fred = FRED()

    key = opts.key

    run = opts.run

    try:
        int(run)
    except:
        if run != "ave" and run != "all":
            print "Invalid parameter for run, needs to be an integer, all or ave"
            sys.exit(1)

    fred_run = FRED_RUN(fred,key)
    numberDays = int(fred_run.get_param("days"))
    outputsAve = fred_run.outputsAve

    apolloDB = apollo.ApolloDB()
    apolloDB.connect()

    #for day in outputsAve:
#	print "Day = %d I = %d"%(day['Day'],day['I'])

    #ApolloConn = MySQLdb.connect(host="warhol-fred.psc.edu",
#				 user="apolloext",
#				 db="apollo")
    #ApolloCursor = ApolloConn.cursor()

    ## Fill in the run table
    SQLString = 'INSERT INTO run set label = "' + key + '"'
    apolloDB.query(SQLString)
    runInsertID = apolloDB.insertID()
#    print "Run ID = " + str(runInsertID)
#    sys.exit()
    #ApolloCursor.execute(SQLString)
    #runInsertID = ApolloConn.insert_id()

    stateList = {'S':'susceptible','E':'exposed','I':'infectious','R':'recovered'}
    #axisList = ['disease_status','location']
    locationList = ['42003'] # need to replace with an automatic way of getting this info
    stateInsertIDDict = {}
    #axisInsertIDDict = {}

    ### setup simulated_population_axis
    #for axis in axisList:
    #    ApolloCursor.execute('INSERT INTO population_axis set label = "%s"'%axis)
    #    axisInsertIDDict[axis] = ApolloConn.insert_id()
    axisInsertIDDict = apolloDB._populationAxis()
    #ApolloCursor.execute("SELECT * from population_axis")
    #axisResults = ApolloCursor.fetchall()

    #for row in axisResults:
#	axisInsertIDDict[row[1]] = row[0]

#    print "Axis = " + str(axisInsertIDDict)
#    sys.exit()
    ### setup simulated_populations table
    for state in stateList.keys():
        for location in locationList:
            populationLabel = str(stateList[state]) + " in " + location
            SQLString = 'INSERT INTO simulated_population set label = "' + populationLabel + '"'
            #result = ApolloCursor.execute(SQLString)
	    apolloDB.query(SQLString)
            stateInsertIDDict[(state,location)] = apolloDB.insertID()

    ### Set up the simulated population axis value table
    for state in stateList.keys():
        for location in locationList:
            populationLabelID = stateInsertIDDict[(state,location)]
            axisID = axisInsertIDDict["disease_state"]
            apolloDB.query('INSERT INTO simulated_population_axis_value set ' +\
			   'population_id = "' + str(populationLabelID) + '", ' +\
			   'axis_id = "' + str(axisID) + '", ' +\
			   'value = "' + stateList[state]  + '"')
            axisID = axisInsertIDDict['location']
            apolloDB.query('INSERT INTO simulated_population_axis_value set ' +\
			   'population_id = "' + str(populationLabelID) + '", ' +\
			   'axis_id = "' + str(axisID) + '", ' +\
			   'value = "' + location  + '"')

    for state in stateList.keys():
        for location in locationList:
            for day in outputsAve:
                populationID = stateInsertIDDict[(state,location)]
                SQLString = 'INSERT INTO time_series set '\
                    +'run_id = "' + str(runInsertID) + '", '\
                    +'population_id = "' + str(populationID) + '", '\
                    +'time_step = "' + str(day['Day']) +'", '\
                    +'pop_count = "' + str(day[state]) + '"'
		apolloDB.query(SQLString)
                #result = ApolloCursor.execute(SQLString)

    synth_pop_dir = fred_run.get_param("synthetic_population_directory")
    synth_pop_id = fred_run.get_param("synthetic_population_id")
    synth_pop_prefix = os.path.expandvars(synth_pop_dir + "/" + synth_pop_id + "/" + synth_pop_id)
    synth_pop_filename = synth_pop_prefix + "_synth_people.txt"
    synth_household_filename = synth_pop_prefix + "_synth_households.txt"

    try:
        open(synth_pop_filename,"r")
    except IOError:
        print "Problem opening FRED population file " + synth_pop_filename
        sys.exit(1)

    fred_population = FRED_People_Set(synth_pop_filename)

    numDays = fred_run.get_param("days")
    aveInfByBG = []
    fipsList = []
    time1 = time.time()
    for i in range(0,int(numDays)):
	aveInfByBG.append({})
    InfByBG = []
#    for i in range(0,len(fred_run.infection_file_names)):
#	InfByBG.append([])
#	for j in range(0,int(numDays)):
#	    InfByBG[i].append({})
#
    #for infection_file in fred_run.infection_file_names:
    queues = []
    threads = []
    for j in range(0,len(fred_run.infection_file_names)):
	queues.append(Queue())
	threads.append(Process(target=fillInfection,args=(queues[j],j,fred_run)))

    for t in threads:
	t.start()

    for q in queues:
	InfByBG.append(q.get())

    for t in threads:
	t.join()
#	infection_file = fred_run.infection_file_names[j]
#	run_number = infection_file.split("infections")[1].split(".")[0]
#	print "Run = " + run_number
#	infSet = fred_run.get_infection_set(int(run_number))
#.       #print "Length = %d"%len(infSet.infectionList)
#for i in range(0,len(infSet.infectionList)):
#    day = int(infSet.get('day',i))
#    infected_id = int(infSet.get('host',i))
#    #personRec = fred_population.people[int(infRec.infected_id)]
#    personFips = fred_population.get("stcotrbg",infected_id)
#           #if personFips == "420034754012":
#           #    print "There is someone here on Day %d"%(day)
#           #print personFips
#           if personFips not in fipsList:
#               fipsList.append(personFips)
#    #print "%d %d"%(j,day)
#    if InfByBG[j][day].has_key(personFips) is False:
#	InfByBG[j][day][personFips] = 0.0
#    InfByBG[j][day][personFips] += 1

    ### finish off averages
    numRuns = fred_run.number_of_runs
    for infFile in InfByBG:
	sum = 0
        for day in range(0,len(infFile)):
	    for fips in infFile[day].keys():
		if fips not in fipsList:
		   fipsList.append(fips)
		if fips not in aveInfByBG[day].keys():
		   aveInfByBG[day][fips]=0.0
	        aveInfByBG[day][fips] += infFile[day][fips]
		sum += infFile[day][fips]
	print "Sum = " + str(sum)

    for day in range(0,len(aveInfByBG)):
            for fips in aveInfByBG[day].keys():
                aveInfByBG[day][fips] /= float(numRuns)

    time2 = time.time()
    print "Time to calculate ave = %10.5f seconds"%(time2-time1)
    ### update simulated_population
    fipsInsertIDDict = {}
    #print 'fips list = ' + str(fipsList)
    for fips in fipsList:
        apolloDB.query('INSERT INTO simulated_population set label = "newly exposed in '+fips + '"')
        insertID = apolloDB.insertID()
        fipsInsertIDDict[fips] = insertID
        #if fips == "420034754012":
        #    print "Fips = " + str(fips) + " id = " + str(insertID)
        axisID = axisInsertIDDict['disease_state']
        apolloDB.query('INSERT INTO simulated_population_axis_value set ' +\
		       'population_id = "' + str(insertID) + '", ' +\
		       'axis_id = "' + str(axisID) + '", ' +\
		       'value = "newly exposed"')
        axisID = axisInsertIDDict['location']

        apolloDB.query('INSERT INTO simulated_population_axis_value set ' +\
		       'population_id = "' + str(insertID) + '", ' +\
		       'axis_id = "' + str(axisID) + '", ' +\
		       'value = "' + fips + '"')

    time1 = time.time()
#    for day in range(0,len(aveInfByBG)):
    for day in range(0,numberDays):
        if False: #day > len(aveInfByBG):
            for fips in fipsList:
                SQLString = 'INSERT INTO time_series set '\
                        +'run_id = "' + str(runInsertID) + '", '\
                        +'population_id = "' + str(fipsInsertIDDict[fips]) + '", '\
                        +'time_step = "' + str(day) + '", '\
                        +'pop_count = "0"'
                apolloDB.query(SQLString)
        else:
            for fips in aveInfByBG[day].keys():
            #print "Day = " + str(day) + " and Fips: " + fips
            #print "ave = " + str(int(aveInfByBG[day][fips]))
                SQLString = 'INSERT INTO time_series set '\
                    +'run_id = "' + str(runInsertID) + '", '\
                    +'population_id = "' + str(fipsInsertIDDict[fips]) + '", '\
                    +'time_step = "' + str(day) + '", '\
                    +'pop_count = "' + str(int(aveInfByBG[day][fips])) + '"'
                apolloDB.query(SQLString)

    time2 = time.time()
    print "Time To populate TimeSeries = %10.5f"%(time2-time1)
    apolloDB.close()

<commit_msg>This now only creates one set of simulated population, and only if it does not already exist (the functionality is actually in apollo.py).  <commit_after>import os,sys,string
import math
import optparse
from fred import FRED,FRED_RUN,FRED_Infection_Set,FRED_Locations_Set,FRED_People_Set
import apollo
import time
#from threading import Thread
from multiprocessing import Process,Queue

def fillInfection(q,j,fred_run):
        InfByBG = []
	numDays = fred_run.get_param("days")
	for i in range(0,int(numDays)):
		InfByBG.append({})

        infection_file = fred_run.infection_file_names[j]
        run_number = infection_file.split("infections")[1].split(".")[0]
        print "Run = " + run_number
        infSet = fred_run.get_infection_set(int(run_number))
        #print "Length = %d"%len(infSet.infectionList)
        for i in range(0,len(infSet.infectionList)):
		day = int(infSet.get('day',i))
		infected_id = int(infSet.get('host',i))
            #personRec = fred_population.people[int(infRec.infected_id)]
		personFips = fred_population.get("stcotrbg",infected_id)
            #if personFips == "420034754012":
            #    print "There is someone here on Day %d"%(day)
            #print personFips
	#	if personFips not in fipsList:
	#		fipsList.append(personFips)
            #print "%d %d"%(j,day)
		if InfByBG[day].has_key(personFips) is False:
			InfByBG[day][personFips] = 0.0
		InfByBG[day][personFips] += 1
	q.put(InfByBG)

if __name__ == '__main__':
    parser = optparse.OptionParser(usage="""
    %prog [--help][-k key][-r run_number][-t vis_type]
    -k or --key   Fred Key
    -r or --run   The run number of vizualize
                  ave = produce an average result
                  all = produce an input for all runs

    -w or --time  Turn on profiling
    -d or --debug Turn on debug printing
    """)

    parser.add_option("-k","--key",type="string",
                      help="The FRED key for the run you wish to visualize")
    parser.add_option("-r","--run",type="string",
                      help="The number of the run you would like to visualize (number,ave, or all)",
                      default=1)
    parser.add_option("-w","--time",action="store_true",
                      default=False)
    parser.add_option("-d","--debug",action="store_true",
                      default=False)

    opts,args=parser.parse_args()

    ### This only currently works for allegheny county

    #### Initialize the FRED SIMS
    fred = FRED()

    key = opts.key

    run = opts.run

    try:
        int(run)
    except:
        if run != "ave" and run != "all":
            print "Invalid parameter for run, needs to be an integer, all or ave"
            sys.exit(1)

    fred_run = FRED_RUN(fred,key)
    numberDays = int(fred_run.get_param("days"))
    outputsAve = fred_run.outputsAve

    apolloDB = apollo.ApolloDB()
    apolloDB.connect()

    ## Fill in the run table
    SQLString = 'INSERT INTO run set label = "' + key + '"'
    apolloDB.query(SQLString)
    runInsertID = apolloDB.insertID()

    stateList = {'S':'susceptible','E':'exposed','I':'infectious','R':'recovered'}
    locationList = ['42003'] # need to replace with an automatic way of getting this info
    stateInsertIDDict = {}

    ### setup simulated_population_axis

    axisInsertIDDict = apolloDB._populationAxis()

    ### setup simulated_populations table
    for state in stateList.keys():
        for location in locationList:
	    stateInsertIDDict[(state,location)] =\
			  apolloDB.checkAndAddSimulatedPopulation(state,location)

    for state in stateList.keys():
        for location in locationList:
            for day in outputsAve:
                populationID = stateInsertIDDict[(state,location)]
                SQLString = 'INSERT INTO time_series set '\
                    +'run_id = "' + str(runInsertID) + '", '\
                    +'population_id = "' + str(populationID) + '", '\
                    +'time_step = "' + str(day['Day']) +'", '\
                    +'pop_count = "' + str(day[state]) + '"'
		apolloDB.query(SQLString)

    synth_pop_dir = fred_run.get_param("synthetic_population_directory")
    synth_pop_id = fred_run.get_param("synthetic_population_id")
    synth_pop_prefix = os.path.expandvars(synth_pop_dir + "/" + synth_pop_id + "/" + synth_pop_id)
    synth_pop_filename = synth_pop_prefix + "_synth_people.txt"
    synth_household_filename = synth_pop_prefix + "_synth_households.txt"

    try:
        open(synth_pop_filename,"r")
    except IOError:
        print "Problem opening FRED population file " + synth_pop_filename
        sys.exit(1)

    fred_population = FRED_People_Set(synth_pop_filename)

    numDays = fred_run.get_param("days")
    aveInfByBG = []
    fipsList = []
    time1 = time.time()
    for i in range(0,int(numDays)):
	aveInfByBG.append({})
    InfByBG = []

    #for infection_file in fred_run.infection_file_names:
    queues = []
    threads = []
    for j in range(0,len(fred_run.infection_file_names)):
	queues.append(Queue())
	threads.append(Process(target=fillInfection,args=(queues[j],j,fred_run)))

    for t in threads:
	t.start()

    for q in queues:
	InfByBG.append(q.get())

    for t in threads:
	t.join()

    ### finish off averages
    numRuns = fred_run.number_of_runs
    for infFile in InfByBG:
	sum = 0
        for day in range(0,len(infFile)):
	    for fips in infFile[day].keys():
		if fips not in fipsList:
		   fipsList.append(fips)
		if fips not in aveInfByBG[day].keys():
		   aveInfByBG[day][fips]=0.0
	        aveInfByBG[day][fips] += infFile[day][fips]
		sum += infFile[day][fips]
	print "Sum = " + str(sum)

    for day in range(0,len(aveInfByBG)):
            for fips in aveInfByBG[day].keys():
                aveInfByBG[day][fips] /= float(numRuns)

    time2 = time.time()
    print "Time to calculate ave = %10.5f seconds"%(time2-time1)
    ### update simulated_population
    fipsInsertIDDict = {}
    #print 'fips list = ' + str(fipsList)
    for fips in fipsList:
	insertID = apolloDB.checkAndAddSimulatedPopulation('newly exposed',fips)

        fipsInsertIDDict[fips] = insertID

    time1 = time.time()
#    for day in range(0,len(aveInfByBG)):
#    SQLString = 'INSERT INTO time_series (run_id,population_id,time_step,pop_count)'\
#                        +' VALUES'
    for day in range(0,numberDays):
        if False: #day > len(aveInfByBG):
            for fips in fipsList:
                SQLString = 'INSERT INTO time_series set '\
                        +'run_id = "' + str(runInsertID) + '", '\
                        +'population_id = "' + str(fipsInsertIDDict[fips]) + '", '\
                        +'time_step = "' + str(day) + '", '\
                        +'pop_count = "0"'
                apolloDB.query(SQLString)
        else:
	    if(len(aveInfByBG[day].keys())>0):
		SQLString = 'INSERT INTO time_series (run_id,population_id,time_step,pop_count)'\
                        +' VALUES'

            	for fips in aveInfByBG[day].keys():
            #print "Day = " + str(day) + " and Fips: " + fips
            #print "ave = " + str(int(aveInfByBG[day][fips]))
                	SQLString += '('\
                    		+'"'+str(runInsertID) + '", '\
                    		+'"' + str(fipsInsertIDDict[fips]) + '",'\
                    		+'"' + str(day) + '", '\
                    		+'"' + str(int(aveInfByBG[day][fips])) + '"),'
    		SQLString = SQLString[:-1]

    		apolloDB.query(SQLString)

    time2 = time.time()
    print "Time To populate TimeSeries = %10.5f"%(time2-time1)
    apolloDB.close()

<|endoftext|>


<commit_before>#!/usr/bin/env python3

# written by sqall
# twitter: https://twitter.com/sqall01
# blog: https://h4des.org
# github: https://github.com/sqall01
#
# Licensed under the GNU Affero General Public License, version 3.

import re
import os
import time
from typing import Optional, Union
from .number import _NumberSensor
from ..globalData.sensorObjects import SensorDataFloat, SensorDataInt, SensorDataType, SensorErrorState

class RaspberryPiDS18b20Sensor(_NumberSensor):
    """
    Represents one DS18b20 sensor connected to the Raspberry Pi.
    """

    def __init__(self):
        _NumberSensor.__init__(self)

        self._unit = "°C"

        # Used for logging.
        self._log_tag = os.path.basename(__file__)

        # Set sensor to hold float data.
        self.sensorDataType = SensorDataType.FLOAT
        self.data = None  # Sensor initialization fails if no data could be collected.

        # The file of the sensor that should be parsed.
        self._sensor_file = None

        # The name of the sensor that should be parsed.
        self.sensorName = None

        # The interval in seconds in which an update of the current held data
        # should be sent to the server.
        self.interval = None

        # The time the last update of the data was sent to the server.
        self._last_update = None

        self._last_temperature_update = 0.0

        # As long as errors occurring during the fetching of data are encoded as negative values,
        # we need the lowest value that we use for our threshold check.
        self._sane_lowest_value = -273.0

        # This sensor type string is used for log messages.
        self._log_desc = "Temperature"

    def _get_data(self) -> Optional[Union[SensorDataInt, SensorDataFloat]]:
        """
        Internal function that reads the data of the DS18b20 sensor.

        :return: temperature value or None
        """

        utc_timestamp = int(time.time())
        if (utc_timestamp - self._last_temperature_update) > self.interval:
            self._last_temperature_update = utc_timestamp

            try:
                with open(self._sensor_file, 'r') as fp:

                    # File content looks like this:
                    # 2d 00 4b 46 ff ff 04 10 b3 : crc=b3 YES
                    # 2d 00 4b 46 ff ff 04 10 b3 t=22500
                    fp.readline()
                    line = fp.readline()

                    reMatch = re.match("([0-9a-f]{2} ){9}t=([+-]?[0-9]+)", line)
                    if reMatch:
                        return SensorDataFloat(float(reMatch.group(2)) / 1000, self._unit)

                    else:
                        self._log_error(self._log_tag, "Could not parse DS18B20 sensor file.")
                        self._set_error_state(SensorErrorState.ProcessingError, "Could not parse DS18B20 sensor file.")

            except Exception as e:
                self._log_exception(self._log_tag, "Could not read DS18B20 sensor file.")
                self._set_error_state(SensorErrorState.ProcessingError, "Could not read DS18B20 sensor file: " + str(e))

            return None

        else:
            return self.data

    def initialize(self) -> bool:
        self.state = 1 - self.triggerState

        self._sensor_file = "/sys/bus/w1/devices/" \
                            + self.sensorName \
                            + "/w1_slave"

        # First time the temperature is updated is done in a blocking way.
        utc_timestamp = int(time.time())
        self.data = self._get_data()
        self._last_temperature_update = utc_timestamp

        if self.data is None:
            return False

        self._last_update = utc_timestamp
        self._optional_data = {"sensorName": self.sensorName}

        return True
<commit_msg>changed parsing error logging output<commit_after>#!/usr/bin/env python3

# written by sqall
# twitter: https://twitter.com/sqall01
# blog: https://h4des.org
# github: https://github.com/sqall01
#
# Licensed under the GNU Affero General Public License, version 3.

import re
import os
import time
from typing import Optional, Union
from .number import _NumberSensor
from ..globalData.sensorObjects import SensorDataFloat, SensorDataInt, SensorDataType, SensorErrorState

class RaspberryPiDS18b20Sensor(_NumberSensor):
    """
    Represents one DS18b20 sensor connected to the Raspberry Pi.
    """

    def __init__(self):
        _NumberSensor.__init__(self)

        self._unit = "°C"

        # Used for logging.
        self._log_tag = os.path.basename(__file__)

        # Set sensor to hold float data.
        self.sensorDataType = SensorDataType.FLOAT
        self.data = None  # Sensor initialization fails if no data could be collected.

        # The file of the sensor that should be parsed.
        self._sensor_file = None

        # The name of the sensor that should be parsed.
        self.sensorName = None

        # The interval in seconds in which an update of the current held data
        # should be sent to the server.
        self.interval = None

        # The time the last update of the data was sent to the server.
        self._last_update = None

        self._last_temperature_update = 0.0

        # As long as errors occurring during the fetching of data are encoded as negative values,
        # we need the lowest value that we use for our threshold check.
        self._sane_lowest_value = -273.0

        # This sensor type string is used for log messages.
        self._log_desc = "Temperature"

    def _get_data(self) -> Optional[Union[SensorDataInt, SensorDataFloat]]:
        """
        Internal function that reads the data of the DS18b20 sensor.

        :return: temperature value or None
        """

        utc_timestamp = int(time.time())
        if (utc_timestamp - self._last_temperature_update) > self.interval:
            self._last_temperature_update = utc_timestamp

            try:
                with open(self._sensor_file, 'r') as fp:

                    # File content looks like this:
                    # 2d 00 4b 46 ff ff 04 10 b3 : crc=b3 YES
                    # 2d 00 4b 46 ff ff 04 10 b3 t=22500
                    fp.readline()
                    line = fp.readline()

                    reMatch = re.match("([0-9a-f]{2} ){9}t=([+-]?[0-9]+)", line)
                    if reMatch:
                        return SensorDataFloat(float(reMatch.group(2)) / 1000, self._unit)

                    else:
                        self._log_error(self._log_tag, "Could not parse DS18B20 sensor line: %s" % line)
                        self._set_error_state(SensorErrorState.ProcessingError,
                                              "Could not parse DS18B20 sensor line: %s" % line)

            except Exception as e:
                self._log_exception(self._log_tag, "Could not read DS18B20 sensor file.")
                self._set_error_state(SensorErrorState.ProcessingError, "Could not read DS18B20 sensor file: " + str(e))

            return None

        else:
            return self.data

    def initialize(self) -> bool:
        self.state = 1 - self.triggerState

        self._sensor_file = "/sys/bus/w1/devices/" \
                            + self.sensorName \
                            + "/w1_slave"

        # First time the temperature is updated is done in a blocking way.
        utc_timestamp = int(time.time())
        self.data = self._get_data()
        self._last_temperature_update = utc_timestamp

        if self.data is None:
            return False

        self._last_update = utc_timestamp
        self._optional_data = {"sensorName": self.sensorName}

        return True
<|endoftext|>


<commit_before># -*- coding: utf8 -*-

__author__ = 'sergey'

"""
Class for Old Zstd v0.0.1 compression helper
"""

from dedupsqlfs.compression import BaseCompression

class Zstd001Compression(BaseCompression):

    _method_name = "zstd001"

    _minimal_size = 26

    _has_comp_level_options = False

    def isDataMayBeCompressed(self, data):
        """
        Disallow compression
        It's deprecated version is ZSTD

        @param data:
        @return:
        """
        return False

    pass
<commit_msg>Try to decompress all versions here<commit_after># -*- coding: utf8 -*-

__author__ = 'sergey'

"""
Class for Old Zstd v0.0.1 compression helper
"""

from dedupsqlfs.compression import BaseCompression

class Zstd001Compression(BaseCompression):

    _method_name = "zstd001"

    _minimal_size = 26

    _has_comp_level_options = False

    _func_decomp_new = None

    def _init_module(self):
        super()._init_module()

        module = __import__('zstd')
        self._func_decomp_new = getattr(module, "decompress")
        return

    def isDataMayBeCompressed(self, data):
        """
        Disallow compression
        It's deprecated version is ZSTD

        @param data:
        @return:
        """
        return False

    def decompressData(self, cdata):
        """
        Try to decompress by old version
        If can't - try new version

        Thats because maybe some data was compressed before migrations arraived

        @param cdata:
        @return:
        """
        try:
            data = super().decompressData(cdata)
        except (Exception):
            data = False
            pass

        if data is False:
            data = self._func_decomp_new(cdata)

        return data
<|endoftext|>


<commit_before>## $Id$
##
## This file is part of CDS Invenio.
## Copyright (C) 2002, 2003, 2004, 2005, 2006, 2007, 2008 CERN.
##
## CDS Invenio is free software; you can redistribute it and/or
## modify it under the terms of the GNU General Public License as
## published by the Free Software Foundation; either version 2 of the
## License, or (at your option) any later version.
##
## CDS Invenio is distributed in the hope that it will be useful, but
## WITHOUT ANY WARRANTY; without even the implied warranty of
## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
## General Public License for more details.
##
## You should have received a copy of the GNU General Public License
## along with CDS Invenio; if not, write to the Free Software Foundation, Inc.,
## 59 Temple Place, Suite 330, Boston, MA 02111-1307, USA.

"""CDS Invenio BibCirculation Administrator Interface."""

__revision__ = ""

import sys
import invenio.bibcirculationadminlib as bal
from invenio.config import CFG_SITE_LANG
from invenio.urlutils import wash_url_argument
"""
http://cdsweb.cern.ch/admin/bibcirculation/bibcirculationadmin.py
"""

def index(req, ln=CFG_SITE_LANG):
    """
    """
    return bal.index(req, ln)

def manage_holdings(req, ln=CFG_SITE_LANG):
    """
    """
    return bal.manage_holdings(req, ln)

def borrower_search(req, ln=CFG_SITE_LANG):
    """
    http://cdsweb.cern.ch/admin/bibcirculation/bibcirculationadmin.py/borrowers_search

    """
    return bal.borrower_search(req, ln)

def borrower_search_test(req, column=None, string=None,
                         user=None, ln=CFG_SITE_LANG):
    """
    http://cdsweb.cern.ch/admin/bibcirculation/bibcirculationadmin.py/borrowers_search_test

    """
    return bal.borrower_search_test(req, column, string, user, ln)

def holdings_search(req, ln=CFG_SITE_LANG):
    """
    http://cdsweb.cern.ch/admin/bibcirculation/bibcirculationadmin.py/holdings_search

    """
    return bal.holdings_search(req, ln)

def borrower_notification(req, borrower_id=None,
                          template=None, to_borrower=None,
                          message=None, search_button=None,
                          load_button=None, string=None,
                          column=None, subject=None,
                          send_button=None, ln=CFG_SITE_LANG):
    """
    http://cdsweb.cern.ch/admin/bibcirculation/bibcirculationadmin.py/borrower_notification

    """
    return bal.borrower_notification(req, borrower_id, template,
                                     to_borrower, message, search_button,
                                     load_button, string, column, subject,
                                     send_button, ln)

def get_pending_loan_request(req, show=None, ln=CFG_SITE_LANG):
    """
    http://cdsweb.cern.ch/admin/bibcirculation/bibcirculationadmin.py/get_pending_loan_request

    """
    return bal.get_pending_loan_request(req, show, ln)

def item_search_result(req, p=None, f=None, ln=CFG_SITE_LANG):
    """
    http://cdsweb.cern.ch/admin/bibcirculation/bibcirculationadmin.py/item_search_result
    """

    return bal.item_search_result(req, p, f, ln)

def loan_return(req, ln=CFG_SITE_LANG):
    """
    http://cdsweb.cern.ch/admin/bibcirculation/bibcirculationadmin.py/loan_return

    """
    return bal.loan_return(req, ln)

def loan_on_desk(req, column=None, string=None,
                 borrower_id=None, confirm_button=None,
                 barcode=None, borrower_name=None, ln=CFG_SITE_LANG):
    """
    http://cdsweb.cern.ch/admin/bibcirculation/bibcirculationadmin.py/loan_on_desk

    """

    return bal.loan_on_desk(req, column, string,
                            borrower_id, confirm_button,
                            barcode, borrower_name, ln)

def loan_on_desk_confirm(req, barcode=None, borrower_id=None, ln=CFG_SITE_LANG):
    """
    http://cdsweb.cern.ch/admin/bibcirculation/bibcirculationadmin.py/loan_on_desk_confirm

    """
    return bal.loan_on_desk_confirm(req, barcode, borrower_id, ln)

def register_new_loan(req, barcode=None, borrower_id=None, ln=CFG_SITE_LANG):
    """
    http://cdsweb.cern.ch/admin/bibcirculation/bibcirculationadmin.py/register_new_loan

    """
    return bal.register_new_loan(req, barcode, borrower_id, ln)

def loan_return_confirm(req, barcode=None, ln=CFG_SITE_LANG):
    """
    http://cdsweb.cern.ch/admin/bibcirculation/bibcirculationadmin.py/loan_return_confirm

    """
    return bal.loan_return_confirm(req, barcode, ln)

def get_next_waiting_loan_request(req, recID=None, barcode=None,
                                  ln=CFG_SITE_LANG):
    """
    http://cdsweb.cern.ch/admin/bibcirculation/bibcirculationadmin.py/get_next_waiting_loan_request

    """
    return bal.get_next_waiting_loan_request(req, recID, barcode, ln)

def update_loan_request_status(req, check_id_list=None, approve_button=None,
                               cancel_button=None, ln=CFG_SITE_LANG):
    """
    http://cdsweb.cern.ch/admin/bibcirculation/bibcirculationadmin.py/update_loan_request_status

    """

    check_id_list = wash_url_argument(check_id_list, 'list')

    return bal.update_loan_request_status(req, check_id_list,
                                          approve_button, cancel_button, ln)

def update_next_loan_request_status(req, check_id=None, approve_button=None,
                                    cancel_button=None, barcode=None,
                                    ln=CFG_SITE_LANG):
    """
    http://cdsweb.cern.ch/admin/bibcirculation/bibcirculationadmin.py/update_next_loan_request_status

    """
    return bal.update_next_loan_request_status(req, check_id,
                                               approve_button,
                                               cancel_button,
                                               barcode, ln)

def all_requests(req, orderby=None, ln=CFG_SITE_LANG):
    """
    http://cdsweb.cern.ch/admin/bibcirculation/bibcirculationadmin.py/all_requests

    """
    return bal.all_requests(req, orderby, ln)

def all_loans(req, show=None, ln=CFG_SITE_LANG):
    """
    http://cdsweb.cern.ch/admin/bibcirculation/bibcirculationadmin.py/all_loans

    """
    return bal.all_loans(req, show, ln)

def all_requests_for_item(req, recid=None, orderby=None, ln=CFG_SITE_LANG):
    """
    http://cdsweb.cern.ch/admin/bibcirculation/bibcirculationadmin.py/borrowers_search

    """
    return bal.get_all_requests_for_item(req, recid, orderby, ln)

def get_borrower_details(req, ln=CFG_SITE_LANG, borrower_id=None):
    """
    http://cdsweb.cern.ch/admin/bibcirculation/bibcirculationadmin.py/borrowers_search

    """
    return bal.get_borrower_details(req, ln, borrower_id)

def get_item_details(req, ln=CFG_SITE_LANG, recid=None):
    """
    http://cdsweb.cern.ch/admin/bibcirculation/bibcirculationadmin.py/borrowers_search

    """
    return bal.get_item_details(req, ln, recid)

def get_library_details(req, ln=CFG_SITE_LANG, libid=None):
    """
    http://cdsweb.cern.ch/admin/bibcirculation/bibcirculationadmin.py/borrowers_search

    """
    return bal.get_library_details(req, ln, libid)

def get_borrower_requests_details(req, orderby=None, ln=CFG_SITE_LANG,
                                  notify_button=None, borrower_id=None):
    """
    http://cdsweb.cern.ch/admin/bibcirculation/bibcirculationadmin.py/borrowers_search

    """
    return bal.get_borrower_requests_details(req, orderby, ln,
                                             notify_button, borrower_id)

def get_borrower_loans_details(req, orderby=None, ln=CFG_SITE_LANG,
                               notify_button=None, barcode=None,
                               borrower_id=None):
    """
    http://cdsweb.cern.ch/admin/bibcirculation/bibcirculationadmin.py/borrowers_search

    """
    return bal.get_borrower_loans_details(req, orderby, ln,
                                          notify_button, barcode, borrower_id)

def search_result(req, column, str, ln=CFG_SITE_LANG):
    """
    """
    return bal.search_result(req, column, str, ln)

def help_contactsupport(req, ln=CFG_SITE_LANG):
    """
    http://cdsweb.cern.ch/admin/bibcirculation/bibcirculationadmin.py/borrowers_search

    """
    return bal.help_contactsupport(req, ln)

<commit_msg>Changes on method loan_on_desk.<commit_after>## $Id$
##
## This file is part of CDS Invenio.
## Copyright (C) 2002, 2003, 2004, 2005, 2006, 2007, 2008 CERN.
##
## CDS Invenio is free software; you can redistribute it and/or
## modify it under the terms of the GNU General Public License as
## published by the Free Software Foundation; either version 2 of the
## License, or (at your option) any later version.
##
## CDS Invenio is distributed in the hope that it will be useful, but
## WITHOUT ANY WARRANTY; without even the implied warranty of
## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
## General Public License for more details.
##
## You should have received a copy of the GNU General Public License
## along with CDS Invenio; if not, write to the Free Software Foundation, Inc.,
## 59 Temple Place, Suite 330, Boston, MA 02111-1307, USA.

"""CDS Invenio BibCirculation Administrator Interface."""

__revision__ = ""

import sys
import invenio.bibcirculationadminlib as bal
from invenio.config import CFG_SITE_LANG
from invenio.urlutils import wash_url_argument
"""
http://cdsweb.cern.ch/admin/bibcirculation/bibcirculationadmin.py
"""

def index(req, ln=CFG_SITE_LANG):
    """
    """
    return bal.index(req, ln)

def manage_holdings(req, ln=CFG_SITE_LANG):
    """
    """
    return bal.manage_holdings(req, ln)

def borrower_search(req, ln=CFG_SITE_LANG):
    """
    http://cdsweb.cern.ch/admin/bibcirculation/bibcirculationadmin.py/borrowers_search

    """
    return bal.borrower_search(req, ln)

def borrower_search_test(req, column=None, string=None,
                         user=None, ln=CFG_SITE_LANG):
    """
    http://cdsweb.cern.ch/admin/bibcirculation/bibcirculationadmin.py/borrowers_search_test

    """
    return bal.borrower_search_test(req, column, string, user, ln)

def holdings_search(req, ln=CFG_SITE_LANG):
    """
    http://cdsweb.cern.ch/admin/bibcirculation/bibcirculationadmin.py/holdings_search

    """
    return bal.holdings_search(req, ln)

def borrower_notification(req, borrower_id=None,
                          template=None, to_borrower=None,
                          message=None, search_button=None,
                          load_button=None, string=None,
                          column=None, subject=None,
                          send_button=None, ln=CFG_SITE_LANG):
    """
    http://cdsweb.cern.ch/admin/bibcirculation/bibcirculationadmin.py/borrower_notification

    """
    return bal.borrower_notification(req, borrower_id, template,
                                     to_borrower, message, search_button,
                                     load_button, string, column, subject,
                                     send_button, ln)

def get_pending_loan_request(req, show=None, ln=CFG_SITE_LANG):
    """
    http://cdsweb.cern.ch/admin/bibcirculation/bibcirculationadmin.py/get_pending_loan_request

    """
    return bal.get_pending_loan_request(req, show, ln)

def item_search_result(req, p=None, f=None, ln=CFG_SITE_LANG):
    """
    http://cdsweb.cern.ch/admin/bibcirculation/bibcirculationadmin.py/item_search_result
    """

    return bal.item_search_result(req, p, f, ln)

def loan_return(req, ln=CFG_SITE_LANG):
    """
    http://cdsweb.cern.ch/admin/bibcirculation/bibcirculationadmin.py/loan_return

    """
    return bal.loan_return(req, ln)

def loan_on_desk(req, column=None, string=None,
                 borrower="", confirm_button=None,
                 barcode="", borrower_name=None, ln=CFG_SITE_LANG):
    """
    http://cdsweb.cern.ch/admin/bibcirculation/bibcirculationadmin.py/loan_on_desk

    """

    return bal.loan_on_desk(req, column, string,
                            borrower, confirm_button,
                            barcode, borrower_name, ln)

def loan_on_desk_confirm(req, barcode=None, borrower_id=None, ln=CFG_SITE_LANG):
    """
    http://cdsweb.cern.ch/admin/bibcirculation/bibcirculationadmin.py/loan_on_desk_confirm

    """
    return bal.loan_on_desk_confirm(req, barcode, borrower_id, ln)

def register_new_loan(req, barcode=None, borrower_id=None, ln=CFG_SITE_LANG):
    """
    http://cdsweb.cern.ch/admin/bibcirculation/bibcirculationadmin.py/register_new_loan

    """
    barcode = wash_url_argument(barcode, 'list')

    return bal.register_new_loan(req, barcode, borrower_id, ln)

def loan_return_confirm(req, barcode=None, ln=CFG_SITE_LANG):
    """
    http://cdsweb.cern.ch/admin/bibcirculation/bibcirculationadmin.py/loan_return_confirm

    """
    return bal.loan_return_confirm(req, barcode, ln)

def get_next_waiting_loan_request(req, recID=None, barcode=None,
                                  ln=CFG_SITE_LANG):
    """
    http://cdsweb.cern.ch/admin/bibcirculation/bibcirculationadmin.py/get_next_waiting_loan_request

    """
    return bal.get_next_waiting_loan_request(req, recID, barcode, ln)

def update_loan_request_status(req, check_id_list=None, approve_button=None,
                               cancel_button=None, ln=CFG_SITE_LANG):
    """
    http://cdsweb.cern.ch/admin/bibcirculation/bibcirculationadmin.py/update_loan_request_status

    """

    check_id_list = wash_url_argument(check_id_list, 'list')

    return bal.update_loan_request_status(req, check_id_list,
                                          approve_button, cancel_button, ln)

def update_next_loan_request_status(req, check_id=None, approve_button=None,
                                    cancel_button=None, barcode=None,
                                    ln=CFG_SITE_LANG):
    """
    http://cdsweb.cern.ch/admin/bibcirculation/bibcirculationadmin.py/update_next_loan_request_status

    """
    return bal.update_next_loan_request_status(req, check_id,
                                               approve_button,
                                               cancel_button,
                                               barcode, ln)

def all_requests(req, orderby=None, ln=CFG_SITE_LANG):
    """
    http://cdsweb.cern.ch/admin/bibcirculation/bibcirculationadmin.py/all_requests

    """
    return bal.all_requests(req, orderby, ln)

def all_loans(req, show=None, ln=CFG_SITE_LANG):
    """
    http://cdsweb.cern.ch/admin/bibcirculation/bibcirculationadmin.py/all_loans

    """
    return bal.all_loans(req, show, ln)

def all_requests_for_item(req, recid=None, orderby=None, ln=CFG_SITE_LANG):
    """
    http://cdsweb.cern.ch/admin/bibcirculation/bibcirculationadmin.py/borrowers_search

    """
    return bal.get_all_requests_for_item(req, recid, orderby, ln)

def get_borrower_details(req, ln=CFG_SITE_LANG, borrower_id=None):
    """
    http://cdsweb.cern.ch/admin/bibcirculation/bibcirculationadmin.py/borrowers_search

    """
    return bal.get_borrower_details(req, ln, borrower_id)

def get_item_details(req, ln=CFG_SITE_LANG, recid=None):
    """
    http://cdsweb.cern.ch/admin/bibcirculation/bibcirculationadmin.py/borrowers_search

    """
    return bal.get_item_details(req, ln, recid)

def get_library_details(req, ln=CFG_SITE_LANG, libid=None):
    """
    http://cdsweb.cern.ch/admin/bibcirculation/bibcirculationadmin.py/borrowers_search

    """
    return bal.get_library_details(req, ln, libid)

def get_borrower_requests_details(req, orderby=None, ln=CFG_SITE_LANG,
                                  notify_button=None, borrower_id=None):
    """
    http://cdsweb.cern.ch/admin/bibcirculation/bibcirculationadmin.py/borrowers_search

    """
    return bal.get_borrower_requests_details(req, orderby, ln,
                                             notify_button, borrower_id)

def get_borrower_loans_details(req, orderby=None, ln=CFG_SITE_LANG,
                               notify_button=None, barcode=None,
                               borrower_id=None):
    """
    http://cdsweb.cern.ch/admin/bibcirculation/bibcirculationadmin.py/borrowers_search

    """
    return bal.get_borrower_loans_details(req, orderby, ln,
                                          notify_button, barcode, borrower_id)

def search_result(req, column, str, ln=CFG_SITE_LANG):
    """
    """
    return bal.search_result(req, column, str, ln)

def help_contactsupport(req, ln=CFG_SITE_LANG):
    """
    http://cdsweb.cern.ch/admin/bibcirculation/bibcirculationadmin.py/borrowers_search

    """
    return bal.help_contactsupport(req, ln)

<|endoftext|>