<commit_before>from django.db import models
from django.db import transaction
from django.db.models import signals
from django.dispatch import receiver
from django.contrib.auth.models import User
from django.utils import timezone
from django.core.exceptions import ValidationError

class Capsule(models.Model):
	name = models.CharField(max_length=64, unique=True)
	price = models.IntegerField()
	# TODO: description(text, url?)
	def __unicode__(self):
		return self.name
	def get_price(self):
		return self.price
	class Meta:
		ordering = ['pk']

class Purchase(models.Model):
	name = models.CharField(max_length=64)
	begin_date = models.DateTimeField()
	end_date = models.DateTimeField()
	is_closed = models.BooleanField(default=False)
	def __unicode__(self):
		return self.name
	def clean(self):
		if self.begin_date >= self.end_date:
			raise ValidationError('begin_date >= end_date')
		if self.end_date <= timezone.now():
			raise ValidationError('end_date is not in the future')

class PurchaseItem(models.Model):
	purchase = models.ForeignKey(Purchase)
	capsule = models.ForeignKey(Capsule, related_name='+')
	price = models.IntegerField() # 'Since price can be changed, each PurchaseItem has price at the Purchase moment'
	quantity_accepted = models.IntegerField(default=0, editable=False) # TODO: replace with aggregate
	quantity_grouped = models.IntegerField(default=0, editable=False) # TODO: ..
	quantity_queued = models.IntegerField(default=0, editable=False) # TODO: ..
	class Meta:
		unique_together = (("purchase", "capsule"))
	def __unicode__(self):
		return self.purchase.__unicode__() + ": " + self.capsule.__unicode__()

class Request(models.Model):
	purchaseitem = models.ForeignKey(PurchaseItem)
	user = models.ForeignKey(User, related_name='+')
	quantity_accepted = models.IntegerField(default=0, editable=False)
	quantity_grouped = models.IntegerField(default=0, editable=False)
	quantity_queued = models.IntegerField(default=0)
	date = models.DateTimeField(auto_now=True)
	#class Meta:
	#	unique_together = (("purchaseitem", "user"))

class RequestGroup(models.Model):
	purchaseitem = models.ForeignKey(PurchaseItem, editable=False)
	priority = models.BooleanField(default=False, editable=False)
	quantity_accepted = models.IntegerField(default=0, editable=False)
	quantity_grouped = models.IntegerField(default=0, editable=False)
	date = models.DateTimeField(editable=False)
	# TODO: many-to-many fields??
	def clean(self):
		if self.quantity%10 != 0:
			raise ValidationError('quantity of RequestGroup should be multiples of 10')

class RequestGroupItem(models.Model):
	requestgroup = models.ForeignKey(RequestGroup)
	request = models.ForeignKey(Request)
	quantity = models.IntegerField()

@receiver(signals.post_save, sender=Request)
@transaction.commit_on_success
def group_request(sender, instance, created, **kwargs):
	if created == False:
		return
	g_unit = 10 # TODO: settings
	if instance.quantity_accepted > 0 or instance.quantity_grouped > 0:
		print("DEBUG: ERROR")
	instance.purchaseitem.quantity_queued += instance.quantity_queued
	instance.purchaseitem.save()
	if instance.quantity_queued >= g_unit:
		mypr = True
		rgqty = (instance.quantity_queued//g_unit)*g_unit
		instance.purchaseitem.quantity_queued -= rgqty
		instance.purchaseitem.quantity_grouped += rgqty
		instance.purchaseitem.save()
		instance.quantity_queued -= rgqty
		instance.quantity_grouped += rgqty
		instance.save()
		rg = RequestGroup(purchaseitem=instance.purchaseitem, priority=mypr, quantity_grouped=rgqty, date=instance.date)
		rg.save()
		rgitem = RequestGroupItem(requestgroup=rg, request=instance, quantity=rgqty)
		rgitem.save()
		accept_request(rg)
	# remainders
	requests = Request.objects.filter(purchaseitem=instance.purchaseitem, quantity_queued__gt=0).order_by('date')
	qtysum = requests.aggregate(models.Sum('quantity_queued'))['quantity_queued__sum']
	if qtysum >= g_unit:
		mypr = False
		rgqty = (qtysum//g_unit)*g_unit
		instance.purchaseitem.quantity_queued -= rgqty
		instance.purchaseitem.quantity_grouped += rgqty
		instance.purchaseitem.save()
		rg = RequestGroup(purchaseitem=instance.purchaseitem, priority=mypr, quantity_grouped=rgqty, date=instance.date)
		rg.save()
		tmp = rgqty
		for r in requests.all():
			mod = min(r.quantity_queued, tmp, g_unit)
			r.quantity_queued -= mod
			r.quantity_grouped += mod
			r.save()
			rgitem = RequestGroupItem(requestgroup=rg, request=r, quantity=mod)
			rgitem.save()
			tmp -= mod
			if tmp == 0:
				break
		if tmp != 0:
			print('ERROR')
		accept_request(rg)

def accept_request(instance):
	p_unit = 50 # TODO: settings
	rgs = RequestGroup.objects.filter(purchaseitem=instance.purchaseitem, quantity_grouped__gt=0).order_by('date')
	qtysum = rgs.aggregate(models.Sum('quantity_grouped'))['quantity_grouped__sum']
	if qtysum >= p_unit:
		p_qty = (qtysum//p_unit)*p_unit
		instance.purchaseitem.quantity_grouped -= p_qty
		instance.purchaseitem.quantity_accepted += p_qty
		instance.purchaseitem.save()
		tmp = p_qty
		for rg in rgs.all():
			mod = min(rg.quantity_grouped, tmp, p_unit)
			rg.quantity_grouped -= mod
			rg.quantity_accepted += mod
			rg.save()
			rgitems = RequestGroupItem.objects.filter(requestgroup=rg)
			tmp2 = mod
			for rgitem in rgitems.all():
				mod2 = min(rgitem.quantity, tmp2)
				rgitem.request.quantity_grouped -= mod2
				rgitem.request.quantity_accepted += mod2
				rgitem.request.save()
				tmp2 -= mod2
			tmp -= mod
			if tmp == 0:
				break
		if tmp != 0:
			print('ERROR')
<commit_msg>Fix a bug in accept_request()<commit_after>from django.db import models
from django.db import transaction
from django.db.models import signals
from django.dispatch import receiver
from django.contrib.auth.models import User
from django.utils import timezone
from django.core.exceptions import ValidationError

class Capsule(models.Model):
	name = models.CharField(max_length=64, unique=True)
	price = models.IntegerField()
	# TODO: description(text, url?)
	def __unicode__(self):
		return self.name
	def get_price(self):
		return self.price
	class Meta:
		ordering = ['pk']

class Purchase(models.Model):
	name = models.CharField(max_length=64)
	begin_date = models.DateTimeField()
	end_date = models.DateTimeField()
	is_closed = models.BooleanField(default=False)
	def __unicode__(self):
		return self.name
	def clean(self):
		if self.begin_date >= self.end_date:
			raise ValidationError('begin_date >= end_date')
		if self.end_date <= timezone.now():
			raise ValidationError('end_date is not in the future')

class PurchaseItem(models.Model):
	purchase = models.ForeignKey(Purchase)
	capsule = models.ForeignKey(Capsule, related_name='+')
	price = models.IntegerField() # 'Since price can be changed, each PurchaseItem has price at the Purchase moment'
	#quantity_accepted = models.IntegerField(default=0, editable=False) # TODO: replace with aggregate
	#quantity_grouped = models.IntegerField(default=0, editable=False) # TODO: ..
	#quantity_queued = models.IntegerField(default=0, editable=False) # TODO: ..
	def quantity_accepted(self):
		return self.request_set.aggregate(models.Sum('quantity_accepted'))['quantity_accepted__sum']
	def quantity_grouped(self):
		return self.request_set.aggregate(models.Sum('quantity_grouped'))['quantity_grouped__sum']
	def quantity_queued(self):
		return self.request_set.aggregate(models.Sum('quantity_queued'))['quantity_queued__sum']
	class Meta:
		unique_together = (("purchase", "capsule"))
	def __unicode__(self):
		return self.purchase.__unicode__() + ": " + self.capsule.__unicode__()

class Request(models.Model):
	purchaseitem = models.ForeignKey(PurchaseItem)
	user = models.ForeignKey(User, related_name='+')
	quantity_accepted = models.IntegerField(default=0, editable=False)
	quantity_grouped = models.IntegerField(default=0, editable=False)
	quantity_queued = models.IntegerField(default=0)
	date = models.DateTimeField(auto_now=True)
	#class Meta:
	#	unique_together = (("purchaseitem", "user"))

class RequestGroup(models.Model):
	purchaseitem = models.ForeignKey(PurchaseItem, editable=False)
	priority = models.BooleanField(default=False, editable=False)
	quantity_accepted = models.IntegerField(default=0, editable=False)
	quantity_grouped = models.IntegerField(default=0, editable=False)
	date = models.DateTimeField(editable=False)
	# TODO: many-to-many fields??
	def clean(self):
		if self.quantity%10 != 0:
			raise ValidationError('quantity of RequestGroup should be multiples of 10')

class RequestGroupItem(models.Model):
	requestgroup = models.ForeignKey(RequestGroup)
	request = models.ForeignKey(Request)
	quantity = models.IntegerField()

@receiver(signals.post_save, sender=Request)
@transaction.commit_on_success
def group_request(sender, instance, created, **kwargs):
	if created == False:
		return
	g_unit = 10 # TODO: settings
	if instance.quantity_accepted > 0 or instance.quantity_grouped > 0:
		print("DEBUG: ERROR")
	#instance.purchaseitem.quantity_queued += instance.quantity_queued
	#instance.purchaseitem.save()
	if instance.quantity_queued >= g_unit:
		mypr = True
		rgqty = (instance.quantity_queued//g_unit)*g_unit
		#instance.purchaseitem.quantity_queued -= rgqty
		#instance.purchaseitem.quantity_grouped += rgqty
		#instance.purchaseitem.save()
		instance.quantity_queued -= rgqty
		instance.quantity_grouped += rgqty
		instance.save()
		rg = RequestGroup(purchaseitem=instance.purchaseitem, priority=mypr, quantity_grouped=rgqty, date=instance.date)
		rg.save()
		rgitem = RequestGroupItem(requestgroup=rg, request=instance, quantity=rgqty)
		rgitem.save()
		accept_request(rg)
	# remainders
	requests = Request.objects.filter(purchaseitem=instance.purchaseitem, quantity_queued__gt=0).order_by('date')
	qtysum = requests.aggregate(models.Sum('quantity_queued'))['quantity_queued__sum']
	if qtysum >= g_unit:
		mypr = False
		rgqty = (qtysum//g_unit)*g_unit
		#instance.purchaseitem.quantity_queued -= rgqty
		#instance.purchaseitem.quantity_grouped += rgqty
		#instance.purchaseitem.save()
		rg = RequestGroup(purchaseitem=instance.purchaseitem, priority=mypr, quantity_grouped=rgqty, date=instance.date)
		rg.save()
		tmp = rgqty
		for r in requests.all():
			mod = min(r.quantity_queued, tmp, g_unit)
			r.quantity_queued -= mod
			r.quantity_grouped += mod
			r.save()
			rgitem = RequestGroupItem(requestgroup=rg, request=r, quantity=mod)
			rgitem.save()
			tmp -= mod
			if tmp == 0:
				break
		if tmp != 0:
			print('ERROR')
		accept_request(rg)

def accept_request(instance):
	p_unit = 50 # TODO: settings
	rgs = RequestGroup.objects.filter(purchaseitem__purchase=instance.purchaseitem.purchase, quantity_grouped__gt=0).order_by('date')
	qtysum = rgs.aggregate(models.Sum('quantity_grouped'))['quantity_grouped__sum']
	if qtysum >= p_unit:
		p_qty = (qtysum//p_unit)*p_unit
		tmp = p_qty
		for rg in rgs.all():
			mod = min(rg.quantity_grouped, tmp, p_unit)
			#rg.purchaseitem.quantity_grouped -= mod
			#rg.purchaseitem.quantity_accepted += mod
			#rg.purchaseitem.save()
			rg.quantity_grouped -= mod
			rg.quantity_accepted += mod
			rg.save()
			rgitems = RequestGroupItem.objects.filter(requestgroup=rg)
			tmp2 = mod
			for rgitem in rgitems.all():
				mod2 = min(rgitem.quantity, tmp2)
				rgitem.request.quantity_grouped -= mod2
				rgitem.request.quantity_accepted += mod2
				rgitem.request.save()
				tmp2 -= mod2
			tmp -= mod
			if tmp == 0:
				break
		if tmp != 0:
			print('ERROR')
<|endoftext|>


<commit_before># (c) Fugro GeoServices. MIT licensed, see LICENSE.rst.
from __future__ import absolute_import

from subprocess import call
import logging
import os
import shutil

from django.conf import settings
from django.contrib.auth.models import User
from pandas.io.parsers import read_csv
from pandas import DataFrame
from tslib.readers import PiXmlReader
from datetime import datetime, timedelta

from ddsc_worker.import_auth import get_auth
from ddsc_worker.import_auth import get_remoteid_by_filename
from ddsc_worker.import_auth import get_timestamp_by_filename
from ddsc_worker.geoserver.catalog import Catalog
logger = logging.getLogger(__name__)

pd = getattr(settings, 'IMPORTER_PATH')
ERROR_file = pd['storage_base_path'] + pd['rejected_file']
OK_file = pd['storage_base_path'] + pd['accepted_file']
gs_setting = getattr(settings, 'IMPORTER_GEOSERVER')

def data_convert(src):
    try:
        logger.debug("[x] converting %r to pandas object", src)
        # Half the time read_csv fails with: "IndexError: list index out of
        # range". Quite bizar, but asking for some file metadata first,
        # e.g. the file size, seems to solve the problem. Performing
        # a sleep does not solve the problem. Something fishy about
        # accessing a file on a mounted Windows share from Ubuntu?
        logger.debug("%s is %d bytes", src, os.path.getsize(src))
        tsOBJ = read_csv(
            src, index_col=0,
            parse_dates=True,
            names=['SensorID', 'value'],
        )
    except Exception, e:
        logger.error(e, exc_info=True)
        logger.error("[x] %r _FAILED to be converted", src)
        data_move(src, ERROR_file)
        raise e
    tsOBJ = tsOBJ.sort()
    logger.debug("[x] %r _converted & sorted", src)
    return tsOBJ

def write2_cassandra(tsOBJ_yes, ts, src):
    try:
        tsOBJ_yes = tsOBJ_yes.tz_localize('UTC')
        del tsOBJ_yes['SensorID']
        ts.set_events(tsOBJ_yes)
        ts.save()
        logger.debug("[x] %r _written" % (src))
    except Exception, e:
        logger.error(
            "[x] %r _FAILED to be written to cassandra",
            src, exc_info=True
        )
        data_move(src, ERROR_file)
        raise e

def data_delete(wStatus, src):
    if wStatus == 1:
        os.remove(src)
        if 1 == 2:
            logger.error('something went wrong')
            pass
        else:
            logger.debug("[x] %r _deleted" % (src))
    else:
        pass

def import_csv(src, usr_id):
    usr = User.objects.get(pk=usr_id)
    tsobj = data_convert(src)
    tsgrouped = tsobj.groupby('SensorID')
    nr = len(tsgrouped)
    nr = str(nr)
    logger.debug('There are %r timeseries in file : %r' % (nr, src))
    auth_tag = 0
    for name, tsobj_grouped in tsgrouped:
        remoteid = tsobj_grouped['SensorID'][0]
        ts = get_auth(usr, remoteid)
        if ts is False:
            logger.debug(
                '[x] Timeseries %r has been rejected because of authorization',
                remoteid
            )
            auth_tag += 1
        else:
            write2_cassandra(tsobj_grouped, ts, src)
    if str(auth_tag) == nr:
        logger.error('[x] File:--%r-- has been fully rejected' % src)
        data_move(src, ERROR_file)
        raise Exception("[x] In : %r  All of the timeseries failed"
            " to be imported because of auth" % (src))
    elif auth_tag == 0:
        data_move(src, OK_file)
        logger.info('[x] File:--%r-- has been successfully imported' % src)
    else:
        data_move(src, ERROR_file)
        logger.warning('[x] File:--%r-- has been only partly imported' % src)
        raise Exception("[x] In : %r  Some of the timeseries failed"
            " to be imported because of auth" % (src))

def data_move(src, dst):
    if not os.path.exists(dst):
        os.makedirs(dst)
    try:
        shutil.move(src, dst)
        if os.path.isdir(dst):
            logger.debug("[x] Moved %r to %r" % (src, dst))
            return os.path.join(dst, os.path.split(src)[1])
        else:
            logger.debug("[x] Moved %r to %r" % (src, dst))
            return dst
    except EnvironmentError as e:
        logger.error(e)
        return

def import_file(src, filename, dst, usr_id):
    usr = User.objects.get(pk=usr_id)
    logger.debug("[x] Importing %r to DB" % filename)
    timestamp = get_timestamp_by_filename(filename)

    remoteid = get_remoteid_by_filename(filename)
    ts = get_auth(usr, remoteid)

    if ts is not False:
        str_year = str(timestamp.year[0])
        str_month = str(timestamp.month[0])
        str_day = str(timestamp.day[0])
        store_dst = dst + ts.uuid + '/' + str_year +\
            '-' + str_month + '-' + str_day + '/'
        store_dstf = store_dst + filename
        values = {"value": store_dstf}

        ts.set_event(timestamp[0], values)
        ts.save()
        data_move(src + filename, store_dst)
        logger.info(
            '[x] File:--%r-- has been successfully imported' % (src +
            filename))
    else:
        logger.error('[x] File:--%r-- has been rejected' % (src + filename))
        data_move(src + filename, ERROR_file)
        raise Exception("[x] %r _FAILED to be imported" % (src + filename))

def import_geotiff(src, filename, dst, usr_id):
    usr = User.objects.get(pk=usr_id)
    src = src + filename

    logger.debug("[x] Importing %r to DB" % filename)
    timestamp = get_timestamp_by_filename(filename)

    remoteid = get_remoteid_by_filename(filename)
    filename_without_ext =  filename.replace('.geotiff', '')
    ts = get_auth(usr, remoteid)

    logger.debug("[x] publishing %r into GeoServer..." % src)

    if ts is not False:
        str_year = str(timestamp.year[0])
        str_month = str(timestamp.month[0])
        str_day = str(timestamp.day[0])
        store_dst = dst + ts.name + '/' + \
        str_year + '-' + str_month + '-' + str_day + '/'

        cat = Catalog(gs_setting['geoserver_url'] + '/rest',
            gs_setting['geoserver_username'],
            gs_setting['geoserver_password'], True)

        workspace = cat.get_workspace(gs_setting['geoserver_workspace'])
        if workspace == None:
            cat.create_workspace(gs_setting['geoserver_workspace'], gs_setting['geoserver_workspace'])

        try:
            data_move(src, store_dst)
            store = cat.create_coveragestore(filename_without_ext, store_dst + filename, workspace, True)
        except Exception, e:
            data_move(src, ERROR_file)
            logger.error('[x] File:--%r-- has been filed to publish' % src)
            raise Exception("[x] %r _FAILED to be imported" % src + ' reason--%r' % e )

        values = {"value": filename_without_ext}
        logger.debug("[x] Published %r to GeoServer " % src)
        ts.set_event(timestamp[0], values)
        ts.save()
        logger.info("[x] File:--%r-- has been successfully imported" % src)
    else:
        logger.error('[x] File:--%r-- has been rejected' % src)
        data_move(src, ERROR_file)
        raise Exception("[x] %r _FAILED to be imported" % src)

def import_pi_xml(src, usr_id):
    logger.info("[x] Importing %r" % src)
    reader = PiXmlReader(src)

    usr = User.objects.get(pk=usr_id)

    for md, df in reader.get_series():
        loc = md['header']['locationId']  # required
        para = md['header']['parameterId']  # required
        unit = md['header']['timeStep']['@unit']  # required
        div = md['header']['timeStep'].get('@divider', '')  # optional
        mul = md['header']['timeStep'].get('@multiplier', '')  # optional
        remote_id = loc + '::' + para + '::' + unit + '::' + div + '::' + mul
        ts = get_auth(usr, remote_id)
        if ts is False:
            data_move(src, ERROR_file)
            logger.error(
                '[x] File:--%r-- has been rejected because of authorization' %
                src)
            raise Exception("[x] %r _FAILED to be imported" % (src))
        else:
            try:
                del df['flag']
                ts.set_events(df)
                ts.save()
                logger.debug("[x] %r _written" % (src))
            except:
                logger.error(
                    "[x] %r _FAILED to be written to cassandra" % (src))
                data_move(src, ERROR_file)
                raise Exception('piXML file: %r ERROR to convert!' % src)

    data_move(src, OK_file)
    logger.info('[x] File:--%r-- has been successfully imported' % src)

def file_ignored(src, fileExtension):
    logger.error('[x]--Warning-- * %r' % fileExtension +
        ' FILE: %r is not acceptable' % src)
    dst = pd['storage_base_path'] + pd['rejected_file']
    data_move(src, dst)
    raise Exception("[x] %r _FAILED to be imported" % src)

def import_lmw(DestinationPath, admFileName, datFileName, kwaFileName):
    ## get the user which in this case should be LMW I guess
    usr = User.objects.get(username='lmw_ddsc')
    adm_src = admFileName
    dat_src = datFileName
    kwa_src = kwaFileName

    tsobj_indx = read_lmw(adm_src, dat_src, kwa_src)
    tsgrouped = tsobj_indx.groupby('SensorID')
    nr = len(tsgrouped)
    nr = str(nr)
    logger.debug('There are %r timeseries in file : %r' % (nr, adm_src))
    for name, tsobj_grouped in tsgrouped:
        remoteid = tsobj_grouped['SensorID'][0]
        ts = get_auth(usr, remoteid)  # user object and remote id
        if ts is False:
            data_move(adm_src, ERROR_file)
            data_move(dat_src, ERROR_file)
            data_move(kwa_src, ERROR_file)
            logger.error(
                '[x] File:--%r-- has been rejected because of authorization' %
                adm_src)
            raise Exception("[x] %r _FAILED to be imported" % (adm_src))
        else:
            tsobjYes = tsobj_grouped
            write2_cassandra(tsobjYes, ts, dat_src)

    data_move(adm_src, OK_file)
    data_move(dat_src, OK_file)
    data_move(kwa_src, OK_file)
    logger.info('[x] File:--%r-- has been successfully imported' % adm_src)

def read_lmw(admFile, datFile, kwaFile):
    with open(admFile) as f:
        administration = f.readlines()
    with open(datFile) as f:
        data = f.readlines()
    with open(kwaFile) as f:
        data_quality = f.readlines()

    if len(administration) != len(data):
        raise Exception("Input data is not of same length.")

    # LMW interval in minutes
    interval = 10
    val_series = []
    timestamp_series = []
    remoteid_series = []
    quality_series = []
    zom_win = []

    for i in range(len(administration)):
        values = administration[i].split(",")
        # Get the id of the timeserie
        timeseriesId = values[0].strip() +\
            "_" + values[1].strip() + "_" + values[3].strip()
        # Get the time of the first value
        if values[7].find('MET') == -1:
            zom_win = 'summer'

        values[7] = values[7].replace("JAN", "01")
        values[7] = values[7].replace("FEB", "02")
        values[7] = values[7].replace("MRT", "03")
        values[7] = values[7].replace("APR", "04")
        values[7] = values[7].replace("MEI", "05")
        values[7] = values[7].replace("JUN", "06")
        values[7] = values[7].replace("JUL", "07")
        values[7] = values[7].replace("AUG", "08")
        values[7] = values[7].replace("SEP", "09")
        values[7] = values[7].replace("OKT", "10")
        values[7] = values[7].replace("NOV", "11")
        values[7] = values[7].replace("DEC", "12")
        values[7] = values[7].replace("Z03", "")
        values[7] = values[7].replace("MET", "")
        values[7] = values[7].strip()

        if zom_win == 'summer':
            timeFirstValue = datetime.strptime(values[7], "%d-%m-%y %H:%M") -\
                timedelta(0, 0, 0, 0, 120)
        else:
            timeFirstValue = datetime.strptime(values[7], "%d-%m-%y %H:%M") -\
                timedelta(0, 0, 0, 0, 60)
        # Get all the measurements
        measurements = data[i].split(",")
        quality = data_quality[i].split(",")

        if len(measurements) != 7:
            raise Exception("Invalid number of measurements for timeserie.")

        if len(quality) != 7:
            raise Exception("Invalid number of quality flags for timeserie.")

        counter = 0
        for j in range(6):
            value = measurements[j].strip()
            value_flag = int(quality[j])
            if value != "f" and value != "n":
                TimeForValue = timeFirstValue +\
                    timedelta(0, 0, 0, 0, interval * j)
                val_series.append(float(value))
                timestamp_series.append(TimeForValue)
                remoteid_series.append(timeseriesId)
                counter += 1
                if value_flag in [10, 30, 50, 70]:
                    quality_series.append('0')
                elif value_flag in [2, 22, 24, 28, 42, 44, 48, 62, 68]:
                    quality_series.append('3')
                else:
                    quality_series.append('6')

    tsobj = DataFrame([remoteid_series,
                       val_series, quality_series])
    tsobj = tsobj.transpose()
    tsobj.columns = ['SensorID', 'value', 'flag']
    tstamp = DataFrame(timestamp_series, columns=['ts'])
    tsobj_indexed = tsobj.set_index(tstamp['ts'])
    return tsobj_indexed

def make_file_name_unique(file_name):
    pos = file_name.find('.')
    time_string = datetime.utcnow().strftime("%Y%m%d%H%M%S_%f")
    new_name = file_name[0:pos] + '_' + time_string + file_name[pos:]
    return new_name
<commit_msg>geotiff no longer copied to geoserver data dir<commit_after># (c) Fugro GeoServices. MIT licensed, see LICENSE.rst.
from __future__ import absolute_import

from subprocess import call
import logging
import os
import shutil

from django.conf import settings
from django.contrib.auth.models import User
from pandas.io.parsers import read_csv
from pandas import DataFrame
from tslib.readers import PiXmlReader
from datetime import datetime, timedelta

from ddsc_worker.import_auth import get_auth
from ddsc_worker.import_auth import get_remoteid_by_filename
from ddsc_worker.import_auth import get_timestamp_by_filename
from ddsc_worker.geoserver.catalog import Catalog
logger = logging.getLogger(__name__)

pd = getattr(settings, 'IMPORTER_PATH')
ERROR_file = pd['storage_base_path'] + pd['rejected_file']
OK_file = pd['storage_base_path'] + pd['accepted_file']
gs_setting = getattr(settings, 'IMPORTER_GEOSERVER')

def data_convert(src):
    try:
        logger.debug("[x] converting %r to pandas object", src)
        # Half the time read_csv fails with: "IndexError: list index out of
        # range". Quite bizar, but asking for some file metadata first,
        # e.g. the file size, seems to solve the problem. Performing
        # a sleep does not solve the problem. Something fishy about
        # accessing a file on a mounted Windows share from Ubuntu?
        logger.debug("%s is %d bytes", src, os.path.getsize(src))
        tsOBJ = read_csv(
            src, index_col=0,
            parse_dates=True,
            names=['SensorID', 'value'],
        )
    except Exception, e:
        logger.error(e, exc_info=True)
        logger.error("[x] %r _FAILED to be converted", src)
        data_move(src, ERROR_file)
        raise e
    tsOBJ = tsOBJ.sort()
    logger.debug("[x] %r _converted & sorted", src)
    return tsOBJ

def write2_cassandra(tsOBJ_yes, ts, src):
    try:
        tsOBJ_yes = tsOBJ_yes.tz_localize('UTC')
        del tsOBJ_yes['SensorID']
        ts.set_events(tsOBJ_yes)
        ts.save()
        logger.debug("[x] %r _written" % (src))
    except Exception, e:
        logger.error(
            "[x] %r _FAILED to be written to cassandra",
            src, exc_info=True
        )
        data_move(src, ERROR_file)
        raise e

def data_delete(wStatus, src):
    if wStatus == 1:
        os.remove(src)
        if 1 == 2:
            logger.error('something went wrong')
            pass
        else:
            logger.debug("[x] %r _deleted" % (src))
    else:
        pass

def import_csv(src, usr_id):
    usr = User.objects.get(pk=usr_id)
    tsobj = data_convert(src)
    tsgrouped = tsobj.groupby('SensorID')
    nr = len(tsgrouped)
    nr = str(nr)
    logger.debug('There are %r timeseries in file : %r' % (nr, src))
    auth_tag = 0
    for name, tsobj_grouped in tsgrouped:
        remoteid = tsobj_grouped['SensorID'][0]
        ts = get_auth(usr, remoteid)
        if ts is False:
            logger.debug(
                '[x] Timeseries %r has been rejected because of authorization',
                remoteid
            )
            auth_tag += 1
        else:
            write2_cassandra(tsobj_grouped, ts, src)
    if str(auth_tag) == nr:
        logger.error('[x] File:--%r-- has been fully rejected' % src)
        data_move(src, ERROR_file)
        raise Exception("[x] In : %r  All of the timeseries failed"
            " to be imported because of auth" % (src))
    elif auth_tag == 0:
        data_move(src, OK_file)
        logger.info('[x] File:--%r-- has been successfully imported' % src)
    else:
        data_move(src, ERROR_file)
        logger.warning('[x] File:--%r-- has been only partly imported' % src)
        raise Exception("[x] In : %r  Some of the timeseries failed"
            " to be imported because of auth" % (src))

def data_move(src, dst):
    if not os.path.exists(dst):
        os.makedirs(dst)
    try:
        shutil.move(src, dst)
        if os.path.isdir(dst):
            logger.debug("[x] Moved %r to %r" % (src, dst))
            return os.path.join(dst, os.path.split(src)[1])
        else:
            logger.debug("[x] Moved %r to %r" % (src, dst))
            return dst
    except EnvironmentError as e:
        logger.error(e)
        return

def import_file(src, filename, dst, usr_id):
    usr = User.objects.get(pk=usr_id)
    logger.debug("[x] Importing %r to DB" % filename)
    timestamp = get_timestamp_by_filename(filename)

    remoteid = get_remoteid_by_filename(filename)
    ts = get_auth(usr, remoteid)

    if ts is not False:
        str_year = str(timestamp.year[0])
        str_month = str(timestamp.month[0])
        str_day = str(timestamp.day[0])
        store_dst = dst + ts.uuid + '/' + str_year +\
            '-' + str_month + '-' + str_day + '/'
        store_dstf = store_dst + filename
        values = {"value": store_dstf}

        ts.set_event(timestamp[0], values)
        ts.save()
        data_move(src + filename, store_dst)
        logger.info(
            '[x] File:--%r-- has been successfully imported' % (src +
            filename))
    else:
        logger.error('[x] File:--%r-- has been rejected' % (src + filename))
        data_move(src + filename, ERROR_file)
        raise Exception("[x] %r _FAILED to be imported" % (src + filename))

def import_geotiff(src, filename, dst, usr_id):
    usr = User.objects.get(pk=usr_id)
    src = src + filename

    logger.debug("[x] Importing %r to DB" % filename)
    timestamp = get_timestamp_by_filename(filename)

    uuid = get_remoteid_by_filename(filename)
    filename_without_ext =  filename.replace('.geotiff', '')
    ts = get_auth(usr, uuid)

    logger.debug("[x] publishing %r into GeoServer..." % src)

    if ts is not False:
        str_year = str(timestamp.year[0])
        str_month = str(timestamp.month[0])
        str_day = str(timestamp.day[0])
        store_dst = dst + uuid + '/' + \
        str_year + '-' + str_month + '-' + str_day + '/'

        cat = Catalog(gs_setting['geoserver_url'] + '/rest',
            gs_setting['geoserver_username'],
            gs_setting['geoserver_password'], True)

        workspace = cat.get_workspace(gs_setting['geoserver_workspace'])
        if workspace == None:
            cat.create_workspace(gs_setting['geoserver_workspace'], gs_setting['geoserver_workspace'])

        try:
            data_move(src, store_dst)
            #store = cat.create_coveragestore(filename_without_ext, store_dst + filename, workspace, True)
            create_geotiff(cat, gs_setting['geoserver_workspace'], filename_without_ext, store_dst + filename)
        except Exception, e:
            data_move(src, ERROR_file)
            logger.error('[x] File:--%r-- has been filed to publish' % src)
            raise Exception("[x] %r _FAILED to be imported" % src + ' reason--%r' % e )

        values = {"value": store_dst + filename}

        logger.debug("[x] Published %r to GeoServer " % src)
        ts.set_event(timestamp[0], values)
        ts.save()
        logger.info("[x] File:--%r-- has been successfully imported" % src)
    else:
        logger.error('[x] File:--%r-- has been rejected' % src)
        data_move(src, ERROR_file)
        raise Exception("[x] %r _FAILED to be imported" % src)

def import_pi_xml(src, usr_id):
    logger.info("[x] Importing %r" % src)
    reader = PiXmlReader(src)

    usr = User.objects.get(pk=usr_id)

    for md, df in reader.get_series():
        loc = md['header']['locationId']  # required
        para = md['header']['parameterId']  # required
        unit = md['header']['timeStep']['@unit']  # required
        div = md['header']['timeStep'].get('@divider', '')  # optional
        mul = md['header']['timeStep'].get('@multiplier', '')  # optional
        remote_id = loc + '::' + para + '::' + unit + '::' + div + '::' + mul
        ts = get_auth(usr, remote_id)
        if ts is False:
            data_move(src, ERROR_file)
            logger.error(
                '[x] File:--%r-- has been rejected because of authorization' %
                src)
            raise Exception("[x] %r _FAILED to be imported" % (src))
        else:
            try:
                del df['flag']
                ts.set_events(df)
                ts.save()
                logger.debug("[x] %r _written" % (src))
            except:
                logger.error(
                    "[x] %r _FAILED to be written to cassandra" % (src))
                data_move(src, ERROR_file)
                raise Exception('piXML file: %r ERROR to convert!' % src)

    data_move(src, OK_file)
    logger.info('[x] File:--%r-- has been successfully imported' % src)

def file_ignored(src, fileExtension):
    logger.error('[x]--Warning-- * %r' % fileExtension +
        ' FILE: %r is not acceptable' % src)
    dst = pd['storage_base_path'] + pd['rejected_file']
    data_move(src, dst)
    raise Exception("[x] %r _FAILED to be imported" % src)

def import_lmw(DestinationPath, admFileName, datFileName, kwaFileName):
    ## get the user which in this case should be LMW I guess
    usr = User.objects.get(username='lmw_ddsc')
    adm_src = admFileName
    dat_src = datFileName
    kwa_src = kwaFileName

    tsobj_indx = read_lmw(adm_src, dat_src, kwa_src)
    tsgrouped = tsobj_indx.groupby('SensorID')
    nr = len(tsgrouped)
    nr = str(nr)
    logger.debug('There are %r timeseries in file : %r' % (nr, adm_src))
    for name, tsobj_grouped in tsgrouped:
        remoteid = tsobj_grouped['SensorID'][0]
        ts = get_auth(usr, remoteid)  # user object and remote id
        if ts is False:
            data_move(adm_src, ERROR_file)
            data_move(dat_src, ERROR_file)
            data_move(kwa_src, ERROR_file)
            logger.error(
                '[x] File:--%r-- has been rejected because of authorization' %
                adm_src)
            raise Exception("[x] %r _FAILED to be imported" % (adm_src))
        else:
            tsobjYes = tsobj_grouped
            write2_cassandra(tsobjYes, ts, dat_src)

    data_move(adm_src, OK_file)
    data_move(dat_src, OK_file)
    data_move(kwa_src, OK_file)
    logger.info('[x] File:--%r-- has been successfully imported' % adm_src)

def read_lmw(admFile, datFile, kwaFile):
    with open(admFile) as f:
        administration = f.readlines()
    with open(datFile) as f:
        data = f.readlines()
    with open(kwaFile) as f:
        data_quality = f.readlines()

    if len(administration) != len(data):
        raise Exception("Input data is not of same length.")

    # LMW interval in minutes
    interval = 10
    val_series = []
    timestamp_series = []
    remoteid_series = []
    quality_series = []
    zom_win = []

    for i in range(len(administration)):
        values = administration[i].split(",")
        # Get the id of the timeserie
        timeseriesId = values[0].strip() +\
            "_" + values[1].strip() + "_" + values[3].strip()
        # Get the time of the first value
        if values[7].find('MET') == -1:
            zom_win = 'summer'

        values[7] = values[7].replace("JAN", "01")
        values[7] = values[7].replace("FEB", "02")
        values[7] = values[7].replace("MRT", "03")
        values[7] = values[7].replace("APR", "04")
        values[7] = values[7].replace("MEI", "05")
        values[7] = values[7].replace("JUN", "06")
        values[7] = values[7].replace("JUL", "07")
        values[7] = values[7].replace("AUG", "08")
        values[7] = values[7].replace("SEP", "09")
        values[7] = values[7].replace("OKT", "10")
        values[7] = values[7].replace("NOV", "11")
        values[7] = values[7].replace("DEC", "12")
        values[7] = values[7].replace("Z03", "")
        values[7] = values[7].replace("MET", "")
        values[7] = values[7].strip()

        if zom_win == 'summer':
            timeFirstValue = datetime.strptime(values[7], "%d-%m-%y %H:%M") -\
                timedelta(0, 0, 0, 0, 120)
        else:
            timeFirstValue = datetime.strptime(values[7], "%d-%m-%y %H:%M") -\
                timedelta(0, 0, 0, 0, 60)
        # Get all the measurements
        measurements = data[i].split(",")
        quality = data_quality[i].split(",")

        if len(measurements) != 7:
            raise Exception("Invalid number of measurements for timeserie.")

        if len(quality) != 7:
            raise Exception("Invalid number of quality flags for timeserie.")

        counter = 0
        for j in range(6):
            value = measurements[j].strip()
            value_flag = int(quality[j])
            if value != "f" and value != "n":
                TimeForValue = timeFirstValue +\
                    timedelta(0, 0, 0, 0, interval * j)
                val_series.append(float(value))
                timestamp_series.append(TimeForValue)
                remoteid_series.append(timeseriesId)
                counter += 1
                if value_flag in [10, 30, 50, 70]:
                    quality_series.append('0')
                elif value_flag in [2, 22, 24, 28, 42, 44, 48, 62, 68]:
                    quality_series.append('3')
                else:
                    quality_series.append('6')

    tsobj = DataFrame([remoteid_series,
                       val_series, quality_series])
    tsobj = tsobj.transpose()
    tsobj.columns = ['SensorID', 'value', 'flag']
    tstamp = DataFrame(timestamp_series, columns=['ts'])
    tsobj_indexed = tsobj.set_index(tstamp['ts'])
    return tsobj_indexed

def make_file_name_unique(file_name):
    pos = file_name.find('.')
    time_string = datetime.utcnow().strftime("%Y%m%d%H%M%S_%f")
    new_name = file_name[0:pos] + '_' + time_string + file_name[pos:]
    return new_name

def create_geotiff(catelog, work_space, store_name, file_path):
     headers = {
            "Content-type": "application/xml",
            "Accept": "application/xml"
     }
     message = "url:file:" + file_path

     url = catelog.service_url + "/workspaces/" +\
         work_space + "/coveragestores/" +\
         store_name + "/external.geotiff"
     response, content = catelog.http.request(url, "PUT", message, headers)

     if (response.status != 201):
         raise Exception(content)
<|endoftext|>


<commit_before>from cog.models import *
from cog.models.auth import userHasContributorPermission
from django.forms import ModelForm, ModelMultipleChoiceField, NullBooleanSelect
from django.db import models
from django.contrib.admin.widgets import FilteredSelectMultiple
from django import forms
from django.forms import ModelForm, Textarea, TextInput, Select, CheckboxSelectMultiple
from django.core.exceptions import ObjectDoesNotExist
from os.path import basename
import re
from cog.utils import *
from django.db.models import Q
import operator
from cog.forms.forms_utils import validate_image
import magic
from cog.constants import VALID_MIME_TYPES
from cog.utils import default_clean_field

# list of invalid characters in uploaded documents file names
INVALID_CHARS = "[^a-zA-Z0-9_\-\.\/\s]"

class NewsForm(ModelForm):

    # extra fields not present in model
    peer_projects = ModelMultipleChoiceField(queryset=Project.objects.all(), required=False,
                                             widget=CheckboxSelectMultiple)
    parent_projects = ModelMultipleChoiceField(queryset=Project.objects.all(), required=False,
                                               widget=CheckboxSelectMultiple(attrs={'vertical-align': 'bottom'}))
    child_projects = ModelMultipleChoiceField(queryset=Project.objects.all(), required=False,
                                              widget=CheckboxSelectMultiple)

    # override __init__ method to customize the list of choices for the parent/peer/child projects
    def __init__(self, project, user, *args, **kwargs):

        super(NewsForm, self).__init__(*args, **kwargs)

        self.fields['parent_projects'].queryset = \
            Project.objects.filter(self._buildQuerySet(project.parents.all(), user)).distinct().order_by('short_name')
        self.fields['peer_projects'].queryset = \
            Project.objects.filter(self._buildQuerySet(project.peers.all(), user)).distinct().order_by('short_name')
        self.fields['child_projects'].queryset = \
            Project.objects.filter(self._buildQuerySet(project.children(), user)).distinct().order_by('short_name')

        # on update only: pre-populate extra fields with current selection
        if self.instance.id:
            self.fields['parent_projects'].initial = self.instance.other_projects.all()
            self.fields['peer_projects'].initial = self.instance.other_projects.all()
            self.fields['child_projects'].initial = self.instance.other_projects.all()

    # method to build a query set that contains only the projects the User has access to.
    # the User must be a Contributor to post news
    def _buildQuerySet(self, projects, user):
        qs = Q(pk=0)  # start with an empty query set - does not match any project
        for p in projects:
            if userHasContributorPermission(user, p):
                qs = qs | Q(pk=p.id)
        return qs

    def clean_title(self):
        return xss_clean_field(self, 'title')

    def clean_text(self):
        return xss_clean_field(self, 'text')

    class Meta:
        model = News
        fields = "__all__"

class DocForm(ModelForm):

    # extra field not present in model,
    # used for redirection to other URLs after for has been successfully submitted
    redirect = forms.CharField(required=False)

    # additional extra field to optionally create a resource under the selected folder
    folder = forms.ModelChoiceField(queryset=None, required=False)

    # override __init__ method to provide a filtered list of options for the bookmark folder
    def __init__(self, project, *args, **kwargs):

        super(DocForm, self).__init__(*args, **kwargs)

        # filter folders by project and active state
        # order by name in the form pull down
        self.fields['folder'].queryset = \
            Folder.objects.filter(project=project).filter(active=True).distinct().order_by('name')

    def clean(self):
        """Override clean method to check that file size does not exceed limit.
            At this point the file is still in memory only,
            so if error is thrown there is no need to remove it from disk."""
        cleaned_data = self.cleaned_data
        thefile = cleaned_data.get("file")
        title = cleaned_data.get("title")
        description = cleaned_data.get("description")

        if not thefile:
            self._errors["file"] = self.error_class(["Sorry, the file is empty."])
            return cleaned_data

        if re.search(INVALID_CHARS, thefile.name):
            self._errors['file'] = self.error_class(["Sorry, the filename contains invalid characters. "
                                                     "It can only contain letters, numbers, spaces, and _ - . /"])

        if re.search(INVALID_CHARS, title):
            self._errors['title'] = self.error_class(["Sorry, the document title contains invalid characters. "
                                                      "It can only contain letters, numbers, spaces, and _ - . /"])
        if re.search(INVALID_CHARS, description):
            self._errors['description'] = self.error_class(["Sorry, the document description contains invalid characters. "
                                                            "It can only contain letters, numbers, spaces, and _ - . /"])

        project = cleaned_data['project']
        if thefile.size > project.maxUploadSize:
            self._errors["file"] = self.error_class(["Sorry, the file size exceeds the maximum allowed."])

        # validate the file content
        # must write the file to a temporary location to validate it
        # choose to write to $SITE_MEDIA/tmp/file.name
        if len(self._errors)==0:

            file_ext = str(os.path.splitext(thefile.name)[1])
            mime_type = magic.from_buffer(thefile.read(1024), mime=True)
            print "Validating file extension=%s, mime type=%s" % (file_ext, mime_type)
            if not file_ext:
                self._errors["file"] = self.error_class(["File name must have an extension"])
            elif file_ext.lower() not in VALID_MIME_TYPES.keys():
                self._errors["file"] = self.error_class(["File extension %s is not supported" % file_ext])
            elif mime_type not in VALID_MIME_TYPES[file_ext.lower()]:
                self._errors["file"] = self.error_class(["File extension %s does not match its valid mime type" % file_ext])

        return cleaned_data

    class Meta:
        model = Doc
        exclude = ('author', 'publication_date', 'update_date',)

class UploadImageForm(forms.Form):
    # note: field MUST be named 'upload' as this is the parameter named used by CKeditor
    upload = forms.ImageField()

    # override 'clean' method to validate the image field
    def clean(self):

        # invoke superclass cleaning method
        super(UploadImageForm, self).clean()

        # validate image
        validate_image(self, 'upload')

        return self.cleaned_data
<commit_msg>adding periods to messages<commit_after>from cog.models import *
from cog.models.auth import userHasContributorPermission
from django.forms import ModelForm, ModelMultipleChoiceField, NullBooleanSelect
from django.db import models
from django.contrib.admin.widgets import FilteredSelectMultiple
from django import forms
from django.forms import ModelForm, Textarea, TextInput, Select, CheckboxSelectMultiple
from django.core.exceptions import ObjectDoesNotExist
from os.path import basename
import re
from cog.utils import *
from django.db.models import Q
import operator
from cog.forms.forms_utils import validate_image
import magic
from cog.constants import VALID_MIME_TYPES
from cog.utils import default_clean_field

# list of invalid characters in uploaded documents file names
INVALID_CHARS = "[^a-zA-Z0-9_\-\.\/\s]"

class NewsForm(ModelForm):

    # extra fields not present in model
    peer_projects = ModelMultipleChoiceField(queryset=Project.objects.all(), required=False,
                                             widget=CheckboxSelectMultiple)
    parent_projects = ModelMultipleChoiceField(queryset=Project.objects.all(), required=False,
                                               widget=CheckboxSelectMultiple(attrs={'vertical-align': 'bottom'}))
    child_projects = ModelMultipleChoiceField(queryset=Project.objects.all(), required=False,
                                              widget=CheckboxSelectMultiple)

    # override __init__ method to customize the list of choices for the parent/peer/child projects
    def __init__(self, project, user, *args, **kwargs):

        super(NewsForm, self).__init__(*args, **kwargs)

        self.fields['parent_projects'].queryset = \
            Project.objects.filter(self._buildQuerySet(project.parents.all(), user)).distinct().order_by('short_name')
        self.fields['peer_projects'].queryset = \
            Project.objects.filter(self._buildQuerySet(project.peers.all(), user)).distinct().order_by('short_name')
        self.fields['child_projects'].queryset = \
            Project.objects.filter(self._buildQuerySet(project.children(), user)).distinct().order_by('short_name')

        # on update only: pre-populate extra fields with current selection
        if self.instance.id:
            self.fields['parent_projects'].initial = self.instance.other_projects.all()
            self.fields['peer_projects'].initial = self.instance.other_projects.all()
            self.fields['child_projects'].initial = self.instance.other_projects.all()

    # method to build a query set that contains only the projects the User has access to.
    # the User must be a Contributor to post news
    def _buildQuerySet(self, projects, user):
        qs = Q(pk=0)  # start with an empty query set - does not match any project
        for p in projects:
            if userHasContributorPermission(user, p):
                qs = qs | Q(pk=p.id)
        return qs

    def clean_title(self):
        return xss_clean_field(self, 'title')

    def clean_text(self):
        return xss_clean_field(self, 'text')

    class Meta:
        model = News
        fields = "__all__"

class DocForm(ModelForm):

    # extra field not present in model,
    # used for redirection to other URLs after for has been successfully submitted
    redirect = forms.CharField(required=False)

    # additional extra field to optionally create a resource under the selected folder
    folder = forms.ModelChoiceField(queryset=None, required=False)

    # override __init__ method to provide a filtered list of options for the bookmark folder
    def __init__(self, project, *args, **kwargs):

        super(DocForm, self).__init__(*args, **kwargs)

        # filter folders by project and active state
        # order by name in the form pull down
        self.fields['folder'].queryset = \
            Folder.objects.filter(project=project).filter(active=True).distinct().order_by('name')

    def clean(self):
        """Override clean method to check that file size does not exceed limit.
            At this point the file is still in memory only,
            so if error is thrown there is no need to remove it from disk."""
        cleaned_data = self.cleaned_data
        thefile = cleaned_data.get("file")
        title = cleaned_data.get("title")
        description = cleaned_data.get("description")

        if not thefile:
            self._errors["file"] = self.error_class(["Sorry, the file is empty."])
            return cleaned_data

        if re.search(INVALID_CHARS, thefile.name):
            self._errors['file'] = self.error_class(["Sorry, the filename contains invalid characters. "
                                                     "It can only contain letters, numbers, spaces, and _ - . /"])

        if re.search(INVALID_CHARS, title):
            self._errors['title'] = self.error_class(["Sorry, the document title contains invalid characters. "
                                                      "It can only contain letters, numbers, spaces, and _ - . /"])
        if re.search(INVALID_CHARS, description):
            self._errors['description'] = self.error_class(["Sorry, the document description contains invalid characters. "
                                                            "It can only contain letters, numbers, spaces, and _ - . /"])

        project = cleaned_data['project']
        if thefile.size > project.maxUploadSize:
            self._errors["file"] = self.error_class(["Sorry, the file size exceeds the maximum allowed."])

        # validate the file content
        # must write the file to a temporary location to validate it
        # choose to write to $SITE_MEDIA/tmp/file.name
        if len(self._errors)==0:

            file_ext = str(os.path.splitext(thefile.name)[1])
            mime_type = magic.from_buffer(thefile.read(1024), mime=True)
            print "Validating file extension=%s, mime type=%s" % (file_ext, mime_type)
            if not file_ext:
                self._errors["file"] = self.error_class(["File name must have an extension."])
            elif file_ext.lower() not in VALID_MIME_TYPES.keys():
                self._errors["file"] = self.error_class(["File extension %s is not supported." % file_ext])
            elif mime_type not in VALID_MIME_TYPES[file_ext.lower()]:
                self._errors["file"] = self.error_class(["File extension %s does not match its valid mime type." % file_ext])

        return cleaned_data

    class Meta:
        model = Doc
        exclude = ('author', 'publication_date', 'update_date',)

class UploadImageForm(forms.Form):
    # note: field MUST be named 'upload' as this is the parameter named used by CKeditor
    upload = forms.ImageField()

    # override 'clean' method to validate the image field
    def clean(self):

        # invoke superclass cleaning method
        super(UploadImageForm, self).clean()

        # validate image
        validate_image(self, 'upload')

        return self.cleaned_data
<|endoftext|>


<commit_before># cython.* namespace for pure mode.
__version__ = "0.21b1"

# BEGIN shameless copy from Cython/minivect/minitypes.py

class _ArrayType(object):

    is_array = True
    subtypes = ['dtype']

    def __init__(self, dtype, ndim, is_c_contig=False, is_f_contig=False,
                 inner_contig=False, broadcasting=None):
        self.dtype = dtype
        self.ndim = ndim
        self.is_c_contig = is_c_contig
        self.is_f_contig = is_f_contig
        self.inner_contig = inner_contig or is_c_contig or is_f_contig
        self.broadcasting = broadcasting

    def __repr__(self):
        axes = [":"] * self.ndim
        if self.is_c_contig:
            axes[-1] = "::1"
        elif self.is_f_contig:
            axes[0] = "::1"

        return "%s[%s]" % (self.dtype, ", ".join(axes))

def index_type(base_type, item):
    """
    Support array type creation by slicing, e.g. double[:, :] specifies
    a 2D strided array of doubles. The syntax is the same as for
    Cython memoryviews.
    """
    assert isinstance(item, (tuple, slice))

    class InvalidTypeSpecification(Exception):
        pass

    def verify_slice(s):
        if s.start or s.stop or s.step not in (None, 1):
            raise InvalidTypeSpecification(
                "Only a step of 1 may be provided to indicate C or "
                "Fortran contiguity")

    if isinstance(item, tuple):
        step_idx = None
        for idx, s in enumerate(item):
            verify_slice(s)
            if s.step and (step_idx or idx not in (0, len(item) - 1)):
                raise InvalidTypeSpecification(
                    "Step may only be provided once, and only in the "
                    "first or last dimension.")

            if s.step == 1:
                step_idx = idx

        return _ArrayType(base_type, len(item),
                          is_c_contig=step_idx == len(item) - 1,
                          is_f_contig=step_idx == 0)
    else:
        verify_slice(item)
        return _ArrayType(base_type, 1, is_c_contig=bool(item.step))

# END shameless copy

compiled = False

_Unspecified = object()

# Function decorators

def _empty_decorator(x):
    return x

def locals(**arg_types):
    return _empty_decorator

def test_assert_path_exists(*paths):
    return _empty_decorator

def test_fail_if_path_exists(*paths):
    return _empty_decorator

class _EmptyDecoratorAndManager(object):
    def __call__(self, x):
        return x
    def __enter__(self):
        pass
    def __exit__(self, exc_type, exc_value, traceback):
        pass

cclass = ccall = cfunc = _EmptyDecoratorAndManager()

returns = lambda type_arg: _EmptyDecoratorAndManager()

final = internal = type_version_tag = no_gc_clear = _empty_decorator

def inline(f, *args, **kwds):
  if isinstance(f, basestring):
    from Cython.Build.Inline import cython_inline
    return cython_inline(f, *args, **kwds)
  else:
    assert len(args) == len(kwds) == 0
    return f

def compile(f):
    from Cython.Build.Inline import RuntimeCompiledFunction
    return RuntimeCompiledFunction(f)

# Special functions

def cdiv(a, b):
    q = a / b
    if q < 0:
        q += 1

def cmod(a, b):
    r = a % b
    if (a*b) < 0:
        r -= b
    return r

# Emulated language constructs

def cast(type, *args):
    if hasattr(type, '__call__'):
        return type(*args)
    else:
        return args[0]

def sizeof(arg):
    return 1

def typeof(arg):
    return arg.__class__.__name__
    # return type(arg)

def address(arg):
    return pointer(type(arg))([arg])

def declare(type=None, value=_Unspecified, **kwds):
    if type not in (None, object) and hasattr(type, '__call__'):
        if value is not _Unspecified:
            return type(value)
        else:
            return type()
    else:
        return value

class _nogil(object):
    """Support for 'with nogil' statement
    """
    def __enter__(self):
        pass
    def __exit__(self, exc_class, exc, tb):
        return exc_class is None

nogil = _nogil()
gil = _nogil()
del _nogil

# Emulated types

class CythonMetaType(type):

    def __getitem__(type, ix):
        return array(type, ix)

CythonTypeObject = CythonMetaType('CythonTypeObject', (object,), {})

class CythonType(CythonTypeObject):

    def _pointer(self, n=1):
        for i in range(n):
            self = pointer(self)
        return self

class PointerType(CythonType):

    def __init__(self, value=None):
        if isinstance(value, (ArrayType, PointerType)):
            self._items = [cast(self._basetype, a) for a in value._items]
        elif isinstance(value, list):
            self._items = [cast(self._basetype, a) for a in value]
        elif value is None or value == 0:
            self._items = []
        else:
            raise ValueError

    def __getitem__(self, ix):
        if ix < 0:
            raise IndexError("negative indexing not allowed in C")
        return self._items[ix]

    def __setitem__(self, ix, value):
        if ix < 0:
            raise IndexError("negative indexing not allowed in C")
        self._items[ix] = cast(self._basetype, value)

    def __eq__(self, value):
        if value is None and not self._items:
            return True
        elif type(self) != type(value):
            return False
        else:
            return not self._items and not value._items

    def __repr__(self):
        return "%s *" % (self._basetype,)

class ArrayType(PointerType):

    def __init__(self):
        self._items = [None] * self._n

class StructType(CythonType):

    def __init__(self, cast_from=_Unspecified, **data):
        if cast_from is not _Unspecified:
            # do cast
            if len(data) > 0:
                raise ValueError('Cannot accept keyword arguments when casting.')
            if type(cast_from) is not type(self):
                raise ValueError('Cannot cast from %s'%cast_from)
            for key, value in cast_from.__dict__.items():
                setattr(self, key, value)
        else:
            for key, value in data.iteritems():
                setattr(self, key, value)

    def __setattr__(self, key, value):
        if key in self._members:
            self.__dict__[key] = cast(self._members[key], value)
        else:
            raise AttributeError("Struct has no member '%s'" % key)

class UnionType(CythonType):

    def __init__(self, cast_from=_Unspecified, **data):
        if cast_from is not _Unspecified:
            # do type cast
            if len(data) > 0:
                raise ValueError('Cannot accept keyword arguments when casting.')
            if isinstance(cast_from, dict):
                datadict = cast_from
            elif type(cast_from) is type(self):
                datadict = cast_from.__dict__
            else:
                raise ValueError('Cannot cast from %s'%cast_from)
        else:
            datadict = data
        if len(datadict) > 1:
            raise AttributeError("Union can only store one field at a time.")
        for key, value in datadict.iteritems():
            setattr(self, key, value)

    def __setattr__(self, key, value):
        if key in '__dict__':
            CythonType.__setattr__(self, key, value)
        elif key in self._members:
            self.__dict__ = {key: cast(self._members[key], value)}
        else:
            raise AttributeError("Union has no member '%s'" % key)

def pointer(basetype):
    class PointerInstance(PointerType):
        _basetype = basetype
    return PointerInstance

def array(basetype, n):
    class ArrayInstance(ArrayType):
        _basetype = basetype
        _n = n
    return ArrayInstance

def struct(**members):
    class StructInstance(StructType):
        _members = members
    for key in members:
        setattr(StructInstance, key, None)
    return StructInstance

def union(**members):
    class UnionInstance(UnionType):
        _members = members
    for key in members:
        setattr(UnionInstance, key, None)
    return UnionInstance

class typedef(CythonType):

    def __init__(self, type, name=None):
        self._basetype = type
        self.name = name

    def __call__(self, *arg):
        value = cast(self._basetype, *arg)
        return value

    def __repr__(self):
        return self.name or str(self._basetype)

    __getitem__ = index_type

class _FusedType(CythonType):
    pass

def fused_type(*args):
    if not args:
        raise TypeError("Expected at least one type as argument")

    # Find the numeric type with biggest rank if all types are numeric
    rank = -1
    for type in args:
        if type not in (py_int, py_long, py_float, py_complex):
            break

        if type_ordering.index(type) > rank:
            result_type = type
    else:
        return result_type

    # Not a simple numeric type, return a fused type instance. The result
    # isn't really meant to be used, as we can't keep track of the context in
    # pure-mode. Casting won't do anything in this case.
    return _FusedType()

def _specialized_from_args(signatures, args, kwargs):
    "Perhaps this should be implemented in a TreeFragment in Cython code"
    raise Exception("yet to be implemented")

py_int = typedef(int, "int")
try:
    py_long = typedef(long, "long")
except NameError: # Py3
    py_long = typedef(int, "long")
py_float = typedef(float, "float")
py_complex = typedef(complex, "double complex")

# Predefined types

int_types = ['char', 'short', 'Py_UNICODE', 'int', 'Py_UCS4', 'long', 'longlong', 'Py_ssize_t', 'size_t']
float_types = ['longdouble', 'double', 'float']
complex_types = ['longdoublecomplex', 'doublecomplex', 'floatcomplex', 'complex']
other_types = ['bint', 'void']

to_repr = {
    'longlong': 'long long',
    'longdouble': 'long double',
    'longdoublecomplex': 'long double complex',
    'doublecomplex': 'double complex',
    'floatcomplex': 'float complex',
}.get

gs = globals()

for name in int_types:
    reprname = to_repr(name, name)
    gs[name] = typedef(py_int, reprname)
    if name not in ('Py_UNICODE', 'Py_UCS4') and not name.endswith('size_t'):
        gs['u'+name] = typedef(py_int, "unsigned " + reprname)
        gs['s'+name] = typedef(py_int, "signed " + reprname)

for name in float_types:
    gs[name] = typedef(py_float, to_repr(name, name))

for name in complex_types:
    gs[name] = typedef(py_complex, to_repr(name, name))

bint = typedef(bool, "bint")
void = typedef(int, "void")

for t in int_types + float_types + complex_types + other_types:
    for i in range(1, 4):
        gs["%s_%s" % ('p'*i, t)] = globals()[t]._pointer(i)

void = typedef(None, "void")
NULL = p_void(0)

integral = floating = numeric = _FusedType()

type_ordering = [py_int, py_long, py_float, py_complex]

class CythonDotParallel(object):
    """
    The cython.parallel module.
    """

    __all__ = ['parallel', 'prange', 'threadid']

    def parallel(self, num_threads=None):
        return nogil

    def prange(self, start=0, stop=None, step=1, schedule=None, nogil=False):
        if stop is None:
            stop = start
            start = 0
        return range(start, stop, step)

    def threadid(self):
        return 0

    # def threadsavailable(self):
        # return 1

import sys
sys.modules['cython.parallel'] = CythonDotParallel()
del sys
<commit_msg>prepare release of Cython 0.21 beta 2<commit_after># cython.* namespace for pure mode.
__version__ = "0.21b2"

# BEGIN shameless copy from Cython/minivect/minitypes.py

class _ArrayType(object):

    is_array = True
    subtypes = ['dtype']

    def __init__(self, dtype, ndim, is_c_contig=False, is_f_contig=False,
                 inner_contig=False, broadcasting=None):
        self.dtype = dtype
        self.ndim = ndim
        self.is_c_contig = is_c_contig
        self.is_f_contig = is_f_contig
        self.inner_contig = inner_contig or is_c_contig or is_f_contig
        self.broadcasting = broadcasting

    def __repr__(self):
        axes = [":"] * self.ndim
        if self.is_c_contig:
            axes[-1] = "::1"
        elif self.is_f_contig:
            axes[0] = "::1"

        return "%s[%s]" % (self.dtype, ", ".join(axes))

def index_type(base_type, item):
    """
    Support array type creation by slicing, e.g. double[:, :] specifies
    a 2D strided array of doubles. The syntax is the same as for
    Cython memoryviews.
    """
    assert isinstance(item, (tuple, slice))

    class InvalidTypeSpecification(Exception):
        pass

    def verify_slice(s):
        if s.start or s.stop or s.step not in (None, 1):
            raise InvalidTypeSpecification(
                "Only a step of 1 may be provided to indicate C or "
                "Fortran contiguity")

    if isinstance(item, tuple):
        step_idx = None
        for idx, s in enumerate(item):
            verify_slice(s)
            if s.step and (step_idx or idx not in (0, len(item) - 1)):
                raise InvalidTypeSpecification(
                    "Step may only be provided once, and only in the "
                    "first or last dimension.")

            if s.step == 1:
                step_idx = idx

        return _ArrayType(base_type, len(item),
                          is_c_contig=step_idx == len(item) - 1,
                          is_f_contig=step_idx == 0)
    else:
        verify_slice(item)
        return _ArrayType(base_type, 1, is_c_contig=bool(item.step))

# END shameless copy

compiled = False

_Unspecified = object()

# Function decorators

def _empty_decorator(x):
    return x

def locals(**arg_types):
    return _empty_decorator

def test_assert_path_exists(*paths):
    return _empty_decorator

def test_fail_if_path_exists(*paths):
    return _empty_decorator

class _EmptyDecoratorAndManager(object):
    def __call__(self, x):
        return x
    def __enter__(self):
        pass
    def __exit__(self, exc_type, exc_value, traceback):
        pass

cclass = ccall = cfunc = _EmptyDecoratorAndManager()

returns = lambda type_arg: _EmptyDecoratorAndManager()

final = internal = type_version_tag = no_gc_clear = _empty_decorator

def inline(f, *args, **kwds):
  if isinstance(f, basestring):
    from Cython.Build.Inline import cython_inline
    return cython_inline(f, *args, **kwds)
  else:
    assert len(args) == len(kwds) == 0
    return f

def compile(f):
    from Cython.Build.Inline import RuntimeCompiledFunction
    return RuntimeCompiledFunction(f)

# Special functions

def cdiv(a, b):
    q = a / b
    if q < 0:
        q += 1

def cmod(a, b):
    r = a % b
    if (a*b) < 0:
        r -= b
    return r

# Emulated language constructs

def cast(type, *args):
    if hasattr(type, '__call__'):
        return type(*args)
    else:
        return args[0]

def sizeof(arg):
    return 1

def typeof(arg):
    return arg.__class__.__name__
    # return type(arg)

def address(arg):
    return pointer(type(arg))([arg])

def declare(type=None, value=_Unspecified, **kwds):
    if type not in (None, object) and hasattr(type, '__call__'):
        if value is not _Unspecified:
            return type(value)
        else:
            return type()
    else:
        return value

class _nogil(object):
    """Support for 'with nogil' statement
    """
    def __enter__(self):
        pass
    def __exit__(self, exc_class, exc, tb):
        return exc_class is None

nogil = _nogil()
gil = _nogil()
del _nogil

# Emulated types

class CythonMetaType(type):

    def __getitem__(type, ix):
        return array(type, ix)

CythonTypeObject = CythonMetaType('CythonTypeObject', (object,), {})

class CythonType(CythonTypeObject):

    def _pointer(self, n=1):
        for i in range(n):
            self = pointer(self)
        return self

class PointerType(CythonType):

    def __init__(self, value=None):
        if isinstance(value, (ArrayType, PointerType)):
            self._items = [cast(self._basetype, a) for a in value._items]
        elif isinstance(value, list):
            self._items = [cast(self._basetype, a) for a in value]
        elif value is None or value == 0:
            self._items = []
        else:
            raise ValueError

    def __getitem__(self, ix):
        if ix < 0:
            raise IndexError("negative indexing not allowed in C")
        return self._items[ix]

    def __setitem__(self, ix, value):
        if ix < 0:
            raise IndexError("negative indexing not allowed in C")
        self._items[ix] = cast(self._basetype, value)

    def __eq__(self, value):
        if value is None and not self._items:
            return True
        elif type(self) != type(value):
            return False
        else:
            return not self._items and not value._items

    def __repr__(self):
        return "%s *" % (self._basetype,)

class ArrayType(PointerType):

    def __init__(self):
        self._items = [None] * self._n

class StructType(CythonType):

    def __init__(self, cast_from=_Unspecified, **data):
        if cast_from is not _Unspecified:
            # do cast
            if len(data) > 0:
                raise ValueError('Cannot accept keyword arguments when casting.')
            if type(cast_from) is not type(self):
                raise ValueError('Cannot cast from %s'%cast_from)
            for key, value in cast_from.__dict__.items():
                setattr(self, key, value)
        else:
            for key, value in data.iteritems():
                setattr(self, key, value)

    def __setattr__(self, key, value):
        if key in self._members:
            self.__dict__[key] = cast(self._members[key], value)
        else:
            raise AttributeError("Struct has no member '%s'" % key)

class UnionType(CythonType):

    def __init__(self, cast_from=_Unspecified, **data):
        if cast_from is not _Unspecified:
            # do type cast
            if len(data) > 0:
                raise ValueError('Cannot accept keyword arguments when casting.')
            if isinstance(cast_from, dict):
                datadict = cast_from
            elif type(cast_from) is type(self):
                datadict = cast_from.__dict__
            else:
                raise ValueError('Cannot cast from %s'%cast_from)
        else:
            datadict = data
        if len(datadict) > 1:
            raise AttributeError("Union can only store one field at a time.")
        for key, value in datadict.iteritems():
            setattr(self, key, value)

    def __setattr__(self, key, value):
        if key in '__dict__':
            CythonType.__setattr__(self, key, value)
        elif key in self._members:
            self.__dict__ = {key: cast(self._members[key], value)}
        else:
            raise AttributeError("Union has no member '%s'" % key)

def pointer(basetype):
    class PointerInstance(PointerType):
        _basetype = basetype
    return PointerInstance

def array(basetype, n):
    class ArrayInstance(ArrayType):
        _basetype = basetype
        _n = n
    return ArrayInstance

def struct(**members):
    class StructInstance(StructType):
        _members = members
    for key in members:
        setattr(StructInstance, key, None)
    return StructInstance

def union(**members):
    class UnionInstance(UnionType):
        _members = members
    for key in members:
        setattr(UnionInstance, key, None)
    return UnionInstance

class typedef(CythonType):

    def __init__(self, type, name=None):
        self._basetype = type
        self.name = name

    def __call__(self, *arg):
        value = cast(self._basetype, *arg)
        return value

    def __repr__(self):
        return self.name or str(self._basetype)

    __getitem__ = index_type

class _FusedType(CythonType):
    pass

def fused_type(*args):
    if not args:
        raise TypeError("Expected at least one type as argument")

    # Find the numeric type with biggest rank if all types are numeric
    rank = -1
    for type in args:
        if type not in (py_int, py_long, py_float, py_complex):
            break

        if type_ordering.index(type) > rank:
            result_type = type
    else:
        return result_type

    # Not a simple numeric type, return a fused type instance. The result
    # isn't really meant to be used, as we can't keep track of the context in
    # pure-mode. Casting won't do anything in this case.
    return _FusedType()

def _specialized_from_args(signatures, args, kwargs):
    "Perhaps this should be implemented in a TreeFragment in Cython code"
    raise Exception("yet to be implemented")

py_int = typedef(int, "int")
try:
    py_long = typedef(long, "long")
except NameError: # Py3
    py_long = typedef(int, "long")
py_float = typedef(float, "float")
py_complex = typedef(complex, "double complex")

# Predefined types

int_types = ['char', 'short', 'Py_UNICODE', 'int', 'Py_UCS4', 'long', 'longlong', 'Py_ssize_t', 'size_t']
float_types = ['longdouble', 'double', 'float']
complex_types = ['longdoublecomplex', 'doublecomplex', 'floatcomplex', 'complex']
other_types = ['bint', 'void']

to_repr = {
    'longlong': 'long long',
    'longdouble': 'long double',
    'longdoublecomplex': 'long double complex',
    'doublecomplex': 'double complex',
    'floatcomplex': 'float complex',
}.get

gs = globals()

for name in int_types:
    reprname = to_repr(name, name)
    gs[name] = typedef(py_int, reprname)
    if name not in ('Py_UNICODE', 'Py_UCS4') and not name.endswith('size_t'):
        gs['u'+name] = typedef(py_int, "unsigned " + reprname)
        gs['s'+name] = typedef(py_int, "signed " + reprname)

for name in float_types:
    gs[name] = typedef(py_float, to_repr(name, name))

for name in complex_types:
    gs[name] = typedef(py_complex, to_repr(name, name))

bint = typedef(bool, "bint")
void = typedef(int, "void")

for t in int_types + float_types + complex_types + other_types:
    for i in range(1, 4):
        gs["%s_%s" % ('p'*i, t)] = globals()[t]._pointer(i)

void = typedef(None, "void")
NULL = p_void(0)

integral = floating = numeric = _FusedType()

type_ordering = [py_int, py_long, py_float, py_complex]

class CythonDotParallel(object):
    """
    The cython.parallel module.
    """

    __all__ = ['parallel', 'prange', 'threadid']

    def parallel(self, num_threads=None):
        return nogil

    def prange(self, start=0, stop=None, step=1, schedule=None, nogil=False):
        if stop is None:
            stop = start
            start = 0
        return range(start, stop, step)

    def threadid(self):
        return 0

    # def threadsavailable(self):
        # return 1

import sys
sys.modules['cython.parallel'] = CythonDotParallel()
del sys
<|endoftext|>


<commit_before># encoding: utf-8
"""
manager.py

Created by Thomas Mangin on 2011-11-30.
Copyright (c) 2011-2013  Exa Networks. All rights reserved.
"""

import time
from Queue import Queue

from .worker import Redirector

from exaproxy.util.log.logger import Logger
# Do we really need to call join() on the thread as we are stoppin on our own ?

class RedirectorManager (object):
	def __init__ (self,configuration,poller):
		self.configuration = configuration

		self.low = configuration.redirector.minimum       # minimum number of workers at all time
		self.high = configuration.redirector.maximum      # maximum numbe of workers at all time
		self.program = configuration.redirector.program   # what program speaks the squid redirector API

		self.nbq = 0                      # number of request waiting to be filtered
		self.nextid = 1                   # incremental number to make the name of the next worker
		self.queue = Queue()              # queue with HTTP headers to process
		self.poller = poller              # poller interface that checks for events on sockets
		self.worker = {}                  # our workers threads
		self.closing = {}                 # workers that are currently closing and must be joined with when they are ready
		self.running = True               # we are running

		self.log = Logger('manager', configuration.log.manager)

	def _getid(self):
		id = str(self.nextid)
		self.nextid +=1
		return id

	def _spawn (self):
		"""add one worker to the pool"""
		wid = self._getid()

		worker = Redirector(self.configuration,wid,self.queue,self.program)
		self.poller.addReadSocket('read_workers', worker.response_box_read)
		self.worker[wid] = worker
		self.log.info("added a worker")
		self.log.info("we have %d workers. defined range is ( %d / %d )" % (len(self.worker),self.low,self.high))
		self.worker[wid].start()

	def spawn (self,number=1):
		"""create the set number of worker"""
		self.log.info("spawning %d more worker" % number)
		for _ in range(number):
			self._spawn()

	def respawn (self):
		"""make sure we reach the minimum number of workers"""
		number = max(min(len(self.worker),self.high),self.low)
		for wid in set(self.worker):
			self.reap(wid)
		self.spawn(number)

	def reap (self,wid):
		self.log.info('we are killing worker %s' % wid)
		worker = self.worker[wid]
		self.worker.pop(wid)
		self.closing[wid] = worker
		worker.stop()  # will cause the worker to stop when it can

	def start (self):
		"""spawn our minimum number of workers"""
		self.log.info("starting workers.")
		self.spawn(max(0,self.low-len(self.worker)))

	def stop (self):
		"""tell all our worker to stop reading the queue and stop"""
		self.running = False
		threads = self.worker.values() + self.closing.values()
		if len(self.worker):
			self.log.info("stopping %d workers." % len(self.worker))
			for wid in set(self.worker):
				self.reap(wid)
			for thread in threads:
				self.request(None, None, None, 'nop')
			for thread in threads:
				thread.destroyProcess()
				thread.join()

		self.worker = {}
		self.closing = {}

	def _oldest (self):
		"""find the oldest worker"""
		oldest = None
		past = time.time()
		for wid in set(self.worker):
			creation = self.worker[wid].creation
			if creation < past:
				past = creation
				oldest = self.worker[wid]
		return oldest

	def provision (self):
		"""manage our workers to make sure we have enough to consume the queue"""
		if not self.running:
			return

		size = self.nbq
		num_workers = len(self.worker)

		# we are now overprovisioned
		if size < num_workers:
			if size <= self.low:
				return
			self.log.info("we have too many workers, killing the oldest")
			# if we have to kill one, at least stop the one who had the most chance to memory leak :)
			worker = self._oldest()
			if worker:
				self.reap(worker.wid)
		# we need more workers
		else:
			# bad we are bleeding workers !
			if num_workers < self.low:
				self.log.info("we lost some workers, respawing")
				self.respawn()
			# nothing we can do we have reach our limit
			if num_workers >= self.high:
				self.log.warning("we need more workers by we reach our ceiling ! help !")
				return
			# try to figure a good number to add ..
			# no less than one, no more than to reach self.high, lower between self.low and a quarter of the allowed growth
			nb_to_add = int(min(max(1,min(self.low,(self.high-self.low)/4)),self.high-num_workers))
			self.log.warning("we are low on workers, adding a few (%d)" % nb_to_add)
			self.spawn(nb_to_add)

	def request(self, client_id, peer, request, source):
		self.nbq += 1
		return self.queue.put((client_id,peer,request,source,False))

	def getDecision(self, box):
		# NOTE: reads may block if we send badly formatted data
		self.nbq -=1
		try:
			r_buffer = box.read(3)
			while r_buffer.isdigit():
				r_buffer += box.read(1)

			if ':' in r_buffer:
				size, response = r_buffer.split(':', 1)
				if size.isdigit():
					size = int(size)
				else:
					size, response = None, None
			else:   # not a netstring
				size, response = None, None

			if size is not None:
				required = size + 1 - len(response)
				response += box.read(required)

			if response is not None:
				if response.endswith(','):
					response = response[:-1]
				else:
					response = None

		except ValueError:  # I/O operation on closed file
			worker = self.worker.get(box, None)
			if worker is not None:
				worker.destroyProcess()

			response = None
		except TypeError:
			response = None

		try:
			if response:
				client_id, command, decision = response.split('\0', 2)
			else:
				client_id = None
				command = None
				decision = None

		except (ValueError, TypeError):
			client_id = None
			command = None
			decision = None

		if command == 'requeue':
			_client_id, _peer, _source, _header = response.split('\0', 3)
			self.queue.put((_client_id,_peer,_header,_source,True))

			client_id = None
			command = None
			decision = None

		elif command == 'hangup':
			wid = decision
			client_id = None
			command = None
			decision = None

			worker = self.worker.pop(wid, None)

			if worker:
				self.poller.removeReadSocket('read_workers', worker.response_box_read)
				worker.shutdown()
				worker.join()
			else:
				worker = self.closing.pop(wid, None)

		elif command == 'stats':
			wid, timestamp, stats = decision
			self.storeStats(timestamp, wid, stats)

			client_id = None
			command = None
			decision = None

		return client_id, command, decision

	def showInternalError(self):
		return 'file', '\0'.join(('200', 'internal_error.html'))

	def requestStats(self):
		for wid, worker in self.worker.iteritems():
			worker.requestStats()

	def storeStats(self, timestamp, wid, stats):
		pairs = (d.split('=',1) for d in stats.split('?', 1).split('&'))
		d = self.cache.setdefault(timestamp, {})

		for k, v in pairs:
			d.setdefault(k, []).append(v)
<commit_msg>make sure we shutdown the worker when they were already in closing state<commit_after># encoding: utf-8
"""
manager.py

Created by Thomas Mangin on 2011-11-30.
Copyright (c) 2011-2013  Exa Networks. All rights reserved.
"""

import time
from Queue import Queue

from .worker import Redirector

from exaproxy.util.log.logger import Logger
# Do we really need to call join() on the thread as we are stoppin on our own ?

class RedirectorManager (object):
	def __init__ (self,configuration,poller):
		self.configuration = configuration

		self.low = configuration.redirector.minimum       # minimum number of workers at all time
		self.high = configuration.redirector.maximum      # maximum numbe of workers at all time
		self.program = configuration.redirector.program   # what program speaks the squid redirector API

		self.nbq = 0                      # number of request waiting to be filtered
		self.nextid = 1                   # incremental number to make the name of the next worker
		self.queue = Queue()              # queue with HTTP headers to process
		self.poller = poller              # poller interface that checks for events on sockets
		self.worker = {}                  # our workers threads
		self.closing = {}                 # workers that are currently closing and must be joined with when they are ready
		self.running = True               # we are running

		self.log = Logger('manager', configuration.log.manager)

	def _getid(self):
		id = str(self.nextid)
		self.nextid +=1
		return id

	def _spawn (self):
		"""add one worker to the pool"""
		wid = self._getid()

		worker = Redirector(self.configuration,wid,self.queue,self.program)
		self.poller.addReadSocket('read_workers', worker.response_box_read)
		self.worker[wid] = worker
		self.log.info("added a worker")
		self.log.info("we have %d workers. defined range is ( %d / %d )" % (len(self.worker),self.low,self.high))
		self.worker[wid].start()

	def spawn (self,number=1):
		"""create the set number of worker"""
		self.log.info("spawning %d more worker" % number)
		for _ in range(number):
			self._spawn()

	def respawn (self):
		"""make sure we reach the minimum number of workers"""
		number = max(min(len(self.worker),self.high),self.low)
		for wid in set(self.worker):
			self.reap(wid)
		self.spawn(number)

	def reap (self,wid):
		self.log.info('we are killing worker %s' % wid)
		worker = self.worker[wid]
		self.worker.pop(wid)
		self.closing[wid] = worker
		worker.stop()  # will cause the worker to stop when it can

	def start (self):
		"""spawn our minimum number of workers"""
		self.log.info("starting workers.")
		self.spawn(max(0,self.low-len(self.worker)))

	def stop (self):
		"""tell all our worker to stop reading the queue and stop"""
		self.running = False
		threads = self.worker.values() + self.closing.values()
		if len(self.worker):
			self.log.info("stopping %d workers." % len(self.worker))
			for wid in set(self.worker):
				self.reap(wid)
			for thread in threads:
				self.request(None, None, None, 'nop')
			for thread in threads:
				thread.destroyProcess()
				thread.join()

		self.worker = {}
		self.closing = {}

	def _oldest (self):
		"""find the oldest worker"""
		oldest = None
		past = time.time()
		for wid in set(self.worker):
			creation = self.worker[wid].creation
			if creation < past:
				past = creation
				oldest = self.worker[wid]
		return oldest

	def provision (self):
		"""manage our workers to make sure we have enough to consume the queue"""
		if not self.running:
			return

		size = self.nbq
		num_workers = len(self.worker)

		# we are now overprovisioned
		if size < num_workers:
			if size <= self.low:
				return
			self.log.info("we have too many workers, killing the oldest")
			# if we have to kill one, at least stop the one who had the most chance to memory leak :)
			worker = self._oldest()
			if worker:
				self.reap(worker.wid)
		# we need more workers
		else:
			# bad we are bleeding workers !
			if num_workers < self.low:
				self.log.info("we lost some workers, respawing")
				self.respawn()
			# nothing we can do we have reach our limit
			if num_workers >= self.high:
				self.log.warning("we need more workers by we reach our ceiling ! help !")
				return
			# try to figure a good number to add ..
			# no less than one, no more than to reach self.high, lower between self.low and a quarter of the allowed growth
			nb_to_add = int(min(max(1,min(self.low,(self.high-self.low)/4)),self.high-num_workers))
			self.log.warning("we are low on workers, adding a few (%d)" % nb_to_add)
			self.spawn(nb_to_add)

	def request(self, client_id, peer, request, source):
		self.nbq += 1
		return self.queue.put((client_id,peer,request,source,False))

	def getDecision(self, box):
		# NOTE: reads may block if we send badly formatted data
		self.nbq -=1
		try:
			r_buffer = box.read(3)
			while r_buffer.isdigit():
				r_buffer += box.read(1)

			if ':' in r_buffer:
				size, response = r_buffer.split(':', 1)
				if size.isdigit():
					size = int(size)
				else:
					size, response = None, None
			else:   # not a netstring
				size, response = None, None

			if size is not None:
				required = size + 1 - len(response)
				response += box.read(required)

			if response is not None:
				if response.endswith(','):
					response = response[:-1]
				else:
					response = None

		except ValueError:  # I/O operation on closed file
			worker = self.worker.get(box, None)
			if worker is not None:
				worker.destroyProcess()

			response = None
		except TypeError:
			response = None

		try:
			if response:
				client_id, command, decision = response.split('\0', 2)
			else:
				client_id = None
				command = None
				decision = None

		except (ValueError, TypeError):
			client_id = None
			command = None
			decision = None

		if command == 'requeue':
			_client_id, _peer, _source, _header = response.split('\0', 3)
			self.queue.put((_client_id,_peer,_header,_source,True))

			client_id = None
			command = None
			decision = None

		elif command == 'hangup':
			wid = decision
			client_id = None
			command = None
			decision = None

			worker = self.worker.pop(wid, None)

			if worker:
				self.poller.removeReadSocket('read_workers', worker.response_box_read)
			else:
				worker = self.closing.pop(wid, None)

			if worker:
				worker.shutdown()
				worker.join()

		elif command == 'stats':
			wid, timestamp, stats = decision
			self.storeStats(timestamp, wid, stats)

			client_id = None
			command = None
			decision = None

		return client_id, command, decision

	def showInternalError(self):
		return 'file', '\0'.join(('200', 'internal_error.html'))

	def requestStats(self):
		for wid, worker in self.worker.iteritems():
			worker.requestStats()

	def storeStats(self, timestamp, wid, stats):
		pairs = (d.split('=',1) for d in stats.split('?', 1).split('&'))
		d = self.cache.setdefault(timestamp, {})

		for k, v in pairs:
			d.setdefault(k, []).append(v)
<|endoftext|>


<commit_before>import re
import sys

errors_encounter = 0
pattern = re.compile(r"_\(([\"']{,3})(?P<message>((?!\1).)*)\1(\s*,\s*context\s*=\s*([\"'])(?P<py_context>((?!\5).)*)\5)*(\s*,\s*(.)*?\s*(,\s*([\"'])(?P<js_context>((?!\11).)*)\11)*)*\)")
start_pattern = re.compile(r"_{1,2}\([\"'`]{1,3}.*[a-zA-Z]")

# _('this is valid')
# _('{0} {1}')
# _("{0} {1}")
# _("{0} valid")
# _("valid {0}")

# skip first argument
files = sys.argv[1:]
files_to_scan = [_file for _file in files if _file.endswith(('.py', '.js'))]

for _file in files_to_scan:
	with open(_file, 'r') as f:
		print(f'Checking: {_file}')
		file_lines = f.readlines()
		for line_number, line in enumerate(file_lines, 1):
			start_matches = start_pattern.search(line)
			if start_matches:
				match = pattern.search(line)
				if not match and line.endswith(',\n'):
					# concat remaining text to validate multiline pattern
					line = "".join(file_lines[line_number - 1:])
					line = line[start_matches.start() + 1:]
					match = pattern.match(line)

				if not match:
					errors_encounter += 1
					print(f'\nTranslation syntax error at line number: {line_number + 1}\n{line.strip()[:100]}')

if errors_encounter > 0:
	print('\nYou can visit "https://frappeframework.com/docs/user/en/translations" to resolve this error.')
	sys.exit(1)
else:
	print('\nGood To Go!')
<commit_msg>fix: eager lookup<commit_after>import re
import sys

errors_encounter = 0
pattern = re.compile(r"_\(([\"']{,3})(?P<message>((?!\1).)*)\1(\s*,\s*context\s*=\s*([\"'])(?P<py_context>((?!\5).)*)\5)*(\s*,\s*(.)*?\s*(,\s*([\"'])(?P<js_context>((?!\11).)*)\11)*)*\)")
start_pattern = re.compile(r"_{1,2}\([\"'`]{1,3}.*?[a-zA-Z]")

# _('this is valid')
# _('{0} {1}')
# _("{0} {1}")
# _("{0} valid")
# _("valid {0}")

# skip first argument
files = sys.argv[1:]
files_to_scan = [_file for _file in files if _file.endswith(('.py', '.js'))]

for _file in files_to_scan:
	with open(_file, 'r') as f:
		print(f'Checking: {_file}')
		file_lines = f.readlines()
		for line_number, line in enumerate(file_lines, 1):
			start_matches = start_pattern.search(line)
			if start_matches:
				match = pattern.search(line)
				if not match and line.endswith(',\n'):
					# concat remaining text to validate multiline pattern
					line = "".join(file_lines[line_number - 1:])
					line = line[start_matches.start() + 1:]
					match = pattern.match(line)

				if not match:
					errors_encounter += 1
					print(f'\nTranslation syntax error at line number: {line_number + 1}\n{line.strip()[:100]}')

if errors_encounter > 0:
	print('\nYou can visit "https://frappeframework.com/docs/user/en/translations" to resolve this error.')
	sys.exit(1)
else:
	print('\nGood To Go!')
<|endoftext|>