<commit_before>"""
Run integration tests from pixsim through redshifts

python -m desispec.test.integration_test
"""
from __future__ import absolute_import, print_function
import sys
import os
import random
import time

import numpy as np
from astropy.io import fits

from desispec.pipeline import runcmd
from desispec import io
from desispec.log import get_logger

#- prevent nose from trying to run this test since it takes too long
__test__ = False

def check_env():
    """
    Check required environment variables; raise RuntimeException if missing
    """
    log = get_logger()
    #- template locations
    missing_env = False
    if 'DESI_BASIS_TEMPLATES' not in os.environ:
        log.warning('missing $DESI_BASIS_TEMPLATES needed for simulating spectra'.format(name))
        missing_env = True

    if not os.path.isdir(os.getenv('DESI_BASIS_TEMPLATES')):
        log.warning('missing $DESI_BASIS_TEMPLATES directory')
        log.warning('e.g. see NERSC:/project/projectdirs/desi/spectro/templates/basis_templates/v1.0')
        missing_env = True

    for name in (
        'DESI_SPECTRO_SIM', 'DESI_SPECTRO_REDUX', 'PIXPROD', 'PRODNAME'):
        if name not in os.environ:
            log.warning("missing ${0}".format(name))
            missing_env = True

    if missing_env:
        log.warning("Why are these needed?")
        log.warning("    Simulations written to $DESI_SPECTRO_SIM/$PIXPROD/")
        log.warning("    Raw data read from $DESI_SPECTRO_DATA/")
        log.warning("    Spectro pipeline output written to $DESI_SPECTRO_REDUX/$PRODNAME/")
        log.warning("    Templates are read from $DESI_BASIS_TEMPLATES")

    #- Wait until end to raise exception so that we report everything that
    #- is missing before actually failing
    if missing_env:
        log.critical("missing env vars; exiting without running pipeline")
        sys.exit(1)

    #- Override $DESI_SPECTRO_DATA to match $DESI_SPECTRO_SIM/$PIXPROD
    os.environ['DESI_SPECTRO_DATA'] = os.path.join(os.getenv('DESI_SPECTRO_SIM'), os.getenv('PIXPROD'))

#- TODO: fix usage of night to be something other than today
def integration_test(night=None, nspec=5, clobber=False):
    """Run an integration test from raw data simulations through redshifts

    Args:
        night (str, optional): YEARMMDD, defaults to current night
        nspec (int, optional): number of spectra to include
        clobber (bool, optional): rerun steps even if outputs already exist

    Raises:
        RuntimeError if any script fails

    """
    log = get_logger()
    #- YEARMMDD string, rolls over at noon not midnight
    if night is None:
        night = time.strftime('%Y%m%d', time.localtime(time.time()-12*3600))

    #- check for required environment variables
    check_env()

    #- parameter dictionary that will later be used for formatting commands
    params = dict(night=night, nspec=nspec)

    #-----
    #- Input fibermaps, spectra, and pixel-level raw data
    for expid, flavor in zip([0,1,2], ['flat', 'arc', 'science']):
        cmd = "newexp-desi --flavor {flavor} --nspec {nspec} --night {night} --expid {expid}".format(
            expid=expid, flavor=flavor, **params)
        fibermap = io.findfile('fibermap', night, expid)
        simspec = '{}/simspec-{:08d}.fits'.format(os.path.dirname(fibermap), expid)
        inputs = []
        outputs = [fibermap, simspec]
        if runcmd(cmd, inputs, outputs, clobber) != 0:
            raise RuntimeError('pixsim newexp failed for {} exposure {}'.format(flavor, expid))

        cmd = "pixsim-desi --nspec {nspec} --night {night} --expid {expid}".format(expid=expid, **params)
        inputs = [fibermap, simspec]
        outputs = list()
        for camera in ['b0', 'r0', 'z0']:
            pixfile = io.findfile('pix', night, expid, camera)
            outputs.append(pixfile)
            outputs.append(os.path.join(os.path.dirname(pixfile), os.path.basename(pixfile).replace('pix-', 'simpix-')))
        if runcmd(cmd, inputs, outputs, clobber) != 0:
            raise RuntimeError('pixsim failed for {} exposure {}'.format(flavor, expid))

    #-----
    #- Extract

    waverange = dict(
        b = "3570,5940,1.0",
        r = "5630,7740,1.0",
        z = "7440,9830,1.0",
        )
    for expid in [0,1,2]:
        for channel in ['b', 'r', 'z']:
            camera = channel+'0'
            pixfile = io.findfile('pix', night, expid, camera)
            psffile = '{}/data/specpsf/psf-{}.fits'.format(os.getenv('DESIMODEL'), channel)
            framefile = io.findfile('frame', night, expid, camera)
            cmd = "exspec -i {pix} -p {psf} --specmin 0 --nspec {nspec} -w {wave} -o {frame}".format(
                pix=pixfile, psf=psffile, wave=waverange[channel], frame=framefile, **params)

            inputs = [pixfile, psffile]
            outputs = [framefile,]
            if runcmd(cmd, inputs, outputs, clobber) != 0:
                raise RuntimeError('extraction failed for {} expid {}'.format(camera, expid))

    #-----
    #- Fiber flat
    expid = 0
    for channel in ['b', 'r', 'z']:
        camera = channel+"0"
        framefile = io.findfile('frame', night, expid, camera)
        fiberflat = io.findfile('fiberflat', night, expid, camera)
        cmd = "desi_compute_fiberflat.py --infile {frame} --outfile {fiberflat}".format(
            frame=framefile, fiberflat=fiberflat, **params)
        inputs = [framefile,]
        outputs = [fiberflat,]
        if runcmd(cmd, inputs, outputs, clobber) != 0:
            raise RuntimeError('fiberflat failed for '+camera)

    #-----
    #- Sky model
    flat_expid = 0
    expid = 2
    for channel in ['b', 'r', 'z']:
        camera = channel+"0"
        framefile = io.findfile('frame', night, expid, camera)
        fibermap = io.findfile('fibermap', night, expid)
        fiberflat = io.findfile('fiberflat', night, flat_expid, camera)
        skyfile = io.findfile('sky', night, expid, camera)
        cmd="desi_compute_sky.py --infile {frame} --fibermap {fibermap} --fiberflat {fiberflat} --outfile {sky}".format(
            frame=framefile, fibermap=fibermap, fiberflat=fiberflat, sky=skyfile, **params)
        inputs = [framefile, fibermap, fiberflat]
        outputs = [skyfile, ]
        if runcmd(cmd, inputs, outputs, clobber) != 0:
            raise RuntimeError('sky model failed for '+camera)

    #-----
    #- Fit standard stars
    if 'STD_TEMPLATES' in os.environ:
        std_templates = os.getenv('STD_TEMPLATES')
    else:
        std_templates = os.getenv('DESI_ROOT')+'/spectro/templates/star_templates/v1.0/stdstar_templates_v1.0.fits'

    stdstarfile = io.findfile('stdstars', night, expid, spectrograph=0)
    cmd = """desi_fit_stdstars.py --spectrograph 0 \
      --fibermap {fibermap} \
      --fiberflatexpid {flat_expid} \
      --models {std_templates} --outfile {stdstars}""".format(
        flat_expid=flat_expid, fibermap=fibermap, std_templates=std_templates,
        stdstars=stdstarfile)

    inputs = [fibermap, std_templates]
    outputs = [stdstarfile,]
    if runcmd(cmd, inputs, outputs, clobber) != 0:
        raise RuntimeError('fitting stdstars failed')

    #-----
    #- Flux calibration
    for channel in ['b', 'r', 'z']:
        camera = channel+"0"
        framefile = io.findfile('frame', night, expid, camera)
        fibermap  = io.findfile('fibermap', night, expid)
        fiberflat = io.findfile('fiberflat', night, flat_expid, camera)
        skyfile   = io.findfile('sky', night, expid, camera)
        calibfile = io.findfile('calib', night, expid, camera)

        #- Compute flux calibration vector
        cmd = """desi_compute_fluxcalibration.py \
          --infile {frame} --fibermap {fibermap} --fiberflat {fiberflat} --sky {sky} \
          --models {stdstars} --outfile {calib}""".format(
            frame=framefile, fibermap=fibermap, fiberflat=fiberflat, sky=skyfile,
            stdstars=stdstarfile, calib=calibfile,
            )
        inputs = [framefile, fibermap, fiberflat, skyfile, stdstarfile]
        outputs = [calibfile,]
        if runcmd(cmd, inputs, outputs, clobber) != 0:
            raise RuntimeError('flux calibration failed for '+camera)

        #- Apply the flux calibration to write a cframe file
        cframefile = io.findfile('cframe', night, expid, camera)
        cmd = """desi_process_exposure.py \
          --infile {frame} --fiberflat {fiberflat} --sky {sky} --calib {calib} \
          --outfile {cframe}""".format(frame=framefile, fibermap=fibermap,
            fiberflat=fiberflat, sky=skyfile, calib=calibfile, cframe=cframefile)
        inputs = [framefile, fiberflat, skyfile, calibfile]
        outputs = [cframefile, ]
        if runcmd(cmd, inputs, outputs, clobber) != 0:
            raise RuntimeError('combining calibration steps failed for '+camera)

    #-----
    #- Bricks
    inputs = list()
    for camera in ['b0', 'r0', 'z0']:
        inputs.append( io.findfile('cframe', night, expid, camera) )

    outputs = list()
    fibermap = io.read_fibermap(io.findfile('fibermap', night, expid))
    bricks = set(fibermap['BRICKNAME'])
    for b in bricks:
        for channel in ['b', 'r', 'z']:
            outputs.append( io.findfile('brick', brickname=b, band=channel))

    cmd = "desi_make_bricks.py --night "+night
    if runcmd(cmd, inputs, outputs, clobber) != 0:
        raise RuntimeError('brick generation failed')

    #-----
    #- Redshifts!
    for b in bricks:
        inputs = [io.findfile('brick', brickname=b, band=channel) for channel in ['b', 'r', 'z']]
        zbestfile = io.findfile('zbest', brickname=b)
        outputs = [zbestfile, ]
        cmd = "desi_zfind.py --brick {} -o {}".format(b, zbestfile)
        if runcmd(cmd, inputs, outputs, clobber) != 0:
            raise RuntimeError('redshifts failed for brick '+b)

    #-----
    #- Did it work?
    #- (this combination of fibermap, simspec, and zbest is a pain)
    simdir = os.path.dirname(io.findfile('fibermap', night=night, expid=expid))
    simspec = '{}/simspec-{:08d}.fits'.format(simdir, expid)
    siminfo = fits.getdata(simspec, 'METADATA')

    print()
    print("--------------------------------------------------")
    print("Brick     True  z        ->  Class  z        zwarn")
    # print("3338p190  SKY   0.00000  ->  QSO    1.60853   12   - ok")
    for b in bricks:
        zbest = io.read_zbest(io.findfile('zbest', brickname=b))
        for i in range(len(zbest.z)):
            if zbest.type[i] == 'ssp_em_galaxy':
                objtype = 'GAL'
            elif zbest.type[i] == 'spEigenStar':
                objtype = 'STAR'
            else:
                objtype = zbest.type[i]

            z, zwarn = zbest.z[i], zbest.zwarn[i]

            j = np.where(fibermap['TARGETID'] == zbest.targetid[i])[0][0]
            truetype = siminfo['OBJTYPE'][j]
            truez = siminfo['REDSHIFT'][j]
            dv = 3e5*(z-truez)/(1+truez)
            if truetype == 'SKY' and zwarn > 0:
                status = 'ok'
            elif zwarn == 0:
                if truetype == 'LRG' and objtype == 'GAL' and abs(dv) < 150:
                    status = 'ok'
                elif truetype == 'ELG' and objtype == 'GAL' and abs(dv) < 150:
                    status = 'ok'
                elif truetype == 'QSO' and objtype == 'QSO' and abs(dv) < 750:
                    status = 'ok'
                elif truetype == 'STD' and objtype == 'STAR':
                    status = 'ok'
                else:
                    status = 'OOPS'
            else:
                status = 'OOPS'
            print('{0}  {1:4s} {2:8.5f}  -> {3:5s} {4:8.5f} {5:4d}  - {6}'.format(
                b, truetype, truez, objtype, z, zwarn, status))

    print("--------------------------------------------------")

if __name__ == '__main__':
    integration_test()
<commit_msg>fix science -> dark usage for integration test<commit_after>"""
Run integration tests from pixsim through redshifts

python -m desispec.test.integration_test
"""
from __future__ import absolute_import, print_function
import sys
import os
import random
import time

import numpy as np
from astropy.io import fits

from desispec.pipeline import runcmd
from desispec import io
from desispec.log import get_logger

#- prevent nose from trying to run this test since it takes too long
__test__ = False

def check_env():
    """
    Check required environment variables; raise RuntimeException if missing
    """
    log = get_logger()
    #- template locations
    missing_env = False
    if 'DESI_BASIS_TEMPLATES' not in os.environ:
        log.warning('missing $DESI_BASIS_TEMPLATES needed for simulating spectra'.format(name))
        missing_env = True

    if not os.path.isdir(os.getenv('DESI_BASIS_TEMPLATES')):
        log.warning('missing $DESI_BASIS_TEMPLATES directory')
        log.warning('e.g. see NERSC:/project/projectdirs/desi/spectro/templates/basis_templates/v1.0')
        missing_env = True

    for name in (
        'DESI_SPECTRO_SIM', 'DESI_SPECTRO_REDUX', 'PIXPROD', 'PRODNAME'):
        if name not in os.environ:
            log.warning("missing ${0}".format(name))
            missing_env = True

    if missing_env:
        log.warning("Why are these needed?")
        log.warning("    Simulations written to $DESI_SPECTRO_SIM/$PIXPROD/")
        log.warning("    Raw data read from $DESI_SPECTRO_DATA/")
        log.warning("    Spectro pipeline output written to $DESI_SPECTRO_REDUX/$PRODNAME/")
        log.warning("    Templates are read from $DESI_BASIS_TEMPLATES")

    #- Wait until end to raise exception so that we report everything that
    #- is missing before actually failing
    if missing_env:
        log.critical("missing env vars; exiting without running pipeline")
        sys.exit(1)

    #- Override $DESI_SPECTRO_DATA to match $DESI_SPECTRO_SIM/$PIXPROD
    os.environ['DESI_SPECTRO_DATA'] = os.path.join(os.getenv('DESI_SPECTRO_SIM'), os.getenv('PIXPROD'))

#- TODO: fix usage of night to be something other than today
def integration_test(night=None, nspec=5, clobber=False):
    """Run an integration test from raw data simulations through redshifts

    Args:
        night (str, optional): YEARMMDD, defaults to current night
        nspec (int, optional): number of spectra to include
        clobber (bool, optional): rerun steps even if outputs already exist

    Raises:
        RuntimeError if any script fails

    """
    log = get_logger()
    #- YEARMMDD string, rolls over at noon not midnight
    if night is None:
        night = time.strftime('%Y%m%d', time.localtime(time.time()-12*3600))

    #- check for required environment variables
    check_env()

    #- parameter dictionary that will later be used for formatting commands
    params = dict(night=night, nspec=nspec)

    #-----
    #- Input fibermaps, spectra, and pixel-level raw data
    for expid, flavor in zip([0,1,2], ['flat', 'arc', 'dark']):
        cmd = "newexp-desi --flavor {flavor} --nspec {nspec} --night {night} --expid {expid}".format(
            expid=expid, flavor=flavor, **params)
        fibermap = io.findfile('fibermap', night, expid)
        simspec = '{}/simspec-{:08d}.fits'.format(os.path.dirname(fibermap), expid)
        inputs = []
        outputs = [fibermap, simspec]
        if runcmd(cmd, inputs, outputs, clobber) != 0:
            raise RuntimeError('pixsim newexp failed for {} exposure {}'.format(flavor, expid))

        cmd = "pixsim-desi --nspec {nspec} --night {night} --expid {expid}".format(expid=expid, **params)
        inputs = [fibermap, simspec]
        outputs = list()
        for camera in ['b0', 'r0', 'z0']:
            pixfile = io.findfile('pix', night, expid, camera)
            outputs.append(pixfile)
            outputs.append(os.path.join(os.path.dirname(pixfile), os.path.basename(pixfile).replace('pix-', 'simpix-')))
        if runcmd(cmd, inputs, outputs, clobber) != 0:
            raise RuntimeError('pixsim failed for {} exposure {}'.format(flavor, expid))

    #-----
    #- Extract

    waverange = dict(
        b = "3570,5940,1.0",
        r = "5630,7740,1.0",
        z = "7440,9830,1.0",
        )
    for expid in [0,1,2]:
        for channel in ['b', 'r', 'z']:
            camera = channel+'0'
            pixfile = io.findfile('pix', night, expid, camera)
            psffile = '{}/data/specpsf/psf-{}.fits'.format(os.getenv('DESIMODEL'), channel)
            framefile = io.findfile('frame', night, expid, camera)
            cmd = "exspec -i {pix} -p {psf} --specmin 0 --nspec {nspec} -w {wave} -o {frame}".format(
                pix=pixfile, psf=psffile, wave=waverange[channel], frame=framefile, **params)

            inputs = [pixfile, psffile]
            outputs = [framefile,]
            if runcmd(cmd, inputs, outputs, clobber) != 0:
                raise RuntimeError('extraction failed for {} expid {}'.format(camera, expid))

    #-----
    #- Fiber flat
    expid = 0
    for channel in ['b', 'r', 'z']:
        camera = channel+"0"
        framefile = io.findfile('frame', night, expid, camera)
        fiberflat = io.findfile('fiberflat', night, expid, camera)
        cmd = "desi_compute_fiberflat.py --infile {frame} --outfile {fiberflat}".format(
            frame=framefile, fiberflat=fiberflat, **params)
        inputs = [framefile,]
        outputs = [fiberflat,]
        if runcmd(cmd, inputs, outputs, clobber) != 0:
            raise RuntimeError('fiberflat failed for '+camera)

    #-----
    #- Sky model
    flat_expid = 0
    expid = 2
    for channel in ['b', 'r', 'z']:
        camera = channel+"0"
        framefile = io.findfile('frame', night, expid, camera)
        fibermap = io.findfile('fibermap', night, expid)
        fiberflat = io.findfile('fiberflat', night, flat_expid, camera)
        skyfile = io.findfile('sky', night, expid, camera)
        cmd="desi_compute_sky.py --infile {frame} --fibermap {fibermap} --fiberflat {fiberflat} --outfile {sky}".format(
            frame=framefile, fibermap=fibermap, fiberflat=fiberflat, sky=skyfile, **params)
        inputs = [framefile, fibermap, fiberflat]
        outputs = [skyfile, ]
        if runcmd(cmd, inputs, outputs, clobber) != 0:
            raise RuntimeError('sky model failed for '+camera)

    #-----
    #- Fit standard stars
    if 'STD_TEMPLATES' in os.environ:
        std_templates = os.getenv('STD_TEMPLATES')
    else:
        std_templates = os.getenv('DESI_ROOT')+'/spectro/templates/star_templates/v1.0/stdstar_templates_v1.0.fits'

    stdstarfile = io.findfile('stdstars', night, expid, spectrograph=0)
    cmd = """desi_fit_stdstars.py --spectrograph 0 \
      --fibermap {fibermap} \
      --fiberflatexpid {flat_expid} \
      --models {std_templates} --outfile {stdstars}""".format(
        flat_expid=flat_expid, fibermap=fibermap, std_templates=std_templates,
        stdstars=stdstarfile)

    inputs = [fibermap, std_templates]
    outputs = [stdstarfile,]
    if runcmd(cmd, inputs, outputs, clobber) != 0:
        raise RuntimeError('fitting stdstars failed')

    #-----
    #- Flux calibration
    for channel in ['b', 'r', 'z']:
        camera = channel+"0"
        framefile = io.findfile('frame', night, expid, camera)
        fibermap  = io.findfile('fibermap', night, expid)
        fiberflat = io.findfile('fiberflat', night, flat_expid, camera)
        skyfile   = io.findfile('sky', night, expid, camera)
        calibfile = io.findfile('calib', night, expid, camera)

        #- Compute flux calibration vector
        cmd = """desi_compute_fluxcalibration.py \
          --infile {frame} --fibermap {fibermap} --fiberflat {fiberflat} --sky {sky} \
          --models {stdstars} --outfile {calib}""".format(
            frame=framefile, fibermap=fibermap, fiberflat=fiberflat, sky=skyfile,
            stdstars=stdstarfile, calib=calibfile,
            )
        inputs = [framefile, fibermap, fiberflat, skyfile, stdstarfile]
        outputs = [calibfile,]
        if runcmd(cmd, inputs, outputs, clobber) != 0:
            raise RuntimeError('flux calibration failed for '+camera)

        #- Apply the flux calibration to write a cframe file
        cframefile = io.findfile('cframe', night, expid, camera)
        cmd = """desi_process_exposure.py \
          --infile {frame} --fiberflat {fiberflat} --sky {sky} --calib {calib} \
          --outfile {cframe}""".format(frame=framefile, fibermap=fibermap,
            fiberflat=fiberflat, sky=skyfile, calib=calibfile, cframe=cframefile)
        inputs = [framefile, fiberflat, skyfile, calibfile]
        outputs = [cframefile, ]
        if runcmd(cmd, inputs, outputs, clobber) != 0:
            raise RuntimeError('combining calibration steps failed for '+camera)

    #-----
    #- Bricks
    inputs = list()
    for camera in ['b0', 'r0', 'z0']:
        inputs.append( io.findfile('cframe', night, expid, camera) )

    outputs = list()
    fibermap = io.read_fibermap(io.findfile('fibermap', night, expid))
    bricks = set(fibermap['BRICKNAME'])
    for b in bricks:
        for channel in ['b', 'r', 'z']:
            outputs.append( io.findfile('brick', brickname=b, band=channel))

    cmd = "desi_make_bricks.py --night "+night
    if runcmd(cmd, inputs, outputs, clobber) != 0:
        raise RuntimeError('brick generation failed')

    #-----
    #- Redshifts!
    for b in bricks:
        inputs = [io.findfile('brick', brickname=b, band=channel) for channel in ['b', 'r', 'z']]
        zbestfile = io.findfile('zbest', brickname=b)
        outputs = [zbestfile, ]
        cmd = "desi_zfind.py --brick {} -o {}".format(b, zbestfile)
        if runcmd(cmd, inputs, outputs, clobber) != 0:
            raise RuntimeError('redshifts failed for brick '+b)

    #-----
    #- Did it work?
    #- (this combination of fibermap, simspec, and zbest is a pain)
    simdir = os.path.dirname(io.findfile('fibermap', night=night, expid=expid))
    simspec = '{}/simspec-{:08d}.fits'.format(simdir, expid)
    siminfo = fits.getdata(simspec, 'METADATA')

    print()
    print("--------------------------------------------------")
    print("Brick     True  z        ->  Class  z        zwarn")
    # print("3338p190  SKY   0.00000  ->  QSO    1.60853   12   - ok")
    for b in bricks:
        zbest = io.read_zbest(io.findfile('zbest', brickname=b))
        for i in range(len(zbest.z)):
            if zbest.type[i] == 'ssp_em_galaxy':
                objtype = 'GAL'
            elif zbest.type[i] == 'spEigenStar':
                objtype = 'STAR'
            else:
                objtype = zbest.type[i]

            z, zwarn = zbest.z[i], zbest.zwarn[i]

            j = np.where(fibermap['TARGETID'] == zbest.targetid[i])[0][0]
            truetype = siminfo['OBJTYPE'][j]
            truez = siminfo['REDSHIFT'][j]
            dv = 3e5*(z-truez)/(1+truez)
            if truetype == 'SKY' and zwarn > 0:
                status = 'ok'
            elif zwarn == 0:
                if truetype == 'LRG' and objtype == 'GAL' and abs(dv) < 150:
                    status = 'ok'
                elif truetype == 'ELG' and objtype == 'GAL' and abs(dv) < 150:
                    status = 'ok'
                elif truetype == 'QSO' and objtype == 'QSO' and abs(dv) < 750:
                    status = 'ok'
                elif truetype == 'STD' and objtype == 'STAR':
                    status = 'ok'
                else:
                    status = 'OOPS'
            else:
                status = 'OOPS'
            print('{0}  {1:4s} {2:8.5f}  -> {3:5s} {4:8.5f} {5:4d}  - {6}'.format(
                b, truetype, truez, objtype, z, zwarn, status))

    print("--------------------------------------------------")

if __name__ == '__main__':
    integration_test()
<|endoftext|>


<commit_before>#!/bin/bash
source /usr/local/bin/config_env

PRE=""
CMD="python manage.py celery worker -Q feeds --concurrency=2 --scheduler=djcelery.schedulers.DatabaseScheduler --loglevel=DEBUG -E $CELERY_OPTS --settings=unisubs_settings"

cd $APP_DIR
if [ ! -z "$NEW_RELIC_LICENSE_KEY" ] ; then
    pip install -U newrelic
    PRE="newrelic-admin run-program "
fi

echo "Starting Feed Worker..."
$PRE $CMD
<commit_msg>Let's use concurrency=1 just to be super paranoid about the load<commit_after>#!/bin/bash
source /usr/local/bin/config_env

PRE=""
CMD="python manage.py celery worker -Q feeds --concurrency=1 --scheduler=djcelery.schedulers.DatabaseScheduler --loglevel=DEBUG -E $CELERY_OPTS --settings=unisubs_settings"

cd $APP_DIR
if [ ! -z "$NEW_RELIC_LICENSE_KEY" ] ; then
    pip install -U newrelic
    PRE="newrelic-admin run-program "
fi

echo "Starting Feed Worker..."
$PRE $CMD
<|endoftext|>


<commit_before>"""
.. module:: NURBS
    :platform: Unix, Windows
    :synopsis: A data storage and evaluation class for NURBS curves and surfaces

.. moduleauthor:: Onur Rauf Bingol

"""

import sys
import geomdl.BSpline as BSpline
import geomdl.utilities as utils

class Curve(BSpline.Curve):
    """ A data storage and evaluation class for 3D NURBS curves.

    **Data Storage**

    The following properties are present in this class:

    * order
    * degree
    * knotvector
    * delta
    * ctrlpts
    * weights
    * curvepts

    The function :func:`.read_ctrlpts()` provides an easy way to read weighted control points from a text file.
    Additional details for the text format can be found in `FORMATS.md <https://github.com/orbingol/NURBS-Python/blob/master/FORMATS.md>`_ file.

    .. note:: Control points are stored as a list of (x*w, y*w, z*w, w) coordinates

    **Evaluation**

    The evaluation methods class are:

    * :func:`.evaluate()`
    * :func:`.derivatives()`
    * :func:`.tangent()`
    * :func:`.insert_knot()`

    Please check the function reference for the details.

    .. note::

        If you update any of the data storage elements after the curve evaluation, the surface points stored in :py:attr:`~curvepts` property will be deleted automatically.
    """
    def __init__(self):
        super(Curve, self).__init__()
        # Override dimension variable
        self._dimension = 4  # 3D coordinates + weights
        self._rational = True

    @property
    def ctrlpts(self):
        """ Control points.

        :getter: Gets un-weighted control points. Use :py:attr:`~weights` to get weights vector.
        :setter: Sets weighted control points
        :type: list
        """
        ret_list = []
        for pt in self._control_points:
            temp = []
            for idx in range(self._dimension - 1):
                temp.append(float(pt[idx] / pt[-1]))
            ret_list.append(tuple(temp))
        return tuple(ret_list)

    @property
    def weights(self):
        """ Weights vector.

        :getter: Extracts the weights vector from weighted control points array
        :type: list
        """
        weights = []
        for pt in self._control_points:
            weights.append(pt[-1])
        return tuple(weights)

    # Prepares control points for exporting as a CSV file
    def _get_ctrlpts_for_exporting(self):
        """ Prepares control points for exporting as a CSV file.

        :return: list of control points
        :rtype: list
        """
        ctrlpts_noweight = []
        for pt in self._control_points:
            temp = []
            for idx in range(self._dimension - 1):
                temp.append(float(pt[idx] / pt[-1]))
            ctrlpts_noweight.append(temp)
        return ctrlpts_noweight

    # Prepares and returns the CSV file header
    def _get_csv_header(self):
        """ Prepares and returns the CSV file header.

        :return: header of the CSV file
        :rtype: str
        """
        return "coord x, coord y, coord z, scalar\n"

    # Evaluates the rational curve at the given parameter
    def curvept(self, u=-1, check_vars=True):
        """ Evaluates the NURBS curve at the given u parameter.

        :param u: parameter
        :type u: float
        :param check_vars: flag to disable variable checking (only for internal eval functions)
        :type check_vars: bool
        :return: evaluated curve point at the given knot value
        :rtype: list
        """
        if check_vars:
            # Check all parameters are set before the curve evaluation
            self._check_variables()
            # Check u parameters are correct
            if u < 0.0 or u > 1.0:
                raise ValueError('"u" value should be between 0 and 1.')

        # Algorithm A4.1
        span = utils.find_span(self._degree, tuple(self._knot_vector), len(self._control_points), u)
        basis = utils.basis_functions(self._degree, tuple(self._knot_vector), span, u)
        cptw = [0.0 for x in range(self._dimension)]
        for i in range(0, self._degree + 1):
            cptw[:] = [elem1 + (basis[i] * elem2) for elem1, elem2 in zip(cptw, self._control_points[span - self._degree + i])]

        # Divide by weight
        cpt = []
        for idx in range(self._dimension - 1):
            cpt.append(float(cptw[idx] / cptw[-1]))

        return cpt

    # Evaluates the rational curve derivative
    def derivatives(self, u=-1, order=0):
        """ Evaluates n-th order curve derivatives at the given u from the rational curve.

        :param u: knot value
        :type u: float
        :param order: derivative order
        :type order: integer
        :return: A list containing up to {order}-th derivative of the curve
        :rtype: list
        """
        # Call the parent function to evaluate A(u) and w(u) derivatives
        CKw = super(Curve, self).derivatives(u, order)

        # Algorithm A4.2
        CK = [[None for x in range(self._dimension - 1)] for y in range(order + 1)]
        for k in range(0, order + 1):
            v = []
            for idx in range(self._dimension - 1):
                v.append(CKw[idx])

            for i in range(1, order + 1):
                v[:] = [tmp - (utils.binomial_coefficient(k, i) * CKw[i][-1] * drv) for tmp, drv in zip(v, CK[k - i])]
            CK[k][:] = [tmp / CKw[0][-1] for tmp in v]

        # Return C(u) derivatives
        return CK

class Curve2D(Curve):
    """ A data storage and evaluation class for 2D NURBS curves.

    **Data Storage**

    The following properties are present in this class:

    * order
    * degree
    * knotvector
    * delta
    * ctrlpts
    * weights
    * curvepts

    The function :func:`.read_ctrlpts()` provides an easy way to read weighted control points from a text file.
    Additional details for the text format can be found in `FORMATS.md <https://github.com/orbingol/NURBS-Python/blob/master/FORMATS.md>`_ file.

    .. note:: Control points are stored as a list of (x*w, y*w, w) coordinates

    **Evaluation**

    The evaluation methods class are:

    * :func:`.evaluate()`
    * :func:`.derivatives()`
    * :func:`.tangent()`
    * :func:`.insert_knot()`

    Please check the function reference for the details.

    .. note::

        If you update any of the data storage elements after the curve evaluation, the surface points stored in :py:attr:`~curvepts` property will be deleted automatically.
    """
    def __init__(self):
        super(Curve2D, self).__init__()
        # Override dimension variable
        self._dimension = 3  # 2D coordinates + weights
        self._rational = True

    # Prepares and returns the CSV file header
    def _get_csv_header(self):
        """ Prepares and returns the CSV file header.

        :return: header of the CSV file
        :rtype: str
        """
        return "coord x, coord y, scalar\n"

class Surface(BSpline.Surface):
    """ A data storage and evaluation class for NURBS surfaces.

    **Data Storage**

    The following properties are present in this class:

    * order_u
    * order_v
    * degree_u
    * degree_v
    * knotvector_u
    * knotvector_v
    * delta
    * ctrlpts
    * ctrlpts2D
    * weights
    * surfpts

    The function :func:`.read_ctrlpts()` provides an easy way to read control points from a text file.
    Additional details for the text format can be found in `FORMATS.md <https://github.com/orbingol/NURBS-Python/blob/master/FORMATS.md>`_ file.

    .. note:: Control points are stored as a list of (x*w, y*w, z*w, w) coordinates

    **Evaluation**

    The evaluation methods are:

    * :func:`.evaluate()`
    * :func:`.derivatives()`
    * :func:`.tangent()`
    * :func:`.normal()`
    * :func:`.insert_knot()`

    Please check the function reference for the details.

    .. note::

        If you update any of the data storage elements after the surface evaluation, the surface points stored in :py:attr:`~surfpts` property will be deleted automatically.
    """
    def __init__(self):
        super(Surface, self).__init__()
        # Override dimension variable
        self._dimension = 4  # 3D coordinates + weights
        self._rational = True

    @property
    def ctrlpts(self):
        """ Control points.

        :getter: Gets un-weighted control points. Use :py:attr:`~weights` to get weights vector.
        :setter: Sets weighted control points.
        :type: list
        """
        ret_list = []
        for pt in self._control_points:
            temp = []
            for idx in range(self._dimension - 1):
                temp.append(float(pt[idx] / pt[-1]))
            ret_list.append(tuple(temp))
        return tuple(ret_list)

    @property
    def weights(self):
        """ Weights vector.

        :getter: Extracts the weights vector from weighted control points array
        :type: list
        """
        weights = []
        for pt in self._control_points:
            weights.append(pt[-1])
        return tuple(weights)

    # Prepares control points for exporting as a CSV file
    def _get_ctrlpts_for_exporting(self):
        """ Prepares control points for exporting as a CSV file.

        :return: list of control points
        :rtype: list
        """
        ctrlpts_noweight = []
        for pt in self._control_points:
            temp = []
            for idx in range(self._dimension - 1):
                temp.append(float(pt[idx] / pt[-1]))
            ctrlpts_noweight.append(temp)
        return ctrlpts_noweight

    # Prepares and returns the CSV file header
    def _get_csv_header(self):
        """ Prepares and returns the CSV file header.

        :return: header of the CSV file
        :rtype: str
        """
        return "coord x, coord y, coord z, scalar\n"

    # Evaluates rational surface at the given (u,v) parameters
    def surfpt(self, u=-1, v=-1, check_vars=True):
        """ Evaluates the NURBS surface at the given (u,v) parameters.

        :param u: parameter in the U direction
        :type u: float
        :param v: parameter in the V direction
        :type v: float
        :param check_vars: flag to disable variable checking (only for internal eval functions)
        :type check_vars: bool
        :return: evaluated surface point at the given knot values
        :rtype: list
        """
        if check_vars:
            # Check all parameters are set before the surface evaluation
            self._check_variables()
            # Check u and v parameters are correct
            utils.check_uv(u, v)

        # Algorithm A4.3
        span_v = utils.find_span(self._degree_v, tuple(self._knot_vector_v), self._control_points_size_v, v)
        basis_v = utils.basis_functions(self._degree_v, tuple(self._knot_vector_v), span_v, v)
        span_u = utils.find_span(self._degree_u, tuple(self._knot_vector_u), self._control_points_size_u, u)
        basis_u = utils.basis_functions(self._degree_u, tuple(self._knot_vector_u), span_u, u)
        idx_u = span_u - self._degree_u
        sptw = [0.0 for x in range(self._dimension)]

        for l in range(0, self._degree_v + 1):
            temp = [0.0 for x in range(self._dimension)]
            idx_v = span_v - self._degree_v + l
            for k in range(0, self._degree_u + 1):
                temp[:] = [tmp + (basis_u[k] * cp) for tmp, cp in zip(temp, self._control_points2D[idx_u + k][idx_v])]
            sptw[:] = [ptw + (basis_v[l] * tmp) for ptw, tmp in zip(sptw, temp)]

        # Divide by weight
        spt = []
        for idx in range(self._dimension - 1):
            spt.append(float(sptw[idx] / sptw[-1]))

        return spt

    # Evaluates n-th order rational surface derivatives at the given (u,v) parameter
    def derivatives(self, u=-1, v=-1, order=0):
        """ Evaluates n-th order surface derivatives at the given (u,v) parameter from the rational surface.

        * SKL[0][0] will be the surface point itself
        * SKL[0][1] will be the 1st derivative w.r.t. v
        * SKL[2][1] will be the 2nd derivative w.r.t. u and 1st derivative w.r.t. v

        :param u: parameter in the U direction
        :type u: float
        :param v: parameter in the V direction
        :type v: float
        :param order: derivative order
        :type order: integer
        :return: A list SKL, where SKL[k][l] is the derivative of the surface S(u,v) w.r.t. u k times and v l times
        :rtype: list
        """
        # Call the parent function to evaluate A(u) and w(u) derivatives
        SKLw = super(Surface, self).derivatives(u, v, order)

        # Algorithm A4.4
        du = min(self._degree_u, order)
        dv = min(self._degree_v, order)

        # Generate an empty list of derivatives
        SKL = [[[None for x in range(self._dimension)] for y in range(dv + 1)] for z in range(du + 1)]

        for k in range(0, order + 1):
            for l in range(0, order - k + 1):
                v = []
                for idx in range(self._dimension - 1):
                    v.append(SKLw[idx])

                for j in range(1, l + 1):
                    v[:] = [tmp - (utils.binomial_coefficient(l, j) * SKLw[0][j][-1] * drv) for tmp, drv in zip(v, SKL[k][l - j])]
                for i in range(1, k + 1):
                    v[:] = [tmp - (utils.binomial_coefficient(k, i) * SKLw[i][0][-1] * drv) for tmp, drv in zip(v, SKL[k - i][l])]
                    v2 = [0.0 for x in range(self._dimension - 1)]
                    for j in range(1, l + 1):
                        v2[:] = [tmp + (utils.binomial_coefficient(l, j) * SKLw[i][j][-1] * drv) for tmp, drv in zip(v2, SKL[k - i][l - j])]
                    v[:] = [tmp - (utils.binomial_coefficient(k, i) * tmp2) for tmp, tmp2 in zip(v, v2)]

                SKL[k][l][:] = [tmp / SKLw[0][0][-1] for tmp in v]

        # Return S(u,v) derivatives
        return SKL
<commit_msg>Add derivatives2 method as a wrapper to derivatives in NURBS module<commit_after>"""
.. module:: NURBS
    :platform: Unix, Windows
    :synopsis: A data storage and evaluation class for NURBS curves and surfaces

.. moduleauthor:: Onur Rauf Bingol

"""

import sys
import geomdl.BSpline as BSpline
import geomdl.utilities as utils

class Curve(BSpline.Curve):
    """ A data storage and evaluation class for 3D NURBS curves.

    **Data Storage**

    The following properties are present in this class:

    * order
    * degree
    * knotvector
    * delta
    * ctrlpts
    * weights
    * curvepts

    The function :func:`.read_ctrlpts()` provides an easy way to read weighted control points from a text file.
    Additional details for the text format can be found in `FORMATS.md <https://github.com/orbingol/NURBS-Python/blob/master/FORMATS.md>`_ file.

    .. note:: Control points are stored as a list of (x*w, y*w, z*w, w) coordinates

    **Evaluation**

    The evaluation methods class are:

    * :func:`.evaluate()`
    * :func:`.derivatives()`
    * :func:`.tangent()`
    * :func:`.insert_knot()`

    Please check the function reference for the details.

    .. note::

        If you update any of the data storage elements after the curve evaluation, the surface points stored in :py:attr:`~curvepts` property will be deleted automatically.
    """
    def __init__(self):
        super(Curve, self).__init__()
        # Override dimension variable
        self._dimension = 4  # 3D coordinates + weights
        self._rational = True

    @property
    def ctrlpts(self):
        """ Control points.

        :getter: Gets un-weighted control points. Use :py:attr:`~weights` to get weights vector.
        :setter: Sets weighted control points
        :type: list
        """
        ret_list = []
        for pt in self._control_points:
            temp = []
            for idx in range(self._dimension - 1):
                temp.append(float(pt[idx] / pt[-1]))
            ret_list.append(tuple(temp))
        return tuple(ret_list)

    @property
    def weights(self):
        """ Weights vector.

        :getter: Extracts the weights vector from weighted control points array
        :type: list
        """
        weights = []
        for pt in self._control_points:
            weights.append(pt[-1])
        return tuple(weights)

    # Prepares control points for exporting as a CSV file
    def _get_ctrlpts_for_exporting(self):
        """ Prepares control points for exporting as a CSV file.

        :return: list of control points
        :rtype: list
        """
        ctrlpts_noweight = []
        for pt in self._control_points:
            temp = []
            for idx in range(self._dimension - 1):
                temp.append(float(pt[idx] / pt[-1]))
            ctrlpts_noweight.append(temp)
        return ctrlpts_noweight

    # Prepares and returns the CSV file header
    def _get_csv_header(self):
        """ Prepares and returns the CSV file header.

        :return: header of the CSV file
        :rtype: str
        """
        return "coord x, coord y, coord z, scalar\n"

    # Evaluates the rational curve at the given parameter
    def curvept(self, u=-1, check_vars=True):
        """ Evaluates the NURBS curve at the given u parameter.

        :param u: parameter
        :type u: float
        :param check_vars: flag to disable variable checking (only for internal eval functions)
        :type check_vars: bool
        :return: evaluated curve point at the given knot value
        :rtype: list
        """
        if check_vars:
            # Check all parameters are set before the curve evaluation
            self._check_variables()
            # Check u parameters are correct
            if u < 0.0 or u > 1.0:
                raise ValueError('"u" value should be between 0 and 1.')

        # Algorithm A4.1
        span = utils.find_span(self._degree, tuple(self._knot_vector), len(self._control_points), u)
        basis = utils.basis_functions(self._degree, tuple(self._knot_vector), span, u)
        cptw = [0.0 for x in range(self._dimension)]
        for i in range(0, self._degree + 1):
            cptw[:] = [elem1 + (basis[i] * elem2) for elem1, elem2 in zip(cptw, self._control_points[span - self._degree + i])]

        # Divide by weight
        cpt = []
        for idx in range(self._dimension - 1):
            cpt.append(float(cptw[idx] / cptw[-1]))

        return cpt

    # Evaluates the rational curve derivative
    def derivatives(self, u=-1, order=0):
        """ Evaluates n-th order curve derivatives at the given u from the rational curve.

        :param u: knot value
        :type u: float
        :param order: derivative order
        :type order: integer
        :return: A list containing up to {order}-th derivative of the curve
        :rtype: list
        """
        # Call the parent function to evaluate A(u) and w(u) derivatives
        CKw = super(Curve, self).derivatives(u, order)

        # Algorithm A4.2
        CK = [[None for x in range(self._dimension - 1)] for y in range(order + 1)]
        for k in range(0, order + 1):
            v = []
            for idx in range(self._dimension - 1):
                v.append(CKw[idx])

            for i in range(1, order + 1):
                v[:] = [tmp - (utils.binomial_coefficient(k, i) * CKw[i][-1] * drv) for tmp, drv in zip(v, CK[k - i])]
            CK[k][:] = [tmp / CKw[0][-1] for tmp in v]

        # Return C(u) derivatives
        return CK

    # Evaluates the rational curve derivative
    def derivatives2(self, u=-1, order=0):
        """ Evaluates n-th order curve derivatives at the given u from the rational curve.

        :param u: knot value
        :type u: float
        :param order: derivative order
        :type order: integer
        :return: A list containing up to {order}-th derivative of the curve
        :rtype: list
        """
        return self.derivatives(u, order)

class Curve2D(Curve):
    """ A data storage and evaluation class for 2D NURBS curves.

    **Data Storage**

    The following properties are present in this class:

    * order
    * degree
    * knotvector
    * delta
    * ctrlpts
    * weights
    * curvepts

    The function :func:`.read_ctrlpts()` provides an easy way to read weighted control points from a text file.
    Additional details for the text format can be found in `FORMATS.md <https://github.com/orbingol/NURBS-Python/blob/master/FORMATS.md>`_ file.

    .. note:: Control points are stored as a list of (x*w, y*w, w) coordinates

    **Evaluation**

    The evaluation methods class are:

    * :func:`.evaluate()`
    * :func:`.derivatives()`
    * :func:`.tangent()`
    * :func:`.insert_knot()`

    Please check the function reference for the details.

    .. note::

        If you update any of the data storage elements after the curve evaluation, the surface points stored in :py:attr:`~curvepts` property will be deleted automatically.
    """
    def __init__(self):
        super(Curve2D, self).__init__()
        # Override dimension variable
        self._dimension = 3  # 2D coordinates + weights
        self._rational = True

    # Prepares and returns the CSV file header
    def _get_csv_header(self):
        """ Prepares and returns the CSV file header.

        :return: header of the CSV file
        :rtype: str
        """
        return "coord x, coord y, scalar\n"

class Surface(BSpline.Surface):
    """ A data storage and evaluation class for NURBS surfaces.

    **Data Storage**

    The following properties are present in this class:

    * order_u
    * order_v
    * degree_u
    * degree_v
    * knotvector_u
    * knotvector_v
    * delta
    * ctrlpts
    * ctrlpts2D
    * weights
    * surfpts

    The function :func:`.read_ctrlpts()` provides an easy way to read control points from a text file.
    Additional details for the text format can be found in `FORMATS.md <https://github.com/orbingol/NURBS-Python/blob/master/FORMATS.md>`_ file.

    .. note:: Control points are stored as a list of (x*w, y*w, z*w, w) coordinates

    **Evaluation**

    The evaluation methods are:

    * :func:`.evaluate()`
    * :func:`.derivatives()`
    * :func:`.tangent()`
    * :func:`.normal()`
    * :func:`.insert_knot()`

    Please check the function reference for the details.

    .. note::

        If you update any of the data storage elements after the surface evaluation, the surface points stored in :py:attr:`~surfpts` property will be deleted automatically.
    """
    def __init__(self):
        super(Surface, self).__init__()
        # Override dimension variable
        self._dimension = 4  # 3D coordinates + weights
        self._rational = True

    @property
    def ctrlpts(self):
        """ Control points.

        :getter: Gets un-weighted control points. Use :py:attr:`~weights` to get weights vector.
        :setter: Sets weighted control points.
        :type: list
        """
        ret_list = []
        for pt in self._control_points:
            temp = []
            for idx in range(self._dimension - 1):
                temp.append(float(pt[idx] / pt[-1]))
            ret_list.append(tuple(temp))
        return tuple(ret_list)

    @property
    def weights(self):
        """ Weights vector.

        :getter: Extracts the weights vector from weighted control points array
        :type: list
        """
        weights = []
        for pt in self._control_points:
            weights.append(pt[-1])
        return tuple(weights)

    # Prepares control points for exporting as a CSV file
    def _get_ctrlpts_for_exporting(self):
        """ Prepares control points for exporting as a CSV file.

        :return: list of control points
        :rtype: list
        """
        ctrlpts_noweight = []
        for pt in self._control_points:
            temp = []
            for idx in range(self._dimension - 1):
                temp.append(float(pt[idx] / pt[-1]))
            ctrlpts_noweight.append(temp)
        return ctrlpts_noweight

    # Prepares and returns the CSV file header
    def _get_csv_header(self):
        """ Prepares and returns the CSV file header.

        :return: header of the CSV file
        :rtype: str
        """
        return "coord x, coord y, coord z, scalar\n"

    # Evaluates rational surface at the given (u,v) parameters
    def surfpt(self, u=-1, v=-1, check_vars=True):
        """ Evaluates the NURBS surface at the given (u,v) parameters.

        :param u: parameter in the U direction
        :type u: float
        :param v: parameter in the V direction
        :type v: float
        :param check_vars: flag to disable variable checking (only for internal eval functions)
        :type check_vars: bool
        :return: evaluated surface point at the given knot values
        :rtype: list
        """
        if check_vars:
            # Check all parameters are set before the surface evaluation
            self._check_variables()
            # Check u and v parameters are correct
            utils.check_uv(u, v)

        # Algorithm A4.3
        span_v = utils.find_span(self._degree_v, tuple(self._knot_vector_v), self._control_points_size_v, v)
        basis_v = utils.basis_functions(self._degree_v, tuple(self._knot_vector_v), span_v, v)
        span_u = utils.find_span(self._degree_u, tuple(self._knot_vector_u), self._control_points_size_u, u)
        basis_u = utils.basis_functions(self._degree_u, tuple(self._knot_vector_u), span_u, u)
        idx_u = span_u - self._degree_u
        sptw = [0.0 for x in range(self._dimension)]

        for l in range(0, self._degree_v + 1):
            temp = [0.0 for x in range(self._dimension)]
            idx_v = span_v - self._degree_v + l
            for k in range(0, self._degree_u + 1):
                temp[:] = [tmp + (basis_u[k] * cp) for tmp, cp in zip(temp, self._control_points2D[idx_u + k][idx_v])]
            sptw[:] = [ptw + (basis_v[l] * tmp) for ptw, tmp in zip(sptw, temp)]

        # Divide by weight
        spt = []
        for idx in range(self._dimension - 1):
            spt.append(float(sptw[idx] / sptw[-1]))

        return spt

    # Evaluates n-th order rational surface derivatives at the given (u,v) parameter
    def derivatives(self, u=-1, v=-1, order=0):
        """ Evaluates n-th order surface derivatives at the given (u,v) parameter from the rational surface.

        * SKL[0][0] will be the surface point itself
        * SKL[0][1] will be the 1st derivative w.r.t. v
        * SKL[2][1] will be the 2nd derivative w.r.t. u and 1st derivative w.r.t. v

        :param u: parameter in the U direction
        :type u: float
        :param v: parameter in the V direction
        :type v: float
        :param order: derivative order
        :type order: integer
        :return: A list SKL, where SKL[k][l] is the derivative of the surface S(u,v) w.r.t. u k times and v l times
        :rtype: list
        """
        # Call the parent function to evaluate A(u) and w(u) derivatives
        SKLw = super(Surface, self).derivatives(u, v, order)

        # Algorithm A4.4
        du = min(self._degree_u, order)
        dv = min(self._degree_v, order)

        # Generate an empty list of derivatives
        SKL = [[[None for x in range(self._dimension)] for y in range(dv + 1)] for z in range(du + 1)]

        for k in range(0, order + 1):
            for l in range(0, order - k + 1):
                v = []
                for idx in range(self._dimension - 1):
                    v.append(SKLw[idx])

                for j in range(1, l + 1):
                    v[:] = [tmp - (utils.binomial_coefficient(l, j) * SKLw[0][j][-1] * drv) for tmp, drv in zip(v, SKL[k][l - j])]
                for i in range(1, k + 1):
                    v[:] = [tmp - (utils.binomial_coefficient(k, i) * SKLw[i][0][-1] * drv) for tmp, drv in zip(v, SKL[k - i][l])]
                    v2 = [0.0 for x in range(self._dimension - 1)]
                    for j in range(1, l + 1):
                        v2[:] = [tmp + (utils.binomial_coefficient(l, j) * SKLw[i][j][-1] * drv) for tmp, drv in zip(v2, SKL[k - i][l - j])]
                    v[:] = [tmp - (utils.binomial_coefficient(k, i) * tmp2) for tmp, tmp2 in zip(v, v2)]

                SKL[k][l][:] = [tmp / SKLw[0][0][-1] for tmp in v]

        # Return S(u,v) derivatives
        return SKL
<|endoftext|>


<commit_before>#!/usr/bin/env python3

import os

from doit import get_var
from ruamel import yaml

from api.autocert.config import _update_config, CONFIG_YML, DOT_CONFIG_YML

DOIT_CONFIG = {
    'default_tasks': ['deploy', 'rmimages', 'rmvolumes', 'count'],
    'verbosity': 2,
}

USER = os.getenv('USER')

MINIMUM_DOCKER_COMPOSE_VERSION = '1.6'

LOG_LEVELS = [
    'DEBUG',
    'INFO',
    'WARNING',
    'ERROR',
    'CRITICAL',
]

class UnknownPkgmgrError(Exception):
    def __init__(self):
        super(UnknownPkgmgrError, self).__init__('unknown pkgmgr!')

def check_hash(program):
    from subprocess import check_call, CalledProcessError
    try:
        check_call('hash {program}'.format(**locals()), shell=True)
        return True
    except CalledProcessError as cpe:
        return False

def get_pkgmgr():
    if check_hash('dpkg'):
        return 'deb'
    elif check_hash('rpm'):
        return 'rpm'
    raise UnknownPkgmgrError

def task_count():
    '''
    use the cloc utility to count lines of code
    '''
    excludes = [
        'dist',
        'venv',
        '__pycache__',
        'auto_cert_cli.egg-info',
    ]
    excludes = '--exclude-dir=' + ','.join(excludes)
    scandir = os.path.dirname(__file__)
    return {
        'actions': [
            'cloc {excludes} {scandir}'.format(**locals()),
        ],
        'uptodate': [
            lambda: not check_hash('cloc'),
        ],
    }

def task_checkreqs():
    '''
    check for required software
    '''
    DEBS = [
        'docker-engine',
        #'libpq-dev',
    ]
    RPMS = [
        'docker-engine',
        #'postgresql-devel',
    ]
    return {
        'deb': {
            'actions': ['dpkg -s ' + deb for deb in DEBS],
        },
        'rpm': {
            'actions': ['rpm -q ' + rpm for rpm in RPMS],
        }
    }[get_pkgmgr()]

def task_dockercompose():
    '''
    assert docker-compose version ({0}) or higher
    '''
    from utils.function import format_docstr
    format_docstr(task_dockercompose, MINIMUM_DOCKER_COMPOSE_VERSION)
    def check_docker_compose():
        import re
        from subprocess import check_output
        from packaging.version import parse as version_parse
        pattern = '(docker-compose version) ([0-9.]+)(, build [0-9]+)'
        output = check_output('docker-compose --version', shell=True).decode('utf-8').strip()
        regex = re.compile(pattern)
        match = regex.search(output)
        version = match.groups()[1]
        assert version_parse(version) >= version_parse(MINIMUM_DOCKER_COMPOSE_VERSION)

    return {
        'actions': [
            check_docker_compose,
        ],
    }

def task_noroot():
    '''
    make sure script isn't run as root
    '''
    then = 'echo "   DO NOT RUN AS ROOT!"; echo; exit 1'
    bash = 'if [[ $(id -u) -eq 0 ]]; then {0}; fi'.format(then)
    return {
        'actions': [
            'bash -c \'{0}\''.format(bash),
        ],
    }

def task_test():
    '''
    setup venv and run pytest
    '''
    return {
        'task_dep': [
            'noroot',
            'config',
        ],
        'actions': [
            'virtualenv --python=$(which python3) venv',
            'venv/bin/pip3 install --upgrade pip',
            'venv/bin/pip install -r api/requirements.txt',
            'venv/bin/pip install -r tests/requirements.txt',
            'venv/bin/pytest -v tests/',
        ],
    }

def task_version():
    '''
    write git describe to VERSION file
    '''
    return {
        'actions': [
            'git describe | xargs echo -n > api/VERSION',
        ],
    }

def task_deploy():
    '''
    deloy flask app via docker-compose
    '''
    return {
        'task_dep': [
            'noroot',
            'checkreqs',
            'version',
            'test',
            'config',
            'dockercompose'
        ],
        'actions': [
            'docker-compose build',
            'docker-compose up --remove-orphans -d',
        ],
    }

def task_rmimages():
    '''
    remove dangling docker images
    '''
    query = '`docker images -q -f dangling=true`'
    return {
        'actions': [
            'docker rmi {query}'.format(**locals()),
        ],
        'uptodate': [
            '[ -z "{query}" ] && exit 0 || exit 1'.format(**locals()),
        ],
    }

def task_rmvolumes():
    '''
    remove dangling docker volumes
    '''
    query = '`docker volume ls -q -f dangling=true`'
    return {
        'actions': [
            'docker volume rm {query}'.format(**locals()),
        ],
        'uptodate': [
            '[ -z "{query}" ] && exit 0 || exit 1'.format(**locals()),
        ],
    }

def task_config():
    '''
    write config.yml -> .config.yml
    '''
    log_level = 'WARNING'
    filename = '{0}/LOG_LEVEL'.format(os.path.dirname(__file__))
    if os.path.isfile(filename):
        log_level = open(filename).read().strip()
    log_level = get_var('LOG_LEVEL', log_level)
    if log_level not in LOG_LEVELS:
        raise UnknownLogLevelError(log_level)
    punch = '''
    logging:
        loggers:
            api:
                level: {log_level}
        handlers:
            console:
                level: {log_level}
    '''.format(**locals())
    return {
        'actions': [
            'echo "cp {CONFIG_YML}\n-> {DOT_CONFIG_YML}"'.format(**globals()),
            'echo "setting LOG_LEVEL={log_level}"'.format(**locals()),
            'cp {CONFIG_YML} {DOT_CONFIG_YML}'.format(**globals()),
            lambda: _update_config(DOT_CONFIG_YML, yaml.safe_load(punch)),
        ]
    }

def task_example():
    '''
    cp|strip config.yml -> config.yml.example
    '''
    apikey = '82_CHAR_APIKEY'
    punch = '''
    authorities:
        digicert:
            apikey: {apikey}
    destinations:
        zeus:
            apikey: {apikey}
    '''.format(**locals())
    return {
        'actions': [
            'cp {CONFIG_YML}.example {CONFIG_YML}.bak'.format(**globals()),
            'cp {CONFIG_YML} {CONFIG_YML}.example'.format(**globals()),
            lambda: _update_config(CONFIG_YML+'.example', yaml.safe_load(punch)),
        ],
    }

def task_tidy():
    '''
    delete cached files
    '''
    TIDY_FILES = [
        '.doit.db/',
        'venv/',
        'api/VERSION',
    ]
    return {
        'actions': [
            'rm -rf ' + ' '.join(TIDY_FILES),
            'find . | grep -E "(__pycache__|\.pyc$)" | xargs rm -rf',
        ],
    }

def task_nuke():
    '''
    git clean and reset
    '''
    return {
        'task_dep': ['tidy'],
        'actions': [
            'docker-compose kill',
            'docker-compose rm -f',
            'git clean -fd',
            'git reset --hard HEAD',
        ],
    }

def task_setup():
    '''
    setup venv
    '''
    from utils.version import version
    return {
        'actions': [
            'rm -rf auto_cert_cli.egg-info/ venv/ dist/ __pycache__/',
            'virtualenv --python=python3 venv',
            'venv/bin/pip3 install --upgrade pip',
            'venv/bin/pip3 install -r cli/requirements.txt',
            'venv/bin/python3 ./setup.py install',
            'unzip -l venv/lib/python3.5/site-packages/auto_cert_cli-{0}-py3.5.egg'.format(version()),
        ],
    }

if __name__ == '__main__':
    print('should be run with doit installed')
    import doit
    doit.run(globals())
<commit_msg>removed unneeded as cpe expression<commit_after>#!/usr/bin/env python3

import os

from doit import get_var
from ruamel import yaml

from api.autocert.config import _update_config, CONFIG_YML, DOT_CONFIG_YML

DOIT_CONFIG = {
    'default_tasks': ['deploy', 'rmimages', 'rmvolumes', 'count'],
    'verbosity': 2,
}

USER = os.getenv('USER')

MINIMUM_DOCKER_COMPOSE_VERSION = '1.6'

LOG_LEVELS = [
    'DEBUG',
    'INFO',
    'WARNING',
    'ERROR',
    'CRITICAL',
]

class UnknownPkgmgrError(Exception):
    def __init__(self):
        super(UnknownPkgmgrError, self).__init__('unknown pkgmgr!')

def check_hash(program):
    from subprocess import check_call, CalledProcessError
    try:
        check_call('hash {program}'.format(**locals()), shell=True)
        return True
    except CalledProcessError:
        return False

def get_pkgmgr():
    if check_hash('dpkg'):
        return 'deb'
    elif check_hash('rpm'):
        return 'rpm'
    raise UnknownPkgmgrError

def task_count():
    '''
    use the cloc utility to count lines of code
    '''
    excludes = [
        'dist',
        'venv',
        '__pycache__',
        'auto_cert_cli.egg-info',
    ]
    excludes = '--exclude-dir=' + ','.join(excludes)
    scandir = os.path.dirname(__file__)
    return {
        'actions': [
            'cloc {excludes} {scandir}'.format(**locals()),
        ],
        'uptodate': [
            lambda: not check_hash('cloc'),
        ],
    }

def task_checkreqs():
    '''
    check for required software
    '''
    DEBS = [
        'docker-engine',
        #'libpq-dev',
    ]
    RPMS = [
        'docker-engine',
        #'postgresql-devel',
    ]
    return {
        'deb': {
            'actions': ['dpkg -s ' + deb for deb in DEBS],
        },
        'rpm': {
            'actions': ['rpm -q ' + rpm for rpm in RPMS],
        }
    }[get_pkgmgr()]

def task_dockercompose():
    '''
    assert docker-compose version ({0}) or higher
    '''
    from utils.function import format_docstr
    format_docstr(task_dockercompose, MINIMUM_DOCKER_COMPOSE_VERSION)
    def check_docker_compose():
        import re
        from subprocess import check_output
        from packaging.version import parse as version_parse
        pattern = '(docker-compose version) ([0-9.]+)(, build [0-9]+)'
        output = check_output('docker-compose --version', shell=True).decode('utf-8').strip()
        regex = re.compile(pattern)
        match = regex.search(output)
        version = match.groups()[1]
        assert version_parse(version) >= version_parse(MINIMUM_DOCKER_COMPOSE_VERSION)

    return {
        'actions': [
            check_docker_compose,
        ],
    }

def task_noroot():
    '''
    make sure script isn't run as root
    '''
    then = 'echo "   DO NOT RUN AS ROOT!"; echo; exit 1'
    bash = 'if [[ $(id -u) -eq 0 ]]; then {0}; fi'.format(then)
    return {
        'actions': [
            'bash -c \'{0}\''.format(bash),
        ],
    }

def task_test():
    '''
    setup venv and run pytest
    '''
    return {
        'task_dep': [
            'noroot',
            'config',
        ],
        'actions': [
            'virtualenv --python=$(which python3) venv',
            'venv/bin/pip3 install --upgrade pip',
            'venv/bin/pip install -r api/requirements.txt',
            'venv/bin/pip install -r tests/requirements.txt',
            'venv/bin/pytest -v tests/',
        ],
    }

def task_version():
    '''
    write git describe to VERSION file
    '''
    return {
        'actions': [
            'git describe | xargs echo -n > api/VERSION',
        ],
    }

def task_deploy():
    '''
    deloy flask app via docker-compose
    '''
    return {
        'task_dep': [
            'noroot',
            'checkreqs',
            'version',
            'test',
            'config',
            'dockercompose'
        ],
        'actions': [
            'docker-compose build',
            'docker-compose up --remove-orphans -d',
        ],
    }

def task_rmimages():
    '''
    remove dangling docker images
    '''
    query = '`docker images -q -f dangling=true`'
    return {
        'actions': [
            'docker rmi {query}'.format(**locals()),
        ],
        'uptodate': [
            '[ -z "{query}" ] && exit 0 || exit 1'.format(**locals()),
        ],
    }

def task_rmvolumes():
    '''
    remove dangling docker volumes
    '''
    query = '`docker volume ls -q -f dangling=true`'
    return {
        'actions': [
            'docker volume rm {query}'.format(**locals()),
        ],
        'uptodate': [
            '[ -z "{query}" ] && exit 0 || exit 1'.format(**locals()),
        ],
    }

def task_config():
    '''
    write config.yml -> .config.yml
    '''
    log_level = 'WARNING'
    filename = '{0}/LOG_LEVEL'.format(os.path.dirname(__file__))
    if os.path.isfile(filename):
        log_level = open(filename).read().strip()
    log_level = get_var('LOG_LEVEL', log_level)
    if log_level not in LOG_LEVELS:
        raise UnknownLogLevelError(log_level)
    punch = '''
    logging:
        loggers:
            api:
                level: {log_level}
        handlers:
            console:
                level: {log_level}
    '''.format(**locals())
    return {
        'actions': [
            'echo "cp {CONFIG_YML}\n-> {DOT_CONFIG_YML}"'.format(**globals()),
            'echo "setting LOG_LEVEL={log_level}"'.format(**locals()),
            'cp {CONFIG_YML} {DOT_CONFIG_YML}'.format(**globals()),
            lambda: _update_config(DOT_CONFIG_YML, yaml.safe_load(punch)),
        ]
    }

def task_example():
    '''
    cp|strip config.yml -> config.yml.example
    '''
    apikey = '82_CHAR_APIKEY'
    punch = '''
    authorities:
        digicert:
            apikey: {apikey}
    destinations:
        zeus:
            apikey: {apikey}
    '''.format(**locals())
    return {
        'actions': [
            'cp {CONFIG_YML}.example {CONFIG_YML}.bak'.format(**globals()),
            'cp {CONFIG_YML} {CONFIG_YML}.example'.format(**globals()),
            lambda: _update_config(CONFIG_YML+'.example', yaml.safe_load(punch)),
        ],
    }

def task_tidy():
    '''
    delete cached files
    '''
    TIDY_FILES = [
        '.doit.db/',
        'venv/',
        'api/VERSION',
    ]
    return {
        'actions': [
            'rm -rf ' + ' '.join(TIDY_FILES),
            'find . | grep -E "(__pycache__|\.pyc$)" | xargs rm -rf',
        ],
    }

def task_nuke():
    '''
    git clean and reset
    '''
    return {
        'task_dep': ['tidy'],
        'actions': [
            'docker-compose kill',
            'docker-compose rm -f',
            'git clean -fd',
            'git reset --hard HEAD',
        ],
    }

def task_setup():
    '''
    setup venv
    '''
    from utils.version import version
    return {
        'actions': [
            'rm -rf auto_cert_cli.egg-info/ venv/ dist/ __pycache__/',
            'virtualenv --python=python3 venv',
            'venv/bin/pip3 install --upgrade pip',
            'venv/bin/pip3 install -r cli/requirements.txt',
            'venv/bin/python3 ./setup.py install',
            'unzip -l venv/lib/python3.5/site-packages/auto_cert_cli-{0}-py3.5.egg'.format(version()),
        ],
    }

if __name__ == '__main__':
    print('should be run with doit installed')
    import doit
    doit.run(globals())
<|endoftext|>


<commit_before>from cStringIO import StringIO

from redis import Redis
from PIL import Image
import requests

import config

redis = Redis(
    host=config.REDIS_HOST,
    port=config.REDIS_PORT,
    db=config.REDIS_DB
)

base_url = 'http://s3.amazonaws.com/MinecraftSkins/{0}.png'

class Avatar(object):

    box = (8, 8, 16, 16)
    helm_box = (40, 8, 48, 16)
    expiry = config.IMG_CACHE_TIMEOUT

    def __init__(self, username, size=48, helmet='h'):
        self.username = username
        self.size = (size, size)
        self.helmet = True if helmet == 'h' else False
        self.url = base_url.format(username)
        self.key = '{0}_{1}_{2}'.format(size, helmet, username)

        if username.lower() == 'char':
            # Ensure the case is correct
            # and set expiry to one week
            self.username = 'char'
            self.expiry = 60 * 60 * 24 * 7

    def skin(self):
        r = requests.get(self.url)
        if r.status_code == 403:
            # Probably not the best error to raise,
            # but at least it won't be raised by
            # something else
            raise NotImplementedError

        return StringIO(r.content)

    def helmet_exists(self, helmet, bg):
        helmet = helmet.load()
        for y in xrange(8):
            for x in xrange(8):
                if not helmet[x, y] == bg:
                    return True
        return False

    def render(self):
        skin = Image.open(self.skin())
        bg = skin.load()[0, 0]

        head = skin.copy().crop(self.box)
        helm = skin.crop(self.helm_box)

        if self.helmet and self.helmet_exists(helm, bg):
            head = head.convert('RGBA')
            helm = helm.convert('RGBA')
            head.paste(helm, None, helm)

        head = head.resize(self.size)

        out = StringIO()
        head.save(out, 'png')

        redis.setex(self.key, out.getvalue(), self.expiry)
        return out.getvalue()
<commit_msg>Prefix helmet_exists() with _ and mvoe it above skin()<commit_after>from cStringIO import StringIO

from redis import Redis
from PIL import Image
import requests

import config

redis = Redis(
    host=config.REDIS_HOST,
    port=config.REDIS_PORT,
    db=config.REDIS_DB
)

base_url = 'http://s3.amazonaws.com/MinecraftSkins/{0}.png'

class Avatar(object):

    box = (8, 8, 16, 16)
    helm_box = (40, 8, 48, 16)
    expiry = config.IMG_CACHE_TIMEOUT

    def __init__(self, username, size=48, helmet='h'):
        self.username = username
        self.size = (size, size)
        self.helmet = True if helmet == 'h' else False
        self.url = base_url.format(username)
        self.key = '{0}_{1}_{2}'.format(size, helmet, username)

        if username.lower() == 'char':
            # Ensure the case is correct
            # and set expiry to one week
            self.username = 'char'
            self.expiry = 60 * 60 * 24 * 7

    def _helmet_exists(self, helmet, bg):
        helmet = helmet.load()
        for y in xrange(8):
            for x in xrange(8):
                if not helmet[x, y] == bg:
                    return True
        return False

    def skin(self):
        r = requests.get(self.url)
        if r.status_code == 403:
            # Probably not the best error to raise,
            # but at least it won't be raised by
            # something else
            raise NotImplementedError

        return StringIO(r.content)

    def render(self):
        skin = Image.open(self.skin())
        bg = skin.load()[0, 0]

        head = skin.copy().crop(self.box)
        helm = skin.crop(self.helm_box)

        if self.helmet and self._helmet_exists(helm, bg):
            head = head.convert('RGBA')
            helm = helm.convert('RGBA')
            head.paste(helm, None, helm)

        head = head.resize(self.size)

        out = StringIO()
        head.save(out, 'png')

        redis.setex(self.key, out.getvalue(), self.expiry)
        return out.getvalue()
<|endoftext|>