<commit_before>import copy
import datetime

from django.db import models

from history import manager

class HistoricalRecords(object):
    def contribute_to_class(self, cls, name):
        self.manager_name = name
        models.signals.class_prepared.connect(self.finalize, sender=cls)

    def finalize(self, sender, **kwargs):
        history_model = self.create_history_model(sender)

        # The HistoricalRecords object will be discarded,
        # so the signal handlers can't use weak references.
        models.signals.post_save.connect(self.post_save, sender=sender,
                                         weak=False)
        models.signals.post_delete.connect(self.post_delete, sender=sender,
                                           weak=False)

        descriptor = manager.HistoryDescriptor(history_model)
        setattr(sender, self.manager_name, descriptor)

    def create_history_model(self, model):
        """
        Creates a historical model to associate with the model provided.
        """
        attrs = self.copy_fields(model)
        attrs.update(self.get_extra_fields(model))
        attrs.update(Meta=type('Meta', (), self.get_meta_options(model)))
        name = 'Historical%s' % model._meta.object_name
        return type(name, (models.Model,), attrs)

    def copy_fields(self, model):
        """
        Creates copies of the model's original fields, returning
        a dictionary mapping field name to copied field object.
        """
        # Though not strictly a field, this attribute
        # is required for a model to function properly.
        fields = {'__module__': model.__module__}

        for field in model._meta.fields:
            field = copy.copy(field)
            if isinstance(field, models.AutoField):
                # The historical model gets its own AutoField, so any
                # existing one must be replaced with an IntegerField.
                field.__class__ = models.IntegerField

            if field.primary_key or field.unique:
                # Unique fields can no longer be guaranteed unique,
                # but they should still be indexed for faster lookups.
                field.primary_key = False
                field._unique = False
                field.db_index = True
            fields[field.name] = field

        return fields

    def get_extra_fields(self, model):
        """
        Returns a dictionary of fields that will be added to the historical
        record model, in addition to the ones returned by copy_fields below.
        """
        rel_nm = '_%s_history' % model._meta.object_name.lower()
        return {
            'history_id': models.AutoField(primary_key=True),
            'history_date': models.DateTimeField(default=datetime.datetime.now),
            'history_type': models.CharField(max_length=1, choices=(
                ('+', 'Created'),
                ('~', 'Changed'),
                ('-', 'Deleted'),
            )),
            'history_object': HistoricalObjectDescriptor(model),
            '__unicode__': lambda self: u'%s as of %s' % (self.history_object,
                                                          self.history_date)
        }

    def get_meta_options(self, model):
        """
        Returns a dictionary of fields that will be added to
        the Meta inner class of the historical record model.
        """
        return {
            'ordering': ('-history_id',),
            'get_latest_by': 'history_id'
        }

    def post_save(self, instance, created, **kwargs):
        self.create_historical_record(instance, created and '+' or '~')

    def post_delete(self, instance, **kwargs):
        self.create_historical_record(instance, '-')

    def create_historical_record(self, instance, type):
        manager = getattr(instance, self.manager_name)
        attrs = {}
        for field in instance._meta.fields:
            attrs[field.attname] = getattr(instance, field.attname)
        manager.create(history_type=type, **attrs)

class HistoricalObjectDescriptor(object):
    def __init__(self, model):
        self.model = model

    def __get__(self, instance, owner):
        values = (getattr(instance, f.attname) for f in self.model._meta.fields)
        return self.model(*values)
<commit_msg>Make historical records compatible with Django 1.4 timezones, the model now uses django.utils.timezone to get the current time. If Django 1.3 or lower is used, the datetime module will be used.<commit_after>import copy

try:
    from django.utils import timezone as datetime
except:
    from datetime import datetime as datetime

from django.db import models

from history import manager

class HistoricalRecords(object):
    def contribute_to_class(self, cls, name):
        self.manager_name = name
        models.signals.class_prepared.connect(self.finalize, sender=cls)

    def finalize(self, sender, **kwargs):
        history_model = self.create_history_model(sender)

        # The HistoricalRecords object will be discarded,
        # so the signal handlers can't use weak references.
        models.signals.post_save.connect(self.post_save, sender=sender,
                                         weak=False)
        models.signals.post_delete.connect(self.post_delete, sender=sender,
                                           weak=False)

        descriptor = manager.HistoryDescriptor(history_model)
        setattr(sender, self.manager_name, descriptor)

    def create_history_model(self, model):
        """
        Creates a historical model to associate with the model provided.
        """
        attrs = self.copy_fields(model)
        attrs.update(self.get_extra_fields(model))
        attrs.update(Meta=type('Meta', (), self.get_meta_options(model)))
        name = 'Historical%s' % model._meta.object_name
        return type(name, (models.Model,), attrs)

    def copy_fields(self, model):
        """
        Creates copies of the model's original fields, returning
        a dictionary mapping field name to copied field object.
        """
        # Though not strictly a field, this attribute
        # is required for a model to function properly.
        fields = {'__module__': model.__module__}

        for field in model._meta.fields:
            field = copy.copy(field)
            if isinstance(field, models.AutoField):
                # The historical model gets its own AutoField, so any
                # existing one must be replaced with an IntegerField.
                field.__class__ = models.IntegerField

            if field.primary_key or field.unique:
                # Unique fields can no longer be guaranteed unique,
                # but they should still be indexed for faster lookups.
                field.primary_key = False
                field._unique = False
                field.db_index = True
            fields[field.name] = field

        return fields

    def get_extra_fields(self, model):
        """
        Returns a dictionary of fields that will be added to the historical
        record model, in addition to the ones returned by copy_fields below.
        """
        rel_nm = '_%s_history' % model._meta.object_name.lower()
        return {
            'history_id': models.AutoField(primary_key=True),
            'history_date': models.DateTimeField(default=datetime.now()),
            'history_type': models.CharField(max_length=1, choices=(
                ('+', 'Created'),
                ('~', 'Changed'),
                ('-', 'Deleted'),
            )),
            'history_object': HistoricalObjectDescriptor(model),
            '__unicode__': lambda self: u'%s as of %s' % (self.history_object,
                                                          self.history_date)
        }

    def get_meta_options(self, model):
        """
        Returns a dictionary of fields that will be added to
        the Meta inner class of the historical record model.
        """
        return {
            'ordering': ('-history_id',),
            'get_latest_by': 'history_id'
        }

    def post_save(self, instance, created, **kwargs):
        self.create_historical_record(instance, created and '+' or '~')

    def post_delete(self, instance, **kwargs):
        self.create_historical_record(instance, '-')

    def create_historical_record(self, instance, type):
        manager = getattr(instance, self.manager_name)
        attrs = {}
        for field in instance._meta.fields:
            attrs[field.attname] = getattr(instance, field.attname)
        manager.create(history_type=type, **attrs)

class HistoricalObjectDescriptor(object):
    def __init__(self, model):
        self.model = model

    def __get__(self, instance, owner):
        values = (getattr(instance, f.attname) for f in self.model._meta.fields)
        return self.model(*values)
<|endoftext|>


<commit_before># Copyright 2010 University Of Southern California
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
import sys
import os
import time
import cPickle as pickle
from threading import Thread
from mule import log, config

try:
	import bsddb3.db as bdb
except ImportError:
	import bsddb.db as bdb

version = tuple([int(y) for y in bdb.__version__.split(".")])
if version < (4,7):
	raise ImportError('bsddb version 4.7 or later required')
del version

def with_transaction(method):
	def with_transaction(self, *args, **kwargs):
		if len(args)>0 and isinstance(args[0],Database):
			return method(self, *args, **kwargs)
		else:
			txn = self.env.txn_begin()
			try:
				result = method(self, txn, *args, **kwargs)
				txn.commit()
				return result
			except:
				txn.abort()
				raise
	return with_transaction

class CheckpointThread(Thread):
	def __init__(self, env, interval=300):
		Thread.__init__(self)
		self.setDaemon(True)
		self.log = log.get_log("ckpt_thread")
		self.env = env
		self.interval = interval

	def run(self):
		while True:
			try:
				time.sleep(self.interval)
				self.log.info("checkpointing database")
				self.env.txn_checkpoint(0,0)
			except:
				t, e, tb = sys.exc_info()
				self.log.exception(e)

class Database(object):
	max_txns = 1000

	def __init__(self, path, name, duplicates=False):
		self.path = path
		self.dbpath = os.path.join(self.path, name)

		if not os.path.isdir(self.path):
			os.makedirs(self.path)

		self.env = bdb.DBEnv()
		self.env.set_tx_max(self.max_txns)
		self.env.set_lk_max_lockers(self.max_txns*2)
		self.env.set_lk_max_locks(self.max_txns*2)
		self.env.set_lk_max_objects(self.max_txns*2)
		self.env.set_flags(bdb.DB_TXN_NOSYNC, True)
		if bdb.version() > (4,7):
			self.env.log_set_config(bdb.DB_LOG_AUTO_REMOVE, True)
		else:
			self.env.log_set_config(bdb.DB_LOG_AUTOREMOVE, True)
		self.env.open(self.path, bdb.DB_CREATE | bdb.DB_INIT_LOCK |
				bdb.DB_INIT_LOG | bdb.DB_INIT_MPOOL | bdb.DB_INIT_TXN |
				bdb.DB_RECOVER | bdb.DB_THREAD)

		self.db = bdb.DB(self.env)
		if duplicates:
			self.db.set_flags(bdb.DB_DUPSORT)
		if bdb.version() > (4,1):
			txn = self.env.txn_begin()
			self.db.open(self.dbpath, name, flags=bdb.DB_CREATE|bdb.DB_THREAD, dbtype=bdb.DB_BTREE, txn=txn)
			txn.commit()
		else:
			self.db.open(self.dbpath, name, flags=bdb.DB_CREATE|bdb.DB_THREAD, dbtype=bdb.DB_BTREE)

	def close(self):
		self.env.log_flush()
		self.db.close()
		self.env.close()

class RLSDatabase(Database):
	def __init__(self):
		self.log = log.get_log("rls_database")
		home = config.get_home()
		path = os.path.join(home, "var", "rls")
		Database.__init__(self, path, "rls", duplicates=True)

	@with_transaction
	def add(self, txn, lfn, pfn):
		cur = self.db.cursor(txn)
		try:
			current = cur.get_both(lfn, pfn)
			if current is None:
				cur.put(lfn, pfn, flags=bdb.DB_KEYLAST)
		finally:
			cur.close()

	@with_transaction
	def delete(self, txn, lfn, pfn=None):
		cur = self.db.cursor(txn)
		try:
			if pfn is None:
				current = cur.set(lfn)
				while current is not None:
					cur.delete()
					current = cur.next_dup()
			else:
				current = cur.set_both(lfn, pfn)
				if current is not None:
					cur.delete()
		finally:
			cur.close()

	def lookup(self, lfn):
		cur = self.db.cursor()
		try:
			result = []
			current = cur.set(lfn)
			while current is not None:
				result.append(current[1])
				current = cur.next_dup()
			return result
		finally:
			cur.close()

class CacheDatabase(Database):
	def __init__(self):
		self.log = log.get_log("cache_database")
		home = config.get_home()
		path = os.path.join(home, "var", "cache")
		Database.__init__(self, path, "cache", duplicates=False)

	def get(self, lfn):
		cur = self.db.cursor()
		try:
			current = cur.set(lfn)
			if current is not None:
				return pickle.loads(current[1])
			else:
				return None
		finally:
			cur.close()

	@with_transaction
	def put(self, txn, lfn):
		next = { 'status': 'unready', 'uuid': None }
		self.db.put(lfn, pickle.dumps(next), txn, bdb.DB_NOOVERWRITE)

	@with_transaction
	def remove(self, txn, lfn):
		self.db.delete(lfn, txn)

	def list(self):
		cur = self.db.cursor()
		try:
			result = []
			current = cur.first()
			while current is not None:
				rec = pickle.loads(current[1])
				rec['lfn'] = current[0]
				result.append(rec)
				current = cur.next()
			return result
		finally:
			cur.close()

	@with_transaction
	def update(self, txn, lfn, status, uuid):
		next = { 'status': status, 'uuid': uuid }
		self.db.put(lfn, pickle.dumps(next), txn)

if __name__ == '__main__':
	log.configure()
	db = CacheDatabase()
	try:
		if db.put("b"):
			rec = db.get("b")
			if rec['status'] == "unready":
				db.update("b", "123123123123")
		print db.get("b")
	finally:
		db.close()
<commit_msg>No autoremove on logs for old bdb<commit_after># Copyright 2010 University Of Southern California
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
import sys
import os
import time
import cPickle as pickle
from threading import Thread
from mule import log, config

try:
	import bsddb3.db as bdb
except ImportError:
	import bsddb.db as bdb

version = tuple([int(y) for y in bdb.__version__.split(".")])
if version < (4,7):
	raise ImportError('bsddb version 4.7 or later required')
del version

def with_transaction(method):
	def with_transaction(self, *args, **kwargs):
		if len(args)>0 and isinstance(args[0],Database):
			return method(self, *args, **kwargs)
		else:
			txn = self.env.txn_begin()
			try:
				result = method(self, txn, *args, **kwargs)
				txn.commit()
				return result
			except:
				txn.abort()
				raise
	return with_transaction

class CheckpointThread(Thread):
	def __init__(self, env, interval=300):
		Thread.__init__(self)
		self.setDaemon(True)
		self.log = log.get_log("ckpt_thread")
		self.env = env
		self.interval = interval

	def run(self):
		while True:
			try:
				time.sleep(self.interval)
				self.log.info("checkpointing database")
				self.env.txn_checkpoint(0,0)
			except:
				t, e, tb = sys.exc_info()
				self.log.exception(e)

class Database(object):
	max_txns = 1000

	def __init__(self, path, name, duplicates=False):
		self.path = path
		self.dbpath = os.path.join(self.path, name)

		if not os.path.isdir(self.path):
			os.makedirs(self.path)

		self.env = bdb.DBEnv()
		self.env.set_tx_max(self.max_txns)
		self.env.set_lk_max_lockers(self.max_txns*2)
		self.env.set_lk_max_locks(self.max_txns*2)
		self.env.set_lk_max_objects(self.max_txns*2)
		self.env.set_flags(bdb.DB_TXN_NOSYNC, True)
		if bdb.version() > (4,7):
			self.env.log_set_config(bdb.DB_LOG_AUTO_REMOVE, True)
		self.env.open(self.path, bdb.DB_CREATE | bdb.DB_INIT_LOCK |
				bdb.DB_INIT_LOG | bdb.DB_INIT_MPOOL | bdb.DB_INIT_TXN |
				bdb.DB_RECOVER | bdb.DB_THREAD)

		self.db = bdb.DB(self.env)
		if duplicates:
			self.db.set_flags(bdb.DB_DUPSORT)
		if bdb.version() > (4,1):
			txn = self.env.txn_begin()
			self.db.open(self.dbpath, name, flags=bdb.DB_CREATE|bdb.DB_THREAD, dbtype=bdb.DB_BTREE, txn=txn)
			txn.commit()
		else:
			self.db.open(self.dbpath, name, flags=bdb.DB_CREATE|bdb.DB_THREAD, dbtype=bdb.DB_BTREE)

	def close(self):
		self.env.log_flush()
		self.db.close()
		self.env.close()

class RLSDatabase(Database):
	def __init__(self):
		self.log = log.get_log("rls_database")
		home = config.get_home()
		path = os.path.join(home, "var", "rls")
		Database.__init__(self, path, "rls", duplicates=True)

	@with_transaction
	def add(self, txn, lfn, pfn):
		cur = self.db.cursor(txn)
		try:
			current = cur.get_both(lfn, pfn)
			if current is None:
				cur.put(lfn, pfn, flags=bdb.DB_KEYLAST)
		finally:
			cur.close()

	@with_transaction
	def delete(self, txn, lfn, pfn=None):
		cur = self.db.cursor(txn)
		try:
			if pfn is None:
				current = cur.set(lfn)
				while current is not None:
					cur.delete()
					current = cur.next_dup()
			else:
				current = cur.set_both(lfn, pfn)
				if current is not None:
					cur.delete()
		finally:
			cur.close()

	def lookup(self, lfn):
		cur = self.db.cursor()
		try:
			result = []
			current = cur.set(lfn)
			while current is not None:
				result.append(current[1])
				current = cur.next_dup()
			return result
		finally:
			cur.close()

class CacheDatabase(Database):
	def __init__(self):
		self.log = log.get_log("cache_database")
		home = config.get_home()
		path = os.path.join(home, "var", "cache")
		Database.__init__(self, path, "cache", duplicates=False)

	def get(self, lfn):
		cur = self.db.cursor()
		try:
			current = cur.set(lfn)
			if current is not None:
				return pickle.loads(current[1])
			else:
				return None
		finally:
			cur.close()

	@with_transaction
	def put(self, txn, lfn):
		next = { 'status': 'unready', 'uuid': None }
		self.db.put(lfn, pickle.dumps(next), txn, bdb.DB_NOOVERWRITE)

	@with_transaction
	def remove(self, txn, lfn):
		self.db.delete(lfn, txn)

	def list(self):
		cur = self.db.cursor()
		try:
			result = []
			current = cur.first()
			while current is not None:
				rec = pickle.loads(current[1])
				rec['lfn'] = current[0]
				result.append(rec)
				current = cur.next()
			return result
		finally:
			cur.close()

	@with_transaction
	def update(self, txn, lfn, status, uuid):
		next = { 'status': status, 'uuid': uuid }
		self.db.put(lfn, pickle.dumps(next), txn)

if __name__ == '__main__':
	log.configure()
	db = CacheDatabase()
	try:
		if db.put("b"):
			rec = db.get("b")
			if rec['status'] == "unready":
				db.update("b", "123123123123")
		print db.get("b")
	finally:
		db.close()
<|endoftext|>


<commit_before>from popen2 import Popen3

class File:
    def __init__(self):
        self.origFile = None
        self.newFile = None
        self.origInfo = None
        self.newInfo = None
        self.data = None

def parseFile(lines, linenum, lastline, filename):
    file = File()

    if (lines[linenum].startswith('--- ') and \
        lines[linenum + 1].startswith('+++ ')) or \
       (lines[linenum].startswith('*** ') and \
        lines[linenum + 1].startswith('--- ')):

        # Unified or Context diff
        file.origFile, file.origInfo = lines[linenum].split(None, 2)[1:]
        file.newFile,  file.newInfo  = lines[linenum + 1].split(None, 2)[1:]
    else:
        raise Exception('Unable to recognize diff format')

    if lines[lastline].startswith("diff "):
        lastline -= 1
    elif lines[lastline].startswith("====================") and \
         lines[lastline - 1].startswith("Index: "):
        lastline -= 2

    file.data = '\n'.join(lines[linenum:lastline])

    return file

def parse(data):
    p = Popen3('lsdiff -n')
    p.tochild.write(data)
    p.tochild.close()
    r = p.fromchild.read().strip()
    ret = p.wait()

    if ret:
        raise Exception('Error running lsdiff')

    lines = data.splitlines()
    files = []

    current_slice = r.splitlines()
    next_slice = current_slice[1:] + ['%d' % len(lines)]
    for current, next in zip(current_slice, next_slice):
        begin, file = current.split()
        end = next.split()[0]

        files.append(parseFile(lines, int(begin) - 1, int(end) - 1, file))

    return files
<commit_msg>I like this variable name better<commit_after>from popen2 import Popen3

class File:
    def __init__(self):
        self.origFile = None
        self.newFile = None
        self.origInfo = None
        self.newInfo = None
        self.data = None

def parseFile(lines, linenum, lastline, filename):
    file = File()

    if (lines[linenum].startswith('--- ') and \
        lines[linenum + 1].startswith('+++ ')) or \
       (lines[linenum].startswith('*** ') and \
        lines[linenum + 1].startswith('--- ')):

        # Unified or Context diff
        file.origFile, file.origInfo = lines[linenum].split(None, 2)[1:]
        file.newFile,  file.newInfo  = lines[linenum + 1].split(None, 2)[1:]
    else:
        raise Exception('Unable to recognize diff format')

    if lines[lastline].startswith("diff "):
        lastline -= 1
    elif lines[lastline].startswith("====================") and \
         lines[lastline - 1].startswith("Index: "):
        lastline -= 2

    file.data = '\n'.join(lines[linenum:lastline])

    return file

def parse(data):
    p = Popen3('lsdiff -n')
    p.tochild.write(data)
    p.tochild.close()
    r = p.fromchild.read().strip()
    failure = p.wait()

    if failure:
        raise Exception('Error running lsdiff')

    lines = data.splitlines()
    files = []

    current_slice = r.splitlines()
    next_slice = current_slice[1:] + ['%d' % len(lines)]
    for current, next in zip(current_slice, next_slice):
        begin, file = current.split()
        end = next.split()[0]

        files.append(parseFile(lines, int(begin) - 1, int(end) - 1, file))

    return files
<|endoftext|>


<commit_before>import os
import urllib2
from os.path import join as pjoin
import nibabel as nib
from dipy.core.gradients import gradient_table
from dipy.io.gradients import read_bvals_bvecs

def fetch_beijing_dti():
    """ Download a DTI dataset with 64 gradient directions
    """
    dipy_home = pjoin(os.path.expanduser('~'), '.dipy')
    url = 'https://www.dropbox.com/sh/ybl8vquht8pdc3s/'
    uraw = url + 'NfhK5WsYRO/DTI64.nii.gz?dl=1'
    ubval = url + 'HyX6GEQSSV/DTI64.bval?dl=1'
    ubvec = url + 'Xtrvr5jSOs/DTI64.bvec?dl=1'
    ureadme = url + 'HoGsocT6hZ/beijingEnhanced.txt?dl=1'
    folder = pjoin(dipy_home, 'beijing_dti')

    if not os.path.exists(folder):
        print('Creating new directory %s' % folder)
        os.makedirs(folder)
        print('Downloading raw DTI data ...')
        opener = urllib2.urlopen(uraw)
        open(pjoin(folder, 'DTI64.nii.gz'), 'wb').write(opener.read())

        opener = urllib2.urlopen(ubval)
        open(pjoin(folder, 'DTI64.bval'), 'w').write(opener.read())

        opener = urllib2.urlopen(ubvec)
        open(pjoin(folder, 'DTI64.bvec'), 'w').write(opener.read())

        opener = urllib2.urlopen(ureadme)
        open(pjoin(folder, 'beijingEnhanced.txt'), 'w').write(opener.read())

        print('Done.')
        print('Files copied in folder %s' % folder)
        print('See BeijingEnhanced.txt for DATA AGREEMENT LICENSE.')
        print('This is only a single brain DTI dataset which we use for some of the dipy tutorials.')
        print('For the complete datasets please visit :')
        print('http://fcon_1000.projects.nitrc.org/indi/retro/BeijingEnhanced.html')

    else:
        raise IOError('Dataset already in place. If you want to fetch again please first remove folder %s ' % folder)

def read_beijing_dti():
    """ Load Beijing dataset

    Returns
    -------
    img : obj,
        Nifti1Image
    gtab : obj,
        GradientTable
    """
    dipy_home = pjoin(os.path.expanduser('~'), '.dipy')
    folder = pjoin(dipy_home, 'beijing_dti')
    fraw = pjoin(folder, 'DTI64.nii.gz')
    fbval = pjoin(folder, 'DTI64.bval')
    fbvec = pjoin(folder, 'DTI64.bvec')
    bvals, bvecs = read_bvals_bvecs(fbval, fbvec)
    gtab = gradient_table(bvals, bvecs)
    img = nib.load(fraw)
    return img, gtab

def fetch_taiwan_ntu_dsi():
    """ Download a DSI dataset with 203 gradient directions
    """
    dipy_home = pjoin(os.path.expanduser('~'), '.dipy')
    uraw = 'http://dl.dropbox.com/u/2481924/taiwan_ntu_dsi.nii.gz'
    ubval = 'http://dl.dropbox.com/u/2481924/tawian_ntu_dsi.bval'
    ubvec = 'http://dl.dropbox.com/u/2481924/taiwan_ntu_dsi.bvec'
    ureadme = 'http://dl.dropbox.com/u/2481924/license_taiwan_ntu_dsi.txt'
    folder = pjoin(dipy_home, 'taiwan_ntu_dsi')
    if not os.path.exists(folder):
        print('Creating new directory %s' % folder)
        os.makedirs(folder)

        print('Downloading raw DSI data (91MB)...')

        opener = urllib2.urlopen(uraw)
        open(pjoin(folder, 'DSI203.nii.gz'), 'wb').write(opener.read())

        opener = urllib2.urlopen(ubval)
        open(pjoin(folder, 'DSI203.bval'), 'w').write(opener.read())

        opener = urllib2.urlopen(ubvec)
        open(pjoin(folder, 'DSI203.bvec'), 'w').write(opener.read())

        opener = urllib2.urlopen(ubvec)
        open(pjoin(folder, 'DSI203_license.txt'), 'w').write(opener.read())

        print('Done.')
        print('Files copied in folder %s' % folder)
        print('See DSI203_license.txt for LICENSE.')

    else:
        raise IOError('Dataset already in place. If you want to fetch again please first remove folder %s ' % folder)

def read_taiwan_ntu_dsi():
    """ Load Tawian NTU dataset

    Returns
    -------
    img : obj,
        Nifti1Image
    gtab : obj,
        GradientTable
    """
    dipy_home = pjoin(os.path.expanduser('~'), '.dipy')
    folder = pjoin(dipy_home, 'taiwan_ntu_dsi')
    fraw = pjoin(folder, 'DSI203.nii.gz')
    fbval = pjoin(folder, 'DSI203.bval')
    fbvec = pjoin(folder, 'DSI203.bvec')
    bvals, bvecs = read_bvals_bvecs(fbval, fbvec)
    gtab = gradient_table(bvals, bvecs)
    img = nib.load(fraw)
    return img, gtab

<commit_msg>DOC added size of dataset in beijing<commit_after>import os
import urllib2
from os.path import join as pjoin
import nibabel as nib
from dipy.core.gradients import gradient_table
from dipy.io.gradients import read_bvals_bvecs

def fetch_beijing_dti():
    """ Download a DTI dataset with 64 gradient directions
    """
    dipy_home = pjoin(os.path.expanduser('~'), '.dipy')
    url = 'https://www.dropbox.com/sh/ybl8vquht8pdc3s/'
    uraw = url + 'NfhK5WsYRO/DTI64.nii.gz?dl=1'
    ubval = url + 'HyX6GEQSSV/DTI64.bval?dl=1'
    ubvec = url + 'Xtrvr5jSOs/DTI64.bvec?dl=1'
    ureadme = url + 'HoGsocT6hZ/beijingEnhanced.txt?dl=1'
    folder = pjoin(dipy_home, 'beijing_dti')

    if not os.path.exists(folder):
        print('Creating new directory %s' % folder)
        os.makedirs(folder)
        print('Downloading raw DTI data (51MB)...')
        opener = urllib2.urlopen(uraw)
        open(pjoin(folder, 'DTI64.nii.gz'), 'wb').write(opener.read())

        opener = urllib2.urlopen(ubval)
        open(pjoin(folder, 'DTI64.bval'), 'w').write(opener.read())

        opener = urllib2.urlopen(ubvec)
        open(pjoin(folder, 'DTI64.bvec'), 'w').write(opener.read())

        opener = urllib2.urlopen(ureadme)
        open(pjoin(folder, 'beijingEnhanced.txt'), 'w').write(opener.read())

        print('Done.')
        print('Files copied in folder %s' % folder)
        print('See BeijingEnhanced.txt for DATA AGREEMENT LICENSE.')
        print('This is only a single brain DTI dataset which we use for some of the dipy tutorials.')
        print('For the complete datasets please visit :')
        print('http://fcon_1000.projects.nitrc.org/indi/retro/BeijingEnhanced.html')

    else:
        raise IOError('Dataset already in place. If you want to fetch again please first remove folder %s ' % folder)

def read_beijing_dti():
    """ Load Beijing dataset

    Returns
    -------
    img : obj,
        Nifti1Image
    gtab : obj,
        GradientTable
    """
    dipy_home = pjoin(os.path.expanduser('~'), '.dipy')
    folder = pjoin(dipy_home, 'beijing_dti')
    fraw = pjoin(folder, 'DTI64.nii.gz')
    fbval = pjoin(folder, 'DTI64.bval')
    fbvec = pjoin(folder, 'DTI64.bvec')
    bvals, bvecs = read_bvals_bvecs(fbval, fbvec)
    gtab = gradient_table(bvals, bvecs)
    img = nib.load(fraw)
    return img, gtab

def fetch_taiwan_ntu_dsi():
    """ Download a DSI dataset with 203 gradient directions
    """
    dipy_home = pjoin(os.path.expanduser('~'), '.dipy')
    uraw = 'http://dl.dropbox.com/u/2481924/taiwan_ntu_dsi.nii.gz'
    ubval = 'http://dl.dropbox.com/u/2481924/tawian_ntu_dsi.bval'
    ubvec = 'http://dl.dropbox.com/u/2481924/taiwan_ntu_dsi.bvec'
    ureadme = 'http://dl.dropbox.com/u/2481924/license_taiwan_ntu_dsi.txt'
    folder = pjoin(dipy_home, 'taiwan_ntu_dsi')
    if not os.path.exists(folder):
        print('Creating new directory %s' % folder)
        os.makedirs(folder)

        print('Downloading raw DSI data (91MB)...')

        opener = urllib2.urlopen(uraw)
        open(pjoin(folder, 'DSI203.nii.gz'), 'wb').write(opener.read())

        opener = urllib2.urlopen(ubval)
        open(pjoin(folder, 'DSI203.bval'), 'w').write(opener.read())

        opener = urllib2.urlopen(ubvec)
        open(pjoin(folder, 'DSI203.bvec'), 'w').write(opener.read())

        opener = urllib2.urlopen(ubvec)
        open(pjoin(folder, 'DSI203_license.txt'), 'w').write(opener.read())

        print('Done.')
        print('Files copied in folder %s' % folder)
        print('See DSI203_license.txt for LICENSE.')

    else:
        raise IOError('Dataset already in place. If you want to fetch again please first remove folder %s ' % folder)

def read_taiwan_ntu_dsi():
    """ Load Tawian NTU dataset

    Returns
    -------
    img : obj,
        Nifti1Image
    gtab : obj,
        GradientTable
    """
    dipy_home = pjoin(os.path.expanduser('~'), '.dipy')
    folder = pjoin(dipy_home, 'taiwan_ntu_dsi')
    fraw = pjoin(folder, 'DSI203.nii.gz')
    fbval = pjoin(folder, 'DSI203.bval')
    fbvec = pjoin(folder, 'DSI203.bvec')
    bvals, bvecs = read_bvals_bvecs(fbval, fbvec)
    gtab = gradient_table(bvals, bvecs)
    img = nib.load(fraw)
    return img, gtab

<|endoftext|>


<commit_before>from __future__ import print_function, division, absolute_import

from collections import Iterable
import socket
import sys

from tornado import gen

def funcname(func):
    """Get the name of a function."""
    while hasattr(func, 'func'):
        func = func.func
    try:
        return func.__name__
    except:
        return str(func)

def get_ip():
    return [(s.connect(('8.8.8.8', 80)), s.getsockname()[0], s.close())
        for s in [socket.socket(socket.AF_INET, socket.SOCK_DGRAM)]][0][1]

from contextlib import contextmanager

@contextmanager
def ignoring(*exceptions):
    try:
        yield
    except exceptions:
        pass

@gen.coroutine
def ignore_exceptions(coroutines, *exceptions):
    """ Process list of coroutines, ignoring certain exceptions

    >>> coroutines = [cor(...) for ...]  # doctest: +SKIP
    >>> x = yield ignore_exceptions(coroutines, TypeError)  # doctest: +SKIP
    """
    wait_iterator = gen.WaitIterator(*coroutines)
    results = []
    while not wait_iterator.done():
        with ignoring(*exceptions):
            result = yield wait_iterator.next()
            results.append(result)
    raise gen.Return(results)

@gen.coroutine
def All(*args):
    """ Wait on many tasks at the same time

    Err once any of the tasks err.

    See https://github.com/tornadoweb/tornado/issues/1546
    """
    if len(args) == 1 and isinstance(args[0], Iterable):
        args = args[0]
    tasks = gen.WaitIterator(*args)
    results = [None for _ in args]
    while not tasks.done():
        result = yield tasks.next()
        results[tasks.current_index] = result
    raise gen.Return(results)

def sync(loop, func, *args, **kwargs):
    """ Run coroutine in loop running in separate thread """
    from threading import Event
    e = Event()
    result = [None]

    @gen.coroutine
    def f():
        result[0] = yield func(*args, **kwargs)
        e.set()

    a = loop.add_callback(f)
    e.wait()
    return result[0]

import logging
logging.basicConfig(format='%(name)s - %(levelname)s - %(message)s',
                    level=logging.DEBUG)

# http://stackoverflow.com/questions/21234772/python-tornado-disable-logging-to-stderr
stream = logging.StreamHandler(sys.stderr)
stream.setLevel(logging.CRITICAL)
logging.getLogger('tornado').addHandler(stream)
logging.getLogger('tornado').propagate = False
<commit_msg>add tmp_text function to utils<commit_after>from __future__ import print_function, division, absolute_import

from collections import Iterable
import os
import socket
import sys
import tempfile

from tornado import gen

def funcname(func):
    """Get the name of a function."""
    while hasattr(func, 'func'):
        func = func.func
    try:
        return func.__name__
    except:
        return str(func)

def get_ip():
    return [(s.connect(('8.8.8.8', 80)), s.getsockname()[0], s.close())
        for s in [socket.socket(socket.AF_INET, socket.SOCK_DGRAM)]][0][1]

from contextlib import contextmanager

@contextmanager
def ignoring(*exceptions):
    try:
        yield
    except exceptions:
        pass

@gen.coroutine
def ignore_exceptions(coroutines, *exceptions):
    """ Process list of coroutines, ignoring certain exceptions

    >>> coroutines = [cor(...) for ...]  # doctest: +SKIP
    >>> x = yield ignore_exceptions(coroutines, TypeError)  # doctest: +SKIP
    """
    wait_iterator = gen.WaitIterator(*coroutines)
    results = []
    while not wait_iterator.done():
        with ignoring(*exceptions):
            result = yield wait_iterator.next()
            results.append(result)
    raise gen.Return(results)

@gen.coroutine
def All(*args):
    """ Wait on many tasks at the same time

    Err once any of the tasks err.

    See https://github.com/tornadoweb/tornado/issues/1546
    """
    if len(args) == 1 and isinstance(args[0], Iterable):
        args = args[0]
    tasks = gen.WaitIterator(*args)
    results = [None for _ in args]
    while not tasks.done():
        result = yield tasks.next()
        results[tasks.current_index] = result
    raise gen.Return(results)

def sync(loop, func, *args, **kwargs):
    """ Run coroutine in loop running in separate thread """
    from threading import Event
    e = Event()
    result = [None]

    @gen.coroutine
    def f():
        result[0] = yield func(*args, **kwargs)
        e.set()

    a = loop.add_callback(f)
    e.wait()
    return result[0]

@contextmanager
def tmp_text(filename, text):
    fn = os.path.join(tempfile.gettempdir(), filename)
    with open(fn, 'w') as f:
        f.write(text)

    try:
        yield fn
    finally:
        if os.path.exists(fn):
            os.remove(fn)

import logging
logging.basicConfig(format='%(name)s - %(levelname)s - %(message)s',
                    level=logging.DEBUG)

# http://stackoverflow.com/questions/21234772/python-tornado-disable-logging-to-stderr
stream = logging.StreamHandler(sys.stderr)
stream.setLevel(logging.CRITICAL)
logging.getLogger('tornado').addHandler(stream)
logging.getLogger('tornado').propagate = False
<|endoftext|>


<commit_before>"""
  Tone class used for make a nice Tone
"""

import math

import numpy as np

from .Play import *

ms = math.pow(10, -3)

class Tone:
    """
      Make tones at frequence tf (Hz) with a sampling rate at fs (Hz).
      Uses a cosine to help remove clicking defined for attack (ms)
      Gain is for full scale of 16-bit signed integer
    """

    def __init__(self, _sample_freq=8000.0, _tone_freq=600.0,
                 _attack=10.0, _gain=10000.0):
        self.fs = _sample_freq
        self.tf = _tone_freq
        self.attack = _attack
        self.gain = _gain
        self.raisedCosine()
        self.p = Play(_sample_freq)

    def raisedCosine(self):
        self.rc = (1 + np.cos(np.arange(-math.pi,
                                        0,
                                        math.pi / (self.attack * ms
                                                   * self.fs)))) * .5

    def createTone(self, Nms):
        """create a tone of Nms (ms) length"""
        if Nms < 2 * self.attack:
            raise ValueError
        self.tone = np.sin(2.0 * math.pi
                           * np.arange(0,
                                       Nms * ms, 1 / self.fs) * self.tf)
        w = np.ones(len(self.tone))
        w[:self.rc.shape[0]] = self.rc
        w = np.flip(w, 0)
        w[:self.rc.shape[0]] = self.rc
        self.tone = np.multiply(self.tone, w)
        return self.tone

    def play(self):
        """play the tone, needs to be created with createTone"""
        p.play(self.tone, self.gain)
<commit_msg>changed default attack (window region) to 15ms from 10ms)<commit_after>"""
  Tone class used for make a nice Tone
"""

import math

import numpy as np

from .Play import *

ms = math.pow(10, -3)

class Tone:
    """
      Make tones at frequence tf (Hz) with a sampling rate at fs (Hz).
      Uses a cosine to help remove clicking defined for attack (ms)
      Gain is for full scale of 16-bit signed integer
    """

    def __init__(self, _sample_freq=8000.0, _tone_freq=600.0,
                 _attack=15.0, _gain=10000.0):
        self.fs = _sample_freq
        self.tf = _tone_freq
        self.attack = _attack
        self.gain = _gain
        self.raisedCosine()
        self.p = Play(_sample_freq)

    def raisedCosine(self):
        self.rc = (1 + np.cos(np.arange(-math.pi,
                                        0,
                                        math.pi / (self.attack * ms
                                                   * self.fs)))) * .5

    def createTone(self, Nms):
        """create a tone of Nms (ms) length"""
        if Nms < 2 * self.attack:
            raise ValueError
        self.tone = np.sin(2.0 * math.pi
                           * np.arange(0,
                                       Nms * ms, 1 / self.fs) * self.tf)
        w = np.ones(len(self.tone))
        w[:self.rc.shape[0]] = self.rc
        w = np.flip(w, 0)
        w[:self.rc.shape[0]] = self.rc
        self.tone = np.multiply(self.tone, w)
        return self.tone

    def play(self):
        """play the tone, needs to be created with createTone"""
        p.play(self.tone, self.gain)
<|endoftext|>


<commit_before>"""
Caching framework.

This module defines set of cache backends that all conform to a simple API.
In a nutshell, a cache is a set of values -- which can be any object that
may be pickled -- identified by string keys.  For the complete API, see
the abstract Cache object, below.

Client code should not access a cache backend directly; instead
it should use the get_cache() function.  This function will look at
settings.CACHE_BACKEND and use that to create and load a cache object.

The CACHE_BACKEND setting is a quasi-URI; examples are:

    memcached://127.0.0.1:11211/    A memcached backend; the server is running
                                    on localhost port 11211.  You can use
                                    multiple memcached servers by separating
                                    them with semicolons.

    db://tablename/                 A database backend in a table named
                                    "tablename". This table should be created
                                    with "django-admin createcachetable".

    file:///var/tmp/django_cache/   A file-based cache stored in the directory
                                    /var/tmp/django_cache/.

    simple:///                      A simple single-process memory cache; you
                                    probably don't want to use this except for
                                    testing. Note that this cache backend is
                                    NOT threadsafe!

    locmem:///                      A more sophisticaed local memory cache;
                                    this is multi-process- and thread-safe.

All caches may take arguments; these are given in query-string style.  Valid
arguments are:

    timeout
        Default timeout, in seconds, to use for the cache.  Defaults
        to 5 minutes (300 seconds).

    max_entries
        For the simple, file, and database backends, the maximum number of
        entries allowed in the cache before it is cleaned.  Defaults to
        300.

    cull_percentage
        The percentage of entries that are culled when max_entries is reached.
        The actual percentage is 1/cull_percentage, so set cull_percentage=3 to
        cull 1/3 of the entries when max_entries is reached.

        A value of 0 for cull_percentage means that the entire cache will be
        dumped when max_entries is reached.  This makes culling *much* faster
        at the expense of more cache misses.

For example:

    memcached://127.0.0.1:11211/?timeout=60
    db://tablename/?timeout=120&max_entries=500&cull_percentage=4

Invalid arguments are silently ignored, as are invalid values of known
arguments.
"""

##############
# Exceptions #
##############

class InvalidCacheBackendError(Exception):
    pass

################################
# Abstract base implementation #
################################

class _Cache:
    def __init__(self, params):
        timeout = params.get('timeout', 300)
        try:
            timeout = int(timeout)
        except (ValueError, TypeError):
            timeout = 300
        self.default_timeout = timeout

    def get(self, key, default=None):
        '''
        Fetch a given key from the cache.  If the key does not exist, return
        default, which itself defaults to None.
        '''
        raise NotImplementedError

    def set(self, key, value, timeout=None):
        '''
        Set a value in the cache.  If timeout is given, that timeout will be
        used for the key; otherwise the default cache timeout will be used.
        '''
        raise NotImplementedError

    def delete(self, key):
        '''
        Delete a key from the cache, failing silently.
        '''
        raise NotImplementedError

    def get_many(self, keys):
        '''
        Fetch a bunch of keys from the cache.  For certain backends (memcached,
        pgsql) this can be *much* faster when fetching multiple values.

        Returns a dict mapping each key in keys to its value.  If the given
        key is missing, it will be missing from the response dict.
        '''
        d = {}
        for k in keys:
            val = self.get(k)
            if val is not None:
                d[k] = val
        return d

    def has_key(self, key):
        '''
        Returns True if the key is in the cache and has not expired.
        '''
        return self.get(key) is not None

###########################
# memcached cache backend #
###########################

try:
    import memcache
except ImportError:
    _MemcachedCache = None
else:
    class _MemcachedCache(_Cache):
        "Memcached cache backend."
        def __init__(self, server, params):
            _Cache.__init__(self, params)
            self._cache = memcache.Client(server.split(';'))

        def get(self, key, default=None):
            val = self._cache.get(key)
            if val is None:
                return default
            else:
                return val

        def set(self, key, value, timeout=0):
            self._cache.set(key, value, timeout)

        def delete(self, key):
            self._cache.delete(key)

        def get_many(self, keys):
            return self._cache.get_multi(keys)

##################################
# Single-process in-memory cache #
##################################

import time

class _SimpleCache(_Cache):
    "Simple single-process in-memory cache."
    def __init__(self, host, params):
        _Cache.__init__(self, params)
        self._cache = {}
        self._expire_info = {}

        max_entries = params.get('max_entries', 300)
        try:
            self._max_entries = int(max_entries)
        except (ValueError, TypeError):
            self._max_entries = 300

        cull_frequency = params.get('cull_frequency', 3)
        try:
            self._cull_frequency = int(cull_frequency)
        except (ValueError, TypeError):
            self._cull_frequency = 3

    def get(self, key, default=None):
        now = time.time()
        exp = self._expire_info.get(key)
        if exp is None:
            return default
        elif exp < now:
            del self._cache[key]
            del self._expire_info[key]
            return default
        else:
            return self._cache[key]

    def set(self, key, value, timeout=None):
        if len(self._cache) >= self._max_entries:
            self._cull()
        if timeout is None:
            timeout = self.default_timeout
        self._cache[key] = value
        self._expire_info[key] = time.time() + timeout

    def delete(self, key):
        try:
            del self._cache[key]
        except KeyError:
            pass
        try:
            del self._expire_info[key]
        except KeyError:
            pass

    def has_key(self, key):
        return self._cache.has_key(key)

    def _cull(self):
        if self._cull_frequency == 0:
            self._cache.clear()
            self._expire_info.clear()
        else:
            doomed = [k for (i, k) in enumerate(self._cache) if i % self._cull_frequency == 0]
            for k in doomed:
                self.delete(k)

###############################
# Thread-safe in-memory cache #
###############################

try:
    import cPickle as pickle
except ImportError:
    import pickle
import copy
from django.utils.synch import RWLock

class _LocMemCache(_SimpleCache):
    "Thread-safe in-memory cache."
    def __init__(self, host, params):
        _SimpleCache.__init__(self, host, params)
        self._lock = RWLock()

    def get(self, key, default=None):
        should_delete = False
        self._lock.reader_enters()
        try:
            now = time.time()
            exp = self._expire_info.get(key)
            if exp is None:
                return default
            elif exp < now:
                should_delete = True
            else:
                return copy.deepcopy(self._cache[key])
        finally:
            self._lock.reader_leaves()
        if should_delete:
            self._lock.writer_enters()
            try:
                del self._cache[key]
                del self._expire_info[key]
                return default
            finally:
                self._lock.writer_leaves()

    def set(self, key, value, timeout=None):
        self._lock.writer_enters()
        try:
            _SimpleCache.set(self, key, value, timeout)
        finally:
            self._lock.writer_leaves()

    def delete(self, key):
        self._lock.writer_enters()
        try:
            _SimpleCache.delete(self, key)
        finally:
            self._lock.writer_leaves()

####################
# File-based cache #
####################

import os
import urllib

class _FileCache(_SimpleCache):
    "File-based cache."
    def __init__(self, dir, params):
        self._dir = dir
        if not os.path.exists(self._dir):
            self._createdir()
        _SimpleCache.__init__(self, dir, params)
        del self._cache
        del self._expire_info

    def get(self, key, default=None):
        fname = self._key_to_file(key)
        try:
            f = open(fname, 'rb')
            exp = pickle.load(f)
            now = time.time()
            if exp < now:
                f.close()
                os.remove(fname)
            else:
                return pickle.load(f)
        except (IOError, pickle.PickleError):
            pass
        return default

    def set(self, key, value, timeout=None):
        fname = self._key_to_file(key)
        if timeout is None:
            timeout = self.default_timeout
        try:
            filelist = os.listdir(self._dir)
        except (IOError, OSError):
            self._createdir()
            filelist = []
        if len(filelist) > self._max_entries:
            self._cull(filelist)
        try:
            f = open(fname, 'wb')
            now = time.time()
            pickle.dump(now + timeout, f, 2)
            pickle.dump(value, f, 2)
        except (IOError, OSError):
            raise

    def delete(self, key):
        try:
            os.remove(self._key_to_file(key))
        except (IOError, OSError):
            pass

    def has_key(self, key):
        return os.path.exists(self._key_to_file(key))

    def _cull(self, filelist):
        if self.cull_frequency == 0:
            doomed = filelist
        else:
            doomed = [k for (i, k) in enumerate(filelist) if i % self._cull_frequency == 0]
        for fname in doomed:
            try:
                os.remove(os.path.join(self._dir, fname))
            except (IOError, OSError):
                pass

    def _createdir(self):
        try:
            os.makedirs(self._dir)
        except OSError:
            raise EnvironmentError, "Cache directory '%s' does not exist and could not be created'" % self._dir

    def _key_to_file(self, key):
        return os.path.join(self._dir, urllib.quote_plus(key))

#############
# SQL cache #
#############

import base64
from django.core.db import db, DatabaseError
from datetime import datetime

class _DBCache(_Cache):
    "SQL cache backend."
    def __init__(self, table, params):
        _Cache.__init__(self, params)
        self._table = table
        max_entries = params.get('max_entries', 300)
        try:
            self._max_entries = int(max_entries)
        except (ValueError, TypeError):
            self._max_entries = 300
        cull_frequency = params.get('cull_frequency', 3)
        try:
            self._cull_frequency = int(cull_frequency)
        except (ValueError, TypeError):
            self._cull_frequency = 3

    def get(self, key, default=None):
        cursor = db.cursor()
        cursor.execute("SELECT cache_key, value, expires FROM %s WHERE cache_key = %%s" % self._table, [key])
        row = cursor.fetchone()
        if row is None:
            return default
        now = datetime.now()
        if row[2] < now:
            cursor.execute("DELETE FROM %s WHERE cache_key = %%s" % self._table, [key])
            db.commit()
            return default
        return pickle.loads(base64.decodestring(row[1]))

    def set(self, key, value, timeout=None):
        if timeout is None:
            timeout = self.default_timeout
        cursor = db.cursor()
        cursor.execute("SELECT COUNT(*) FROM %s" % self._table)
        num = cursor.fetchone()[0]
        now = datetime.now().replace(microsecond=0)
        exp = datetime.fromtimestamp(time.time() + timeout).replace(microsecond=0)
        if num > self._max_entries:
            self._cull(cursor, now)
        encoded = base64.encodestring(pickle.dumps(value, 2)).strip()
        cursor.execute("SELECT cache_key FROM %s WHERE cache_key = %%s" % self._table, [key])
        try:
            if cursor.fetchone():
                cursor.execute("UPDATE %s SET value = %%s, expires = %%s WHERE cache_key = %%s" % self._table, [encoded, str(exp), key])
            else:
                cursor.execute("INSERT INTO %s (cache_key, value, expires) VALUES (%%s, %%s, %%s)" % self._table, [key, encoded, str(exp)])
        except DatabaseError:
            # To be threadsafe, updates/inserts are allowed to fail silently
            pass
        else:
            db.commit()

    def delete(self, key):
        cursor = db.cursor()
        cursor.execute("DELETE FROM %s WHERE cache_key = %%s" % self._table, [key])
        db.commit()

    def has_key(self, key):
        cursor = db.cursor()
        cursor.execute("SELECT cache_key FROM %s WHERE cache_key = %%s" % self._table, [key])
        return cursor.fetchone() is not None

    def _cull(self, cursor, now):
        if self._cull_frequency == 0:
            cursor.execute("DELETE FROM %s" % self._table)
        else:
            cursor.execute("DELETE FROM %s WHERE expires < %%s" % self._table, [str(now)])
            cursor.execute("SELECT COUNT(*) FROM %s" % self._table)
            num = cursor.fetchone()[0]
            if num > self._max_entries:
                cursor.execute("SELECT cache_key FROM %s ORDER BY cache_key LIMIT 1 OFFSET %%s" % self._table, [num / self._cull_frequency])
                cursor.execute("DELETE FROM %s WHERE cache_key < %%s" % self._table, [cursor.fetchone()[0]])

##########################################
# Read settings and load a cache backend #
##########################################

from cgi import parse_qsl

_BACKENDS = {
    'memcached' : _MemcachedCache,
    'simple'    : _SimpleCache,
    'locmem'    : _LocMemCache,
    'file'      : _FileCache,
    'db'        : _DBCache,
}

def get_cache(backend_uri):
    if backend_uri.find(':') == -1:
        raise InvalidCacheBackendError("Backend URI must start with scheme://")
    scheme, rest = backend_uri.split(':', 1)
    if not rest.startswith('//'):
        raise InvalidCacheBackendError("Backend URI must start with scheme://")
    if scheme not in _BACKENDS.keys():
        raise InvalidCacheBackendError("%r is not a valid cache backend" % scheme)

    host = rest[2:]
    qpos = rest.find('?')
    if qpos != -1:
        params = dict(parse_qsl(rest[qpos+1:]))
        host = rest[2:qpos]
    else:
        params = {}
    if host.endswith('/'):
        host = host[:-1]

    return _BACKENDS[scheme](host, params)

from django.conf.settings import CACHE_BACKEND
cache = get_cache(CACHE_BACKEND)
<commit_msg>Fixed #761; thanks, Eugene<commit_after>"""
Caching framework.

This module defines set of cache backends that all conform to a simple API.
In a nutshell, a cache is a set of values -- which can be any object that
may be pickled -- identified by string keys.  For the complete API, see
the abstract Cache object, below.

Client code should not access a cache backend directly; instead
it should use the get_cache() function.  This function will look at
settings.CACHE_BACKEND and use that to create and load a cache object.

The CACHE_BACKEND setting is a quasi-URI; examples are:

    memcached://127.0.0.1:11211/    A memcached backend; the server is running
                                    on localhost port 11211.  You can use
                                    multiple memcached servers by separating
                                    them with semicolons.

    db://tablename/                 A database backend in a table named
                                    "tablename". This table should be created
                                    with "django-admin createcachetable".

    file:///var/tmp/django_cache/   A file-based cache stored in the directory
                                    /var/tmp/django_cache/.

    simple:///                      A simple single-process memory cache; you
                                    probably don't want to use this except for
                                    testing. Note that this cache backend is
                                    NOT threadsafe!

    locmem:///                      A more sophisticaed local memory cache;
                                    this is multi-process- and thread-safe.

All caches may take arguments; these are given in query-string style.  Valid
arguments are:

    timeout
        Default timeout, in seconds, to use for the cache.  Defaults
        to 5 minutes (300 seconds).

    max_entries
        For the simple, file, and database backends, the maximum number of
        entries allowed in the cache before it is cleaned.  Defaults to
        300.

    cull_percentage
        The percentage of entries that are culled when max_entries is reached.
        The actual percentage is 1/cull_percentage, so set cull_percentage=3 to
        cull 1/3 of the entries when max_entries is reached.

        A value of 0 for cull_percentage means that the entire cache will be
        dumped when max_entries is reached.  This makes culling *much* faster
        at the expense of more cache misses.

For example:

    memcached://127.0.0.1:11211/?timeout=60
    db://tablename/?timeout=120&max_entries=500&cull_percentage=4

Invalid arguments are silently ignored, as are invalid values of known
arguments.
"""

##############
# Exceptions #
##############

class InvalidCacheBackendError(Exception):
    pass

################################
# Abstract base implementation #
################################

class _Cache:
    def __init__(self, params):
        timeout = params.get('timeout', 300)
        try:
            timeout = int(timeout)
        except (ValueError, TypeError):
            timeout = 300
        self.default_timeout = timeout

    def get(self, key, default=None):
        '''
        Fetch a given key from the cache.  If the key does not exist, return
        default, which itself defaults to None.
        '''
        raise NotImplementedError

    def set(self, key, value, timeout=None):
        '''
        Set a value in the cache.  If timeout is given, that timeout will be
        used for the key; otherwise the default cache timeout will be used.
        '''
        raise NotImplementedError

    def delete(self, key):
        '''
        Delete a key from the cache, failing silently.
        '''
        raise NotImplementedError

    def get_many(self, keys):
        '''
        Fetch a bunch of keys from the cache.  For certain backends (memcached,
        pgsql) this can be *much* faster when fetching multiple values.

        Returns a dict mapping each key in keys to its value.  If the given
        key is missing, it will be missing from the response dict.
        '''
        d = {}
        for k in keys:
            val = self.get(k)
            if val is not None:
                d[k] = val
        return d

    def has_key(self, key):
        '''
        Returns True if the key is in the cache and has not expired.
        '''
        return self.get(key) is not None

###########################
# memcached cache backend #
###########################

try:
    import memcache
except ImportError:
    _MemcachedCache = None
else:
    class _MemcachedCache(_Cache):
        "Memcached cache backend."
        def __init__(self, server, params):
            _Cache.__init__(self, params)
            self._cache = memcache.Client(server.split(';'))

        def get(self, key, default=None):
            val = self._cache.get(key)
            if val is None:
                return default
            else:
                return val

        def set(self, key, value, timeout=0):
            self._cache.set(key, value, timeout)

        def delete(self, key):
            self._cache.delete(key)

        def get_many(self, keys):
            return self._cache.get_multi(keys)

##################################
# Single-process in-memory cache #
##################################

import time

class _SimpleCache(_Cache):
    "Simple single-process in-memory cache."
    def __init__(self, host, params):
        _Cache.__init__(self, params)
        self._cache = {}
        self._expire_info = {}

        max_entries = params.get('max_entries', 300)
        try:
            self._max_entries = int(max_entries)
        except (ValueError, TypeError):
            self._max_entries = 300

        cull_frequency = params.get('cull_frequency', 3)
        try:
            self._cull_frequency = int(cull_frequency)
        except (ValueError, TypeError):
            self._cull_frequency = 3

    def get(self, key, default=None):
        now = time.time()
        exp = self._expire_info.get(key)
        if exp is None:
            return default
        elif exp < now:
            del self._cache[key]
            del self._expire_info[key]
            return default
        else:
            return self._cache[key]

    def set(self, key, value, timeout=None):
        if len(self._cache) >= self._max_entries:
            self._cull()
        if timeout is None:
            timeout = self.default_timeout
        self._cache[key] = value
        self._expire_info[key] = time.time() + timeout

    def delete(self, key):
        try:
            del self._cache[key]
        except KeyError:
            pass
        try:
            del self._expire_info[key]
        except KeyError:
            pass

    def has_key(self, key):
        return self._cache.has_key(key)

    def _cull(self):
        if self._cull_frequency == 0:
            self._cache.clear()
            self._expire_info.clear()
        else:
            doomed = [k for (i, k) in enumerate(self._cache) if i % self._cull_frequency == 0]
            for k in doomed:
                self.delete(k)

###############################
# Thread-safe in-memory cache #
###############################

try:
    import cPickle as pickle
except ImportError:
    import pickle
import copy
from django.utils.synch import RWLock

class _LocMemCache(_SimpleCache):
    "Thread-safe in-memory cache."
    def __init__(self, host, params):
        _SimpleCache.__init__(self, host, params)
        self._lock = RWLock()

    def get(self, key, default=None):
        should_delete = False
        self._lock.reader_enters()
        try:
            now = time.time()
            exp = self._expire_info.get(key)
            if exp is None:
                return default
            elif exp < now:
                should_delete = True
            else:
                return copy.deepcopy(self._cache[key])
        finally:
            self._lock.reader_leaves()
        if should_delete:
            self._lock.writer_enters()
            try:
                del self._cache[key]
                del self._expire_info[key]
                return default
            finally:
                self._lock.writer_leaves()

    def set(self, key, value, timeout=None):
        self._lock.writer_enters()
        try:
            _SimpleCache.set(self, key, value, timeout)
        finally:
            self._lock.writer_leaves()

    def delete(self, key):
        self._lock.writer_enters()
        try:
            _SimpleCache.delete(self, key)
        finally:
            self._lock.writer_leaves()

####################
# File-based cache #
####################

import os
import urllib

class _FileCache(_SimpleCache):
    "File-based cache."
    def __init__(self, dir, params):
        self._dir = dir
        if not os.path.exists(self._dir):
            self._createdir()
        _SimpleCache.__init__(self, dir, params)
        del self._cache
        del self._expire_info

    def get(self, key, default=None):
        fname = self._key_to_file(key)
        try:
            f = open(fname, 'rb')
            exp = pickle.load(f)
            now = time.time()
            if exp < now:
                f.close()
                os.remove(fname)
            else:
                return pickle.load(f)
        except (IOError, pickle.PickleError):
            pass
        return default

    def set(self, key, value, timeout=None):
        fname = self._key_to_file(key)
        if timeout is None:
            timeout = self.default_timeout
        try:
            filelist = os.listdir(self._dir)
        except (IOError, OSError):
            self._createdir()
            filelist = []
        if len(filelist) > self._max_entries:
            self._cull(filelist)
        try:
            f = open(fname, 'wb')
            now = time.time()
            pickle.dump(now + timeout, f, 2)
            pickle.dump(value, f, 2)
        except (IOError, OSError):
            raise

    def delete(self, key):
        try:
            os.remove(self._key_to_file(key))
        except (IOError, OSError):
            pass

    def has_key(self, key):
        return os.path.exists(self._key_to_file(key))

    def _cull(self, filelist):
        if self._cull_frequency == 0:
            doomed = filelist
        else:
            doomed = [k for (i, k) in enumerate(filelist) if i % self._cull_frequency == 0]
        for fname in doomed:
            try:
                os.remove(os.path.join(self._dir, fname))
            except (IOError, OSError):
                pass

    def _createdir(self):
        try:
            os.makedirs(self._dir)
        except OSError:
            raise EnvironmentError, "Cache directory '%s' does not exist and could not be created'" % self._dir

    def _key_to_file(self, key):
        return os.path.join(self._dir, urllib.quote_plus(key))

#############
# SQL cache #
#############

import base64
from django.core.db import db, DatabaseError
from datetime import datetime

class _DBCache(_Cache):
    "SQL cache backend."
    def __init__(self, table, params):
        _Cache.__init__(self, params)
        self._table = table
        max_entries = params.get('max_entries', 300)
        try:
            self._max_entries = int(max_entries)
        except (ValueError, TypeError):
            self._max_entries = 300
        cull_frequency = params.get('cull_frequency', 3)
        try:
            self._cull_frequency = int(cull_frequency)
        except (ValueError, TypeError):
            self._cull_frequency = 3

    def get(self, key, default=None):
        cursor = db.cursor()
        cursor.execute("SELECT cache_key, value, expires FROM %s WHERE cache_key = %%s" % self._table, [key])
        row = cursor.fetchone()
        if row is None:
            return default
        now = datetime.now()
        if row[2] < now:
            cursor.execute("DELETE FROM %s WHERE cache_key = %%s" % self._table, [key])
            db.commit()
            return default
        return pickle.loads(base64.decodestring(row[1]))

    def set(self, key, value, timeout=None):
        if timeout is None:
            timeout = self.default_timeout
        cursor = db.cursor()
        cursor.execute("SELECT COUNT(*) FROM %s" % self._table)
        num = cursor.fetchone()[0]
        now = datetime.now().replace(microsecond=0)
        exp = datetime.fromtimestamp(time.time() + timeout).replace(microsecond=0)
        if num > self._max_entries:
            self._cull(cursor, now)
        encoded = base64.encodestring(pickle.dumps(value, 2)).strip()
        cursor.execute("SELECT cache_key FROM %s WHERE cache_key = %%s" % self._table, [key])
        try:
            if cursor.fetchone():
                cursor.execute("UPDATE %s SET value = %%s, expires = %%s WHERE cache_key = %%s" % self._table, [encoded, str(exp), key])
            else:
                cursor.execute("INSERT INTO %s (cache_key, value, expires) VALUES (%%s, %%s, %%s)" % self._table, [key, encoded, str(exp)])
        except DatabaseError:
            # To be threadsafe, updates/inserts are allowed to fail silently
            pass
        else:
            db.commit()

    def delete(self, key):
        cursor = db.cursor()
        cursor.execute("DELETE FROM %s WHERE cache_key = %%s" % self._table, [key])
        db.commit()

    def has_key(self, key):
        cursor = db.cursor()
        cursor.execute("SELECT cache_key FROM %s WHERE cache_key = %%s" % self._table, [key])
        return cursor.fetchone() is not None

    def _cull(self, cursor, now):
        if self._cull_frequency == 0:
            cursor.execute("DELETE FROM %s" % self._table)
        else:
            cursor.execute("DELETE FROM %s WHERE expires < %%s" % self._table, [str(now)])
            cursor.execute("SELECT COUNT(*) FROM %s" % self._table)
            num = cursor.fetchone()[0]
            if num > self._max_entries:
                cursor.execute("SELECT cache_key FROM %s ORDER BY cache_key LIMIT 1 OFFSET %%s" % self._table, [num / self._cull_frequency])
                cursor.execute("DELETE FROM %s WHERE cache_key < %%s" % self._table, [cursor.fetchone()[0]])

##########################################
# Read settings and load a cache backend #
##########################################

from cgi import parse_qsl

_BACKENDS = {
    'memcached' : _MemcachedCache,
    'simple'    : _SimpleCache,
    'locmem'    : _LocMemCache,
    'file'      : _FileCache,
    'db'        : _DBCache,
}

def get_cache(backend_uri):
    if backend_uri.find(':') == -1:
        raise InvalidCacheBackendError("Backend URI must start with scheme://")
    scheme, rest = backend_uri.split(':', 1)
    if not rest.startswith('//'):
        raise InvalidCacheBackendError("Backend URI must start with scheme://")
    if scheme not in _BACKENDS.keys():
        raise InvalidCacheBackendError("%r is not a valid cache backend" % scheme)

    host = rest[2:]
    qpos = rest.find('?')
    if qpos != -1:
        params = dict(parse_qsl(rest[qpos+1:]))
        host = rest[2:qpos]
    else:
        params = {}
    if host.endswith('/'):
        host = host[:-1]

    return _BACKENDS[scheme](host, params)

from django.conf.settings import CACHE_BACKEND
cache = get_cache(CACHE_BACKEND)
<|endoftext|>


<commit_before>""" Views for Django """
from operator import ne
from django.core import serializers
from django.shortcuts import get_object_or_404, render
from django.http import HttpResponseRedirect
from django.contrib.auth import logout as auth_logout
from django.shortcuts import redirect
from django.contrib.auth.decorators import login_required
from django.db.models import Sum
from django.core.mail import EmailMessage

import datetime
from duckiehunt.secrets.settings import *
from .models import Duck, DuckLocation, DuckLocationPhoto
from .forms import DuckForm
from duck import media
from haversine import haversine, Unit

def index(request):
    """ / path """
    duck_location_list = DuckLocation.objects.all()
    duck_location_json = serializers.serialize('json', duck_location_list)
    map_data = {
        'width': '100%',
        'height': '800px',
        'focus_lat': 23,
        'focus_long': 10,
        'focus_zoom': 2,
        'location_list': duck_location_json,
        'duck_location_id': 0,
    }

    return render(request, 'duck/main.html', {'map': map_data})

def detail(request, duck_id):
    """ /duck/# path """
    duck = get_object_or_404(Duck, pk=duck_id)
    photos = DuckLocationPhoto.objects.filter(duck_location__duck_id=duck_id)

    # SCREW YOU DJANGO FOR NOT SUPPORT USING SELECT_RELATED + SERIALIZATION
    # https://medium.com/@PyGuyCharles/python-sql-to-json-and-beyond-3e3a36d32853
    # https://stackoverflow.com/questions/34666892/trying-to-serialize-a-queryset-that-uses-select-related-cant-obtain-fields-o
    # https://docs.djangoproject.com/en/2.1/topics/serialization/#serialization-of-natural-keys
    duck_location_list = DuckLocation.objects.filter(duck_id=duck_id)
    duck_location_json = serializers.serialize('json', duck_location_list, use_natural_foreign_keys=True)
    map_data = {
        'width': '100%',
        'height': '400px',
        'focus_lat': 0,
        'focus_long': 0,
        'focus_zoom': 2,
        'location_list': duck_location_json,
        'duck_location_id': 0,
    }
    duck_dropdown_list = Duck.objects.all()

    return render(request, 'duck/detail.html',
                  {'duck': duck, 'photos': photos, 'map': map_data,
                   'duck_location_list': duck_location_list, 'duck_list': duck_dropdown_list})

def location(request, duck_location_id):
    """ Show /duck/$duck_id """
    duck_location = get_object_or_404(DuckLocation, pk=duck_location_id)

    photos = DuckLocationPhoto.objects.filter(duck_location_id=duck_location_id)

    duck_location_list = DuckLocation.objects.filter(duck_id=duck_location.duck.duck_id)
    duck_location_json = serializers.serialize('json', duck_location_list, use_natural_foreign_keys=True)
    map_data = {
        'width': '100%',
        'height': '400px',
        'focus_lat': 0,
        'focus_long': 0,
        'focus_zoom': 15,
        'location_list': duck_location_json,
        'duck_location_id': duck_location_id,
    }
    duck_dropdown_list = Duck.objects.all()

    return render(request, 'duck/location.html',
                  {'photos': photos, 'map': map_data, 'duck': duck_location.duck,
                   'duck_location': duck_location, 'duck_location_list': duck_location_list,
                   'duck_list': duck_dropdown_list})

def duck_list(request):
    """ lists all ducks """
    ducks = Duck.objects.all()
    return render(request, 'duck/list.html', {'duck_list': ducks})

def faq(request):
    """ shows faq """
    return render(request, 'duck/faq.html')

def tos(request):
    """ shows terms of service """
    return render(request, 'duck/tos.html')

def privacy(request):
    """ shows privacy """
    return render(request, 'duck/privacy.html')

@login_required
def mark(request):
    """ Adds a duck, location, photo and link from webform """
    # if this is a POST request we need to process the form data
    if request.method == 'POST':
        # create a form instance and populate it with data from the request:
        form = DuckForm(request.POST)
        # check whether it's valid:
        if form.is_valid():
            duck_id = form.cleaned_data['duck_id']
            try:
                duck = Duck.objects.get(pk=duck_id)
                if duck.name == 'Unnamed' and form.cleaned_data['name'] != 'Unnamed':
                    duck.name = form.cleaned_data['name']
            except Duck.DoesNotExist:
                name = form.cleaned_data['name'] if form.cleaned_data['name'] else 'Unnamed'
                duck = Duck(duck_id=duck_id,
                            name=name, approved='Y',
                            create_time=datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S'),
                            comments='')

            # Calculate the distance since last location
            last_duck_location = DuckLocation.objects.filter(duck_id=duck_id).order_by('-date_time')[0]
            distance_travelled = haversine((last_duck_location.latitude, last_duck_location.longitude),
                                           (form.cleaned_data['lat'], form.cleaned_data['lng']), unit=Unit.MILES)
            duck_location = DuckLocation(duck=duck,
                                         latitude=form.cleaned_data['lat'],
                                         longitude=form.cleaned_data['lng'],
                                         location=form.cleaned_data['location'],
                                         date_time=form.cleaned_data['date_time'],
                                         comments=form.cleaned_data['comments'],
                                         distance_to=round(distance_travelled, 2),
                                         user=request.user,
                                         approved='Y')
            duck_location.save()
            if request.FILES and request.FILES['image']:
                photo_info = media.handle_uploaded_file(request.FILES['image'], duck_id,
                                                        duck.name, form.cleaned_data['comments'])
                duck_location_photo = DuckLocationPhoto(duck_location=duck_location,
                                                        flickr_photo_id=photo_info['id'],
                                                        flickr_thumbnail_url=photo_info['sizes']['Small 320']['source'])
                duck_location_photo.save()

            duck.total_distance = round(DuckLocation.objects.filter(duck_id=duck_id).aggregate(Sum('distance_to'))['distance_to__sum'], 2)
            duck.save()

            # redirect to a new URL:
            new_location_url = '/location/' + str(duck_location.duck_location_id)

            msg = EmailMessage(
                'Duckiehunt Update: Duck #' + str(duck_id),
                'Duck #' + str(duck_id) + ' has moved!
' + BASE_URL + new_location_url,
                EMAIL_FROM, EMAIL_TO
            )
            msg.content_subtype = "html"
            msg.send()

            return HttpResponseRedirect(new_location_url)
    # if a GET (or any other method) we'll create a blank form
    else:
        form = DuckForm()

    map_data = {
        'width': '100%',
        'height': '400px',
        'focus_lat': 35,
        'focus_long': -30,
        'focus_zoom': 1,
        'location_list': [],
        'duck_location_id': 0,
    }
    return render(request, 'duck/mark.html', {'form': form, 'map': map_data})

def logout(request):
    """Logs out user"""
    auth_logout(request)
    return redirect('/')

def login(request):
    """Home view, displays login mechanism"""
    return render(request, 'duck/login.html')

def profile(request):
    """ Show profile data """
    print(request.user.username)
    return render(request, 'duck/profile.html', {'user': request.user})<commit_msg>Found issue where new duck add fails<commit_after>""" Views for Django """
from operator import ne
from django.core import serializers
from django.shortcuts import get_object_or_404, render
from django.http import HttpResponseRedirect
from django.contrib.auth import logout as auth_logout
from django.shortcuts import redirect
from django.contrib.auth.decorators import login_required
from django.db.models import Sum
from django.core.mail import EmailMessage
from django.contrib.auth.models import User

import datetime
from duckiehunt.secrets.settings import *
from .models import Duck, DuckLocation, DuckLocationPhoto
from .forms import DuckForm
from duck import media
from haversine import haversine, Unit

def index(request):
    """ / path """
    duck_location_list = DuckLocation.objects.all()
    duck_location_json = serializers.serialize('json', duck_location_list)
    map_data = {
        'width': '100%',
        'height': '800px',
        'focus_lat': 23,
        'focus_long': 10,
        'focus_zoom': 2,
        'location_list': duck_location_json,
        'duck_location_id': 0,
    }

    return render(request, 'duck/main.html', {'map': map_data})

def detail(request, duck_id):
    """ /duck/# path """
    duck = get_object_or_404(Duck, pk=duck_id)
    photos = DuckLocationPhoto.objects.filter(duck_location__duck_id=duck_id)

    # SCREW YOU DJANGO FOR NOT SUPPORT USING SELECT_RELATED + SERIALIZATION
    # https://medium.com/@PyGuyCharles/python-sql-to-json-and-beyond-3e3a36d32853
    # https://stackoverflow.com/questions/34666892/trying-to-serialize-a-queryset-that-uses-select-related-cant-obtain-fields-o
    # https://docs.djangoproject.com/en/2.1/topics/serialization/#serialization-of-natural-keys
    duck_location_list = DuckLocation.objects.filter(duck_id=duck_id)
    duck_location_json = serializers.serialize('json', duck_location_list, use_natural_foreign_keys=True)
    map_data = {
        'width': '100%',
        'height': '400px',
        'focus_lat': 0,
        'focus_long': 0,
        'focus_zoom': 2,
        'location_list': duck_location_json,
        'duck_location_id': 0,
    }
    duck_dropdown_list = Duck.objects.all()

    return render(request, 'duck/detail.html',
                  {'duck': duck, 'photos': photos, 'map': map_data,
                   'duck_location_list': duck_location_list, 'duck_list': duck_dropdown_list})

def location(request, duck_location_id):
    """ Show /duck/$duck_id """
    duck_location = get_object_or_404(DuckLocation, pk=duck_location_id)

    photos = DuckLocationPhoto.objects.filter(duck_location_id=duck_location_id)

    duck_location_list = DuckLocation.objects.filter(duck_id=duck_location.duck.duck_id)
    duck_location_json = serializers.serialize('json', duck_location_list, use_natural_foreign_keys=True)
    map_data = {
        'width': '100%',
        'height': '400px',
        'focus_lat': 0,
        'focus_long': 0,
        'focus_zoom': 15,
        'location_list': duck_location_json,
        'duck_location_id': duck_location_id,
    }
    duck_dropdown_list = Duck.objects.all()

    return render(request, 'duck/location.html',
                  {'photos': photos, 'map': map_data, 'duck': duck_location.duck,
                   'duck_location': duck_location, 'duck_location_list': duck_location_list,
                   'duck_list': duck_dropdown_list})

def duck_list(request):
    """ lists all ducks """
    ducks = Duck.objects.all()
    return render(request, 'duck/list.html', {'duck_list': ducks})

def faq(request):
    """ shows faq """
    return render(request, 'duck/faq.html')

def tos(request):
    """ shows terms of service """
    return render(request, 'duck/tos.html')

def privacy(request):
    """ shows privacy """
    return render(request, 'duck/privacy.html')

@login_required
def mark(request):
    """ Adds a duck, location, photo and link from webform """
    # if this is a POST request we need to process the form data
    if request.method == 'POST':
        # create a form instance and populate it with data from the request:
        form = DuckForm(request.POST)
        # check whether it's valid:
        if form.is_valid():
            duck_id = form.cleaned_data['duck_id']
            try:
                duck = Duck.objects.get(pk=duck_id)
                if duck.name == 'Unnamed' and form.cleaned_data['name'] != 'Unnamed':
                    duck.name = form.cleaned_data['name']
            except Duck.DoesNotExist:
                name = form.cleaned_data['name'] if form.cleaned_data['name'] else 'Unnamed'
                duck = Duck(duck_id=duck_id,
                            name=name, approved='Y',
                            create_time=datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S'),
                            comments='')
                duck.save()
                # Create new DuckLocation from it's origin
                tommy = User.objects.get(id=1)
                duck_location_start = DuckLocation(duck=duck,
                             latitude='32.95159763382337',
                             longitude='-96.90789423886032',
                             location='Carrollton Plaza Arts Center, Carrollton, TX',
                             date_time='2008-08-16 20:00:00',
                             comments='Just got married!',
                             distance_to=0,
                             user=tommy,
                             approved='Y')
                duck_location_start.save()

            # Calculate the distance since last location
            last_duck_location = DuckLocation.objects.filter(duck_id=duck_id).order_by('-date_time')[0]
            distance_travelled = haversine((last_duck_location.latitude, last_duck_location.longitude),
                                           (form.cleaned_data['lat'], form.cleaned_data['lng']), unit=Unit.MILES)
            duck_location = DuckLocation(duck=duck,
                                         latitude=form.cleaned_data['lat'],
                                         longitude=form.cleaned_data['lng'],
                                         location=form.cleaned_data['location'],
                                         date_time=form.cleaned_data['date_time'],
                                         comments=form.cleaned_data['comments'],
                                         distance_to=round(distance_travelled, 2),
                                         user=request.user,
                                         approved='Y')
            duck_location.save()
            if request.FILES and request.FILES['image']:
                photo_info = media.handle_uploaded_file(request.FILES['image'], duck_id,
                                                        duck.name, form.cleaned_data['comments'])
                duck_location_photo = DuckLocationPhoto(duck_location=duck_location,
                                                        flickr_photo_id=photo_info['id'],
                                                        flickr_thumbnail_url=photo_info['sizes']['Small 320']['source'])
                duck_location_photo.save()

            duck.total_distance = round(DuckLocation.objects.filter(duck_id=duck_id).aggregate(Sum('distance_to'))['distance_to__sum'], 2)
            duck.save()

            # redirect to a new URL:
            new_location_url = '/location/' + str(duck_location.duck_location_id)

            msg = EmailMessage(
                'Duckiehunt Update: Duck #' + str(duck_id),
                'Duck #' + str(duck_id) + ' has moved!
' + BASE_URL + new_location_url,
                EMAIL_FROM, EMAIL_TO
            )
            msg.content_subtype = "html"
            msg.send()

            return HttpResponseRedirect(new_location_url)
    # if a GET (or any other method) we'll create a blank form
    else:
        form = DuckForm()

    map_data = {
        'width': '100%',
        'height': '400px',
        'focus_lat': 35,
        'focus_long': -30,
        'focus_zoom': 1,
        'location_list': [],
        'duck_location_id': 0,
    }
    return render(request, 'duck/mark.html', {'form': form, 'map': map_data})

def logout(request):
    """Logs out user"""
    auth_logout(request)
    return redirect('/')

def login(request):
    """Home view, displays login mechanism"""
    return render(request, 'duck/login.html')

def profile(request):
    """ Show profile data """
    print(request.user.username)
    return render(request, 'duck/profile.html', {'user': request.user})<|endoftext|>


<commit_before>from django.db import models
from django.conf import settings

class UserOTP(models.Model):

    OTP_TYPES = (
                ('HOTP', 'hotp'),
                ('TOTP', 'totp'),
    )

    user = models.OneToOneField(settings.AUTH_USER_MODEL)
    otp_type = models.CharField(max_length=20, choices=OTP_TYPES)
    secret_key = models.CharField(max_length=100, blank=True)

def is_mfa_enabled(user):
    try:
        user_otp = user.userotp
        return True
    except:
        return False
<commit_msg>Remove try/except from determing if mfa is enabled<commit_after>from django.conf import settings
from django.db import models

class UserOTP(models.Model):

    OTP_TYPES = (
        ('HOTP', 'hotp'),
        ('TOTP', 'totp'),
    )

    user = models.OneToOneField(settings.AUTH_USER_MODEL)
    otp_type = models.CharField(max_length=20, choices=OTP_TYPES)
    secret_key = models.CharField(max_length=100, blank=True)

def is_mfa_enabled(user):
    """
    Determine if a user has MFA enabled
    """
    return hasattr(user, 'userotp')
<|endoftext|>


<commit_before># import os
# import sys
import django

# myPath = os.path.dirname(os.path.abspath(__file__))
# sys.path.insert(0, myPath)

VERSION = (0, 9, 4)

default_app_config = 'django_q.apps.DjangoQConfig'

# root imports will slowly be deprecated.
# please import from the relevant sub modules
if django.VERSION[:2] < (1, 9):
    from .tasks import async_task, schedule, result, result_group, fetch, fetch_group, count_group, delete_group, queue_size
    from .models import Task, Schedule, Success, Failure
    from .cluster import Cluster
    from .status import Stat
    from .brokers import get_broker

__all__ = ['conf', 'cluster', 'models', 'tasks']
<commit_msg>Removes root imports<commit_after>VERSION = (1, 0, 0)

default_app_config = 'django_q.apps.DjangoQConfig'

__all__ = ['conf', 'cluster', 'models', 'tasks']
<|endoftext|>


<commit_before>from django.contrib import admin
from django.contrib.admin import ModelAdmin as BaseAdmin
from django.contrib.admin.templatetags.admin_static import static
from django.utils.translation import ugettext_lazy as _

from .models import Ca, Cert

class AbstractAdmin(BaseAdmin):
    """
    ModelAdmin for TimeStampedEditableModel
    """
    list_display = ['name',
                    'key_length',
                    'digest',
                    'created',
                    'modified']
    search_fields = ('name', 'serial_number', 'common_name')
    actions_on_bottom = True
    save_on_top = True
    # custom attribute
    readonly_edit = ('key_length',
                     'digest',
                     'validity_start',
                     'validity_end',
                     'country_code',
                     'state',
                     'city',
                     'organization',
                     'email',
                     'common_name',
                     'serial_number',
                     'certificate',
                     'private_key')

    class Media:
        css = {'all': (static('django-x509/css/admin.css'),)}

    def __init__(self, *args, **kwargs):
        self.readonly_fields += ('created', 'modified')
        super(AbstractAdmin, self).__init__(*args, **kwargs)

    def get_readonly_fields(self, request, obj=None):
        # edit
        if obj:
            return self.readonly_edit + self.readonly_fields
        # add
        else:
            return self.readonly_fields

    def get_fields(self, request, obj=None):
        fields = super(AbstractAdmin, self).get_fields(request, obj)
        # edit
        if obj and 'extensions' in fields:
            fields.remove('extensions')
        return fields

class CaAdmin(AbstractAdmin):
    pass

class CertAdmin(AbstractAdmin):
    list_filter = ('ca', 'revoked', 'created',)
    readonly_fields = ('revoked', 'revoked_at',)
    fields = ['name',
              'ca',
              'notes',
              'revoked',
              'revoked_at',
              'key_length',
              'digest',
              'validity_start',
              'validity_end',
              'country_code',
              'state',
              'city',
              'organization',
              'email',
              'common_name',
              'extensions',
              'serial_number',
              'certificate',
              'private_key',
              'created',
              'modified']

    actions = ['revoke_action']

    def revoke_action(self, request, queryset):
        rows = 0
        for cert in queryset:
            cert.revoke()
            rows += 1
        if rows == 1:
            bit = '1 certificate was'
        else:
            bit = '{0} certificates were'.format(rows)
        message = '{0} revoked.'.format(bit)
        self.message_user(request, _(message))

    revoke_action.short_description = _('Revoke selected certificates')

CertAdmin.list_display = AbstractAdmin.list_display[:]
CertAdmin.list_display.insert(1, 'ca')
CertAdmin.list_display.insert(4, 'revoked')
CertAdmin.readonly_edit = AbstractAdmin.readonly_edit[:]
CertAdmin.readonly_edit += ('ca',)

admin.site.register(Ca, CaAdmin)
admin.site.register(Cert, CertAdmin)
<commit_msg>[admin] Show link to CA in cert admin<commit_after>from django.contrib import admin
from django.contrib.admin import ModelAdmin as BaseAdmin
from django.contrib.admin.templatetags.admin_static import static
from django.core.urlresolvers import reverse
from django.utils.html import format_html
from django.utils.translation import ugettext_lazy as _

from .models import Ca, Cert

class AbstractAdmin(BaseAdmin):
    """
    ModelAdmin for TimeStampedEditableModel
    """
    list_display = ['name',
                    'key_length',
                    'digest',
                    'created',
                    'modified']
    search_fields = ('name', 'serial_number', 'common_name')
    actions_on_bottom = True
    save_on_top = True
    # custom attribute
    readonly_edit = ('key_length',
                     'digest',
                     'validity_start',
                     'validity_end',
                     'country_code',
                     'state',
                     'city',
                     'organization',
                     'email',
                     'common_name',
                     'serial_number',
                     'certificate',
                     'private_key')

    class Media:
        css = {'all': (static('django-x509/css/admin.css'),)}

    def __init__(self, *args, **kwargs):
        self.readonly_fields += ('created', 'modified')
        super(AbstractAdmin, self).__init__(*args, **kwargs)

    def get_readonly_fields(self, request, obj=None):
        # edit
        if obj:
            return self.readonly_edit + self.readonly_fields
        # add
        else:
            return self.readonly_fields

    def get_fields(self, request, obj=None):
        fields = super(AbstractAdmin, self).get_fields(request, obj)
        # edit
        if obj and 'extensions' in fields:
            fields.remove('extensions')
        return fields

class CaAdmin(AbstractAdmin):
    pass

class CertAdmin(AbstractAdmin):
    list_filter = ('ca', 'revoked', 'created',)
    list_select_related = ('ca',)
    readonly_fields = ('revoked', 'revoked_at',)
    fields = ['name',
              'ca',
              'notes',
              'revoked',
              'revoked_at',
              'key_length',
              'digest',
              'validity_start',
              'validity_end',
              'country_code',
              'state',
              'city',
              'organization',
              'email',
              'common_name',
              'extensions',
              'serial_number',
              'certificate',
              'private_key',
              'created',
              'modified']

    actions = ['revoke_action']

    def ca_url(self, obj):
        url = reverse('admin:django_x509_ca_change', args=[obj.ca.id])
        return format_html("{text}",
                           url=url,
                           text=obj.ca.name)
    ca_url.short_description = 'CA'
    def revoke_action(self, request, queryset):
        rows = 0
        for cert in queryset:
            cert.revoke()
            rows += 1
        if rows == 1:
            bit = '1 certificate was'
        else:
            bit = '{0} certificates were'.format(rows)
        message = '{0} revoked.'.format(bit)
        self.message_user(request, _(message))

    revoke_action.short_description = _('Revoke selected certificates')

CertAdmin.list_display = AbstractAdmin.list_display[:]
CertAdmin.list_display.insert(1, 'ca_url')
CertAdmin.list_display.insert(4, 'revoked')
CertAdmin.readonly_edit = AbstractAdmin.readonly_edit[:]
CertAdmin.readonly_edit += ('ca',)

admin.site.register(Ca, CaAdmin)
admin.site.register(Cert, CertAdmin)
<|endoftext|>