<commit_before>import os
import mmap
from numba import jit
import numexpr as ne
import numpy as np
from ..utils import Timer
from ..utils.conf import info

def memory_map(filename, access=mmap.ACCESS_WRITE):
    size = os.path.getsize(filename)
    fd = os.open(filename, os.O_RDWR)
    return mmap.mmap(fd, size, access=access)

@jit(cache=True, nopython=True)
def peakdet(v, delta, x = None):

    # initiate with .1 because numba doesn't support empty list
    maxtab = [(.1,.1)]
    mintab = [(.1,.1)]
    if x is None:
        x = np.arange(len(v))

    mn, mx = np.Inf, -np.Inf
    mnpos, mxpos = np.NaN, np.NaN
    lookformax = True
    for i in np.arange(len(v)):
        this = v[i]
        if this > mx:
            mx = this
            mxpos = x[i]
        if this < mn:
            mn = this
            mnpos = x[i]
        if lookformax:
            if this < mx-delta:
                maxtab.append((mxpos, mx))
                mn = this
                mnpos = x[i]
                lookformax = False
        else:
            if this > mn+delta:
                mintab.append((mnpos, mn))
                mx = this
                mxpos = x[i]
                lookformax = True
    return np.array(maxtab[1:]), np.array(mintab[1:])

class bload(object):
    '''
    load bin file data with file format
    init the nCh and fs when create the instance

    bf = Binload(nCh=16,fs=30000)
    bf.load('./137_36_shankD_116.dat')

    1. bf.t is time; bf.asarray(binpoint) return data
    2. bf.asarray(binpoint=14) to convert int to fix-point
    3. bf.to_threshold(k=4.5) to export median-based threshold
    4. (bf._npts, bf._nCh, bf._nbytes) are metadata
    '''

    def __init__(self, nCh=16, fs=30000):
        self._nCh = nCh
        self.fs = float(fs)

    def load(self, file_name, dtype='int32'):
        '''
        bin.load('filename','int16')
        bin.load('filename','float32')
        '''
        self.mm   = memory_map(file_name)
        self.npmm = np.memmap(file_name, dtype=dtype, mode='r')
        self.dtype = dtype
        self._npts = len(self.npmm)/self._nCh #full #pts/ch
        self._nbytes = self.npmm.nbytes
        info("#############  load data  ###################")
        info('{0} loaded, it contains: '.format(file_name))
        info('{0} * {1} points ({2} bytes) '.format(self._npts, self._nCh, self._nbytes))
        info('{0} channels with sampling rate of {1:.4f} '.format(self._nCh, self.fs))
        info('{0:.3f} secs ({1:.3f} mins) of data'.format(self._npts/self.fs, self._npts/self.fs/60))
        info("#############################################")

        dt = 1/self.fs
        self.t = np.linspace(0,self._npts*dt,self._npts,endpoint='false')

    def __repr__(self):
        return self.info0 + self.info1 + self.info2 + self.info3

    def asarray(self, binpoint=13):
        ne.set_num_threads(32)
        data = np.asarray(self.npmm).reshape(-1, self._nCh)
        _scale = np.float32(2**binpoint)
        return ne.evaluate('data/_scale')

    def reorder_by_chip(self, nchips=5):
        nch_perchip = self._nCh/nchips
        self.npmm = self.npmm.reshape(-1, nch_perchip, nchips)
        self.npmm = self.npmm.swapaxes(1,2)
        self.npmm = self.npmm.reshape(-1, self._nCh)
        info('reordered with nchips={0} and nch_perchip={1}'.format(nchips,nch_perchip))

    def to_threshold(self, data, k=4.5):
        # QQ threshold for spike detection
        data = data[::20,:]
        thres_arr = -np.median(ne.evaluate('abs(data)*k/0.675'), axis=0)
        return thres_arr

    def detect_spks(self, delta=.3):
        '''
            detect spikes by peakdet and threshhold. (channel by channel)

            return
            -------
            [0],t  : array-like
                the time of spikes
            [1],ch : array-like
                the channel number of each spikes. so len(t) == len(ch)
        '''
        with Timer('[MODEL] Binload -- covert to data'):
            data = self.asarray()

        with Timer('[MODEL] Binload -- threshholds'):
            threshholds = self.to_threshold(data)

        t = np.array([], dtype=np.int64)
        ch = np.array([], dtype=np.int32)

        for i in range(data.shape[1]):
            with Timer('[MODEL] Binload -- cal peakdet for ' + str(i)):
                _, mintab = peakdet(data[:,i],delta)
            spks = mintab[mintab[:,1] < threshholds[i]].astype(np.int64)
            t = np.hstack((t, spks[:,0]))
            ch = np.hstack((ch, np.full(spks[:,0].shape, i, dtype=np.int64)))

        return np.array([t, ch])
<commit_msg>Binload use torch to solve memory issue<commit_after>import os
import mmap
from numba import jit
import numexpr as ne
import numpy as np
import torch
from ..utils import Timer
from ..utils.conf import info

def memory_map(filename, access=mmap.ACCESS_WRITE):
    size = os.path.getsize(filename)
    fd = os.open(filename, os.O_RDWR)
    return mmap.mmap(fd, size, access=access)

@jit(cache=True, nopython=True)
def peakdet(v, delta, x = None):

    # initiate with .1 because numba doesn't support empty list
    maxtab = [(.1,.1)]
    mintab = [(.1,.1)]
    if x is None:
        x = np.arange(len(v))

    mn, mx = np.Inf, -np.Inf
    mnpos, mxpos = np.NaN, np.NaN
    lookformax = True
    for i in np.arange(len(v)):
        this = v[i]
        if this > mx:
            mx = this
            mxpos = x[i]
        if this < mn:
            mn = this
            mnpos = x[i]
        if lookformax:
            if this < mx-delta:
                maxtab.append((mxpos, mx))
                mn = this
                mnpos = x[i]
                lookformax = False
        else:
            if this > mn+delta:
                mintab.append((mnpos, mn))
                mx = this
                mxpos = x[i]
                lookformax = True
    return np.array(maxtab[1:]), np.array(mintab[1:])

class bload(object):
    '''
    load bin file data with file format
    init the nCh and fs when create the instance

    bf = Binload(nCh=16,fs=30000)
    bf.load('./137_36_shankD_116.dat')

    1. bf.t is time; bf.asarray(binpoint) return data
    2. bf.asarray(binpoint=14) to convert int to fix-point
    3. bf.to_threshold(k=4.5) to export median-based threshold
    4. (bf._npts, bf._nCh, bf._nbytes) are metadata
    '''

    def __init__(self, nCh=160, fs=25000):
        self._nCh = nCh
        self.fs = float(fs)

    def load(self, file_name, dtype='int32'):
        '''
        bin.load('filename','int16')
        bin.load('filename','float32')
        '''
        self.mm   = memory_map(file_name)
        self.npmm = np.memmap(file_name, dtype=dtype, mode='r')
        self.dtype = dtype
        self._npts = len(self.npmm)/self._nCh #full #pts/ch
        self._nbytes = self.npmm.nbytes
        self.data = torch.from_numpy(self.npmm)
        info("#############  load data  ###################")
        info('{0} loaded, it contains: '.format(file_name))
        info('{0} * {1} points ({2} bytes) '.format(self._npts, self._nCh, self._nbytes))
        info('{0} channels with sampling rate of {1:.4f} '.format(self._nCh, self.fs))
        info('{0:.3f} secs ({1:.3f} mins) of data'.format(self._npts/self.fs, self._npts/self.fs/60))
        info("#############################################")

        dt = 1/self.fs
        self.t = np.linspace(0,self._npts*dt,self._npts,endpoint='false')

    def __repr__(self):
        return self.info0 + self.info1 + self.info2 + self.info3

    def asarray(self, binpoint=13):
        ne.set_num_threads(32)
        data = self.data.reshape(-1, self._nCh).numpy()
        _scale = np.float32(2**binpoint)
        self.data = torch.from_numpy(ne.evaluate('data/_scale'))
        return self.data.numpy()

    def reorder_by_chip(self, nchips=5):
        nch_perchip = self._nCh/nchips
        # self.npmm = self.npmm.reshape(-1, nch_perchip, nchips)
        # self.npmm = self.npmm.swapaxes(1,2)
        # self.npmm = self.npmm.reshape(-1, self._nCh)
        self.data = self.data.reshape(-1,nch_perchip, nchips).transpose(1,2).reshape(-1, self._nCh)
        info('reordered with nchips={0} and nch_perchip={1}'.format(nchips,nch_perchip))

    def to_threshold(self, data, k=4.5):
        # QQ threshold for spike detection
        data = data[::20,:]
        thres_arr = -np.median(ne.evaluate('abs(data)*k/0.675'), axis=0)
        return thres_arr

    def detect_spks(self, delta=.3):
        '''
            detect spikes by peakdet and threshhold. (channel by channel)

            return
            -------
            [0],t  : array-like
                the time of spikes
            [1],ch : array-like
                the channel number of each spikes. so len(t) == len(ch)
        '''
        with Timer('[MODEL] Binload -- covert to data'):
            data = self.asarray()

        with Timer('[MODEL] Binload -- threshholds'):
            threshholds = self.to_threshold(data)

        t = np.array([], dtype=np.int64)
        ch = np.array([], dtype=np.int32)

        for i in range(data.shape[1]):
            with Timer('[MODEL] Binload -- cal peakdet for ' + str(i)):
                _, mintab = peakdet(data[:,i],delta)
            spks = mintab[mintab[:,1] < threshholds[i]].astype(np.int64)
            t = np.hstack((t, spks[:,0]))
            ch = np.hstack((ch, np.full(spks[:,0].shape, i, dtype=np.int64)))

        return np.array([t, ch])
<|endoftext|>


<commit_before>import ipyparallel as ipp
import numpy as np
from vispy import app

dpgmm_def = '''
def dpgmm(max_cluster_n=10):
    from sklearn.mixture import BayesianGaussianMixture as DPGMM
    dpgmm = DPGMM(
        n_components=max_cluster_n, covariance_type='full', weight_concentration_prior=1e-3,
        weight_concentration_prior_type='dirichlet_process', init_params="kmeans",
        max_iter=300, random_state=0, verbose=0, verbose_interval=10) # init can be "kmeans" or "random"
    dpgmm.fit(data)
    label = dpgmm.predict(data)
    return label
'''

dpgmm_exec = 'label = dpgmm({})'

class multicore(object):
    """docstring for cpu"""
    def __init__(self, cpu_No):
        super(multicore, self).__init__()
        self.cpu_No = cpu_No
        self.cpu = ipp.Client()
        self.dview = self.cpu[:]
        self.dview.block = False
        self.cpu_ready = {}

    def set_data(self, cpu_id, dict):
        self.cpu[cpu_id].push(dict)

    def _import(self, cpu_id, func_def):
        self.cpu[cpu_id].execute(func_def)

    def _exec(self, cpu_id, func_exe, *args):
        cmd = func_exe.format(args[0])
        self.cpu_ready[cpu_id] = self.cpu[cpu_id].execute(cmd)

    def test_cpu_ready(self, cpu_id):
        return self.cpu_ready[cpu_id].ready()

    @property
    def cpu_ready_list(self):
        try:
            return [self.test_cpu_ready(cpu_id) for cpu_id in range(self.cpu_No)]
        except:
            return [True]*self.cpu_No

    def get(self, cpu_id, var):
        return self.cpu[cpu_id].pull(var).get()

    def __getitem__(self, cpu_id):
        return self.cpu[cpu_id]

class DPGMM_IPY(multicore):
    """docstring for DPGMM_multicore"""
    def __init__(self, cpu_No):
        super(DPGMM_IPY, self).__init__(cpu_No)
        for i in range(cpu_No):
            self._import(i, dpgmm_def)

    def __call__(self, fet):
        for i in range(self.cpu_No):
            self.set_data(i, {'data': fet[i]})

    def run(self, group_id):
        self._exec(group_id, dpgmm_exec, 4)

    def run_all(self):
        for i in range(self.cpu_No):
            self._exec(i, dpgmm_exec, 4)

def report_cpu_status(ev):
    group_id = dpgmm[0]['group_id']
    print('cpu ready list:', dpgmm.cpu_ready_list)
    if dpgmm.cpu_ready_list[0]:
        dpgmm[0]['data'] = fet[group_id]
        dpgmm[0].execute('label = dpgmm(4)')
        labels = dpgmm[0]['label']
        print(group_id, labels)
    dpgmm[0]['group_id'] += 1
    if dpgmm[0]['group_id'] == group_No:
        timer.stop()
        app.quit()

if __name__ == '__main__':
    # 1. Test base class multicore
    # backend = multicore(cpu_No=8)
    # for i in range(8):
    #     backend.set_data(i, {'data': np.random.randn(10000,4)})
    #     backend._import(i, dpgmm_def)
    #     backend._exec(i, dpgmm_exec, 4)

    # for i in range(8):
    #     clu = backend.get(i, 'label')
    #     ar  = backend.test_cpu_ready(i)
    #     print('cpu'+str(i), clu)
    #     print('cpu'+str(i), ar)

    # 2. Test sequential background execution
    # cpu_No = 1
    # group_No = 8
    # fet = np.random.randn(group_No, 10000, 4)
    # dpgmm = DPGMM_IPY(cpu_No=cpu_No)
    # dpgmm(fet)
    # dpgmm[0]['group_id'] = 0
    # timer = app.Timer(connect=report_cpu_status, interval=0.5)
    # timer.start()
    # app.run()

    # 3. Test parallel background execution
    cpu_No = 8
    group_No = 8
    fet = np.random.randn(group_No, 10000, 4)
    dpgmm = DPGMM_IPY(cpu_No=cpu_No)
    dpgmm(fet)
    dpgmm.run_all()
    print('cpu ready list:', dpgmm.cpu_ready_list)
    for i in range(dpgmm.cpu_No):
        print(dpgmm[i]['label'])
    print('cpu ready list:', dpgmm.cpu_ready_list)<commit_msg>DPGMM_IPY report_cpu_status<commit_after>import ipyparallel as ipp
import numpy as np
from vispy import app

dpgmm_def = '''
def dpgmm(max_cluster_n=10):
    from sklearn.mixture import BayesianGaussianMixture as DPGMM
    dpgmm = DPGMM(
        n_components=max_cluster_n, covariance_type='full', weight_concentration_prior=1e-3,
        weight_concentration_prior_type='dirichlet_process', init_params="kmeans",
        max_iter=300, random_state=0, verbose=0, verbose_interval=10) # init can be "kmeans" or "random"
    dpgmm.fit(data)
    label = dpgmm.predict(data)
    return label
'''

dpgmm_exec = 'label = dpgmm({})'

class multicore(object):
    """docstring for cpu"""
    def __init__(self, cpu_No):
        super(multicore, self).__init__()
        self.cpu_No = cpu_No
        self.cpu = ipp.Client()
        self.dview = self.cpu[:]
        self.dview.block = False
        self.cpu_ready = {}

    def set_data(self, cpu_id, dict):
        self.cpu[cpu_id].push(dict)

    def _import(self, cpu_id, func_def):
        self.cpu[cpu_id].execute(func_def)

    def _exec(self, cpu_id, func_exe, *args):
        cmd = func_exe.format(args[0])
        self.cpu_ready[cpu_id] = self.cpu[cpu_id].execute(cmd)

    def test_cpu_ready(self, cpu_id):
        return self.cpu_ready[cpu_id].ready()

    @property
    def cpu_ready_list(self):
        try:
            return [self.test_cpu_ready(cpu_id) for cpu_id in range(self.cpu_No)]
        except:
            return [True]*self.cpu_No

    def get(self, cpu_id, var):
        return self.cpu[cpu_id].pull(var).get()

    def __getitem__(self, cpu_id):
        return self.cpu[cpu_id]

class DPGMM_IPY(multicore):
    """docstring for DPGMM_multicore"""
    def __init__(self, cpu_No):
        super(DPGMM_IPY, self).__init__(cpu_No)
        for i in range(cpu_No):
            self._import(i, dpgmm_def)

    def __call__(self, fet):
        for i in range(self.cpu_No):
            self.set_data(i, {'data': fet[i]})

    def run(self, group_id, nclu_max=8):
        self._exec(group_id, dpgmm_exec, nclu_max)

    def run_all(self, nclu_max=8):
        for i in range(self.cpu_No):
            self._exec(i, dpgmm_exec, nclu_max)

def sequential_clustering(ev):
    group_id = dpgmm[0]['group_id']
    print('cpu ready list:', dpgmm.cpu_ready_list)
    if dpgmm.cpu_ready_list[0]:
        dpgmm[0]['data'] = fet[group_id]
        dpgmm[0].execute('label = dpgmm(4)')
        labels = dpgmm[0]['label']
        print(group_id, labels)
    dpgmm[0]['group_id'] += 1
    if dpgmm[0]['group_id'] == group_No:
        timer.stop()
        app.quit()

def report_cpu_status(ev):
    print('cpu ready list:', dpgmm.cpu_ready_list)
    if np.all(dpgmm.cpu_ready_list):
        app.quit()

if __name__ == '__main__':
    # 1. Test base class multicore
    # backend = multicore(cpu_No=8)
    # for i in range(8):
    #     backend.set_data(i, {'data': np.random.randn(10000,4)})
    #     backend._import(i, dpgmm_def)
    #     backend._exec(i, dpgmm_exec, 4)

    # for i in range(8):
    #     clu = backend.get(i, 'label')
    #     ar  = backend.test_cpu_ready(i)
    #     print('cpu'+str(i), clu)
    #     print('cpu'+str(i), ar)

    # 2. Test sequential background execution
    # cpu_No = 1
    # group_No = 8
    # fet = np.random.randn(group_No, 10000, 4)
    # dpgmm = DPGMM_IPY(cpu_No=cpu_No)
    # dpgmm(fet)
    # dpgmm[0]['group_id'] = 0
    # timer = app.Timer(connect=report_cpu_status, interval=0.5)
    # timer.start()
    # app.run()

    # 3. Test parallel background execution
    cpu_No = 8
    group_No = 8
    fet = np.random.randn(group_No, 10000, 4)
    dpgmm = DPGMM_IPY(cpu_No=cpu_No)
    dpgmm(fet)
    dpgmm.run_all()
    timer = app.Timer(connect=report_cpu_status, interval=0.3)
    timer.start()
    app.run()

    # print('cpu ready list:', dpgmm.cpu_ready_list)
    # for i in range(dpgmm.cpu_No):
    #     print(dpgmm[i]['label'])
    # print('cpu ready list:', dpgmm.cpu_ready_list)<|endoftext|>


<commit_before>from __future__ import absolute_import
from __future__ import division
from __future__ import unicode_literals

import codecs
import datetime
import errno
import hashlib
import os
import shutil
import tempfile

try:
    from urllib.parse import quote as urlquote
except ImportError:
    from urllib import quote as urlquote

from collections import Mapping, Iterable

from twisted.internet import defer

def is_delta(item):
    """
    Returns True if the given item looks like a delta message.
    """

    if not isinstance(item, Mapping):
        return False

    if not 'inserts' in item or not isinstance(item['inserts'], Iterable):
        return False

    if not 'deletes' in item or not isinstance(item['deletes'], Iterable):
        return False

    if not 'data' in item or not isinstance(item['data'], Mapping):
        return False

    return True

def is_delta_empty(item):
    return len(item['inserts']) == 0 and len(item['deletes']) == 0

class ExtractorBase(object):
    """
    Abstract base class for extractors.
    """

    def __call__(self, item, send):
        if is_delta(item) and not is_delta_empty(item):
            for oid in item['inserts']:
                self.extract(oid, item['data'][oid])

        send(item, self)

    def extract(self, key, doc):
        """
        Change the incoming insertable document.
        """
        raise NotImplementedError("Subclasses must implement this method")

class Extractor(ExtractorBase):

    def __init__(self, func, *args, **kwds):
        self._extract_func = func
        self._extract_args = args
        self._extract_kwds = kwds

    def extract(self, key, doc):
        self._extract_func(key, doc, *self._extract_args, **self._extract_kwds)

class Filter(object):

    def __init__(self, filter):
        self.filter = filter
        self._keys = set()

    def __call__(self, item, send):
        if is_delta(item) and not is_delta_empty(item):
            self._filter(item)
            if not is_delta_empty(item):
                send(item, self)

    def _filter(self, msg):
        """
        Applies the filter to a delta item.
        """
        orig_deletes = msg.pop('deletes')
        orig_inserts = msg.pop('inserts')
        orig_data = msg.pop('data')

        # Compute set of effectively deleted keys.
        keys_deleted = set(orig_deletes).intersection(self._keys)

        # Filter inserted data.
        data = {oid: orig_data[oid] for oid in orig_inserts if self.filter(oid, orig_data[oid])}

        # List of keys effectively processed.
        keys_inserted = set(data.keys())

        # Also send delete-events for keys which are modified by this message.
        keys_affected = keys_deleted.union(self._keys.intersection(keys_inserted))
        self._keys.difference_update(keys_affected - keys_inserted)
        self._keys.update(keys_inserted - keys_affected)

        # Reconstruct the message.
        msg['deletes'] = list(keys_affected)
        msg['inserts'] = list(keys_inserted)
        msg['data'] = data

class MapReduce(object):

    @staticmethod
    def map_default(key, value):
        yield key, value

    @staticmethod
    def map_chain(*map_funcs):
        def map_func(key, value):
            for f in map_funcs:
                for i in f(key, value):
                    yield i

        return map_func

    @staticmethod
    def reduce_default(key, values):
        return {k: v for doc in values for k, v in doc.items()}

    @staticmethod
    def finalize_default(key, value):
        return value

    SORT_DEFAULT = {
        'key': lambda triple: triple[0]
    }

    def _dummy_coiterate(self, iterator):
        for x in iterator:
            continue

        return defer.succeed(iterator)

    def __init__(self, map=None, reduce=None, finalize=None, sort=None, coiterate=None):
        self.map = map if map else self.map_default
        self.reduce = reduce if reduce else self.reduce_default
        self.finalize = finalize if finalize else self.finalize_default
        self.sort = sort if sort else self.SORT_DEFAULT

        if coiterate == True:
            from twisted.internet.task import coiterate

        if coiterate:
            self.coiterate = coiterate
        else:
            self.coiterate = self._dummy_coiterate

        self._triples = []
        self._keys = set()

    @defer.inlineCallbacks
    def __call__(self, item, send):
        if is_delta(item) and not is_delta_empty(item):
            yield self.coiterate(self._mapreduce(item))
            if not is_delta_empty(item):
                send(item, self)

    def _mapreduce(self, msg):
        # Remove triples for which we received a delete command.
        keys_deleted = self._delete(msg.pop('deletes'))

        # Compute all triples from the insert-set by applying the map function
        # and store them.
        keys_mapped = set()
        orig_data = msg.pop('data')
        for oid in msg.pop('inserts'):
            # FIXME: Maybe defer to another thread and yield
            tuples = self._map_one(oid, orig_data[oid])
            keys_mapped.update(self._add(oid, tuples))

        # Compute the list of triples which will be used in the subsequent
        # reduction step.
        triples = [t for t in self._triples if t[0] in keys_mapped.union(keys_deleted)]
        triples.sort(**self.sort)

        # Partition the data-set and apply the reduce function.
        buffer = {}
        for key, values in self._generate_partitions(triples):
            try:
                values = buffer[key] + values
            except KeyError:
                pass

            buffer[key] = [self._reduce_one(key, values)] if len(values) > 1 else values
            yield

        # Finalize reduced values and construct the data dictionary.
        data = {key: self.finalize(key, values[0]) for key, values in buffer.items()}

        # List of keys effectively processed.
        keys_inserted = set(data.keys())

        # Also send delete-events for keys which are modified by this message.
        keys_affected = keys_deleted.union(self._keys.intersection(keys_inserted))
        self._keys.difference_update(keys_affected - keys_inserted)
        self._keys.update(keys_inserted - keys_affected)

        # Reconstruct the message.
        msg['deletes'] = list(keys_affected)
        msg['inserts'] = list(keys_inserted)
        msg['data'] = data

    def _map_one(self, key, value):
        return list(self.map(key, value))

    def _reduce_one(self, key, values):
        return self.reduce(key, values)

    def _generate_partitions(self, triples):
        if len(triples) > 0:
            current_key, value, oid_ignored = triples[0]
            current_values = [value]

            for key, value, oid_ignored in triples[1:]:
                if key != current_key:
                    if len(current_values) > 0:
                        yield current_key, current_values
                    current_key = key
                    current_values = []

                current_values.append(value)

            if len(current_values) > 0:
                yield current_key, current_values

    def _delete(self, oids):
        keys_deleted = set([t[0] for t in self._triples if t[2] in oids])
        self._triples = [t for t in self._triples if t[2] not in oids]
        return keys_deleted

    def _add(self, oid, tuples):
        columns = list(zip(*tuples)) + [[oid] * len(tuples)]
        keys_inserted = set(columns[0])
        self._triples += list(zip(*columns))
        return keys_inserted

class SetComputedValue(ExtractorBase):
    def __init__(self, destkey, func):
        self.destkey = destkey
        self.func = func

    def extract(self, key, doc):
        doc[self.destkey] = self.func(key, doc)

class Loadfile(ExtractorBase):
    def __init__(self, key='path', destkey='content', encoding='utf-8'):
        self.key = key
        self.destkey = destkey
        self.encoding = encoding

    def extract(self, key, doc):
        path = doc[self.key]
        with codecs.open(path, encoding=self.encoding, mode='r') as stream:
            doc[self.destkey] = stream.read()

class Savefile(ExtractorBase):
    def __init__(self, key='content', destkey='savepath', encoding='utf-8', clear=False):
        self.key = key
        self.destkey = destkey
        self.encoding = encoding
        self.clear = clear

    def extract(self, key, doc):
        path = doc[self.destkey]
        with codecs.open(path, encoding=self.encoding, mode='w') as stream:
            stream.write(doc[self.key])
        if self.clear:
            del doc[self.key]

class Symlink(ExtractorBase):
    def __init__(self, key='path', destkey='linkpath'):
        self.key = key
        self.destkey = destkey

    def extract(self, key, doc):
        path = doc[self.key]
        linkpath = doc[self.destkey]
        os.symlink(path, linkpath)

class Fileurl(ExtractorBase):
    def __init__(self, key='savepath', destkey='content_url', basedir='', baseurl=''):
        self.key = key
        self.destkey = destkey
        self.basedir = basedir
        self.baseurl = baseurl

    def extract(self, key, doc):
        assert doc[self.key].startswith(self.basedir), 'Cannot generate URLs for files outside of the basedir'
        relpath = doc[self.key][len(self.basedir):].lstrip('/')
        doc[self.destkey] = self.baseurl + urlquote(relpath.encode('utf-8'))

class ContentHash(ExtractorBase):
    def __init__(self, key='content', destkey='content_hash', encoding='utf-8', hashalgo='sha1', hashseed=None):
        self.key = key
        self.destkey = destkey
        self.encoding = encoding
        self.hashalgo = hashalgo
        self.hashseed = hashseed
        self.hashobj = None

    def extract(self, key, doc):
        if self.hashobj is None:
            self.hashobj = hashlib.new(self.hashalgo)
            if self.hashseed is not None:
                self.hashobj.update(self.hashseed)

        content = doc[self.key]
        if self.encoding is not None:
            content = content.encode('utf-8')

        doc[self.destkey] = hashlib.sha1(content).hexdigest()

class Cachedir(object):
    def __init__(self, directory=None, destkey='cachedir', hashalgo='sha1', hashseed=None):
        self.directory = directory
        self.destkey = destkey
        self.hashalgo = hashalgo
        self.hashseed = hashseed
        self.hashobj = None
        self.slicelen = 2
        self._dirclean = False

    def __call__(self, item, send):
        if is_delta(item):
            if self.directory is None:
                self.directory = tempfile.mkdtemp()
                self._dirclean = True

            oids = set(item['deletes'] + item['inserts'])
            pathmap = self._generate_pathmap(self.directory, oids)

            for oid, path in pathmap.items():
                shutil.rmtree(path, ignore_errors=(oid in item['inserts']))

            for oid in item['inserts']:
                path = pathmap[oid]
                os.makedirs(path)
                item['data'][oid][self.destkey] = path

        send(item, self)

    def detach(self):
        if self._dirclean:
            shutil.rmtree(self.directory, ignore_errors=True)

    def _generate_pathmap(self, basedir, oids):
        result = {}

        if self.hashobj is None:
            self.hashobj = hashlib.new(self.hashalgo)
            if self.hashseed is not None:
                self.hashobj.update(self.hashseed)

        for oid in oids:
            hashobj = self.hashobj.copy()
            hashobj.update(str(oid).encode('utf-8'))
            digest = hashobj.hexdigest()
            outer, inner = digest[:self.slicelen], digest[self.slicelen:]
            result[oid] = os.path.join(basedir, outer, inner)

        return result

class LockError(RuntimeError):
    pass

class _Lockfile:
    """A separate object allowing proper closing of a temporary file's
    underlying file descriptor
    """

    fd = None
    close_called = False

    @classmethod
    def open(cls, lockpath):
        try:
            fd = os.open(lockpath, os.O_CREAT | os.O_EXCL | os.O_WRONLY, 0o600)
            return cls(fd, lockpath)
        except OSError as err:
            if err.errno in [errno.EAGAIN, errno.EEXIST, errno.EDEADLK, errno.ENOENT]:
                raise LockError()
            else:
                raise

    # Cache the unlinker so we don't get spurious errors at
    # shutdown when the module-level "os" is None'd out.
    def close(self, os_unlink=os.unlink, os_close=os.close):
        if not self.close_called and self.fd is not None:
            self.close_called = True
            try:
                os_close(self.fd)
            finally:
                os_unlink(self.lockpath)

    def __init__(self, fd, lockpath):
        self.fd = fd
        self.lockpath = lockpath

    # Need to ensure the file is deleted on __del__
    def __del__(self):
        self.close()

class LockingProcessor(object):
    def __init__(self, key='lockpath'):
        # oid -> lock.
        self._acquired = {}

        # list of (oid, doc).
        self._pending = []

        self.key = key
        self.out = object()
        self.out_retry = object()
        self.out_locked = object()

    def __call__(self, item, send):
        data = {}
        pending = []

        # Try to acquire a lockfile for all pending and new inserts.
        for oid, doc in self._inserts(item):
            try:
                self._acquired[oid] = _Lockfile.open(doc[self.key])
                data[oid] = doc
            except LockError:
                pending.append((oid, doc))

        # Reconstruct and send the message to the locked output port.
        pending_oids = set(oid for oid, doc in self._pending)
        item['deletes'] = [oid for oid in item['deletes'] if oid not in pending_oids]
        item['inserts'] = list(data.keys())
        item['data'] = data

        if not is_delta_empty(item):
            send(item, self.out_locked)

        # Push to retry output port.
        self._pending = pending
        if len(self._pending) > 0:
            retry_item = {
                'date': datetime.datetime.now(),
                'deletes': (),
                'inserts': (),
                'data': {}
            }
            send(retry_item, self.out_retry)

    def release(self, item, send):
        """
        Release the locks and send the message to the default output port.
        """
        for oid in item['inserts']:
            self._acquired.pop(oid).close()

        send(item, self.out)

    def attach(self, dispatcher, reactor):
        pass

    def detach(self):
        for lockfile in self._acquired.values():
            lockfile.close()

        self._acquired = {}
        self._pending = []

    def _inserts(self, item):
        """
        Iterates over pending and new inserts.
        """
        for oid, doc in self._pending:
            if oid not in item['deletes']:
                yield (oid, doc)

        for oid in item['inserts']:
            doc = item['data'][oid]
            yield (oid, doc)

    @property
    def dependencies(self):
        yield (self, self.out_locked)
        yield (self, self.out_retry)
        yield (self.release, self.out)
<commit_msg>Extract abstract base class<commit_after>from __future__ import absolute_import
from __future__ import division
from __future__ import unicode_literals

import codecs
import datetime
import errno
import hashlib
import os
import shutil
import tempfile

try:
    from urllib.parse import quote as urlquote
except ImportError:
    from urllib import quote as urlquote

from collections import Mapping, Iterable

from twisted.internet import defer

def is_delta(item):
    """
    Returns True if the given item looks like a delta message.
    """

    if not isinstance(item, Mapping):
        return False

    if not 'inserts' in item or not isinstance(item['inserts'], Iterable):
        return False

    if not 'deletes' in item or not isinstance(item['deletes'], Iterable):
        return False

    if not 'data' in item or not isinstance(item['data'], Mapping):
        return False

    return True

def is_delta_empty(item):
    return len(item['inserts']) == 0 and len(item['deletes']) == 0

class ExtractorBase(object):
    """
    Abstract base class for extractors.
    """

    def __call__(self, item, send):
        if is_delta(item) and not is_delta_empty(item):
            for oid in item['inserts']:
                self.extract(oid, item['data'][oid])

        send(item, self)

    def extract(self, key, doc):
        """
        Change the incoming insertable document.
        """
        raise NotImplementedError("Subclasses must implement this method")

class Extractor(ExtractorBase):

    def __init__(self, func, *args, **kwds):
        self._extract_func = func
        self._extract_args = args
        self._extract_kwds = kwds

    def extract(self, key, doc):
        self._extract_func(key, doc, *self._extract_args, **self._extract_kwds)

class Filter(object):

    def __init__(self, filter):
        self.filter = filter
        self._keys = set()

    def __call__(self, item, send):
        if is_delta(item) and not is_delta_empty(item):
            self._filter(item)
            if not is_delta_empty(item):
                send(item, self)

    def _filter(self, msg):
        """
        Applies the filter to a delta item.
        """
        orig_deletes = msg.pop('deletes')
        orig_inserts = msg.pop('inserts')
        orig_data = msg.pop('data')

        # Compute set of effectively deleted keys.
        keys_deleted = set(orig_deletes).intersection(self._keys)

        # Filter inserted data.
        data = {oid: orig_data[oid] for oid in orig_inserts if self.filter(oid, orig_data[oid])}

        # List of keys effectively processed.
        keys_inserted = set(data.keys())

        # Also send delete-events for keys which are modified by this message.
        keys_affected = keys_deleted.union(self._keys.intersection(keys_inserted))
        self._keys.difference_update(keys_affected - keys_inserted)
        self._keys.update(keys_inserted - keys_affected)

        # Reconstruct the message.
        msg['deletes'] = list(keys_affected)
        msg['inserts'] = list(keys_inserted)
        msg['data'] = data

class MapReduceBase(object):

    def map(self, key, value):
        yield key, value

    def reduce(self, key, values):
        return {k: v for doc in values for k, v in doc.items()}

    def finalize(self, key, value):
        return value

    SORT_DEFAULT = {
        'key': lambda triple: triple[0]
    }

    def _dummy_coiterate(self, iterator):
        for x in iterator:
            continue

        return defer.succeed(iterator)

    def __init__(self, coiterate=None):
        self.sort = self.SORT_DEFAULT

        if coiterate == True:
            from twisted.internet.task import coiterate

        if coiterate:
            self.coiterate = coiterate
        else:
            self.coiterate = self._dummy_coiterate

        self._triples = []
        self._keys = set()

    @defer.inlineCallbacks
    def __call__(self, item, send):
        if is_delta(item) and not is_delta_empty(item):
            yield self.coiterate(self._mapreduce(item))
            if not is_delta_empty(item):
                send(item, self)

    def _mapreduce(self, msg):
        # Remove triples for which we received a delete command.
        keys_deleted = self._delete(msg.pop('deletes'))

        # Compute all triples from the insert-set by applying the map function
        # and store them.
        keys_mapped = set()
        orig_data = msg.pop('data')
        for oid in msg.pop('inserts'):
            # FIXME: Maybe defer to another thread and yield
            tuples = self._map_one(oid, orig_data[oid])
            keys_mapped.update(self._add(oid, tuples))

        # Compute the list of triples which will be used in the subsequent
        # reduction step.
        triples = [t for t in self._triples if t[0] in keys_mapped.union(keys_deleted)]
        triples.sort(**self.sort)

        # Partition the data-set and apply the reduce function.
        buffer = {}
        for key, values in self._generate_partitions(triples):
            try:
                values = buffer[key] + values
            except KeyError:
                pass

            buffer[key] = [self._reduce_one(key, values)] if len(values) > 1 else values
            yield

        # Finalize reduced values and construct the data dictionary.
        data = {key: self.finalize(key, values[0]) for key, values in buffer.items()}

        # List of keys effectively processed.
        keys_inserted = set(data.keys())

        # Also send delete-events for keys which are modified by this message.
        keys_affected = keys_deleted.union(self._keys.intersection(keys_inserted))
        self._keys.difference_update(keys_affected - keys_inserted)
        self._keys.update(keys_inserted - keys_affected)

        # Reconstruct the message.
        msg['deletes'] = list(keys_affected)
        msg['inserts'] = list(keys_inserted)
        msg['data'] = data

    def _map_one(self, key, value):
        return list(self.map(key, value))

    def _reduce_one(self, key, values):
        return self.reduce(key, values)

    def _generate_partitions(self, triples):
        if len(triples) > 0:
            current_key, value, oid_ignored = triples[0]
            current_values = [value]

            for key, value, oid_ignored in triples[1:]:
                if key != current_key:
                    if len(current_values) > 0:
                        yield current_key, current_values
                    current_key = key
                    current_values = []

                current_values.append(value)

            if len(current_values) > 0:
                yield current_key, current_values

    def _delete(self, oids):
        keys_deleted = set([t[0] for t in self._triples if t[2] in oids])
        self._triples = [t for t in self._triples if t[2] not in oids]
        return keys_deleted

    def _add(self, oid, tuples):
        columns = list(zip(*tuples)) + [[oid] * len(tuples)]
        keys_inserted = set(columns[0])
        self._triples += list(zip(*columns))
        return keys_inserted

class MapReduce(MapReduceBase):

    @staticmethod
    def map_default(key, value):
        yield key, value

    @staticmethod
    def map_chain(*map_funcs):
        def map_func(key, value):
            for f in map_funcs:
                for i in f(key, value):
                    yield i

        return map_func

    @staticmethod
    def reduce_default(key, values):
        return {k: v for doc in values for k, v in doc.items()}

    @staticmethod
    def finalize_default(key, value):
        return value

    def __init__(self, map=None, reduce=None, finalize=None, sort=None, coiterate=None):
        super(MapReduce, self).__init__(coiterate=coiterate)
        self._map_func = map if map else self.map_default
        self._reduce_func = reduce if reduce else self.reduce_default
        self._finalize_func = finalize if finalize else self.finalize_default
        self.sort = sort if sort else self.SORT_DEFAULT

    def map(self, key, value):
        for item in self._map_func(key, value):
            yield item

    def reduce(self, key, values):
        return self._reduce_func(key, values)

    def finalize(self, key, value):
        return self._finalize_func(key, value)

class SetComputedValue(ExtractorBase):
    def __init__(self, destkey, func):
        self.destkey = destkey
        self.func = func

    def extract(self, key, doc):
        doc[self.destkey] = self.func(key, doc)

class Loadfile(ExtractorBase):
    def __init__(self, key='path', destkey='content', encoding='utf-8'):
        self.key = key
        self.destkey = destkey
        self.encoding = encoding

    def extract(self, key, doc):
        path = doc[self.key]
        with codecs.open(path, encoding=self.encoding, mode='r') as stream:
            doc[self.destkey] = stream.read()

class Savefile(ExtractorBase):
    def __init__(self, key='content', destkey='savepath', encoding='utf-8', clear=False):
        self.key = key
        self.destkey = destkey
        self.encoding = encoding
        self.clear = clear

    def extract(self, key, doc):
        path = doc[self.destkey]
        with codecs.open(path, encoding=self.encoding, mode='w') as stream:
            stream.write(doc[self.key])
        if self.clear:
            del doc[self.key]

class Symlink(ExtractorBase):
    def __init__(self, key='path', destkey='linkpath'):
        self.key = key
        self.destkey = destkey

    def extract(self, key, doc):
        path = doc[self.key]
        linkpath = doc[self.destkey]
        os.symlink(path, linkpath)

class Fileurl(ExtractorBase):
    def __init__(self, key='savepath', destkey='content_url', basedir='', baseurl=''):
        self.key = key
        self.destkey = destkey
        self.basedir = basedir
        self.baseurl = baseurl

    def extract(self, key, doc):
        assert doc[self.key].startswith(self.basedir), 'Cannot generate URLs for files outside of the basedir'
        relpath = doc[self.key][len(self.basedir):].lstrip('/')
        doc[self.destkey] = self.baseurl + urlquote(relpath.encode('utf-8'))

class ContentHash(ExtractorBase):
    def __init__(self, key='content', destkey='content_hash', encoding='utf-8', hashalgo='sha1', hashseed=None):
        self.key = key
        self.destkey = destkey
        self.encoding = encoding
        self.hashalgo = hashalgo
        self.hashseed = hashseed
        self.hashobj = None

    def extract(self, key, doc):
        if self.hashobj is None:
            self.hashobj = hashlib.new(self.hashalgo)
            if self.hashseed is not None:
                self.hashobj.update(self.hashseed)

        content = doc[self.key]
        if self.encoding is not None:
            content = content.encode('utf-8')

        doc[self.destkey] = hashlib.sha1(content).hexdigest()

class Cachedir(object):
    def __init__(self, directory=None, destkey='cachedir', hashalgo='sha1', hashseed=None):
        self.directory = directory
        self.destkey = destkey
        self.hashalgo = hashalgo
        self.hashseed = hashseed
        self.hashobj = None
        self.slicelen = 2
        self._dirclean = False

    def __call__(self, item, send):
        if is_delta(item):
            if self.directory is None:
                self.directory = tempfile.mkdtemp()
                self._dirclean = True

            oids = set(item['deletes'] + item['inserts'])
            pathmap = self._generate_pathmap(self.directory, oids)

            for oid, path in pathmap.items():
                shutil.rmtree(path, ignore_errors=(oid in item['inserts']))

            for oid in item['inserts']:
                path = pathmap[oid]
                os.makedirs(path)
                item['data'][oid][self.destkey] = path

        send(item, self)

    def detach(self):
        if self._dirclean:
            shutil.rmtree(self.directory, ignore_errors=True)

    def _generate_pathmap(self, basedir, oids):
        result = {}

        if self.hashobj is None:
            self.hashobj = hashlib.new(self.hashalgo)
            if self.hashseed is not None:
                self.hashobj.update(self.hashseed)

        for oid in oids:
            hashobj = self.hashobj.copy()
            hashobj.update(str(oid).encode('utf-8'))
            digest = hashobj.hexdigest()
            outer, inner = digest[:self.slicelen], digest[self.slicelen:]
            result[oid] = os.path.join(basedir, outer, inner)

        return result

class LockError(RuntimeError):
    pass

class _Lockfile:
    """A separate object allowing proper closing of a temporary file's
    underlying file descriptor
    """

    fd = None
    close_called = False

    @classmethod
    def open(cls, lockpath):
        try:
            fd = os.open(lockpath, os.O_CREAT | os.O_EXCL | os.O_WRONLY, 0o600)
            return cls(fd, lockpath)
        except OSError as err:
            if err.errno in [errno.EAGAIN, errno.EEXIST, errno.EDEADLK, errno.ENOENT]:
                raise LockError()
            else:
                raise

    # Cache the unlinker so we don't get spurious errors at
    # shutdown when the module-level "os" is None'd out.
    def close(self, os_unlink=os.unlink, os_close=os.close):
        if not self.close_called and self.fd is not None:
            self.close_called = True
            try:
                os_close(self.fd)
            finally:
                os_unlink(self.lockpath)

    def __init__(self, fd, lockpath):
        self.fd = fd
        self.lockpath = lockpath

    # Need to ensure the file is deleted on __del__
    def __del__(self):
        self.close()

class LockingProcessor(object):
    def __init__(self, key='lockpath'):
        # oid -> lock.
        self._acquired = {}

        # list of (oid, doc).
        self._pending = []

        self.key = key
        self.out = object()
        self.out_retry = object()
        self.out_locked = object()

    def __call__(self, item, send):
        data = {}
        pending = []

        # Try to acquire a lockfile for all pending and new inserts.
        for oid, doc in self._inserts(item):
            try:
                self._acquired[oid] = _Lockfile.open(doc[self.key])
                data[oid] = doc
            except LockError:
                pending.append((oid, doc))

        # Reconstruct and send the message to the locked output port.
        pending_oids = set(oid for oid, doc in self._pending)
        item['deletes'] = [oid for oid in item['deletes'] if oid not in pending_oids]
        item['inserts'] = list(data.keys())
        item['data'] = data

        if not is_delta_empty(item):
            send(item, self.out_locked)

        # Push to retry output port.
        self._pending = pending
        if len(self._pending) > 0:
            retry_item = {
                'date': datetime.datetime.now(),
                'deletes': (),
                'inserts': (),
                'data': {}
            }
            send(retry_item, self.out_retry)

    def release(self, item, send):
        """
        Release the locks and send the message to the default output port.
        """
        for oid in item['inserts']:
            self._acquired.pop(oid).close()

        send(item, self.out)

    def attach(self, dispatcher, reactor):
        pass

    def detach(self):
        for lockfile in self._acquired.values():
            lockfile.close()

        self._acquired = {}
        self._pending = []

    def _inserts(self, item):
        """
        Iterates over pending and new inserts.
        """
        for oid, doc in self._pending:
            if oid not in item['deletes']:
                yield (oid, doc)

        for oid in item['inserts']:
            doc = item['data'][oid]
            yield (oid, doc)

    @property
    def dependencies(self):
        yield (self, self.out_locked)
        yield (self, self.out_retry)
        yield (self.release, self.out)
<|endoftext|>


<commit_before>from google.appengine.ext import ndb
import webapp2
import json

class Boat(ndb.Model):
    name = ndb.StringProperty(required=True)
    type = ndb.StringProperty()
    length = ndb.IntegerProperty()
    at_sea = ndb.BooleanProperty(default=True)

class Slip(ndb.Model):
    number = ndb.IntegerProperty(required=True)
    #current_boat = ndb.StructuredProperty(Boat)
    current_boat = ndb.StringProperty()
    arrival_date = ndb.StringProperty()

class Fish(ndb.Model):
    name = ndb.StringProperty(required=True)
    numFriends = ndb.IntegerProperty()

# This updates Slip's arrival_date
class SlipHandler2(webapp2.RequestHandler):
    def put(self, id=None):
        s = ndb.Key(urlsafe=id).get()
        slip_data = json.loads(self.request.body)
        s.arrival_date = slip_data['arrival_date']
        s.put()
        self.response.write("wonky")

# updates a slip's boat name and arrival
# put /slip/slip_id/putboat
# {'name' ; "some name", 'arrival_date': "some date"}
class SlipHandler3(webapp2.RequestHandler):
    def put(self, id=None):
        s = ndb.Key(urlsafe=id).get()
        slip_data = json.loads(self.request.body)
        #s.current_boat = Boat("jasmine")
        #s.current_boat = "jasminefoo" # line that breaks
        s.current_boat = slip_data['name']
        s.arrival_date = slip_data['arrival_date']
        s.put()

        # also the boat you put in the slip should not be at sea
        b = ndb.Key(urlsafe=s.current_boat).get() # this should get the boat
        b.at_sea = False  # change to at_sea is False
        b.put()
        self.response.write("you changed the boat in the slip")

class SlipHandler(webapp2.RequestHandler):
    def post(self, id=None):
        slip_data = json.loads(self.request.body)
        new_slip = Slip(number=slip_data['number'])
        new_slip.put()
        slip_dict = new_slip.to_dict()
        slip_dict['self'] = '/slip/' + new_slip.key.urlsafe()
        self.response.write(json.dumps(slip_dict))

    # to delete a slip /slip/slip_id/
    # make it so that if a slip is occupied
    # that the boat becomes 'at_sea' = true
    def delete(self, id=None):
        if id:
            s = ndb.Key(urlsafe=id).get()
            s_d = s.to_dict()
            #s_d['self'] = "/slip/" + id
            #self.response.write(s_d)
            if s.current_boat != None:   ##### OMG its NOne not null. stupid!
                #boat_data = json.loads(self.request.body) # get the boat data
                b = ndb.Key(urlsafe=s.current_boat).get() # this should get the boat
                b.at_sea = True  # change to at_sea is true
                b.put()
                s.key.delete() # this deletes the slip
                self.response.write(id)

        else:
            self.response.write("You cannot delete a slip with no id")

    def get(self, id=None):
        if id:
            s = ndb.Key(urlsafe=id).get()
            s_d = s.to_dict()
            s_d['self'] = "/slip/" + id
            #self.response.write(s.key())
            self.response.write(json.dumps(s_d))
        else:

            #results = Slip.query().fetch()
            get_slip_query_results = [get_slip_query.to_dict()
                                      for get_slip_query in Slip.query()]

            self.response.write(json.dumps(get_slip_query_results))

class BoatHandler(webapp2.RequestHandler):
    def post(self):
        boat_data = json.loads(self.request.body)
        #new_boat = Boat(name=boat_data['name'], length=boat_data['length'])
        if 'type' not in boat_data:
            if 'length' not in boat_data:
                # means name only in data
                new_boat = Boat(name=boat_data['name'])
            else:
                # means name + length
                new_boat = Boat(name=boat_data['name'], length=boat_data['length'])
        elif 'length' not in boat_data:
            # name + type
            new_boat = Boat(name=boat_data['name'], type=boat_data['type'])
        else:
            # name + type + length
            new_boat = Boat(name=boat_data['name'], type=boat_data['type'], length=boat_data['length'])

        new_boat.put()
        boat_dict = new_boat.to_dict()
        boat_dict['self'] = '/boat/' + new_boat.key.urlsafe()
        self.response.write(json.dumps(boat_dict))

    def get(self, id=None):
        if id:
            b = ndb.Key(urlsafe=id).get()
            b_d = b.to_dict()
            b_d['self'] = "/boat/" + id
            self.response.write(json.dumps(b_d))
        else:
            get_boat_query_results = [get_boat_query.to_dict()
                                      for get_boat_query in Boat.query()]

            self.response.write(json.dumps(get_boat_query_results))

    def delete(self, id=None):
        if id:
            b = ndb.Key(urlsafe=id).get()
            b_d = b.to_dict()
            b.key.delete()

        else:
            self.response.write("You cannot delete a Boat with no id")

    # for boat to change at_sea
    # have to use {"at_sea": false}  Note the small f Python uses F
    #  need to update so that putting boat at_sea empties out the slip
    def put(self, id=None):
        boat_data = json.loads(self.request.body)
        if id:
            b = ndb.Key(urlsafe=id).get()
            if boat_data['at_sea'] == False:
                b.at_sea = False
                b.put()
            elif boat_data['at_sea'] == True:
                b.at_sea = True
                b.put()
            #b.at_sea = boat_data['at_sea']
            #b.put()
            self.response.write("you updated at sea to ")

class FishHandler(webapp2.RequestHandler):
    def post(self):
        fish_data = json.loads(self.request.body)
        new_fish = Fish(name=fish_data['name'])
        new_fish.put()
        fish_dict = new_fish.to_dict()
        fish_dict['self'] = '/fish/' + new_fish.key.urlsafe()
        self.response.write(json.dumps(fish_dict))

    def get(self, id=None):
        if id:
            f = ndb.Key(urlsafe=id).get()
            f_d = f.to_dict()
            f_d['self'] = "/fish/" + id
            self.response.write(json.dumps(f_d))
        else:
            self.response.write("There are no fish")

class MainPage(webapp2.RequestHandler):
    def get(self):
        self.response.headers['Content-Type'] = 'text/plain'
        self.response.write('Hello, World!')

app = webapp2.WSGIApplication([
    ('/', MainPage),
    ('/fish', FishHandler),
    ('/fish/(.*)', FishHandler),
    ('/boat/(.*)', BoatHandler),
    ('/boat', BoatHandler),
    ('/slip/(.*)/date', SlipHandler2),
    ('/slip/(.*)/putboat', SlipHandler3),
    ('/slip', SlipHandler),
    ('/slip/(.*)', SlipHandler),

], debug=True)
<commit_msg>can move boat to the sea for 2<commit_after>from google.appengine.ext import ndb
import webapp2
import json

class Boat(ndb.Model):
    name = ndb.StringProperty(required=True)
    type = ndb.StringProperty()
    length = ndb.IntegerProperty()
    at_sea = ndb.BooleanProperty(default=True)

class Slip(ndb.Model):
    number = ndb.IntegerProperty(required=True)
    #current_boat = ndb.StructuredProperty(Boat)
    current_boat = ndb.StringProperty()
    arrival_date = ndb.StringProperty()

class Fish(ndb.Model):
    name = ndb.StringProperty(required=True)
    numFriends = ndb.IntegerProperty()

# This updates Slip's arrival_date
class SlipHandler2(webapp2.RequestHandler):
    def put(self, id=None):
        s = ndb.Key(urlsafe=id).get()
        slip_data = json.loads(self.request.body)
        s.arrival_date = slip_data['arrival_date']
        s.put()
        self.response.write("wonky")

# updates a slip's boat name and arrival
# put /slip/slip_id/putboat
# {'name' ; "some name", 'arrival_date': "some date"}
class SlipHandler3(webapp2.RequestHandler):
    def put(self, id=None):
        s = ndb.Key(urlsafe=id).get()
        slip_data = json.loads(self.request.body)
        #s.current_boat = Boat("jasmine")
        #s.current_boat = "jasminefoo" # line that breaks
        s.current_boat = slip_data['name']
        s.arrival_date = slip_data['arrival_date']
        s.put()

        # also the boat you put in the slip should not be at sea
        b = ndb.Key(urlsafe=s.current_boat).get() # this should get the boat
        b.at_sea = False  # change to at_sea is False
        b.put()
        self.response.write("you changed the boat in the slip")

class SlipHandler(webapp2.RequestHandler):
    def post(self, id=None):
        slip_data = json.loads(self.request.body)
        new_slip = Slip(number=slip_data['number'])
        new_slip.put()
        slip_dict = new_slip.to_dict()
        slip_dict['self'] = '/slip/' + new_slip.key.urlsafe()
        self.response.write(json.dumps(slip_dict))

    # to delete a slip /slip/slip_id/
    # make it so that if a slip is occupied
    # that the boat becomes 'at_sea' = true
    def delete(self, id=None):
        if id:
            s = ndb.Key(urlsafe=id).get()
            s_d = s.to_dict()
            #s_d['self'] = "/slip/" + id
            #self.response.write(s_d)
            if s.current_boat != None:   ##### OMG its NOne not null. stupid!
                #boat_data = json.loads(self.request.body) # get the boat data
                b = ndb.Key(urlsafe=s.current_boat).get() # this should get the boat
                b.at_sea = True  # change to at_sea is true
                b.put()
                s.key.delete() # this deletes the slip
                self.response.write(id)

        else:
            self.response.write("You cannot delete a slip with no id")

    def get(self, id=None):
        if id:
            s = ndb.Key(urlsafe=id).get()
            s_d = s.to_dict()
            s_d['self'] = "/slip/" + id
            #self.response.write(s.key())
            self.response.write(json.dumps(s_d))
        else:

            #results = Slip.query().fetch()
            get_slip_query_results = [get_slip_query.to_dict()
                                      for get_slip_query in Slip.query()]

            self.response.write(json.dumps(get_slip_query_results))

# this class for boat handles (2) move boat to the sea
# and free up the slip
class BoatHandler2(webapp2.RequestHandler):
    def put(self, id=None):
        # this changes the boat to the sea
        if id:
            b = ndb.Key(urlsafe=id).get()
            if b.at_sea == False:
                b.at_sea = True # set to true, moved boat to the sea
                b.put()
                self.response.write("moved boat to the sea")
            else:
                self.response.write("boat already at sea")

class BoatHandler(webapp2.RequestHandler):
    def post(self):
        boat_data = json.loads(self.request.body)
        #new_boat = Boat(name=boat_data['name'], length=boat_data['length'])
        if 'type' not in boat_data:
            if 'length' not in boat_data:
                # means name only in data
                new_boat = Boat(name=boat_data['name'])
            else:
                # means name + length
                new_boat = Boat(name=boat_data['name'], length=boat_data['length'])
        elif 'length' not in boat_data:
            # name + type
            new_boat = Boat(name=boat_data['name'], type=boat_data['type'])
        else:
            # name + type + length
            new_boat = Boat(name=boat_data['name'], type=boat_data['type'], length=boat_data['length'])

        new_boat.put()
        boat_dict = new_boat.to_dict()
        boat_dict['self'] = '/boat/' + new_boat.key.urlsafe()
        self.response.write(json.dumps(boat_dict))

    def get(self, id=None):
        if id:
            b = ndb.Key(urlsafe=id).get()
            b_d = b.to_dict()
            b_d['self'] = "/boat/" + id
            self.response.write(json.dumps(b_d))
        else:
            get_boat_query_results = [get_boat_query.to_dict()
                                      for get_boat_query in Boat.query()]

            self.response.write(json.dumps(get_boat_query_results))

    def delete(self, id=None):
        if id:
            b = ndb.Key(urlsafe=id).get()
            b_d = b.to_dict()
            b.key.delete()

        else:
            self.response.write("You cannot delete a Boat with no id")

    # for boat to change at_sea
    # have to use {"at_sea": false}  Note the small f Python uses F
    #  need to update so that putting boat at_sea empties out the slip
    def put(self, id=None):
        boat_data = json.loads(self.request.body)
        if id:
            b = ndb.Key(urlsafe=id).get()
            if boat_data['at_sea'] == False:
                b.at_sea = False
                b.put()
            elif boat_data['at_sea'] == True:
                b.at_sea = True
                b.put()
            #b.at_sea = boat_data['at_sea']
            #b.put()
            self.response.write("you updated at sea to ")

class FishHandler(webapp2.RequestHandler):
    def post(self):
        fish_data = json.loads(self.request.body)
        new_fish = Fish(name=fish_data['name'])
        new_fish.put()
        fish_dict = new_fish.to_dict()
        fish_dict['self'] = '/fish/' + new_fish.key.urlsafe()
        self.response.write(json.dumps(fish_dict))

    def get(self, id=None):
        if id:
            f = ndb.Key(urlsafe=id).get()
            f_d = f.to_dict()
            f_d['self'] = "/fish/" + id
            self.response.write(json.dumps(f_d))
        else:
            self.response.write("There are no fish")

class MainPage(webapp2.RequestHandler):
    def get(self):
        self.response.headers['Content-Type'] = 'text/plain'
        self.response.write('Hello, World!')

app = webapp2.WSGIApplication([

    ('/fish', FishHandler),
    ('/fish/(.*)', FishHandler),
    ('/boat/(.*)/tosea', BoatHandler2),
    ('/boat/(.*)', BoatHandler),
    ('/boat', BoatHandler),
    ('/slip/(.*)/date', SlipHandler2),
    ('/slip/(.*)/putboat', SlipHandler3),
    ('/slip', SlipHandler),
    ('/slip/(.*)', SlipHandler),
    ('/', MainPage),

], debug=True)
<|endoftext|>


<commit_before>###########################################################################
#          (C) Vrije Universiteit, Amsterdam (the Netherlands)            #
#                                                                         #
# This file is part of AmCAT - The Amsterdam Content Analysis Toolkit     #
#                                                                         #
# AmCAT is free software: you can redistribute it and/or modify it under  #
# the terms of the GNU Affero General Public License as published by the  #
# Free Software Foundation, either version 3 of the License, or (at your  #
# option) any later version.                                              #
#                                                                         #
# AmCAT is distributed in the hope that it will be useful, but WITHOUT    #
# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or   #
# FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero General Public     #
# License for more details.                                               #
#                                                                         #
# You should have received a copy of the GNU Affero General Public        #
# License along with AmCAT.  If not, see <http://www.gnu.org/licenses/>.  #
###########################################################################

from django.conf.urls import patterns, url, include
from annotator.views import codingjob, overview

article_patterns = patterns('',
    url(r'^sentences$', codingjob.articleSentences, name='annotator-article-sentences'),
    url(r'^articlecodings$', codingjob.articleCodings, name='annotator-article-codings'),
    url(r'^unitcodings$', codingjob.unitCodings, name='annotator-unit-codings'),
    url(r'^storecodings$', codingjob.storeCodings, name='annotator-store-codings'),
)

codingjob_patterns = patterns('',
    url('^$', codingjob.index),

    url(r'^fields$', codingjob.fields, name='annotator-codingjob-fields'),
    url(r'^articles$', codingjob.articles, name='annotator-codingjob-articles'),
    url(r'^article/(?P<article_id>\d+)/', include(article_patterns)),
)

urlpatterns = patterns('',
    url(r'^$', overview.index),
    url(r'^overview$', overview.index, name='annotator-overview'),
    url(r"^codingjob/(?P<codingjob_id>\d+)$", codingjob.redirect, name="annotator-codingjob"),
    url(r"project/(?P<project_id>\d+)/codingjob/(?P<codingjob_id>\d+)/", include(codingjob_patterns)),
)
<commit_msg>Deleted entry from urls.py, as view has been removed.<commit_after>###########################################################################
#          (C) Vrije Universiteit, Amsterdam (the Netherlands)            #
#                                                                         #
# This file is part of AmCAT - The Amsterdam Content Analysis Toolkit     #
#                                                                         #
# AmCAT is free software: you can redistribute it and/or modify it under  #
# the terms of the GNU Affero General Public License as published by the  #
# Free Software Foundation, either version 3 of the License, or (at your  #
# option) any later version.                                              #
#                                                                         #
# AmCAT is distributed in the hope that it will be useful, but WITHOUT    #
# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or   #
# FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero General Public     #
# License for more details.                                               #
#                                                                         #
# You should have received a copy of the GNU Affero General Public        #
# License along with AmCAT.  If not, see <http://www.gnu.org/licenses/>.  #
###########################################################################

from django.conf.urls import patterns, url, include
from annotator.views import codingjob, overview

article_patterns = patterns('',
    url(r'^articlecodings$', codingjob.articleCodings, name='annotator-article-codings'),
    url(r'^unitcodings$', codingjob.unitCodings, name='annotator-unit-codings'),
    url(r'^storecodings$', codingjob.storeCodings, name='annotator-store-codings'),
)

codingjob_patterns = patterns('',
    url('^$', codingjob.index),

    url(r'^fields$', codingjob.fields, name='annotator-codingjob-fields'),
    url(r'^articles$', codingjob.articles, name='annotator-codingjob-articles'),
    url(r'^article/(?P<article_id>\d+)/', include(article_patterns)),
)

urlpatterns = patterns('',
    url(r'^$', overview.index),
    url(r'^overview$', overview.index, name='annotator-overview'),
    url(r"^codingjob/(?P<codingjob_id>\d+)$", codingjob.redirect, name="annotator-codingjob"),
    url(r"project/(?P<project_id>\d+)/codingjob/(?P<codingjob_id>\d+)/", include(codingjob_patterns)),
)
<|endoftext|>


<commit_before>"""Tests for appengine.produced_item."""

import os.path
import unittest

from factorio import names
from factorio import recipe

from appengine import icons
from appengine import produced_item

class TestProducedItem(unittest.TestCase):
    def setUp(self):
        icons.init(os.path.join(os.path.dirname(__file__), 'icons'))
        names.names_from_json(
            open(os.path.join(os.path.dirname(__file__),
                              'test-names.json'), 'r'))
        recipe.Recipe.recipes_from_json(
            open(os.path.join(os.path.dirname(__file__),
                              'test-recipe.json'), 'r'))

    def test_base_product(self):
        rates = recipe.calculate_required_production_rates([('copper-ore', 5)])

        copper_ore_item = (
            produced_item.required_production_rates_to_produced_items(rates)[0])
        self.assertEqual(copper_ore_item.is_user_requested, True)
        self.assertEqual(copper_ore_item.name, 'copper-ore')
        self.assertEqual(copper_ore_item.username, 'Copper ore')
        self.assertEqual(copper_ore_item.icon,
                         '/appengine/tests/icons/copper-ore.png')
        self.assertEqual(
            copper_ore_item.url,
            'http://www.factorioforums.com/wiki/index.php?title=Copper_ore')
        self.assertEqual(copper_ore_item.required_production_rate, 5)
        self.assertIsNone(copper_ore_item.production_machine_username)
        self.assertIsNone(copper_ore_item.num_production_machines_required)
        self.assertIsNone(copper_ore_item.production_machine_icon)
        self.assertIsNone(copper_ore_item.production_machine_url)

    def test_smelting_product(self):
        rates = recipe.calculate_required_production_rates(
            [('copper-plate', 5)])

        copper_plate_item, copper_ore_item = (
            produced_item.required_production_rates_to_produced_items(rates))
        self.assertEqual(copper_plate_item.is_user_requested, True)
        self.assertEqual(copper_plate_item.name, 'copper-plate')
        self.assertEqual(copper_plate_item.username, 'Copper plate')
        self.assertEqual(copper_plate_item.icon,
                         '/appengine/tests/icons/copper-plate.png')
        self.assertEqual(
            copper_plate_item.url,
            'http://www.factorioforums.com/wiki/index.php?title=Copper_plate')
        self.assertEqual(copper_plate_item.required_production_rate, 5)
        self.assertEqual(copper_plate_item.production_machine_username,
                         'Electric furnace')
        # <rate> = <#-machines> * <#-recipe-results> * <machine-speed>
        #          ---------------------------------------------------
        #                        <energy-required>
        # 5 = x * 1 * 2 / 3.5
        # 5 = 4/7x
        # (7/4)5 = x
        # x = 8.75
        self.assertEqual(copper_plate_item.num_production_machines_required, 9)
        self.assertEqual(
            copper_plate_item.production_machine_icon,
            '/appengine/tests/icons/electric-furnace.png')
        self.assertEqual(
            copper_plate_item.production_machine_url,
            'http://www.factorioforums.com/wiki/index.php?title='
            'Electric_furnace')

        self.assertEqual(copper_ore_item.is_user_requested, False)
        self.assertEqual(copper_ore_item.name, 'copper-ore')

    def test_chemistry_product(self):
        rates = recipe.calculate_required_production_rates(
            [('sulfur', 100)])

        sulfur_item, petrolium_gas_item, water_item = (
            produced_item.required_production_rates_to_produced_items(rates))
        self.assertEqual(sulfur_item.is_user_requested, True)
        self.assertEqual(sulfur_item.name, 'sulfur')
        self.assertEqual(sulfur_item.username, 'Sulfur')
        self.assertEqual(sulfur_item.icon,
                         '/appengine/tests/icons/sulfur.png')
        self.assertEqual(
            sulfur_item.url,
            'http://www.factorioforums.com/wiki/index.php?title=Sulfur')
        self.assertEqual(sulfur_item.required_production_rate, 100)
        self.assertEqual(sulfur_item.production_machine_username,
                         'Chemical plant')
        # <rate> = <#-machines> * <#-recipe-results> * <machine-speed>
        #          ---------------------------------------------------
        #                        <energy-required>
        # 100 = x * 2 * 1.25 / 1
        # 100 = 2.5x
        # 100/2.5 = x
        # x = 40
        self.assertEqual(sulfur_item.num_production_machines_required, 40)
        self.assertEqual(
            sulfur_item.production_machine_icon,
            '/appengine/tests/icons/chemical-plant.png')
        self.assertEqual(
            sulfur_item.production_machine_url,
            'http://www.factorioforums.com/wiki/index.php?title='
            'Chemical_plant')

        self.assertEqual(petrolium_gas_item.is_user_requested, False)
        self.assertEqual(petrolium_gas_item.name, 'petroleum-gas')

        self.assertEqual(water_item.is_user_requested, False)
        self.assertEqual(water_item.name, 'water')

    def test_crafting_product(self):
        rates = recipe.calculate_required_production_rates(
            [('copper-cable', 20)])

        copper_cable_item, copper_plate_item, copper_ore_item = (
            produced_item.required_production_rates_to_produced_items(rates))
        self.assertEqual(copper_cable_item.is_user_requested, True)
        self.assertEqual(copper_cable_item.name, 'copper-cable')
        self.assertEqual(copper_cable_item.username, 'Copper cable')
        self.assertEqual(copper_cable_item.icon,
                         '/appengine/tests/icons/copper-cable.png')
        self.assertEqual(
            copper_cable_item.url,
            'http://www.factorioforums.com/wiki/index.php?title=Copper_cable')
        self.assertEqual(copper_cable_item.required_production_rate, 20)
        self.assertEqual(copper_cable_item.production_machine_username,
                         'Assembling machine 3')
        # <rate> = <#-machines> * <#-recipe-results> * <machine-speed>
        #          ---------------------------------------------------
        #                        <energy-required>
        # 20 = x * 2 * 1.25 / 0.5
        # 20 = 5x
        # 20/5 = x
        # x = 4
        self.assertEqual(copper_cable_item.num_production_machines_required, 4)
        self.assertEqual(
            copper_cable_item.production_machine_icon,
            '/appengine/tests/icons/assembling-machine-3.png')
        self.assertEqual(
            copper_cable_item.production_machine_url,
            'http://www.factorioforums.com/wiki/index.php?title='
            'Assembling_machine_3')

        self.assertEqual(copper_plate_item.is_user_requested, False)
        self.assertEqual(copper_plate_item.name, 'copper-plate')

        self.assertEqual(copper_ore_item.is_user_requested, False)
        self.assertEqual(copper_ore_item.name, 'copper-ore')

    def test_refined_fluids_have_no_production_machine(self):
        rates = recipe.calculate_required_production_rates(
            [('petroleum-gas', 100)])

        # Should only have one item i.e. 'light-oil-cracking' should not be
        # used to make 'petrolium-gas' from 'light-oil'.
        petrolium_gas_item, = (
            produced_item.required_production_rates_to_produced_items(rates))

        self.assertEqual(petrolium_gas_item.is_user_requested, True)
        self.assertEqual(petrolium_gas_item.name, 'petroleum-gas')
        self.assertEqual(petrolium_gas_item.required_production_rate, 100)
        # All production machine information should be None.
        self.assertIsNone(petrolium_gas_item.production_machine_username)
        self.assertIsNone(petrolium_gas_item.num_production_machines_required)
        self.assertIsNone(petrolium_gas_item.production_machine_icon)
        self.assertIsNone(petrolium_gas_item.production_machine_url)

    def test_missing_product_name(self):
        rates = recipe.calculate_required_production_rates(
            [('iron-plate', 100)])

        iron_plate_item, iron_ore_item, = (
            produced_item.required_production_rates_to_produced_items(rates))

        self.assertEqual(iron_plate_item.is_user_requested, True)
        self.assertEqual(iron_plate_item.name, 'iron-plate')
        self.assertEqual(iron_plate_item.username, 'iron-plate')
        self.assertEqual(iron_plate_item.icon,
                         '/appengine/tests/icons/iron-plate.png')
        self.assertIsNone(iron_plate_item.url)
        self.assertEqual(iron_plate_item.required_production_rate, 100)
        self.assertEqual(iron_plate_item.production_machine_username,
                         'Electric furnace')
        # <rate> = <#-machines> * <#-recipe-results> * <machine-speed>
        #          ---------------------------------------------------
        #                        <energy-required>
        # 100 = x * 1 * 2 / 3.5
        # 100 = 4/7x
        # (7/4)100 = x
        # x = 175
        self.assertEqual(iron_plate_item.num_production_machines_required,
                         175)
        self.assertEqual(
            iron_plate_item.production_machine_icon,
            '/appengine/tests/icons/electric-furnace.png')
        self.assertEqual(
            iron_plate_item.production_machine_url,
            'http://www.factorioforums.com/wiki/index.php?title='
            'Electric_furnace')

        self.assertEqual(iron_ore_item.is_user_requested, False)
        self.assertEqual(iron_ore_item.name, 'iron-ore')

    def test_missing_product_icon(self):
        rates = recipe.calculate_required_production_rates(
            [('iron-ore', 100)])

        # Should only have one item i.e. 'light-oil-cracking' should not be
        # used to make 'petrolium-gas' from 'light-oil'.
        iron_ore_item, = (
            produced_item.required_production_rates_to_produced_items(rates))

        self.assertEqual(iron_ore_item.is_user_requested, True)
        self.assertEqual(iron_ore_item.name, 'iron-ore')
        self.assertEqual(iron_ore_item.username, 'Iron ore')
        self.assertEqual(iron_ore_item.icon, '/img/missing-item.png')
        self.assertEqual(
            iron_ore_item.url,
            'http://www.factorioforums.com/wiki/index.php?title=Iron_ore')
        self.assertEqual(iron_ore_item.required_production_rate, 100)

if __name__ == '__main__':
    unittest.main()
<commit_msg>Remove incorrect copy/paste comment<commit_after>"""Tests for appengine.produced_item."""

import os.path
import unittest

from factorio import names
from factorio import recipe

from appengine import icons
from appengine import produced_item

class TestProducedItem(unittest.TestCase):
    def setUp(self):
        icons.init(os.path.join(os.path.dirname(__file__), 'icons'))
        names.names_from_json(
            open(os.path.join(os.path.dirname(__file__),
                              'test-names.json'), 'r'))
        recipe.Recipe.recipes_from_json(
            open(os.path.join(os.path.dirname(__file__),
                              'test-recipe.json'), 'r'))

    def test_base_product(self):
        rates = recipe.calculate_required_production_rates([('copper-ore', 5)])

        copper_ore_item = (
            produced_item.required_production_rates_to_produced_items(rates)[0])
        self.assertEqual(copper_ore_item.is_user_requested, True)
        self.assertEqual(copper_ore_item.name, 'copper-ore')
        self.assertEqual(copper_ore_item.username, 'Copper ore')
        self.assertEqual(copper_ore_item.icon,
                         '/appengine/tests/icons/copper-ore.png')
        self.assertEqual(
            copper_ore_item.url,
            'http://www.factorioforums.com/wiki/index.php?title=Copper_ore')
        self.assertEqual(copper_ore_item.required_production_rate, 5)
        self.assertIsNone(copper_ore_item.production_machine_username)
        self.assertIsNone(copper_ore_item.num_production_machines_required)
        self.assertIsNone(copper_ore_item.production_machine_icon)
        self.assertIsNone(copper_ore_item.production_machine_url)

    def test_smelting_product(self):
        rates = recipe.calculate_required_production_rates(
            [('copper-plate', 5)])

        copper_plate_item, copper_ore_item = (
            produced_item.required_production_rates_to_produced_items(rates))
        self.assertEqual(copper_plate_item.is_user_requested, True)
        self.assertEqual(copper_plate_item.name, 'copper-plate')
        self.assertEqual(copper_plate_item.username, 'Copper plate')
        self.assertEqual(copper_plate_item.icon,
                         '/appengine/tests/icons/copper-plate.png')
        self.assertEqual(
            copper_plate_item.url,
            'http://www.factorioforums.com/wiki/index.php?title=Copper_plate')
        self.assertEqual(copper_plate_item.required_production_rate, 5)
        self.assertEqual(copper_plate_item.production_machine_username,
                         'Electric furnace')
        # <rate> = <#-machines> * <#-recipe-results> * <machine-speed>
        #          ---------------------------------------------------
        #                        <energy-required>
        # 5 = x * 1 * 2 / 3.5
        # 5 = 4/7x
        # (7/4)5 = x
        # x = 8.75
        self.assertEqual(copper_plate_item.num_production_machines_required, 9)
        self.assertEqual(
            copper_plate_item.production_machine_icon,
            '/appengine/tests/icons/electric-furnace.png')
        self.assertEqual(
            copper_plate_item.production_machine_url,
            'http://www.factorioforums.com/wiki/index.php?title='
            'Electric_furnace')

        self.assertEqual(copper_ore_item.is_user_requested, False)
        self.assertEqual(copper_ore_item.name, 'copper-ore')

    def test_chemistry_product(self):
        rates = recipe.calculate_required_production_rates(
            [('sulfur', 100)])

        sulfur_item, petrolium_gas_item, water_item = (
            produced_item.required_production_rates_to_produced_items(rates))
        self.assertEqual(sulfur_item.is_user_requested, True)
        self.assertEqual(sulfur_item.name, 'sulfur')
        self.assertEqual(sulfur_item.username, 'Sulfur')
        self.assertEqual(sulfur_item.icon,
                         '/appengine/tests/icons/sulfur.png')
        self.assertEqual(
            sulfur_item.url,
            'http://www.factorioforums.com/wiki/index.php?title=Sulfur')
        self.assertEqual(sulfur_item.required_production_rate, 100)
        self.assertEqual(sulfur_item.production_machine_username,
                         'Chemical plant')
        # <rate> = <#-machines> * <#-recipe-results> * <machine-speed>
        #          ---------------------------------------------------
        #                        <energy-required>
        # 100 = x * 2 * 1.25 / 1
        # 100 = 2.5x
        # 100/2.5 = x
        # x = 40
        self.assertEqual(sulfur_item.num_production_machines_required, 40)
        self.assertEqual(
            sulfur_item.production_machine_icon,
            '/appengine/tests/icons/chemical-plant.png')
        self.assertEqual(
            sulfur_item.production_machine_url,
            'http://www.factorioforums.com/wiki/index.php?title='
            'Chemical_plant')

        self.assertEqual(petrolium_gas_item.is_user_requested, False)
        self.assertEqual(petrolium_gas_item.name, 'petroleum-gas')

        self.assertEqual(water_item.is_user_requested, False)
        self.assertEqual(water_item.name, 'water')

    def test_crafting_product(self):
        rates = recipe.calculate_required_production_rates(
            [('copper-cable', 20)])

        copper_cable_item, copper_plate_item, copper_ore_item = (
            produced_item.required_production_rates_to_produced_items(rates))
        self.assertEqual(copper_cable_item.is_user_requested, True)
        self.assertEqual(copper_cable_item.name, 'copper-cable')
        self.assertEqual(copper_cable_item.username, 'Copper cable')
        self.assertEqual(copper_cable_item.icon,
                         '/appengine/tests/icons/copper-cable.png')
        self.assertEqual(
            copper_cable_item.url,
            'http://www.factorioforums.com/wiki/index.php?title=Copper_cable')
        self.assertEqual(copper_cable_item.required_production_rate, 20)
        self.assertEqual(copper_cable_item.production_machine_username,
                         'Assembling machine 3')
        # <rate> = <#-machines> * <#-recipe-results> * <machine-speed>
        #          ---------------------------------------------------
        #                        <energy-required>
        # 20 = x * 2 * 1.25 / 0.5
        # 20 = 5x
        # 20/5 = x
        # x = 4
        self.assertEqual(copper_cable_item.num_production_machines_required, 4)
        self.assertEqual(
            copper_cable_item.production_machine_icon,
            '/appengine/tests/icons/assembling-machine-3.png')
        self.assertEqual(
            copper_cable_item.production_machine_url,
            'http://www.factorioforums.com/wiki/index.php?title='
            'Assembling_machine_3')

        self.assertEqual(copper_plate_item.is_user_requested, False)
        self.assertEqual(copper_plate_item.name, 'copper-plate')

        self.assertEqual(copper_ore_item.is_user_requested, False)
        self.assertEqual(copper_ore_item.name, 'copper-ore')

    def test_refined_fluids_have_no_production_machine(self):
        rates = recipe.calculate_required_production_rates(
            [('petroleum-gas', 100)])

        # Should only have one item i.e. 'light-oil-cracking' should not be
        # used to make 'petrolium-gas' from 'light-oil'.
        petrolium_gas_item, = (
            produced_item.required_production_rates_to_produced_items(rates))

        self.assertEqual(petrolium_gas_item.is_user_requested, True)
        self.assertEqual(petrolium_gas_item.name, 'petroleum-gas')
        self.assertEqual(petrolium_gas_item.required_production_rate, 100)
        # All production machine information should be None.
        self.assertIsNone(petrolium_gas_item.production_machine_username)
        self.assertIsNone(petrolium_gas_item.num_production_machines_required)
        self.assertIsNone(petrolium_gas_item.production_machine_icon)
        self.assertIsNone(petrolium_gas_item.production_machine_url)

    def test_missing_product_name(self):
        rates = recipe.calculate_required_production_rates(
            [('iron-plate', 100)])

        iron_plate_item, iron_ore_item, = (
            produced_item.required_production_rates_to_produced_items(rates))

        self.assertEqual(iron_plate_item.is_user_requested, True)
        self.assertEqual(iron_plate_item.name, 'iron-plate')
        self.assertEqual(iron_plate_item.username, 'iron-plate')
        self.assertEqual(iron_plate_item.icon,
                         '/appengine/tests/icons/iron-plate.png')
        self.assertIsNone(iron_plate_item.url)
        self.assertEqual(iron_plate_item.required_production_rate, 100)
        self.assertEqual(iron_plate_item.production_machine_username,
                         'Electric furnace')
        # <rate> = <#-machines> * <#-recipe-results> * <machine-speed>
        #          ---------------------------------------------------
        #                        <energy-required>
        # 100 = x * 1 * 2 / 3.5
        # 100 = 4/7x
        # (7/4)100 = x
        # x = 175
        self.assertEqual(iron_plate_item.num_production_machines_required,
                         175)
        self.assertEqual(
            iron_plate_item.production_machine_icon,
            '/appengine/tests/icons/electric-furnace.png')
        self.assertEqual(
            iron_plate_item.production_machine_url,
            'http://www.factorioforums.com/wiki/index.php?title='
            'Electric_furnace')

        self.assertEqual(iron_ore_item.is_user_requested, False)
        self.assertEqual(iron_ore_item.name, 'iron-ore')

    def test_missing_product_icon(self):
        rates = recipe.calculate_required_production_rates(
            [('iron-ore', 100)])

        iron_ore_item, = (
            produced_item.required_production_rates_to_produced_items(rates))

        self.assertEqual(iron_ore_item.is_user_requested, True)
        self.assertEqual(iron_ore_item.name, 'iron-ore')
        self.assertEqual(iron_ore_item.username, 'Iron ore')
        self.assertEqual(iron_ore_item.icon, '/img/missing-item.png')
        self.assertEqual(
            iron_ore_item.url,
            'http://www.factorioforums.com/wiki/index.php?title=Iron_ore')
        self.assertEqual(iron_ore_item.required_production_rate, 100)

if __name__ == '__main__':
    unittest.main()
<|endoftext|>


<commit_before>from django.views.decorators.csrf import csrf_exempt
from django.views.decorators.http import require_http_methods
from django.conf import settings
from django import http
from django.contrib.auth.models import User

from .mailgun import check_webhook_signature
from .tasks import send_announcement
from studygroups.models import Profile

from email.utils import parseaddr, formataddr
import logging

logger = logging.getLogger(__name__)

@csrf_exempt
@require_http_methods(['POST'])
def announce_webhook(request):
    """ This view is called by mailgun """
    # Verify mailgun call
    token = request.POST.get('token')
    timestamp = request.POST.get('timestamp')
    signature = request.POST.get('signature')
    authenticated = check_webhook_signature(settings.MAILGUN_API_KEY, token, timestamp, signature)
    if not authenticated:
        logger.warn('Authentication failed for announce_webhook')
        return http.HttpResponse(status=401) # TODO should this be 406 of just 200?

    # Get message details
    sender = request.POST.get('sender')
    from_ = request.POST.get('from')
    subject = request.POST.get('subject')
    body_text = request.POST.get('body-plain')
    body_html = request.POST.get('body-html')

    # Verify that email was sent from a staff user
    # parse sender to make sure it's not in the format John Silver <john@mail.net>
    user_name, user_email = parseaddr(sender)
    user = User.objects.filter(email=user_email).first()
    if not user or user.is_staff == False:
        logger.warn('Message sent to announce email from non-staff user')
        return http.HttpResponse(status=406)

    from_name = '{0} from P2PU'.format(user.first_name)
    from_ = formataddr((from_name, settings.ANNOUNCE_EMAIL))

    # Send announcement message to all users that opted in
    send_announcement.delay(from_, subject, body_text, body_html)

    return http.HttpResponse(status=200)

@csrf_exempt
def mailchimp_webhook(request, webhook_secret):
    """ change profile.communications_opt_in if subscriber has an account """

    if request.method == 'GET':
        return http.HttpResponse(status=200)

    import json
    logger.error(json.dumps(request.POST))

    if webhook_secret != settings.MAILCHIMP_WEBHOOK_SECRET:
        return http.HttpResponse(status=404)

    list_id = request.POST.get('data[id]')
    if list_id != settings.MAILCHIMP_LIST_ID:
        logger.warning('mailchimp webhook called with invalid list id')
        return http.HttpResponse(status=404)

    email = request.POST.get('data[email]')
    if User.objects.filter(email__iexact=email).count() == 0:
        return http.HttpResponse(status=200)

    profile = Profile.objects.get(user__email=email)
    if request.POST.get('type') == 'unsubscribe':
        profile.communication_opt_in = False
    elif request.POST.get('type') == 'subscribe':
        profile.communication_opt_in = True
    profile.save()
    return http.HttpResponse(status=200)
<commit_msg>Use correct field for mailchimp list id in webhook<commit_after>from django.views.decorators.csrf import csrf_exempt
from django.views.decorators.http import require_http_methods
from django.conf import settings
from django import http
from django.contrib.auth.models import User

from .mailgun import check_webhook_signature
from .tasks import send_announcement
from studygroups.models import Profile

from email.utils import parseaddr, formataddr
import logging

logger = logging.getLogger(__name__)

@csrf_exempt
@require_http_methods(['POST'])
def announce_webhook(request):
    """ This view is called by mailgun """
    # Verify mailgun call
    token = request.POST.get('token')
    timestamp = request.POST.get('timestamp')
    signature = request.POST.get('signature')
    authenticated = check_webhook_signature(settings.MAILGUN_API_KEY, token, timestamp, signature)
    if not authenticated:
        logger.warn('Authentication failed for announce_webhook')
        return http.HttpResponse(status=401) # TODO should this be 406 of just 200?

    # Get message details
    sender = request.POST.get('sender')
    from_ = request.POST.get('from')
    subject = request.POST.get('subject')
    body_text = request.POST.get('body-plain')
    body_html = request.POST.get('body-html')

    # Verify that email was sent from a staff user
    # parse sender to make sure it's not in the format John Silver <john@mail.net>
    user_name, user_email = parseaddr(sender)
    user = User.objects.filter(email=user_email).first()
    if not user or user.is_staff == False:
        logger.warn('Message sent to announce email from non-staff user')
        return http.HttpResponse(status=406)

    from_name = '{0} from P2PU'.format(user.first_name)
    from_ = formataddr((from_name, settings.ANNOUNCE_EMAIL))

    # Send announcement message to all users that opted in
    send_announcement.delay(from_, subject, body_text, body_html)

    return http.HttpResponse(status=200)

@csrf_exempt
def mailchimp_webhook(request, webhook_secret):
    """ change profile.communications_opt_in if subscriber has an account """

    if request.method == 'GET':
        return http.HttpResponse(status=200)

    import json
    logger.error(json.dumps(request.POST))

    if webhook_secret != settings.MAILCHIMP_WEBHOOK_SECRET:
        return http.HttpResponse(status=404)

    list_id = request.POST.get('data[list_id]')
    if list_id != settings.MAILCHIMP_LIST_ID:
        logger.warning('mailchimp webhook called with invalid list id')
        return http.HttpResponse(status=404)

    email = request.POST.get('data[email]')
    if User.objects.filter(email__iexact=email).count() == 0:
        return http.HttpResponse(status=200)

    profile = Profile.objects.get(user__email=email)
    if request.POST.get('type') == 'unsubscribe':
        profile.communication_opt_in = False
    elif request.POST.get('type') == 'subscribe':
        profile.communication_opt_in = True
    profile.save()
    return http.HttpResponse(status=200)
<|endoftext|>