<commit_before>from __future__ import print_function, division
import warnings
import sys, os
try:
    import platform
    if platform.python_version().startswith('2'):
        warnings.warn("Python 2 is old and yucky, please use 3", RuntimeWarning)
except ImportError:
    sys.exit(2)
from Bio.Seq import Seq
import pandas as pd
import pickle
import numpy as np
import scipy.stats
import utils
from multiprocessing import Pool, Manager, cpu_count
import itertools, time
from timeit import default_timer as timer

def process_data(hdf5_datastorepath, allele_pkl_path = None, experimental_info_csv_path=None, shuffle=False, seed=None):
    print("Loading data...", flush=True, end="")
    idx = pd.IndexSlice
    raw_barcode_data = pd.read_hdf(hdf5_datastorepath, key="grouped_data")

    if allele_pkl_path is None:
        allele_pkl_path = os.path.join(*[os.path.split(hdf5_datastorepath)[0], "allele_dic_with_WT.pkl"])
    with open(allele_pkl_path, "rb") as f:
        barcode_mutant_map = pd.DataFrame(pickle.load(f)).T.reset_index()
    print("Done.\n\nMapping...", flush=True, end="")

    barcode_mutant_map.columns = ["barcodes", "positions", "codons"]
    barcode_mutant_map["barcodes"] = barcode_mutant_map["barcodes"].apply(lambda x: str(Seq(x).reverse_complement()))
    barcode_mutant_map["barcodes"] = barcode_mutant_map["barcodes"].astype(np.str)
    raw_barcode_data = raw_barcode_data.reset_index()

    barcode_mutant_map["WT"] = barcode_mutant_map["codons"] == "WT"
    # add dummy value for WT barcodes
    barcode_mutant_map["amino acids"] = "@"
    barcode_mutant_map.loc[~barcode_mutant_map["WT"], "codons"] = barcode_mutant_map.loc[~barcode_mutant_map["WT"], "codons"].apply(lambda x: str(Seq(x).transcribe()))
    barcode_mutant_map.loc[~barcode_mutant_map["WT"], "amino acids"] = barcode_mutant_map.loc[~barcode_mutant_map["WT"], "codons"].apply(lambda x: str(Seq(x).translate()))

    mapped_barcode_data = raw_barcode_data.merge(barcode_mutant_map, on="barcodes", copy=False)
    print("Done.", flush=True)

    processed_barcodes = mapped_barcode_data.pivot_table(index=["group", "days", "timepoints", "barcodes", "codons","amino acids", "positions"],
                                                         values=["counts"])
    processed_barcodes = processed_barcodes.unstack("timepoints")

    # Onion d1t2 is simply a repeat of t1 so we'll toss it
    processed_barcodes.loc[idx["Onion", "d1"], idx["counts", "t2"]] = np.nan

    # shuffles the counts within each group, day, timepoint
    if shuffle:
        if seed is not None:
            np.random.seed(seed)
        print("\nShuffling...", flush=True, end="")
        processed_barcodes = processed_barcodes.unstack("group").unstack("days")
        count_values = processed_barcodes.values.T
        list(map(np.random.shuffle, count_values))
        processed_barcodes.loc[:, :] = count_values.T
        processed_barcodes = processed_barcodes.stack("group").stack("days").reorder_levels([4, 5, 0, 1, 2, 3])
        processed_barcodes.sort_index(inplace=True)
        np.random.seed()
        print("Done.", flush=True)

    # mapping transitions between timepoints, -1 indicates True to False aka NaN to not-NaN which should be eliminated
    # checks if any such transition occurs in a row
    has_nan_to_val_trans = np.any(np.diff(pd.isnull(processed_barcodes).values.view(np.int8)) == -1, axis=1)
    print("Throwing out barcodes that have nans prior to counts...\n{}".format(processed_barcodes[has_nan_to_val_trans].groupby(level="group").size()))
    processed_barcodes = processed_barcodes[~has_nan_to_val_trans]

    sums = processed_barcodes.unstack("group").unstack("days").reorder_levels([0, 2, 3, 1], axis=1).sort_index(level=["group", "days"], axis=1).sum()

    rel_freq = processed_barcodes.unstack("group").unstack("days").reorder_levels([0, 2, 3, 1], axis=1).sort_index(level=["group", "days"], axis=1)/sums
    rel_freq = rel_freq.stack("group").stack("days").reorder_levels([4, 5, 0, 1, 2, 3]).sort_index(level=["group", "days"])

    medians = rel_freq.unstack("group").unstack("days").loc[idx[:, "WT"], :].median()

    rel_wt = rel_freq.unstack("group").unstack("days")/medians

    rel_wt = rel_wt.stack("group").stack("days").reorder_levels([4, 5, 0, 1, 2, 3]).sort_index(level=["group", "days"])
    rel_wt.columns.set_levels(["rel_wt"], level=0, inplace=True)
    rel_wt = np.log2(rel_wt)

    if experimental_info_csv_path is None:
        experimental_info_csv_path = os.path.join(*[os.path.split(hdf5_datastorepath)[0], "truseq_primers.csv"])
    rel_wt_gen_times = utils.load_gen_times(rel_wt, experimental_info_csv_path)

    all_tp = rel_wt_gen_times[np.all(pd.notnull(rel_wt_gen_times["rel_wt"]), axis=1)]

    start = timer()
    slopes = groupby_parallel(all_tp.groupby(level=["group", "days", "amino acids"]), linregress_df)
    print("\nRegressed on {} slopes in {:.1f}s".format(len(slopes), timer() - start))

    return slopes

def linregress_wrapper(xs, ys):
    slope, intercept, r, p, stderr = scipy.stats.linregress(xs, ys)
    return pd.Series({"slope": slope, "r^2": r**2, "intercept": intercept, "stderr": stderr},
                     index=["slope", "intercept", "stderr"], name="stats")

def linregress_df(args):
    df, q = args
    result = df.apply(lambda x: linregress_wrapper(x.values[3:], x.values[:3]), axis=1)
    q.put(0)
    return result

def groupby_parallel(groupby_df, func):
    num_cpus = cpu_count()
    print("\nUsing {} cpus in parallel...\n\nPercent complete: ".format(num_cpus), end="")
    with Pool(num_cpus) as pool:
        m = Manager()
        q = m.Queue()
        result = pool.map_async(func, [(group, q) for name, group in groupby_df])
        for i, elem in enumerate(itertools.cycle('\|/-')):
            if result.ready():
                break
            size = q.qsize()
            print("Percent complete: {:.0%} {}".format(size/len(groupby_df), elem), end="\r")
            time.sleep(0.25)
        print("Percent complete: {:.0%}".format(1))
    return pd.concat(result.get())

<commit_msg>don't shuffle wt counts because we normalize against them<commit_after>from __future__ import print_function, division
import warnings
import sys, os
try:
    import platform
    if platform.python_version().startswith('2'):
        warnings.warn("Python 2 is old and yucky, please use 3", RuntimeWarning)
except ImportError:
    sys.exit(2)
from Bio.Seq import Seq
import pandas as pd
import pickle
import numpy as np
import scipy.stats
import utils
from multiprocessing import Pool, Manager, cpu_count
import itertools, time
from timeit import default_timer as timer

def process_data(hdf5_datastorepath, allele_pkl_path = None, experimental_info_csv_path=None, shuffle=False, seed=None):
    print("Loading data...", flush=True, end="")
    idx = pd.IndexSlice
    raw_barcode_data = pd.read_hdf(hdf5_datastorepath, key="grouped_data")

    if allele_pkl_path is None:
        allele_pkl_path = os.path.join(*[os.path.split(hdf5_datastorepath)[0], "allele_dic_with_WT.pkl"])
    with open(allele_pkl_path, "rb") as f:
        barcode_mutant_map = pd.DataFrame(pickle.load(f)).T.reset_index()
    print("Done.\n\nMapping...", flush=True, end="")

    barcode_mutant_map.columns = ["barcodes", "positions", "codons"]
    barcode_mutant_map["barcodes"] = barcode_mutant_map["barcodes"].apply(lambda x: str(Seq(x).reverse_complement()))
    barcode_mutant_map["barcodes"] = barcode_mutant_map["barcodes"].astype(np.str)
    raw_barcode_data = raw_barcode_data.reset_index()

    barcode_mutant_map["WT"] = barcode_mutant_map["codons"] == "WT"
    # add dummy value for WT barcodes
    barcode_mutant_map["amino acids"] = "@"
    barcode_mutant_map.loc[~barcode_mutant_map["WT"], "codons"] = barcode_mutant_map.loc[~barcode_mutant_map["WT"], "codons"].apply(lambda x: str(Seq(x).transcribe()))
    barcode_mutant_map.loc[~barcode_mutant_map["WT"], "amino acids"] = barcode_mutant_map.loc[~barcode_mutant_map["WT"], "codons"].apply(lambda x: str(Seq(x).translate()))

    mapped_barcode_data = raw_barcode_data.merge(barcode_mutant_map, on="barcodes", copy=False)
    print("Done.", flush=True)

    processed_barcodes = mapped_barcode_data.pivot_table(index=["group", "days", "timepoints", "barcodes", "codons","amino acids", "positions"],
                                                         values=["counts"])
    processed_barcodes = processed_barcodes.unstack("timepoints")

    # Onion d1t2 is simply a repeat of t1 so we'll toss it
    processed_barcodes.loc[idx["Onion", "d1"], idx["counts", "t2"]] = np.nan

    # shuffles the counts within each group, day, timepoint
    if shuffle:
        if seed is not None:
            np.random.seed(seed)
        print("\nShuffling...", flush=True, end="")
        wts = processed_barcodes.loc[idx[:, :, :, "WT"], :]
        no_wts = processed_barcodes[~processed_barcodes.index.isin(wts.index)].unstack("group").unstack("days")
        count_values = no_wts.values.T
        list(map(np.random.shuffle, count_values))
        no_wts.loc[:, :] = count_values.T
        no_wts = no_wts.stack("group").stack("days").reorder_levels([4, 5, 0, 1, 2, 3])
        processed_barcodes = pd.concat([no_wts, wts])
        processed_barcodes.sort_index(inplace=True)
        np.random.seed()
        print("Done.", flush=True)

    # mapping transitions between timepoints, -1 indicates True to False aka NaN to not-NaN which should be eliminated
    # checks if any such transition occurs in a row
    has_nan_to_val_trans = np.any(np.diff(pd.isnull(processed_barcodes).values.view(np.int8)) == -1, axis=1)
    print("Throwing out barcodes that have nans prior to counts...\n{}".format(processed_barcodes[has_nan_to_val_trans].groupby(level="group").size()))
    processed_barcodes = processed_barcodes[~has_nan_to_val_trans]

    sums = processed_barcodes.unstack("group").unstack("days").reorder_levels([0, 2, 3, 1], axis=1).sort_index(level=["group", "days"], axis=1).sum()

    rel_freq = processed_barcodes.unstack("group").unstack("days").reorder_levels([0, 2, 3, 1], axis=1).sort_index(level=["group", "days"], axis=1)/sums
    rel_freq = rel_freq.stack("group").stack("days").reorder_levels([4, 5, 0, 1, 2, 3]).sort_index(level=["group", "days"])

    medians = rel_freq.unstack("group").unstack("days").loc[idx[:, "WT"], :].median()

    rel_wt = rel_freq.unstack("group").unstack("days")/medians

    rel_wt = rel_wt.stack("group").stack("days").reorder_levels([4, 5, 0, 1, 2, 3]).sort_index(level=["group", "days"])
    rel_wt.columns.set_levels(["rel_wt"], level=0, inplace=True)
    rel_wt = np.log2(rel_wt)

    if experimental_info_csv_path is None:
        experimental_info_csv_path = os.path.join(*[os.path.split(hdf5_datastorepath)[0], "truseq_primers.csv"])
    rel_wt_gen_times = utils.load_gen_times(rel_wt, experimental_info_csv_path)

    all_tp = rel_wt_gen_times[np.all(pd.notnull(rel_wt_gen_times["rel_wt"]), axis=1)]

    start = timer()
    slopes = groupby_parallel(all_tp.groupby(level=["group", "days", "amino acids"]), linregress_df)
    print("\nRegressed on {} slopes in {:.1f}s".format(len(slopes), timer() - start))

    return slopes

def linregress_wrapper(xs, ys):
    slope, intercept, r, p, stderr = scipy.stats.linregress(xs, ys)
    return pd.Series({"slope": slope, "r^2": r**2, "intercept": intercept, "stderr": stderr},
                     index=["slope", "intercept", "stderr"], name="stats")

def linregress_df(args):
    df, q = args
    result = df.apply(lambda x: linregress_wrapper(x.values[3:], x.values[:3]), axis=1)
    q.put(0)
    return result

def groupby_parallel(groupby_df, func):
    num_cpus = cpu_count()
    print("\nUsing {} cpus in parallel...\n\nPercent complete: ".format(num_cpus), end="")
    with Pool(num_cpus) as pool:
        m = Manager()
        q = m.Queue()
        result = pool.map_async(func, [(group, q) for name, group in groupby_df])
        for i, elem in enumerate(itertools.cycle('\|/-')):
            if result.ready():
                break
            size = q.qsize()
            print("Percent complete: {:.0%} {}".format(size/len(groupby_df), elem), end="\r")
            time.sleep(0.25)
        print("Percent complete: {:.0%}".format(1))
    return pd.concat(result.get())

<|endoftext|>


<commit_before># Module:   pollers
# Date:     15th September 2008
# Author:   James Mills <prologic@shortcircuit.net.au>

"""Poller Components for asynchronous file and socket I/O.

This module contains Poller components that enable polling of file or socket
descriptors for read/write events. Pollers:
- Select
- Poll
- EPoll
"""

from errno import *
from time import time
from select import select
from select import error as SelectError
from socket import error as SocketError

try:
    from select import poll
    from select import POLLIN, POLLOUT, POLLHUP, POLLERR, POLLNVAL
    HAS_POLL = 1
except ImportError:
    HAS_POLL = 0

try:
    from select import epoll
    from select import EPOLLIN, EPOLLOUT, EPOLLHUP, EPOLLERR
    HAS_EPOLL = 2
except ImportError:
    try:
        from select26 import epoll
        from select26 import EPOLLIN, EPOLLOUT, EPOLLHUP, EPOLLERR
        HAS_EPOLL = 1
    except ImportError:
        HAS_EPOLL = 0

try:
    # Kqueue
    from select import kqueue, kevent, KQ_FILTER_READ, KQ_EV_ERROR
    from select import KQ_FILTER_WRITE, KQ_EV_ADD, KQ_EV_DELETE, KQ_EV_EOF
    HAS_KQUEUE = 1
except ImportError:
    HAS_KQUEUE = 0

from events import Event
from components import BaseComponent

if HAS_POLL:
    _POLL_DISCONNECTED = (POLLHUP | POLLERR | POLLNVAL)

if HAS_EPOLL:
    _EPOLL_DISCONNECTED = (EPOLLHUP | EPOLLERR)

TIMEOUT = 0.2

###
### Events
###

class Read(Event): pass
class Write(Event): pass
class Error(Event): pass
class Disconnect(Event): pass

class BasePoller(BaseComponent):

    channel = None

    def __init__(self, timeout=TIMEOUT, channel=channel):
        super(BasePoller, self).__init__(channel=channel)

        self.timeout = timeout

        self._read = []
        self._write = []
        self._targets = {}

    def addReader(self, source, fd):
        channel = getattr(source, "channel", "*")
        self._read.append(fd)
        self._targets[fd] = channel

    def addWriter(self, source, fd):
        channel = getattr(source, "channel", "*")
        self._write.append(fd)
        self._targets[fd] = channel

    def removeReader(self, fd):
        if fd in self._read:
            self._read.remove(fd)
        if fd in self._targets:
            del self._targets[fd]

    def removeWriter(self, fd):
        if fd in self._write:
            self._write.remove(fd)
        if not (fd in self._read or fd in self._write):
            del self._targets[fd]

    def isReading(self, fd):
        return fd in self._read

    def isWriting(self, fd):
        return fd in self._write

    def discard(self, fd):
        if fd in self._read:
            self._read.remove(fd)
        if fd in self._write:
            self._write.remove(fd)
        if fd in self._targets:
            del self._targets[fd]

    def getTarget(self, fd):
        return self._targets.get(fd, self.manager)

class Select(BasePoller):
    """Select(...) -> new Select Poller Component

    Creates a new Select Poller Component that uses the select poller
    implementation. This poller is not reccomneded but is available for legacy
    reasons as most systems implement select-based polling for backwards
    compatibility.
    """

    channel = "select"

    def __init__(self, timeout=TIMEOUT, channel=channel):
        super(Select, self).__init__(timeout, channel=channel)

        self._ts = time()
        self._load = 0.0

    def _preenDescriptors(self):
        for socks in (self._read[:], self._write[:]):
            for sock in socks:
                try:
                    select([sock], [sock], [sock], 0)
                except Exception, e:
                    self.discard(sock)

    def __tick__(self):
        try:
            if not any([self._read, self._write]):
                return
            r, w, _ = select(self._read, self._write, [], self.timeout)
            if r or w:
                self.timeout = 0.0
            dtime = time() - self._ts
            self._ts = time()
            if not dtime == 0.0:
                load = float(len(r) + len(w)) / dtime
                if load > self._load:
                    if self.timeout > 0.1:
                        self.timeout -= 0.001
                else:
                    if self.timeout < 0.5:
                        self.timeout += 0.001
                self._load = load
        except ValueError, e:
            # Possibly a file descriptor has gone negative?
            return self._preenDescriptors()
        except TypeError, e:
            # Something *totally* invalid (object w/o fileno, non-integral
            # result) was passed
            return self._preenDescriptors()
        except (SelectError, SocketError, IOError), e:
            # select(2) encountered an error
            if e[0] in (0, 2):
                # windows does this if it got an empty list
                if (not self._read) and (not self._write):
                    return
                else:
                    raise
            elif e[0] == EINTR:
                return
            elif e[0] == EBADF:
                return self._preenDescriptors()
            else:
                # OK, I really don't know what's going on.  Blow up.
                raise

        for sock in w:
            if self.isWriting(sock):
                self.push(Write(sock), "_write", self.getTarget(sock))

        for sock in r:
            if self.isReading(sock):
                self.push(Read(sock), "_read", self.getTarget(sock))

class Poll(BasePoller):
    """Poll(...) -> new Poll Poller Component

    Creates a new Poll Poller Component that uses the poll poller
    implementation.
    """

    channel = "poll"

    def __init__(self, timeout=TIMEOUT, channel=channel):
        super(Poll, self).__init__(timeout, channel=channel)

        self._map = {}
        self._poller = poll()

    def _updateRegistration(self, fd):
        fileno = fd.fileno()

        try:
            self._poller.unregister(fileno)
        except KeyError:
            pass

        mask = 0

        if fd in self._read:
            mask = mask | POLLIN
        if fd in self._write:
            mask = mask | POLLOUT

        if mask:
            self._poller.register(fd, mask)
            self._map[fileno] = fd
        else:
            super(Poll, self).discard(fd)
            del self._map[fileno]

    def addReader(self, source, fd):
        super(Poll, self).addReader(source, fd)
        self._updateRegistration(fd)

    def addWriter(self, source, fd):
        super(Poll, self).addWriter(source, fd)
        self._updateRegistration(fd)

    def removeReader(self, fd):
        super(Poll, self).removeReader(fd)
        self._updateRegistration(fd)

    def removeWriter(self, fd):
        super(Poll, self).removeWriter(fd)
        self._updateRegistration(fd)

    def discard(self, fd):
        super(Poll, self).discard(fd)
        self._updateRegistration(fd)

    def __tick__(self):
        try:
            l = self._poller.poll(self.timeout)
        except SelectError, e:
            if e[0] == EINTR:
                return
            else:
                raise

        for fileno, event in l:
            self._process(fileno, event)

    def _process(self, fileno, event):
        if fileno not in self._map:
            return

        fd = self._map[fileno]

        if event & _POLL_DISCONNECTED and not (event & POLLIN):
            self.push(Disconnect(fd), "_disconnect", self.getTarget(fd))
            self._poller.unregister(fileno)
            super(Poll, self).discard(fd)
            del self._map[fileno]
        else:
            try:
                if event & POLLIN:
                    self.push(Read(fd), "_read", self.getTarget(fd))
                if event & POLLOUT:
                    self.push(Write(fd), "_write", self.getTarget(fd))
            except Exception, e:
                self.push(Error(fd, e), "_error", self.getTarget(fd))
                self.push(Disconnect(fd), "_disconnect", self.getTarget(fd))
                self._poller.unregister(fileno)
                super(Poll, self).discard(fd)
                del self._map[fileno]

class EPoll(BasePoller):
    """EPoll(...) -> new EPoll Poller Component

    Creates a new EPoll Poller Component that uses the epoll poller
    implementation.
    """

    channel = "epoll"

    def __init__(self, timeout=TIMEOUT, channel=channel):
        super(EPoll, self).__init__(timeout, channel=channel)

        self._map = {}
        self._poller = epoll()

    def _updateRegistration(self, fd):
        try:
            fileno = fd.fileno()
            self._poller.unregister(fileno)
        except (SocketError, IOError), e:
            if e[0] == EBADF:
                keys = [k for k, v in self._map.items() if v == fd]
                for key in keys:
                    del self._map[key]

        mask = 0

        if fd in self._read:
            mask = mask | EPOLLIN
        if fd in self._write:
            mask = mask | EPOLLOUT

        if mask:
            self._poller.register(fd, mask)
            self._map[fileno] = fd
        else:
            super(EPoll, self).discard(fd)

    def addReader(self, source, fd):
        super(EPoll, self).addReader(source, fd)
        self._updateRegistration(fd)

    def addWriter(self, source, fd):
        super(EPoll, self).addWriter(source, fd)
        self._updateRegistration(fd)

    def removeReader(self, fd):
        super(EPoll, self).removeReader(fd)
        self._updateRegistration(fd)

    def removeWriter(self, fd):
        super(EPoll, self).removeWriter(fd)
        self._updateRegistration(fd)

    def discard(self, fd):
        super(EPoll, self).discard(fd)
        self._updateRegistration(fd)

    def __tick__(self):
        try:
            l = self._poller.poll(self.timeout)
        except IOError, e:
            if e[0] == EINTR:
                return
        except SelectError, e:
            if e[0] == EINTR:
                return
            else:
                raise

        for fileno, event in l:
            self._process(fileno, event)

    def _process(self, fileno, event):
        if fileno not in self._map:
            return

        fd = self._map[fileno]

        if event & _EPOLL_DISCONNECTED and not (event & POLLIN):
            self.push(Disconnect(fd), "_disconnect", self.getTarget(fd))
            self._poller.unregister(fileno)
            super(EPoll, self).discard(fd)
            del self._map[fileno]
        else:
            try:
                if event & EPOLLIN:
                    self.push(Read(fd), "_read", self.getTarget(fd))
                if event & EPOLLOUT:
                    self.push(Write(fd), "_write", self.getTarget(fd))
            except Exception, e:
                self.push(Error(fd, e), "_error", self.getTarget(fd))
                self.push(Disconnect(fd), "_disconnect", self.getTarget(fd))
                self._poller.unregister(fileno)
                super(EPoll, self).discard(fd)
                del self._map[fileno]

class KQueue(BasePoller):
    """KQueue(...) -> new KQueue Poller Component

    Creates a new KQueue Poller Component that uses the kqueue poller
    implementation.
    """

    channel = "kqueue"

    def __init__(self, timeout=0.00001, channel=channel):
        super(KQueue, self).__init__(timeout, channel=channel)
        self._map = {}
        self._poller = kqueue()

    def addReader(self, source, sock):
        super(KQueue, self).addReader(source, sock)
        self._map[sock.fileno()] = sock
        self._poller.control([kevent(sock,
            KQ_FILTER_READ, KQ_EV_ADD)], 0)

    def addWriter(self, source, sock):
        super(KQueue, self).addWriter(source, sock)
        self._map[sock.fileno()] = sock
        self._poller.control([kevent(sock,
            KQ_FILTER_WRITE, KQ_EV_ADD)], 0)

    def removeReader(self, sock):
        super(KQueue, self).removeReader(sock)
        self._poller.control([kevent(sock,
            KQ_FILTER_READ, KQ_EV_DELETE)], 0)

    def removeWriter(self, sock):
        super(KQueue, self).removeWriter(sock)
        self._poller.control([kevent(sock,
            KQ_FILTER_WRITE, KQ_EV_DELETE)], 0)

    def discard(self, sock):
        super(KQueue, self).discard(sock)
        del self._map[sock.fileno()]
        self._poller.control([kevent(sock,
            KQ_FILTER_WRITE|KQ_FILTER_READ, KQ_EV_DELETE)], 0)

    def __tick__(self):
        try:
            l = self._poller.control(None, 1000, self.timeout)
        except SelectError, e:
            if e[0] == EINTR:
                return
            else:
                raise

        for event in l:
            self._process(event)

    def _process(self, event):
        if event.ident not in self._map:
            # shouldn't happen ?
            # we unregister the socket since we don't care about it anymore
            self._poller.control(
                [kevent(event.ident, event.filter, KQ_EV_DELETE)], 0)
            return

        sock = self._map[event.ident]

        if event.flags & KQ_EV_ERROR:
            self.push(Error(sock, "error"), "_error", self.getTarget(sock))
        elif event.flags & KQ_EV_EOF:
            self.push(Disconnect(sock), "_disconnect", self.getTarget(sock))
        elif event.filter == KQ_FILTER_WRITE:
            self.push(Write(sock), "_write", self.getTarget(sock))
        elif event.filter == KQ_FILTER_READ:
            self.push(Read(sock), "_read", self.getTarget(sock))

### FIXME: The EPoll poller has some weird performance issues :/
#if HAS_EPOLL:
#    Poller = EPoll
#else:
#    Poller = Select

Poller = Select

__all__ = ("BasePoller", "Poller", "Select", "Poll", "EPoll", "KQueue")
<commit_msg>- circuits.core: Removed dynamic timeout code from ``Select`` poller.<commit_after># Module:   pollers
# Date:     15th September 2008
# Author:   James Mills <prologic@shortcircuit.net.au>

"""Poller Components for asynchronous file and socket I/O.

This module contains Poller components that enable polling of file or socket
descriptors for read/write events. Pollers:
- Select
- Poll
- EPoll
"""

from errno import *
from time import time
from select import select
from select import error as SelectError
from socket import error as SocketError

try:
    from select import poll
    from select import POLLIN, POLLOUT, POLLHUP, POLLERR, POLLNVAL
    HAS_POLL = 1
except ImportError:
    HAS_POLL = 0

try:
    from select import epoll
    from select import EPOLLIN, EPOLLOUT, EPOLLHUP, EPOLLERR
    HAS_EPOLL = 2
except ImportError:
    try:
        from select26 import epoll
        from select26 import EPOLLIN, EPOLLOUT, EPOLLHUP, EPOLLERR
        HAS_EPOLL = 1
    except ImportError:
        HAS_EPOLL = 0

try:
    # Kqueue
    from select import kqueue, kevent, KQ_FILTER_READ, KQ_EV_ERROR
    from select import KQ_FILTER_WRITE, KQ_EV_ADD, KQ_EV_DELETE, KQ_EV_EOF
    HAS_KQUEUE = 1
except ImportError:
    HAS_KQUEUE = 0

from events import Event
from components import BaseComponent

if HAS_POLL:
    _POLL_DISCONNECTED = (POLLHUP | POLLERR | POLLNVAL)

if HAS_EPOLL:
    _EPOLL_DISCONNECTED = (EPOLLHUP | EPOLLERR)

TIMEOUT = 0.01 # 10ms timeout

###
### Events
###

class Read(Event): pass
class Write(Event): pass
class Error(Event): pass
class Disconnect(Event): pass

class BasePoller(BaseComponent):

    channel = None

    def __init__(self, timeout=TIMEOUT, channel=channel):
        super(BasePoller, self).__init__(channel=channel)

        self.timeout = timeout

        self._read = []
        self._write = []
        self._targets = {}

    def addReader(self, source, fd):
        channel = getattr(source, "channel", "*")
        self._read.append(fd)
        self._targets[fd] = channel

    def addWriter(self, source, fd):
        channel = getattr(source, "channel", "*")
        self._write.append(fd)
        self._targets[fd] = channel

    def removeReader(self, fd):
        if fd in self._read:
            self._read.remove(fd)
        if fd in self._targets:
            del self._targets[fd]

    def removeWriter(self, fd):
        if fd in self._write:
            self._write.remove(fd)
        if not (fd in self._read or fd in self._write):
            del self._targets[fd]

    def isReading(self, fd):
        return fd in self._read

    def isWriting(self, fd):
        return fd in self._write

    def discard(self, fd):
        if fd in self._read:
            self._read.remove(fd)
        if fd in self._write:
            self._write.remove(fd)
        if fd in self._targets:
            del self._targets[fd]

    def getTarget(self, fd):
        return self._targets.get(fd, self.manager)

class Select(BasePoller):
    """Select(...) -> new Select Poller Component

    Creates a new Select Poller Component that uses the select poller
    implementation. This poller is not reccomneded but is available for legacy
    reasons as most systems implement select-based polling for backwards
    compatibility.
    """

    channel = "select"

    def __init__(self, timeout=TIMEOUT, channel=channel):
        super(Select, self).__init__(timeout, channel=channel)

        self._ts = time()
        self._load = 0.0

    def _preenDescriptors(self):
        for socks in (self._read[:], self._write[:]):
            for sock in socks:
                try:
                    select([sock], [sock], [sock], 0)
                except Exception, e:
                    self.discard(sock)

    def __tick__(self):
        try:
            if not any([self._read, self._write]):
                return
            r, w, _ = select(self._read, self._write, [], self.timeout)
        except ValueError, e:
            # Possibly a file descriptor has gone negative?
            return self._preenDescriptors()
        except TypeError, e:
            # Something *totally* invalid (object w/o fileno, non-integral
            # result) was passed
            return self._preenDescriptors()
        except (SelectError, SocketError, IOError), e:
            # select(2) encountered an error
            if e[0] in (0, 2):
                # windows does this if it got an empty list
                if (not self._read) and (not self._write):
                    return
                else:
                    raise
            elif e[0] == EINTR:
                return
            elif e[0] == EBADF:
                return self._preenDescriptors()
            else:
                # OK, I really don't know what's going on.  Blow up.
                raise

        for sock in w:
            if self.isWriting(sock):
                self.push(Write(sock), "_write", self.getTarget(sock))

        for sock in r:
            if self.isReading(sock):
                self.push(Read(sock), "_read", self.getTarget(sock))

class Poll(BasePoller):
    """Poll(...) -> new Poll Poller Component

    Creates a new Poll Poller Component that uses the poll poller
    implementation.
    """

    channel = "poll"

    def __init__(self, timeout=TIMEOUT, channel=channel):
        super(Poll, self).__init__(timeout, channel=channel)

        self._map = {}
        self._poller = poll()

    def _updateRegistration(self, fd):
        fileno = fd.fileno()

        try:
            self._poller.unregister(fileno)
        except KeyError:
            pass

        mask = 0

        if fd in self._read:
            mask = mask | POLLIN
        if fd in self._write:
            mask = mask | POLLOUT

        if mask:
            self._poller.register(fd, mask)
            self._map[fileno] = fd
        else:
            super(Poll, self).discard(fd)
            del self._map[fileno]

    def addReader(self, source, fd):
        super(Poll, self).addReader(source, fd)
        self._updateRegistration(fd)

    def addWriter(self, source, fd):
        super(Poll, self).addWriter(source, fd)
        self._updateRegistration(fd)

    def removeReader(self, fd):
        super(Poll, self).removeReader(fd)
        self._updateRegistration(fd)

    def removeWriter(self, fd):
        super(Poll, self).removeWriter(fd)
        self._updateRegistration(fd)

    def discard(self, fd):
        super(Poll, self).discard(fd)
        self._updateRegistration(fd)

    def __tick__(self):
        try:
            l = self._poller.poll(self.timeout)
        except SelectError, e:
            if e[0] == EINTR:
                return
            else:
                raise

        for fileno, event in l:
            self._process(fileno, event)

    def _process(self, fileno, event):
        if fileno not in self._map:
            return

        fd = self._map[fileno]

        if event & _POLL_DISCONNECTED and not (event & POLLIN):
            self.push(Disconnect(fd), "_disconnect", self.getTarget(fd))
            self._poller.unregister(fileno)
            super(Poll, self).discard(fd)
            del self._map[fileno]
        else:
            try:
                if event & POLLIN:
                    self.push(Read(fd), "_read", self.getTarget(fd))
                if event & POLLOUT:
                    self.push(Write(fd), "_write", self.getTarget(fd))
            except Exception, e:
                self.push(Error(fd, e), "_error", self.getTarget(fd))
                self.push(Disconnect(fd), "_disconnect", self.getTarget(fd))
                self._poller.unregister(fileno)
                super(Poll, self).discard(fd)
                del self._map[fileno]

class EPoll(BasePoller):
    """EPoll(...) -> new EPoll Poller Component

    Creates a new EPoll Poller Component that uses the epoll poller
    implementation.
    """

    channel = "epoll"

    def __init__(self, timeout=TIMEOUT, channel=channel):
        super(EPoll, self).__init__(timeout, channel=channel)

        self._map = {}
        self._poller = epoll()

    def _updateRegistration(self, fd):
        try:
            fileno = fd.fileno()
            self._poller.unregister(fileno)
        except (SocketError, IOError), e:
            if e[0] == EBADF:
                keys = [k for k, v in self._map.items() if v == fd]
                for key in keys:
                    del self._map[key]

        mask = 0

        if fd in self._read:
            mask = mask | EPOLLIN
        if fd in self._write:
            mask = mask | EPOLLOUT

        if mask:
            self._poller.register(fd, mask)
            self._map[fileno] = fd
        else:
            super(EPoll, self).discard(fd)

    def addReader(self, source, fd):
        super(EPoll, self).addReader(source, fd)
        self._updateRegistration(fd)

    def addWriter(self, source, fd):
        super(EPoll, self).addWriter(source, fd)
        self._updateRegistration(fd)

    def removeReader(self, fd):
        super(EPoll, self).removeReader(fd)
        self._updateRegistration(fd)

    def removeWriter(self, fd):
        super(EPoll, self).removeWriter(fd)
        self._updateRegistration(fd)

    def discard(self, fd):
        super(EPoll, self).discard(fd)
        self._updateRegistration(fd)

    def __tick__(self):
        try:
            l = self._poller.poll(self.timeout)
        except IOError, e:
            if e[0] == EINTR:
                return
        except SelectError, e:
            if e[0] == EINTR:
                return
            else:
                raise

        for fileno, event in l:
            self._process(fileno, event)

    def _process(self, fileno, event):
        if fileno not in self._map:
            return

        fd = self._map[fileno]

        if event & _EPOLL_DISCONNECTED and not (event & POLLIN):
            self.push(Disconnect(fd), "_disconnect", self.getTarget(fd))
            self._poller.unregister(fileno)
            super(EPoll, self).discard(fd)
            del self._map[fileno]
        else:
            try:
                if event & EPOLLIN:
                    self.push(Read(fd), "_read", self.getTarget(fd))
                if event & EPOLLOUT:
                    self.push(Write(fd), "_write", self.getTarget(fd))
            except Exception, e:
                self.push(Error(fd, e), "_error", self.getTarget(fd))
                self.push(Disconnect(fd), "_disconnect", self.getTarget(fd))
                self._poller.unregister(fileno)
                super(EPoll, self).discard(fd)
                del self._map[fileno]

class KQueue(BasePoller):
    """KQueue(...) -> new KQueue Poller Component

    Creates a new KQueue Poller Component that uses the kqueue poller
    implementation.
    """

    channel = "kqueue"

    def __init__(self, timeout=0.00001, channel=channel):
        super(KQueue, self).__init__(timeout, channel=channel)
        self._map = {}
        self._poller = kqueue()

    def addReader(self, source, sock):
        super(KQueue, self).addReader(source, sock)
        self._map[sock.fileno()] = sock
        self._poller.control([kevent(sock,
            KQ_FILTER_READ, KQ_EV_ADD)], 0)

    def addWriter(self, source, sock):
        super(KQueue, self).addWriter(source, sock)
        self._map[sock.fileno()] = sock
        self._poller.control([kevent(sock,
            KQ_FILTER_WRITE, KQ_EV_ADD)], 0)

    def removeReader(self, sock):
        super(KQueue, self).removeReader(sock)
        self._poller.control([kevent(sock,
            KQ_FILTER_READ, KQ_EV_DELETE)], 0)

    def removeWriter(self, sock):
        super(KQueue, self).removeWriter(sock)
        self._poller.control([kevent(sock,
            KQ_FILTER_WRITE, KQ_EV_DELETE)], 0)

    def discard(self, sock):
        super(KQueue, self).discard(sock)
        del self._map[sock.fileno()]
        self._poller.control([kevent(sock,
            KQ_FILTER_WRITE|KQ_FILTER_READ, KQ_EV_DELETE)], 0)

    def __tick__(self):
        try:
            l = self._poller.control(None, 1000, self.timeout)
        except SelectError, e:
            if e[0] == EINTR:
                return
            else:
                raise

        for event in l:
            self._process(event)

    def _process(self, event):
        if event.ident not in self._map:
            # shouldn't happen ?
            # we unregister the socket since we don't care about it anymore
            self._poller.control(
                [kevent(event.ident, event.filter, KQ_EV_DELETE)], 0)
            return

        sock = self._map[event.ident]

        if event.flags & KQ_EV_ERROR:
            self.push(Error(sock, "error"), "_error", self.getTarget(sock))
        elif event.flags & KQ_EV_EOF:
            self.push(Disconnect(sock), "_disconnect", self.getTarget(sock))
        elif event.filter == KQ_FILTER_WRITE:
            self.push(Write(sock), "_write", self.getTarget(sock))
        elif event.filter == KQ_FILTER_READ:
            self.push(Read(sock), "_read", self.getTarget(sock))

### FIXME: The EPoll poller has some weird performance issues :/
#if HAS_EPOLL:
#    Poller = EPoll
#else:
#    Poller = Select

Poller = Select

__all__ = ("BasePoller", "Poller", "Select", "Poll", "EPoll", "KQueue")
<|endoftext|>


<commit_before># cito - The Xenon1T experiments software trigger
# Copyright 2013.  All rights reserved.
# https://github.com/tunnell/cito
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are
# met:
#
#     * Redistributions of source code must retain the above copyright
# notice, this list of conditions and the following disclaimer.
#     * Redistributions in binary form must reproduce the above
# copyright notice, this list of conditions and the following disclaimer
# in the documentation and/or other materials provided with the
# distribution.
#     * Neither the name of the Xenon experiment, nor the names of its
# contributors may be used to endorse or promote products derived from
# this software without specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
"""Perform operations on sets of blocks
"""

import logging

import numpy as np
from scipy import signal
from scipy.stats import norm
import scipy

from cito.helpers import xedb
from cito.helpers import InterfaceV1724Swig
from cito.helpers import InterfaceV1724
from cito.helpers import CaenBlockParsing

def filter_samples(values):
    """Apply a filter

    :rtype : dict
    :param values: Values to filter
    :type values: np.array
    """
    new_values = np.zeros_like(values)

    print('\tprefilter')
    my_filter = norm(0, 100)  # mu=0, sigma=100
    filter_values = 600 * [0.]
    for j in range(-300, 300):
        filter_values[j] = my_filter.pdf(j)

    print('\tapply_filter')
    for i in range(values.size):
        for j in range(-300, 300):
            if j > 0 and j < values.size:
                new_values += values[i] * filter_values[j]

    return new_values

def find_peaks(values, threshold=10000):
    """Find peaks within list of values.

    Uses scipy to find peaks above a threshold.

    Args:
        values (list):  The 'y' values to find a peak in.
        threshold (int): Threshold in ADC counts required for peaks.

    Returns:
       list: Peaks

    """
    extrema = scipy.signal.argrelmax(values)
    high_extrema = []
    for i in extrema:
        if values[i] > threshold:
            high_extrema.append(i)
    return high_extrema

#@profile
def get_sum_waveform(cursor, offset, n_samples):
    """Get inverted sum waveform from mongo

    Args:
        cursor (iterable):  An iterable object of documents containing Caen
                           blocks.  This can be a pymongo Cursor.
        offset (int): An integer start time
        n_samples (int): How many samples to store

    Returns:
       dict: Results dictionary with key 'size' and 'occurences'

    """
    log = logging.getLogger('waveform')
    # Longer int since summing many, otherwise wrap around.
    # TODO: check that unsigned 16 bit doesn't work?  Or bit shift (i.e. avg) or
    # dividie by some nubmer
    #log.debug('Number of samples for sum waveform: %d', n_samples)
    import time

    some_time = time.time()
    #occurences = np.zeros(n_samples, dtype=np.int16)
    #print('It took', (time.time() - some_time), 'to allocate memory')
    size = 0
    scale = 10 # how to scale samples
    assert CaenBlockParsing.setup_sum_waveform_buffer(n_samples);
    CaenBlockParsing.setup_return_buffer(1000)

    for doc in cursor:
        data = xedb.get_data_from_doc(doc)
        time_correction = doc['triggertime'] - offset

        temp_size = InterfaceV1724.get_block_size(data, False)

        #  2 bytes are a sample
        #result = InterfaceV1724Swig.get_waveform(data, )

        n = int(temp_size * 2)
        a = np.fromstring(data, dtype='uint32')

        assert (len(data) != 0)
        print(doc['_id'])
        CaenBlockParsing.inplace(a)
        CaenBlockParsing.put_samples_into_occurences(time_correction, scale)

        size += 4 * temp_size

    occurences = CaenBlockParsing.get_sum_waveform(n_samples)
    #print("size", size)
    #print('len', len(occurences[1]), n_samples)
    results = {}
    results['size'] = size
    results['occurences'] = occurences[1]
    #raise ValueError()
    #print(occurences)
    #print('sum', np.sum(occurences[1]))

    return results
<commit_msg>debugging<commit_after># cito - The Xenon1T experiments software trigger
# Copyright 2013.  All rights reserved.
# https://github.com/tunnell/cito
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are
# met:
#
#     * Redistributions of source code must retain the above copyright
# notice, this list of conditions and the following disclaimer.
#     * Redistributions in binary form must reproduce the above
# copyright notice, this list of conditions and the following disclaimer
# in the documentation and/or other materials provided with the
# distribution.
#     * Neither the name of the Xenon experiment, nor the names of its
# contributors may be used to endorse or promote products derived from
# this software without specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
"""Perform operations on sets of blocks
"""

import logging

import numpy as np
from scipy import signal
from scipy.stats import norm
import scipy

from cito.helpers import xedb
from cito.helpers import InterfaceV1724Swig
from cito.helpers import InterfaceV1724
from cito.helpers import CaenBlockParsing

def filter_samples(values):
    """Apply a filter

    :rtype : dict
    :param values: Values to filter
    :type values: np.array
    """
    new_values = np.zeros_like(values)

    print('\tprefilter')
    my_filter = norm(0, 100)  # mu=0, sigma=100
    filter_values = 600 * [0.]
    for j in range(-300, 300):
        filter_values[j] = my_filter.pdf(j)

    print('\tapply_filter')
    for i in range(values.size):
        for j in range(-300, 300):
            if j > 0 and j < values.size:
                new_values += values[i] * filter_values[j]

    return new_values

def find_peaks(values, threshold=10000):
    """Find peaks within list of values.

    Uses scipy to find peaks above a threshold.

    Args:
        values (list):  The 'y' values to find a peak in.
        threshold (int): Threshold in ADC counts required for peaks.

    Returns:
       list: Peaks

    """
    extrema = scipy.signal.argrelmax(values)
    high_extrema = []
    for i in extrema:
        if values[i] > threshold:
            high_extrema.append(i)
    return high_extrema

#@profile
def get_sum_waveform(cursor, offset, n_samples):
    """Get inverted sum waveform from mongo

    Args:
        cursor (iterable):  An iterable object of documents containing Caen
                           blocks.  This can be a pymongo Cursor.
        offset (int): An integer start time
        n_samples (int): How many samples to store

    Returns:
       dict: Results dictionary with key 'size' and 'occurences'

    """
    log = logging.getLogger('waveform')
    # Longer int since summing many, otherwise wrap around.
    # TODO: check that unsigned 16 bit doesn't work?  Or bit shift (i.e. avg) or
    # dividie by some nubmer
    #log.debug('Number of samples for sum waveform: %d', n_samples)
    import time

    some_time = time.time()
    #occurences = np.zeros(n_samples, dtype=np.int16)
    #print('It took', (time.time() - some_time), 'to allocate memory')
    size = 0
    scale = 10 # how to scale samples
    assert CaenBlockParsing.setup_sum_waveform_buffer(n_samples);
    CaenBlockParsing.setup_return_buffer(1000)

    for doc in cursor:
        data = xedb.get_data_from_doc(doc)
        time_correction = doc['triggertime'] - offset

        temp_size = InterfaceV1724.get_block_size(data, False)

        #  2 bytes are a sample
        #result = InterfaceV1724Swig.get_waveform(data, )

        n = int(temp_size * 2)
        a = np.fromstring(data, dtype='uint32')

        #assert (len(data) != 0)
        print(doc['_id'])
        CaenBlockParsing.inplace(a)
        CaenBlockParsing.put_samples_into_occurences(time_correction, scale)

        size += 4 * temp_size

    occurences = CaenBlockParsing.get_sum_waveform(n_samples)
    #print("size", size)
    #print('len', len(occurences[1]), n_samples)
    results = {}
    results['size'] = size
    results['occurences'] = occurences[1]
    #raise ValueError()
    #print(occurences)
    #print('sum', np.sum(occurences[1]))

    return results
<|endoftext|>


<commit_before>import json
from math import ceil
import subprocess
import sys

def bit_mask(w):
    """
    :note: duplication with pyMathBitPrecise in order to keep this script without non-std dependencies
    """
    return (1 << w) - 1

def select_bit_range(val: int, bits_start: int, bits_len: int):
    """
    Get sequence of bits from an int value

    :note: duplication with pyMathBitPrecise in order to keep this script without non-std dependencies
    """
    val >>= bits_start
    return val & bit_mask(bits_len)

def words_to_int(words, word_size, size):
    end_bytes = size % word_size
    if end_bytes != 0:
        words[-1] &= bit_mask(8 * end_bytes)

    res = 0
    for w in reversed(words):
        res <<= 8 * word_size
        res |= w
    return res

class DebugBusMonitorCtl():
    """
    A tool used to interact with a :class:`hwtLib.abstract.debug_bus_monitor.DebugBusMonitor`
    """
    REG_NAME_MEMORY_SIZE = 0x0
    REG_NAME_MEMORY_OFFSET = 0x4
    REG_DATA_MEMORY = 0x8

    def __init__(self, addr):
        self.addr = addr
        self.name_memory = None
        self.data_memory_size = None

    def load_name_memory(self):
        size = self.read_int(self.REG_NAME_MEMORY_SIZE, 4)
        offset = self.read_int(self.REG_NAME_MEMORY_OFFSET, 4)
        name_memory = self.read(offset, size)
        name_memory = (name_memory).decode("utf-8")
        name_memory = json.loads(name_memory)
        data_width = self.get_data_memory_width(name_memory)
        self.name_memory = name_memory
        self.data_memory_size = ceil(data_width / 8)

    def get_data_memory_width(self, name_memory):
        if isinstance(name_memory, list):
            offset, width = name_memory
            return offset + width
        else:
            w = 0
            for _, v in name_memory.items():
                w = max(w, self.get_data_memory_width(v))
            return w

    def read_int(self, addr, size):
        v = self.read(addr, size)
        v = int.from_bytes(v, "little")
        return v

    def read(self, addr, size):
        addr += self.addr
        raise NotImplementedError("Implement this in your implementation of this abstract class")

    def _dump_txt_indent(self, out, indent):
        for _ in range(indent):
            out.write("  ")

    def _dump_txt(self, out, name_memory, data, indent):
        for k, v in name_memory.items():
            if isinstance(v, list):
                self._dump_txt_indent(out, indent)
                out.write(k)
                out.write(": ")
                bits_start, bits_len = v
                val = select_bit_range(data, bits_start, bits_len)
                if bits_len == 1:
                    out.write(f"{val}:d")
                else:
                    out.write(f"0x{val:x}")
                out.write("\n")
            else:
                self._dump_txt_indent(out, indent)
                out.write(k)
                out.write(":\n")
                self._dump_txt(out, v, data, indent + 1)

    def dump_txt(self, out=sys.stdout):
        if self.name_memory is None:
            self.load_name_memory()

        data = self.read_int(self.REG_DATA_MEMORY, self.data_memory_size)
        self._dump_txt(out, self.name_memory, data, 0)

class DebugBusMonitorCtlDevmem(DebugBusMonitorCtl):

    def __init__(self, addr):
        DebugBusMonitorCtl.__init__(self, addr)
        self.devmem = "devmem"

    def read(self, addr, size):
        addr += self.addr
        word_size = 0x4
        words = []
        for _ in range(ceil(size / word_size)):
            s = subprocess.check_output([self.devmem, f"0x{addr:x}"])
            s = s.decode("utf-8")
            d = int(s.strip(), 16)
            words.append(d)
            addr += word_size

        return words_to_int(words, word_size, size).to_bytes(size, "little")

if __name__ == "__main__":
    import argparse
    parser = argparse.ArgumentParser(description='Dump a values from DebugBusMonitor instance.')
    parser.add_argument('address', metavar='ADDR', type=lambda x: int(x,0),
                        help='base address of component')
    parser.add_argument('--devmem', default="devmem", type=str,
                        help='the devmem tool to use')
    parser.add_argument('--memory-desc', dest='mem_desc', default=None, type=str,
                        help='path to a file with a json specification of memory space of the signals')

    args = parser.parse_args()
    db = DebugBusMonitorCtlDevmem(args.address)
    db.devmem = args.devmem
    if args.mem_desc:
        with open(args.mem_desc) as fp:
            name_memory = json.load(fp)
        data_width = db.get_data_memory_width(name_memory)
        db.name_memory = name_memory
        db.data_memory_size = ceil(data_width / 8)

    db.dump_txt(sys.stdout)
<commit_msg>DebugBusMonitorCtl: fix format after f-string rewrite<commit_after>import json
from math import ceil
import subprocess
import sys

def bit_mask(w):
    """
    :note: duplication with pyMathBitPrecise in order to keep this script without non-std dependencies
    """
    return (1 << w) - 1

def select_bit_range(val: int, bits_start: int, bits_len: int):
    """
    Get sequence of bits from an int value

    :note: duplication with pyMathBitPrecise in order to keep this script without non-std dependencies
    """
    val >>= bits_start
    return val & bit_mask(bits_len)

def words_to_int(words, word_size, size):
    end_bytes = size % word_size
    if end_bytes != 0:
        words[-1] &= bit_mask(8 * end_bytes)

    res = 0
    for w in reversed(words):
        res <<= 8 * word_size
        res |= w
    return res

class DebugBusMonitorCtl():
    """
    A tool used to interact with a :class:`hwtLib.abstract.debug_bus_monitor.DebugBusMonitor`
    """
    REG_NAME_MEMORY_SIZE = 0x0
    REG_NAME_MEMORY_OFFSET = 0x4
    REG_DATA_MEMORY = 0x8

    def __init__(self, addr):
        self.addr = addr
        self.name_memory = None
        self.data_memory_size = None

    def load_name_memory(self):
        size = self.read_int(self.REG_NAME_MEMORY_SIZE, 4)
        offset = self.read_int(self.REG_NAME_MEMORY_OFFSET, 4)
        name_memory = self.read(offset, size)
        name_memory = (name_memory).decode("utf-8")
        name_memory = json.loads(name_memory)
        data_width = self.get_data_memory_width(name_memory)
        self.name_memory = name_memory
        self.data_memory_size = ceil(data_width / 8)

    def get_data_memory_width(self, name_memory):
        if isinstance(name_memory, list):
            offset, width = name_memory
            return offset + width
        else:
            w = 0
            for _, v in name_memory.items():
                w = max(w, self.get_data_memory_width(v))
            return w

    def read_int(self, addr, size):
        v = self.read(addr, size)
        v = int.from_bytes(v, "little")
        return v

    def read(self, addr, size):
        addr += self.addr
        raise NotImplementedError("Implement this in your implementation of this abstract class")

    def _dump_txt_indent(self, out, indent):
        for _ in range(indent):
            out.write("  ")

    def _dump_txt(self, out, name_memory, data, indent):
        for k, v in name_memory.items():
            if isinstance(v, list):
                self._dump_txt_indent(out, indent)
                out.write(k)
                out.write(": ")
                bits_start, bits_len = v
                val = select_bit_range(data, bits_start, bits_len)
                if bits_len == 1:
                    out.write(f"{val:d}")
                else:
                    out.write(f"0x{val:x}")
                out.write("\n")
            else:
                self._dump_txt_indent(out, indent)
                out.write(k)
                out.write(":\n")
                self._dump_txt(out, v, data, indent + 1)

    def dump_txt(self, out=sys.stdout):
        if self.name_memory is None:
            self.load_name_memory()

        data = self.read_int(self.REG_DATA_MEMORY, self.data_memory_size)
        self._dump_txt(out, self.name_memory, data, 0)

class DebugBusMonitorCtlDevmem(DebugBusMonitorCtl):

    def __init__(self, addr):
        DebugBusMonitorCtl.__init__(self, addr)
        self.devmem = "devmem"

    def read(self, addr, size):
        addr += self.addr
        word_size = 0x4
        words = []
        for _ in range(ceil(size / word_size)):
            s = subprocess.check_output([self.devmem, f"0x{addr:x}"])
            s = s.decode("utf-8")
            d = int(s.strip(), 16)
            words.append(d)
            addr += word_size

        return words_to_int(words, word_size, size).to_bytes(size, "little")

if __name__ == "__main__":
    import argparse
    parser = argparse.ArgumentParser(description='Dump a values from DebugBusMonitor instance.')
    parser.add_argument('address', metavar='ADDR', type=lambda x: int(x,0),
                        help='base address of component')
    parser.add_argument('--devmem', default="devmem", type=str,
                        help='the devmem tool to use')
    parser.add_argument('--memory-desc', dest='mem_desc', default=None, type=str,
                        help='path to a file with a json specification of memory space of the signals')

    args = parser.parse_args()
    db = DebugBusMonitorCtlDevmem(args.address)
    db.devmem = args.devmem
    if args.mem_desc:
        with open(args.mem_desc) as fp:
            name_memory = json.load(fp)
        data_width = db.get_data_memory_width(name_memory)
        db.name_memory = name_memory
        db.data_memory_size = ceil(data_width / 8)

    db.dump_txt(sys.stdout)
<|endoftext|>


<commit_before>import time
import hashlib
import isodate
import logging

from util import *
from twisted.internet import task
from twisted.internet.defer import inlineCallbacks, returnValue
from autobahn.websocket import WebSocketServerProtocol, WebSocketServerFactory
from autobahn.resource import WebSocketResource

SEND_REGISTRATIONS = 10

YOUTUBE_STATS_URL = 'https://www.googleapis.com/youtube/v3/videos?part=contentDetails,statistics&id={id}&key={api_key}'
SOUNDCLOUD_STATS_URL = 'http://api.soundcloud.com/tracks/{id}?client_id={api_key}'

class BillyRadioProtocol(WebSocketServerProtocol):

    def onConnect(self, request):
        self.factory.join(request.peer, self)

    def onMessage(self, payload, isBinary):
        if not isBinary:
            json_payload = json.loads(payload)
            type = json_payload.get('type', None)
            if type == 'register' and 'name' in json_payload and 'radio_id' in json_payload:
                reactor.callLater(0, self.factory.register, self.peer, json_payload['name'], json_payload['radio_id'])
            elif type == 'unregister' and 'radio_id' in json_payload:
                self.factory.unregister(self.peer, json_payload['radio_id'])

    def connectionLost(self, reason):
        WebSocketServerProtocol.connectionLost(self, reason)
        self.factory.leave(self.peer)

class BillyRadioFactory(WebSocketServerFactory):

    def __init__(self, *args, **kwargs):
        self.database = kwargs.pop('database')
        self.config = kwargs.pop('config')

        WebSocketServerFactory.__init__(self, *args, **kwargs)

        self.logger = logging.getLogger(__name__)

        self.connections = {}
        self.stations = {}

        task.LoopingCall(self.fetch_playlist).start(300, now=False)
        task.LoopingCall(self.send_status).start(30)

    def join(self, peer, connection):
        self.connections[peer] = connection
        self.logger.debug('Peer %s joined', peer)

    def leave(self, peer):
        if peer in self.connections:
            del self.connections[peer]
            self.logger.debug('Peer %s left', peer)

        for radio_id in self.stations.keys():
            self.unregister(peer, radio_id)

    def send(self, message, peers=None):
        if peers is not None:
            connections = [self.connections[peer] for peer in peers if peer in self.connections]
        else:
            connections = self.connections.values()

        for connection in connections:
            connection.sendMessage(json.dumps(message))

    def send_status(self, radio_id=None, peers=None):
        stations = {radio_id: self.stations[radio_id]} if radio_id != None else self.stations
        for radio_id, station in stations.iteritems():
            if station.start_time > 0:
                self.send({'type': 'status',
                           'radio_id': radio_id,
                           'position': station.get_play_position()}, peers or station.get_peers())

    def send_data(self, radio_id, peers=None):
        station = self.stations[radio_id]
        self.send({'type': 'data',
                   'radio_id': radio_id,
                   'tracks': station.get_tracks(),
                   'registrations': station.get_registrations()}, peers)

    @inlineCallbacks
    def register(self, peer, user_name, radio_id):
        if radio_id not in self.stations:
            self.stations[radio_id] = BillyRadioStation(radio_id, self.config, self.database)
            yield self.stations[radio_id].update_tracks()

        station = self.stations[radio_id]
        peers = station.get_peers()
        station.register(peer, user_name)
        self.send_data(radio_id, [peer])
        self.send_status(radio_id, [peer])

        registration = station.get_registration(peer)
        message = {'type': 'registered',
                   'user_id': registration['user_id'],
                   'user_name': registration['user_name'],
                   'radio_id': radio_id,
                   'time': registration['time']}
        self.send(message, peers=peers)

    def unregister(self, peer, radio_id):
        station = self.stations[radio_id]
        registration = station.unregister(peer)
        if registration:
            message = {'type': 'unregistered',
                       'user_id': registration['user_id'],
                       'radio_id': radio_id}
            self.send(message, station.get_peers())

    @inlineCallbacks
    def fetch_playlist(self):
        self.logger.info('Checking tracks')

        for radio_id, station in self.stations.iteritems():
            updated = yield station.update_tracks()

            if updated:
                # Notify everyone
                self.send_data(radio_id)
                self.send_status(radio_id)

            self.logger.info('Done checking tracks')

class BillyRadioStation(object):
    def __init__(self, radio_id, config, database):
        self.logger = logging.getLogger(__name__)

        self.start_time = 0
        self.radio_id = radio_id
        self.config = config
        self.database = database
        self.listeners = {}
        self.tracks = []

        radio = self.database.get_radio(self.radio_id)
        self.session_id = radio['session_id']
        self.playlist_name = radio['playlist_name']

    def register(self, peer, user_name):
        self.listeners[peer] = {'user_id': hashlib.sha1(str(peer)).hexdigest(),
                                'user_name': user_name,
                                'time': int(time.time())}

    def unregister(self, peer):
        return self.listeners.pop(peer, None)

    def get_registration(self, peer):
        return self.listeners.get(peer, None)

    def get_registrations(self, limit=SEND_REGISTRATIONS):
        return sorted(self.listeners.values(), key=lambda x: x['time'])[:limit]

    def get_play_position(self):
        play_position = int((time.time() - self.start_time))

        track_index = 0
        track = self.tracks[0]
        while play_position > track['duration']:
            play_position -= track['duration']
            track_index += 1
            track_index = track_index % len(self.tracks)
            track = self.tracks[track_index]

        return (track_index, play_position)

    @inlineCallbacks
    def update_tracks(self):
        self.logger.info('Checking tracks for radio %s', self.radio_id)

        session = self.database.get_session(self.session_id)
        tracks = session['playlists'][self.playlist_name]['tracks']

        # Have the tracks changed?
        if [t['link'] for t in self.tracks] != [t['link'] for t in tracks]:
            # Update stats
            yt_api_key = self.config.get('sources', 'youtube_api_key')
            sc_api_key = self.config.get('sources', 'soundcloud_api_key')

            for track in tracks:
                if track['link'].startswith('youtube:'):
                    url = YOUTUBE_STATS_URL.format(api_key=yt_api_key, id=track['link'][8:])
                    response = yield get_request(url)
                    response_dict = response.json
                    track['duration'] = int(isodate.parse_duration(response_dict['items'][0]['contentDetails']['duration']).total_seconds())
                    track['musicinfo'] = track.get('musicinfo', {})
                    track['musicinfo']['playback_count'] = response_dict['items'][0]['statistics']['viewCount']
                    track['musicinfo']['comment_count'] = response_dict['items'][0]['statistics']['commentCount']

                elif track['link'].startswith('soundcloud:'):
                    url = SOUNDCLOUD_STATS_URL.format(api_key=sc_api_key, id=track['link'][11:])
                    response = yield get_request(url)
                    response_dict = response.json
                    track['duration'] = response_dict['duration'] / 1000
                    track['musicinfo'] = track.get('musicinfo', {})
                    track['musicinfo']['playback_count'] = response_dict['playback_count']
                    track['musicinfo']['comment_count'] = response_dict['comment_count']
                    track['musicinfo']['favorite_count'] = response_dict['favoritings_count']

            # Update tracks
            self.tracks = tracks
            self.start_time = int(time.time())

            returnValue(True)
        returnValue(False)

    def get_tracks(self):
        return self.tracks

    def get_peers(self):
        return self.listeners.keys()

<commit_msg>Notify radio client with unregister is successful<commit_after>import time
import hashlib
import isodate
import logging

from util import *
from twisted.internet import task
from twisted.internet.defer import inlineCallbacks, returnValue
from autobahn.websocket import WebSocketServerProtocol, WebSocketServerFactory
from autobahn.resource import WebSocketResource

SEND_REGISTRATIONS = 10

YOUTUBE_STATS_URL = 'https://www.googleapis.com/youtube/v3/videos?part=contentDetails,statistics&id={id}&key={api_key}'
SOUNDCLOUD_STATS_URL = 'http://api.soundcloud.com/tracks/{id}?client_id={api_key}'

class BillyRadioProtocol(WebSocketServerProtocol):

    def onConnect(self, request):
        self.factory.join(request.peer, self)

    def onMessage(self, payload, isBinary):
        if not isBinary:
            json_payload = json.loads(payload)
            type = json_payload.get('type', None)
            if type == 'register' and 'name' in json_payload and 'radio_id' in json_payload:
                reactor.callLater(0, self.factory.register, self.peer, json_payload['name'], json_payload['radio_id'])
            elif type == 'unregister' and 'radio_id' in json_payload:
                self.factory.unregister(self.peer, json_payload['radio_id'])

    def connectionLost(self, reason):
        WebSocketServerProtocol.connectionLost(self, reason)
        self.factory.leave(self.peer)

class BillyRadioFactory(WebSocketServerFactory):

    def __init__(self, *args, **kwargs):
        self.database = kwargs.pop('database')
        self.config = kwargs.pop('config')

        WebSocketServerFactory.__init__(self, *args, **kwargs)

        self.logger = logging.getLogger(__name__)

        self.connections = {}
        self.stations = {}

        task.LoopingCall(self.fetch_playlist).start(300, now=False)
        task.LoopingCall(self.send_status).start(30)

    def join(self, peer, connection):
        self.connections[peer] = connection
        self.logger.debug('Peer %s joined', peer)

    def leave(self, peer):
        if peer in self.connections:
            del self.connections[peer]
            self.logger.debug('Peer %s left', peer)

        for radio_id in self.stations.keys():
            self.unregister(peer, radio_id)

    def send(self, message, peers=None):
        if peers is not None:
            connections = [self.connections[peer] for peer in peers if peer in self.connections]
        else:
            connections = self.connections.values()

        for connection in connections:
            connection.sendMessage(json.dumps(message))

    def send_status(self, radio_id=None, peers=None):
        stations = {radio_id: self.stations[radio_id]} if radio_id != None else self.stations
        for radio_id, station in stations.iteritems():
            if station.start_time > 0:
                self.send({'type': 'status',
                           'radio_id': radio_id,
                           'position': station.get_play_position()}, peers or station.get_peers())

    def send_data(self, radio_id, peers=None):
        station = self.stations[radio_id]
        self.send({'type': 'data',
                   'radio_id': radio_id,
                   'tracks': station.get_tracks(),
                   'registrations': station.get_registrations()}, peers)

    @inlineCallbacks
    def register(self, peer, user_name, radio_id):
        if radio_id not in self.stations:
            self.stations[radio_id] = BillyRadioStation(radio_id, self.config, self.database)
            yield self.stations[radio_id].update_tracks()

        station = self.stations[radio_id]
        peers = station.get_peers()
        station.register(peer, user_name)
        self.send_data(radio_id, [peer])
        self.send_status(radio_id, [peer])

        registration = station.get_registration(peer)
        message = {'type': 'registered',
                   'user_id': registration['user_id'],
                   'user_name': registration['user_name'],
                   'radio_id': radio_id,
                   'time': registration['time']}
        self.send(message, peers=peers)

    def unregister(self, peer, radio_id):
        station = self.stations[radio_id]
        peers = station.get_peers()
        registration = station.unregister(peer)
        if registration:
            message = {'type': 'unregistered',
                       'user_id': registration['user_id'],
                       'radio_id': radio_id}
            self.send(message, peers=peers)

    @inlineCallbacks
    def fetch_playlist(self):
        self.logger.info('Checking tracks')

        for radio_id, station in self.stations.iteritems():
            updated = yield station.update_tracks()

            if updated:
                # Notify everyone
                self.send_data(radio_id)
                self.send_status(radio_id)

            self.logger.info('Done checking tracks')

class BillyRadioStation(object):
    def __init__(self, radio_id, config, database):
        self.logger = logging.getLogger(__name__)

        self.start_time = 0
        self.radio_id = radio_id
        self.config = config
        self.database = database
        self.listeners = {}
        self.tracks = []

        radio = self.database.get_radio(self.radio_id)
        self.session_id = radio['session_id']
        self.playlist_name = radio['playlist_name']

    def register(self, peer, user_name):
        self.listeners[peer] = {'user_id': hashlib.sha1(str(peer)).hexdigest(),
                                'user_name': user_name,
                                'time': int(time.time())}

    def unregister(self, peer):
        return self.listeners.pop(peer, None)

    def get_registration(self, peer):
        return self.listeners.get(peer, None)

    def get_registrations(self, limit=SEND_REGISTRATIONS):
        return sorted(self.listeners.values(), key=lambda x: x['time'])[:limit]

    def get_play_position(self):
        play_position = int((time.time() - self.start_time))

        track_index = 0
        track = self.tracks[0]
        while play_position > track['duration']:
            play_position -= track['duration']
            track_index += 1
            track_index = track_index % len(self.tracks)
            track = self.tracks[track_index]

        return (track_index, play_position)

    @inlineCallbacks
    def update_tracks(self):
        self.logger.info('Checking tracks for radio %s', self.radio_id)

        session = self.database.get_session(self.session_id)
        tracks = session['playlists'][self.playlist_name]['tracks']

        # Have the tracks changed?
        if [t['link'] for t in self.tracks] != [t['link'] for t in tracks]:
            # Update stats
            yt_api_key = self.config.get('sources', 'youtube_api_key')
            sc_api_key = self.config.get('sources', 'soundcloud_api_key')

            for track in tracks:
                if track['link'].startswith('youtube:'):
                    url = YOUTUBE_STATS_URL.format(api_key=yt_api_key, id=track['link'][8:])
                    response = yield get_request(url)
                    response_dict = response.json
                    track['duration'] = int(isodate.parse_duration(response_dict['items'][0]['contentDetails']['duration']).total_seconds())
                    track['musicinfo'] = track.get('musicinfo', {})
                    track['musicinfo']['playback_count'] = response_dict['items'][0]['statistics']['viewCount']
                    track['musicinfo']['comment_count'] = response_dict['items'][0]['statistics']['commentCount']

                elif track['link'].startswith('soundcloud:'):
                    url = SOUNDCLOUD_STATS_URL.format(api_key=sc_api_key, id=track['link'][11:])
                    response = yield get_request(url)
                    response_dict = response.json
                    track['duration'] = response_dict['duration'] / 1000
                    track['musicinfo'] = track.get('musicinfo', {})
                    track['musicinfo']['playback_count'] = response_dict['playback_count']
                    track['musicinfo']['comment_count'] = response_dict['comment_count']
                    track['musicinfo']['favorite_count'] = response_dict['favoritings_count']

            # Update tracks
            self.tracks = tracks
            self.start_time = int(time.time())

            returnValue(True)
        returnValue(False)

    def get_tracks(self):
        return self.tracks

    def get_peers(self):
        return self.listeners.keys()

<|endoftext|>


<commit_before># This file is part of the Juju GUI, which lets users view and manage Juju
# environments within a graphical interface (https://launchpad.net/juju-gui).
# Copyright (C) 2013 Canonical Ltd.
#
# This program is free software: you can redistribute it and/or modify it under
# the terms of the GNU Affero General Public License version 3, as published by
# the Free Software Foundation.
#
# This program is distributed in the hope that it will be useful, but WITHOUT
# ANY WARRANTY; without even the implied warranties of MERCHANTABILITY,
# SATISFACTORY QUALITY, or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
# Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program.  If not, see <http://www.gnu.org/licenses/>.

"""Juju GUI server distribution file."""
<commit_msg>Add setup file.<commit_after># This file is part of the Juju GUI, which lets users view and manage Juju
# environments within a graphical interface (https://launchpad.net/juju-gui).
# Copyright (C) 2013 Canonical Ltd.
#
# This program is free software: you can redistribute it and/or modify it under
# the terms of the GNU Affero General Public License version 3, as published by
# the Free Software Foundation.
#
# This program is distributed in the hope that it will be useful, but WITHOUT
# ANY WARRANTY; without even the implied warranties of MERCHANTABILITY,
# SATISFACTORY QUALITY, or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
# Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program.  If not, see <http://www.gnu.org/licenses/>.

"""Juju GUI server distribution file."""

from distutils.core import setup
import os

ROOT = os.path.abspath(os.path.dirname(__file__))
PROJECT_NAME = 'guiserver'

project = __import__(PROJECT_NAME)
readme_path = os.path.join(ROOT, '..', 'README.md')

os.chdir(ROOT)
setup(
    name=PROJECT_NAME,
    version=project.get_version(),
    description=project.__doc__,
    long_description=open(readme_path).read(),
    author='The Juju GUI team',
    author_email='juju-gui@lists.ubuntu.com',
    url='https://launchpad.net/juju-gui',
    keywords='juju gui server',
    packages=[
        PROJECT_NAME,
        '{0}.tests'.format(PROJECT_NAME),
    ],
    scripts=['runserver.py'],
    classifiers=[
        'Development Status :: 3 - Alpha',
        'Environment :: Web Environment',
        'Intended Audience :: System Administrators',
        'License :: OSI Approved :: GNU Affero General Public License v3',
        'Operating System :: POSIX',
        'Programming Language :: Python',
        'Topic :: System :: Installation/Setup',
    ],
)
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
##############################################################################
#
#    Copyright (C) 2012+ BREMSKERL-REIBBELAGWERKE EMMERLING GmbH & Co. KG
#    Author Marco Dieckhoff
#    Copyright (C) 2013 Agile Business Group sagl (<http://www.agilebg.com>)
#
#    This program is free software: you can redistribute it and/or modify
#    it under the terms of the GNU Affero General Public License as
#    published by the Free Software Foundation, either version 3 of the
#    License, or (at your option) any later version.
#
#    This program is distributed in the hope that it will be useful,
#    but WITHOUT ANY WARRANTY; without even the implied warranty of
#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#    GNU Affero General Public License for more details.
#
#    You should have received a copy of the GNU Affero General Public License
#    along with this program.  If not, see <http://www.gnu.org/licenses/>.
#
##############################################################################
{
    "name": "Allows back-dating of stock moves",
    "version": "1.0",
    'author' : 'Marco Dieckhoff, BREMSKERL - Agile Business Group',
    "category": "Stock Logistics",
    'website' : 'www.bremskerl.com - www.agilebg.com',
    "depends": ["stock"],
    "description": "Allows back-dating of stock moves",
    'data': ["view/stock_view.xml"],
    'demo': [],
    'installable': True,
}<commit_msg>[FIX] authors<commit_after># -*- coding: utf-8 -*-
##############################################################################
#
#    Copyright (C) 2012+ BREMSKERL-REIBBELAGWERKE EMMERLING GmbH & Co. KG
#    Author Marco Dieckhoff
#    Copyright (C) 2013 Agile Business Group sagl (<http://www.agilebg.com>)
#
#    This program is free software: you can redistribute it and/or modify
#    it under the terms of the GNU Affero General Public License as
#    published by the Free Software Foundation, either version 3 of the
#    License, or (at your option) any later version.
#
#    This program is distributed in the hope that it will be useful,
#    but WITHOUT ANY WARRANTY; without even the implied warranty of
#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#    GNU Affero General Public License for more details.
#
#    You should have received a copy of the GNU Affero General Public License
#    along with this program.  If not, see <http://www.gnu.org/licenses/>.
#
##############################################################################
{
    "name": "Allows back-dating of stock moves",
    "version": "1.0",
    'author' : ['Marco Dieckhoff, BREMSKERL', 'Agile Business Group'],
    "category": "Stock Logistics",
    'website' : 'www.bremskerl.com',
    "depends": ["stock"],
    "description": "Allows back-dating of stock moves",
    'data': ["view/stock_view.xml"],
    'demo': [],
    'installable': True,
}
<|endoftext|>


<commit_before># Copyright 2019 Open Source Integrators
# Copyright 2019 Eficent Business and IT Consulting Services, S.L.
# License LGPL-3.0 or later (http://www.gnu.org/licenses/agpl).

{
    "name": "Stock Request Submit",
    "summary": "Add submit state on Stock Requests",
    "version": "12.0.1.0.0",
    "license": "LGPL-3",
    "website": "https://github.com/stock-logistics-warehouse",
    "author": "Open Source Integrators, "
              "Odoo Community Association (OCA)",
    "category": "Warehouse Management",
    'depends': ['stock_request'],
    'data': [
        'views/stock_request_order_views.xml',
        'views/stock_request_views.xml',
    ],
    "installable": True,
    'uninstall_hook': 'uninstall_hook',
}
<commit_msg>stock_request_submit 12.0.1.0.1<commit_after># Copyright 2019 Open Source Integrators
# Copyright 2019 Eficent Business and IT Consulting Services, S.L.
# License LGPL-3.0 or later (http://www.gnu.org/licenses/agpl).

{
    "name": "Stock Request Submit",
    "summary": "Add submit state on Stock Requests",
    "version": "12.0.1.0.1",
    "license": "LGPL-3",
    "website": "https://github.com/stock-logistics-warehouse",
    "author": "Open Source Integrators, "
              "Odoo Community Association (OCA)",
    "category": "Warehouse Management",
    'depends': ['stock_request'],
    'data': [
        'views/stock_request_order_views.xml',
        'views/stock_request_views.xml',
    ],
    "installable": True,
    'uninstall_hook': 'uninstall_hook',
}
<|endoftext|>