<commit_before># Copyright (c) 2012-2013, 2017-2019 ARM Limited
# All rights reserved.
#
# The license below extends only to copyright in the software and shall
# not be construed as granting a license to any other intellectual
# property including but not limited to intellectual property relating
# to a hardware implementation of the functionality of the software
# licensed hereunder.  You may use the software subject to the license
# terms below provided that you ensure that this notice is replicated
# unmodified and in its entirety in all distributions of the software,
# modified or unmodified, in source code or in binary form.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are
# met: redistributions of source code must retain the above copyright
# notice, this list of conditions and the following disclaimer;
# redistributions in binary form must reproduce the above copyright
# notice, this list of conditions and the following disclaimer in the
# documentation and/or other materials provided with the distribution;
# neither the name of the copyright holders nor the names of its
# contributors may be used to endorse or promote products derived from
# this software without specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#
# Authors: Andreas Sandberg

from m5.params import *
from m5.proxy import *
from m5.util.fdthelper import *
from m5.SimObject import SimObject

from m5.objects.Device import PioDevice, BasicPioDevice
from m5.objects.Platform import Platform

class BaseGic(PioDevice):
    type = 'BaseGic'
    abstract = True
    cxx_header = "dev/arm/base_gic.hh"

    # Used for DTB autogeneration
    _state = FdtState(addr_cells=0, interrupt_cells=3)

    platform = Param.Platform(Parent.any, "Platform this device is part of.")

    gicd_iidr = Param.UInt32(0,
        "Distributor Implementer Identification Register")
    gicd_pidr = Param.UInt32(0,
        "Peripheral Identification Register")
    gicc_iidr = Param.UInt32(0,
        "CPU Interface Identification Register")
    gicv_iidr = Param.UInt32(0,
        "VM CPU Interface Identification Register")

    def interruptCells(self, int_type, int_num, int_flag):
        """
        Interupt cells generation helper:
        Following specifications described in

        Documentation/devicetree/bindings/interrupt-controller/arm,gic.txt
        """
        assert self._state.interrupt_cells == 3
        return [ int_type, int_num, int_flag ]

class ArmInterruptPin(SimObject):
    type = 'ArmInterruptPin'
    cxx_header = "dev/arm/base_gic.hh"
    cxx_class = "ArmInterruptPinGen"
    abstract = True

    platform = Param.Platform(Parent.any, "Platform with interrupt controller")
    num = Param.UInt32("Interrupt number in GIC")

class ArmSPI(ArmInterruptPin):
    type = 'ArmSPI'
    cxx_header = "dev/arm/base_gic.hh"
    cxx_class = "ArmSPIGen"

class ArmPPI(ArmInterruptPin):
    type = 'ArmPPI'
    cxx_header = "dev/arm/base_gic.hh"
    cxx_class = "ArmPPIGen"

class GicV2(BaseGic):
    type = 'GicV2'
    cxx_header = "dev/arm/gic_v2.hh"

    dist_addr = Param.Addr("Address for distributor")
    cpu_addr = Param.Addr("Address for cpu")
    cpu_size = Param.Addr(0x2000, "Size of cpu register bank")
    dist_pio_delay = Param.Latency('10ns', "Delay for PIO r/w to distributor")
    cpu_pio_delay = Param.Latency('10ns', "Delay for PIO r/w to cpu interface")
    int_latency = Param.Latency('10ns', "Delay for interrupt to get to CPU")
    it_lines = Param.UInt32(128, "Number of interrupt lines supported (max = 1020)")
    gem5_extensions = Param.Bool(False, "Enable gem5 extensions")

class Gic400(GicV2):
    """
    As defined in:
    "ARM Generic Interrupt Controller Architecture" version 2.0
    "CoreLink GIC-400 Generic Interrupt Controller" revision r0p1
    """
    gicd_pidr = 0x002bb490
    gicd_iidr = 0x0200143B
    gicc_iidr = 0x0202143B

    # gicv_iidr same as gicc_idr
    gicv_iidr = gicc_iidr

class Gicv2mFrame(SimObject):
    type = 'Gicv2mFrame'
    cxx_header = "dev/arm/gic_v2m.hh"
    spi_base = Param.UInt32(0x0, "Frame SPI base number");
    spi_len = Param.UInt32(0x0, "Frame SPI total number");
    addr = Param.Addr("Address for frame PIO")

class Gicv2m(PioDevice):
    type = 'Gicv2m'
    cxx_header = "dev/arm/gic_v2m.hh"

    pio_delay = Param.Latency('10ns', "Delay for PIO r/w")
    gic = Param.BaseGic(Parent.any, "Gic on which to trigger interrupts")
    frames = VectorParam.Gicv2mFrame([], "Power of two number of frames")

class VGic(PioDevice):
    type = 'VGic'
    cxx_header = "dev/arm/vgic.hh"
    gic = Param.BaseGic(Parent.any, "Gic to use for interrupting")
    platform = Param.Platform(Parent.any, "Platform this device is part of.")
    vcpu_addr = Param.Addr(0, "Address for vcpu interfaces")
    hv_addr = Param.Addr(0, "Address for hv control")
    pio_delay = Param.Latency('10ns', "Delay for PIO r/w")
   # The number of list registers is not currently configurable at runtime.
    maint_int = Param.UInt32("HV maintenance interrupt number")

    # gicv_iidr same as gicc_idr
    gicv_iidr = Param.UInt32(Self.gic.gicc_iidr,
        "VM CPU Interface Identification Register")

    def generateDeviceTree(self, state):
        gic = self.gic.unproxy(self)

        node = FdtNode("interrupt-controller")
        node.appendCompatible(["gem5,gic", "arm,cortex-a15-gic",
                               "arm,cortex-a9-gic"])
        node.append(gic._state.interruptCellsProperty())
        node.append(gic._state.addrCellsProperty())
        node.append(FdtProperty("interrupt-controller"))

        regs = (
            state.addrCells(gic.dist_addr) +
            state.sizeCells(0x1000) +
            state.addrCells(gic.cpu_addr) +
            state.sizeCells(0x1000) +
            state.addrCells(self.hv_addr) +
            state.sizeCells(0x2000) +
            state.addrCells(self.vcpu_addr) +
            state.sizeCells(0x2000) )

        node.append(FdtPropertyWords("reg", regs))
        node.append(FdtPropertyWords("interrupts",
                                     [1, int(self.maint_int)-16, 0xf04]))

        node.appendPhandle(gic)

        yield node

class Gicv3Its(BasicPioDevice):
    type = 'Gicv3Its'
    cxx_header = "dev/arm/gic_v3_its.hh"

    dma = MasterPort("DMA port")
    pio_size = Param.Unsigned(0x20000, "Gicv3Its pio size")

    # CIL [36] = 0: ITS supports 16-bit CollectionID
    # Devbits [17:13] = 0b100011: ITS supports 23 DeviceID bits
    # ID_bits [12:8] = 0b11111: ITS supports 31 EventID bits
    gits_typer = Param.UInt64(0x30023F01, "GITS_TYPER RO value")

class Gicv3(BaseGic):
    type = 'Gicv3'
    cxx_header = "dev/arm/gic_v3.hh"

    # Used for DTB autogeneration
    _state = FdtState(addr_cells=2, interrupt_cells=3)

    its = Param.Gicv3Its(Gicv3Its(), "GICv3 Interrupt Translation Service")

    dist_addr = Param.Addr("Address for distributor")
    dist_pio_delay = Param.Latency('10ns', "Delay for PIO r/w to distributor")
    redist_addr = Param.Addr("Address for redistributors")
    redist_pio_delay = Param.Latency('10ns',
            "Delay for PIO r/w to redistributors")
    it_lines = Param.UInt32(1020,
            "Number of interrupt lines supported (max = 1020)")

    maint_int = Param.ArmInterruptPin(
        "HV maintenance interrupt."
        "ARM strongly recommends that maintenance interrupts "
        "are configured to use INTID 25 (PPI Interrupt).")

    cpu_max = Param.Unsigned(256,
        "Maximum number of PE. This is affecting the maximum number of "
        "redistributors")

    gicv4 = Param.Bool(True, "GICv4 extension available")
<commit_msg>dev-arm: Enable DTB autogeneration in GICv3<commit_after># Copyright (c) 2012-2013, 2017-2019 ARM Limited
# All rights reserved.
#
# The license below extends only to copyright in the software and shall
# not be construed as granting a license to any other intellectual
# property including but not limited to intellectual property relating
# to a hardware implementation of the functionality of the software
# licensed hereunder.  You may use the software subject to the license
# terms below provided that you ensure that this notice is replicated
# unmodified and in its entirety in all distributions of the software,
# modified or unmodified, in source code or in binary form.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are
# met: redistributions of source code must retain the above copyright
# notice, this list of conditions and the following disclaimer;
# redistributions in binary form must reproduce the above copyright
# notice, this list of conditions and the following disclaimer in the
# documentation and/or other materials provided with the distribution;
# neither the name of the copyright holders nor the names of its
# contributors may be used to endorse or promote products derived from
# this software without specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#
# Authors: Andreas Sandberg

from m5.params import *
from m5.proxy import *
from m5.util.fdthelper import *
from m5.SimObject import SimObject

from m5.objects.Device import PioDevice, BasicPioDevice
from m5.objects.Platform import Platform

class BaseGic(PioDevice):
    type = 'BaseGic'
    abstract = True
    cxx_header = "dev/arm/base_gic.hh"

    # Used for DTB autogeneration
    _state = FdtState(addr_cells=0, interrupt_cells=3)

    platform = Param.Platform(Parent.any, "Platform this device is part of.")

    gicd_iidr = Param.UInt32(0,
        "Distributor Implementer Identification Register")
    gicd_pidr = Param.UInt32(0,
        "Peripheral Identification Register")
    gicc_iidr = Param.UInt32(0,
        "CPU Interface Identification Register")
    gicv_iidr = Param.UInt32(0,
        "VM CPU Interface Identification Register")

    def interruptCells(self, int_type, int_num, int_flag):
        """
        Interupt cells generation helper:
        Following specifications described in

        Documentation/devicetree/bindings/interrupt-controller/arm,gic.txt
        """
        assert self._state.interrupt_cells == 3
        return [ int_type, int_num, int_flag ]

class ArmInterruptPin(SimObject):
    type = 'ArmInterruptPin'
    cxx_header = "dev/arm/base_gic.hh"
    cxx_class = "ArmInterruptPinGen"
    abstract = True

    platform = Param.Platform(Parent.any, "Platform with interrupt controller")
    num = Param.UInt32("Interrupt number in GIC")

class ArmSPI(ArmInterruptPin):
    type = 'ArmSPI'
    cxx_header = "dev/arm/base_gic.hh"
    cxx_class = "ArmSPIGen"

class ArmPPI(ArmInterruptPin):
    type = 'ArmPPI'
    cxx_header = "dev/arm/base_gic.hh"
    cxx_class = "ArmPPIGen"

class GicV2(BaseGic):
    type = 'GicV2'
    cxx_header = "dev/arm/gic_v2.hh"

    dist_addr = Param.Addr("Address for distributor")
    cpu_addr = Param.Addr("Address for cpu")
    cpu_size = Param.Addr(0x2000, "Size of cpu register bank")
    dist_pio_delay = Param.Latency('10ns', "Delay for PIO r/w to distributor")
    cpu_pio_delay = Param.Latency('10ns', "Delay for PIO r/w to cpu interface")
    int_latency = Param.Latency('10ns', "Delay for interrupt to get to CPU")
    it_lines = Param.UInt32(128, "Number of interrupt lines supported (max = 1020)")
    gem5_extensions = Param.Bool(False, "Enable gem5 extensions")

class Gic400(GicV2):
    """
    As defined in:
    "ARM Generic Interrupt Controller Architecture" version 2.0
    "CoreLink GIC-400 Generic Interrupt Controller" revision r0p1
    """
    gicd_pidr = 0x002bb490
    gicd_iidr = 0x0200143B
    gicc_iidr = 0x0202143B

    # gicv_iidr same as gicc_idr
    gicv_iidr = gicc_iidr

class Gicv2mFrame(SimObject):
    type = 'Gicv2mFrame'
    cxx_header = "dev/arm/gic_v2m.hh"
    spi_base = Param.UInt32(0x0, "Frame SPI base number");
    spi_len = Param.UInt32(0x0, "Frame SPI total number");
    addr = Param.Addr("Address for frame PIO")

class Gicv2m(PioDevice):
    type = 'Gicv2m'
    cxx_header = "dev/arm/gic_v2m.hh"

    pio_delay = Param.Latency('10ns', "Delay for PIO r/w")
    gic = Param.BaseGic(Parent.any, "Gic on which to trigger interrupts")
    frames = VectorParam.Gicv2mFrame([], "Power of two number of frames")

class VGic(PioDevice):
    type = 'VGic'
    cxx_header = "dev/arm/vgic.hh"
    gic = Param.BaseGic(Parent.any, "Gic to use for interrupting")
    platform = Param.Platform(Parent.any, "Platform this device is part of.")
    vcpu_addr = Param.Addr(0, "Address for vcpu interfaces")
    hv_addr = Param.Addr(0, "Address for hv control")
    pio_delay = Param.Latency('10ns', "Delay for PIO r/w")
   # The number of list registers is not currently configurable at runtime.
    maint_int = Param.UInt32("HV maintenance interrupt number")

    # gicv_iidr same as gicc_idr
    gicv_iidr = Param.UInt32(Self.gic.gicc_iidr,
        "VM CPU Interface Identification Register")

    def generateDeviceTree(self, state):
        gic = self.gic.unproxy(self)

        node = FdtNode("interrupt-controller")
        node.appendCompatible(["gem5,gic", "arm,cortex-a15-gic",
                               "arm,cortex-a9-gic"])
        node.append(gic._state.interruptCellsProperty())
        node.append(gic._state.addrCellsProperty())
        node.append(FdtProperty("interrupt-controller"))

        regs = (
            state.addrCells(gic.dist_addr) +
            state.sizeCells(0x1000) +
            state.addrCells(gic.cpu_addr) +
            state.sizeCells(0x1000) +
            state.addrCells(self.hv_addr) +
            state.sizeCells(0x2000) +
            state.addrCells(self.vcpu_addr) +
            state.sizeCells(0x2000) )

        node.append(FdtPropertyWords("reg", regs))
        node.append(FdtPropertyWords("interrupts",
                                     [1, int(self.maint_int)-16, 0xf04]))

        node.appendPhandle(gic)

        yield node

class Gicv3Its(BasicPioDevice):
    type = 'Gicv3Its'
    cxx_header = "dev/arm/gic_v3_its.hh"

    dma = MasterPort("DMA port")
    pio_size = Param.Unsigned(0x20000, "Gicv3Its pio size")

    # CIL [36] = 0: ITS supports 16-bit CollectionID
    # Devbits [17:13] = 0b100011: ITS supports 23 DeviceID bits
    # ID_bits [12:8] = 0b11111: ITS supports 31 EventID bits
    gits_typer = Param.UInt64(0x30023F01, "GITS_TYPER RO value")

class Gicv3(BaseGic):
    type = 'Gicv3'
    cxx_header = "dev/arm/gic_v3.hh"

    # Used for DTB autogeneration
    _state = FdtState(addr_cells=2, size_cells=2, interrupt_cells=3)

    its = Param.Gicv3Its(Gicv3Its(), "GICv3 Interrupt Translation Service")

    dist_addr = Param.Addr("Address for distributor")
    dist_pio_delay = Param.Latency('10ns', "Delay for PIO r/w to distributor")
    redist_addr = Param.Addr("Address for redistributors")
    redist_pio_delay = Param.Latency('10ns',
            "Delay for PIO r/w to redistributors")
    it_lines = Param.UInt32(1020,
            "Number of interrupt lines supported (max = 1020)")

    maint_int = Param.ArmInterruptPin(
        "HV maintenance interrupt."
        "ARM strongly recommends that maintenance interrupts "
        "are configured to use INTID 25 (PPI Interrupt).")

    cpu_max = Param.Unsigned(256,
        "Maximum number of PE. This is affecting the maximum number of "
        "redistributors")

    gicv4 = Param.Bool(True, "GICv4 extension available")

    def interruptCells(self, int_type, int_num, int_flag):
        """
        Interupt cells generation helper:
        Following specifications described in

        Documentation/devicetree/bindings/interrupt-controller/arm,gic-v3.txt
        """
        prop = self._state.interruptCells(0)
        assert len(prop) >= 3
        prop[0] = int_type
        prop[1] = int_num
        prop[2] = int_flag
        return prop

    def generateDeviceTree(self, state):
        node = FdtNode("interrupt-controller")
        node.appendCompatible(["arm,gic-v3"])
        node.append(self._state.interruptCellsProperty())
        node.append(self._state.addrCellsProperty())
        node.append(self._state.sizeCellsProperty())
        node.append(FdtProperty("interrupt-controller"))

        regs = (
            state.addrCells(self.dist_addr) +
            state.sizeCells(0x10000) +
            state.addrCells(self.redist_addr) +
            state.sizeCells(0x2000000) )

        node.append(FdtPropertyWords("reg", regs))
        node.append(FdtPropertyWords("interrupts",
            self.interruptCells(1, int(self.maint_int.num)-16, 0xf04)))

        node.appendPhandle(self)

        yield node
<|endoftext|>


<commit_before>import tensorflow as tf

def discriminator_loss(loss_func: str, d_real: tf.Tensor, d_fake: tf.Tensor):
    real_loss, fake_loss = 0., 0.
    if loss_func in {'wgan', 'wgan-gp'}:
        real_loss = -d_real
        fake_loss = d_fake
    if loss_func == 'lsgan':
        real_loss = tf.math.squared_difference(d_real, 1.0)
        fake_loss = tf.square(d_fake)
    if loss_func in {'dragan', 'gan'}:
        real_loss = tf.nn.sigmoid_cross_entropy_with_logits(labels=tf.ones_like(d_real), logits=d_real)
        fake_loss = tf.nn.sigmoid_cross_entropy_with_logits(labels=tf.zeros_like(d_fake), logits=d_fake)
    if loss_func == 'hinge':
        real_loss = tf.math.maximum(0., 1.0 - d_real)
        fake_loss = tf.math.maximum(0., 1.0 + d_fake)
    return fake_loss + real_loss

def generator_loss(loss_func: str, d_fake: tf.Tensor):
    fake_loss = 0.
    if loss_func in {'wgan', 'wgan-gp'}:
        fake_loss = -d_fake
    if loss_func == 'lsgan':
        fake_loss = tf.math.squared_difference(d_fake, 1.0)
    if loss_func in {'dragan', 'gan'}:
        fake_loss = tf.nn.sigmoid_cross_entropy_with_logits(labels=tf.ones_like(d_fake), logits=d_fake)
    if loss_func == 'hinge':
        fake_loss = -d_fake
    return fake_loss
<commit_msg>update: losses.py<commit_after>import tensorflow as tf

def discriminator_loss(loss_func: str, d_real: tf.Tensor, d_fake: tf.Tensor):
    real_loss, fake_loss = 0.0, 0.0
    if loss_func in {'wgan', 'wgan-gp'}:
        real_loss = -d_real
        fake_loss = d_fake
    if loss_func == 'lsgan':
        real_loss = tf.math.squared_difference(d_real, 1.0)
        fake_loss = tf.square(d_fake)
    if loss_func in {'dragan', 'gan'}:
        real_loss = tf.nn.sigmoid_cross_entropy_with_logits(labels=tf.ones_like(d_real), logits=d_real)
        fake_loss = tf.nn.sigmoid_cross_entropy_with_logits(labels=tf.zeros_like(d_fake), logits=d_fake)
    if loss_func == 'hinge':
        real_loss = tf.math.maximum(0.0, 1.0 - d_real)
        fake_loss = tf.math.maximum(0.0, 1.0 + d_fake)
    return fake_loss + real_loss

def generator_loss(loss_func: str, d_fake: tf.Tensor):
    fake_loss = 0.0
    if loss_func in {'wgan', 'wgan-gp'}:
        fake_loss = -d_fake
    if loss_func == 'lsgan':
        fake_loss = tf.math.squared_difference(d_fake, 1.0)
    if loss_func in {'dragan', 'gan'}:
        fake_loss = tf.nn.sigmoid_cross_entropy_with_logits(labels=tf.ones_like(d_fake), logits=d_fake)
    if loss_func == 'hinge':
        fake_loss = -d_fake
    return fake_loss
<|endoftext|>


<commit_before>#!/usr/bin/env python
# from __future__ import print_function

import os
from collections import defaultdict

from peyotl import get_logger
from peyotl.utility.str_util import StringIO

from taxalotl.ott_schema import (read_taxonomy_to_get_single_taxon,
                                 )
from taxalotl.partitions import (fill_empty_anc_of_mapping,
                                 MISC_DIRNAME,
                                 PREORDER_PART_LIST,
                                 NAME_TO_PARTS_SUBSETS)
from taxalotl.resource_wrapper import ResourceWrapper, TaxonomyWrapper
from taxalotl.tax_partition import (get_roots_for_subset, )
from taxalotl.util import get_true_false_repsonse

_LOG = get_logger(__name__)

OTT_PARTMAP = {
    'Archaea': frozenset([996421]),
    'Bacteria': frozenset([844192]),
    'Eukaryota': frozenset([304358]),
    'SAR': frozenset([5246039]),
    'Haptophyta': frozenset([151014]),
    'Rhodophyta': frozenset([878953]),
    'Archaeplastida': frozenset([5268475]),
    'Glaucophyta': frozenset([664970]),
    'Chloroplastida': frozenset([361838]),
    'Fungi': frozenset([352914]),
    'Metazoa': frozenset([691846]),
    'Annelida': frozenset([941620]),
    'Arthropoda': frozenset([632179]),
    'Malacostraca': frozenset([212701]),
    'Arachnida': frozenset([511967]),
    'Insecta': frozenset([1062253]),
    'Diptera': frozenset([661378]),
    'Coleoptera': frozenset([865243]),
    'Lepidoptera': frozenset([965954]),
    'Hymenoptera': frozenset([753726]),
    'Bryozoa': frozenset([442934]),
    'Chordata': frozenset([125642]),
    'Cnidaria': frozenset([641033]),
    'Ctenophora': frozenset([641212]),
    'Mollusca': frozenset([802117]),
    'Nematoda': frozenset([395057]),
    'Platyhelminthes': frozenset([555379]),
    'Porifera': frozenset([67819]),
    'Viruses': frozenset([4807313]),
}

# Unused separation taxa: cellular organisms	93302

OTT_3_SEPARATION_TAXA = OTT_PARTMAP

def ott_fetch_root_taxon_for_partition(res, parts_key, root_id):
    tax_dir = res.get_taxdir_for_root_of_part(parts_key)
    if not tax_dir:
        _LOG.info('No taxon file found for {}'.format(parts_key))
        return None
    # _LOG.info('{} root should be in {}'.format(parts_key, tax_dir))
    taxon_obj = read_taxonomy_to_get_single_taxon(tax_dir, root_id)
    if not tax_dir:
        _LOG.info(
            'Root taxon for {} with ID {} not found in {}'.format(parts_key, root_id, tax_dir))
        return None
    return taxon_obj

def ott_build_paritition_maps(res):
    # srcs = {i[0]:i[1] for i in res.inputs if i[0] and is_str_type(i[1])}
    src_pre_to_map = {}
    all_part_keys = set()
    for part_key in PREORDER_PART_LIST:
        if part_key != 'Life':
            all_part_keys.add(part_key)
        all_part_keys.update(NAME_TO_PARTS_SUBSETS[part_key])
    for pk in all_part_keys:
        if pk == MISC_DIRNAME:
            continue
        tax_dir = res.get_taxdir_for_part(pk)
        roots = get_roots_for_subset(tax_dir, res.get_misc_taxon_dir_for_part(pk))
        _LOG.info("{} -> {} roots = {}".format(pk, tax_dir, roots))
        if not roots:
            continue
        assert len(roots) == 1
        root_id, taxon = roots.popitem()
        _LOG.info('root for {} has sources: {}'.format(pk, taxon.src_dict))
        for src_pre, src_ids in taxon.src_dict.items():
            src_pre_to_map.setdefault(src_pre, {})[pk] = src_ids
    import copy
    filled = {}
    for src_pre, mapping in src_pre_to_map.items():
        pm = copy.deepcopy(mapping)
        fill_empty_anc_of_mapping(mapping)
        filled[src_pre] = {k: list(v) for k, v in mapping.items()}
        for k, v in mapping.items():
            pv = pm.get(k)
            if pv is None:
                _LOG.info("{}: {} -> (None -> {})".format(src_pre, k, v))
            else:
                assert v == pv
    # @TODO: hard copying the fact that OTT was built with SILVA-115
    if 'silva' in filled:
        filled['silva_115'] = filled['silva']
        del filled['silva']
    # @TODO: make this automatic not generic
    gbif = filled['gbif']
    assert "Chloroplastida" not in gbif
    gbif["Chloroplastida"] = [13, 9, 7707728, 7819616, 36, 35]

    return filled

UNSTABLE_SRC_PREFIXES = frozenset(['h2007', 'study713', 'https', 'http'])

class NewSeparator(object):
    def __init__(self, ott_taxon_obj):
        self.taxon = ott_taxon_obj
        self.sub_separators = {}

    def __str__(self):
        out = StringIO()
        self.write_str(out)
        return out.getvalue()

    def write_str(self, out, pref=''):
        t = self.taxon
        if pref:
            pref = pref + '/'
        out.write('{}{} ott{} sources={}\n'.format(pref,
                                                   t.name_that_is_unique,
                                                   t.id,
                                                   t.src_dict))
        for name, sep in self.sub_separators.items():
            if pref:
                ns = pref + name
            else:
                ns = name
            sep.write_str(out, ns)

    def as_dict(self):
        d = {"name": self.taxon.name, "uniqname": self.taxon.name_that_is_unique}
        sd = {}
        for k, v in self.taxon.src_dict.items():
            vl = list(v)
            vl.sort()
            sd[k] = vl
        d["src_dict"] = sd
        s = {}
        for v in self.sub_separators.values():
            s[v.taxon.id] = v.as_dict()
        if s:
            d["sub"] = s
        return d

    def num_sub_separators(self):
        n = 0
        for el in self.sub_separators.values():
            n += 1 + el.num_sub_separators()
        return n

class NestedNewSeparator(object):
    def __init__(self):
        self.separators = {}

    def __bool__(self):
        return bool(self.separators)

    def add_separtors_for_tree(self, tree, sep_ids):
        self._add_separtors_for_descendants(tree.root, sep_ids, self.separators)

    def _add_separtors_for_descendants(self, nd, sep_ids, par_dict):
        child_refs = nd.children_refs
        if not child_refs:
            return
        for child in child_refs:
            if child.id in sep_ids:
                if nd.id in sep_ids and len(child_refs) == 1:
                    # suppress monotypic separators children as separators
                    self._add_separtors_for_descendants(child, sep_ids, par_dict)
                else:
                    ns = NewSeparator(child)
                    par_dict[child.id] = ns
                    self._add_separtors_for_descendants(child, sep_ids, ns.sub_separators)
            else:
                self._add_separtors_for_descendants(child, sep_ids, par_dict)

    def __str__(self):
        out = StringIO()
        for name, sep in self.separators.items():
            out.write('top-level name={}\n'.format(name))
            sep.write_str(out)
        return out.getvalue()

    def as_dict(self):
        d = {}
        for el in self.separators.values():
            d[el.taxon.id] = el.as_dict()
        return d

    def num_separators(self):
        n = 0
        for el in self.separators.values():
            n += 1 + el.num_sub_separators()
        return n

def _add_nst_subtree_el_to_dict(rd, nst_el, par_to_child):
    sep_taxon, children = nst_el
    if sep_taxon is not None:
        nst = NewSeparator(sep_taxon)
        nd = nst.sub_separators
        rd[sep_taxon.name_that_is_unique] = nst
    else:
        nd = rd
    for c in children:
        next_el = par_to_child[c]
        _add_nst_subtree_el_to_dict(nd, next_el, par_to_child)

NON_SEP_RANKS = frozenset(['forma', 'no rank - terminal', 'species',
                           'species group', 'species subgroup', 'varietas', 'variety', ])
MIN_SEP_SIZE = 1000

def get_stable_source_keys(taxon):
    all_src_keys = taxon.src_dict.keys()
    filtered = [i for i in all_src_keys if not i.startswith('additions')]
    r = [i for i in filtered if i not in UNSTABLE_SRC_PREFIXES]
    r.sort()
    return r

def _diagnose_relevant_sources(tree):
    source_to_count = defaultdict(int)
    leaf_set = set()
    src_prefix_set = set()
    for v in tree.id_to_taxon.values():
        if v.children_refs is None:
            leaf_set.add(v.id)
        filtered = get_stable_source_keys(v)
        src_prefix_set.update(filtered)
        for src in filtered:
            source_to_count[src] += 1
    # We'll define a "minor" source as one that is found in 5000 fold fewer taxa as
    #   the most common source for a slice. This is arbitrary.
    FOLD_DIFF = 5000
    max_seen = max(source_to_count.values())
    min_cutoff = 0 if max_seen < FOLD_DIFF else max_seen // FOLD_DIFF
    ac_src = [k for k, v in source_to_count.items() if v >= min_cutoff]
    ac_src.sort()
    _LOG.info("Relevant sources appear to be: {}".format(ac_src))
    return frozenset(ac_src)

PART_KEY_TO_REL_SRC_SET = {
    'Chordata': frozenset(['gbif', 'irmng', 'ncbi', 'worms']),
}

def add_confirmed_sep(nns, tree, list_num_id_taxon):
    if list_num_id_taxon:
        r = tree.root
        m = 'The current partition subtree "{}" has {} tips below it.'
        _LOG.info(m.format(r.name_that_is_unique, r.num_tips_below))
    top_sep_set = set()
    for nt, i, obj in list_num_id_taxon:
        if top_sep_set:
            already_sep = False
            for anc_obj in tree.to_root_gen(obj):
                if anc_obj.id in top_sep_set:
                    already_sep = True
                    break
            if already_sep:
                continue
        m = '"{}" has {} tips below it.'.format(obj.name_that_is_unique, nt)
        p = '{} Enter (y) to treat is a separator: '.format(m)
        if get_true_false_repsonse(p):
            top_sep_set.add(i)
    if top_sep_set:
        nns.add_separtors_for_tree(tree, top_sep_set)

# noinspection PyAbstractClass
class OTTaxonomyWrapper(TaxonomyWrapper):
    resource_type = 'open tree taxonomy'
    schema = {'ott'}

    def __init__(self, obj, parent=None, refs=None):
        TaxonomyWrapper.__init__(self, obj, parent=parent, refs=refs)

    def get_source_for_sep_or_part(self, current_partition_key):
        try:
            return PART_KEY_TO_REL_SRC_SET[current_partition_key]
        except:
            dn = self.config.get_fragment_from_part_name(current_partition_key)
            higher = os.path.split(os.path.split(dn)[0])[-1]
            if higher.lower() == 'life':
                raise ValueError('partition/separator name "{}" not found')
            return self.get_source_for_sep_or_part(higher)

    def diagnose_new_separators(self, current_partition_key):
        tax_forest = self.get_taxon_forest_for_partition(current_partition_key)
        nns = NestedNewSeparator()

        try:
            ac_src = self.get_source_for_sep_or_part(current_partition_key)
        except:
            m = 'Might try calling _diagnose_relevant_sources for "{}", but in Jan 2019 moved to ' \
                'making this hard coded in PART_KEY_TO_REL_SRC_SET.'
            raise NotImplementedError(m.format(current_partition_key))
        _LOG.info("Relevant sources for {} are recorded as to be: {}".format(current_partition_key,
                                                                             ac_src))
        for tree in tax_forest.trees:
            tree.add_num_tips_below()
            assert ac_src
            nst = []
            for i, obj in tree.id_to_taxon.items():
                if obj is tree.root:
                    continue  # no point in partitioning at the root taxon
                if not obj.children_refs:
                    continue  # no point in partitioning leaves...
                sk_for_obj = set(get_stable_source_keys(obj))
                if sk_for_obj.issuperset(ac_src):
                    if obj.num_tips_below >= MIN_SEP_SIZE:
                        if not obj.rank or (obj.rank not in NON_SEP_RANKS):
                            nst.append((obj.num_tips_below, i, obj))
            nst.sort(reverse=True)
            add_confirmed_sep(nns, tree, nst)

        if len(nns.separators) == 0:
            _LOG.info('No new separators found for "{}"'.format(current_partition_key))
            return None
        rel_dir_for_part = self.config.get_fragment_from_part_name(current_partition_key)
        return {rel_dir_for_part: nns}

    def build_paritition_maps(self):
        return ott_build_paritition_maps(self)

    def get_primary_partition_map(self):
        return OTT_PARTMAP

    def input_dict(self):
        d = {}
        for k, v in self.inputs:
            d[k] = v
        return d

# noinspection PyAbstractClass
class OTTaxonomyIdListWrapper(ResourceWrapper):
    resource_type = 'open tree taxonomy idlist'
    schema = {resource_type, 'ott id csv'}
    _norm_filename = 'by_qid.csv'

    def __init__(self, obj, parent=None, refs=None):
        ResourceWrapper.__init__(self, obj, parent=parent, refs=refs)
<commit_msg>diagnose new separators is broken<commit_after>#!/usr/bin/env python
# from __future__ import print_function

import os
from collections import defaultdict

from peyotl import get_logger
from peyotl.utility.str_util import StringIO

from taxalotl.ott_schema import (read_taxonomy_to_get_single_taxon,
                                 )
from taxalotl.partitions import (fill_empty_anc_of_mapping,
                                 MISC_DIRNAME,
                                 PREORDER_PART_LIST,
                                 NAME_TO_PARTS_SUBSETS)
from taxalotl.resource_wrapper import ResourceWrapper, TaxonomyWrapper
from taxalotl.tax_partition import (get_roots_for_subset, )
from taxalotl.util import get_true_false_repsonse

_LOG = get_logger(__name__)

OTT_PARTMAP = {
    'Archaea': frozenset([996421]),
    'Bacteria': frozenset([844192]),
    'Eukaryota': frozenset([304358]),
    'SAR': frozenset([5246039]),
    'Haptophyta': frozenset([151014]),
    'Rhodophyta': frozenset([878953]),
    'Archaeplastida': frozenset([5268475]),
    'Glaucophyta': frozenset([664970]),
    'Chloroplastida': frozenset([361838]),
    'Fungi': frozenset([352914]),
    'Metazoa': frozenset([691846]),
    'Annelida': frozenset([941620]),
    'Arthropoda': frozenset([632179]),
    'Malacostraca': frozenset([212701]),
    'Arachnida': frozenset([511967]),
    'Insecta': frozenset([1062253]),
    'Diptera': frozenset([661378]),
    'Coleoptera': frozenset([865243]),
    'Lepidoptera': frozenset([965954]),
    'Hymenoptera': frozenset([753726]),
    'Bryozoa': frozenset([442934]),
    'Chordata': frozenset([125642]),
    'Cnidaria': frozenset([641033]),
    'Ctenophora': frozenset([641212]),
    'Mollusca': frozenset([802117]),
    'Nematoda': frozenset([395057]),
    'Platyhelminthes': frozenset([555379]),
    'Porifera': frozenset([67819]),
    'Viruses': frozenset([4807313]),
}

# Unused separation taxa: cellular organisms	93302

OTT_3_SEPARATION_TAXA = OTT_PARTMAP

def ott_fetch_root_taxon_for_partition(res, parts_key, root_id):
    tax_dir = res.get_taxdir_for_root_of_part(parts_key)
    if not tax_dir:
        _LOG.info('No taxon file found for {}'.format(parts_key))
        return None
    # _LOG.info('{} root should be in {}'.format(parts_key, tax_dir))
    taxon_obj = read_taxonomy_to_get_single_taxon(tax_dir, root_id)
    if not tax_dir:
        _LOG.info(
            'Root taxon for {} with ID {} not found in {}'.format(parts_key, root_id, tax_dir))
        return None
    return taxon_obj

def ott_build_paritition_maps(res):
    # srcs = {i[0]:i[1] for i in res.inputs if i[0] and is_str_type(i[1])}
    src_pre_to_map = {}
    all_part_keys = set()
    for part_key in PREORDER_PART_LIST:
        if part_key != 'Life':
            all_part_keys.add(part_key)
        all_part_keys.update(NAME_TO_PARTS_SUBSETS[part_key])
    for pk in all_part_keys:
        if pk == MISC_DIRNAME:
            continue
        tax_dir = res.get_taxdir_for_part(pk)
        roots = get_roots_for_subset(tax_dir, res.get_misc_taxon_dir_for_part(pk))
        _LOG.info("{} -> {} roots = {}".format(pk, tax_dir, roots))
        if not roots:
            continue
        assert len(roots) == 1
        root_id, taxon = roots.popitem()
        _LOG.info('root for {} has sources: {}'.format(pk, taxon.src_dict))
        for src_pre, src_ids in taxon.src_dict.items():
            src_pre_to_map.setdefault(src_pre, {})[pk] = src_ids
    import copy
    filled = {}
    for src_pre, mapping in src_pre_to_map.items():
        pm = copy.deepcopy(mapping)
        fill_empty_anc_of_mapping(mapping)
        filled[src_pre] = {k: list(v) for k, v in mapping.items()}
        for k, v in mapping.items():
            pv = pm.get(k)
            if pv is None:
                _LOG.info("{}: {} -> (None -> {})".format(src_pre, k, v))
            else:
                assert v == pv
    # @TODO: hard copying the fact that OTT was built with SILVA-115
    if 'silva' in filled:
        filled['silva_115'] = filled['silva']
        del filled['silva']
    # @TODO: make this automatic not generic
    gbif = filled['gbif']
    assert "Chloroplastida" not in gbif
    gbif["Chloroplastida"] = [13, 9, 7707728, 7819616, 36, 35]

    return filled

UNSTABLE_SRC_PREFIXES = frozenset(['h2007', 'study713', 'https', 'http'])

class NewSeparator(object):
    def __init__(self, ott_taxon_obj):
        self.taxon = ott_taxon_obj
        self.sub_separators = {}

    def __str__(self):
        out = StringIO()
        self.write_str(out)
        return out.getvalue()

    def write_str(self, out, pref=''):
        t = self.taxon
        if pref:
            pref = pref + '/'
        out.write('{}{} ott{} sources={}\n'.format(pref,
                                                   t.name_that_is_unique,
                                                   t.id,
                                                   t.src_dict))
        for name, sep in self.sub_separators.items():
            if pref:
                ns = pref + name
            else:
                ns = name
            sep.write_str(out, ns)

    def as_dict(self):
        d = {"name": self.taxon.name, "uniqname": self.taxon.name_that_is_unique}
        sd = {}
        for k, v in self.taxon.src_dict.items():
            vl = list(v)
            vl.sort()
            sd[k] = vl
        d["src_dict"] = sd
        s = {}
        for v in self.sub_separators.values():
            s[v.taxon.id] = v.as_dict()
        if s:
            d["sub"] = s
        return d

    def num_sub_separators(self):
        n = 0
        for el in self.sub_separators.values():
            n += 1 + el.num_sub_separators()
        return n

class NestedNewSeparator(object):
    def __init__(self):
        self.separators = {}

    def __bool__(self):
        return bool(self.separators)

    def add_separtors_for_tree(self, tree, sep_ids):
        self._add_separtors_for_descendants(tree.root, sep_ids, self.separators)

    def _add_separtors_for_descendants(self, nd, sep_ids, par_dict):
        child_refs = nd.children_refs
        if not child_refs:
            return
        for child in child_refs:
            if child.id in sep_ids:
                if nd.id in sep_ids and len(child_refs) == 1:
                    # suppress monotypic separators children as separators
                    self._add_separtors_for_descendants(child, sep_ids, par_dict)
                else:
                    ns = NewSeparator(child)
                    par_dict[child.id] = ns
                    self._add_separtors_for_descendants(child, sep_ids, ns.sub_separators)
            else:
                self._add_separtors_for_descendants(child, sep_ids, par_dict)

    def __str__(self):
        out = StringIO()
        for name, sep in self.separators.items():
            out.write('top-level name={}\n'.format(name))
            sep.write_str(out)
        return out.getvalue()

    def as_dict(self):
        d = {}
        for el in self.separators.values():
            d[el.taxon.id] = el.as_dict()
        return d

    def num_separators(self):
        n = 0
        for el in self.separators.values():
            n += 1 + el.num_sub_separators()
        return n

def _add_nst_subtree_el_to_dict(rd, nst_el, par_to_child):
    sep_taxon, children = nst_el
    if sep_taxon is not None:
        nst = NewSeparator(sep_taxon)
        nd = nst.sub_separators
        rd[sep_taxon.name_that_is_unique] = nst
    else:
        nd = rd
    for c in children:
        next_el = par_to_child[c]
        _add_nst_subtree_el_to_dict(nd, next_el, par_to_child)

NON_SEP_RANKS = frozenset(['forma', 'no rank - terminal', 'species',
                           'species group', 'species subgroup', 'varietas', 'variety', ])
MIN_SEP_SIZE = 1000

def get_stable_source_keys(taxon):
    all_src_keys = taxon.src_dict.keys()
    filtered = [i for i in all_src_keys if not i.startswith('additions')]
    r = [i for i in filtered if i not in UNSTABLE_SRC_PREFIXES]
    r.sort()
    return r

def _diagnose_relevant_sources(tree):
    source_to_count = defaultdict(int)
    leaf_set = set()
    src_prefix_set = set()
    for v in tree.id_to_taxon.values():
        if v.children_refs is None:
            leaf_set.add(v.id)
        filtered = get_stable_source_keys(v)
        src_prefix_set.update(filtered)
        for src in filtered:
            source_to_count[src] += 1
    # We'll define a "minor" source as one that is found in 5000 fold fewer taxa as
    #   the most common source for a slice. This is arbitrary.
    FOLD_DIFF = 5000
    max_seen = max(source_to_count.values())
    min_cutoff = 0 if max_seen < FOLD_DIFF else max_seen // FOLD_DIFF
    ac_src = [k for k, v in source_to_count.items() if v >= min_cutoff]
    ac_src.sort()
    _LOG.info("Relevant sources appear to be: {}".format(ac_src))
    return frozenset(ac_src)

MINIMAL_REL_SRC_SET = frozenset(['gbif', 'irmng', 'ncbi']),
PART_KEY_TO_REL_SRC_SET = {
    'Chordata': frozenset(['gbif', 'irmng', 'ncbi', 'worms']),
}

def add_confirmed_sep(nns, tree, list_num_id_taxon):
    if list_num_id_taxon:
        r = tree.root
        m = 'The current partition subtree "{}" has {} tips below it.'
        _LOG.info(m.format(r.name_that_is_unique, r.num_tips_below))
    top_sep_set = set()
    for nt, i, obj in list_num_id_taxon:
        if top_sep_set:
            already_sep = False
            for anc_obj in tree.to_root_gen(obj):
                if anc_obj.id in top_sep_set:
                    already_sep = True
                    break
            if already_sep:
                continue
        m = '"{}" has {} tips below it.'.format(obj.name_that_is_unique, nt)
        p = '{} Enter (y) to treat is a separator: '.format(m)
        if get_true_false_repsonse(p):
            top_sep_set.add(i)
    if top_sep_set:
        nns.add_separtors_for_tree(tree, top_sep_set)

# noinspection PyAbstractClass
class OTTaxonomyWrapper(TaxonomyWrapper):
    resource_type = 'open tree taxonomy'
    schema = {'ott'}

    def __init__(self, obj, parent=None, refs=None):
        TaxonomyWrapper.__init__(self, obj, parent=parent, refs=refs)

    def get_source_for_sep_or_part(self, current_partition_key):
        try:
            return PART_KEY_TO_REL_SRC_SET.get(current_partition_key, MINIMAL_REL_SRC_SET)
        except:
            dn = self.config.get_fragment_from_part_name(current_partition_key)
            higher = os.path.split(os.path.split(dn)[0])[-1]
            if higher.lower() == 'life':
                raise ValueError('partition/separator name "{}" not found'.format(current_partition_key))
            return self.get_source_for_sep_or_part(higher)

    def diagnose_new_separators(self, current_partition_key):
        tax_forest = self.get_taxon_forest_for_partition(current_partition_key)
        nns = NestedNewSeparator()
        if tax_forest:
            try:
                ac_src = self.get_source_for_sep_or_part(current_partition_key)
            except:
                m = 'Might try calling _diagnose_relevant_sources for "{}", but in Jan 2019 moved to ' \
                    'making this hard coded in PART_KEY_TO_REL_SRC_SET.'
                raise NotImplementedError(m.format(current_partition_key))
            _LOG.info("Relevant sources for {} are recorded as to be: {}".format(current_partition_key,
                                                                                ac_src))
            for tree in tax_forest.trees:
                tree.add_num_tips_below()
                assert ac_src
                nst = []
                for i, obj in tree.id_to_taxon.items():
                    if obj is tree.root:
                        continue  # no point in partitioning at the root taxon
                    if not obj.children_refs:
                        continue  # no point in partitioning leaves...
                    sk_for_obj = set(get_stable_source_keys(obj))
                    if sk_for_obj.issuperset(ac_src):
                        if obj.num_tips_below >= MIN_SEP_SIZE:
                            if not obj.rank or (obj.rank not in NON_SEP_RANKS):
                                nst.append((obj.num_tips_below, i, obj))
                nst.sort(reverse=True)
                add_confirmed_sep(nns, tree, nst)
        if len(nns.separators) == 0:
            _LOG.info('No new separators found for "{}"'.format(current_partition_key))
            return None
        rel_dir_for_part = self.config.get_fragment_from_part_name(current_partition_key)
        return {rel_dir_for_part: nns}

    def build_paritition_maps(self):
        return ott_build_paritition_maps(self)

    def get_primary_partition_map(self):
        return OTT_PARTMAP

    def input_dict(self):
        d = {}
        for k, v in self.inputs:
            d[k] = v
        return d

# noinspection PyAbstractClass
class OTTaxonomyIdListWrapper(ResourceWrapper):
    resource_type = 'open tree taxonomy idlist'
    schema = {resource_type, 'ott id csv'}
    _norm_filename = 'by_qid.csv'

    def __init__(self, obj, parent=None, refs=None):
        ResourceWrapper.__init__(self, obj, parent=parent, refs=refs)
<|endoftext|>


<commit_before>#!/usr/bin/env python3
# Script for finding the names of diary entries.

# Should be able to return filenames corresponding to specific timestamps,
# ranges of entries, single entries and new entries (with a given date).

import config
from os import makedirs, listdir
from os.path import realpath, join, basename, dirname, exists, isfile, getmtime
import math
import datetime
import time
import argparse
import re
from subprocess import check_output, call
from itertools import islice
import sys
import shelve
import textwrap

# TODO move these into a settings module
pad_char = '='
color_middle = '\033[1;34m'
color_padding = '\033[0;34m'
color_end = '\033[0m'

# This might not work on non-Linux OSes.
# TODO find a more general solution and move to settings module.
terminal_width = int(check_output('tput cols', shell=True).strip())

# Open cache shelf.
cache_path = config.dir_entries_cache
if not exists(cache_path):
    makedirs(cache_path)
cache_shelf = shelve.open(cache_path + '/range-cache', writeback=False)

def cached(func):
    '''Cache the results of a function, recalculate when cache is outdated.'''

    # TODO preserve name and docstring
    def wrapper(self, *args):

        id = self.timestamp
        func_tuple = (func.__name__, args)

        cached_attrs = cache_shelf.get(id, None)

        if cached_attrs is None or cached_attrs['mtime'] != self.getmtime():
            cache_shelf[id] = { 'mtime': self.getmtime() }

        elif func_tuple in cached_attrs:
            return cached_attrs[func_tuple]

        result = func(self, *args)
        cached_attrs = cache_shelf[id]
        cached_attrs[func_tuple] = result
        cache_shelf[id] = cached_attrs
        return result

    return wrapper

# TODO move all formatting related methods to a separate module
#      only keey the _gen_text() method
class Entry():
    '''Encapsulates entry manipulation functionality.'''

    _filename_re = re.compile(
            r'^diary-(-?[0-9]+)-([a-zA-Z0-9_][a-zA-Z0-9_-]*)\.([a-z]+)$')

    def __init__(self, *path_components):
        self.pathname = realpath(join(*path_components))
        match = Entry._filename_re.match(basename(self.pathname))
        self.timestamp, self.device_name, self.extension = match.groups()

    def mkdir(self):
        '''Create the entry's base directory (if it does not exist).'''
        directory = dirname(self.pathname)
        if not exists(directory):
            makedirs(directory)

    def exists(self):
        '''Return True if the entry exists.'''
        return isfile(self.pathname)

    def getmtime(self):
        '''Return the timestamp when the entry was last modified.'''
        if not self.exists():
            return None
        else:
            return getmtime(self.pathname)

    @cached
    def contains(self, search_string):
        '''Return True if the entry contains the given search string.'''
        command = 'grep -qi "{}" "{}"'.format(search_string, self.pathname)
        return call(command, shell=True) == 0

    @cached
    def get_date_string(self, format='%A %d %B %Y %I:%M%p'):
        '''Return formatted string representing the entry creation date.'''
        # TODO add additional information (like today/yesterday/in the future).
        return time.strftime(format, time.localtime(float(self.timestamp)))

    @cached
    def wordcount(self):
        '''Return the number of space separated words in the entry.'''
        # TODO do wordcount in python
        command = 'wc -w < "{}"'.format(self.pathname)
        return int(check_output(command, shell=True).strip())

    def _gen_text(self):
        '''Return a generator over the lines of the entry.'''
        with open(self.pathname) as f:
            for line in f:
                yield line.strip()

    # TODO add search term highlighting after this stage
    # TODO skip final lines if they are empty?
    # TODO add markdown formatting?
    # TODO put header generation in separate method?
    def _gen_formatted(self, width, header):

        if header:
            left = pad_char + '{} words'.format(self.wordcount())
            right = str(self.timestamp) + pad_char
            middle = ' {} '.format(self.get_date_string())

            padding_left = pad_char * int(width/2 - len(left) - len(middle)/2)
            padding_right = pad_char * (width - len(left) -
                                len(padding_left) - len(middle) - len(right))

            header_string = color_padding + left + padding_left + color_end + \
                            color_middle + middle + color_end + \
                            color_padding + padding_right + right + color_end

            yield header_string

        wrapper = textwrap.TextWrapper(width=width)

        for line in self._gen_text():
            # TODO do this more efficiently?
            if line.strip() == '':
                yield ''
            for wrapped_line in wrapper.wrap(line):
                yield wrapped_line

        yield ''
        yield ''

    @cached
    def _formatted(self, width, header):
        return '\n'.join(self._gen_formatted(width, header))

    def formatted(self, width=terminal_width, header=True):
        '''Return a list of formatted lines, wrapped to width.'''
        return self._formatted(width, header)

    @cached
    def tags(self):
        '''Return a list of all tags occurring in the entry text.'''

        # TODO do this in python using _gen_text()
        command = r'grep -o "#\S\+\b" "{}" || true'.format(self.pathname)
        matches = check_output(command, shell=True, universal_newlines=True)

        matches = matches.split('\n')
        matches = [ match.lstrip('#').rstrip() for match in matches ]
        matches = [ match for match in matches if len(match)>0 ]

        return matches

# TODO move this to the Entry class definition.
def new_entry(timestamp=None, device_name=config.device_name):
    '''Return a new (not currently existing) entry.

    Note that the directory structure may not exist.'''

    timestamp = int(time.time()) if timestamp is None else int(timestamp)
    month = time.strftime('%Y-%m', time.localtime(timestamp))
    filename = 'diary-{}-{}.txt'.format(timestamp, device_name)

    return [Entry(config.dir_entries, month, filename)]

def _find_with_command(command):
    results = check_output(command, shell=True, universal_newlines=True)
    return [ Entry(result) for result in results.split('\n') if len(result)>0 ]

def find_by_timestamp(timestamp):
    '''Returns any entries with the given timestamp.'''
    command = 'find "{}" -iname "*-{}-*"'.format(config.dir_entries, timestamp)
    return _find_with_command(command)

def modified_since(timestamp=0):
    '''Returns all entries last modified after the given timestamp.'''
    command = 'find "{}" -type f -newermt @{}'.format(config.dir_entries, timestamp)
    return _find_with_command(command)

def walk_all_entries(reverse=False):
    '''Iterates over all entries.'''
    for month in sorted(listdir(config.dir_entries), reverse=reverse):
        for filename in sorted(listdir(join(config.dir_entries, month)),
                               reverse=reverse):
            yield Entry(config.dir_entries, month, filename)

def range_of_entries(slice_args):
    '''Returns all entries in the given range.'''

    start, stop, step = slice_args

    # If start, stop and step are all positive (or None) then use islice.
    if (step is None or step > 0) and \
       (start is None or start >= 0) and \
       (stop is None or stop >=0):
        return islice(walk_all_entries(), start, stop, step)

    # If start, stop and step are all negative (or None) then use islice.
    elif (step is not None and step < 0) and \
         (start is None or start < 0) and \
         (stop is None or stop < 0):
        start = None if start is None else -1*start
        stop = None if stop is None else -1*stop
        step = None if step is None else -1*step
        return islice(walk_all_entries(reverse=True), start, stop, step)

    # If there is a mixture of positives and negatives then use regular slice.
    else:
        return list(walk_all_entries())[start:stop:step]

def last(num_entries=0):
    '''Return the most recent entries in decending order.'''
    return range_of_entries((None, -1*num_entries, -1))

def single_entry(index):
    '''Return a single entry at the given index.'''
    if index >= 0:
        return range_of_entries((index, index+1, 1))
    else:
        return range_of_entries((None if index==-1 else index+1, index, -1))

def filter_entries(search_string, entries=None):
    '''Filter entries by presence of the (regular expression) search string.

    If entries is None then search all entries in reverse order.'''

    if entries is None:
        entries = walk_all_entries(reverse=True)

    for entry in entries:
        if entry.contains(search_string):
            yield entry

def range_type(string, range_re=re.compile(
            r'^([+_-]?[0-9]+)?:([+_-]?[0-9]+)?(?::([+_-]?[0-9]+))?$')):

    '''Returns (start, stop, step) for the given slice.'''
    match = range_re.match(string)
    if not match:
        raise argparse.ArgumentTypeError('invalid range')

    start, stop, step = match.groups()
    start = None if start is None else int(start.replace('_', '-'))
    stop = None if stop is None else int(stop.replace('_', '-'))
    step = None if step is None else int(step.replace('_', '-'))

    return (start, stop, step)

class QueueAction(argparse.Action):
    '''Appends (dest, args) pairs to 'queue' in the returned Namespace.'''
    def __call__(self, parser, namespace, value, option_string=None):
        if not hasattr(namespace, 'queue'):
            namespace.queue = []
        namespace.queue.append( (self.dest, value) )

parser = argparse.ArgumentParser(description='Get entry filenames.')

parser.add_argument('-n', '--new', nargs='?', type=int,
                    dest='new_entry', metavar='NEW',
                    help='new entry with given timestamp',
                    action=QueueAction, default=argparse.SUPPRESS)

parser.add_argument('-r', '--range', type=range_type,
                    dest='range_of_entries', metavar='SLICE',
                    help='range of entries',
                    action=QueueAction, default=argparse.SUPPRESS)

parser.add_argument('-i', '--index', type=int,
                    dest='single_entry', metavar='INDEX',
                    help='single entry at INDEX',
                    action=QueueAction, default=argparse.SUPPRESS)

parser.add_argument('-t', '--timestamp', type=int,
                    dest='find_by_timestamp', metavar='TIMESTAMP',
                    help='all entries created on TIMESTAMP',
                    action=QueueAction, default=argparse.SUPPRESS)

parser.add_argument('-m', '--modified', type=int,
                    dest='modified_since', metavar='MODIFIED',
                    help='all entries modified since MODIFIED',
                    action=QueueAction, default=argparse.SUPPRESS)

parser.add_argument('-s', '--search', type=str,
                    dest='filter_entries', metavar='SEARCH',
                    help='all entries containing SEARCH',
                    action=QueueAction, default=argparse.SUPPRESS)

def process_args():
    '''Return a list of all entries specified by the command line args.'''

    args = parser.parse_args()

    if hasattr(args, 'queue'):
        for func_name, value in args.queue:
            for entry in globals()[func_name](value):
                yield entry

# If running as script then print the filenames of all entries.
if __name__ == '__main__':

    for entry in process_args():
        print(entry.pathname)
<commit_msg>Removed unused import statements<commit_after>#!/usr/bin/env python3
# Script for finding the names of diary entries.

# Should be able to return filenames corresponding to specific timestamps,
# ranges of entries, single entries and new entries (with a given date).

import config
from os import makedirs, listdir
from os.path import realpath, join, basename, dirname, exists, isfile, getmtime
import time
import argparse
import re
from subprocess import check_output, call
from itertools import islice
import shelve
import textwrap

# TODO move these into a settings module
pad_char = '='
color_middle = '\033[1;34m'
color_padding = '\033[0;34m'
color_end = '\033[0m'

# This might not work on non-Linux OSes.
# TODO find a more general solution and move to settings module.
terminal_width = int(check_output('tput cols', shell=True).strip())

# Open cache shelf.
cache_path = config.dir_entries_cache
if not exists(cache_path):
    makedirs(cache_path)
cache_shelf = shelve.open(cache_path + '/range-cache', writeback=False)

def cached(func):
    '''Cache the results of a function, recalculate when cache is outdated.'''

    # TODO preserve name and docstring
    def wrapper(self, *args):

        id = self.timestamp
        func_tuple = (func.__name__, args)

        cached_attrs = cache_shelf.get(id, None)

        if cached_attrs is None or cached_attrs['mtime'] != self.getmtime():
            cache_shelf[id] = { 'mtime': self.getmtime() }

        elif func_tuple in cached_attrs:
            return cached_attrs[func_tuple]

        result = func(self, *args)
        cached_attrs = cache_shelf[id]
        cached_attrs[func_tuple] = result
        cache_shelf[id] = cached_attrs
        return result

    return wrapper

# TODO move all formatting related methods to a separate module
#      only keey the _gen_text() method
class Entry():
    '''Encapsulates entry manipulation functionality.'''

    _filename_re = re.compile(
            r'^diary-(-?[0-9]+)-([a-zA-Z0-9_][a-zA-Z0-9_-]*)\.([a-z]+)$')

    def __init__(self, *path_components):
        self.pathname = realpath(join(*path_components))
        match = Entry._filename_re.match(basename(self.pathname))
        self.timestamp, self.device_name, self.extension = match.groups()

    def mkdir(self):
        '''Create the entry's base directory (if it does not exist).'''
        directory = dirname(self.pathname)
        if not exists(directory):
            makedirs(directory)

    def exists(self):
        '''Return True if the entry exists.'''
        return isfile(self.pathname)

    def getmtime(self):
        '''Return the timestamp when the entry was last modified.'''
        if not self.exists():
            return None
        else:
            return getmtime(self.pathname)

    @cached
    def contains(self, search_string):
        '''Return True if the entry contains the given search string.'''
        command = 'grep -qi "{}" "{}"'.format(search_string, self.pathname)
        return call(command, shell=True) == 0

    @cached
    def get_date_string(self, format='%A %d %B %Y %I:%M%p'):
        '''Return formatted string representing the entry creation date.'''
        # TODO add additional information (like today/yesterday/in the future).
        return time.strftime(format, time.localtime(float(self.timestamp)))

    @cached
    def wordcount(self):
        '''Return the number of space separated words in the entry.'''
        # TODO do wordcount in python
        command = 'wc -w < "{}"'.format(self.pathname)
        return int(check_output(command, shell=True).strip())

    def _gen_text(self):
        '''Return a generator over the lines of the entry.'''
        with open(self.pathname) as f:
            for line in f:
                yield line.strip()

    # TODO add search term highlighting after this stage
    # TODO skip final lines if they are empty?
    # TODO add markdown formatting?
    # TODO put header generation in separate method?
    def _gen_formatted(self, width, header):

        if header:
            left = pad_char + '{} words'.format(self.wordcount())
            right = str(self.timestamp) + pad_char
            middle = ' {} '.format(self.get_date_string())

            padding_left = pad_char * int(width/2 - len(left) - len(middle)/2)
            padding_right = pad_char * (width - len(left) -
                                len(padding_left) - len(middle) - len(right))

            header_string = color_padding + left + padding_left + color_end + \
                            color_middle + middle + color_end + \
                            color_padding + padding_right + right + color_end

            yield header_string

        wrapper = textwrap.TextWrapper(width=width)

        for line in self._gen_text():
            # TODO do this more efficiently?
            if line.strip() == '':
                yield ''
            for wrapped_line in wrapper.wrap(line):
                yield wrapped_line

        yield ''
        yield ''

    @cached
    def _formatted(self, width, header):
        return '\n'.join(self._gen_formatted(width, header))

    def formatted(self, width=terminal_width, header=True):
        '''Return a list of formatted lines, wrapped to width.'''
        return self._formatted(width, header)

    @cached
    def tags(self):
        '''Return a list of all tags occurring in the entry text.'''

        # TODO do this in python using _gen_text()
        command = r'grep -o "#\S\+\b" "{}" || true'.format(self.pathname)
        matches = check_output(command, shell=True, universal_newlines=True)

        matches = matches.split('\n')
        matches = [ match.lstrip('#').rstrip() for match in matches ]
        matches = [ match for match in matches if len(match)>0 ]

        return matches

# TODO move this to the Entry class definition.
def new_entry(timestamp=None, device_name=config.device_name):
    '''Return a new (not currently existing) entry.

    Note that the directory structure may not exist.'''

    timestamp = int(time.time()) if timestamp is None else int(timestamp)
    month = time.strftime('%Y-%m', time.localtime(timestamp))
    filename = 'diary-{}-{}.txt'.format(timestamp, device_name)

    return [Entry(config.dir_entries, month, filename)]

def _find_with_command(command):
    results = check_output(command, shell=True, universal_newlines=True)
    return [ Entry(result) for result in results.split('\n') if len(result)>0 ]

def find_by_timestamp(timestamp):
    '''Returns any entries with the given timestamp.'''
    command = 'find "{}" -iname "*-{}-*"'.format(config.dir_entries, timestamp)
    return _find_with_command(command)

def modified_since(timestamp=0):
    '''Returns all entries last modified after the given timestamp.'''
    command = 'find "{}" -type f -newermt @{}'.format(config.dir_entries, timestamp)
    return _find_with_command(command)

def walk_all_entries(reverse=False):
    '''Iterates over all entries.'''
    for month in sorted(listdir(config.dir_entries), reverse=reverse):
        for filename in sorted(listdir(join(config.dir_entries, month)),
                               reverse=reverse):
            yield Entry(config.dir_entries, month, filename)

def range_of_entries(slice_args):
    '''Returns all entries in the given range.'''

    start, stop, step = slice_args

    # If start, stop and step are all positive (or None) then use islice.
    if (step is None or step > 0) and \
       (start is None or start >= 0) and \
       (stop is None or stop >=0):
        return islice(walk_all_entries(), start, stop, step)

    # If start, stop and step are all negative (or None) then use islice.
    elif (step is not None and step < 0) and \
         (start is None or start < 0) and \
         (stop is None or stop < 0):
        start = None if start is None else -1*start
        stop = None if stop is None else -1*stop
        step = None if step is None else -1*step
        return islice(walk_all_entries(reverse=True), start, stop, step)

    # If there is a mixture of positives and negatives then use regular slice.
    else:
        return list(walk_all_entries())[start:stop:step]

def last(num_entries=0):
    '''Return the most recent entries in decending order.'''
    return range_of_entries((None, -1*num_entries, -1))

def single_entry(index):
    '''Return a single entry at the given index.'''
    if index >= 0:
        return range_of_entries((index, index+1, 1))
    else:
        return range_of_entries((None if index==-1 else index+1, index, -1))

def filter_entries(search_string, entries=None):
    '''Filter entries by presence of the (regular expression) search string.

    If entries is None then search all entries in reverse order.'''

    if entries is None:
        entries = walk_all_entries(reverse=True)

    for entry in entries:
        if entry.contains(search_string):
            yield entry

def range_type(string, range_re=re.compile(
            r'^([+_-]?[0-9]+)?:([+_-]?[0-9]+)?(?::([+_-]?[0-9]+))?$')):

    '''Returns (start, stop, step) for the given slice.'''
    match = range_re.match(string)
    if not match:
        raise argparse.ArgumentTypeError('invalid range')

    start, stop, step = match.groups()
    start = None if start is None else int(start.replace('_', '-'))
    stop = None if stop is None else int(stop.replace('_', '-'))
    step = None if step is None else int(step.replace('_', '-'))

    return (start, stop, step)

class QueueAction(argparse.Action):
    '''Appends (dest, args) pairs to 'queue' in the returned Namespace.'''
    def __call__(self, parser, namespace, value, option_string=None):
        if not hasattr(namespace, 'queue'):
            namespace.queue = []
        namespace.queue.append( (self.dest, value) )

parser = argparse.ArgumentParser(description='Get entry filenames.')

parser.add_argument('-n', '--new', nargs='?', type=int,
                    dest='new_entry', metavar='NEW',
                    help='new entry with given timestamp',
                    action=QueueAction, default=argparse.SUPPRESS)

parser.add_argument('-r', '--range', type=range_type,
                    dest='range_of_entries', metavar='SLICE',
                    help='range of entries',
                    action=QueueAction, default=argparse.SUPPRESS)

parser.add_argument('-i', '--index', type=int,
                    dest='single_entry', metavar='INDEX',
                    help='single entry at INDEX',
                    action=QueueAction, default=argparse.SUPPRESS)

parser.add_argument('-t', '--timestamp', type=int,
                    dest='find_by_timestamp', metavar='TIMESTAMP',
                    help='all entries created on TIMESTAMP',
                    action=QueueAction, default=argparse.SUPPRESS)

parser.add_argument('-m', '--modified', type=int,
                    dest='modified_since', metavar='MODIFIED',
                    help='all entries modified since MODIFIED',
                    action=QueueAction, default=argparse.SUPPRESS)

parser.add_argument('-s', '--search', type=str,
                    dest='filter_entries', metavar='SEARCH',
                    help='all entries containing SEARCH',
                    action=QueueAction, default=argparse.SUPPRESS)

def process_args():
    '''Return a list of all entries specified by the command line args.'''

    args = parser.parse_args()

    if hasattr(args, 'queue'):
        for func_name, value in args.queue:
            for entry in globals()[func_name](value):
                yield entry

# If running as script then print the filenames of all entries.
if __name__ == '__main__':

    for entry in process_args():
        print(entry.pathname)
<|endoftext|>


<commit_before>import miniboa,os, subprocess, sys
import pexpect, re
from subprocess import PIPE, STDOUT
from buckPasser import run
clients = []

class Client(object):
    def __init__(self, client):
        self.client = client
        self.process = pexpect.spawnu('./buckPasser.py')
        self.process.setecho(0)
    @property
    def active(self):
        return self.client.active

    @property
    def cmd_ready(self):
        return self.client.cmd_ready

    def get_command(self):
        return self.client.get_command()

def on_connect(client):
    clients.append(Client(client))

def process_clients():
    """
    Check each client, if client.cmd_ready == True then there is a line of
    input available via client.get_command().
    """
    for client in clients:
        if client.active and client.cmd_ready:
            # If the client sends input echo it to the chat room
            try:
                interact(client)
            except:
                client.client.deactivate()

def interact(client):
    choice = client.process.expect([re.compile('\n*/>$'), '\n'])
    if choice == 1:
        if len(client.process.before) == 0:
            return
        client.client.send(client.process.before + '\n')
    else:
        msg = client.get_command().strip()
        print('msg info',msg, len(msg))
        client.client.send(client.process.before + client.process.after[:-3] )
        client.process.sendline(msg)

if __name__ == "__main__":
    tn = miniboa.TelnetServer(
        port=7000,
        address='127.0.0.1',
        on_connect = on_connect
    )
    while True:
        tn.poll()
        if len(clients) > 0:
            process_clients()
<commit_msg>removed off by 1 input error<commit_after>import miniboa,os, subprocess, sys
import pexpect, re
from subprocess import PIPE, STDOUT
from buckPasser import run
clients = []

class Client(object):
    def __init__(self):
        self.process = pexpect.spawnu('python3', ['buckPasser.py'])
        self.process.setecho(0)

    def sendMessage(self, msg):
        print(msg, end = '')

    def getInput(self):
        '''
        assuming commandline
        '''
        return input()

    def expectInteract(self):
        '''
        Gets input and puts output from the expect process.
        Returns a string to be sent to the user and None otherwise
        '''
        choice = self.process.expect([re.compile('/>$'), '\n'])
        #print("choice ", choice)
        if choice == 0:
            self.sendMessage(self.process.before + self.process.after[:-3])
            msg = self.getInput()
            #print('msg info',msg, len(msg))
            self.process.sendline(msg)

        elif choice == 1:
            if len(self.process.before) > 0:
                self.sendMessage(self.process.before + '\n')

    def interact(self):
        '''
        Interacts with the client
        '''
        self.expectInteract()

class TelnetClient(Client):
    def __init__(self, client):
        Client.__init__(self)
        self.client = client

    @property
    def active(self):
        return self.client.active

    @property
    def cmd_ready(self):
        return self.client.cmd_ready

    def sendMessage(self,msg):
        self.client.send(msg)

    def getInput(self):
        return client.client.get_command().strip()

    def close(self):
        self.client.deactivate()

def on_connect(client):
    clients.append(TelnetClient(client))

def process_clients():
    '''
    Check each client, if client.cmd_ready == True then there is a line of
    input available via client.getInput.
    '''
    for client in clients:
        if client.active and client.cmd_ready:
            # If the client sends input interacte with it
            try:
                client.interact()
            except:
                client.close()

if __name__ == "__main__":
    tn = miniboa.TelnetServer(
        port=7000,
        address='127.0.0.1',
        on_connect = on_connect
    )
    while True:
        tn.poll()
        if len(clients) > 0:
            process_clients()
<|endoftext|>


<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-
"""
Copyright 2010 Asidev s.r.l.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
"""

from aybu.core.models import Base
from aybu.core.models import Banner
from aybu.core.models import File
from aybu.core.models import Image
from aybu.core.models import Language
from aybu.core.models import ExternalLink
from aybu.core.models import ExternalLinkInfo
from aybu.core.models import InternalLink
from aybu.core.models import InternalLinkInfo
from aybu.core.models import Menu
from aybu.core.models import MenuInfo
from aybu.core.models import Node
from aybu.core.models import NodeInfo
from aybu.core.models import Page
from aybu.core.models import PageInfo
from aybu.core.models import Section
from aybu.core.models import SectionInfo
from aybu.core.models import Setting
from aybu.core.models import SettingType
from aybu.core.models import Keyword
from aybu.core.models import Theme
from aybu.core.models import Group
from aybu.core.models import User
from aybu.core.models import View
from aybu.core.models import ViewDescription

__all__ = ['Base', 'File', 'Image', 'Banner', 'Language',
           'Node', 'Menu', 'Page', 'Section', 'ExternalLink', 'InternalLink',
           'MenuInfo', 'NodeInfo', 'Setting', 'SettingType', 'Keyword', 'Theme',
           'User', 'Group', 'View', 'ViewDescription',
           'ExternalLinkInfo', 'InternalLinkInfo', 'PageInfo', 'SectionInfo']
<commit_msg>Removed models modules, import models directly from core<commit_after><|endoftext|>


<commit_before>from __future__ import division

def frequencies(temperament, octaves_low, octaves_high, base_freq = 440.0):
    freq = []
    for octave in range(1, octaves_low + 1):
        freq =  [(base_freq / 2 ** octave) * note for note in temperament] + freq
    for octave in range(octaves_high):
        freq += [(base_freq * 2 ** octave) * note for note in temperament]
    return freq

def equal_temperament():
    return [ 2. ** (i/12)  for i in range(12)]
<commit_msg>definition of cents<commit_after>from __future__ import division
import math

def frequencies(temperament, octaves_low, octaves_high, base_freq = 440.0):
    freq = []
    for octave in range(1, octaves_low + 1):
        freq =  [(base_freq / 2 ** octave) * note for note in temperament] + freq
    for octave in range(octaves_high):
        freq += [(base_freq * 2 ** octave) * note for note in temperament]
    return freq

def equal_temperament():
    return [ 2. ** (i/12)  for i in range(12)]

def to_cent(temperament):
    a = temperament[0]
    return [int(1200 * math.log(t / a, 2) + .5) for t in temperament]
<|endoftext|>


<commit_before>import logging
import yaml

# Tested on Azure 2.0 API
from azure.common.credentials import ServicePrincipalCredentials
from azure.mgmt.compute import (
    ComputeManagementClient,
    )
from azure.mgmt.resource.subscriptions import SubscriptionClient
from msrestazure.azure_exceptions import CloudError
from simplestreams.json2streams import Item

from make_aws_image_streams import (
    get_parameters,
    make_aws_items,
    write_item_streams,
    )

MS_VSTUDIO = 'MicrosoftVisualStudio'
MS_SERVER = 'MicrosoftWindowsServer'
WINDOWS = 'Windows'
WINDOWS_SERVER = 'WindowsServer'
IMAGE_SPEC = [
    ('win81', MS_VSTUDIO, WINDOWS, '8.1-Enterprise-N'),
    ('win10', MS_VSTUDIO, WINDOWS, '10-Enterprise-N'),
    ('win2012', MS_SERVER, WINDOWS_SERVER, '2012-Datacenter'),
    ('win2012r2', MS_SERVER, WINDOWS_SERVER, '2012-R2-Datacenter'),
    ('centos7', 'OpenLogic', 'CentOS', '7.1'),
]

def get_azure_credentials(all_credentials):
    azure_dict = all_credentials['azure']['credentials']
    subscription_id = azure_dict['subscription-id']
    return subscription_id, ServicePrincipalCredentials(
        client_id=azure_dict['application-id'],
        secret=azure_dict['application-password'],
        tenant=azure_dict['tenant-id'],
        subscription_id=azure_dict['subscription-id'],
        )

def get_image_versions(client, region, region_name):
    endpoint = client.config.base_url
    for full_spec in IMAGE_SPEC:
        spec = full_spec[1:]
        try:
            versions = client.virtual_machine_images.list(region, *spec)
        except CloudError:
            logging.warning('Could not find {} {} {} in {region}'.format(*spec,
                            region=region))
            continue
        for version in versions:
            yield make_item(version, full_spec, region_name, endpoint)

def make_item(version, full_spec, region_name, endpoint):
    URN = ':'.join(full_spec[1:] + (version.name,))
    product_name = (
        'com.ubuntu.cloud:server:centos7:amd64' if full_spec[2] == 'CentOS'
        else 'com.ubuntu.cloud:windows')
    return Item(
        'com.ubuntu.cloud:released:azure',
        product_name,
        version.name, version.location, {
            'arch': 'amd64',
            'virt': 'Hyper-V',
            'region': region_name,
            'id': URN,
            'label': 'release',
            'endpoint': endpoint,
            'release': full_spec[0],
            }
        )

def make_azure_items(all_credentials):
    subscription_id, credentials = get_azure_credentials(all_credentials)
    sub_client = SubscriptionClient(credentials)
    client = ComputeManagementClient(credentials, subscription_id)
    items = []
    for region in sub_client.subscriptions.list_locations(subscription_id):
        logging.info('Retrieving image data in {}'.format(region.display_name))
        items.extend(get_image_versions(
            client, region.name, region.display_name))
    return items
<commit_msg>Fix lint.<commit_after>import logging

# Tested on Azure 2.0 API
from azure.common.credentials import ServicePrincipalCredentials
from azure.mgmt.compute import (
    ComputeManagementClient,
    )
from azure.mgmt.resource.subscriptions import SubscriptionClient
from msrestazure.azure_exceptions import CloudError
from simplestreams.json2streams import Item

MS_VSTUDIO = 'MicrosoftVisualStudio'
MS_SERVER = 'MicrosoftWindowsServer'
WINDOWS = 'Windows'
WINDOWS_SERVER = 'WindowsServer'
IMAGE_SPEC = [
    ('win81', MS_VSTUDIO, WINDOWS, '8.1-Enterprise-N'),
    ('win10', MS_VSTUDIO, WINDOWS, '10-Enterprise-N'),
    ('win2012', MS_SERVER, WINDOWS_SERVER, '2012-Datacenter'),
    ('win2012r2', MS_SERVER, WINDOWS_SERVER, '2012-R2-Datacenter'),
    ('centos7', 'OpenLogic', 'CentOS', '7.1'),
]

def get_azure_credentials(all_credentials):
    azure_dict = all_credentials['azure']['credentials']
    subscription_id = azure_dict['subscription-id']
    return subscription_id, ServicePrincipalCredentials(
        client_id=azure_dict['application-id'],
        secret=azure_dict['application-password'],
        tenant=azure_dict['tenant-id'],
        subscription_id=azure_dict['subscription-id'],
        )

def get_image_versions(client, region, region_name):
    endpoint = client.config.base_url
    for full_spec in IMAGE_SPEC:
        spec = full_spec[1:]
        try:
            versions = client.virtual_machine_images.list(region, *spec)
        except CloudError:
            logging.warning('Could not find {} {} {} in {region}'.format(*spec,
                            region=region))
            continue
        for version in versions:
            yield make_item(version, full_spec, region_name, endpoint)

def make_item(version, full_spec, region_name, endpoint):
    URN = ':'.join(full_spec[1:] + (version.name,))
    product_name = (
        'com.ubuntu.cloud:server:centos7:amd64' if full_spec[2] == 'CentOS'
        else 'com.ubuntu.cloud:windows')
    return Item(
        'com.ubuntu.cloud:released:azure',
        product_name,
        version.name, version.location, {
            'arch': 'amd64',
            'virt': 'Hyper-V',
            'region': region_name,
            'id': URN,
            'label': 'release',
            'endpoint': endpoint,
            'release': full_spec[0],
            }
        )

def make_azure_items(all_credentials):
    subscription_id, credentials = get_azure_credentials(all_credentials)
    sub_client = SubscriptionClient(credentials)
    client = ComputeManagementClient(credentials, subscription_id)
    items = []
    for region in sub_client.subscriptions.list_locations(subscription_id):
        logging.info('Retrieving image data in {}'.format(region.display_name))
        items.extend(get_image_versions(
            client, region.name, region.display_name))
    return items
<|endoftext|>


<commit_before>#    Copyright 2013, 2014 Joshua Charles Campbell
#
#    This file is part of UnnaturalCode.
#
#    UnnaturalCode is free software: you can redistribute it and/or modify
#    it under the terms of the GNU Affero General Public License as published by
#    the Free Software Foundation, either version 3 of the License, or
#    (at your option) any later version.
#
#    UnnaturalCode is distributed in the hope that it will be useful,
#    but WITHOUT ANY WARRANTY; without even the implied warranty of
#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#    GNU Affero General Public License for more details.
#
#    You should have received a copy of the GNU Affero General Public License
#    along with UnnaturalCode.  If not, see <http://www.gnu.org/licenses/>.

from unnaturalcode.ucUtil import *
from unnaturalcode.mitlmCorpus import *
from unnaturalcode.pythonSource import *
from operator import itemgetter

from logging import debug, info, warning, error

class sourceModel(object):

    def __init__(self, cm=mitlmCorpus(), language=pythonSource, windowSize=20):
        self.cm = cm
        self.lang = language
        self.windowSize = windowSize

    def trainFile(self, files):
        """Blindly train on a set of files whether or not it compiles..."""
        files = [files] if isinstance(files, str) else files
        assert isinstance(files, list)
        for fi in files:
            sourceCode = slurp(fi)
            self.trainString(sourceCode)

    def stringifyAll(self, lexemes):
        """Clean up a list of lexemes and convert it to a list of strings"""
        return [i.stringify() for i in lexemes]

    def corpify(self, lexemes):
        """Corpify a string"""
        return self.cm.corpify(self.stringifyAll(lexemes))

    def sourceToScrubbed(self, sourceCode):
        return self.lang(sourceCode).scrubbed()

    def trainLexemes(self, lexemes):
        """Train on a lexeme sequence."""
        return self.cm.addToCorpus(self.stringifyAll(lexemes))

    def trainString(self, sourceCode):
        """Train on a source code string"""
        return self.trainLexemes(self.sourceToScrubbed(sourceCode))

    def queryString(self, sourceCode):
        return self.queryLexed(self.lang(sourceCode))

    def queryLexed(self, lexemes):
        return self.cm.queryCorpus(self.stringifyAll(lexemes))

    def predictLexed(self, lexemes):
        return self.cm.predictCorpus(self.stringifyAll(lexemes))

    def windowedQuery(self, lexemes, returnWindows=True):
        lastWindowStarts = len(lexemes)-self.windowSize
        if lastWindowStarts < 1:
          return [(lexemes, self.queryLexed(lexemes))]
        r = []
        for i in range(0,lastWindowStarts+1): # remember range is [)
            end = i+self.windowSize
            w = lexemes[i:end] # remember range is [)
            e = self.queryLexed(w)
            if returnWindows:
                r.append( (w,e) )
            else:
                r.append( (False,e) )
        return r

    def worstWindows(self, lexemes):
        lexemes = lexemes.scrubbed()
        unsorted = self.windowedQuery(lexemes)
        return sorted(unsorted, key=itemgetter(1), reverse=True)

    def release(self):
        self.cm.release()
<commit_msg>fix boog<commit_after>#    Copyright 2013, 2014 Joshua Charles Campbell
#
#    This file is part of UnnaturalCode.
#
#    UnnaturalCode is free software: you can redistribute it and/or modify
#    it under the terms of the GNU Affero General Public License as published by
#    the Free Software Foundation, either version 3 of the License, or
#    (at your option) any later version.
#
#    UnnaturalCode is distributed in the hope that it will be useful,
#    but WITHOUT ANY WARRANTY; without even the implied warranty of
#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#    GNU Affero General Public License for more details.
#
#    You should have received a copy of the GNU Affero General Public License
#    along with UnnaturalCode.  If not, see <http://www.gnu.org/licenses/>.

from unnaturalcode.ucUtil import *
from unnaturalcode.mitlmCorpus import *
from unnaturalcode.pythonSource import *
from operator import itemgetter

from logging import debug, info, warning, error

class sourceModel(object):

    def __init__(self, cm=mitlmCorpus(), language=pythonSource, windowSize=20):
        self.cm = cm
        self.lang = language
        self.windowSize = windowSize

    def trainFile(self, files):
        """Blindly train on a set of files whether or not it compiles..."""
        files = [files] if isinstance(files, str) else files
        assert isinstance(files, list)
        for fi in files:
            sourceCode = slurp(fi)
            self.trainString(sourceCode)

    def stringifyAll(self, lexemes):
        """Clean up a list of lexemes and convert it to a list of strings"""
        return [i.stringify() for i in lexemes]

    def corpify(self, lexemes):
        """Corpify a string"""
        return self.cm.corpify(self.stringifyAll(lexemes))

    def sourceToScrubbed(self, sourceCode):
        return self.lang(sourceCode).scrubbed()

    def trainLexemes(self, lexemes):
        """Train on a lexeme sequence."""
        return self.cm.addToCorpus(self.stringifyAll(lexemes))

    def trainString(self, sourceCode):
        """Train on a source code string"""
        return self.trainLexemes(self.sourceToScrubbed(sourceCode))

    def queryString(self, sourceCode):
        return self.queryLexed(self.lang(sourceCode))

    def queryLexed(self, lexemes):
        return self.cm.queryCorpus(self.stringifyAll(lexemes))

    def predictLexed(self, lexemes):
        return self.cm.predictCorpus(self.stringifyAll(lexemes))

    def windowedQuery(self, lexemes, returnWindows=True):
        lastWindowStarts = len(lexemes)-self.windowSize
        if lastWindowStarts < 1:
            if returnWindows:
                return [(lexemes, self.queryLexed(lexemes))]
            else:
                return [(False, self.queryLexed(lexemes))]
        r = []
        for i in range(0,lastWindowStarts+1): # remember range is [)
            end = i+self.windowSize
            w = lexemes[i:end] # remember range is [)
            e = self.queryLexed(w)
            if returnWindows:
                r.append( (w,e) )
            else:
                r.append( (False,e) )
        return r

    def worstWindows(self, lexemes):
        lexemes = lexemes.scrubbed()
        unsorted = self.windowedQuery(lexemes)
        return sorted(unsorted, key=itemgetter(1), reverse=True)

    def release(self):
        self.cm.release()
<|endoftext|>