<commit_before>/*
 * This file is open source software, licensed to you under the terms
 * of the Apache License, Version 2.0 (the "License").  See the NOTICE file
 * distributed with this work for additional information regarding copyright
 * ownership.  You may not use this file except in compliance with the License.
 *
 * You may obtain a copy of the License at
 *
 *   http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing,
 * software distributed under the License is distributed on an
 * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
 * KIND, either express or implied.  See the License for the
 * specific language governing permissions and limitations
 * under the License.
 */
/*
 * Copyright (C) 2014 Cloudius Systems, Ltd.
 */

#include "resource.hh"
#include "core/align.hh"

namespace resource {

size_t calculate_memory(configuration c, size_t available_memory, float panic_factor = 1) {
    size_t default_reserve_memory = std::max<size_t>(1 << 30, 0.05 * available_memory) * panic_factor;
    available_memory -= c.reserve_memory.value_or(default_reserve_memory);
    size_t mem = c.total_memory.value_or(available_memory);
    if (mem > available_memory) {
        throw std::runtime_error("insufficient physical memory");
    }
    return mem;
}

}

#ifdef HAVE_HWLOC

#include "util/defer.hh"
#include "core/print.hh"
#include <hwloc.h>
#include <unordered_map>
#include <boost/range/irange.hpp>

cpu_set_t cpuid_to_cpuset(unsigned cpuid) {
    cpu_set_t cs;
    CPU_ZERO(&cs);
    CPU_SET(cpuid, &cs);
    return cs;
}

namespace resource {

size_t div_roundup(size_t num, size_t denom) {
    return (num + denom - 1) / denom;
}

static unsigned find_memory_depth(hwloc_topology_t& topology) {
    auto depth = hwloc_get_type_depth(topology, HWLOC_OBJ_PU);
    auto obj = hwloc_get_next_obj_by_depth(topology, depth, nullptr);

    while (!obj->memory.local_memory && obj) {
        obj = hwloc_get_ancestor_obj_by_depth(topology, --depth, obj);
    }
    assert(obj);
    return depth;
}

static size_t alloc_from_node(cpu& this_cpu, hwloc_obj_t node, std::unordered_map<hwloc_obj_t, size_t>& used_mem, size_t alloc) {
    auto taken = std::min(node->memory.local_memory - used_mem[node], alloc);
    if (taken) {
        used_mem[node] += taken;
        auto node_id = hwloc_bitmap_first(node->nodeset);
        assert(node_id != -1);
        this_cpu.mem.push_back({taken, unsigned(node_id)});
    }
    return taken;
}

struct distribute_objects {
    std::vector<hwloc_cpuset_t> cpu_sets;
    hwloc_obj_t root;

    distribute_objects(hwloc_topology_t& topology, size_t nobjs) : cpu_sets(nobjs), root(hwloc_get_root_obj(topology)) {
#if HWLOC_API_VERSION >= 0x00010900
        hwloc_distrib(topology, &root, 1, cpu_sets.data(), cpu_sets.size(), INT_MAX, 0);
#else
        hwloc_distribute(topology, root, cpu_sets.data(), cpu_sets.size(), INT_MAX);
#endif
    }

    ~distribute_objects() {
        for (auto&& cs : cpu_sets) {
            hwloc_bitmap_free(cs);
        }
    }
    std::vector<hwloc_cpuset_t>& operator()() {
        return cpu_sets;
    }
};

static io_queue_topology
allocate_io_queues(hwloc_topology_t& topology, configuration c, std::vector<cpu> cpus) {
    unsigned num_io_queues = c.io_queues.value_or(cpus.size());
    unsigned max_io_requests = c.max_io_requests.value_or(128 * num_io_queues);

    unsigned depth = find_memory_depth(topology);
    auto node_of_shard = [&topology, &cpus, &depth] (unsigned shard) {
        auto pu = hwloc_get_pu_obj_by_os_index(topology, cpus[shard].cpu_id);
        auto node = hwloc_get_ancestor_obj_by_depth(topology, depth, pu);
        return hwloc_bitmap_first(node->nodeset);
    };

    // There are two things we are trying to achieve by populating a numa_nodes map.
    //
    // The first is to find out how many nodes we have in the system. We can't use
    // hwloc for that, because at this point we are not longer talking about the physical system,
    // but the actual booted seastar server instead. So if we have restricted the run to a subset
    // of the available processors, counting topology nodes won't spur the same result.
    //
    // Secondly, we need to find out which processors live in each node. For a reason similar to the
    // above, hwloc won't do us any good here. Later on, we will use this information to assign
    // shards to coordinators that are node-local to themselves.
    std::unordered_map<unsigned, std::set<unsigned>> numa_nodes;
    for (auto shard: boost::irange(0, int(cpus.size()))) {
        auto node_id = node_of_shard(shard);

        if (numa_nodes.count(node_id) == 0) {
            numa_nodes.emplace(node_id, std::set<unsigned>());
        }
        numa_nodes.at(node_id).insert(shard);
    }

    io_queue_topology ret;
    ret.shard_to_coordinator.resize(cpus.size());

    // If we have more than one node, we will mandate at least one coordinator
    // per node. It simplifies the coordinator assignment and in real scenarios
    // we don't want to be passing things around to the other side of the box
    // anyway. We could silently adjust, but it is better to avoid surprises.
    if ((num_io_queues < numa_nodes.size()) || (num_io_queues > cpus.size())) {
        auto msg = sprint("Invalid number of IO queues. Asked for %d. Minimum value is %d, maximum %d", num_io_queues, numa_nodes.size(), cpus.size());
        throw std::runtime_error(std::move(msg));
    }

    auto find_shard = [&cpus] (unsigned cpu_id) {
        auto idx = 0u;
        for (auto& c: cpus) {
            if (c.cpu_id == cpu_id) {
                return idx;
            }
            idx++;
        }
        assert(0);
    };

    auto cpu_sets = distribute_objects(topology, num_io_queues);
    // First step: distribute the IO queues given the information returned in cpu_sets.
    // If there is one IO queue per processor, only this loop will be executed.
    std::unordered_map<unsigned, std::vector<unsigned>> node_coordinators;
    for (auto&& cs : cpu_sets()) {
        auto io_coordinator = find_shard(hwloc_bitmap_first(cs));

        ret.coordinators.emplace_back(io_queue{io_coordinator, std::max(max_io_requests / num_io_queues , 1u)});
        // If a processor is a coordinator, it is also obviously a coordinator of itself
        ret.shard_to_coordinator[io_coordinator] = io_coordinator;

        auto node_id = node_of_shard(io_coordinator);
        if (node_coordinators.count(node_id) == 0) {
            node_coordinators.emplace(node_id, std::vector<unsigned>());
        }
        node_coordinators.at(node_id).push_back(io_coordinator);
        numa_nodes[node_id].erase(io_coordinator);
    }

    // If there are more processors than coordinators, we will have to assign them to existing
    // coordinators. We always do that within the same NUMA node.
    for (auto& node: numa_nodes) {
        auto cid_idx = 0;
        for (auto& remaining_shard: node.second) {
            auto idx = cid_idx++ % node_coordinators.at(node.first).size();
            auto io_coordinator = node_coordinators.at(node.first)[idx];
            ret.shard_to_coordinator[remaining_shard] = io_coordinator;
        }
    }

    return ret;
}

resources allocate(configuration c) {
    hwloc_topology_t topology;
    hwloc_topology_init(&topology);
    auto free_hwloc = defer([&] { hwloc_topology_destroy(topology); });
    hwloc_topology_load(topology);
    if (c.cpu_set) {
        auto bm = hwloc_bitmap_alloc();
        auto free_bm = defer([&] { hwloc_bitmap_free(bm); });
        for (auto idx : *c.cpu_set) {
            hwloc_bitmap_set(bm, idx);
        }
        auto r = hwloc_topology_restrict(topology, bm,
                HWLOC_RESTRICT_FLAG_ADAPT_DISTANCES
                | HWLOC_RESTRICT_FLAG_ADAPT_MISC
                | HWLOC_RESTRICT_FLAG_ADAPT_IO);
        if (r == -1) {
            if (errno == ENOMEM) {
                throw std::bad_alloc();
            }
            if (errno == EINVAL) {
                throw std::runtime_error("bad cpuset");
            }
            abort();
        }
    }
    auto machine_depth = hwloc_get_type_depth(topology, HWLOC_OBJ_MACHINE);
    assert(hwloc_get_nbobjs_by_depth(topology, machine_depth) == 1);
    auto machine = hwloc_get_obj_by_depth(topology, machine_depth, 0);
    auto available_memory = machine->memory.total_memory;
    // hwloc doesn't account for kernel reserved memory, so set panic_factor = 2
    size_t mem = calculate_memory(c, available_memory, 2);
    unsigned available_procs = hwloc_get_nbobjs_by_type(topology, HWLOC_OBJ_PU);
    unsigned procs = c.cpus.value_or(available_procs);
    if (procs > available_procs) {
        throw std::runtime_error("insufficient processing units");
    }
    auto mem_per_proc = align_down<size_t>(mem / procs, 2 << 20);

    resources ret;
    std::unordered_map<hwloc_obj_t, size_t> topo_used_mem;
    std::vector<std::pair<cpu, size_t>> remains;
    size_t remain;
    unsigned depth = find_memory_depth(topology);

    auto cpu_sets = distribute_objects(topology, procs);

    // Divide local memory to cpus
    for (auto&& cs : cpu_sets()) {
        auto cpu_id = hwloc_bitmap_first(cs);
        assert(cpu_id != -1);
        auto pu = hwloc_get_pu_obj_by_os_index(topology, cpu_id);
        auto node = hwloc_get_ancestor_obj_by_depth(topology, depth, pu);
        cpu this_cpu;
        this_cpu.cpu_id = cpu_id;
        remain = mem_per_proc - alloc_from_node(this_cpu, node, topo_used_mem, mem_per_proc);

        remains.emplace_back(std::move(this_cpu), remain);
    }

    // Divide the rest of the memory
    for (auto&& r : remains) {
        cpu this_cpu;
        size_t remain;
        std::tie(this_cpu, remain) = r;
        auto pu = hwloc_get_pu_obj_by_os_index(topology, this_cpu.cpu_id);
        auto node = hwloc_get_ancestor_obj_by_depth(topology, depth, pu);
        auto obj = node;

        while (remain) {
            remain -= alloc_from_node(this_cpu, obj, topo_used_mem, remain);
            do {
                obj = hwloc_get_next_obj_by_depth(topology, depth, obj);
            } while (!obj);
            if (obj == node)
                break;
        }
        assert(!remain);
        ret.cpus.push_back(std::move(this_cpu));
    }

    ret.io_queues = allocate_io_queues(topology, c, ret.cpus);
    return ret;
}

unsigned nr_processing_units() {
    hwloc_topology_t topology;
    hwloc_topology_init(&topology);
    auto free_hwloc = defer([&] { hwloc_topology_destroy(topology); });
    hwloc_topology_load(topology);
    return hwloc_get_nbobjs_by_type(topology, HWLOC_OBJ_PU);
}

}

#else

#include "resource.hh"
#include <unistd.h>

namespace resource {

// Without hwloc, we don't support tuning the number of IO queues. So each CPU gets their.
static io_queue_topology
allocate_io_queues(configuration c, std::vector<cpu> cpus) {
    io_queue_topology ret;

    unsigned nr_cpus = unsigned(cpus.size());
    unsigned max_io_requests = c.max_io_requests.value_or(128 * nr_cpus);

    ret.shard_to_coordinator.resize(nr_cpus);
    ret.coordinators.resize(nr_cpus);

    for (unsigned shard = 0; shard < nr_cpus; ++shard) {
        ret.shard_to_coordinator[shard] = shard;
        ret.coordinators[shard].capacity =  std::max(max_io_requests / nr_cpus, 1u);
        ret.coordinators[shard].id = shard;
    }
    return ret;
}

resources allocate(configuration c) {
    resources ret;

    auto available_memory = ::sysconf(_SC_PAGESIZE) * size_t(::sysconf(_SC_PHYS_PAGES));
    auto mem = calculate_memory(c, available_memory);
    auto cpuset_procs = c.cpu_set ? c.cpu_set->size() : nr_processing_units();
    auto procs = c.cpus.value_or(cpuset_procs);
    ret.cpus.reserve(procs);
    for (unsigned i = 0; i < procs; ++i) {
        ret.cpus.push_back(cpu{i, {{mem / procs, 0}}});
    }

    ret.io_queues = allocate_io_queues(c, ret.cpus);
    return ret;
}

unsigned nr_processing_units() {
    return ::sysconf(_SC_NPROCESSORS_ONLN);
}

}

#endif
<commit_msg>resource: fix failures on low-memory machines<commit_after>/*
 * This file is open source software, licensed to you under the terms
 * of the Apache License, Version 2.0 (the "License").  See the NOTICE file
 * distributed with this work for additional information regarding copyright
 * ownership.  You may not use this file except in compliance with the License.
 *
 * You may obtain a copy of the License at
 *
 *   http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing,
 * software distributed under the License is distributed on an
 * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
 * KIND, either express or implied.  See the License for the
 * specific language governing permissions and limitations
 * under the License.
 */
/*
 * Copyright (C) 2014 Cloudius Systems, Ltd.
 */

#include "resource.hh"
#include "core/align.hh"

namespace resource {

size_t calculate_memory(configuration c, size_t available_memory, float panic_factor = 1) {
    size_t default_reserve_memory = std::max<size_t>(1 << 30, 0.05 * available_memory) * panic_factor;
    auto reserve = c.reserve_memory.value_or(default_reserve_memory);
    size_t min_memory = 500'000'000;
    if (available_memory >= reserve + min_memory) {
        available_memory -= reserve;
    } else {
        // Allow starting up even in low memory configurations (e.g. 2GB boot2docker VM)
        available_memory = min_memory;
    }
    size_t mem = c.total_memory.value_or(available_memory);
    if (mem > available_memory) {
        throw std::runtime_error("insufficient physical memory");
    }
    return mem;
}

}

#ifdef HAVE_HWLOC

#include "util/defer.hh"
#include "core/print.hh"
#include <hwloc.h>
#include <unordered_map>
#include <boost/range/irange.hpp>

cpu_set_t cpuid_to_cpuset(unsigned cpuid) {
    cpu_set_t cs;
    CPU_ZERO(&cs);
    CPU_SET(cpuid, &cs);
    return cs;
}

namespace resource {

size_t div_roundup(size_t num, size_t denom) {
    return (num + denom - 1) / denom;
}

static unsigned find_memory_depth(hwloc_topology_t& topology) {
    auto depth = hwloc_get_type_depth(topology, HWLOC_OBJ_PU);
    auto obj = hwloc_get_next_obj_by_depth(topology, depth, nullptr);

    while (!obj->memory.local_memory && obj) {
        obj = hwloc_get_ancestor_obj_by_depth(topology, --depth, obj);
    }
    assert(obj);
    return depth;
}

static size_t alloc_from_node(cpu& this_cpu, hwloc_obj_t node, std::unordered_map<hwloc_obj_t, size_t>& used_mem, size_t alloc) {
    auto taken = std::min(node->memory.local_memory - used_mem[node], alloc);
    if (taken) {
        used_mem[node] += taken;
        auto node_id = hwloc_bitmap_first(node->nodeset);
        assert(node_id != -1);
        this_cpu.mem.push_back({taken, unsigned(node_id)});
    }
    return taken;
}

struct distribute_objects {
    std::vector<hwloc_cpuset_t> cpu_sets;
    hwloc_obj_t root;

    distribute_objects(hwloc_topology_t& topology, size_t nobjs) : cpu_sets(nobjs), root(hwloc_get_root_obj(topology)) {
#if HWLOC_API_VERSION >= 0x00010900
        hwloc_distrib(topology, &root, 1, cpu_sets.data(), cpu_sets.size(), INT_MAX, 0);
#else
        hwloc_distribute(topology, root, cpu_sets.data(), cpu_sets.size(), INT_MAX);
#endif
    }

    ~distribute_objects() {
        for (auto&& cs : cpu_sets) {
            hwloc_bitmap_free(cs);
        }
    }
    std::vector<hwloc_cpuset_t>& operator()() {
        return cpu_sets;
    }
};

static io_queue_topology
allocate_io_queues(hwloc_topology_t& topology, configuration c, std::vector<cpu> cpus) {
    unsigned num_io_queues = c.io_queues.value_or(cpus.size());
    unsigned max_io_requests = c.max_io_requests.value_or(128 * num_io_queues);

    unsigned depth = find_memory_depth(topology);
    auto node_of_shard = [&topology, &cpus, &depth] (unsigned shard) {
        auto pu = hwloc_get_pu_obj_by_os_index(topology, cpus[shard].cpu_id);
        auto node = hwloc_get_ancestor_obj_by_depth(topology, depth, pu);
        return hwloc_bitmap_first(node->nodeset);
    };

    // There are two things we are trying to achieve by populating a numa_nodes map.
    //
    // The first is to find out how many nodes we have in the system. We can't use
    // hwloc for that, because at this point we are not longer talking about the physical system,
    // but the actual booted seastar server instead. So if we have restricted the run to a subset
    // of the available processors, counting topology nodes won't spur the same result.
    //
    // Secondly, we need to find out which processors live in each node. For a reason similar to the
    // above, hwloc won't do us any good here. Later on, we will use this information to assign
    // shards to coordinators that are node-local to themselves.
    std::unordered_map<unsigned, std::set<unsigned>> numa_nodes;
    for (auto shard: boost::irange(0, int(cpus.size()))) {
        auto node_id = node_of_shard(shard);

        if (numa_nodes.count(node_id) == 0) {
            numa_nodes.emplace(node_id, std::set<unsigned>());
        }
        numa_nodes.at(node_id).insert(shard);
    }

    io_queue_topology ret;
    ret.shard_to_coordinator.resize(cpus.size());

    // If we have more than one node, we will mandate at least one coordinator
    // per node. It simplifies the coordinator assignment and in real scenarios
    // we don't want to be passing things around to the other side of the box
    // anyway. We could silently adjust, but it is better to avoid surprises.
    if ((num_io_queues < numa_nodes.size()) || (num_io_queues > cpus.size())) {
        auto msg = sprint("Invalid number of IO queues. Asked for %d. Minimum value is %d, maximum %d", num_io_queues, numa_nodes.size(), cpus.size());
        throw std::runtime_error(std::move(msg));
    }

    auto find_shard = [&cpus] (unsigned cpu_id) {
        auto idx = 0u;
        for (auto& c: cpus) {
            if (c.cpu_id == cpu_id) {
                return idx;
            }
            idx++;
        }
        assert(0);
    };

    auto cpu_sets = distribute_objects(topology, num_io_queues);
    // First step: distribute the IO queues given the information returned in cpu_sets.
    // If there is one IO queue per processor, only this loop will be executed.
    std::unordered_map<unsigned, std::vector<unsigned>> node_coordinators;
    for (auto&& cs : cpu_sets()) {
        auto io_coordinator = find_shard(hwloc_bitmap_first(cs));

        ret.coordinators.emplace_back(io_queue{io_coordinator, std::max(max_io_requests / num_io_queues , 1u)});
        // If a processor is a coordinator, it is also obviously a coordinator of itself
        ret.shard_to_coordinator[io_coordinator] = io_coordinator;

        auto node_id = node_of_shard(io_coordinator);
        if (node_coordinators.count(node_id) == 0) {
            node_coordinators.emplace(node_id, std::vector<unsigned>());
        }
        node_coordinators.at(node_id).push_back(io_coordinator);
        numa_nodes[node_id].erase(io_coordinator);
    }

    // If there are more processors than coordinators, we will have to assign them to existing
    // coordinators. We always do that within the same NUMA node.
    for (auto& node: numa_nodes) {
        auto cid_idx = 0;
        for (auto& remaining_shard: node.second) {
            auto idx = cid_idx++ % node_coordinators.at(node.first).size();
            auto io_coordinator = node_coordinators.at(node.first)[idx];
            ret.shard_to_coordinator[remaining_shard] = io_coordinator;
        }
    }

    return ret;
}

resources allocate(configuration c) {
    hwloc_topology_t topology;
    hwloc_topology_init(&topology);
    auto free_hwloc = defer([&] { hwloc_topology_destroy(topology); });
    hwloc_topology_load(topology);
    if (c.cpu_set) {
        auto bm = hwloc_bitmap_alloc();
        auto free_bm = defer([&] { hwloc_bitmap_free(bm); });
        for (auto idx : *c.cpu_set) {
            hwloc_bitmap_set(bm, idx);
        }
        auto r = hwloc_topology_restrict(topology, bm,
                HWLOC_RESTRICT_FLAG_ADAPT_DISTANCES
                | HWLOC_RESTRICT_FLAG_ADAPT_MISC
                | HWLOC_RESTRICT_FLAG_ADAPT_IO);
        if (r == -1) {
            if (errno == ENOMEM) {
                throw std::bad_alloc();
            }
            if (errno == EINVAL) {
                throw std::runtime_error("bad cpuset");
            }
            abort();
        }
    }
    auto machine_depth = hwloc_get_type_depth(topology, HWLOC_OBJ_MACHINE);
    assert(hwloc_get_nbobjs_by_depth(topology, machine_depth) == 1);
    auto machine = hwloc_get_obj_by_depth(topology, machine_depth, 0);
    auto available_memory = machine->memory.total_memory;
    // hwloc doesn't account for kernel reserved memory, so set panic_factor = 2
    size_t mem = calculate_memory(c, available_memory, 2);
    unsigned available_procs = hwloc_get_nbobjs_by_type(topology, HWLOC_OBJ_PU);
    unsigned procs = c.cpus.value_or(available_procs);
    if (procs > available_procs) {
        throw std::runtime_error("insufficient processing units");
    }
    auto mem_per_proc = align_down<size_t>(mem / procs, 2 << 20);

    resources ret;
    std::unordered_map<hwloc_obj_t, size_t> topo_used_mem;
    std::vector<std::pair<cpu, size_t>> remains;
    size_t remain;
    unsigned depth = find_memory_depth(topology);

    auto cpu_sets = distribute_objects(topology, procs);

    // Divide local memory to cpus
    for (auto&& cs : cpu_sets()) {
        auto cpu_id = hwloc_bitmap_first(cs);
        assert(cpu_id != -1);
        auto pu = hwloc_get_pu_obj_by_os_index(topology, cpu_id);
        auto node = hwloc_get_ancestor_obj_by_depth(topology, depth, pu);
        cpu this_cpu;
        this_cpu.cpu_id = cpu_id;
        remain = mem_per_proc - alloc_from_node(this_cpu, node, topo_used_mem, mem_per_proc);

        remains.emplace_back(std::move(this_cpu), remain);
    }

    // Divide the rest of the memory
    for (auto&& r : remains) {
        cpu this_cpu;
        size_t remain;
        std::tie(this_cpu, remain) = r;
        auto pu = hwloc_get_pu_obj_by_os_index(topology, this_cpu.cpu_id);
        auto node = hwloc_get_ancestor_obj_by_depth(topology, depth, pu);
        auto obj = node;

        while (remain) {
            remain -= alloc_from_node(this_cpu, obj, topo_used_mem, remain);
            do {
                obj = hwloc_get_next_obj_by_depth(topology, depth, obj);
            } while (!obj);
            if (obj == node)
                break;
        }
        assert(!remain);
        ret.cpus.push_back(std::move(this_cpu));
    }

    ret.io_queues = allocate_io_queues(topology, c, ret.cpus);
    return ret;
}

unsigned nr_processing_units() {
    hwloc_topology_t topology;
    hwloc_topology_init(&topology);
    auto free_hwloc = defer([&] { hwloc_topology_destroy(topology); });
    hwloc_topology_load(topology);
    return hwloc_get_nbobjs_by_type(topology, HWLOC_OBJ_PU);
}

}

#else

#include "resource.hh"
#include <unistd.h>

namespace resource {

// Without hwloc, we don't support tuning the number of IO queues. So each CPU gets their.
static io_queue_topology
allocate_io_queues(configuration c, std::vector<cpu> cpus) {
    io_queue_topology ret;

    unsigned nr_cpus = unsigned(cpus.size());
    unsigned max_io_requests = c.max_io_requests.value_or(128 * nr_cpus);

    ret.shard_to_coordinator.resize(nr_cpus);
    ret.coordinators.resize(nr_cpus);

    for (unsigned shard = 0; shard < nr_cpus; ++shard) {
        ret.shard_to_coordinator[shard] = shard;
        ret.coordinators[shard].capacity =  std::max(max_io_requests / nr_cpus, 1u);
        ret.coordinators[shard].id = shard;
    }
    return ret;
}

resources allocate(configuration c) {
    resources ret;

    auto available_memory = ::sysconf(_SC_PAGESIZE) * size_t(::sysconf(_SC_PHYS_PAGES));
    auto mem = calculate_memory(c, available_memory);
    auto cpuset_procs = c.cpu_set ? c.cpu_set->size() : nr_processing_units();
    auto procs = c.cpus.value_or(cpuset_procs);
    ret.cpus.reserve(procs);
    for (unsigned i = 0; i < procs; ++i) {
        ret.cpus.push_back(cpu{i, {{mem / procs, 0}}});
    }

    ret.io_queues = allocate_io_queues(c, ret.cpus);
    return ret;
}

unsigned nr_processing_units() {
    return ::sysconf(_SC_NPROCESSORS_ONLN);
}

}

#endif
<|endoftext|>


<commit_before>/*
* Copyright 2016 Nu-book Inc.
* Copyright 2016 ZXing authors
* Copyright 2022 Axel Waggershauser
*/
// SPDX-License-Identifier: Apache-2.0

#include "GS1.h"

#include "ZXContainerAlgorithms.h"

namespace ZXing {

struct AiInfo
{
	std::string_view aiPrefix;
	int _fieldSize;	// if negative, the length is variable and abs(length) give the max size

	bool isVariableLength() const noexcept { return _fieldSize < 0; }
	int fieldSize() const noexcept { return std::abs(_fieldSize); }
	int aiSize() const
	{
		if ((aiPrefix[0] == '3' && Contains("1234569", aiPrefix[1])) || aiPrefix == "703" || aiPrefix == "723")
			return 4;
		else
			return Size(aiPrefix);
	}
};

// GS1 General Specifications Release 22.0 (Jan 22, 2022)
static const AiInfo aiInfos[] = {
// TWO_DIGIT_DATA_LENGTH
	{ "00", 18 },
	{ "01", 14 },
	{ "02", 14 },

	{ "10", -20 },
	{ "11", 6 },
	{ "12", 6 },
	{ "13", 6 },
	{ "15", 6 },
	{ "16", 6 },
	{ "17", 6 },

	{ "20", 2 },
	{ "21", -20 },
	{ "22", -20 },

	{ "30", -8 },
	{ "37", -8 },

	//internal company codes
	{ "90", -30 },
	{ "91", -90 },
	{ "92", -90 },
	{ "93", -90 },
	{ "94", -90 },
	{ "95", -90 },
	{ "96", -90 },
	{ "97", -90 },
	{ "98", -90 },
	{ "99", -90 },

//THREE_DIGIT_DATA_LENGTH
	{ "235", -28 },
	{ "240", -30 },
	{ "241", -30 },
	{ "242", -6 },
	{ "243", -20 },
	{ "250", -30 },
	{ "251", -30 },
	{ "253", -30 },
	{ "254", -20 },
	{ "255", -25 },

	{ "400", -30 },
	{ "401", -30 },
	{ "402", 17 },
	{ "403", -30 },
	{ "410", 13 },
	{ "411", 13 },
	{ "412", 13 },
	{ "413", 13 },
	{ "414", 13 },
	{ "415", 13 },
	{ "416", 13 },
	{ "417", 13 },
	{ "420", -20 },
	{ "421", -12 },
	{ "422", 3 },
	{ "423", -15 },
	{ "424", 3 },
	{ "425", -15 },
	{ "426", 3 },
	{ "427", -3 },

	{ "710", -20 },
	{ "711", -20 },
	{ "712", -20 },
	{ "713", -20 },
	{ "714", -20 },
	{ "715", -20 },

//THREE_DIGIT_PLUS_DIGIT_DATA_LENGTH
	{ "310", 6 },
	{ "311", 6 },
	{ "312", 6 },
	{ "313", 6 },
	{ "314", 6 },
	{ "315", 6 },
	{ "316", 6 },
	{ "320", 6 },
	{ "321", 6 },
	{ "322", 6 },
	{ "323", 6 },
	{ "324", 6 },
	{ "325", 6 },
	{ "326", 6 },
	{ "327", 6 },
	{ "328", 6 },
	{ "329", 6 },
	{ "330", 6 },
	{ "331", 6 },
	{ "332", 6 },
	{ "333", 6 },
	{ "334", 6 },
	{ "335", 6 },
	{ "336", 6 },
	{ "337", 6 },
	{ "340", 6 },
	{ "341", 6 },
	{ "342", 6 },
	{ "343", 6 },
	{ "344", 6 },
	{ "345", 6 },
	{ "346", 6 },
	{ "347", 6 },
	{ "348", 6 },
	{ "349", 6 },
	{ "350", 6 },
	{ "351", 6 },
	{ "352", 6 },
	{ "353", 6 },
	{ "354", 6 },
	{ "355", 6 },
	{ "356", 6 },
	{ "357", 6 },
	{ "360", 6 },
	{ "361", 6 },
	{ "362", 6 },
	{ "363", 6 },
	{ "364", 6 },
	{ "365", 6 },
	{ "366", 6 },
	{ "367", 6 },
	{ "368", 6 },
	{ "369", 6 },
	{ "390", -15 },
	{ "391", -18 },
	{ "392", -15 },
	{ "393", -18 },
	{ "394", 4 },
	{ "395", 6 },
	{ "703", -30 },
	{ "723", -30 },

//FOUR_DIGIT_DATA_LENGTH
	{ "4300", -35 },
	{ "4301", -35 },
	{ "4302", -70 },
	{ "4303", -70 },
	{ "4304", -70 },
	{ "4305", -70 },
	{ "4306", -70 },
	{ "4307", 2 },
	{ "4308", -30 },
	{ "4310", -35 },
	{ "4311", -35 },
	{ "4312", -70 },
	{ "4313", -70 },
	{ "4314", -70 },
	{ "4315", -70 },
	{ "4316", -70 },
	{ "4317", 2 },
	{ "4318", -20 },
	{ "4319", -30 },
	{ "4320", -35 },
	{ "4321", 1 },
	{ "4322", 1 },
	{ "4323", 1 },
	{ "4324", 10 },
	{ "4325", 10 },
	{ "4326", 6 },

	{ "7001", 13 },
	{ "7002", -30 },
	{ "7003", 10 },
	{ "7004", -4 },
	{ "7005", -12 },
	{ "7006", 6 },
	{ "7007", -12 },
	{ "7008", -3 },
	{ "7009", -10 },
	{ "7010", -2 },
	{ "7020", -20 },
	{ "7021", -20 },
	{ "7022", -20 },
	{ "7023", -30 },
	{ "7040", 4 },
	{ "7240", -20 },

	{ "8001", 14 },
	{ "8002", -20 },
	{ "8003", -30 },
	{ "8004", -30 },
	{ "8005", 6 },
	{ "8006", 18 },
	{ "8007", -34 },
	{ "8008", -12 },
	{ "8009", -50 },
	{ "8010", -30 },
	{ "8011", -12 },
	{ "8012", -20 },
	{ "8013", -25 },
	{ "8017", 18 },
	{ "8018", 18 },
	{ "8019", -10 },
	{ "8020", -25 },
	{ "8026", 18 },
	{ "8110", -70 },
	{ "8111", 4 },
	{ "8112", -70 },
	{ "8200", -70 },
};

std::string HRIFromGS1(const std::string& gs1)
{
	//TODO: c++20
	auto starts_with = [](std::string_view str, std::string_view pre) { return str.substr(0, pre.size()) == pre; };
	constexpr char GS = 29; // GS character (29 / 0x1D)

	std::string_view rem = gs1;
	std::string res;

	while (rem.size()) {
		const AiInfo* i = FindIf(aiInfos, [&](const AiInfo& i) { return starts_with(rem, i.aiPrefix); });
		if (i == std::end(aiInfos))
			return {};

		int aiSize = i->aiSize();
		if (Size(rem) < aiSize)
			return {};

		res += '(';
		res += rem.substr(0, aiSize);
		res += ')';
		rem.remove_prefix(aiSize);

		int fieldSize = i->fieldSize();
		if (i->isVariableLength()) {
			auto gsPos = rem.find(GS);
#if 1
			fieldSize = std::min(gsPos == std::string_view::npos ? Size(rem) : static_cast<int>(gsPos), fieldSize);
#else
			// TODO: ignore the 'max field size' part for now as it breaks rssexpanded-3/13.png?
			fieldSize = gsPos == std::string_view::npos ? Size(rem) : static_cast<int>(gsPos);
#endif
		}
		if (fieldSize == 0 || Size(rem) < fieldSize)
			return {};

		res += rem.substr(0, fieldSize);
		rem.remove_prefix(fieldSize);

		if (Size(rem) && rem.front() == GS) {
			// TODO: we have DataBar samples where the fixed-length 422 ends with a GS, sigh...
			if (i->isVariableLength() || i->aiPrefix == "422")
				rem.remove_prefix(1);
			else
				return {};
		}
	}

	return res;
}

} // namespace ZXing
<commit_msg>GS1: accept GS after every field according to spec<commit_after>/*
* Copyright 2016 Nu-book Inc.
* Copyright 2016 ZXing authors
* Copyright 2022 Axel Waggershauser
*/
// SPDX-License-Identifier: Apache-2.0

#include "GS1.h"

#include "ZXContainerAlgorithms.h"

namespace ZXing {

struct AiInfo
{
	std::string_view aiPrefix;
	int _fieldSize;	// if negative, the length is variable and abs(length) give the max size

	bool isVariableLength() const noexcept { return _fieldSize < 0; }
	int fieldSize() const noexcept { return std::abs(_fieldSize); }
	int aiSize() const
	{
		if ((aiPrefix[0] == '3' && Contains("1234569", aiPrefix[1])) || aiPrefix == "703" || aiPrefix == "723")
			return 4;
		else
			return Size(aiPrefix);
	}
};

// GS1 General Specifications Release 22.0 (Jan 22, 2022)
static const AiInfo aiInfos[] = {
// TWO_DIGIT_DATA_LENGTH
	{ "00", 18 },
	{ "01", 14 },
	{ "02", 14 },

	{ "10", -20 },
	{ "11", 6 },
	{ "12", 6 },
	{ "13", 6 },
	{ "15", 6 },
	{ "16", 6 },
	{ "17", 6 },

	{ "20", 2 },
	{ "21", -20 },
	{ "22", -20 },

	{ "30", -8 },
	{ "37", -8 },

	//internal company codes
	{ "90", -30 },
	{ "91", -90 },
	{ "92", -90 },
	{ "93", -90 },
	{ "94", -90 },
	{ "95", -90 },
	{ "96", -90 },
	{ "97", -90 },
	{ "98", -90 },
	{ "99", -90 },

//THREE_DIGIT_DATA_LENGTH
	{ "235", -28 },
	{ "240", -30 },
	{ "241", -30 },
	{ "242", -6 },
	{ "243", -20 },
	{ "250", -30 },
	{ "251", -30 },
	{ "253", -30 },
	{ "254", -20 },
	{ "255", -25 },

	{ "400", -30 },
	{ "401", -30 },
	{ "402", 17 },
	{ "403", -30 },
	{ "410", 13 },
	{ "411", 13 },
	{ "412", 13 },
	{ "413", 13 },
	{ "414", 13 },
	{ "415", 13 },
	{ "416", 13 },
	{ "417", 13 },
	{ "420", -20 },
	{ "421", -12 },
	{ "422", 3 },
	{ "423", -15 },
	{ "424", 3 },
	{ "425", -15 },
	{ "426", 3 },
	{ "427", -3 },

	{ "710", -20 },
	{ "711", -20 },
	{ "712", -20 },
	{ "713", -20 },
	{ "714", -20 },
	{ "715", -20 },

//THREE_DIGIT_PLUS_DIGIT_DATA_LENGTH
	{ "310", 6 },
	{ "311", 6 },
	{ "312", 6 },
	{ "313", 6 },
	{ "314", 6 },
	{ "315", 6 },
	{ "316", 6 },
	{ "320", 6 },
	{ "321", 6 },
	{ "322", 6 },
	{ "323", 6 },
	{ "324", 6 },
	{ "325", 6 },
	{ "326", 6 },
	{ "327", 6 },
	{ "328", 6 },
	{ "329", 6 },
	{ "330", 6 },
	{ "331", 6 },
	{ "332", 6 },
	{ "333", 6 },
	{ "334", 6 },
	{ "335", 6 },
	{ "336", 6 },
	{ "337", 6 },
	{ "340", 6 },
	{ "341", 6 },
	{ "342", 6 },
	{ "343", 6 },
	{ "344", 6 },
	{ "345", 6 },
	{ "346", 6 },
	{ "347", 6 },
	{ "348", 6 },
	{ "349", 6 },
	{ "350", 6 },
	{ "351", 6 },
	{ "352", 6 },
	{ "353", 6 },
	{ "354", 6 },
	{ "355", 6 },
	{ "356", 6 },
	{ "357", 6 },
	{ "360", 6 },
	{ "361", 6 },
	{ "362", 6 },
	{ "363", 6 },
	{ "364", 6 },
	{ "365", 6 },
	{ "366", 6 },
	{ "367", 6 },
	{ "368", 6 },
	{ "369", 6 },
	{ "390", -15 },
	{ "391", -18 },
	{ "392", -15 },
	{ "393", -18 },
	{ "394", 4 },
	{ "395", 6 },
	{ "703", -30 },
	{ "723", -30 },

//FOUR_DIGIT_DATA_LENGTH
	{ "4300", -35 },
	{ "4301", -35 },
	{ "4302", -70 },
	{ "4303", -70 },
	{ "4304", -70 },
	{ "4305", -70 },
	{ "4306", -70 },
	{ "4307", 2 },
	{ "4308", -30 },
	{ "4310", -35 },
	{ "4311", -35 },
	{ "4312", -70 },
	{ "4313", -70 },
	{ "4314", -70 },
	{ "4315", -70 },
	{ "4316", -70 },
	{ "4317", 2 },
	{ "4318", -20 },
	{ "4319", -30 },
	{ "4320", -35 },
	{ "4321", 1 },
	{ "4322", 1 },
	{ "4323", 1 },
	{ "4324", 10 },
	{ "4325", 10 },
	{ "4326", 6 },

	{ "7001", 13 },
	{ "7002", -30 },
	{ "7003", 10 },
	{ "7004", -4 },
	{ "7005", -12 },
	{ "7006", 6 },
	{ "7007", -12 },
	{ "7008", -3 },
	{ "7009", -10 },
	{ "7010", -2 },
	{ "7020", -20 },
	{ "7021", -20 },
	{ "7022", -20 },
	{ "7023", -30 },
	{ "7040", 4 },
	{ "7240", -20 },

	{ "8001", 14 },
	{ "8002", -20 },
	{ "8003", -30 },
	{ "8004", -30 },
	{ "8005", 6 },
	{ "8006", 18 },
	{ "8007", -34 },
	{ "8008", -12 },
	{ "8009", -50 },
	{ "8010", -30 },
	{ "8011", -12 },
	{ "8012", -20 },
	{ "8013", -25 },
	{ "8017", 18 },
	{ "8018", 18 },
	{ "8019", -10 },
	{ "8020", -25 },
	{ "8026", 18 },
	{ "8110", -70 },
	{ "8111", 4 },
	{ "8112", -70 },
	{ "8200", -70 },
};

std::string HRIFromGS1(const std::string& gs1)
{
	//TODO: c++20
	auto starts_with = [](std::string_view str, std::string_view pre) { return str.substr(0, pre.size()) == pre; };
	constexpr char GS = 29; // GS character (29 / 0x1D)

	std::string_view rem = gs1;
	std::string res;

	while (rem.size()) {
		const AiInfo* i = FindIf(aiInfos, [&](const AiInfo& i) { return starts_with(rem, i.aiPrefix); });
		if (i == std::end(aiInfos))
			return {};

		int aiSize = i->aiSize();
		if (Size(rem) < aiSize)
			return {};

		res += '(';
		res += rem.substr(0, aiSize);
		res += ')';
		rem.remove_prefix(aiSize);

		int fieldSize = i->fieldSize();
		if (i->isVariableLength()) {
			auto gsPos = rem.find(GS);
#if 1
			fieldSize = std::min(gsPos == std::string_view::npos ? Size(rem) : static_cast<int>(gsPos), fieldSize);
#else
			// TODO: ignore the 'max field size' part for now as it breaks rssexpanded-3/13.png?
			fieldSize = gsPos == std::string_view::npos ? Size(rem) : static_cast<int>(gsPos);
#endif
		}
		if (fieldSize == 0 || Size(rem) < fieldSize)
			return {};

		res += rem.substr(0, fieldSize);
		rem.remove_prefix(fieldSize);

		// See General Specification v22.0 Section 7.8.6.3: "...the processing routine SHALL tolerate a single separator character
		// immediately following any element string, whether necessary or not..."
		if (Size(rem) && rem.front() == GS)
			rem.remove_prefix(1);
	}

	return res;
}

} // namespace ZXing
<|endoftext|>


<commit_before>// Copyright (c) 2009 The Chromium Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.

// Calculate Crc by calling CRC method in LZMA SDK

#include "courgette/crc.h"

extern "C" {
#include "third_party/lzma_sdk/7zCrc.h"
}

namespace courgette {

uint32 CalculateCrc(const uint8* buffer, size_t size) {
  CrcGenerateTable();
  uint32 crc = 0xffffffffL;
  crc = ~CrcCalc(buffer, size);
  return crc;
}

}  // namespace
<commit_msg>Try a different library for Crc32.<commit_after>// Copyright (c) 2011 The Chromium Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.

#include "courgette/crc.h"

#ifdef OS_CHROMEOS
#  include "zlib.h"
#else
extern "C" {
#  include "third_party/lzma_sdk/7zCrc.h"
}
#endif

#include "base/basictypes.h"

namespace courgette {

uint32 CalculateCrc(const uint8* buffer, size_t size) {
  uint32 crc;

#ifdef OS_CHROMEOS
  // Calculate Crc by calling CRC method in zlib
  crc = crc32(0, buffer, size);
#else
  // Calculate Crc by calling CRC method in LZMA SDK
  CrcGenerateTable();
  crc = CrcCalc(buffer, size);
#endif

  return ~crc;
}

}  // namespace
<|endoftext|>


<commit_before>#include "cpr/redirect.h"
#include <type_traits>

namespace cpr {
PostRedirectFlags operator|(PostRedirectFlags lhs, PostRedirectFlags rhs) {
    return static_cast<PostRedirectFlags>(static_cast<std::underlying_type<PostRedirectFlags>::type>(lhs) | static_cast<std::underlying_type<PostRedirectFlags>::type>(rhs));
}

PostRedirectFlags operator&(PostRedirectFlags lhs, PostRedirectFlags rhs) {
    return static_cast<PostRedirectFlags>(static_cast<std::underlying_type<PostRedirectFlags>::type>(lhs) & static_cast<std::underlying_type<PostRedirectFlags>::type>(rhs));
}

PostRedirectFlags operator^(PostRedirectFlags lhs, PostRedirectFlags rhs) {
    return static_cast<PostRedirectFlags>(static_cast<std::underlying_type<PostRedirectFlags>::type>(lhs) ^ static_cast<std::underlying_type<PostRedirectFlags>::type>(rhs));
}

PostRedirectFlags operator~(PostRedirectFlags flag) {
    return static_cast<PostRedirectFlags>(~static_cast<std::underlying_type<PostRedirectFlags>::type>(flag));
}

PostRedirectFlags& operator|=(PostRedirectFlags& lhs, PostRedirectFlags rhs) {
    lhs = static_cast<PostRedirectFlags>(static_cast<std::underlying_type<PostRedirectFlags>::type>(lhs) | static_cast<std::underlying_type<PostRedirectFlags>::type>(rhs));
    uint8_t tmp = static_cast<uint8_t>(lhs);
    lhs = static_cast<PostRedirectFlags>(tmp);
    return lhs;
}

PostRedirectFlags& operator&=(PostRedirectFlags& lhs, PostRedirectFlags rhs) {
    lhs = static_cast<PostRedirectFlags>(static_cast<std::underlying_type<PostRedirectFlags>::type>(lhs) & static_cast<std::underlying_type<PostRedirectFlags>::type>(rhs));
    return lhs;
}

PostRedirectFlags& operator^=(PostRedirectFlags& lhs, PostRedirectFlags rhs) {
    lhs = static_cast<PostRedirectFlags>(static_cast<std::underlying_type<PostRedirectFlags>::type>(lhs) ^ static_cast<std::underlying_type<PostRedirectFlags>::type>(rhs));
    return lhs;
}

bool any(PostRedirectFlags flag) {
    return flag != PostRedirectFlags::NONE;
}
} // namespace cpr
<commit_msg>Removed cpp14 underlaying type<commit_after>#include "cpr/redirect.h"

namespace cpr {
PostRedirectFlags operator|(PostRedirectFlags lhs, PostRedirectFlags rhs) {
    return static_cast<PostRedirectFlags>(static_cast<uint8_t>(lhs) | static_cast<uint8_t>(rhs));
}

PostRedirectFlags operator&(PostRedirectFlags lhs, PostRedirectFlags rhs) {
    return static_cast<PostRedirectFlags>(static_cast<uint8_t>(lhs) & static_cast<uint8_t>(rhs));
}

PostRedirectFlags operator^(PostRedirectFlags lhs, PostRedirectFlags rhs) {
    return static_cast<PostRedirectFlags>(static_cast<uint8_t>(lhs) ^ static_cast<uint8_t>(rhs));
}

PostRedirectFlags operator~(PostRedirectFlags flag) {
    return static_cast<PostRedirectFlags>(~static_cast<uint8_t>(flag));
}

PostRedirectFlags& operator|=(PostRedirectFlags& lhs, PostRedirectFlags rhs) {
    lhs = static_cast<PostRedirectFlags>(static_cast<uint8_t>(lhs) | static_cast<uint8_t>(rhs));
    uint8_t tmp = static_cast<uint8_t>(lhs);
    lhs = static_cast<PostRedirectFlags>(tmp);
    return lhs;
}

PostRedirectFlags& operator&=(PostRedirectFlags& lhs, PostRedirectFlags rhs) {
    lhs = static_cast<PostRedirectFlags>(static_cast<uint8_t>(lhs) & static_cast<uint8_t>(rhs));
    return lhs;
}

PostRedirectFlags& operator^=(PostRedirectFlags& lhs, PostRedirectFlags rhs) {
    lhs = static_cast<PostRedirectFlags>(static_cast<uint8_t>(lhs) ^ static_cast<uint8_t>(rhs));
    return lhs;
}

bool any(PostRedirectFlags flag) {
    return flag != PostRedirectFlags::NONE;
}
} // namespace cpr
<|endoftext|>


<commit_before>#include "data-student.hpp"
using namespace std;

Semester::Semester() {
	period = 0;
}
Semester::Semester(int p) {
	period = p;
}

void Student::init(string n, int s, int g, string m) {
	name = n;
	startingYear = s;
	gradutationYear = g;
	// parseMajors(m);
}

Student::Student() {
	init("", 2000, 2004, "");
}
Student::Student(string fn) {
	ifstream infile;
	infile.open(fn.c_str());
}

bool Student::hasTakenCourse() {
	return false;
}

void Student::addCourse(const Course& c, const Semester& s) {
	// s.push_back();
}

ostream& Student::getData(ostream &os) {
	os << name << ", ";
	os << "majoring in ";
	for (vector<Major>::iterator i = majors.begin(); i != majors.end(); ++i)
		os << *i << ", ";
	os << " and taking:" << endl;
	for (vector<Course>::iterator i = courses.begin(); i != courses.end(); ++i){
		os << *i << endl;
	}
	return os;
}
void Student::display() {
	if (this == 0)
		cout << *this << endl;
};
ostream& operator<<(ostream& os, Student& item) {
	os << item.getData(os);
	return os;
}
<commit_msg>Enabl parseMajors<commit_after>#include "data-student.hpp"
using namespace std;

Semester::Semester() {
	period = 0;
}
Semester::Semester(int p) {
	period = p;
}

void Student::init(string n, int s, int g, string m) {
	name = n;
	startingYear = s;
	gradutationYear = g;
	parseMajors(m);
}

Student::Student() {
	init("", 2000, 2004, "");
}
Student::Student(string fn) {
	ifstream infile;
	infile.open(fn.c_str());
}

void Student::parseMajors(string str) {
	vector<string> record = split(str, ',');
	for (vector<string>::iterator i = record.begin(); i != record.end(); ++i) {
		string str = removeStartingText(*i, " ");
		Major m = Major(str);
		majors.push_back(m);
	}
}

bool Student::hasTakenCourse() {
	return false;
}

void Student::addCourse(const Course& c, const Semester& s) {
	// s.push_back();
}

ostream& Student::getData(ostream &os) {
	os << name << ", ";
	os << "majoring in ";
	for (vector<Major>::iterator i = majors.begin(); i != majors.end(); ++i)
		os << *i << ", ";
	os << " and taking:" << endl;
	for (vector<Course>::iterator i = courses.begin(); i != courses.end(); ++i){
		os << *i << endl;
	}
	return os;
}
void Student::display() {
	if (this == 0)
		cout << *this << endl;
};
ostream& operator<<(ostream& os, Student& item) {
	os << item.getData(os);
	return os;
}
<|endoftext|>


<commit_before>#include <iostream>
#include "word.h"

using namespace std;

string make_outfile(string namefile);

int main(int argc, char* argv[])
{
        Word word;

        if (argc < 2) {
                cerr << "Provide file to translage" << endl;
                exit(EXIT_FAILURE);
        }
        else if (argc == 2) {
                string outfile = make_outfile(argv[1]);
                word.set_out(outfile);
        }
        else if (argc > 2)
                word.set_out(argv[2]);

        word.set_in(argv[1]);
        word.write();

        word.close_files();
}

string make_outfile(string namefile)
{
        string name = "";
        for (int i = 0; i < (int)namefile.length(); i++) {
                if (namefile[i] == '.') {
                        name = namefile.substr(0,i);
                        name += ".cpp";
                        break;
                }
        }
        if (name == "") {
                cerr << "No .si file found\n";
                exit(EXIT_FAILURE);
        }
        return name;
}
<commit_msg>Documentation updates to si.cpp<commit_after>#include <iostream>
#include "word.h"

using namespace std;

string make_outfile(string namefile);

int main(int argc, char* argv[])
{
        Word word;

        if (argc < 2) {
                cerr << "Usage: si [origin].si [destination].cpp" << endl;
                exit(EXIT_FAILURE);
        }
        else if (argc == 2) {
                string outfile = make_outfile(argv[1]);
                word.set_out(outfile);
        }
        else if (argc > 2)
                word.set_out(argv[2]);

        word.set_in(argv[1]);
        word.write();

        word.close_files();
}

/* if no destination file is provided, this will take the
 * first argument and turn its contents in to the name
 * of the destination file in .cpp format
 */
string make_outfile(string namefile)
{
        string name = "";
        for (int i = 0; i < (int)namefile.length(); i++) {
                if (namefile[i] == '.') {
                        name = namefile.substr(0,i);
                        name += ".cpp";
                        break;
                }
        }
        if (name == "") {
                cerr << "No .si file found\n";
                exit(EXIT_FAILURE);
        }
        return name;
}
<|endoftext|>


<commit_before>#include <fstream>
#include <string>
#include <iostream>
#include <string.h>

#include "io.hh"
#include "str.hh"
#include "conf.hh"
#include "FeatureGenerator.hh"
#include "Recipe.hh"
#include "PhonePool.hh"
#include "PhnReader.hh"
#include "SpeakerConfig.hh"
#include "HmmNetBaumWelch.hh"

using namespace aku;

int max_contexts;
int info;

conf::Config config;
Recipe recipe;
HmmSet model;
FeatureGenerator fea_gen;
FeatureGenerator model_fea_gen;
FeatureGenerator *mfea_gen; // Set to the correct FeatureGenerator object
SpeakerConfig speaker_config(fea_gen);

void
collect_phone_stats(PhnReader *phn_reader, PhonePool *pool)
{
  int f;
  PhnReader::Phn phn;

  while (phn_reader->next_phn_line(phn))
  {
    if (phn.state == -1)
      throw std::string("Context phone tying requires phn files with state numbers!");
    PhonePool::ContextPhoneContainer phone = pool->get_context_phone(
      phn.label[0], phn.state);
    for (f = phn.start; f < phn.end; f++)
    {
      FeatureVec feature = fea_gen.generate(f);
      if (fea_gen.eof())
        break; // EOF in FeatureGenerator
      phone.add_feature(1, feature);
    }
    if (f < phn.end) // EOF in FeatureGenerator
      return;
  }
}

void
hmmnet_collect_phone_stats(HmmNetBaumWelch *seg, PhonePool *pool)
{
  int i;

  while (seg->next_frame())
  {
    std::vector<HmmNetBaumWelch::ArcInfo> arc_info;
    seg->fill_arc_info(arc_info);
    FeatureVec feature = fea_gen.generate(seg->current_frame());
    if (fea_gen.eof())
      break; // EOF in FeatureGenerator

    for (i = 0; i < (int)arc_info.size(); i++)
    {
      int state = -1;
      if (strchr(arc_info[i].label.c_str(), '.') != NULL){
	state = atoi((const char*)(strchr(arc_info[i].label.c_str(),'.')+1));
	arc_info[i].label.erase(arc_info[i].label.find('.', 0), 2);
      }
      if (state < 0)
      {
        fprintf(stderr, "Warning: Invalid label %s, ignoring current file\n",
                arc_info[i].label.c_str());
        return;
      }
      PhonePool::ContextPhoneContainer phone = pool->get_context_phone(
        arc_info[i].label, state);
      phone.add_feature(arc_info[i].prob, feature);
    }
  }
}

void
save_basebind(const std::string &filename, PhonePool *pool)
{
  FILE *fp;

  // Save silence models
  if ((fp = fopen(filename.c_str(), "w")) == NULL)
  {
    fprintf(stderr, "Could not open file %s for writing.\n", filename.c_str());
    exit(1);
  }
  //fprintf(fp, "_ 1 0\n__ 3 1 2 3\n");

  pool->save_to_basebind(fp, 0, max_contexts);
  fclose(fp);
}

int
main(int argc, char *argv[])
{
  PhonePool phone_pool;

  try {
    config("usage: tie [OPTION...]\n")
      ('h', "help", "", "", "display help")
      ('c', "config=FILE", "arg must", "", "feature configuration")
      ('r', "recipe=FILE", "arg must", "", "recipe file")
      ('O', "ophn", "", "", "use output phns for training")
      ('H', "hmmnet", "", "", "use HMM networks for training")
      ('b', "base=BASENAME", "arg", "", "model files (required with --hmmnet)")
      ('C', "mconfig=FILE", "arg", "", "model configuration (optional)")
      ('u', "rule=FILE", "arg must", "", "rule set for triphone state tying")
      ('o', "out=FILE", "arg", "", "write output to HMM model with base name FILE")
      ('B', "basebind=FILE", "arg", "", "write output to basebind FILE")
      ('\0', "count=INT", "arg", "100", "minimum feature count for state clusters")
      ('\0', "sgain=FLOAT", "arg", "0", "minimum loglikelihood gain in cluster splitting")
      ('\0', "mloss=FLOAT", "arg", "0", "cluster merging with maximum loglikelihood loss")
      ('\0', "context=INT", "arg", "1", "maximum number of contexts (default 1=triphones)")
      ('F', "fw-beam=FLOAT", "arg", "0", "Forward beam (for HMM networks)")
      ('W', "bw-beam=FLOAT", "arg", "0", "Backward beam (for HMM networks)")
      ('A', "ac-scale=FLOAT", "arg", "1", "Acoustic scaling (for HMM networks)")
      ('V', "vit", "", "", "Use Viterbi over HMM networks")
      ('S', "speakers=FILE", "arg", "", "speaker configuration file")
      ('i', "info=INT", "arg", "0", "info level")
      ;
    config.default_parse(argc, argv);

    info = config["info"].get_int();
    fea_gen.load_configuration(io::Stream(config["config"].get_str()));

    // Read recipe file
    recipe.read(io::Stream(config["recipe"].get_str()), 0, 0, false);

    if (!(config["out"].specified^config["basebind"].specified))
      throw std::string("Specify either --out or --basebind for output");

    phone_pool.set_clustering_parameters(config["count"].get_int(),
                                         config["sgain"].get_float(),
                                         config["mloss"].get_float());

    phone_pool.load_decision_tree_rules(io::Stream(config["rule"].get_str()));

    max_contexts = config["context"].get_int();

    if (config["speakers"].specified)
    {
      speaker_config.read_speaker_file(
        io::Stream(config["speakers"].get_str()));
    }

    // Initialize triphone tying
    phone_pool.set_dimension(fea_gen.dim());
    phone_pool.set_info(info);

    if (config["hmmnet"].specified)
    {
      if (config["base"].specified)
      {
        model.read_all(config["base"].get_str());
        if (config["mconfig"].specified)
        {
          model_fea_gen.load_configuration(
            io::Stream(config["mconfig"].get_str()));
          if (model_fea_gen.frame_rate() != fea_gen.frame_rate())
            throw str::fmt(256, "Frame rate of the model and the feature generator must match!\nModel frame rate is %.3f, feature generator frame rate is %3.f\n", model_fea_gen.frame_rate(), fea_gen.frame_rate());
          mfea_gen = &model_fea_gen;
        }
        else
          mfea_gen = &fea_gen; // Use the feature configuration
      }
      else
      {
        throw std::string("HMM model is required with HMM networks");
      }
      if (model.dim() != mfea_gen->dim()) {
        throw str::fmt(128,
                       "gaussian dimension is %d but feature dimension is %d",
                       model.dim(), mfea_gen->dim());
      }
    }

    for (int f = 0; f < (int)recipe.infos.size(); f++)
    {
      if (info > 0)
      {
        fprintf(stderr, "Processing file: %s",
                recipe.infos[f].audio_path.c_str());
        if (recipe.infos[f].start_time || recipe.infos[f].end_time)
          fprintf(stderr," (%.2f-%.2f)",recipe.infos[f].start_time,
                  recipe.infos[f].end_time);
        fprintf(stderr,"\n");
      }

      if (config["speakers"].specified)
      {
        speaker_config.set_speaker(recipe.infos[f].speaker_id);
        if (recipe.infos[f].utterance_id.size() > 0)
          speaker_config.set_utterance(recipe.infos[f].utterance_id);
      }

      if (config["hmmnet"].specified)
      {
        // Open files and configure
        HmmNetBaumWelch* lattice = recipe.infos[f].init_hmmnet_files(
          &model, false, mfea_gen, NULL);
        lattice->set_pruning_thresholds(config["bw-beam"].get_float(),
                                        config["fw-beam"].get_float());
        if (config["ac-scale"].specified)
          lattice->set_acoustic_scaling(config["ac-scale"].get_float());
        if (config["vit"].specified)
          lattice->set_mode(HmmNetBaumWelch::MODE_VITERBI);

        if (mfea_gen != &fea_gen)
        {
          // Open the file for the actual feature generator
          fea_gen.open(recipe.infos[f].audio_path);
        }

        double orig_beam = lattice->get_backward_beam();
        int counter = 1;
        bool skip = false;
        while (!lattice->init_utterance_segmentation())
        {
          if (counter >= 5)
          {
            fprintf(stderr, "Could not run Baum-Welch for file %s\n",
                    recipe.infos[f].audio_path.c_str());
            fprintf(stderr, "The HMM network may be incorrect or initial beam too low.\n");
            skip = true;
            break;
          }
          fprintf(stderr,
                  "Warning: Backward phase failed, increasing beam to %.1f\n",
                  ++counter*orig_beam);
          lattice->set_pruning_thresholds(counter*orig_beam, 0);
        }
        if (!skip)
          hmmnet_collect_phone_stats(lattice, &phone_pool);
        if (mfea_gen != &fea_gen) // fea_gen is closed elsewhere
          mfea_gen->close();
        delete lattice;
      }
      else
      {
        PhnReader phn_reader(NULL);
        recipe.infos[f].init_phn_files(NULL, false, false,
                                       config["ophn"].specified, &fea_gen,
                                       &phn_reader);
        phn_reader.set_collect_transition_probs(false);
        collect_phone_stats(&phn_reader, &phone_pool);
        phn_reader.close();
      }

      fea_gen.close();
    }

    phone_pool.finish_statistics();
    phone_pool.decision_tree_cluster_context_phones(max_contexts);
    if (config["mloss"].specified)
      phone_pool.merge_context_phones();

    if (config["out"].specified)
      phone_pool.save_model(config["out"].get_str(), max_contexts);
    else
      save_basebind(config["basebind"].get_str(), &phone_pool);
  }
  catch (std::exception &e) {
    fprintf(stderr, "exception: %s\n", e.what());
    abort();
  }
  catch (std::string &str) {
    fprintf(stderr, "exception: %s\n", str.c_str());
    abort();
  }
}
<commit_msg>Using HMM networks is broken due to HmmNetBaumWelch update<commit_after>#include <fstream>
#include <string>
#include <iostream>
#include <string.h>

#include "io.hh"
#include "str.hh"
#include "conf.hh"
#include "FeatureGenerator.hh"
#include "Recipe.hh"
#include "PhonePool.hh"
#include "PhnReader.hh"
#include "SpeakerConfig.hh"
#include "HmmNetBaumWelch.hh"

using namespace aku;

int max_contexts;
int info;

conf::Config config;
Recipe recipe;
HmmSet model;
FeatureGenerator fea_gen;
FeatureGenerator model_fea_gen;
FeatureGenerator *mfea_gen; // Set to the correct FeatureGenerator object
SpeakerConfig speaker_config(fea_gen);

void
collect_phone_stats(PhnReader *phn_reader, PhonePool *pool)
{
  int f;
  PhnReader::Phn phn;

  while (phn_reader->next_phn_line(phn))
  {
    if (phn.state == -1)
      throw std::string("Context phone tying requires phn files with state numbers!");
    PhonePool::ContextPhoneContainer phone = pool->get_context_phone(
      phn.label[0], phn.state);
    for (f = phn.start; f < phn.end; f++)
    {
      FeatureVec feature = fea_gen.generate(f);
      if (fea_gen.eof())
        break; // EOF in FeatureGenerator
      phone.add_feature(1, feature);
    }
    if (f < phn.end) // EOF in FeatureGenerator
      return;
  }
}

// FIXME: Broken after HmmNetBaumWelch update
// void
// hmmnet_collect_phone_stats(HmmNetBaumWelch *seg, PhonePool *pool)
// {
//   int i;

//   while (seg->next_frame())
//   {
//     std::vector<HmmNetBaumWelch::ArcInfo> arc_info;
//     seg->fill_arc_info(arc_info);
//     FeatureVec feature = fea_gen.generate(seg->current_frame());
//     if (fea_gen.eof())
//       break; // EOF in FeatureGenerator

//     for (i = 0; i < (int)arc_info.size(); i++)
//     {
//       int state = -1;
//       if (strchr(arc_info[i].label.c_str(), '.') != NULL){
// 	state = atoi((const char*)(strchr(arc_info[i].label.c_str(),'.')+1));
// 	arc_info[i].label.erase(arc_info[i].label.find('.', 0), 2);
//       }
//       if (state < 0)
//       {
//         fprintf(stderr, "Warning: Invalid label %s, ignoring current file\n",
//                 arc_info[i].label.c_str());
//         return;
//       }
//       PhonePool::ContextPhoneContainer phone = pool->get_context_phone(
//         arc_info[i].label, state);
//       phone.add_feature(arc_info[i].prob, feature);
//     }
//   }
// }

void
save_basebind(const std::string &filename, PhonePool *pool)
{
  FILE *fp;

  // Save silence models
  if ((fp = fopen(filename.c_str(), "w")) == NULL)
  {
    fprintf(stderr, "Could not open file %s for writing.\n", filename.c_str());
    exit(1);
  }
  //fprintf(fp, "_ 1 0\n__ 3 1 2 3\n");

  pool->save_to_basebind(fp, 0, max_contexts);
  fclose(fp);
}

int
main(int argc, char *argv[])
{
  PhonePool phone_pool;

  try {
    config("usage: tie [OPTION...]\n")
      ('h', "help", "", "", "display help")
      ('c', "config=FILE", "arg must", "", "feature configuration")
      ('r', "recipe=FILE", "arg must", "", "recipe file")
      ('O', "ophn", "", "", "use output phns for training")
      ('H', "hmmnet", "", "", "use HMM networks for training")
      ('b', "base=BASENAME", "arg", "", "model files (required with --hmmnet)")
      ('C', "mconfig=FILE", "arg", "", "model configuration (optional)")
      ('u', "rule=FILE", "arg must", "", "rule set for triphone state tying")
      ('o', "out=FILE", "arg", "", "write output to HMM model with base name FILE")
      ('B', "basebind=FILE", "arg", "", "write output to basebind FILE")
      ('\0', "count=INT", "arg", "100", "minimum feature count for state clusters")
      ('\0', "sgain=FLOAT", "arg", "0", "minimum loglikelihood gain in cluster splitting")
      ('\0', "mloss=FLOAT", "arg", "0", "cluster merging with maximum loglikelihood loss")
      ('\0', "context=INT", "arg", "1", "maximum number of contexts (default 1=triphones)")
      ('F', "fw-beam=FLOAT", "arg", "0", "Forward beam (for HMM networks)")
      ('W', "bw-beam=FLOAT", "arg", "0", "Backward beam (for HMM networks)")
      ('A', "ac-scale=FLOAT", "arg", "1", "Acoustic scaling (for HMM networks)")
      ('V', "vit", "", "", "Use Viterbi over HMM networks")
      ('S', "speakers=FILE", "arg", "", "speaker configuration file")
      ('i', "info=INT", "arg", "0", "info level")
      ;
    config.default_parse(argc, argv);

    info = config["info"].get_int();
    fea_gen.load_configuration(io::Stream(config["config"].get_str()));

    // Read recipe file
    recipe.read(io::Stream(config["recipe"].get_str()), 0, 0, false);

    if (!(config["out"].specified^config["basebind"].specified))
      throw std::string("Specify either --out or --basebind for output");

    phone_pool.set_clustering_parameters(config["count"].get_int(),
                                         config["sgain"].get_float(),
                                         config["mloss"].get_float());

    phone_pool.load_decision_tree_rules(io::Stream(config["rule"].get_str()));

    max_contexts = config["context"].get_int();

    if (config["speakers"].specified)
    {
      speaker_config.read_speaker_file(
        io::Stream(config["speakers"].get_str()));
    }

    // Initialize triphone tying
    phone_pool.set_dimension(fea_gen.dim());
    phone_pool.set_info(info);

    if (config["hmmnet"].specified)
    {
      throw std::string("This feature is currently broken. Fix it?");

      if (config["base"].specified)
      {
        model.read_all(config["base"].get_str());
        if (config["mconfig"].specified)
        {
          model_fea_gen.load_configuration(
            io::Stream(config["mconfig"].get_str()));
          if (model_fea_gen.frame_rate() != fea_gen.frame_rate())
            throw str::fmt(256, "Frame rate of the model and the feature generator must match!\nModel frame rate is %.3f, feature generator frame rate is %3.f\n", model_fea_gen.frame_rate(), fea_gen.frame_rate());
          mfea_gen = &model_fea_gen;
        }
        else
          mfea_gen = &fea_gen; // Use the feature configuration
      }
      else
      {
        throw std::string("HMM model is required with HMM networks");
      }
      if (model.dim() != mfea_gen->dim()) {
        throw str::fmt(128,
                       "gaussian dimension is %d but feature dimension is %d",
                       model.dim(), mfea_gen->dim());
      }
    }

    for (int f = 0; f < (int)recipe.infos.size(); f++)
    {
      if (info > 0)
      {
        fprintf(stderr, "Processing file: %s",
                recipe.infos[f].audio_path.c_str());
        if (recipe.infos[f].start_time || recipe.infos[f].end_time)
          fprintf(stderr," (%.2f-%.2f)",recipe.infos[f].start_time,
                  recipe.infos[f].end_time);
        fprintf(stderr,"\n");
      }

      if (config["speakers"].specified)
      {
        speaker_config.set_speaker(recipe.infos[f].speaker_id);
        if (recipe.infos[f].utterance_id.size() > 0)
          speaker_config.set_utterance(recipe.infos[f].utterance_id);
      }

      if (config["hmmnet"].specified)
      {
        // Open files and configure
        HmmNetBaumWelch* lattice = recipe.infos[f].init_hmmnet_files(
          &model, false, mfea_gen, NULL);
        lattice->set_pruning_thresholds(config["fw-beam"].get_float(),
                                        config["bw-beam"].get_float());
        if (config["ac-scale"].specified)
          lattice->set_acoustic_scaling(config["ac-scale"].get_float());
        if (config["vit"].specified)
          lattice->set_mode(HmmNetBaumWelch::MODE_VITERBI);

        if (mfea_gen != &fea_gen)
        {
          // Open the file for the actual feature generator
          fea_gen.open(recipe.infos[f].audio_path);
        }

        double orig_beam = lattice->get_backward_beam();
        int counter = 1;
        bool skip = false;
        while (!lattice->init_utterance_segmentation())
        {
          if (counter >= 5)
          {
            fprintf(stderr, "Could not run Baum-Welch for file %s\n",
                    recipe.infos[f].audio_path.c_str());
            fprintf(stderr, "The HMM network may be incorrect or initial beam too low.\n");
            skip = true;
            break;
          }
          fprintf(stderr,
                  "Warning: Backward phase failed, increasing beam to %.1f\n",
                  ++counter*orig_beam);
          lattice->set_pruning_thresholds(0, counter*orig_beam);
        }
        // FIXME: Broken after HmmNetBaumWelch fix
        // if (!skip)
        //   hmmnet_collect_phone_stats(lattice, &phone_pool);
        if (mfea_gen != &fea_gen) // fea_gen is closed elsewhere
          mfea_gen->close();
        delete lattice;
      }
      else
      {
        PhnReader phn_reader(NULL);
        recipe.infos[f].init_phn_files(NULL, false, false,
                                       config["ophn"].specified, &fea_gen,
                                       &phn_reader);
        phn_reader.set_collect_transition_probs(false);
        collect_phone_stats(&phn_reader, &phone_pool);
        phn_reader.close();
      }

      fea_gen.close();
    }

    phone_pool.finish_statistics();
    phone_pool.decision_tree_cluster_context_phones(max_contexts);
    if (config["mloss"].specified)
      phone_pool.merge_context_phones();

    if (config["out"].specified)
      phone_pool.save_model(config["out"].get_str(), max_contexts);
    else
      save_basebind(config["basebind"].get_str(), &phone_pool);
  }
  catch (std::exception &e) {
    fprintf(stderr, "exception: %s\n", e.what());
    abort();
  }
  catch (std::string &str) {
    fprintf(stderr, "exception: %s\n", str.c_str());
    abort();
  }
}
<|endoftext|>


<commit_before>/********************************************************************
 * Copyright © 2016 Computational Molecular Biology Group,          *
 *                  Freie Universität Berlin (GER)                  *
 *                                                                  *
 * This file is part of ReaDDy.                                     *
 *                                                                  *
 * ReaDDy is free software: you can redistribute it and/or modify   *
 * it under the terms of the GNU Lesser General Public License as   *
 * published by the Free Software Foundation, either version 3 of   *
 * the License, or (at your option) any later version.              *
 *                                                                  *
 * This program is distributed in the hope that it will be useful,  *
 * but WITHOUT ANY WARRANTY; without even the implied warranty of   *
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the    *
 * GNU Lesser General Public License for more details.              *
 *                                                                  *
 * You should have received a copy of the GNU Lesser General        *
 * Public License along with this program. If not, see              *
 * <http://www.gnu.org/licenses/>.                                  *
 ********************************************************************/

/**
 * << detailed description >>
 *
 * @file Observables.cpp
 * @brief << brief description >>
 * @author clonker
 * @date 27.10.16
 */

#include <future>

#include <readdy/common/thread/scoped_async.h>

#include <readdy/kernel/cpu/observables/CPUObservables.h>
#include <readdy/kernel/cpu/CPUKernel.h>
#include <readdy/kernel/cpu/util/config.h>

namespace readdy {
namespace kernel {
namespace cpu {
namespace observables {

CPUPositions::CPUPositions(CPUKernel *const kernel, unsigned int stride,
                           const std::vector<std::string> &typesToCount) :
        readdy::model::observables::Positions(kernel, stride, typesToCount), kernel(kernel) {}

void CPUPositions::evaluate() {
    result.clear();
    auto &stateModel = kernel->getCPUKernelStateModel();
    const auto &pd = stateModel.getParticleData();
    if (typesToCount.empty()) {
        result = stateModel.getParticlePositions();
    } else {
        for (const auto &e : *stateModel.getParticleData()) {
            if (!e.deactivated &&
                std::find(typesToCount.begin(), typesToCount.end(), e.type) != typesToCount.end()) {
                result.push_back(e.pos);
            }
        }
    }
}

CPUHistogramAlongAxis::CPUHistogramAlongAxis(CPUKernel *const kernel, unsigned int stride,
                                             const std::vector<scalar> &binBorders,
                                             const std::vector<std::string> &typesToCount, unsigned int axis)
        : readdy::model::observables::HistogramAlongAxis(kernel, stride, binBorders, typesToCount, axis),
          kernel(kernel) {
    size = result.size();
}

void CPUHistogramAlongAxis::evaluate() {
    using Iter = readdy::kernel::cpu::CPUStateModel::data_type::const_iterator;

    std::fill(result.begin(), result.end(), 0);

    const auto binBorders = this->binBorders;
    const auto typesToCount = this->typesToCount;
    const auto resultSize = result.size();
    const auto axis = this->axis;
    const auto data = kernel->getCPUKernelStateModel().getParticleData();

    std::vector<std::future<result_type>> updates;
    updates.reserve(kernel->getNThreads());
    auto worker = [binBorders, typesToCount, resultSize, data, axis](std::size_t, Iter from, Iter to, std::promise<result_type>& update) {
        result_type resultUpdate;
        resultUpdate.resize(resultSize);

        for (auto it = from; it != to; ++it) {
            if (!it->deactivated && typesToCount.find(it->type) != typesToCount.end()) {
                auto upperBound = std::upper_bound(binBorders.begin(), binBorders.end(), it->pos[axis]);
                if (upperBound != binBorders.end()) {
                    auto binBordersIdx = upperBound - binBorders.begin();
                    if (binBordersIdx >= 1 && binBordersIdx < resultSize) {
                        ++resultUpdate[binBordersIdx - 1];
                    }
                }
            }
        }

        update.set_value(std::move(resultUpdate));
    };

    {
        const std::size_t grainSize = data->size() / kernel->getNThreads();

        std::vector<std::function<void(std::size_t)>> executables;
        executables.reserve(kernel->getNThreads());
        std::vector<std::promise<result_type>> promises;
        promises.resize(kernel->getNThreads());

        const auto& executor = kernel->executor();

        auto workIter = data->cbegin();
        for (unsigned int i = 0; i < kernel->getNThreads() - 1; ++i) {
            updates.push_back(promises.at(i).get_future());
            executables.push_back(executor.pack(worker, workIter, workIter + grainSize, std::ref(promises.at(i))));
            workIter += grainSize;
        }
        auto& promise = promises.back();
        updates.push_back(promise.get_future());
        executables.push_back(executor.pack(worker, workIter, data->cend(), std::ref(promise)));

        executor.execute_and_wait(std::move(executables));
    }

    for (auto &update : updates) {
        auto vec = std::move(update.get());
        auto it1 = vec.begin();
        auto it2 = result.begin();
        for (; it1 != vec.end(); ++it1, ++it2) {
            *it2 += *it1;
        }
    }
}

CPUNParticles::CPUNParticles(CPUKernel *const kernel, unsigned int stride, std::vector<std::string> typesToCount)
        : readdy::model::observables::NParticles(kernel, stride, std::move(typesToCount)),
          kernel(kernel) {}

void CPUNParticles::evaluate() {
    std::vector<unsigned long> resultVec = {};
    if (typesToCount.empty()) {
        resultVec.push_back(kernel->getCPUKernelStateModel().getParticleData()->size());
    } else {
        resultVec.resize(typesToCount.size());
        const auto &pd = kernel->getCPUKernelStateModel().getParticleData();
        for (const auto &e : *pd) {
            if (!e.deactivated) {
                auto typeIt = std::find(typesToCount.begin(), typesToCount.end(), e.type);
                if (typeIt != typesToCount.end()) {
                    ++resultVec[typeIt - typesToCount.begin()];
                }
            }
        }
    }
    result = std::move(resultVec);
}

CPUForces::CPUForces(CPUKernel *const kernel, unsigned int stride, std::vector<std::string> typesToCount) :
        readdy::model::observables::Forces(kernel, stride, std::move(typesToCount)),
        kernel(kernel) {}

void CPUForces::evaluate() {
    result.clear();
    const auto &pd = kernel->getCPUKernelStateModel().getParticleData();
    if (typesToCount.empty()) {
        result.reserve(pd->size());
    }
    for (const auto &e : *pd) {
        if (!e.deactivated) {
            if (typesToCount.empty()) {
                result.push_back(e.force);
            } else {
                for (auto countedParticleType : typesToCount) {
                    if (e.type == countedParticleType) {
                        result.push_back(e.force);
                        break;
                    }
                }
            }
        }
    }
}

CPUParticles::CPUParticles(CPUKernel *const kernel, unsigned int stride)
        : readdy::model::observables::Particles(kernel, stride), kernel(kernel) {}

void CPUParticles::evaluate() {
    auto &resultTypes = std::get<0>(result);
    auto &resultIds = std::get<1>(result);
    auto &resultPositions = std::get<2>(result);
    resultTypes.clear();
    resultIds.clear();
    resultPositions.clear();
    const auto &particleData = kernel->getCPUKernelStateModel().getParticleData();
    resultTypes.reserve(particleData->size());
    resultIds.reserve(particleData->size());
    resultPositions.reserve(particleData->size());
    for (const auto &entry : *particleData) {
        if (!entry.deactivated) {
            resultTypes.push_back(entry.type);
            resultIds.push_back(entry.id);
            resultPositions.push_back(entry.pos);
        }
    }
}

CPUReactions::CPUReactions(CPUKernel *const kernel, unsigned int stride)
        : Reactions(kernel, stride), kernel(kernel) {}

void CPUReactions::evaluate() {
    const auto& model = kernel->getCPUKernelStateModel();
    const auto& records = model.reactionRecords();
    result.clear();
    result.reserve(records.size());
    result.insert(result.end(), records.begin(), records.end());
}

CPUReactionCounts::CPUReactionCounts(CPUKernel *const kernel, unsigned int stride)
        : ReactionCounts(kernel, stride), kernel(kernel) {}

void CPUReactionCounts::evaluate() {
    readdy::model::observables::ReactionCounts::initializeCounts(result, kernel->context());
    assignCountsToResult(kernel->getCPUKernelStateModel().reactionCounts(), result);
}

}
}
}
}
<commit_msg>fix bug in n_particles observable in CPU kernel<commit_after>/********************************************************************
 * Copyright © 2016 Computational Molecular Biology Group,          *
 *                  Freie Universität Berlin (GER)                  *
 *                                                                  *
 * This file is part of ReaDDy.                                     *
 *                                                                  *
 * ReaDDy is free software: you can redistribute it and/or modify   *
 * it under the terms of the GNU Lesser General Public License as   *
 * published by the Free Software Foundation, either version 3 of   *
 * the License, or (at your option) any later version.              *
 *                                                                  *
 * This program is distributed in the hope that it will be useful,  *
 * but WITHOUT ANY WARRANTY; without even the implied warranty of   *
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the    *
 * GNU Lesser General Public License for more details.              *
 *                                                                  *
 * You should have received a copy of the GNU Lesser General        *
 * Public License along with this program. If not, see              *
 * <http://www.gnu.org/licenses/>.                                  *
 ********************************************************************/

/**
 * << detailed description >>
 *
 * @file Observables.cpp
 * @brief << brief description >>
 * @author clonker
 * @date 27.10.16
 */

#include <future>

#include <readdy/common/thread/scoped_async.h>

#include <readdy/kernel/cpu/observables/CPUObservables.h>
#include <readdy/kernel/cpu/CPUKernel.h>
#include <readdy/kernel/cpu/util/config.h>

namespace readdy {
namespace kernel {
namespace cpu {
namespace observables {

CPUPositions::CPUPositions(CPUKernel *const kernel, unsigned int stride,
                           const std::vector<std::string> &typesToCount) :
        readdy::model::observables::Positions(kernel, stride, typesToCount), kernel(kernel) {}

void CPUPositions::evaluate() {
    result.clear();
    auto &stateModel = kernel->getCPUKernelStateModel();
    const auto &pd = stateModel.getParticleData();
    if (typesToCount.empty()) {
        result = stateModel.getParticlePositions();
    } else {
        for (const auto &e : *stateModel.getParticleData()) {
            if (!e.deactivated &&
                std::find(typesToCount.begin(), typesToCount.end(), e.type) != typesToCount.end()) {
                result.push_back(e.pos);
            }
        }
    }
}

CPUHistogramAlongAxis::CPUHistogramAlongAxis(CPUKernel *const kernel, unsigned int stride,
                                             const std::vector<scalar> &binBorders,
                                             const std::vector<std::string> &typesToCount, unsigned int axis)
        : readdy::model::observables::HistogramAlongAxis(kernel, stride, binBorders, typesToCount, axis),
          kernel(kernel) {
    size = result.size();
}

void CPUHistogramAlongAxis::evaluate() {
    using Iter = readdy::kernel::cpu::CPUStateModel::data_type::const_iterator;

    std::fill(result.begin(), result.end(), 0);

    const auto binBorders = this->binBorders;
    const auto typesToCount = this->typesToCount;
    const auto resultSize = result.size();
    const auto axis = this->axis;
    const auto data = kernel->getCPUKernelStateModel().getParticleData();

    std::vector<std::future<result_type>> updates;
    updates.reserve(kernel->getNThreads());
    auto worker = [binBorders, typesToCount, resultSize, data, axis](std::size_t, Iter from, Iter to, std::promise<result_type>& update) {
        result_type resultUpdate;
        resultUpdate.resize(resultSize);

        for (auto it = from; it != to; ++it) {
            if (!it->deactivated && typesToCount.find(it->type) != typesToCount.end()) {
                auto upperBound = std::upper_bound(binBorders.begin(), binBorders.end(), it->pos[axis]);
                if (upperBound != binBorders.end()) {
                    auto binBordersIdx = upperBound - binBorders.begin();
                    if (binBordersIdx >= 1 && binBordersIdx < resultSize) {
                        ++resultUpdate[binBordersIdx - 1];
                    }
                }
            }
        }

        update.set_value(std::move(resultUpdate));
    };

    {
        const std::size_t grainSize = data->size() / kernel->getNThreads();

        std::vector<std::function<void(std::size_t)>> executables;
        executables.reserve(kernel->getNThreads());
        std::vector<std::promise<result_type>> promises;
        promises.resize(kernel->getNThreads());

        const auto& executor = kernel->executor();

        auto workIter = data->cbegin();
        for (unsigned int i = 0; i < kernel->getNThreads() - 1; ++i) {
            updates.push_back(promises.at(i).get_future());
            executables.push_back(executor.pack(worker, workIter, workIter + grainSize, std::ref(promises.at(i))));
            workIter += grainSize;
        }
        auto& promise = promises.back();
        updates.push_back(promise.get_future());
        executables.push_back(executor.pack(worker, workIter, data->cend(), std::ref(promise)));

        executor.execute_and_wait(std::move(executables));
    }

    for (auto &update : updates) {
        auto vec = std::move(update.get());
        auto it1 = vec.begin();
        auto it2 = result.begin();
        for (; it1 != vec.end(); ++it1, ++it2) {
            *it2 += *it1;
        }
    }
}

CPUNParticles::CPUNParticles(CPUKernel *const kernel, unsigned int stride, std::vector<std::string> typesToCount)
        : readdy::model::observables::NParticles(kernel, stride, std::move(typesToCount)),
          kernel(kernel) {}

void CPUNParticles::evaluate() {
    std::vector<unsigned long> resultVec = {};
    if (typesToCount.empty()) {
        const auto data = kernel->getCPUKernelStateModel().getParticleData();
        resultVec.push_back(data->size() - data->getNDeactivated());
    } else {
        resultVec.resize(typesToCount.size());
        const auto &pd = kernel->getCPUKernelStateModel().getParticleData();
        for (const auto &e : *pd) {
            if (!e.deactivated) {
                auto typeIt = std::find(typesToCount.begin(), typesToCount.end(), e.type);
                if (typeIt != typesToCount.end()) {
                    ++resultVec[typeIt - typesToCount.begin()];
                }
            }
        }
    }
    result = std::move(resultVec);
}

CPUForces::CPUForces(CPUKernel *const kernel, unsigned int stride, std::vector<std::string> typesToCount) :
        readdy::model::observables::Forces(kernel, stride, std::move(typesToCount)),
        kernel(kernel) {}

void CPUForces::evaluate() {
    result.clear();
    const auto &pd = kernel->getCPUKernelStateModel().getParticleData();
    if (typesToCount.empty()) {
        result.reserve(pd->size());
    }
    for (const auto &e : *pd) {
        if (!e.deactivated) {
            if (typesToCount.empty()) {
                result.push_back(e.force);
            } else {
                for (auto countedParticleType : typesToCount) {
                    if (e.type == countedParticleType) {
                        result.push_back(e.force);
                        break;
                    }
                }
            }
        }
    }
}

CPUParticles::CPUParticles(CPUKernel *const kernel, unsigned int stride)
        : readdy::model::observables::Particles(kernel, stride), kernel(kernel) {}

void CPUParticles::evaluate() {
    auto &resultTypes = std::get<0>(result);
    auto &resultIds = std::get<1>(result);
    auto &resultPositions = std::get<2>(result);
    resultTypes.clear();
    resultIds.clear();
    resultPositions.clear();
    const auto &particleData = kernel->getCPUKernelStateModel().getParticleData();
    resultTypes.reserve(particleData->size());
    resultIds.reserve(particleData->size());
    resultPositions.reserve(particleData->size());
    for (const auto &entry : *particleData) {
        if (!entry.deactivated) {
            resultTypes.push_back(entry.type);
            resultIds.push_back(entry.id);
            resultPositions.push_back(entry.pos);
        }
    }
}

CPUReactions::CPUReactions(CPUKernel *const kernel, unsigned int stride)
        : Reactions(kernel, stride), kernel(kernel) {}

void CPUReactions::evaluate() {
    const auto& model = kernel->getCPUKernelStateModel();
    const auto& records = model.reactionRecords();
    result.clear();
    result.reserve(records.size());
    result.insert(result.end(), records.begin(), records.end());
}

CPUReactionCounts::CPUReactionCounts(CPUKernel *const kernel, unsigned int stride)
        : ReactionCounts(kernel, stride), kernel(kernel) {}

void CPUReactionCounts::evaluate() {
    readdy::model::observables::ReactionCounts::initializeCounts(result, kernel->context());
    assignCountsToResult(kernel->getCPUKernelStateModel().reactionCounts(), result);
}

}
}
}
}
<|endoftext|>


<commit_before>#include <gainput/gainput.h>

#ifdef GAINPUT_DEV
#include "GainputMemoryStream.h"

namespace gainput {

MemoryStream::MemoryStream(void* data, size_t length, size_t capacity, bool ownership) :
	data(data),
	length(length),
	capacity(capacity),
	ownership(ownership),
	position(0)
{
	// empty
}

MemoryStream::MemoryStream(size_t capacity, Allocator& allocator) :
	allocator(&allocator),
	length(0),
	capacity(capacity),
	ownership(true),
	position(0)
{
	data = this->allocator->Allocate(capacity);
}

MemoryStream::~MemoryStream()
{
	if (ownership)
	{
		assert(allocator);
		allocator->Deallocate(data);
	}
}

size_t
MemoryStream::Read(void* dest, size_t readLength)
{
	assert(position + readLength <= length);
	memcpy(dest, (void*)( (uint8_t*)data + position), readLength);
	position += readLength;
	return readLength;
}

size_t
MemoryStream::Write(const void* src, size_t writeLength)
{
	assert(position + writeLength <= capacity);
	memcpy((void*)( (uint8_t*)data + position), src, writeLength);
	position += writeLength;
	length += writeLength;
	return writeLength;
}

bool
MemoryStream::SeekBegin(int offset)
{
	if (offset < 0)
	{
		return false;
	}
	position = offset;
	return true;
}

bool
MemoryStream::SeekCurrent(int offset)
{
	if (offset + position > length)
	{
		return false;
	}
	position += offset;
	return true;
}

}
#endif

<commit_msg>Fixed release builds failing with enabled input recording.<commit_after>#include <gainput/gainput.h>

#if defined(GAINPUT_DEV) || defined(GAINPUT_ENABLE_RECORDER)
#include "GainputMemoryStream.h"

namespace gainput {

MemoryStream::MemoryStream(void* data, size_t length, size_t capacity, bool ownership) :
	data(data),
	length(length),
	capacity(capacity),
	ownership(ownership),
	position(0)
{
	// empty
}

MemoryStream::MemoryStream(size_t capacity, Allocator& allocator) :
	allocator(&allocator),
	length(0),
	capacity(capacity),
	ownership(true),
	position(0)
{
	data = this->allocator->Allocate(capacity);
}

MemoryStream::~MemoryStream()
{
	if (ownership)
	{
		assert(allocator);
		allocator->Deallocate(data);
	}
}

size_t
MemoryStream::Read(void* dest, size_t readLength)
{
	assert(position + readLength <= length);
	memcpy(dest, (void*)( (uint8_t*)data + position), readLength);
	position += readLength;
	return readLength;
}

size_t
MemoryStream::Write(const void* src, size_t writeLength)
{
	assert(position + writeLength <= capacity);
	memcpy((void*)( (uint8_t*)data + position), src, writeLength);
	position += writeLength;
	length += writeLength;
	return writeLength;
}

bool
MemoryStream::SeekBegin(int offset)
{
	if (offset < 0)
	{
		return false;
	}
	position = offset;
	return true;
}

bool
MemoryStream::SeekCurrent(int offset)
{
	if (offset + position > length)
	{
		return false;
	}
	position += offset;
	return true;
}

}
#endif

<|endoftext|>