<commit_before>#include "Variant.h"
#include "split.h"
#include <string>
#include <list>
#include <iostream>

using namespace std;
using namespace vcf;

void annotateWithBlankGenotypes(Variant& var, string& annotationTag) {

    var.addFormatField(annotationTag);

    map<string, map<string, vector<string> > >::iterator s     = var.samples.begin();
    map<string, map<string, vector<string> > >::iterator sEnd  = var.samples.end();

    for (; s != sEnd; ++s) {
        map<string, vector<string> >& sample = s->second;
        sample[annotationTag].clear(); // means "no genotype" genotype
        sample[annotationTag].push_back("./."); // means "no genotype" genotype
    }
}

void annotateWithGenotypes(Variant& varA, Variant& varB, string& annotationTag) {

    varA.addFormatField(annotationTag);

    map<string, map<string, vector<string> > >::iterator s     = varA.samples.begin();
    map<string, map<string, vector<string> > >::iterator sEnd  = varA.samples.end();

    map<string, int> varAAlleleInts;
    int i = 1;
    for (vector<string>::iterator a = varA.alt.begin(); a != varA.alt.end(); ++a, ++i) {
        varAAlleleInts[*a] = i;
    }

    map<int, int> varBconvertToVarA; // maps alleles in the second file to allele numbers for the first
    varBconvertToVarA[0] = 0; // reference == reference!
    i = 1;
    for (vector<string>::iterator a = varB.alt.begin(); a != varB.alt.end(); ++a, ++i) {
        map<string, int>::iterator ita = varAAlleleInts.find(*a);
        if (ita != varAAlleleInts.end()) {
            varBconvertToVarA[i] = ita->second;
        }
    }

    for (; s != sEnd; ++s) {
        map<string, vector<string> >& sample = s->second;
        const string& name = s->first;
        map<string, map<string, vector<string> > >::iterator o = varB.samples.find(name);
        sample[annotationTag].clear();
        if (o == varB.samples.end()) {
            sample[annotationTag].push_back("./."); // means "no genotype"
        } else {
            map<string, vector<string> >& other = o->second;
            string& otherGenotype = other["GT"].front();
            // XXX this must compare the genotypes in the two files
            map<int, int> gtB = decomposeGenotype(otherGenotype);
            map<int, int> gtnew;
            for (map<int, int>::iterator g = gtB.begin(); g != gtB.end(); ++g) {
                map<int, int>::iterator f = varBconvertToVarA.find(g->first);
                if (f != varBconvertToVarA.end()) {
                    gtnew[f->second] += g->second;
                } else {
                    gtnew[-1] += g->second;
                }
            }
            sample[annotationTag].push_back(genotypeToString(gtnew));
        }
    }
}

int main(int argc, char** argv) {

    if (argc != 4) {
        cerr << "usage: " << argv[0] << " <annotation-tag> <vcf file> <vcf file>" << endl
             << "annotates genotypes in the first file with genotypes in the second" << endl
             << "adding the genotype as another flag to each sample filed in the first file." << endl
             << "annotation-tag is the name of the sample flag which is added to store the annotation." << endl
             << "also adds a 'has_variant' flag for sites where the second file has a variant." << endl;
        return 1;
    }

    string annotag = argv[1];
    string filenameA = argv[2];
    string filenameB = argv[3];

    if (filenameA == filenameB) {
        cerr << "it won't help to annotate samples with their own genotypes!" << endl;
        return 1;
    }

    VariantCallFile variantFileA;
    if (filenameA == "-") {
        variantFileA.open(std::cin);
    } else {
        variantFileA.open(filenameA);
    }

    VariantCallFile variantFileB;
    if (filenameB == "-") {
        variantFileB.open(std::cin);
    } else {
        variantFileB.open(filenameB);
    }

    if (!variantFileA.is_open() || !variantFileB.is_open()) {
        return 1;
    }

    Variant varA(variantFileA);
    Variant varB(variantFileB);

    // while the first file doesn't match the second positionally,
    // step forward, annotating each genotype record with an empty genotype
    // when the two match, iterate through the genotypes from the first file
    // and get the genotypes reported in the second file

    variantFileA.getNextVariant(varA);
    variantFileB.getNextVariant(varB);

    string line = "##INFO=<ID=" + annotag + ".has_variant,Number=0,Type=Flag,Description=\"True if "
        + annotag + " has a called alternate among samples under comparison.\">";
    variantFileA.addHeaderLine(line);
    line = "##FORMAT=<ID=" + annotag + ",Number=1,Type=String,Description=\"Genotype from "
        + annotag + ".\">";
    variantFileA.addHeaderLine(line);

    cout << variantFileA.header << endl;
	int counter = 0;

    do {
		counter ++;

        // this is broken.  to do it right, it'll be necessary to get reference ids from the fasta reference used to make the alignments...
		// if B is NOT done, and is less than A, read new B.
        if (!variantFileB.done()
            && (varB.sequenceName != varA.sequenceName
                || (varB.sequenceName == varA.sequenceName && varB.position < varA.position)
				|| variantFileA.done())
            ) {
            variantFileB.getNextVariant(varB);
        }

		// if A is not done- and A is less than B, read A.
		// should also read if variant B is done.
        if (!variantFileA.done()
            && (varA.sequenceName != varB.sequenceName
                || (varA.sequenceName == varB.sequenceName && varA.position < varB.position)
				|| variantFileB.done())
            ) {
            cout << varA << endl;
            variantFileA.getNextVariant(varA);
        }

        vector<Variant> varsA;
        vector<Variant> varsB;

        bool hasMultipleAlts = false;

        long int thisPosition = 0;
        string thisSequenceName;
        if (varA.position == varB.position
            && varA.sequenceName == varB.sequenceName) {
            thisPosition = varA.position;
            thisSequenceName = varA.sequenceName;
        }
        while (!variantFileA.done()
               && !variantFileB.done()
               && thisPosition == varA.position
               && thisSequenceName == varA.sequenceName
               && varA.sequenceName == varB.sequenceName
               && varA.position == varB.position) {
            // accumulate all the alts at the current position
            varsA.push_back(varA);
            varsB.push_back(varB);
            if (varA.alt.size() > 1 || varB.alt.size() > 1)
                hasMultipleAlts = true;
            variantFileA.getNextVariant(varA);
            variantFileB.getNextVariant(varB);
        }

        // multiple lines per position
        if (!hasMultipleAlts && (varsA.size() > 1 || varsB.size() > 1)) {

            map<pair<string, string>, Variant> varsAParsed;
            map<pair<string, string>, Variant> varsBParsed;
            for (vector<Variant>::iterator v = varsA.begin(); v != varsA.end(); ++v) {
                varsAParsed[make_pair(v->ref, v->alt.front())] = *v;
            }
            for (vector<Variant>::iterator v = varsB.begin(); v != varsB.end(); ++v) {
                varsBParsed[make_pair(v->ref, v->alt.front())] = *v;
            }

            for (map<pair<string, string>, Variant>::iterator vs = varsAParsed.begin(); vs != varsAParsed.end(); ++vs) {
                Variant& varA = vs->second;
                if (varsBParsed.find(make_pair(varA.ref, varA.alt.front())) != varsBParsed.end()) {
                    Variant& varB = varsBParsed[make_pair(varA.ref, varA.alt.front())]; // TODO cleanup
                    annotateWithGenotypes(varA, varB, annotag);
                    varA.infoFlags[annotag + ".has_variant"] = true;
                } else {
                    annotateWithBlankGenotypes(varA, annotag);
                }
                cout << varA << endl;
            }

        } else if (!varsA.empty() && !varsB.empty()) { // one line per multi-allelic
            Variant& varA = varsA.front();
            Variant& varB = varsB.front();
            annotateWithGenotypes(varA, varB, annotag);
            // XXX TODO, and also allow for records with multiple alts
            // XXX assume that if the other file has a corresponding record, some kind of variation was detected at the same site
            varA.infoFlags[annotag + ".has_variant"] = true;
            cout << varA << endl;
        }
    } while (!variantFileA.done() || !variantFileB.done());

    cerr << "check: " << counter << "\n";
    return 0;

}

<commit_msg>Removed testing prints to serr<commit_after>#include "Variant.h"
#include "split.h"
#include <string>
#include <list>
#include <iostream>

using namespace std;
using namespace vcf;

void annotateWithBlankGenotypes(Variant& var, string& annotationTag) {

    var.addFormatField(annotationTag);

    map<string, map<string, vector<string> > >::iterator s     = var.samples.begin();
    map<string, map<string, vector<string> > >::iterator sEnd  = var.samples.end();

    for (; s != sEnd; ++s) {
        map<string, vector<string> >& sample = s->second;
        sample[annotationTag].clear(); // means "no genotype" genotype
        sample[annotationTag].push_back("./."); // means "no genotype" genotype
    }
}

void annotateWithGenotypes(Variant& varA, Variant& varB, string& annotationTag) {

    varA.addFormatField(annotationTag);

    map<string, map<string, vector<string> > >::iterator s     = varA.samples.begin();
    map<string, map<string, vector<string> > >::iterator sEnd  = varA.samples.end();

    map<string, int> varAAlleleInts;
    int i = 1;
    for (vector<string>::iterator a = varA.alt.begin(); a != varA.alt.end(); ++a, ++i) {
        varAAlleleInts[*a] = i;
    }

    map<int, int> varBconvertToVarA; // maps alleles in the second file to allele numbers for the first
    varBconvertToVarA[0] = 0; // reference == reference!
    i = 1;
    for (vector<string>::iterator a = varB.alt.begin(); a != varB.alt.end(); ++a, ++i) {
        map<string, int>::iterator ita = varAAlleleInts.find(*a);
        if (ita != varAAlleleInts.end()) {
            varBconvertToVarA[i] = ita->second;
        }
    }

    for (; s != sEnd; ++s) {
        map<string, vector<string> >& sample = s->second;
        const string& name = s->first;
        map<string, map<string, vector<string> > >::iterator o = varB.samples.find(name);
        sample[annotationTag].clear();
        if (o == varB.samples.end()) {
            sample[annotationTag].push_back("./."); // means "no genotype"
        } else {
            map<string, vector<string> >& other = o->second;
            string& otherGenotype = other["GT"].front();
            // XXX this must compare the genotypes in the two files
            map<int, int> gtB = decomposeGenotype(otherGenotype);
            map<int, int> gtnew;
            for (map<int, int>::iterator g = gtB.begin(); g != gtB.end(); ++g) {
                map<int, int>::iterator f = varBconvertToVarA.find(g->first);
                if (f != varBconvertToVarA.end()) {
                    gtnew[f->second] += g->second;
                } else {
                    gtnew[-1] += g->second;
                }
            }
            sample[annotationTag].push_back(genotypeToString(gtnew));
        }
    }
}

int main(int argc, char** argv) {

    if (argc != 4) {
        cerr << "usage: " << argv[0] << " <annotation-tag> <vcf file> <vcf file>" << endl
             << "annotates genotypes in the first file with genotypes in the second" << endl
             << "adding the genotype as another flag to each sample filed in the first file." << endl
             << "annotation-tag is the name of the sample flag which is added to store the annotation." << endl
             << "also adds a 'has_variant' flag for sites where the second file has a variant." << endl;
        return 1;
    }

    string annotag = argv[1];
    string filenameA = argv[2];
    string filenameB = argv[3];

    if (filenameA == filenameB) {
        cerr << "it won't help to annotate samples with their own genotypes!" << endl;
        return 1;
    }

    VariantCallFile variantFileA;
    if (filenameA == "-") {
        variantFileA.open(std::cin);
    } else {
        variantFileA.open(filenameA);
    }

    VariantCallFile variantFileB;
    if (filenameB == "-") {
        variantFileB.open(std::cin);
    } else {
        variantFileB.open(filenameB);
    }

    if (!variantFileA.is_open() || !variantFileB.is_open()) {
        return 1;
    }

    Variant varA(variantFileA);
    Variant varB(variantFileB);

    // while the first file doesn't match the second positionally,
    // step forward, annotating each genotype record with an empty genotype
    // when the two match, iterate through the genotypes from the first file
    // and get the genotypes reported in the second file

    variantFileA.getNextVariant(varA);
    variantFileB.getNextVariant(varB);

    string line = "##INFO=<ID=" + annotag + ".has_variant,Number=0,Type=Flag,Description=\"True if "
        + annotag + " has a called alternate among samples under comparison.\">";
    variantFileA.addHeaderLine(line);
    line = "##FORMAT=<ID=" + annotag + ",Number=1,Type=String,Description=\"Genotype from "
        + annotag + ".\">";
    variantFileA.addHeaderLine(line);

    cout << variantFileA.header << endl;

    do {

        // this is broken.  to do it right, it'll be necessary to get reference ids from the fasta reference used to make the alignments...
		// if B is NOT done, and is less than A, read new B.
        if (!variantFileB.done()
            && (varB.sequenceName != varA.sequenceName
                || (varB.sequenceName == varA.sequenceName && varB.position < varA.position)
				|| variantFileA.done())
            ) {
            variantFileB.getNextVariant(varB);
        }

		// if A is not done- and A is less than B, read A.
		// should also read if variant B is done.
        if (!variantFileA.done()
            && (varA.sequenceName != varB.sequenceName
                || (varA.sequenceName == varB.sequenceName && varA.position < varB.position)
				|| variantFileB.done())
            ) {
            cout << varA << endl;
            variantFileA.getNextVariant(varA);
        }

        vector<Variant> varsA;
        vector<Variant> varsB;

        bool hasMultipleAlts = false;

        long int thisPosition = 0;
        string thisSequenceName;
        if (varA.position == varB.position
            && varA.sequenceName == varB.sequenceName) {
            thisPosition = varA.position;
            thisSequenceName = varA.sequenceName;
        }
        while (!variantFileA.done()
               && !variantFileB.done()
               && thisPosition == varA.position
               && thisSequenceName == varA.sequenceName
               && varA.sequenceName == varB.sequenceName
               && varA.position == varB.position) {
            // accumulate all the alts at the current position
            varsA.push_back(varA);
            varsB.push_back(varB);
            if (varA.alt.size() > 1 || varB.alt.size() > 1)
                hasMultipleAlts = true;
            variantFileA.getNextVariant(varA);
            variantFileB.getNextVariant(varB);
        }

        // multiple lines per position
        if (!hasMultipleAlts && (varsA.size() > 1 || varsB.size() > 1)) {

            map<pair<string, string>, Variant> varsAParsed;
            map<pair<string, string>, Variant> varsBParsed;
            for (vector<Variant>::iterator v = varsA.begin(); v != varsA.end(); ++v) {
                varsAParsed[make_pair(v->ref, v->alt.front())] = *v;
            }
            for (vector<Variant>::iterator v = varsB.begin(); v != varsB.end(); ++v) {
                varsBParsed[make_pair(v->ref, v->alt.front())] = *v;
            }

            for (map<pair<string, string>, Variant>::iterator vs = varsAParsed.begin(); vs != varsAParsed.end(); ++vs) {
                Variant& varA = vs->second;
                if (varsBParsed.find(make_pair(varA.ref, varA.alt.front())) != varsBParsed.end()) {
                    Variant& varB = varsBParsed[make_pair(varA.ref, varA.alt.front())]; // TODO cleanup
                    annotateWithGenotypes(varA, varB, annotag);
                    varA.infoFlags[annotag + ".has_variant"] = true;
                } else {
                    annotateWithBlankGenotypes(varA, annotag);
                }
                cout << varA << endl;
            }

        } else if (!varsA.empty() && !varsB.empty()) { // one line per multi-allelic
            Variant& varA = varsA.front();
            Variant& varB = varsB.front();
            annotateWithGenotypes(varA, varB, annotag);
            // XXX TODO, and also allow for records with multiple alts
            // XXX assume that if the other file has a corresponding record, some kind of variation was detected at the same site
            varA.infoFlags[annotag + ".has_variant"] = true;
            cout << varA << endl;
        }

    } while (!variantFileA.done() || !variantFileB.done());

    return 0;

}

<|endoftext|>


<commit_before>/*
 * Copyright 2007-2019 Content Management AG
 * All rights reserved.
 *
 * author: Max Kellermann <mk@cm4all.com>
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions
 * are met:
 *
 * - Redistributions of source code must retain the above copyright
 * notice, this list of conditions and the following disclaimer.
 *
 * - Redistributions in binary form must reproduce the above copyright
 * notice, this list of conditions and the following disclaimer in the
 * documentation and/or other materials provided with the
 * distribution.
 *
 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
 * FOR A PARTICULAR PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE
 * FOUNDATION OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
 * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
 * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
 * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED
 * OF THE POSSIBILITY OF SUCH DAMAGE.
 */

#include "istream_deflate.hxx"
#include "UnusedPtr.hxx"
#include "New.hxx"
#include "FacadeIstream.hxx"
#include "pool/pool.hxx"
#include "fb_pool.hxx"
#include "SliceFifoBuffer.hxx"
#include "event/DeferEvent.hxx"
#include "util/Cast.hxx"
#include "util/ConstBuffer.hxx"
#include "util/DestructObserver.hxx"
#include "util/WritableBuffer.hxx"

#include <zlib.h>

#include <stdexcept>

#include <assert.h>

class ZlibError : public std::runtime_error {
    int code;

public:
    explicit ZlibError(int _code, const char *_msg)
        :std::runtime_error(_msg), code(_code) {}

    int GetCode() const noexcept {
        return code;
    }
};

class DeflateIstream final : public FacadeIstream, DestructAnchor {
    const bool gzip;
    bool z_initialized = false, z_stream_end = false;
    z_stream z;
    bool had_input, had_output;
    bool reading;
    SliceFifoBuffer buffer;

    /**
     * This callback is used to request more data from the input if an
     * OnData() call did not produce any output.  This tries to
     * prevent stalling the stream.
     */
    DeferEvent defer;

public:
    DeflateIstream(struct pool &_pool, UnusedIstreamPtr _input,
                   EventLoop &event_loop, bool _gzip)
        :FacadeIstream(_pool, std::move(_input)),
         gzip(_gzip),
         reading(false),
         defer(event_loop, BIND_THIS_METHOD(OnDeferred))
    {
    }

    ~DeflateIstream() {
        defer.Cancel();
    }

    bool InitZlib() noexcept;

    void DeinitZlib() noexcept {
        if (z_initialized)
            deflateEnd(&z);
    }

    void Abort(std::exception_ptr ep) noexcept {
        DeinitZlib();

        if (HasInput())
            ClearAndCloseInput();

        DestroyError(ep);
    }

    void Abort(int code, const char *msg) noexcept {
        Abort(std::make_exception_ptr(ZlibError(code, msg)));
    }

    /**
     * Submit data from the buffer to our istream handler.
     *
     * @return the number of bytes which were handled, or 0 if the
     * stream was closed
     */
    size_t TryWrite() noexcept;

    /**
     * Starts to write to the buffer.
     *
     * @return a pointer to the writable buffer, or nullptr if there is no
     * room (our istream handler blocks) or if the stream was closed
     */
    WritableBuffer<void> BufferWrite() noexcept {
        buffer.AllocateIfNull(fb_pool_get());
        auto w = buffer.Write();
        if (w.empty() && TryWrite() > 0)
            w = buffer.Write();

        return w.ToVoid();
    }

    void TryFlush() noexcept;

    /**
     * Read from our input until we have submitted some bytes to our
     * istream handler.
     */
    void ForceRead() noexcept;

    void TryFinish() noexcept;

    /* virtual methods from class Istream */

    void _Read() noexcept override {
        if (!buffer.empty())
            TryWrite();
        else if (HasInput())
            ForceRead();
        else
            TryFinish();
    }

    void _Close() noexcept override {
        DeinitZlib();

        if (HasInput())
            input.Close();

        Destroy();
    }

    /* virtual methods from class IstreamHandler */
    size_t OnData(const void *data, size_t length) noexcept override;
    void OnEof() noexcept override;
    void OnError(std::exception_ptr ep) noexcept override;

private:
    int GetWindowBits() const noexcept {
        return MAX_WBITS + gzip * 16;
    }

    void OnDeferred() noexcept {
        assert(HasInput());

        ForceRead();
    }
};

static voidpf
z_alloc(voidpf opaque, uInt items, uInt size) noexcept
{
    struct pool *pool = (struct pool *)opaque;

    return p_malloc(pool, items * size);
}

static void
z_free(voidpf opaque, voidpf address) noexcept
{
    (void)opaque;
    (void)address;
}

bool
DeflateIstream::InitZlib() noexcept
{
    if (z_initialized)
        return true;

    z.zalloc = z_alloc;
    z.zfree = z_free;
    z.opaque = &GetPool();

    int err = deflateInit2(&z, Z_DEFAULT_COMPRESSION,
                           Z_DEFLATED, GetWindowBits(), 8,
                           Z_DEFAULT_STRATEGY);
    if (err != Z_OK) {
        Abort(err, "deflateInit2() failed");
        return false;
    }

    z_initialized = true;
    return true;
}

size_t
DeflateIstream::TryWrite() noexcept
{
    auto r = buffer.Read();
    assert(!r.empty());

    size_t nbytes = InvokeData(r.data, r.size);
    if (nbytes == 0)
        return 0;

    buffer.Consume(nbytes);
    buffer.FreeIfEmpty();

    if (nbytes == r.size && !HasInput() && z_stream_end) {
        DeinitZlib();
        DestroyEof();
        return 0;
    }

    return nbytes;
}

inline void
DeflateIstream::TryFlush() noexcept
{
    assert(!z_stream_end);

    auto w = BufferWrite();
    if (w.empty())
        return;

    z.next_out = (Bytef *)w.data;
    z.avail_out = (uInt)w.size;

    z.next_in = nullptr;
    z.avail_in = 0;

    int err = deflate(&z, Z_SYNC_FLUSH);
    if (err != Z_OK) {
        Abort(err, "deflate(Z_SYNC_FLUSH) failed");
        return;
    }

    buffer.Append(w.size - (size_t)z.avail_out);

    if (!buffer.empty())
        TryWrite();
}

inline void
DeflateIstream::ForceRead() noexcept
{
    assert(!reading);

    const DestructObserver destructed(*this);

    bool had_input2 = false;
    had_output = false;

    while (1) {
        had_input = false;
        reading = true;
        input.Read();
        if (destructed)
            return;

        reading = false;
        if (!HasInput() || had_output)
            return;

        if (!had_input)
            break;

        had_input2 = true;
    }

    if (had_input2)
        TryFlush();
}

void
DeflateIstream::TryFinish() noexcept
{
    assert(!z_stream_end);

    auto w = BufferWrite();
    if (w.empty())
        return;

    z.next_out = (Bytef *)w.data;
    z.avail_out = (uInt)w.size;

    z.next_in = nullptr;
    z.avail_in = 0;

    int err = deflate(&z, Z_FINISH);
    if (err == Z_STREAM_END)
        z_stream_end = true;
    else if (err != Z_OK) {
        Abort(err, "deflate(Z_FINISH) failed");
        return;
    }

    buffer.Append(w.size - (size_t)z.avail_out);

    if (z_stream_end && buffer.empty()) {
        DeinitZlib();
        DestroyEof();
    } else
        TryWrite();
}

/*
 * istream handler
 *
 */

size_t
DeflateIstream::OnData(const void *data, size_t length) noexcept
{
    assert(HasInput());

    auto w = BufferWrite();
    if (w.size < 64) /* reserve space for end-of-stream marker */
        return 0;

    if (!InitZlib())
        return 0;

    had_input = true;

    if (!reading)
        had_output = false;

    z.next_out = (Bytef *)w.data;
    z.avail_out = (uInt)w.size;

    z.next_in = (Bytef *)const_cast<void *>(data);
    z.avail_in = (uInt)length;

    do {
        auto err = deflate(&z, Z_NO_FLUSH);
        if (err != Z_OK) {
            Abort(err, "deflate() failed");
            return 0;
        }

        size_t nbytes = w.size - (size_t)z.avail_out;
        if (nbytes > 0) {
            had_output = true;
            buffer.Append(nbytes);

            const DestructObserver destructed(*this);
            TryWrite();
            if (destructed)
                return 0;
        } else
            break;

        w = BufferWrite();
        if (w.size < 64) /* reserve space for end-of-stream marker */
            break;

        z.next_out = (Bytef *)w.data;
        z.avail_out = (uInt)w.size;
    } while (z.avail_in > 0);

    if (!reading && !had_output)
        /* we received data from our input, but we did not produce any
           output (and we're not looping inside ForceRead()) - to
           avoid stalling the stream, trigger the DeferEvent */
        defer.Schedule();

    return length - (size_t)z.avail_in;
}

void
DeflateIstream::OnEof() noexcept
{
    ClearInput();
    defer.Cancel();

    if (!InitZlib())
        return;

    TryFinish();
}

void
DeflateIstream::OnError(std::exception_ptr ep) noexcept
{
    ClearInput();

    DeinitZlib();

    DestroyError(ep);
}

/*
 * constructor
 *
 */

UnusedIstreamPtr
istream_deflate_new(struct pool &pool, UnusedIstreamPtr input,
                    EventLoop &event_loop, bool gzip) noexcept
{
    return NewIstreamPtr<DeflateIstream>(pool, std::move(input),
                                         event_loop, gzip);

}
<commit_msg>istream/deflate: deinitialize zlib in destructor<commit_after>/*
 * Copyright 2007-2019 Content Management AG
 * All rights reserved.
 *
 * author: Max Kellermann <mk@cm4all.com>
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions
 * are met:
 *
 * - Redistributions of source code must retain the above copyright
 * notice, this list of conditions and the following disclaimer.
 *
 * - Redistributions in binary form must reproduce the above copyright
 * notice, this list of conditions and the following disclaimer in the
 * documentation and/or other materials provided with the
 * distribution.
 *
 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
 * FOR A PARTICULAR PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE
 * FOUNDATION OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
 * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
 * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
 * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED
 * OF THE POSSIBILITY OF SUCH DAMAGE.
 */

#include "istream_deflate.hxx"
#include "UnusedPtr.hxx"
#include "New.hxx"
#include "FacadeIstream.hxx"
#include "pool/pool.hxx"
#include "fb_pool.hxx"
#include "SliceFifoBuffer.hxx"
#include "event/DeferEvent.hxx"
#include "util/Cast.hxx"
#include "util/ConstBuffer.hxx"
#include "util/DestructObserver.hxx"
#include "util/WritableBuffer.hxx"

#include <zlib.h>

#include <stdexcept>

#include <assert.h>

class ZlibError : public std::runtime_error {
    int code;

public:
    explicit ZlibError(int _code, const char *_msg)
        :std::runtime_error(_msg), code(_code) {}

    int GetCode() const noexcept {
        return code;
    }
};

class DeflateIstream final : public FacadeIstream, DestructAnchor {
    const bool gzip;
    bool z_initialized = false, z_stream_end = false;
    z_stream z;
    bool had_input, had_output;
    bool reading;
    SliceFifoBuffer buffer;

    /**
     * This callback is used to request more data from the input if an
     * OnData() call did not produce any output.  This tries to
     * prevent stalling the stream.
     */
    DeferEvent defer;

public:
    DeflateIstream(struct pool &_pool, UnusedIstreamPtr _input,
                   EventLoop &event_loop, bool _gzip)
        :FacadeIstream(_pool, std::move(_input)),
         gzip(_gzip),
         reading(false),
         defer(event_loop, BIND_THIS_METHOD(OnDeferred))
    {
    }

    ~DeflateIstream() {
        defer.Cancel();
        if (z_initialized)
            deflateEnd(&z);
    }

    bool InitZlib() noexcept;

    void Abort(std::exception_ptr ep) noexcept {
        if (HasInput())
            ClearAndCloseInput();

        DestroyError(ep);
    }

    void Abort(int code, const char *msg) noexcept {
        Abort(std::make_exception_ptr(ZlibError(code, msg)));
    }

    /**
     * Submit data from the buffer to our istream handler.
     *
     * @return the number of bytes which were handled, or 0 if the
     * stream was closed
     */
    size_t TryWrite() noexcept;

    /**
     * Starts to write to the buffer.
     *
     * @return a pointer to the writable buffer, or nullptr if there is no
     * room (our istream handler blocks) or if the stream was closed
     */
    WritableBuffer<void> BufferWrite() noexcept {
        buffer.AllocateIfNull(fb_pool_get());
        auto w = buffer.Write();
        if (w.empty() && TryWrite() > 0)
            w = buffer.Write();

        return w.ToVoid();
    }

    void TryFlush() noexcept;

    /**
     * Read from our input until we have submitted some bytes to our
     * istream handler.
     */
    void ForceRead() noexcept;

    void TryFinish() noexcept;

    /* virtual methods from class Istream */

    void _Read() noexcept override {
        if (!buffer.empty())
            TryWrite();
        else if (HasInput())
            ForceRead();
        else
            TryFinish();
    }

    void _Close() noexcept override {
        if (HasInput())
            input.Close();

        Destroy();
    }

    /* virtual methods from class IstreamHandler */
    size_t OnData(const void *data, size_t length) noexcept override;
    void OnEof() noexcept override;
    void OnError(std::exception_ptr ep) noexcept override;

private:
    int GetWindowBits() const noexcept {
        return MAX_WBITS + gzip * 16;
    }

    void OnDeferred() noexcept {
        assert(HasInput());

        ForceRead();
    }
};

static voidpf
z_alloc(voidpf opaque, uInt items, uInt size) noexcept
{
    struct pool *pool = (struct pool *)opaque;

    return p_malloc(pool, items * size);
}

static void
z_free(voidpf opaque, voidpf address) noexcept
{
    (void)opaque;
    (void)address;
}

bool
DeflateIstream::InitZlib() noexcept
{
    if (z_initialized)
        return true;

    z.zalloc = z_alloc;
    z.zfree = z_free;
    z.opaque = &GetPool();

    int err = deflateInit2(&z, Z_DEFAULT_COMPRESSION,
                           Z_DEFLATED, GetWindowBits(), 8,
                           Z_DEFAULT_STRATEGY);
    if (err != Z_OK) {
        Abort(err, "deflateInit2() failed");
        return false;
    }

    z_initialized = true;
    return true;
}

size_t
DeflateIstream::TryWrite() noexcept
{
    auto r = buffer.Read();
    assert(!r.empty());

    size_t nbytes = InvokeData(r.data, r.size);
    if (nbytes == 0)
        return 0;

    buffer.Consume(nbytes);
    buffer.FreeIfEmpty();

    if (nbytes == r.size && !HasInput() && z_stream_end) {
        DestroyEof();
        return 0;
    }

    return nbytes;
}

inline void
DeflateIstream::TryFlush() noexcept
{
    assert(!z_stream_end);

    auto w = BufferWrite();
    if (w.empty())
        return;

    z.next_out = (Bytef *)w.data;
    z.avail_out = (uInt)w.size;

    z.next_in = nullptr;
    z.avail_in = 0;

    int err = deflate(&z, Z_SYNC_FLUSH);
    if (err != Z_OK) {
        Abort(err, "deflate(Z_SYNC_FLUSH) failed");
        return;
    }

    buffer.Append(w.size - (size_t)z.avail_out);

    if (!buffer.empty())
        TryWrite();
}

inline void
DeflateIstream::ForceRead() noexcept
{
    assert(!reading);

    const DestructObserver destructed(*this);

    bool had_input2 = false;
    had_output = false;

    while (1) {
        had_input = false;
        reading = true;
        input.Read();
        if (destructed)
            return;

        reading = false;
        if (!HasInput() || had_output)
            return;

        if (!had_input)
            break;

        had_input2 = true;
    }

    if (had_input2)
        TryFlush();
}

void
DeflateIstream::TryFinish() noexcept
{
    assert(!z_stream_end);

    auto w = BufferWrite();
    if (w.empty())
        return;

    z.next_out = (Bytef *)w.data;
    z.avail_out = (uInt)w.size;

    z.next_in = nullptr;
    z.avail_in = 0;

    int err = deflate(&z, Z_FINISH);
    if (err == Z_STREAM_END)
        z_stream_end = true;
    else if (err != Z_OK) {
        Abort(err, "deflate(Z_FINISH) failed");
        return;
    }

    buffer.Append(w.size - (size_t)z.avail_out);

    if (z_stream_end && buffer.empty()) {
        DestroyEof();
    } else
        TryWrite();
}

/*
 * istream handler
 *
 */

size_t
DeflateIstream::OnData(const void *data, size_t length) noexcept
{
    assert(HasInput());

    auto w = BufferWrite();
    if (w.size < 64) /* reserve space for end-of-stream marker */
        return 0;

    if (!InitZlib())
        return 0;

    had_input = true;

    if (!reading)
        had_output = false;

    z.next_out = (Bytef *)w.data;
    z.avail_out = (uInt)w.size;

    z.next_in = (Bytef *)const_cast<void *>(data);
    z.avail_in = (uInt)length;

    do {
        auto err = deflate(&z, Z_NO_FLUSH);
        if (err != Z_OK) {
            Abort(err, "deflate() failed");
            return 0;
        }

        size_t nbytes = w.size - (size_t)z.avail_out;
        if (nbytes > 0) {
            had_output = true;
            buffer.Append(nbytes);

            const DestructObserver destructed(*this);
            TryWrite();
            if (destructed)
                return 0;
        } else
            break;

        w = BufferWrite();
        if (w.size < 64) /* reserve space for end-of-stream marker */
            break;

        z.next_out = (Bytef *)w.data;
        z.avail_out = (uInt)w.size;
    } while (z.avail_in > 0);

    if (!reading && !had_output)
        /* we received data from our input, but we did not produce any
           output (and we're not looping inside ForceRead()) - to
           avoid stalling the stream, trigger the DeferEvent */
        defer.Schedule();

    return length - (size_t)z.avail_in;
}

void
DeflateIstream::OnEof() noexcept
{
    ClearInput();
    defer.Cancel();

    if (!InitZlib())
        return;

    TryFinish();
}

void
DeflateIstream::OnError(std::exception_ptr ep) noexcept
{
    ClearInput();

    DestroyError(ep);
}

/*
 * constructor
 *
 */

UnusedIstreamPtr
istream_deflate_new(struct pool &pool, UnusedIstreamPtr input,
                    EventLoop &event_loop, bool gzip) noexcept
{
    return NewIstreamPtr<DeflateIstream>(pool, std::move(input),
                                         event_loop, gzip);

}
<|endoftext|>


<commit_before>/***************************************************************
 *
 * Copyright (C) 1990-2007, Condor Team, Computer Sciences Department,
 * University of Wisconsin-Madison, WI.
 *
 * Licensed under the Apache License, Version 2.0 (the "License"); you
 * may not use this file except in compliance with the License.  You may
 * obtain a copy of the License at
 *
 *    http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 *
 ***************************************************************/

#include "condor_common.h"
#include "condor_debug.h"

#include "sysapi.h"
#include "sysapi_externs.h"

#include "condor_sockaddr.h"

static bool net_devices_cached = false;
static std::vector<NetworkDeviceInfo> net_devices_cache;

#if WIN32

#include <Ws2ipdef.h>

bool sysapi_get_network_device_info_raw(std::vector<NetworkDeviceInfo> &devices)
{
	int i,num_interfaces=0,max_interfaces=20;
	LPINTERFACE_INFO interfaces=NULL;
	DWORD bytes_out=0;
	SOCKET sock;

	sock = socket(AF_INET, SOCK_DGRAM, 0);
	if( sock == INVALID_SOCKET ) {
		dprintf(D_ALWAYS,"sysapi_get_network_device_info_raw: socket() failed: (errno %d)\n",
				WSAGetLastError());
		return false;
	}

	while( true ) {
		interfaces = new INTERFACE_INFO[max_interfaces];

		int rc = WSAIoctl(
			sock,
			SIO_GET_INTERFACE_LIST,
			NULL,
			0,
			interfaces,
			sizeof(INTERFACE_INFO)*max_interfaces,
			&bytes_out,
			NULL,
			NULL);

		if( rc == 0 ) { // success
			num_interfaces = bytes_out/sizeof(INTERFACE_INFO);
			break;
		}

		delete [] interfaces;

		int error = WSAGetLastError();
		if( error == WSAEFAULT ) { // not enough space in buffer
			max_interfaces *= 2;
			continue;
		}
		dprintf(D_ALWAYS,"SIO_GET_INTERFACE_LIST failed: %d\n",error);
		closesocket(sock);
		return false;
	}

	for(i=0;i<num_interfaces;i++) {
		char const *ip = NULL;
		if( interfaces[i].iiAddress.Address.sa_family == AF_INET ) {
			ip = inet_ntoa(((struct sockaddr_in *)&interfaces[i].iiAddress)->sin_addr);
		}
		if( ip ) {
			NetworkDeviceInfo inf("",ip);
			devices.push_back(inf);
		}
	}

	delete [] interfaces;
	closesocket(sock);
	return true;
}

#elif HAVE_GETIFADDRS
#include <sys/types.h>
#include <sys/socket.h>
#include <ifaddrs.h>

bool sysapi_get_network_device_info_raw(std::vector<NetworkDeviceInfo> &devices)
{
	struct ifaddrs *ifap_list=NULL;
	if( getifaddrs(&ifap_list) == -1 ) {
		dprintf(D_ALWAYS,"getifaddrs failed: errno=%d: %s\n",errno,strerror(errno));
		return false;
	}
	struct ifaddrs *ifap=ifap_list;
	char ip_buf[INET6_ADDRSTRLEN];
	for(ifap=ifap_list;
		ifap;
		ifap=ifap->ifa_next)
	{
		const char* ip = NULL;
		char const *name = ifap->ifa_name;
		if( ifap->ifa_addr && ifap->ifa_addr->sa_family == AF_INET ) {
			condor_sockaddr addr(ifap->ifa_addr);
			ip = addr.to_ip_string(ip_buf, INET6_ADDRSTRLEN);
		}
		if( ip ) {
			NetworkDeviceInfo inf(name,ip);
			devices.push_back(inf);
		}
	}
	freeifaddrs(ifap_list);

	return true;
}

#elif defined(SIOCGIFCONF)

#include <sys/types.h>
#include <sys/socket.h>
#include <sys/ioctl.h>
#include <net/if.h>

bool sysapi_get_network_device_info_raw(std::vector<NetworkDeviceInfo> &devices)
{
	int sock = socket(AF_INET, SOCK_DGRAM, 0);
	if( sock == -1 ) {
		dprintf(D_ALWAYS,"sysapi_get_network_device_info_raw: socket() failed: (errno %d) %s\n",
				errno,strerror(errno));
		return false;
	}

	struct ifconf ifc,prev;
	memset(&ifc, 0, sizeof(ifc));

	const int max_interfaces = 10000;
	int num_interfaces;
	for(num_interfaces=100;num_interfaces<max_interfaces;num_interfaces+=100) {
		prev.ifc_len = ifc.ifc_len;
		ifc.ifc_len = num_interfaces*sizeof(struct ifreq);

		ifc.ifc_req = (struct ifreq *)malloc(ifc.ifc_len);

		if( ifc.ifc_req == NULL ) {
			dprintf(D_ALWAYS,"sysapi_get_network_device_info_raw: out of memory\n");
			return false;
		}

			// EINVAL (on some platforms) indicates that the buffer
			// is not large enough
		if( ioctl(sock, SIOCGIFCONF, &ifc) < 0 && errno != EINVAL ) {
			dprintf(D_ALWAYS,"sysapi_get_network_device_info_raw: ioctlsocket() failed after allocating buffer: (errno %d) %s\n",
					errno,strerror(errno));
			free( ifc.ifc_req );
			return false;
		}

		if( ifc.ifc_len == prev.ifc_len ) {
				// Getting the same length in successive calls to
				// SIOCGIFCONF is the only reliable way to know that
				// the buffer was big enough, because some
				// implementations silently return truncated results
				// if the buffer isn't big enough.
			break;
		}
		free( ifc.ifc_req );
		ifc.ifc_req = NULL;
	}
	if( num_interfaces > max_interfaces ) {
			// something must be going wrong
		dprintf(D_ALWAYS,"sysapi_get_network_device_info_raw: unexpectedly large SIOCGIFCONF buffer: %d (errno=%d)\n",num_interfaces,errno);
		return false;
	}

	num_interfaces = ifc.ifc_len/sizeof(struct ifreq);

	char ip_buf[INET6_ADDRSTRLEN];
	int i;
	for(i=0; i<num_interfaces; i++) {
		struct ifreq *ifr = &ifc.ifc_req[i];
		char const *name = ifr->ifr_name;
		const char* ip = NULL;

		if( ifr->ifr_addr.sa_family == AF_INET ||ifr->ifr_addr.sa_family == AF_INET6 ) {
			condor_sockaddr addr(&ifr->ifr_addr);
			ip = addr.to_ip_string(ip_buf, INET6_ADDRSTRLEN);
		}
		if( ip ) {
			NetworkDeviceInfo inf(name,ip);
			devices.push_back(inf);
		}
	}
	free( ifc.ifc_req );

	return true;
}

#else

#error sysapi_get_network_device_info() must be implemented for this platform

#endif

bool sysapi_get_network_device_info(std::vector<NetworkDeviceInfo> &devices)
{
	if( net_devices_cached ) {
		devices = net_devices_cache;
		return true;
	}

	bool rc = sysapi_get_network_device_info_raw(devices);

	if( rc ) {
		net_devices_cached = true;
		net_devices_cache = devices;
	}
	return rc;
}

void sysapi_clear_network_device_info_cache() {
	net_devices_cached = false;
}
<commit_msg>sysapi_get_network_device_nifo_raw is now allowed to return IPv6 devices. Not user facing.<commit_after>/***************************************************************
 *
 * Copyright (C) 1990-2007, Condor Team, Computer Sciences Department,
 * University of Wisconsin-Madison, WI.
 *
 * Licensed under the Apache License, Version 2.0 (the "License"); you
 * may not use this file except in compliance with the License.  You may
 * obtain a copy of the License at
 *
 *    http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 *
 ***************************************************************/

#include "condor_common.h"
#include "condor_debug.h"

#include "sysapi.h"
#include "sysapi_externs.h"

#include "condor_sockaddr.h"

static bool net_devices_cached = false;
static std::vector<NetworkDeviceInfo> net_devices_cache;

#if WIN32

#include <Ws2ipdef.h>

bool sysapi_get_network_device_info_raw(std::vector<NetworkDeviceInfo> &devices)
{
	int i,num_interfaces=0,max_interfaces=20;
	LPINTERFACE_INFO interfaces=NULL;
	DWORD bytes_out=0;
	SOCKET sock;

	sock = socket(AF_INET, SOCK_DGRAM, 0);
	if( sock == INVALID_SOCKET ) {
		dprintf(D_ALWAYS,"sysapi_get_network_device_info_raw: socket() failed: (errno %d)\n",
				WSAGetLastError());
		return false;
	}

	while( true ) {
		interfaces = new INTERFACE_INFO[max_interfaces];

		int rc = WSAIoctl(
			sock,
			SIO_GET_INTERFACE_LIST,
			NULL,
			0,
			interfaces,
			sizeof(INTERFACE_INFO)*max_interfaces,
			&bytes_out,
			NULL,
			NULL);

		if( rc == 0 ) { // success
			num_interfaces = bytes_out/sizeof(INTERFACE_INFO);
			break;
		}

		delete [] interfaces;

		int error = WSAGetLastError();
		if( error == WSAEFAULT ) { // not enough space in buffer
			max_interfaces *= 2;
			continue;
		}
		dprintf(D_ALWAYS,"SIO_GET_INTERFACE_LIST failed: %d\n",error);
		closesocket(sock);
		return false;
	}

	for(i=0;i<num_interfaces;i++) {
		char const *ip = NULL;
		if( interfaces[i].iiAddress.Address.sa_family == AF_INET ) {
			ip = inet_ntoa(((struct sockaddr_in *)&interfaces[i].iiAddress)->sin_addr);
		}
		if( ip ) {
			NetworkDeviceInfo inf("",ip);
			devices.push_back(inf);
		}
	}

	delete [] interfaces;
	closesocket(sock);
	return true;
}

#elif HAVE_GETIFADDRS
#include <sys/types.h>
#include <sys/socket.h>
#include <ifaddrs.h>

bool sysapi_get_network_device_info_raw(std::vector<NetworkDeviceInfo> &devices)
{
	struct ifaddrs *ifap_list=NULL;
	if( getifaddrs(&ifap_list) == -1 ) {
		dprintf(D_ALWAYS,"getifaddrs failed: errno=%d: %s\n",errno,strerror(errno));
		return false;
	}
	struct ifaddrs *ifap=ifap_list;
	char ip_buf[INET6_ADDRSTRLEN];
	for(ifap=ifap_list;
		ifap;
		ifap=ifap->ifa_next)
	{
		const char* ip = NULL;
		char const *name = ifap->ifa_name;
		if( ifap->ifa_addr &&
			(ifap->ifa_addr->sa_family == AF_INET  ||
			ifap->ifa_addr->sa_family == AF_INET6)
			) {
			condor_sockaddr addr(ifap->ifa_addr);
			ip = addr.to_ip_string(ip_buf, INET6_ADDRSTRLEN);
		}
		if( ip ) {
			NetworkDeviceInfo inf(name,ip);
			devices.push_back(inf);
		}
	}
	freeifaddrs(ifap_list);

	return true;
}

#elif defined(SIOCGIFCONF)

#include <sys/types.h>
#include <sys/socket.h>
#include <sys/ioctl.h>
#include <net/if.h>

bool sysapi_get_network_device_info_raw(std::vector<NetworkDeviceInfo> &devices)
{
	int sock = socket(AF_INET, SOCK_DGRAM, 0);
	if( sock == -1 ) {
		dprintf(D_ALWAYS,"sysapi_get_network_device_info_raw: socket() failed: (errno %d) %s\n",
				errno,strerror(errno));
		return false;
	}

	struct ifconf ifc,prev;
	memset(&ifc, 0, sizeof(ifc));

	const int max_interfaces = 10000;
	int num_interfaces;
	for(num_interfaces=100;num_interfaces<max_interfaces;num_interfaces+=100) {
		prev.ifc_len = ifc.ifc_len;
		ifc.ifc_len = num_interfaces*sizeof(struct ifreq);

		ifc.ifc_req = (struct ifreq *)malloc(ifc.ifc_len);

		if( ifc.ifc_req == NULL ) {
			dprintf(D_ALWAYS,"sysapi_get_network_device_info_raw: out of memory\n");
			return false;
		}

			// EINVAL (on some platforms) indicates that the buffer
			// is not large enough
		if( ioctl(sock, SIOCGIFCONF, &ifc) < 0 && errno != EINVAL ) {
			dprintf(D_ALWAYS,"sysapi_get_network_device_info_raw: ioctlsocket() failed after allocating buffer: (errno %d) %s\n",
					errno,strerror(errno));
			free( ifc.ifc_req );
			return false;
		}

		if( ifc.ifc_len == prev.ifc_len ) {
				// Getting the same length in successive calls to
				// SIOCGIFCONF is the only reliable way to know that
				// the buffer was big enough, because some
				// implementations silently return truncated results
				// if the buffer isn't big enough.
			break;
		}
		free( ifc.ifc_req );
		ifc.ifc_req = NULL;
	}
	if( num_interfaces > max_interfaces ) {
			// something must be going wrong
		dprintf(D_ALWAYS,"sysapi_get_network_device_info_raw: unexpectedly large SIOCGIFCONF buffer: %d (errno=%d)\n",num_interfaces,errno);
		return false;
	}

	num_interfaces = ifc.ifc_len/sizeof(struct ifreq);

	char ip_buf[INET6_ADDRSTRLEN];
	int i;
	for(i=0; i<num_interfaces; i++) {
		struct ifreq *ifr = &ifc.ifc_req[i];
		char const *name = ifr->ifr_name;
		const char* ip = NULL;

		if( ifr->ifr_addr.sa_family == AF_INET ||ifr->ifr_addr.sa_family == AF_INET6 ) {
			condor_sockaddr addr(&ifr->ifr_addr);
			ip = addr.to_ip_string(ip_buf, INET6_ADDRSTRLEN);
		}
		if( ip ) {
			NetworkDeviceInfo inf(name,ip);
			devices.push_back(inf);
		}
	}
	free( ifc.ifc_req );

	return true;
}

#else

#error sysapi_get_network_device_info() must be implemented for this platform

#endif

bool sysapi_get_network_device_info(std::vector<NetworkDeviceInfo> &devices)
{
	if( net_devices_cached ) {
		devices = net_devices_cache;
		return true;
	}

	bool rc = sysapi_get_network_device_info_raw(devices);

	if( rc ) {
		net_devices_cached = true;
		net_devices_cache = devices;
	}
	return rc;
}

void sysapi_clear_network_device_info_cache() {
	net_devices_cached = false;
}
<|endoftext|>


<commit_before>// Copyright 2018 The Amber Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

#include "src/vulkan/transfer_image.h"

#include <cstring>
#include <limits>
#include <vector>

#include "src/vulkan/command_buffer.h"
#include "src/vulkan/device.h"

namespace amber {
namespace vulkan {
namespace {

const VkImageCreateInfo kDefaultImageInfo = {
    VK_STRUCTURE_TYPE_IMAGE_CREATE_INFO, /* sType */
    nullptr,                             /* pNext */
    0,                                   /* flags */
    VK_IMAGE_TYPE_2D,                    /* imageType */
    VK_FORMAT_R8G8B8A8_UNORM,            /* format */
    {250, 250, 1},                       /* extent */
    1,                                   /* mipLevels */
    1,                                   /* arrayLayers */
    VK_SAMPLE_COUNT_1_BIT,               /* samples */
    VK_IMAGE_TILING_OPTIMAL,             /* tiling */
    0,                                   /* usage */
    VK_SHARING_MODE_EXCLUSIVE,           /* sharingMode */
    0,                                   /* queueFamilyIndexCount */
    nullptr,                             /* pQueueFamilyIndices */
    VK_IMAGE_LAYOUT_UNDEFINED,           /* initialLayout */
};

VkSampleCountFlagBits GetVkSampleCount(uint32_t samples) {
  switch (samples) {
    case 1u:
      return VK_SAMPLE_COUNT_1_BIT;
    case 2u:
      return VK_SAMPLE_COUNT_2_BIT;
    case 4u:
      return VK_SAMPLE_COUNT_4_BIT;
    case 8u:
      return VK_SAMPLE_COUNT_8_BIT;
    case 16u:
      return VK_SAMPLE_COUNT_16_BIT;
    case 32u:
      return VK_SAMPLE_COUNT_32_BIT;
    case 64u:
      return VK_SAMPLE_COUNT_64_BIT;
  }

  return VK_SAMPLE_COUNT_FLAG_BITS_MAX_ENUM;
}

}  // namespace

TransferImage::TransferImage(Device* device,
                             const Format& format,
                             VkImageAspectFlags aspect,
                             VkImageType image_type,
                             uint32_t x,
                             uint32_t y,
                             uint32_t z,
                             uint32_t mip_levels,
                             uint32_t base_mip_level,
                             uint32_t used_mip_levels,
                             uint32_t samples)
    : Resource(device, x * y * z * format.SizeInBytes()),
      image_info_(kDefaultImageInfo),
      aspect_(aspect),
      mip_levels_(mip_levels),
      base_mip_level_(base_mip_level),
      used_mip_levels_(used_mip_levels),
      samples_(samples) {
  image_info_.format = device_->GetVkFormat(format);
  image_info_.imageType = image_type;
  image_info_.extent = {x, y, z};
  image_info_.mipLevels = mip_levels;
  image_info_.samples = GetVkSampleCount(samples);
}

TransferImage::~TransferImage() {
  if (view_ != VK_NULL_HANDLE) {
    device_->GetPtrs()->vkDestroyImageView(device_->GetVkDevice(), view_,
                                           nullptr);
  }

  if (image_ != VK_NULL_HANDLE)
    device_->GetPtrs()->vkDestroyImage(device_->GetVkDevice(), image_, nullptr);

  if (memory_ != VK_NULL_HANDLE)
    device_->GetPtrs()->vkFreeMemory(device_->GetVkDevice(), memory_, nullptr);

  if (host_accessible_memory_ != VK_NULL_HANDLE) {
    UnMapMemory(host_accessible_memory_);
    device_->GetPtrs()->vkFreeMemory(device_->GetVkDevice(),
                                     host_accessible_memory_, nullptr);
  }

  if (host_accessible_buffer_ != VK_NULL_HANDLE) {
    device_->GetPtrs()->vkDestroyBuffer(device_->GetVkDevice(),
                                        host_accessible_buffer_, nullptr);
  }
}

Result TransferImage::Initialize(VkImageUsageFlags usage) {
  if (image_ != VK_NULL_HANDLE)
    return Result("Vulkan::TransferImage was already initialized");

  image_info_.usage = usage;

  if (device_->GetPtrs()->vkCreateImage(device_->GetVkDevice(), &image_info_,
                                        nullptr, &image_) != VK_SUCCESS) {
    return Result("Vulkan::Calling vkCreateImage Fail");
  }

  uint32_t memory_type_index = 0;
  Result r = AllocateAndBindMemoryToVkImage(image_, &memory_,
                                            VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT,
                                            false, &memory_type_index);
  if (!r.IsSuccess())
    return r;

  if (aspect_ & (VK_IMAGE_ASPECT_DEPTH_BIT | VK_IMAGE_ASPECT_STENCIL_BIT) &&
      !(usage & VK_IMAGE_USAGE_DEPTH_STENCIL_ATTACHMENT_BIT)) {
    // Combined depth/stencil image used as a descriptor. Only one aspect can be
    // used for the image view.
    r = CreateVkImageView(VK_IMAGE_ASPECT_DEPTH_BIT);
  } else {
    r = CreateVkImageView(aspect_);
  }

  if (!r.IsSuccess())
    return r;

  // For images, we always make a secondary buffer. When the tiling of an image
  // is optimal, read/write data from CPU does not show correct values. We need
  // a secondary buffer to convert the GPU-optimal data to CPU-readable data
  // and vice versa.
  r = CreateVkBuffer(
      &host_accessible_buffer_,
      VK_BUFFER_USAGE_TRANSFER_SRC_BIT | VK_BUFFER_USAGE_TRANSFER_DST_BIT);
  if (!r.IsSuccess())
    return r;

  memory_type_index = 0;
  r = AllocateAndBindMemoryToVkBuffer(host_accessible_buffer_,
                                      &host_accessible_memory_,
                                      VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT |
                                          VK_MEMORY_PROPERTY_HOST_COHERENT_BIT,
                                      true, &memory_type_index);
  if (!r.IsSuccess())
    return r;

  return MapMemory(host_accessible_memory_);
}

VkImageViewType TransferImage::GetImageViewType() const {
  // TODO(alan-baker): handle other view types.
  // 1D-array, 2D-array, Cube, Cube-array.
  switch (image_info_.imageType) {
    case VK_IMAGE_TYPE_1D:
      return VK_IMAGE_VIEW_TYPE_1D;
    case VK_IMAGE_TYPE_2D:
      return VK_IMAGE_VIEW_TYPE_2D;
    case VK_IMAGE_TYPE_3D:
      return VK_IMAGE_VIEW_TYPE_3D;
    default:
      break;
  }

  // Default to 2D image view.
  return VK_IMAGE_VIEW_TYPE_2D;
}

Result TransferImage::CreateVkImageView(VkImageAspectFlags aspect) {
  VkImageViewCreateInfo image_view_info = VkImageViewCreateInfo();
  image_view_info.sType = VK_STRUCTURE_TYPE_IMAGE_VIEW_CREATE_INFO;
  image_view_info.image = image_;
  image_view_info.viewType = GetImageViewType();
  image_view_info.format = image_info_.format;
  image_view_info.components = {
      VK_COMPONENT_SWIZZLE_R,
      VK_COMPONENT_SWIZZLE_G,
      VK_COMPONENT_SWIZZLE_B,
      VK_COMPONENT_SWIZZLE_A,
  };
  image_view_info.subresourceRange = {
      aspect,           /* aspectMask */
      base_mip_level_,  /* baseMipLevel */
      used_mip_levels_, /* levelCount */
      0,                /* baseArrayLayer */
      1,                /* layerCount */
  };

  if (device_->GetPtrs()->vkCreateImageView(device_->GetVkDevice(),
                                            &image_view_info, nullptr,
                                            &view_) != VK_SUCCESS) {
    return Result("Vulkan::Calling vkCreateImageView Fail");
  }

  return {};
}

VkBufferImageCopy TransferImage::CreateBufferImageCopy(
    VkImageAspectFlags aspect,
    uint32_t mip_level) {
  VkBufferImageCopy copy_region = VkBufferImageCopy();
  if (aspect == VK_IMAGE_ASPECT_STENCIL_BIT) {
    // Store stencil data at the end of the buffer after depth data.
    copy_region.bufferOffset =
        GetSizeInBytes() - image_info_.extent.width * image_info_.extent.height;
  } else {
    copy_region.bufferOffset = 0;
  }
  // Row length of 0 results in tight packing of rows, so the row stride
  // is the number of texels times the texel stride.
  copy_region.bufferRowLength = 0;
  copy_region.bufferImageHeight = 0;
  copy_region.imageSubresource = {
      aspect,    /* aspectMask */
      mip_level, /* mipLevel */
      0,         /* baseArrayLayer */
      1,         /* layerCount */
  };
  copy_region.imageOffset = {0, 0, 0};
  copy_region.imageExtent = {image_info_.extent.width >> mip_level,
                             image_info_.extent.height >> mip_level,
                             image_info_.extent.depth};
  return copy_region;
}

void TransferImage::CopyToHost(CommandBuffer* command_buffer) {
  const VkImageAspectFlagBits aspects[] = {VK_IMAGE_ASPECT_COLOR_BIT,
                                           VK_IMAGE_ASPECT_DEPTH_BIT,
                                           VK_IMAGE_ASPECT_STENCIL_BIT};
  // Copy operations don't support multisample images.
  if (samples_ > 1)
    return;

  std::vector<VkBufferImageCopy> copy_regions;
  uint32_t last_mip_level = used_mip_levels_ == VK_REMAINING_MIP_LEVELS
                                ? mip_levels_
                                : base_mip_level_ + used_mip_levels_;
  for (uint32_t i = base_mip_level_; i < last_mip_level; i++) {
    for (auto aspect : aspects) {
      if (aspect_ & aspect) {
        copy_regions.push_back(CreateBufferImageCopy(aspect, i));
      }
    }
  }

  device_->GetPtrs()->vkCmdCopyImageToBuffer(
      command_buffer->GetVkCommandBuffer(), image_,
      VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL, host_accessible_buffer_,
      static_cast<uint32_t>(copy_regions.size()), copy_regions.data());

  MemoryBarrier(command_buffer);
}

void TransferImage::CopyToDevice(CommandBuffer* command_buffer) {
  // Copy operations don't support multisample images.
  if (samples_ > 1)
    return;

  const VkImageAspectFlagBits aspects[] = {VK_IMAGE_ASPECT_COLOR_BIT,
                                           VK_IMAGE_ASPECT_DEPTH_BIT,
                                           VK_IMAGE_ASPECT_STENCIL_BIT};
  std::vector<VkBufferImageCopy> copy_regions;
  uint32_t last_mip_level = used_mip_levels_ == VK_REMAINING_MIP_LEVELS
                                ? mip_levels_
                                : base_mip_level_ + used_mip_levels_;
  for (uint32_t i = base_mip_level_; i < last_mip_level; i++) {
    for (auto aspect : aspects) {
      if (aspect_ & aspect) {
        copy_regions.push_back(CreateBufferImageCopy(aspect, i));
      }
    }
  }

  device_->GetPtrs()->vkCmdCopyBufferToImage(
      command_buffer->GetVkCommandBuffer(), host_accessible_buffer_, image_,
      VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL,
      static_cast<uint32_t>(copy_regions.size()), copy_regions.data());

  MemoryBarrier(command_buffer);
}

void TransferImage::ImageBarrier(CommandBuffer* command_buffer,
                                 VkImageLayout to_layout,
                                 VkPipelineStageFlags to_stage) {
  if (to_layout == layout_ && to_stage == stage_)
    return;

  VkImageMemoryBarrier barrier = VkImageMemoryBarrier();
  barrier.sType = VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER;
  barrier.oldLayout = layout_;
  barrier.newLayout = to_layout;
  barrier.srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED;
  barrier.dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED;
  barrier.image = image_;
  barrier.subresourceRange = {
      aspect_,                 /* aspectMask */
      0,                       /* baseMipLevel */
      VK_REMAINING_MIP_LEVELS, /* levelCount */
      0,                       /* baseArrayLayer */
      1,                       /* layerCount */
  };

  switch (layout_) {
    case VK_IMAGE_LAYOUT_PREINITIALIZED:
      barrier.srcAccessMask = VK_ACCESS_HOST_WRITE_BIT;
      break;
    case VK_IMAGE_LAYOUT_COLOR_ATTACHMENT_OPTIMAL:
      barrier.srcAccessMask = VK_ACCESS_COLOR_ATTACHMENT_WRITE_BIT;
      break;
    case VK_IMAGE_LAYOUT_DEPTH_STENCIL_ATTACHMENT_OPTIMAL:
      barrier.srcAccessMask = VK_ACCESS_DEPTH_STENCIL_ATTACHMENT_WRITE_BIT;
      break;
    case VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL:
      barrier.srcAccessMask = VK_ACCESS_TRANSFER_WRITE_BIT;
      break;
    case VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL:
      barrier.srcAccessMask = VK_ACCESS_TRANSFER_READ_BIT;
      break;
    default:
      barrier.srcAccessMask = 0;
      break;
  }

  switch (to_layout) {
    case VK_IMAGE_LAYOUT_COLOR_ATTACHMENT_OPTIMAL:
      barrier.dstAccessMask = VK_ACCESS_COLOR_ATTACHMENT_WRITE_BIT;
      break;
    case VK_IMAGE_LAYOUT_DEPTH_STENCIL_ATTACHMENT_OPTIMAL:
      barrier.dstAccessMask = VK_ACCESS_DEPTH_STENCIL_ATTACHMENT_WRITE_BIT;
      break;
    case VK_IMAGE_LAYOUT_DEPTH_STENCIL_READ_ONLY_OPTIMAL:
      barrier.dstAccessMask = VK_ACCESS_DEPTH_STENCIL_ATTACHMENT_READ_BIT;
      break;
    case VK_IMAGE_LAYOUT_SHADER_READ_ONLY_OPTIMAL:
      barrier.dstAccessMask = VK_ACCESS_SHADER_READ_BIT;
      break;
    case VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL:
      barrier.dstAccessMask = VK_ACCESS_TRANSFER_READ_BIT;
      break;
    case VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL:
      barrier.dstAccessMask = VK_ACCESS_TRANSFER_WRITE_BIT;
      break;
    default:
      barrier.dstAccessMask = 0;
      break;
  }

  device_->GetPtrs()->vkCmdPipelineBarrier(command_buffer->GetVkCommandBuffer(),
                                           stage_, to_stage, 0, 0, NULL, 0,
                                           NULL, 1, &barrier);

  layout_ = to_layout;
  stage_ = to_stage;
}

Result TransferImage::AllocateAndBindMemoryToVkImage(
    VkImage image,
    VkDeviceMemory* memory,
    VkMemoryPropertyFlags flags,
    bool force_flags,
    uint32_t* memory_type_index) {
  if (memory_type_index == nullptr) {
    return Result(
        "Vulkan: TransferImage::AllocateAndBindMemoryToVkImage "
        "memory_type_index is "
        "nullptr");
  }

  *memory_type_index = 0;

  if (image == VK_NULL_HANDLE)
    return Result("Vulkan::Given VkImage is VK_NULL_HANDLE");
  if (memory == nullptr)
    return Result("Vulkan::Given VkDeviceMemory pointer is nullptr");

  VkMemoryRequirements requirement;
  device_->GetPtrs()->vkGetImageMemoryRequirements(device_->GetVkDevice(),
                                                   image, &requirement);

  *memory_type_index =
      ChooseMemory(requirement.memoryTypeBits, flags, force_flags);
  if (*memory_type_index == std::numeric_limits<uint32_t>::max())
    return Result("Vulkan::Find Proper Memory Fail");

  Result r = AllocateMemory(memory, requirement.size, *memory_type_index);
  if (!r.IsSuccess())
    return r;

  if (device_->GetPtrs()->vkBindImageMemory(device_->GetVkDevice(), image,
                                            *memory, 0) != VK_SUCCESS) {
    return Result("Vulkan::Calling vkBindImageMemory Fail");
  }

  return {};
}

}  // namespace vulkan
}  // namespace amber
<commit_msg>Fix Vulkan synchronization validation errors (#926)<commit_after>// Copyright 2018 The Amber Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

#include "src/vulkan/transfer_image.h"

#include <cstring>
#include <limits>
#include <vector>

#include "src/vulkan/command_buffer.h"
#include "src/vulkan/device.h"

namespace amber {
namespace vulkan {
namespace {

const VkImageCreateInfo kDefaultImageInfo = {
    VK_STRUCTURE_TYPE_IMAGE_CREATE_INFO, /* sType */
    nullptr,                             /* pNext */
    0,                                   /* flags */
    VK_IMAGE_TYPE_2D,                    /* imageType */
    VK_FORMAT_R8G8B8A8_UNORM,            /* format */
    {250, 250, 1},                       /* extent */
    1,                                   /* mipLevels */
    1,                                   /* arrayLayers */
    VK_SAMPLE_COUNT_1_BIT,               /* samples */
    VK_IMAGE_TILING_OPTIMAL,             /* tiling */
    0,                                   /* usage */
    VK_SHARING_MODE_EXCLUSIVE,           /* sharingMode */
    0,                                   /* queueFamilyIndexCount */
    nullptr,                             /* pQueueFamilyIndices */
    VK_IMAGE_LAYOUT_UNDEFINED,           /* initialLayout */
};

VkSampleCountFlagBits GetVkSampleCount(uint32_t samples) {
  switch (samples) {
    case 1u:
      return VK_SAMPLE_COUNT_1_BIT;
    case 2u:
      return VK_SAMPLE_COUNT_2_BIT;
    case 4u:
      return VK_SAMPLE_COUNT_4_BIT;
    case 8u:
      return VK_SAMPLE_COUNT_8_BIT;
    case 16u:
      return VK_SAMPLE_COUNT_16_BIT;
    case 32u:
      return VK_SAMPLE_COUNT_32_BIT;
    case 64u:
      return VK_SAMPLE_COUNT_64_BIT;
  }

  return VK_SAMPLE_COUNT_FLAG_BITS_MAX_ENUM;
}

}  // namespace

TransferImage::TransferImage(Device* device,
                             const Format& format,
                             VkImageAspectFlags aspect,
                             VkImageType image_type,
                             uint32_t x,
                             uint32_t y,
                             uint32_t z,
                             uint32_t mip_levels,
                             uint32_t base_mip_level,
                             uint32_t used_mip_levels,
                             uint32_t samples)
    : Resource(device, x * y * z * format.SizeInBytes()),
      image_info_(kDefaultImageInfo),
      aspect_(aspect),
      mip_levels_(mip_levels),
      base_mip_level_(base_mip_level),
      used_mip_levels_(used_mip_levels),
      samples_(samples) {
  image_info_.format = device_->GetVkFormat(format);
  image_info_.imageType = image_type;
  image_info_.extent = {x, y, z};
  image_info_.mipLevels = mip_levels;
  image_info_.samples = GetVkSampleCount(samples);
}

TransferImage::~TransferImage() {
  if (view_ != VK_NULL_HANDLE) {
    device_->GetPtrs()->vkDestroyImageView(device_->GetVkDevice(), view_,
                                           nullptr);
  }

  if (image_ != VK_NULL_HANDLE)
    device_->GetPtrs()->vkDestroyImage(device_->GetVkDevice(), image_, nullptr);

  if (memory_ != VK_NULL_HANDLE)
    device_->GetPtrs()->vkFreeMemory(device_->GetVkDevice(), memory_, nullptr);

  if (host_accessible_memory_ != VK_NULL_HANDLE) {
    UnMapMemory(host_accessible_memory_);
    device_->GetPtrs()->vkFreeMemory(device_->GetVkDevice(),
                                     host_accessible_memory_, nullptr);
  }

  if (host_accessible_buffer_ != VK_NULL_HANDLE) {
    device_->GetPtrs()->vkDestroyBuffer(device_->GetVkDevice(),
                                        host_accessible_buffer_, nullptr);
  }
}

Result TransferImage::Initialize(VkImageUsageFlags usage) {
  if (image_ != VK_NULL_HANDLE)
    return Result("Vulkan::TransferImage was already initialized");

  image_info_.usage = usage;

  if (device_->GetPtrs()->vkCreateImage(device_->GetVkDevice(), &image_info_,
                                        nullptr, &image_) != VK_SUCCESS) {
    return Result("Vulkan::Calling vkCreateImage Fail");
  }

  uint32_t memory_type_index = 0;
  Result r = AllocateAndBindMemoryToVkImage(image_, &memory_,
                                            VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT,
                                            false, &memory_type_index);
  if (!r.IsSuccess())
    return r;

  if (aspect_ & (VK_IMAGE_ASPECT_DEPTH_BIT | VK_IMAGE_ASPECT_STENCIL_BIT) &&
      !(usage & VK_IMAGE_USAGE_DEPTH_STENCIL_ATTACHMENT_BIT)) {
    // Combined depth/stencil image used as a descriptor. Only one aspect can be
    // used for the image view.
    r = CreateVkImageView(VK_IMAGE_ASPECT_DEPTH_BIT);
  } else {
    r = CreateVkImageView(aspect_);
  }

  if (!r.IsSuccess())
    return r;

  // For images, we always make a secondary buffer. When the tiling of an image
  // is optimal, read/write data from CPU does not show correct values. We need
  // a secondary buffer to convert the GPU-optimal data to CPU-readable data
  // and vice versa.
  r = CreateVkBuffer(
      &host_accessible_buffer_,
      VK_BUFFER_USAGE_TRANSFER_SRC_BIT | VK_BUFFER_USAGE_TRANSFER_DST_BIT);
  if (!r.IsSuccess())
    return r;

  memory_type_index = 0;
  r = AllocateAndBindMemoryToVkBuffer(host_accessible_buffer_,
                                      &host_accessible_memory_,
                                      VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT |
                                          VK_MEMORY_PROPERTY_HOST_COHERENT_BIT,
                                      true, &memory_type_index);
  if (!r.IsSuccess())
    return r;

  return MapMemory(host_accessible_memory_);
}

VkImageViewType TransferImage::GetImageViewType() const {
  // TODO(alan-baker): handle other view types.
  // 1D-array, 2D-array, Cube, Cube-array.
  switch (image_info_.imageType) {
    case VK_IMAGE_TYPE_1D:
      return VK_IMAGE_VIEW_TYPE_1D;
    case VK_IMAGE_TYPE_2D:
      return VK_IMAGE_VIEW_TYPE_2D;
    case VK_IMAGE_TYPE_3D:
      return VK_IMAGE_VIEW_TYPE_3D;
    default:
      break;
  }

  // Default to 2D image view.
  return VK_IMAGE_VIEW_TYPE_2D;
}

Result TransferImage::CreateVkImageView(VkImageAspectFlags aspect) {
  VkImageViewCreateInfo image_view_info = VkImageViewCreateInfo();
  image_view_info.sType = VK_STRUCTURE_TYPE_IMAGE_VIEW_CREATE_INFO;
  image_view_info.image = image_;
  image_view_info.viewType = GetImageViewType();
  image_view_info.format = image_info_.format;
  image_view_info.components = {
      VK_COMPONENT_SWIZZLE_R,
      VK_COMPONENT_SWIZZLE_G,
      VK_COMPONENT_SWIZZLE_B,
      VK_COMPONENT_SWIZZLE_A,
  };
  image_view_info.subresourceRange = {
      aspect,           /* aspectMask */
      base_mip_level_,  /* baseMipLevel */
      used_mip_levels_, /* levelCount */
      0,                /* baseArrayLayer */
      1,                /* layerCount */
  };

  if (device_->GetPtrs()->vkCreateImageView(device_->GetVkDevice(),
                                            &image_view_info, nullptr,
                                            &view_) != VK_SUCCESS) {
    return Result("Vulkan::Calling vkCreateImageView Fail");
  }

  return {};
}

VkBufferImageCopy TransferImage::CreateBufferImageCopy(
    VkImageAspectFlags aspect,
    uint32_t mip_level) {
  VkBufferImageCopy copy_region = VkBufferImageCopy();
  if (aspect == VK_IMAGE_ASPECT_STENCIL_BIT) {
    // Store stencil data at the end of the buffer after depth data.
    copy_region.bufferOffset =
        GetSizeInBytes() - image_info_.extent.width * image_info_.extent.height;
  } else {
    copy_region.bufferOffset = 0;
  }
  // Row length of 0 results in tight packing of rows, so the row stride
  // is the number of texels times the texel stride.
  copy_region.bufferRowLength = 0;
  copy_region.bufferImageHeight = 0;
  copy_region.imageSubresource = {
      aspect,    /* aspectMask */
      mip_level, /* mipLevel */
      0,         /* baseArrayLayer */
      1,         /* layerCount */
  };
  copy_region.imageOffset = {0, 0, 0};
  copy_region.imageExtent = {image_info_.extent.width >> mip_level,
                             image_info_.extent.height >> mip_level,
                             image_info_.extent.depth};
  return copy_region;
}

void TransferImage::CopyToHost(CommandBuffer* command_buffer) {
  const VkImageAspectFlagBits aspects[] = {VK_IMAGE_ASPECT_COLOR_BIT,
                                           VK_IMAGE_ASPECT_DEPTH_BIT,
                                           VK_IMAGE_ASPECT_STENCIL_BIT};
  // Copy operations don't support multisample images.
  if (samples_ > 1)
    return;

  std::vector<VkBufferImageCopy> copy_regions;
  uint32_t last_mip_level = used_mip_levels_ == VK_REMAINING_MIP_LEVELS
                                ? mip_levels_
                                : base_mip_level_ + used_mip_levels_;
  for (uint32_t i = base_mip_level_; i < last_mip_level; i++) {
    for (auto aspect : aspects) {
      if (aspect_ & aspect) {
        copy_regions.push_back(CreateBufferImageCopy(aspect, i));
      }
    }
  }

  device_->GetPtrs()->vkCmdCopyImageToBuffer(
      command_buffer->GetVkCommandBuffer(), image_,
      VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL, host_accessible_buffer_,
      static_cast<uint32_t>(copy_regions.size()), copy_regions.data());

  MemoryBarrier(command_buffer);
}

void TransferImage::CopyToDevice(CommandBuffer* command_buffer) {
  // Copy operations don't support multisample images.
  if (samples_ > 1)
    return;

  const VkImageAspectFlagBits aspects[] = {VK_IMAGE_ASPECT_COLOR_BIT,
                                           VK_IMAGE_ASPECT_DEPTH_BIT,
                                           VK_IMAGE_ASPECT_STENCIL_BIT};
  std::vector<VkBufferImageCopy> copy_regions;
  uint32_t last_mip_level = used_mip_levels_ == VK_REMAINING_MIP_LEVELS
                                ? mip_levels_
                                : base_mip_level_ + used_mip_levels_;
  for (uint32_t i = base_mip_level_; i < last_mip_level; i++) {
    for (auto aspect : aspects) {
      if (aspect_ & aspect) {
        copy_regions.push_back(CreateBufferImageCopy(aspect, i));
      }
    }
  }

  device_->GetPtrs()->vkCmdCopyBufferToImage(
      command_buffer->GetVkCommandBuffer(), host_accessible_buffer_, image_,
      VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL,
      static_cast<uint32_t>(copy_regions.size()), copy_regions.data());

  MemoryBarrier(command_buffer);
}

void TransferImage::ImageBarrier(CommandBuffer* command_buffer,
                                 VkImageLayout to_layout,
                                 VkPipelineStageFlags to_stage) {
  if (to_layout == layout_ && to_stage == stage_)
    return;

  VkImageMemoryBarrier barrier = VkImageMemoryBarrier();
  barrier.sType = VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER;
  barrier.oldLayout = layout_;
  barrier.newLayout = to_layout;
  barrier.srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED;
  barrier.dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED;
  barrier.image = image_;
  barrier.subresourceRange = {
      aspect_,                 /* aspectMask */
      0,                       /* baseMipLevel */
      VK_REMAINING_MIP_LEVELS, /* levelCount */
      0,                       /* baseArrayLayer */
      1,                       /* layerCount */
  };

  switch (layout_) {
    case VK_IMAGE_LAYOUT_PREINITIALIZED:
      barrier.srcAccessMask = VK_ACCESS_HOST_WRITE_BIT;
      break;
    case VK_IMAGE_LAYOUT_COLOR_ATTACHMENT_OPTIMAL:
      barrier.srcAccessMask = VK_ACCESS_COLOR_ATTACHMENT_WRITE_BIT;
      break;
    case VK_IMAGE_LAYOUT_DEPTH_STENCIL_ATTACHMENT_OPTIMAL:
      barrier.srcAccessMask = VK_ACCESS_DEPTH_STENCIL_ATTACHMENT_WRITE_BIT;
      break;
    case VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL:
      barrier.srcAccessMask = VK_ACCESS_TRANSFER_WRITE_BIT;
      break;
    case VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL:
      barrier.srcAccessMask = VK_ACCESS_TRANSFER_READ_BIT;
      break;
    default:
      barrier.srcAccessMask = 0;
      break;
  }

  switch (to_layout) {
    case VK_IMAGE_LAYOUT_COLOR_ATTACHMENT_OPTIMAL:
      barrier.dstAccessMask = VK_ACCESS_COLOR_ATTACHMENT_READ_BIT |
                              VK_ACCESS_COLOR_ATTACHMENT_WRITE_BIT;
      break;
    case VK_IMAGE_LAYOUT_DEPTH_STENCIL_ATTACHMENT_OPTIMAL:
      barrier.dstAccessMask = VK_ACCESS_DEPTH_STENCIL_ATTACHMENT_READ_BIT |
                              VK_ACCESS_DEPTH_STENCIL_ATTACHMENT_WRITE_BIT;
      break;
    case VK_IMAGE_LAYOUT_DEPTH_STENCIL_READ_ONLY_OPTIMAL:
      barrier.dstAccessMask = VK_ACCESS_DEPTH_STENCIL_ATTACHMENT_READ_BIT;
      break;
    case VK_IMAGE_LAYOUT_SHADER_READ_ONLY_OPTIMAL:
      barrier.dstAccessMask = VK_ACCESS_SHADER_READ_BIT;
      break;
    case VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL:
      barrier.dstAccessMask = VK_ACCESS_TRANSFER_READ_BIT;
      break;
    case VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL:
      barrier.dstAccessMask = VK_ACCESS_TRANSFER_WRITE_BIT;
      break;
    default:
      barrier.dstAccessMask = 0;
      break;
  }

  device_->GetPtrs()->vkCmdPipelineBarrier(command_buffer->GetVkCommandBuffer(),
                                           stage_, to_stage, 0, 0, NULL, 0,
                                           NULL, 1, &barrier);

  layout_ = to_layout;
  stage_ = to_stage;
}

Result TransferImage::AllocateAndBindMemoryToVkImage(
    VkImage image,
    VkDeviceMemory* memory,
    VkMemoryPropertyFlags flags,
    bool force_flags,
    uint32_t* memory_type_index) {
  if (memory_type_index == nullptr) {
    return Result(
        "Vulkan: TransferImage::AllocateAndBindMemoryToVkImage "
        "memory_type_index is "
        "nullptr");
  }

  *memory_type_index = 0;

  if (image == VK_NULL_HANDLE)
    return Result("Vulkan::Given VkImage is VK_NULL_HANDLE");
  if (memory == nullptr)
    return Result("Vulkan::Given VkDeviceMemory pointer is nullptr");

  VkMemoryRequirements requirement;
  device_->GetPtrs()->vkGetImageMemoryRequirements(device_->GetVkDevice(),
                                                   image, &requirement);

  *memory_type_index =
      ChooseMemory(requirement.memoryTypeBits, flags, force_flags);
  if (*memory_type_index == std::numeric_limits<uint32_t>::max())
    return Result("Vulkan::Find Proper Memory Fail");

  Result r = AllocateMemory(memory, requirement.size, *memory_type_index);
  if (!r.IsSuccess())
    return r;

  if (device_->GetPtrs()->vkBindImageMemory(device_->GetVkDevice(), image,
                                            *memory, 0) != VK_SUCCESS) {
    return Result("Vulkan::Calling vkBindImageMemory Fail");
  }

  return {};
}

}  // namespace vulkan
}  // namespace amber
<|endoftext|>


<commit_before>//
//  yas_audio_rendering_connection.cpp
//

#include "yas_audio_rendering_connection.h"

#include "yas_audio_rendering_node.h"

using namespace yas;

audio::rendering_connection::rendering_connection(uint32_t const src_bus_idx, rendering_node const *const src_node,
                                                  audio::format const format)
    : source_bus_idx(src_bus_idx), source_node(src_node), format(std::move(format)) {
}

bool audio::rendering_connection::render(audio::pcm_buffer *const buffer, audio::time const &time) const {
    if (buffer->format() != this->format) {
        return false;
    }

    this->source_node->render_handler()({.buffer = buffer,
                                         .bus_idx = this->source_bus_idx,
                                         .time = time,
                                         .source_connections = this->source_node->source_connections()});

    return true;
}
<commit_msg>fix connection<commit_after>//
//  yas_audio_rendering_connection.cpp
//

#include "yas_audio_rendering_connection.h"

#include "yas_audio_rendering_node.h"

using namespace yas;

audio::rendering_connection::rendering_connection(uint32_t const src_bus_idx, rendering_node const *const src_node,
                                                  audio::format const format)
    : source_bus_idx(src_bus_idx), source_node(src_node), format(std::move(format)) {
}

bool audio::rendering_connection::render(audio::pcm_buffer *const buffer, audio::time const &time) const {
    if (buffer->format() != this->format) {
        return false;
    }

    if (!this->source_node) {
        return false;
    }

    this->source_node->render_handler()({.buffer = buffer,
                                         .bus_idx = this->source_bus_idx,
                                         .time = time,
                                         .source_connections = this->source_node->source_connections()});

    return true;
}
<|endoftext|>


<commit_before>/*******************************************************************************

Licensed to the OpenCOR team under one or more contributor license agreements.
See the NOTICE.txt file distributed with this work for additional information
regarding copyright ownership. The OpenCOR team licenses this file to you under
the Apache License, Version 2.0 (the "License"); you may not use this file
except in compliance with the License. You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software distributed
under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
CONDITIONS OF ANY KIND, either express or implied. See the License for the
specific language governing permissions and limitations under the License.

*******************************************************************************/

//==============================================================================
// Standard file manager
//==============================================================================

#include "corecliutils.h"
#include "filemanager.h"
#include "standardfilemanager.h"
#include "standardsupportplugin.h"

//==============================================================================

namespace OpenCOR {
namespace StandardSupport {

//==============================================================================

StandardFileManager::StandardFileManager() :
    mFiles(Files())
{
    // Create some connections to keep track of some events related to our
    // 'global' file manager

    Core::FileManager *fileManagerInstance = Core::FileManager::instance();

    connect(fileManagerInstance, SIGNAL(fileManaged(const QString &)),
            this, SLOT(manageFile(const QString &)));
    connect(fileManagerInstance, SIGNAL(fileUnmanaged(const QString &)),
            this, SLOT(unmanageFile(const QString &)));

    connect(fileManagerInstance, SIGNAL(fileReloaded(const QString &)),
            this, SLOT(reloadFile(const QString &)));

    connect(fileManagerInstance, SIGNAL(fileRenamed(const QString &, const QString &)),
            this, SLOT(renameFile(const QString &, const QString &)));
}

//==============================================================================

StandardFileManager::~StandardFileManager()
{
    // Remove all the managed files

    foreach (QObject *file, mFiles)
        delete file;
}

//==============================================================================

bool StandardFileManager::isFile(const QString &pFileName)
{
    // If the given file is already managed, then we consider that it's of the
    // right type (e.g. CellML file), even though it may not be of the right
    // type anymore after having been edited and saved, but in this case it's
    // good to keep considering the file as of the right type, so that the user
    // can continue editing it without any problem, for example

    QString nativeFileName = Core::nativeCanonicalFileName(pFileName);

    if (file(nativeFileName))
        return true;

    // The given file is not managed, so do the following:
    //  - Check whether it is a text file and consider it of the right type if,
    //    after having been trimmed, it is empty, or check whether it can be
    //    loaded
    //  - Check whether it is a binary file and consider it of the right type if
    //    it can be loaded
    //  - Consider the file of the right type if it is new

    if (Core::isTextFile(nativeFileName)) {
        QByteArray fileContents;

        if (Core::readFileContentsFromFile(nativeFileName, fileContents)) {
            if (fileContents.trimmed().isEmpty())
                return true;

            return canLoadFile(nativeFileName);
        } else {
            return false;
        }
    } else if (canLoadFile(nativeFileName)) {
        return true;
    } else {
        return Core::FileManager::instance()->isNew(nativeFileName);
    }
}

//==============================================================================

QObject * StandardFileManager::file(const QString &pFileName)
{
    // Return the File object, if any, associated with the given file

    return mFiles.value(Core::nativeCanonicalFileName(pFileName));
}

//==============================================================================

void StandardFileManager::manageFile(const QString &pFileName)
{
    QString nativeFileName = Core::nativeCanonicalFileName(pFileName);

    if (!file(nativeFileName) && isFile(nativeFileName)) {
        // We are dealing with a file, which is not already managed, so we can
        // add it to our list of managed files

        mFiles.insert(nativeFileName, newFile(nativeFileName));
    }
}

//==============================================================================

void StandardFileManager::unmanageFile(const QString &pFileName)
{
    QObject *crtFile = file(pFileName);

    if (crtFile) {
        // We are dealing with a file, so we can remove it from our list of
        // managed files after having deleted it

        delete crtFile;

        mFiles.remove(Core::nativeCanonicalFileName(pFileName));
    }
}

//==============================================================================

void StandardFileManager::reloadFile(const QString &pFileName)
{
    // The file is to be reloaded, so reload it
    // Note: to reload a file here ensures that our different standard-based
    //       views won't each do it, thus saving time and ensuring that a
    //       standard-based view doesn't forget to do it...

    QObject *crtFile = file(pFileName);

    if (crtFile) {
        // The file is managed, but should it still be (i.e. can it still be
        // considered as being a file)?

        if (isFile(pFileName))
            static_cast<StandardFile *>(crtFile)->reload();
        else
            unmanageFile(pFileName);
    } else {
        // The file is not managed, which means that previously it wasn't
        // considered as being a file, but things may be different now, so try
        // to remanage it and load it, if possible

        manageFile(pFileName);

        crtFile = file(pFileName);

        if (crtFile)
            static_cast<StandardFile *>(crtFile)->load();
    }
}

//==============================================================================

void StandardFileManager::renameFile(const QString &pOldFileName,
                                     const QString &pNewFileName)
{
    // The file has been renamed, so we need to update our files mapping, if
    // needed

    QObject *crtFile = file(pOldFileName);

    if (!crtFile)
        return;

    mFiles.insert(pNewFileName, crtFile);
    mFiles.remove(pOldFileName);

    // We also need to ensure that our file object has its file name updated

    static_cast<StandardFile *>(crtFile)->setFileName(pNewFileName);
}

//==============================================================================

}   // namespace StandardSupport
}   // namespace OpenCOR

//==============================================================================
// End of file
//==============================================================================
<commit_msg>Standard file manager: revised StandardFileManager::isFile() so that it doesn't consider new files being necessarily of the right type (useful when duplicating a file) [ci skip].<commit_after>/*******************************************************************************

Licensed to the OpenCOR team under one or more contributor license agreements.
See the NOTICE.txt file distributed with this work for additional information
regarding copyright ownership. The OpenCOR team licenses this file to you under
the Apache License, Version 2.0 (the "License"); you may not use this file
except in compliance with the License. You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software distributed
under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
CONDITIONS OF ANY KIND, either express or implied. See the License for the
specific language governing permissions and limitations under the License.

*******************************************************************************/

//==============================================================================
// Standard file manager
//==============================================================================

#include "corecliutils.h"
#include "filemanager.h"
#include "standardfilemanager.h"
#include "standardsupportplugin.h"

//==============================================================================

namespace OpenCOR {
namespace StandardSupport {

//==============================================================================

StandardFileManager::StandardFileManager() :
    mFiles(Files())
{
    // Create some connections to keep track of some events related to our
    // 'global' file manager

    Core::FileManager *fileManagerInstance = Core::FileManager::instance();

    connect(fileManagerInstance, SIGNAL(fileManaged(const QString &)),
            this, SLOT(manageFile(const QString &)));
    connect(fileManagerInstance, SIGNAL(fileUnmanaged(const QString &)),
            this, SLOT(unmanageFile(const QString &)));

    connect(fileManagerInstance, SIGNAL(fileReloaded(const QString &)),
            this, SLOT(reloadFile(const QString &)));

    connect(fileManagerInstance, SIGNAL(fileRenamed(const QString &, const QString &)),
            this, SLOT(renameFile(const QString &, const QString &)));
}

//==============================================================================

StandardFileManager::~StandardFileManager()
{
    // Remove all the managed files

    foreach (QObject *file, mFiles)
        delete file;
}

//==============================================================================

bool StandardFileManager::isFile(const QString &pFileName)
{
    // If the given file is already managed, then we consider that it's of the
    // right type (e.g. CellML file), even though it may not be of the right
    // type anymore after having been edited and saved, but in this case it's
    // good to keep considering the file as of the right type, so that the user
    // can continue editing it without any problem, for example

    QString nativeFileName = Core::nativeCanonicalFileName(pFileName);

    if (file(nativeFileName))
        return true;

    // The given file is not managed, so consider it of the right type if it is
    // an empty file (after having been trimmed) or it can be loaded

    QByteArray fileContents;

    if (Core::readFileContentsFromFile(nativeFileName, fileContents)) {
        if (fileContents.trimmed().isEmpty())
            return true;

        return canLoadFile(nativeFileName);
    } else {
        return false;
    }
}

//==============================================================================

QObject * StandardFileManager::file(const QString &pFileName)
{
    // Return the File object, if any, associated with the given file

    return mFiles.value(Core::nativeCanonicalFileName(pFileName));
}

//==============================================================================

void StandardFileManager::manageFile(const QString &pFileName)
{
    QString nativeFileName = Core::nativeCanonicalFileName(pFileName);

    if (!file(nativeFileName) && isFile(nativeFileName)) {
        // We are dealing with a file, which is not already managed, so we can
        // add it to our list of managed files

        mFiles.insert(nativeFileName, newFile(nativeFileName));
    }
}

//==============================================================================

void StandardFileManager::unmanageFile(const QString &pFileName)
{
    QObject *crtFile = file(pFileName);

    if (crtFile) {
        // We are dealing with a file, so we can remove it from our list of
        // managed files after having deleted it

        delete crtFile;

        mFiles.remove(Core::nativeCanonicalFileName(pFileName));
    }
}

//==============================================================================

void StandardFileManager::reloadFile(const QString &pFileName)
{
    // The file is to be reloaded, so reload it
    // Note: to reload a file here ensures that our different standard-based
    //       views won't each do it, thus saving time and ensuring that a
    //       standard-based view doesn't forget to do it...

    QObject *crtFile = file(pFileName);

    if (crtFile) {
        // The file is managed, but should it still be (i.e. can it still be
        // considered as being a file)?

        if (isFile(pFileName))
            static_cast<StandardFile *>(crtFile)->reload();
        else
            unmanageFile(pFileName);
    } else {
        // The file is not managed, which means that previously it wasn't
        // considered as being a file, but things may be different now, so try
        // to remanage it and load it, if possible

        manageFile(pFileName);

        crtFile = file(pFileName);

        if (crtFile)
            static_cast<StandardFile *>(crtFile)->load();
    }
}

//==============================================================================

void StandardFileManager::renameFile(const QString &pOldFileName,
                                     const QString &pNewFileName)
{
    // The file has been renamed, so we need to update our files mapping, if
    // needed

    QObject *crtFile = file(pOldFileName);

    if (!crtFile)
        return;

    mFiles.insert(pNewFileName, crtFile);
    mFiles.remove(pOldFileName);

    // We also need to ensure that our file object has its file name updated

    static_cast<StandardFile *>(crtFile)->setFileName(pNewFileName);
}

//==============================================================================

}   // namespace StandardSupport
}   // namespace OpenCOR

//==============================================================================
// End of file
//==============================================================================
<|endoftext|>