<commit_before>"""
ETLT

Copyright 2016 Set Based IT Consultancy

Licence MIT
"""
import abc

from etlt.writer.Writer import Writer

class SqlLoaderWriter(Writer):
    """
    Abstract parent class for loading rows to a table in a database using a SQL statement for loading data from file.
    """
    handlers = {}
    """
    The handlers for writing objects as a field to a CSV file.

    :type dict[str,callable]:
    """

    # ------------------------------------------------------------------------------------------------------------------
    def __init__(self, filename, encoding='utf8'):
        """
        Object constructor.

        :param str filename: The destination file for the rows.
        :param str encoding: The encoding of the text of the destination file.
        """
        Writer.__init__(self)

        self._filename = filename
        """
        The name of the destination file.

        :type str:
        """

        self._encoding = encoding
        """
        The encoding of the text in the destination file.

        :type str:
        """

        self._file = None
        """
        The underling file object.

        :type T:
        """

    # ------------------------------------------------------------------------------------------------------------------
    def __enter__(self):
        self._file = open(self._filename, mode='wt', encoding=self._encoding)

    # ------------------------------------------------------------------------------------------------------------------
    def __exit__(self, exc_type, exc_value, traceback):
        self._file.close()

    # ------------------------------------------------------------------------------------------------------------------
    @abc.abstractmethod
    def get_bulk_load_sql(self, table_name):
        """
        Returns a SQL statement for bulk loading the data writen to the destination file into a table.

        :param str table_name: The name of the table.

        :rtype: str
        """
        raise NotImplementedError()

    # ------------------------------------------------------------------------------------------------------------------
    @staticmethod
    def register_handler(class_name, handler):
        """
        Registers a handler for writing instances of a class as a field to the destination file.

        :param str class_name: The name of the class.
        :param callable handler: The handler. This handler will be called with two arguments: the object which value
                                 must be writen to the destination file, the file handler.
        """
        SqlLoaderWriter.handlers[class_name] = handler

    # ------------------------------------------------------------------------------------------------------------------
    def _write_field(self, value):
        class_name = str(value.__class__)
        if class_name not in self.handlers:
            raise ValueError('No handler has been registered for class: %s' % class_name)
        handler = self.handlers[class_name]
        handler(value, self._file)

# ----------------------------------------------------------------------------------------------------------------------
<commit_msg>Added getters.<commit_after>"""
ETLT

Copyright 2016 Set Based IT Consultancy

Licence MIT
"""
import abc

from etlt.writer.Writer import Writer

class SqlLoaderWriter(Writer):
    """
    Abstract parent class for loading rows to a table in a database using a SQL statement for loading data from file.
    """
    handlers = {}
    """
    The handlers for writing objects as a field to a CSV file.

    :type dict[str,callable]:
    """

    # ------------------------------------------------------------------------------------------------------------------
    def __init__(self, filename, encoding='utf8'):
        """
        Object constructor.

        :param str filename: The destination file for the rows.
        :param str encoding: The encoding of the text of the destination file.
        """
        Writer.__init__(self)

        self._filename = filename
        """
        The name of the destination file.

        :type str:
        """

        self._encoding = encoding
        """
        The encoding of the text in the destination file.

        :type str:
        """

        self._file = None
        """
        The underling file object.

        :type T:
        """

    # ------------------------------------------------------------------------------------------------------------------
    def __enter__(self):
        self._file = open(self._filename, mode='wt', encoding=self._encoding)

    # ------------------------------------------------------------------------------------------------------------------
    def __exit__(self, exc_type, exc_value, traceback):
        self._file.close()

    # ------------------------------------------------------------------------------------------------------------------
    @property
    def filename(self):
        """
        Getter for filename.

        :rtype: list[str]
        """
        return self._filename

    # ------------------------------------------------------------------------------------------------------------------
    @property
    def encoding(self):
        """
        Getter for encoding.

        :rtype: list[str]
        """
        return self._encoding

    # ------------------------------------------------------------------------------------------------------------------
    @abc.abstractmethod
    def get_bulk_load_sql(self, table_name):
        """
        Returns a SQL statement for bulk loading the data writen to the destination file into a table.

        :param str table_name: The name of the table.

        :rtype: str
        """
        raise NotImplementedError()

    # ------------------------------------------------------------------------------------------------------------------
    @staticmethod
    def register_handler(class_name, handler):
        """
        Registers a handler for writing instances of a class as a field to the destination file.

        :param str class_name: The name of the class.
        :param callable handler: The handler. This handler will be called with two arguments: the object which value
                                 must be writen to the destination file, the file handler.
        """
        SqlLoaderWriter.handlers[class_name] = handler

    # ------------------------------------------------------------------------------------------------------------------
    def _write_field(self, value):
        class_name = str(value.__class__)
        if class_name not in self.handlers:
            raise ValueError('No handler has been registered for class: %s' % class_name)
        handler = self.handlers[class_name]
        handler(value, self._file)

# ----------------------------------------------------------------------------------------------------------------------
<|endoftext|>


<commit_before><commit_msg>[BugFix: ZSTACK-43732] further check license body<commit_after><|endoftext|>


<commit_before>""" Test tools module """
from __future__ import division, print_function

import os
from os.path import join as pjoin, split as psplit, abspath, dirname
import shutil

from ..tools import back_tick, ensure_writable, zip2dir, dir2zip

from ..tmpdirs import InTemporaryDirectory

from nose.tools import assert_true, assert_false, assert_equal, assert_raises

def test_back_tick():
    cmd = 'python -c "print(\'Hello\')"'
    assert_equal(back_tick(cmd), "Hello")
    assert_equal(back_tick(cmd, ret_err=True), ("Hello", ""))
    assert_equal(back_tick(cmd, True, False), (b"Hello", b""))
    cmd = 'python -c "raise ValueError()"'
    assert_raises(RuntimeError, back_tick, cmd)

def test_ensure_writable():
    # Test ensure writable decorator
    with InTemporaryDirectory():
        with open('test.bin', 'wt') as fobj:
            fobj.write('A line\n')
        # Set to user rw, else r
        os.chmod('test.bin', 0o644)
        st = os.stat('test.bin')
        @ensure_writable
        def foo(fname):
            pass
        foo('test.bin')
        assert_equal(os.stat('test.bin'), st)
        # No-one can write
        os.chmod('test.bin', 0o444)
        st = os.stat('test.bin')
        foo('test.bin')
        assert_equal(os.stat('test.bin'), st)

def test_zip2():
    # Test utilities to unzip and zip up
    with InTemporaryDirectory():
        os.mkdir('a_dir')
        os.mkdir('zips')
        with open(pjoin('a_dir', 'file1.txt'), 'wt') as fobj:
            fobj.write('File one')
        s_dir = pjoin('a_dir', 's_dir')
        os.mkdir(s_dir)
        with open(pjoin(s_dir, 'file2.txt'), 'wt') as fobj:
            fobj.write('File two')
        zip_fname = pjoin('zips', 'my.zip')
        dir2zip('a_dir', zip_fname)
        zip2dir(zip_fname, 'another_dir')
        assert_equal(os.listdir('another_dir'), ['file1.txt', 's_dir'])
        assert_equal(os.listdir(pjoin('another_dir', 's_dir')), ['file2.txt'])
        # Try zipping from a subdirectory, with a different extension
        dir2zip(s_dir, 'another.ext')
        # Remove original tree just to be sure
        shutil.rmtree('a_dir')
        zip2dir('another.ext', 'third_dir')
        assert_equal(os.listdir('third_dir'), ['file2.txt'])
<commit_msg>RF: add _write_file utility<commit_after>""" Test tools module """
from __future__ import division, print_function

import os
from os.path import join as pjoin, split as psplit, abspath, dirname
import shutil

from ..tools import back_tick, ensure_writable, zip2dir, dir2zip

from ..tmpdirs import InTemporaryDirectory

from nose.tools import assert_true, assert_false, assert_equal, assert_raises

def test_back_tick():
    cmd = 'python -c "print(\'Hello\')"'
    assert_equal(back_tick(cmd), "Hello")
    assert_equal(back_tick(cmd, ret_err=True), ("Hello", ""))
    assert_equal(back_tick(cmd, True, False), (b"Hello", b""))
    cmd = 'python -c "raise ValueError()"'
    assert_raises(RuntimeError, back_tick, cmd)

def test_ensure_writable():
    # Test ensure writable decorator
    with InTemporaryDirectory():
        with open('test.bin', 'wt') as fobj:
            fobj.write('A line\n')
        # Set to user rw, else r
        os.chmod('test.bin', 0o644)
        st = os.stat('test.bin')
        @ensure_writable
        def foo(fname):
            pass
        foo('test.bin')
        assert_equal(os.stat('test.bin'), st)
        # No-one can write
        os.chmod('test.bin', 0o444)
        st = os.stat('test.bin')
        foo('test.bin')
        assert_equal(os.stat('test.bin'), st)

def _write_file(filename, contents):
    with open(filename, 'wt') as fobj:
        fobj.write(contents)

def test_zip2():
    # Test utilities to unzip and zip up
    with InTemporaryDirectory():
        os.mkdir('a_dir')
        os.mkdir('zips')
        _write_file(pjoin('a_dir', 'file1.txt'), 'File one')
        s_dir = pjoin('a_dir', 's_dir')
        os.mkdir(s_dir)
        _write_file(pjoin(s_dir, 'file2.txt'), 'File two')
        zip_fname = pjoin('zips', 'my.zip')
        dir2zip('a_dir', zip_fname)
        zip2dir(zip_fname, 'another_dir')
        assert_equal(os.listdir('another_dir'), ['file1.txt', 's_dir'])
        assert_equal(os.listdir(pjoin('another_dir', 's_dir')), ['file2.txt'])
        # Try zipping from a subdirectory, with a different extension
        dir2zip(s_dir, 'another.ext')
        # Remove original tree just to be sure
        shutil.rmtree('a_dir')
        zip2dir('another.ext', 'third_dir')
        assert_equal(os.listdir('third_dir'), ['file2.txt'])
<|endoftext|>


<commit_before># Copyright 2013 Eucalyptus Systems, Inc.
#
# Redistribution and use of this software in source and binary forms,
# with or without modification, are permitted provided that the following
# conditions are met:
#
#   Redistributions of source code must retain the above copyright notice,
#   this list of conditions and the following disclaimer.
#
#   Redistributions in binary form must reproduce the above copyright
#   notice, this list of conditions and the following disclaimer in the
#   documentation and/or other materials provided with the distribution.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

import hashlib
import multiprocessing
import os
import shutil
import subprocess
import tarfile
import euca2ools.bundle.util
from euca2ools.bundle.util import spawn_process, close_all_fds, pid_exists

def create_bundle_pipeline(infile, outfile, enc_key, enc_iv, tarinfo,
                           debug=False):
    pids = []

    # infile -> tar
    tar_out_r, tar_out_w = euca2ools.bundle.util.open_pipe_fileobjs()
    tar_p = multiprocessing.Process(target=_create_tarball_from_stream,
                                    args=(infile, tar_out_w, tarinfo),
                                    kwargs={'debug': debug})
    tar_p.start()
    pids.append(tar_p.pid)
    infile.close()
    tar_out_w.close()

    # tar -> sha1sum
    digest_out_r, digest_out_w = euca2ools.bundle.util.open_pipe_fileobjs()
    digest_result_r, digest_result_w = multiprocessing.Pipe(duplex=False)
    digest_p = multiprocessing.Process(
        target=_calc_sha1_for_pipe,
        args=(tar_out_r, digest_out_w, digest_result_w))
    digest_p.start()
    pids.append(digest_p.pid)
    digest_out_w.close()
    digest_result_w.close()

    # sha1sum -> gzip
    try:
        gzip = subprocess.Popen(['pigz', '-c'], stdin=digest_out_r,
                                stdout=subprocess.PIPE, close_fds=True,
                                bufsize=-1)
    except OSError:
        gzip = subprocess.Popen(['gzip', '-c'], stdin=digest_out_r,
                                stdout=subprocess.PIPE, close_fds=True,
                                bufsize=-1)
    digest_out_r.close()
    pids.append(gzip.pid)

    # gzip -> openssl
    openssl = subprocess.Popen(['openssl', 'enc', '-e', '-aes-128-cbc',
                                '-K', enc_key, '-iv', enc_iv],
                               stdin=gzip.stdout, stdout=outfile,
                               close_fds=True, bufsize=-1)
    gzip.stdout.close()
    pids.append(openssl.pid)

    # Make sure something calls wait() on every child process
    for pid in pids:
        euca2ools.bundle.util.waitpid_in_thread(pid)

    # Return the connection the caller can use to obtain the final digest
    return digest_result_r

def create_unbundle_pipeline(infile,
                             outfile,
                             enc_key,
                             enc_iv,
                             debug=False):
    """
    Creates a pipeline to perform the unbundle operation on infile input.
    The resulting unbundled image will be written to 'outfile'.

    :param outfile: file  obj to write unbundled image to
    :param enc_key: the encryption key used to bundle the image
    :param enc_iv: the encyrption initialization vector used in the bundle
    :returns multiprocess pipe to read sha1 digest of written image
    """
    openssl = None
    gzip = None
    sha1 = None
    tar = None
    pids = []
    try:
        # infile -> openssl
        openssl = subprocess.Popen(['openssl', 'enc', '-d', '-aes-128-cbc',
                                    '-K', enc_key, '-iv', enc_iv],
                                   stdin=infile, stdout=subprocess.PIPE,
                                   close_fds=True, bufsize=-1)
        pids.append(openssl.pid)
        infile.close()

        # openssl -> gzip
        try:
            gzip = subprocess.Popen(['pigz', '-c', '-d'], stdin=openssl.stdout,
                                    stdout=subprocess.PIPE, close_fds=True,
                                    bufsize=-1)
        except OSError:
            gzip = subprocess.Popen(['gzip', '-c', '-d'], stdin=openssl.stdout,
                                    stdout=subprocess.PIPE, close_fds=True,
                                    bufsize=-1)
        pids.append(gzip.pid)
        openssl.stdout.close()

        #Create pipe file objs to handle i/o...
        sha1_io_r, sha1_io_w = euca2ools.bundle.util.open_pipe_fileobjs()
        sha1_checksum_r, sha1_checksum_w = multiprocessing.Pipe(duplex=False)

        # gzip -> sha1sum
        sha1 = spawn_process(_calc_sha1_for_pipe,
                             infile=gzip.stdout,
                             outfile=sha1_io_w,
                             digest_out_pipe_w=sha1_checksum_w,
                             debug=debug)
        pids.append(sha1.pid)
        gzip.stdout.close()
        sha1_io_w.close()

        # sha1sum -> tar
        tar = spawn_process(_do_tar_extract,
                            infile=sha1_io_r,
                            outfile=outfile,
                            debug=debug)
        pids.append(tar.pid)
        sha1_io_r.close()
        sha1_checksum_w.close()
    except:
        try:
            sha1_checksum_r.close()
        except:
            pass
        raise

    finally:
        # Make sure something calls wait() on every child process
        for pid in pids:
            euca2ools.bundle.util.waitpid_in_thread(pid)
    return sha1_checksum_r

def copy_with_progressbar(infile, outfile, progressbar=None, maxbytes=0):
    """
    Synchronously copy data from infile to outfile, updating a progress bar
    with the total number of bytes copied along the way if one was provided,
    and return the number of bytes copied.

    This method must be run on the main thread.

    :param infile: file obj to read input from
    :param outfile: file obj to write output to
    :param progressbar: progressbar object to update with i/o information
    :param maxbytes: Int maximum number of bytes to write
    """
    bytes_written = 0
    if progressbar:
        progressbar.start()
    try:
        while not infile.closed:
            chunk = infile.read(euca2ools.bundle.pipes._BUFSIZE)
            if chunk:
                if maxbytes and ((bytes_written + len(chunk)) > maxbytes):
                    raise RuntimeError('Amount to be written:{0} will exceed '
                                       'max. Written bytes: {1}/{2}'
                                       .format(len(chunk),
                                               bytes_written,
                                               maxbytes))
                outfile.write(chunk)
                outfile.flush()
                bytes_written += len(chunk)
                if progressbar:
                    progressbar.update(bytes_written)

            else:
                break
    finally:
        if progressbar:
            progressbar.finish()
        infile.close()

def _calc_sha1_for_pipe(infile, outfile, digest_out_pipe_w, debug=False):
    """
    Read data from infile and write it to outfile, calculating a running SHA1
    digest along the way.  When infile hits end-of-file, send the digest in
    hex form to result_mpconn and exit.
    :param infile: file obj providing input for digest
    :param outfile: file obj destination for writing output
    :param digest_out_pipe_w: fileobj to write digest to
    :param debug: boolean used in exception handling
    """
    close_all_fds([infile, outfile, digest_out_pipe_w])
    digest = hashlib.sha1()
    try:
        while True:
            chunk = infile.read(euca2ools.bundle.pipes._BUFSIZE)
            if chunk:
                digest.update(chunk)
                outfile.write(chunk)
                outfile.flush()
            else:
                break
        digest_out_pipe_w.send(digest.hexdigest())
    except IOError:
        # HACK
        if not debug:
            return
        raise
    finally:
        infile.close()
        outfile.close()
        digest_out_pipe_w.close()

def _create_tarball_from_stream(infile, outfile, tarinfo, debug=False):
    close_all_fds(except_fds=[infile, outfile])
    tarball = tarfile.open(mode='w|', fileobj=outfile,
                           bufsize=euca2ools.bundle.pipes._BUFSIZE)
    try:
        tarball.addfile(tarinfo, fileobj=infile)
    except IOError:
        # HACK
        if not debug:
            return
        raise
    finally:
        infile.close()
        tarball.close()
        outfile.close()

def _do_tar_extract(infile, outfile, debug=False):
    """
    Perform tar extract on infile and write to outfile
    :param infile: file obj providing input for tar
    :param outfile: file obj destination for tar output
    :param debug: boolean used in exception handling
    """
    close_all_fds([infile, outfile])
    tarball = tarfile.open(mode='r|', fileobj=infile)
    try:
        tarinfo = tarball.next()
        shutil.copyfileobj(tarball.extractfile(tarinfo), outfile)
    except IOError:
        # HACK
        if not debug:
            return
        raise
    finally:
        infile.close()
        tarball.close()
        outfile.close()
<commit_msg>Actually return a value from copy_with_progressbar<commit_after># Copyright 2013 Eucalyptus Systems, Inc.
#
# Redistribution and use of this software in source and binary forms,
# with or without modification, are permitted provided that the following
# conditions are met:
#
#   Redistributions of source code must retain the above copyright notice,
#   this list of conditions and the following disclaimer.
#
#   Redistributions in binary form must reproduce the above copyright
#   notice, this list of conditions and the following disclaimer in the
#   documentation and/or other materials provided with the distribution.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

import hashlib
import multiprocessing
import os
import shutil
import subprocess
import tarfile
import euca2ools.bundle.util
from euca2ools.bundle.util import spawn_process, close_all_fds, pid_exists

def create_bundle_pipeline(infile, outfile, enc_key, enc_iv, tarinfo,
                           debug=False):
    pids = []

    # infile -> tar
    tar_out_r, tar_out_w = euca2ools.bundle.util.open_pipe_fileobjs()
    tar_p = multiprocessing.Process(target=_create_tarball_from_stream,
                                    args=(infile, tar_out_w, tarinfo),
                                    kwargs={'debug': debug})
    tar_p.start()
    pids.append(tar_p.pid)
    infile.close()
    tar_out_w.close()

    # tar -> sha1sum
    digest_out_r, digest_out_w = euca2ools.bundle.util.open_pipe_fileobjs()
    digest_result_r, digest_result_w = multiprocessing.Pipe(duplex=False)
    digest_p = multiprocessing.Process(
        target=_calc_sha1_for_pipe,
        args=(tar_out_r, digest_out_w, digest_result_w))
    digest_p.start()
    pids.append(digest_p.pid)
    digest_out_w.close()
    digest_result_w.close()

    # sha1sum -> gzip
    try:
        gzip = subprocess.Popen(['pigz', '-c'], stdin=digest_out_r,
                                stdout=subprocess.PIPE, close_fds=True,
                                bufsize=-1)
    except OSError:
        gzip = subprocess.Popen(['gzip', '-c'], stdin=digest_out_r,
                                stdout=subprocess.PIPE, close_fds=True,
                                bufsize=-1)
    digest_out_r.close()
    pids.append(gzip.pid)

    # gzip -> openssl
    openssl = subprocess.Popen(['openssl', 'enc', '-e', '-aes-128-cbc',
                                '-K', enc_key, '-iv', enc_iv],
                               stdin=gzip.stdout, stdout=outfile,
                               close_fds=True, bufsize=-1)
    gzip.stdout.close()
    pids.append(openssl.pid)

    # Make sure something calls wait() on every child process
    for pid in pids:
        euca2ools.bundle.util.waitpid_in_thread(pid)

    # Return the connection the caller can use to obtain the final digest
    return digest_result_r

def create_unbundle_pipeline(infile,
                             outfile,
                             enc_key,
                             enc_iv,
                             debug=False):
    """
    Creates a pipeline to perform the unbundle operation on infile input.
    The resulting unbundled image will be written to 'outfile'.

    :param outfile: file  obj to write unbundled image to
    :param enc_key: the encryption key used to bundle the image
    :param enc_iv: the encyrption initialization vector used in the bundle
    :returns multiprocess pipe to read sha1 digest of written image
    """
    openssl = None
    gzip = None
    sha1 = None
    tar = None
    pids = []
    try:
        # infile -> openssl
        openssl = subprocess.Popen(['openssl', 'enc', '-d', '-aes-128-cbc',
                                    '-K', enc_key, '-iv', enc_iv],
                                   stdin=infile, stdout=subprocess.PIPE,
                                   close_fds=True, bufsize=-1)
        pids.append(openssl.pid)
        infile.close()

        # openssl -> gzip
        try:
            gzip = subprocess.Popen(['pigz', '-c', '-d'], stdin=openssl.stdout,
                                    stdout=subprocess.PIPE, close_fds=True,
                                    bufsize=-1)
        except OSError:
            gzip = subprocess.Popen(['gzip', '-c', '-d'], stdin=openssl.stdout,
                                    stdout=subprocess.PIPE, close_fds=True,
                                    bufsize=-1)
        pids.append(gzip.pid)
        openssl.stdout.close()

        #Create pipe file objs to handle i/o...
        sha1_io_r, sha1_io_w = euca2ools.bundle.util.open_pipe_fileobjs()
        sha1_checksum_r, sha1_checksum_w = multiprocessing.Pipe(duplex=False)

        # gzip -> sha1sum
        sha1 = spawn_process(_calc_sha1_for_pipe,
                             infile=gzip.stdout,
                             outfile=sha1_io_w,
                             digest_out_pipe_w=sha1_checksum_w,
                             debug=debug)
        pids.append(sha1.pid)
        gzip.stdout.close()
        sha1_io_w.close()

        # sha1sum -> tar
        tar = spawn_process(_do_tar_extract,
                            infile=sha1_io_r,
                            outfile=outfile,
                            debug=debug)
        pids.append(tar.pid)
        sha1_io_r.close()
        sha1_checksum_w.close()
    except:
        try:
            sha1_checksum_r.close()
        except:
            pass
        raise

    finally:
        # Make sure something calls wait() on every child process
        for pid in pids:
            euca2ools.bundle.util.waitpid_in_thread(pid)
    return sha1_checksum_r

def copy_with_progressbar(infile, outfile, progressbar=None, maxbytes=0):
    """
    Synchronously copy data from infile to outfile, updating a progress bar
    with the total number of bytes copied along the way if one was provided,
    and return the number of bytes copied.

    This method must be run on the main thread.

    :param infile: file obj to read input from
    :param outfile: file obj to write output to
    :param progressbar: progressbar object to update with i/o information
    :param maxbytes: Int maximum number of bytes to write
    """
    bytes_written = 0
    if progressbar:
        progressbar.start()
    try:
        while not infile.closed:
            chunk = infile.read(euca2ools.bundle.pipes._BUFSIZE)
            if chunk:
                if maxbytes and ((bytes_written + len(chunk)) > maxbytes):
                    raise RuntimeError('Amount to be written:{0} will exceed '
                                       'max. Written bytes: {1}/{2}'
                                       .format(len(chunk),
                                               bytes_written,
                                               maxbytes))
                outfile.write(chunk)
                outfile.flush()
                bytes_written += len(chunk)
                if progressbar:
                    progressbar.update(bytes_written)

            else:
                break
    finally:
        if progressbar:
            progressbar.finish()
        infile.close()
    return bytes_written

def _calc_sha1_for_pipe(infile, outfile, digest_out_pipe_w, debug=False):
    """
    Read data from infile and write it to outfile, calculating a running SHA1
    digest along the way.  When infile hits end-of-file, send the digest in
    hex form to result_mpconn and exit.
    :param infile: file obj providing input for digest
    :param outfile: file obj destination for writing output
    :param digest_out_pipe_w: fileobj to write digest to
    :param debug: boolean used in exception handling
    """
    close_all_fds([infile, outfile, digest_out_pipe_w])
    digest = hashlib.sha1()
    try:
        while True:
            chunk = infile.read(euca2ools.bundle.pipes._BUFSIZE)
            if chunk:
                digest.update(chunk)
                outfile.write(chunk)
                outfile.flush()
            else:
                break
        digest_out_pipe_w.send(digest.hexdigest())
    except IOError:
        # HACK
        if not debug:
            return
        raise
    finally:
        infile.close()
        outfile.close()
        digest_out_pipe_w.close()

def _create_tarball_from_stream(infile, outfile, tarinfo, debug=False):
    close_all_fds(except_fds=[infile, outfile])
    tarball = tarfile.open(mode='w|', fileobj=outfile,
                           bufsize=euca2ools.bundle.pipes._BUFSIZE)
    try:
        tarball.addfile(tarinfo, fileobj=infile)
    except IOError:
        # HACK
        if not debug:
            return
        raise
    finally:
        infile.close()
        tarball.close()
        outfile.close()

def _do_tar_extract(infile, outfile, debug=False):
    """
    Perform tar extract on infile and write to outfile
    :param infile: file obj providing input for tar
    :param outfile: file obj destination for tar output
    :param debug: boolean used in exception handling
    """
    close_all_fds([infile, outfile])
    tarball = tarfile.open(mode='r|', fileobj=infile)
    try:
        tarinfo = tarball.next()
        shutil.copyfileobj(tarball.extractfile(tarinfo), outfile)
    except IOError:
        # HACK
        if not debug:
            return
        raise
    finally:
        infile.close()
        tarball.close()
        outfile.close()
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-

from django.core.management.base import BaseCommand
from django.db import transaction
from django.utils.timezone import now

from django.contrib.webdesign import lorem_ipsum
from django.contrib.contenttypes.models import ContentType

from sampledatahelper.helper import SampleDataHelper

from greenmine.base.users.models import *
from greenmine.projects.models import *
from greenmine.projects.milestones.models import *
from greenmine.projects.userstories.models import *
from greenmine.projects.tasks.models import *
from greenmine.projects.issues.models import *
#from greenmine.projects.questions.models import *
#from greenmine.projects.documents.models import *
from greenmine.projects.wiki.models import *

import random
import datetime

ATTACHMENT_SAMPLE_DATA = [
    "greenmine/projects/management/commands/sample_data",
    [".txt", ]
]

COLOR_CHOICES = [
    "#FC8EAC",
    "#A5694F",
    "#002e33",
    "#67CF00",
    "#71A6D2",
    "#FFF8E7",
    "#4B0082",
    "#007000",
    "#40826D",
    "#708090",
    "#761CEC",
    "#0F0F0F",
    "#D70A53",
    "#CC0000",
    "#FFCC00",
    "#FFFF00",
    "#C0FF33",
    "#B6DA55",
    "#2099DB"]

SUBJECT_CHOICES = [
    "Create the user model",
    "Implement the form",
    "Create the html template",
    "Fixing templates for Django 1.6.",
    "get_actions() does not check for 'delete_selected' in actions",
    "Experimental: modular file types",
    "Add setting to allow regular users to create folders at the root level.",
    "Add tests for bulk operations",
    "Create testsuite with matrix builds",
    "Lighttpd support",
    "Lighttpd x-sendfile support",
    "Added file copying and processing of images (resizing)",
    "Exception is thrown if trying to add a folder with existing name",
    "Feature/improved image admin",
    "Support for bulk actions",
    "Migrate to Python 3 and milk a beautiful cow"]

class Command(BaseCommand):
    sd = SampleDataHelper(seed=12345678901)

    @transaction.atomic
    def handle(self, *args, **options):
        self.users = [User.objects.get(is_superuser=True)]

        # create users
        for x in range(10):
            self.users.append(self.create_user(x))

        # create project
        for x in range(4):
            project = self.create_project(x)

            # added memberships
            computable_project_roles = set()
            for user in self.users:
                role = self.sd.db_object_from_queryset(Role.objects.all())

                Membership.objects.create(
                        project=project,
                        role=role,
                        user=user)

                if role.computable:
                    computable_project_roles.add(role)

            start_date = now() - datetime.timedelta(55)

            # create milestones
            for y in range(self.sd.int(1, 5)):
                end_date = start_date + datetime.timedelta(15)
                milestone = self.create_milestone(project, start_date, end_date)

                # create uss asociated to milestones
                for z in range(self.sd.int(3, 7)):
                    us = self.create_us(project, milestone, computable_project_roles)

                    # create tasks
                    rang = (1, 4) if start_date <= now() and end_date <= now() else (0, 6)
                    for w in range(self.sd.int(*rang)):
                        if start_date <= now() and end_date <= now():
                            task = self.create_task(project, milestone, us, start_date,
                                                    end_date, closed=True)
                        elif start_date <= now() and end_date >= now():
                            task = self.create_task(project, milestone, us, start_date,
                                                    now())
                        else:
                            # No task on not initiated milestones
                            pass

                start_date = end_date

            # created unassociated uss.
            for y in range(self.sd.int(8,15)):
                us = self.create_us(project, None, computable_project_roles)

            # create bugs.
            for y in range(self.sd.int(15,25)):
                bug = self.create_bug(project)

            # create questions.
            #for y in range(self.sd.int(15,25)):
            #    question = self.create_question(project)

            # create a wiki page
            wiki_page = self.create_wiki(project, "home")

    def create_attachment(self, object):
        attachment = Attachment.objects.create(
                project=object.project,
                content_type=ContentType.objects.get_for_model(object.__class__),
                content_object=object,
                object_id=object.id,
                owner=self.sd.db_object_from_queryset(
                          object.project.memberships.all()).user,
                attached_file=self.sd.image_from_directory(*ATTACHMENT_SAMPLE_DATA))

        return attachment

    def create_wiki(self, project, slug):
        wiki_page = WikiPage.objects.create(
                project=project,
                slug=slug,
                content=self.sd.paragraphs(3,15),
                owner=self.sd.db_object_from_queryset(project.memberships.all()).user)

        for i in range(self.sd.int(0, 4)):
            attachment = self.create_attachment(wiki_page)

        return wiki_page

    #def create_question(self, project):
    #    question = Question.objects.create(
    #            project=project,
    #            subject=self.sd.choice(SUBJECT_CHOICES),
    #            content=self.sd.paragraph(),
    #            owner=self.sd.db_object_from_queryset(project.memberships.all()).user,
    #            status=self.sd.db_object_from_queryset(project.question_status.all()),
    #            tags=self.sd.words(1,5).split(" "))
    #
    #    for i in range(self.sd.int(0, 4)):
    #        attachment = self.create_attachment(question)
    #
    #    return question

    def create_bug(self, project):
        bug = Issue.objects.create(
                project=project,
                subject=self.sd.choice(SUBJECT_CHOICES),
                description=self.sd.paragraph(),
                owner=self.sd.db_object_from_queryset(project.memberships.all()).user,
                severity=self.sd.db_object_from_queryset(Severity.objects.filter(
                                                                  project=project)),
                status=self.sd.db_object_from_queryset(IssueStatus.objects.filter(
                                                                   project=project)),
                priority=self.sd.db_object_from_queryset(Priority.objects.filter(
                                                                  project=project)),
                type=self.sd.db_object_from_queryset(IssueType.objects.filter(
                                                               project=project)),
                tags=self.sd.words(1, 5).split(" "))

        for i in range(self.sd.int(0, 4)):
            attachment = self.create_attachment(bug)

        if bug.status.order != 1:
            bug.assigned_to = self.sd.db_object_from_queryset(project.memberships.all()).user
            bug.save()

        return bug

    def create_task(self, project, milestone, us, min_date, max_date, closed=False):
        task = Task(
                subject=self.sd.choice(SUBJECT_CHOICES),
                description=self.sd.paragraph(),
                project=project,
                owner=self.sd.db_object_from_queryset(project.memberships.all()).user,
                milestone=milestone,
                user_story=us,
                finished_date=None,
                assigned_to = self.sd.db_object_from_queryset(project.memberships.all()).user)

        if closed:
            task.status = project.task_statuses.get(order=4)
        else:
            task.status = self.sd.db_object_from_queryset(project.task_statuses.all())

        if task.status.is_closed:
            task.finished_date = self.sd.datetime_between(min_date, max_date)

        task.save()

        for i in range(self.sd.int(0, 4)):
            attachment = self.create_attachment(task)

        return task

    def create_us(self, project, milestone=None, computable_project_roles=list(Role.objects.all())):
        us = UserStory.objects.create(
                subject=self.sd.choice(SUBJECT_CHOICES),
                project=project,
                owner=self.sd.db_object_from_queryset(project.memberships.all()).user,
                description=self.sd.paragraph(),
                milestone=milestone,
                status=self.sd.db_object_from_queryset(project.us_statuses.filter(
                                                                   is_closed=False)),
                tags=self.sd.words(1, 3).split(" ")
        )

        for role_points in us.role_points.filter(role__in=computable_project_roles):
            if milestone:
                role_points.points = self.sd.db_object_from_queryset(
                                us.project.points.exclude(value=None))
            else:
                role_points.points = self.sd.db_object_from_queryset(
                                              us.project.points.all())

            role_points.save()

        for i in range(self.sd.int(0, 4)):
            attachment = self.create_attachment(us)

        return us

    def create_milestone(self, project, start_date, end_date):
        milestone = Milestone.objects.create(
                project=project,
                name='Sprint {0}-{1}-{2}'.format(start_date.year,
                                                 start_date.month,
                                                 start_date.day),
                owner=self.sd.db_object_from_queryset(project.memberships.all()).user,
                created_date=start_date,
                modified_date=start_date,
                estimated_start=start_date,
                estimated_finish=end_date,
                order=10)

        return milestone

    def create_project(self, counter):
        project = Project.objects.create(
                name='Project Example {0}'.format(counter),
                description='Project example {0} description'.format(counter),
                owner=random.choice(self.users),
                public=True,
                total_story_points=self.sd.int(100, 150),
                total_milestones=self.sd.int(5,10))

        return project

    def create_user(self, counter):
        user = User.objects.create(
                username='user-{0}'.format(counter),
                first_name=self.sd.name('es'),
                last_name=self.sd.surname('es'),
                email=self.sd.email(),
                token=''.join(random.sample('abcdef0123456789', 10)),
                color=self.sd.choice(COLOR_CHOICES))

        user.set_password('user{0}'.format(counter))
        user.save()

        return user
<commit_msg>Increased the number of total points of a project for generate more real sample data<commit_after># -*- coding: utf-8 -*-

from django.core.management.base import BaseCommand
from django.db import transaction
from django.utils.timezone import now

from django.contrib.webdesign import lorem_ipsum
from django.contrib.contenttypes.models import ContentType

from sampledatahelper.helper import SampleDataHelper

from greenmine.base.users.models import *
from greenmine.projects.models import *
from greenmine.projects.milestones.models import *
from greenmine.projects.userstories.models import *
from greenmine.projects.tasks.models import *
from greenmine.projects.issues.models import *
#from greenmine.projects.questions.models import *
#from greenmine.projects.documents.models import *
from greenmine.projects.wiki.models import *

import random
import datetime

ATTACHMENT_SAMPLE_DATA = [
    "greenmine/projects/management/commands/sample_data",
    [".txt", ]
]

COLOR_CHOICES = [
    "#FC8EAC",
    "#A5694F",
    "#002e33",
    "#67CF00",
    "#71A6D2",
    "#FFF8E7",
    "#4B0082",
    "#007000",
    "#40826D",
    "#708090",
    "#761CEC",
    "#0F0F0F",
    "#D70A53",
    "#CC0000",
    "#FFCC00",
    "#FFFF00",
    "#C0FF33",
    "#B6DA55",
    "#2099DB"]

SUBJECT_CHOICES = [
    "Create the user model",
    "Implement the form",
    "Create the html template",
    "Fixing templates for Django 1.6.",
    "get_actions() does not check for 'delete_selected' in actions",
    "Experimental: modular file types",
    "Add setting to allow regular users to create folders at the root level.",
    "Add tests for bulk operations",
    "Create testsuite with matrix builds",
    "Lighttpd support",
    "Lighttpd x-sendfile support",
    "Added file copying and processing of images (resizing)",
    "Exception is thrown if trying to add a folder with existing name",
    "Feature/improved image admin",
    "Support for bulk actions",
    "Migrate to Python 3 and milk a beautiful cow"]

class Command(BaseCommand):
    sd = SampleDataHelper(seed=12345678901)

    @transaction.atomic
    def handle(self, *args, **options):
        self.users = [User.objects.get(is_superuser=True)]

        # create users
        for x in range(10):
            self.users.append(self.create_user(x))

        # create project
        for x in range(4):
            project = self.create_project(x)

            # added memberships
            computable_project_roles = set()
            for user in self.users:
                role = self.sd.db_object_from_queryset(Role.objects.all())

                Membership.objects.create(
                        project=project,
                        role=role,
                        user=user)

                if role.computable:
                    computable_project_roles.add(role)

            start_date = now() - datetime.timedelta(55)

            # create milestones
            for y in range(self.sd.int(1, 5)):
                end_date = start_date + datetime.timedelta(15)
                milestone = self.create_milestone(project, start_date, end_date)

                # create uss asociated to milestones
                for z in range(self.sd.int(3, 7)):
                    us = self.create_us(project, milestone, computable_project_roles)

                    # create tasks
                    rang = (1, 4) if start_date <= now() and end_date <= now() else (0, 6)
                    for w in range(self.sd.int(*rang)):
                        if start_date <= now() and end_date <= now():
                            task = self.create_task(project, milestone, us, start_date,
                                                    end_date, closed=True)
                        elif start_date <= now() and end_date >= now():
                            task = self.create_task(project, milestone, us, start_date,
                                                    now())
                        else:
                            # No task on not initiated milestones
                            pass

                start_date = end_date

            # created unassociated uss.
            for y in range(self.sd.int(8,15)):
                us = self.create_us(project, None, computable_project_roles)

            # create bugs.
            for y in range(self.sd.int(15,25)):
                bug = self.create_bug(project)

            # create questions.
            #for y in range(self.sd.int(15,25)):
            #    question = self.create_question(project)

            # create a wiki page
            wiki_page = self.create_wiki(project, "home")

    def create_attachment(self, object):
        attachment = Attachment.objects.create(
                project=object.project,
                content_type=ContentType.objects.get_for_model(object.__class__),
                content_object=object,
                object_id=object.id,
                owner=self.sd.db_object_from_queryset(
                          object.project.memberships.all()).user,
                attached_file=self.sd.image_from_directory(*ATTACHMENT_SAMPLE_DATA))

        return attachment

    def create_wiki(self, project, slug):
        wiki_page = WikiPage.objects.create(
                project=project,
                slug=slug,
                content=self.sd.paragraphs(3,15),
                owner=self.sd.db_object_from_queryset(project.memberships.all()).user)

        for i in range(self.sd.int(0, 4)):
            attachment = self.create_attachment(wiki_page)

        return wiki_page

    #def create_question(self, project):
    #    question = Question.objects.create(
    #            project=project,
    #            subject=self.sd.choice(SUBJECT_CHOICES),
    #            content=self.sd.paragraph(),
    #            owner=self.sd.db_object_from_queryset(project.memberships.all()).user,
    #            status=self.sd.db_object_from_queryset(project.question_status.all()),
    #            tags=self.sd.words(1,5).split(" "))
    #
    #    for i in range(self.sd.int(0, 4)):
    #        attachment = self.create_attachment(question)
    #
    #    return question

    def create_bug(self, project):
        bug = Issue.objects.create(
                project=project,
                subject=self.sd.choice(SUBJECT_CHOICES),
                description=self.sd.paragraph(),
                owner=self.sd.db_object_from_queryset(project.memberships.all()).user,
                severity=self.sd.db_object_from_queryset(Severity.objects.filter(
                                                                  project=project)),
                status=self.sd.db_object_from_queryset(IssueStatus.objects.filter(
                                                                   project=project)),
                priority=self.sd.db_object_from_queryset(Priority.objects.filter(
                                                                  project=project)),
                type=self.sd.db_object_from_queryset(IssueType.objects.filter(
                                                               project=project)),
                tags=self.sd.words(1, 5).split(" "))

        for i in range(self.sd.int(0, 4)):
            attachment = self.create_attachment(bug)

        if bug.status.order != 1:
            bug.assigned_to = self.sd.db_object_from_queryset(project.memberships.all()).user
            bug.save()

        return bug

    def create_task(self, project, milestone, us, min_date, max_date, closed=False):
        task = Task(
                subject=self.sd.choice(SUBJECT_CHOICES),
                description=self.sd.paragraph(),
                project=project,
                owner=self.sd.db_object_from_queryset(project.memberships.all()).user,
                milestone=milestone,
                user_story=us,
                finished_date=None,
                assigned_to = self.sd.db_object_from_queryset(project.memberships.all()).user)

        if closed:
            task.status = project.task_statuses.get(order=4)
        else:
            task.status = self.sd.db_object_from_queryset(project.task_statuses.all())

        if task.status.is_closed:
            task.finished_date = self.sd.datetime_between(min_date, max_date)

        task.save()

        for i in range(self.sd.int(0, 4)):
            attachment = self.create_attachment(task)

        return task

    def create_us(self, project, milestone=None, computable_project_roles=list(Role.objects.all())):
        us = UserStory.objects.create(
                subject=self.sd.choice(SUBJECT_CHOICES),
                project=project,
                owner=self.sd.db_object_from_queryset(project.memberships.all()).user,
                description=self.sd.paragraph(),
                milestone=milestone,
                status=self.sd.db_object_from_queryset(project.us_statuses.filter(
                                                                   is_closed=False)),
                tags=self.sd.words(1, 3).split(" ")
        )

        for role_points in us.role_points.filter(role__in=computable_project_roles):
            if milestone:
                role_points.points = self.sd.db_object_from_queryset(
                                us.project.points.exclude(value=None))
            else:
                role_points.points = self.sd.db_object_from_queryset(
                                              us.project.points.all())

            role_points.save()

        for i in range(self.sd.int(0, 4)):
            attachment = self.create_attachment(us)

        return us

    def create_milestone(self, project, start_date, end_date):
        milestone = Milestone.objects.create(
                project=project,
                name='Sprint {0}-{1}-{2}'.format(start_date.year,
                                                 start_date.month,
                                                 start_date.day),
                owner=self.sd.db_object_from_queryset(project.memberships.all()).user,
                created_date=start_date,
                modified_date=start_date,
                estimated_start=start_date,
                estimated_finish=end_date,
                order=10)

        return milestone

    def create_project(self, counter):
        project = Project.objects.create(
                name='Project Example {0}'.format(counter),
                description='Project example {0} description'.format(counter),
                owner=random.choice(self.users),
                public=True,
                total_story_points=self.sd.int(600, 3000),
                total_milestones=self.sd.int(5,10))

        return project

    def create_user(self, counter):
        user = User.objects.create(
                username='user-{0}'.format(counter),
                first_name=self.sd.name('es'),
                last_name=self.sd.surname('es'),
                email=self.sd.email(),
                token=''.join(random.sample('abcdef0123456789', 10)),
                color=self.sd.choice(COLOR_CHOICES))

        user.set_password('user{0}'.format(counter))
        user.save()

        return user
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
import json

from optparse import make_option

from django.core.management.base import BaseCommand

from django_qbe.forms import QueryByExampleFormSet
from django_qbe.savedqueries.models import SavedQuery
from django_qbe.utils import formats, pickle_decode

class Command(BaseCommand):
    available_formats = ", ".join(formats.keys())
    option_list = BaseCommand.option_list + (
        make_option('--output',
            dest='output',
            default=False,
            help='Export to a file. By default standard output will be used'),
        make_option('--format',
            dest='export_format',
            default="csv",
            help='Format for the returned data: %s' % available_formats),
        make_option('--db-alias',
            dest='db_alias',
            default="default",
            help='Database alias to run the query'),

    )
    args = "query_hash"
    help = "\tExports the results of the query identified by query_hash."
    # can_import_settings = True  # To check db_alias?

    def handle(self, *args, **options):
        # Checking args and options
        if args:
            query_hash = args[0]
        else:
            self.stderr.write(u"Wrong or missing hash code\n")
            return None
        output = options.get("output", None)
        file_name = None
        file_descr = None
        if output:
            file_name = output
            try:
                file_descr = open(file_name, "w")
            except IOError:
                self.stderr.write("Unable to create a file: \"%s\"\n" \
                                  % file_name)
        export_format = options.get("export_format", "csv")
        if export_format not in formats:
            self.stderr.write(u"Wrong format to export: %s\n" % export_format)
            return None
        db_alias = options.get("db_alias", "default")
        saved_queries = SavedQuery.objects.filter(query_hash=query_hash)
        saved_queries_length = len(saved_queries)
        # Making the query
        if saved_queries_length == 1:
            data = saved_queries[0].query_data
            formset = QueryByExampleFormSet(data=data, using=db_alias)
            if formset.is_valid():
                labels = formset.get_labels()
                query = formset.get_raw_query()
                results = formset.get_results(query)
                response = formats[export_format](labels, results)
                if file_descr:
                    file_descr.write(response.content)
                else:
                    self.stdout.write(response.content)
            else:
                self.stderr.write(u"Malformed query: %s\n" % query_hash)
        else:
            self.stderr.write(u"Invalid query_hash, returned %s queries\n" \
                              % saved_queries_length)
        if output:
            file_descr.close()
<commit_msg>use streaming_content with StreamingHttpResponse<commit_after># -*- coding: utf-8 -*-
import json

from optparse import make_option

from django.core.management.base import BaseCommand

from django_qbe.forms import QueryByExampleFormSet
from django_qbe.savedqueries.models import SavedQuery
from django_qbe.utils import formats, pickle_decode

class Command(BaseCommand):
    available_formats = ", ".join(formats.keys())
    option_list = BaseCommand.option_list + (
        make_option('--output',
            dest='output',
            default=False,
            help='Export to a file. By default standard output will be used'),
        make_option('--format',
            dest='export_format',
            default="csv",
            help='Format for the returned data: %s' % available_formats),
        make_option('--db-alias',
            dest='db_alias',
            default="default",
            help='Database alias to run the query'),

    )
    args = "query_hash"
    help = "\tExports the results of the query identified by query_hash."
    # can_import_settings = True  # To check db_alias?

    def handle(self, *args, **options):
        # Checking args and options
        if args:
            query_hash = args[0]
        else:
            self.stderr.write(u"Wrong or missing hash code\n")
            return None
        output = options.get("output", None)
        file_name = None
        file_descr = None
        if output:
            file_name = output
            try:
                file_descr = open(file_name, "w")
            except IOError:
                self.stderr.write("Unable to create a file: \"%s\"\n" \
                                  % file_name)
        export_format = options.get("export_format", "csv")
        if export_format not in formats:
            self.stderr.write(u"Wrong format to export: %s\n" % export_format)
            return None
        db_alias = options.get("db_alias", "default")
        saved_queries = SavedQuery.objects.filter(query_hash=query_hash)
        saved_queries_length = len(saved_queries)
        # Making the query
        if saved_queries_length == 1:
            data = saved_queries[0].query_data
            formset = QueryByExampleFormSet(data=data, using=db_alias)
            if formset.is_valid():
                labels = formset.get_labels()
                query = formset.get_raw_query()
                results = formset.get_results(query)
                response = formats[export_format](labels, results)
                if file_descr:
                    file_descr.write(response.streaming_content)
                else:
                    self.stdout.write(response.streaming_content)
            else:
                self.stderr.write(u"Malformed query: %s\n" % query_hash)
        else:
            self.stderr.write(u"Invalid query_hash, returned %s queries\n" \
                              % saved_queries_length)
        if output:
            file_descr.close()
<|endoftext|>


<commit_before># -*- coding: UTF-8 -*-
from rdflib import Graph, plugin
from rdflib.serializer import Serializer

cases = []
def case(*args): cases.append(args)

case("""@prefix dc: <http://purl.org/dc/terms/> .
<http://example.org/>
    dc:title "Homepage"@en .
""",
{"@vocab": "http://purl.org/dc/terms/", "@language": "en"},
"""
{
    "@context": "/context.jsonld",
    "@id": "http://example.org/",
    "title": "Homepage"
}
""")

case("""@prefix dc: <http://purl.org/dc/terms/> .
<http://example.org/>
    dc:title "Homepage"@en, "Hemsida"@sv .
""",
{"@vocab": "http://purl.org/dc/terms/", "title": {"@container": "@language"}},
"""
{
    "@context": "/context.jsonld",
    "@id": "http://example.org/",
    "title": {
        "en": "Homepage",
        "sv": "Hemsida"
    }
}
""")

case("""@prefix dc: <http://purl.org/dc/terms/> .
<http://example.org/>
    dc:title "Homepage"@en, "Hemsida"@sv .
""",
{"@vocab": "http://purl.org/dc/terms/", "@language": "sv", "title_en": {"@id": "title", "@language": "en"}},
"""
{
    "@context": "/context.jsonld",
    "@id": "http://example.org/",
    "title_en": "Homepage",
    "title": "Hemsida"
}
""")

#    dc:title "Homepage"@en, "Home Page"@en, "Hemsida"@sv .
#    dc:title "Homepage"@en, "Home Page"@en-GB, "Hemsida"@sv .

case("""@prefix dc: <http://purl.org/dc/terms/> .
<http://example.org/easter_island>
    dc:title "Påskön"@sv .
""",
{"@vocab": "http://purl.org/dc/terms/", "@language": "sv"},
"""
{
    "@context": "/context.jsonld",
    "@id": "http://example.org/easter_island",
    "title": "Påskön"
}
""")

case("""
@prefix : <http://example.org/ns#> .
<http://example.org/> :has _:b1 .
""",
{"has": {"@type": "@id", "@id": "http://example.org/ns#has"}},
"""
{
    "@context": "/context.jsonld",
    "@id": "http://example.org",
    "has": "_:b1"
}
""")

case("""
@prefix rdfs: <http://www.w3.org/2000/01/rdf-schema#> .
@prefix : <http://example.org/ns#> .
:Something rdfs:subClassOf :Thing .
""",
{
    "rdfs": "http://www.w3.org/2000/01/rdf-schema#",
    "v": "http://example.org/ns#",
    "rdfs:subClassOf": {"@container": "@set"}
},
"""
{
    "@context": "/context.jsonld",
    "@id": "v:Something",
    "rdfs:subClassOf": [{"@id": "v:Thing"}]
}
""")

case("""
@prefix rdfs: <http://www.w3.org/2000/01/rdf-schema#> .
@prefix : <http://example.org/ns#> .
:Something rdfs:subClassOf :Thing .
""",
{
    "rdfs": "http://www.w3.org/2000/01/rdf-schema#",
    "v": "http://example.org/ns#",
    "subClassOf": {"@id": "rdfs:subClassOf", "@type": "@id", "@container": "@set"}
},
"""
{
    "@context": "/context.jsonld",
    "@id": "v:Something",
    "subClassOf": ["v:Thing"]
}
""")

case("""
@prefix owl: <http://www.w3.org/2002/07/owl#> .
@prefix : <http://example.org/ns#> .
:World owl:unionOf (:Everyhing :Nothing) .
""",
{
    "owl": "http://www.w3.org/2002/07/owl#",
    "v": "http://example.org/ns#"
},
"""
{
    "@context": "/context.jsonld",
    "@id": "v:World",
    "owl:unionOf": {"@list": [{"@id": "v:Everyhing"}, {"@id": "v:Nothing"}]}
}
""")

case("""
@prefix owl: <http://www.w3.org/2002/07/owl#> .
@prefix : <http://example.org/ns#> .
:World owl:unionOf (:Everyhing :Nothing) .
""",
{
    "owl": "http://www.w3.org/2002/07/owl#",
    "v": "http://example.org/ns#",
    "unionOf": {"@id": "owl:unionOf", "@container": "@list"}
},
"""
{
    "@context": "/context.jsonld",
    "@id": "v:World",
    "unionOf": [{"@id": "v:Everyhing"}, {"@id": "v:Nothing"}]
}
""")

case("""
@prefix owl: <http://www.w3.org/2002/07/owl#> .
@prefix : <http://example.org/ns#> .
:World owl:unionOf (:Everyhing :Nothing) .
""",
{
    "owl": "http://www.w3.org/2002/07/owl#",
    "v": "http://example.org/ns#",
    "unionOf": {"@id": "owl:unionOf", "@type": "@id", "@container": "@list"}
},
"""
{
    "@context": "/context.jsonld",
    "@id": "v:World",
    "unionOf": ["v:Everyhing", "v:Nothing"]
}
""")

#def test_cases():
#        yield run, data, context, output
if __name__ == '__main__':
    for data, context, output in cases:
        g = Graph().parse(data=data, format='n3')
        print(g.serialize(format='json-ld', context=context, indent=4))
<commit_msg>Turn test_compaction.py into working tests<commit_after># -*- coding: UTF-8 -*-
from __future__ import unicode_literals
import re
import json
import itertools
from rdflib import Graph
from rdflib.serializer import Serializer

cases = []
def case(*args): cases.append(args)

case("""
@prefix dc: <http://purl.org/dc/terms/> .
<http://example.org/>
    dc:title "Homepage"@en .
""",
{
    "@context": {"@vocab": "http://purl.org/dc/terms/", "@language": "en"},
    "@id": "http://example.org/",
    "title": "Homepage"
}
)

case("""
@prefix dc: <http://purl.org/dc/terms/> .
<http://example.org/>
    dc:title "Homepage"@en, "Hemsida"@sv .
""",
{
    "@context": {"@vocab": "http://purl.org/dc/terms/", "title": {"@container": "@language"}},
    "@id": "http://example.org/",
    "title": {"en": "Homepage", "sv": "Hemsida"}
}
)

case("""
@prefix dc: <http://purl.org/dc/terms/> .
<http://example.org/>
    dc:title "Homepage"@en, "Hemsida"@sv .
""",
{
    "@context": {
        "@vocab": "http://purl.org/dc/terms/",
        "@language": "sv",
        "title_en": {"@id": "title", "@language": "en"}
    },
    "@id": "http://example.org/",
    "title_en": "Homepage",
    "title": "Hemsida"
}
)

# .. Requires set values to be sorted to be predicatble
#case("""
#@prefix dc: <http://purl.org/dc/terms/> .
#<http://example.org/>
#    dc:title "Homepage"@en, "Home Page"@en, "Home Page"@en-GB, "Hemsida"@sv .
#""",
#{
#    "@context": "-||-",
#    "@id": "http://example.org/",
#    "title_en": ["Homepage", "Home Page"],
#    "title": [{"@language": "en-GB", "@value": "Home Page"}, "Hemsida"]
#}
#)

case("""
@prefix dc: <http://purl.org/dc/terms/> .
<http://example.org/easter_island>
    dc:title "Påskön"@sv .
""",
{
    "@context": {"@vocab": "http://purl.org/dc/terms/", "@language": "sv"},
    "@id": "http://example.org/easter_island",
    "title": "Påskön"
}
)

case("""
@prefix : <http://example.org/ns#> .
<http://example.org/> :has _:blank-1 .
""",
{
    "@context": {"has": {"@type": "@id", "@id": "http://example.org/ns#has"}},
    "@id": "http://example.org/",
    "has": "_:blank-1"
}
)

case("""
@prefix rdfs: <http://www.w3.org/2000/01/rdf-schema#> .
@prefix : <http://example.org/ns#> .
:Something rdfs:subClassOf :Thing .
""",
{
    "@context": {
        "rdfs": "http://www.w3.org/2000/01/rdf-schema#",
        "v": "http://example.org/ns#",
        "rdfs:subClassOf": {"@container": "@set"}
    },
    "@id": "v:Something",
    "rdfs:subClassOf": [{"@id": "v:Thing"}]
}
)

case("""
@prefix rdfs: <http://www.w3.org/2000/01/rdf-schema#> .
@prefix : <http://example.org/ns#> .
:Something rdfs:subClassOf :Thing .
""",
{
    "@context": {
        "rdfs": "http://www.w3.org/2000/01/rdf-schema#",
        "v": "http://example.org/ns#",
        "subClassOf": {"@id": "rdfs:subClassOf", "@type": "@id", "@container": "@set"}
    },
    "@id": "v:Something",
    "subClassOf": ["v:Thing"]
}
)

case("""
@prefix owl: <http://www.w3.org/2002/07/owl#> .
@prefix : <http://example.org/ns#> .
:World owl:unionOf (:Everyhing :Nothing) .
""",
{
    "@context": {
        "owl": "http://www.w3.org/2002/07/owl#",
        "v": "http://example.org/ns#"
    },
    "@id": "v:World",
    "owl:unionOf": {"@list": [{"@id": "v:Everyhing"}, {"@id": "v:Nothing"}]}
}
)

case("""
@prefix owl: <http://www.w3.org/2002/07/owl#> .
@prefix : <http://example.org/ns#> .
:World owl:unionOf (:Everyhing :Nothing) .
""",
{
    "@context": {
        "owl": "http://www.w3.org/2002/07/owl#",
        "v": "http://example.org/ns#",
        "unionOf": {"@id": "owl:unionOf", "@container": "@list"}
    },
    "@id": "v:World",
    "unionOf": [{"@id": "v:Everyhing"}, {"@id": "v:Nothing"}]
}
)

case("""
@prefix owl: <http://www.w3.org/2002/07/owl#> .
@prefix : <http://example.org/ns#> .
:World owl:unionOf (:Everyhing :Nothing) .
""",
{
    "@context": {
        "owl": "http://www.w3.org/2002/07/owl#",
        "v": "http://example.org/ns#",
        "unionOf": {"@id": "owl:unionOf", "@type": "@id", "@container": "@list"}
    },
    "@id": "v:World",
    "unionOf": ["v:Everyhing", "v:Nothing"]
}
)

json_kwargs = dict(indent=2, separators=(',', ': '), sort_keys=True, ensure_ascii=False)

def run(data, expected):
    context = expected['@context']
    g = Graph().parse(data=data, format='turtle')
    result = g.serialize(format='json-ld', context=context, **json_kwargs)
    incr = itertools.count(1)
    result = re.sub(r'"_:[^"]+"', lambda m: '"_:blank-%s"' % incr.next(), result)
    expected = json.dumps(expected, **json_kwargs).encode('utf-8')
    assert result == expected

def test_cases():
    for data, expected in cases:
        yield run, data, expected
<|endoftext|>


<commit_before>import os
import sys
import csv
import json
import uuid
import importlib
import datetime
import unicodecsv
from time import time
from copy import deepcopy
from optparse import make_option
from os.path import isfile, join
from django.db import connection, transaction
from django.contrib.auth.models import User
from django.forms.models import model_to_dict
from django.core.management.base import BaseCommand, CommandError
from django.core.exceptions import ValidationError
from arches.app.datatypes.datatypes import DataTypeFactory
from arches.app.models.entity import Entity
from arches.app.models.resource import Resource
from arches.app.models.tile import Tile
from arches.app.models.models import DDataType
from arches.app.models.models import ResourceInstance
from arches.app.models.models import FunctionXGraph
from arches.app.models.models import ResourceXResource
from arches.app.models.models import NodeGroup
from arches.app.models.models import ResourceXResource
from arches.app.models.models import Concept
from arches.app.models.models import Value
from arches.app.models.models import ResourceXResource
from arches.app.models.concept import Concept
from arches.app.models.system_settings import settings
from arches.app.utils.betterJSONSerializer import JSONSerializer, JSONDeserializer
from arches.app.search.search_engine_factory import SearchEngineFactory
from arches.management.commands import utils
from formats.archesjson import JsonReader
from formats.csvfile import CsvReader
from formats.archesfile import ArchesFileReader

class BusinessDataImporter(object):

    def __init__(self, file=None, mapping_file=None, relations_file=None):
        self.business_data = ''
        self.mapping = None
        self.graphs = ''
        self.reference_data = ''
        self.business_data = ''
        self.file_format = ''
        self.relations = ''
        csv.field_size_limit(sys.maxint)

        if not file:
            file = settings.BUSINESS_DATA_FILES
        else:
            file = [file]

        if mapping_file == None:
            try:
                mapping_file = [file[0].split('.')[0] + '.mapping']
            except:
                print '*'*80
                print "ERROR: Mapping file is missing or improperly named. Make sure you have mapping file with the same basename as your business data file and the extension .mapping"
                print '*'*80
                sys.exit()
        else:
            try:
                mapping_file = [mapping_file]
            except:
                print '*'*80
                print "ERROR: Mapping file is missing or improperly named. Make sure you have mapping file with the same basename as your business data file and the extension .mapping"
                print '*'*80
                sys.exit()

        if relations_file == None:
            try:
                relations_file = [file[0].split('.')[0] + '.relations']
            except:
                pass

        for path in relations_file:
            if os.path.exists(path):
                if isfile(join(path)):
                    self.relations = csv.DictReader(open(relations_file[0], 'r'))

        for path in mapping_file:
            if os.path.exists(path):
                if isfile(join(path)):
                    self.mapping = json.load(open(path, 'r'))
                else:
                    self.mapping = None

        for path in file:
            if os.path.exists(path):
                if isfile(join(path)):
                    self.file_format = file[0].split('.')[1]
                    if self.file_format == 'json':
                        with open(file[0], 'rU') as f:
                            archesfile = JSONDeserializer().deserialize(f)
                            if 'graph' in archesfile.keys():
                                self.graphs = archesfile['graph']
                            if 'reference_data' in archesfile.keys():
                                self.reference_data = archesfile['reference_data']
                            if 'business_data' in archesfile.keys():
                                self.business_data = archesfile['business_data']
                    elif self.file_format == 'csv':
                        data = unicodecsv.DictReader(open(file[0], 'r'), encoding='utf-8-sig', restkey='ADDITIONAL', restval='MISSING')
                        self.business_data = list(data)
                else:
                    print str(file) + ' is not a valid file'
            else:
                print path + ' is not a valid path'

    def import_business_data(self, file_format=None, business_data=None, mapping=None, overwrite='append', bulk=False):
        reader = None
        start = time()
        cursor = connection.cursor()

        try:
            if file_format == None:
                file_format = self.file_format
            if business_data == None:
                business_data = self.business_data
            if mapping == None:
                mapping = self.mapping
            if file_format == 'json':
                reader = ArchesFileReader()
                reader.import_business_data(business_data, mapping)
            elif file_format == 'csv':
                if mapping != None:
                    reader = CsvReader()
                    reader.import_business_data(business_data=business_data, mapping=mapping, overwrite=overwrite, bulk=bulk)
                else:
                    print '*'*80
                    print 'ERROR: No mapping file detected. Please indicate one with the \'-c\' paramater or place one in the same directory as your business data.'
                    print '*'*80
                    sys.exit()

            elapsed = (time() - start)
            print 'Time to import_business_data = {0}'.format(datetime.timedelta(seconds=elapsed))

            reader.report_errors()

        finally:
            datatype_factory = DataTypeFactory()
            datatypes = DDataType.objects.all()
            for datatype in datatypes:
                datatype_instance = datatype_factory.get_instance(datatype.datatype)
                datatype_instance.after_update_all()

class ResourceLoader(object):

    def __init__(self):
        self.user = User()
        self.user.first_name = settings.ETL_USERNAME
        self.resources = []
        self.se = SearchEngineFactory().create()

    option_list = BaseCommand.option_list + (
        make_option('--source',
            action='store',
            dest='source',
            default='',
            help='.arches file containing resource records'),
         make_option('--format',
            action='store_true',
            default='arches',
            help='format extension that you would like to load: arches or shp'),
        )

    def load(self, source):
        file_name, file_format = os.path.splitext(source)
        archesjson = False
        if file_format == '.arches':
            reader = ArchesReader()
            print '\nVALIDATING ARCHES FILE ({0})'.format(source)
            reader.validate_file(source)
        elif file_format == '.json':
            archesjson = True
            reader = JsonReader()

        start = time()
        resources = reader.load_file(source)

        print '\nLOADING RESOURCES ({0})'.format(source)
        relationships = None
        related_resource_records = []
        relationships_file = file_name + '.relations'
        elapsed = (time() - start)
        print 'time to parse {0} resources = {1}'.format(file_name, elapsed)
        results = self.resource_list_to_entities(resources, archesjson)
        if os.path.exists(relationships_file):
            relationships = csv.DictReader(open(relationships_file, 'r'), delimiter='|')
            for relationship in relationships:
                related_resource_records.append(self.relate_resources(relationship, results['legacyid_to_entityid'], archesjson))
        else:
            print 'No relationship file'

    # def relate_resources(self, relationship, legacyid_to_entityid, archesjson):
    #     start_date = None if relationship['START_DATE'] in ('', 'None') else relationship['START_DATE']
    #     end_date = None if relationship['END_DATE'] in ('', 'None') else relationship['END_DATE']
    #
    #     if archesjson == False:
    #         relationshiptype_concept = Concept.objects.get(legacyoid = relationship['RELATION_TYPE'])
    #         concept_value = Value.objects.filter(concept = relationshiptype_concept.conceptid).filter(valuetype = 'prefLabel')
    #         entityid1 = legacyid_to_entityid[relationship['RESOURCEID_FROM']]
    #         entityid2 = legacyid_to_entityid[relationship['RESOURCEID_TO']]
    #
    #     else:
    #         concept_value = Value.objects.filter(valueid = relationship['RELATION_TYPE'])
    #         entityid1 = relationship['RESOURCEID_FROM']
    #         entityid2 = relationship['RESOURCEID_TO']
    #
    #     related_resource_record = ResourceXResource(
    #         entityid1 = entityid1,
    #         entityid2 = entityid2,
    #         notes = relationship['NOTES'],
    #         relationshiptype = concept_value[0].valueid,
    #         datestarted = start_date,
    #         dateended = end_date,
    #         )
    #
    #     related_resource_record.save()
    #     self.se.index_data(index='resource_relations', doc_type='all', body=model_to_dict(related_resource_record), idfield='resourcexid')
<commit_msg>modified index of imported files to ensure that the last element of the split filename returns the extension<commit_after>import os
import sys
import csv
import json
import uuid
import importlib
import datetime
import unicodecsv
from time import time
from copy import deepcopy
from optparse import make_option
from os.path import isfile, join
from django.db import connection, transaction
from django.contrib.auth.models import User
from django.forms.models import model_to_dict
from django.core.management.base import BaseCommand, CommandError
from django.core.exceptions import ValidationError
from arches.app.datatypes.datatypes import DataTypeFactory
from arches.app.models.entity import Entity
from arches.app.models.resource import Resource
from arches.app.models.tile import Tile
from arches.app.models.models import DDataType
from arches.app.models.models import ResourceInstance
from arches.app.models.models import FunctionXGraph
from arches.app.models.models import ResourceXResource
from arches.app.models.models import NodeGroup
from arches.app.models.models import ResourceXResource
from arches.app.models.models import Concept
from arches.app.models.models import Value
from arches.app.models.models import ResourceXResource
from arches.app.models.concept import Concept
from arches.app.models.system_settings import settings
from arches.app.utils.betterJSONSerializer import JSONSerializer, JSONDeserializer
from arches.app.search.search_engine_factory import SearchEngineFactory
from arches.management.commands import utils
from formats.archesjson import JsonReader
from formats.csvfile import CsvReader
from formats.archesfile import ArchesFileReader

class BusinessDataImporter(object):

    def __init__(self, file=None, mapping_file=None, relations_file=None):
        self.business_data = ''
        self.mapping = None
        self.graphs = ''
        self.reference_data = ''
        self.business_data = ''
        self.file_format = ''
        self.relations = ''
        csv.field_size_limit(sys.maxint)

        if not file:
            file = settings.BUSINESS_DATA_FILES
        else:
            file = [file]

        if mapping_file == None:
            try:
                mapping_file = [file[0].split('.')[0] + '.mapping']
            except:
                print '*'*80
                print "ERROR: Mapping file is missing or improperly named. Make sure you have mapping file with the same basename as your business data file and the extension .mapping"
                print '*'*80
                sys.exit()
        else:
            try:
                mapping_file = [mapping_file]
            except:
                print '*'*80
                print "ERROR: Mapping file is missing or improperly named. Make sure you have mapping file with the same basename as your business data file and the extension .mapping"
                print '*'*80
                sys.exit()

        if relations_file == None:
            try:
                relations_file = [file[0].split('.')[0] + '.relations']
            except:
                pass

        for path in relations_file:
            if os.path.exists(path):
                if isfile(join(path)):
                    self.relations = csv.DictReader(open(relations_file[0], 'r'))

        for path in mapping_file:
            if os.path.exists(path):
                if isfile(join(path)):
                    self.mapping = json.load(open(path, 'r'))
                else:
                    self.mapping = None

        for path in file:
            if os.path.exists(path):
                if isfile(join(path)):
                    self.file_format = file[0].split('.')[-1]
                    if self.file_format == 'json':
                        with open(file[0], 'rU') as f:
                            archesfile = JSONDeserializer().deserialize(f)
                            if 'graph' in archesfile.keys():
                                self.graphs = archesfile['graph']
                            if 'reference_data' in archesfile.keys():
                                self.reference_data = archesfile['reference_data']
                            if 'business_data' in archesfile.keys():
                                self.business_data = archesfile['business_data']
                    elif self.file_format == 'csv':
                        data = unicodecsv.DictReader(open(file[0], 'r'), encoding='utf-8-sig', restkey='ADDITIONAL', restval='MISSING')
                        self.business_data = list(data)
                else:
                    print str(file) + ' is not a valid file'
            else:
                print path + ' is not a valid path'

    def import_business_data(self, file_format=None, business_data=None, mapping=None, overwrite='append', bulk=False):
        reader = None
        start = time()
        cursor = connection.cursor()

        try:
            if file_format == None:
                file_format = self.file_format
            if business_data == None:
                business_data = self.business_data
            if mapping == None:
                mapping = self.mapping
            if file_format == 'json':
                reader = ArchesFileReader()
                reader.import_business_data(business_data, mapping)
            elif file_format == 'csv':
                if mapping != None:
                    reader = CsvReader()
                    reader.import_business_data(business_data=business_data, mapping=mapping, overwrite=overwrite, bulk=bulk)
                else:
                    print '*'*80
                    print 'ERROR: No mapping file detected. Please indicate one with the \'-c\' paramater or place one in the same directory as your business data.'
                    print '*'*80
                    sys.exit()

            elapsed = (time() - start)
            print 'Time to import_business_data = {0}'.format(datetime.timedelta(seconds=elapsed))

            reader.report_errors()

        finally:
            datatype_factory = DataTypeFactory()
            datatypes = DDataType.objects.all()
            for datatype in datatypes:
                datatype_instance = datatype_factory.get_instance(datatype.datatype)
                datatype_instance.after_update_all()

class ResourceLoader(object):

    def __init__(self):
        self.user = User()
        self.user.first_name = settings.ETL_USERNAME
        self.resources = []
        self.se = SearchEngineFactory().create()

    option_list = BaseCommand.option_list + (
        make_option('--source',
            action='store',
            dest='source',
            default='',
            help='.arches file containing resource records'),
         make_option('--format',
            action='store_true',
            default='arches',
            help='format extension that you would like to load: arches or shp'),
        )

    def load(self, source):
        file_name, file_format = os.path.splitext(source)
        archesjson = False
        if file_format == '.arches':
            reader = ArchesReader()
            print '\nVALIDATING ARCHES FILE ({0})'.format(source)
            reader.validate_file(source)
        elif file_format == '.json':
            archesjson = True
            reader = JsonReader()

        start = time()
        resources = reader.load_file(source)

        print '\nLOADING RESOURCES ({0})'.format(source)
        relationships = None
        related_resource_records = []
        relationships_file = file_name + '.relations'
        elapsed = (time() - start)
        print 'time to parse {0} resources = {1}'.format(file_name, elapsed)
        results = self.resource_list_to_entities(resources, archesjson)
        if os.path.exists(relationships_file):
            relationships = csv.DictReader(open(relationships_file, 'r'), delimiter='|')
            for relationship in relationships:
                related_resource_records.append(self.relate_resources(relationship, results['legacyid_to_entityid'], archesjson))
        else:
            print 'No relationship file'

    # def relate_resources(self, relationship, legacyid_to_entityid, archesjson):
    #     start_date = None if relationship['START_DATE'] in ('', 'None') else relationship['START_DATE']
    #     end_date = None if relationship['END_DATE'] in ('', 'None') else relationship['END_DATE']
    #
    #     if archesjson == False:
    #         relationshiptype_concept = Concept.objects.get(legacyoid = relationship['RELATION_TYPE'])
    #         concept_value = Value.objects.filter(concept = relationshiptype_concept.conceptid).filter(valuetype = 'prefLabel')
    #         entityid1 = legacyid_to_entityid[relationship['RESOURCEID_FROM']]
    #         entityid2 = legacyid_to_entityid[relationship['RESOURCEID_TO']]
    #
    #     else:
    #         concept_value = Value.objects.filter(valueid = relationship['RELATION_TYPE'])
    #         entityid1 = relationship['RESOURCEID_FROM']
    #         entityid2 = relationship['RESOURCEID_TO']
    #
    #     related_resource_record = ResourceXResource(
    #         entityid1 = entityid1,
    #         entityid2 = entityid2,
    #         notes = relationship['NOTES'],
    #         relationshiptype = concept_value[0].valueid,
    #         datestarted = start_date,
    #         dateended = end_date,
    #         )
    #
    #     related_resource_record.save()
    #     self.se.index_data(index='resource_relations', doc_type='all', body=model_to_dict(related_resource_record), idfield='resourcexid')
<|endoftext|>