<commit_before># -*- coding: utf-8 -*-
from salt.modules import artifactory
from salttesting import TestCase
from salttesting.mock import MagicMock

class ArtifactoryTestCase(TestCase):

    org_module_functions = {}

    def __save_module_functions(self):
        for name, val in artifactory.__dict__.iteritems():
            if callable(val):
                self.org_module_functions[name] = val

    def __restore_module_functions(self):
        for name, val in self.org_module_functions.iteritems():
            artifactory.__dict__[name] = val

    def setUp(self):
        self.__save_module_functions()

    def tearDown(self):
        self.__restore_module_functions()

    def test_artifact_get_metadata(self):
        artifactory._get_artifact_metadata_xml = MagicMock(return_value='''<?xml version="1.0" encoding="UTF-8"?>
            <metadata>
              <groupId>com.company.sampleapp.web-module</groupId>
              <artifactId>web</artifactId>
              <versioning>
                <latest>1.1_RC11</latest>
                <release>1.0.1</release>
                <versions>
                  <version>1.0_RC20</version>
                  <version>1.0_RC22</version>
                </versions>
                <lastUpdated>20140623120632</lastUpdated>
              </versioning>
            </metadata>
        ''')
        metadata = artifactory._get_artifact_metadata(artifactory_url='http://artifactory.company.com/artifactory',
                                            repository='libs-releases',
                                            group_id='com.company.sampleapp.web-module',
                                            artifact_id='web')
        self.assertEqual(metadata['latest_version'], '1.1_RC11')

    def test_snapshot_version_get_metadata(self):
        artifactory._get_snapshot_version_metadata_xml = MagicMock(return_value='''<?xml version="1.0" encoding="UTF-8"?>
            <metadata>
              <groupId>com.company.sampleapp.web-module</groupId>
              <artifactId>web</artifactId>
              <version>1.1_RC8-SNAPSHOT</version>
              <versioning>
                <snapshot>
                  <timestamp>20140418.150212</timestamp>
                  <buildNumber>1</buildNumber>
                </snapshot>
                <lastUpdated>20140623104055</lastUpdated>
                <snapshotVersions>
                  <snapshotVersion>
                    <extension>pom</extension>
                    <value>1.1_RC8-20140418.150212-1</value>
                    <updated>20140418150212</updated>
                  </snapshotVersion>
                  <snapshotVersion>
                    <extension>war</extension>
                    <value>1.1_RC8-20140418.150212-1</value>
                    <updated>20140418150212</updated>
                  </snapshotVersion>
                </snapshotVersions>
              </versioning>
            </metadata>
        ''')
        metadata = artifactory._get_snapshot_version_metadata(artifactory_url='http://artifactory.company.com/artifactory',
                                                             repository='libs-releases',
                                                             group_id='com.company.sampleapp.web-module',
                                                             artifact_id='web',
                                                             version='1.1_RC8-SNAPSHOT')
        self.assertEqual(metadata['snapshot_versions']['war'], '1.1_RC8-20140418.150212-1')

    def test_artifact_metadata_url(self):
        metadata_url = artifactory._get_artifact_metadata_url(artifactory_url='http://artifactory.company.com/artifactory',
                                                             repository='libs-releases',
                                                             group_id='com.company.sampleapp.web-module',
                                                             artifact_id='web')

        self.assertEqual(metadata_url, "http://artifactory.company.com/artifactory/libs-releases/com/company/sampleapp/web-module/web/maven-metadata.xml")

    def test_snapshot_version_metadata_url(self):
        metadata_url = artifactory._get_snapshot_version_metadata_url(artifactory_url='http://artifactory.company.com/artifactory',
                                                             repository='libs-snapshots',
                                                             group_id='com.company.sampleapp.web-module',
                                                             artifact_id='web',
                                                             version='1.0_RC10-SNAPSHOT')

        self.assertEqual(metadata_url, "http://artifactory.company.com/artifactory/libs-snapshots/com/company/sampleapp/web-module/web/1.0_RC10-SNAPSHOT/maven-metadata.xml")

    def test_construct_url_for_released_version(self):
        artifact_url, file_name = artifactory._get_release_url(repository='libs-releases',
                                      group_id='com.company.sampleapp.web-module',
                                      artifact_id='web',
                                      packaging='war',
                                      artifactory_url='http://artifactory.company.com/artifactory',
                                      version='1.0_RC20')

        self.assertEqual(artifact_url, "http://artifactory.company.com/artifactory/libs-releases/com/company/sampleapp/web-module/web/1.0_RC20/web-1.0_RC20.war")
        self.assertEqual(file_name, "web-1.0_RC20.war")

    def test_construct_url_for_snapshot_version(self):
        prev_artifactory_get_snapshot_version_metadata = artifactory._get_snapshot_version_metadata
        artifactory._get_snapshot_version_metadata = MagicMock(return_value={'snapshot_versions': {'war': '1.0_RC10-20131127.105838-2'}})

        artifact_url, file_name = artifactory._get_snapshot_url(artifactory_url='http://artifactory.company.com/artifactory',
                                               repository='libs-snapshots',
                                               group_id='com.company.sampleapp.web-module',
                                               artifact_id='web',
                                               version='1.0_RC10-SNAPSHOT',
                                               packaging='war')

        self.assertEqual(artifact_url, "http://artifactory.company.com/artifactory/libs-snapshots/com/company/sampleapp/web-module/web/1.0_RC10-SNAPSHOT/web-1.0_RC10-20131127.105838-2.war")
        self.assertEqual(file_name, "web-1.0_RC10-20131127.105838-2.war")
<commit_msg>Py3 compatibility fixes.<commit_after># -*- coding: utf-8 -*-

# Import Salt testing libs
from salttesting import TestCase
from salttesting.mock import MagicMock
from salttesting.helpers import ensure_in_syspath
ensure_in_syspath('../../')

# Import Salt libs
from salt.modules import artifactory

# Import 3rd-party libs
import salt.ext.six as six

class ArtifactoryTestCase(TestCase):

    org_module_functions = {}

    def __save_module_functions(self):
        for name, val in six.iteritems(artifactory.__dict__):
            if callable(val):
                self.org_module_functions[name] = val

    def __restore_module_functions(self):
        for name, val in six.iteritems(self.org_module_functions):
            artifactory.__dict__[name] = val

    def setUp(self):
        self.__save_module_functions()

    def tearDown(self):
        self.__restore_module_functions()

    def test_artifact_get_metadata(self):
        artifactory._get_artifact_metadata_xml = MagicMock(return_value='''<?xml version="1.0" encoding="UTF-8"?>
            <metadata>
              <groupId>com.company.sampleapp.web-module</groupId>
              <artifactId>web</artifactId>
              <versioning>
                <latest>1.1_RC11</latest>
                <release>1.0.1</release>
                <versions>
                  <version>1.0_RC20</version>
                  <version>1.0_RC22</version>
                </versions>
                <lastUpdated>20140623120632</lastUpdated>
              </versioning>
            </metadata>
        ''')
        metadata = artifactory._get_artifact_metadata(artifactory_url='http://artifactory.company.com/artifactory',
                                            repository='libs-releases',
                                            group_id='com.company.sampleapp.web-module',
                                            artifact_id='web')
        self.assertEqual(metadata['latest_version'], '1.1_RC11')

    def test_snapshot_version_get_metadata(self):
        artifactory._get_snapshot_version_metadata_xml = MagicMock(return_value='''<?xml version="1.0" encoding="UTF-8"?>
            <metadata>
              <groupId>com.company.sampleapp.web-module</groupId>
              <artifactId>web</artifactId>
              <version>1.1_RC8-SNAPSHOT</version>
              <versioning>
                <snapshot>
                  <timestamp>20140418.150212</timestamp>
                  <buildNumber>1</buildNumber>
                </snapshot>
                <lastUpdated>20140623104055</lastUpdated>
                <snapshotVersions>
                  <snapshotVersion>
                    <extension>pom</extension>
                    <value>1.1_RC8-20140418.150212-1</value>
                    <updated>20140418150212</updated>
                  </snapshotVersion>
                  <snapshotVersion>
                    <extension>war</extension>
                    <value>1.1_RC8-20140418.150212-1</value>
                    <updated>20140418150212</updated>
                  </snapshotVersion>
                </snapshotVersions>
              </versioning>
            </metadata>
        ''')
        metadata = artifactory._get_snapshot_version_metadata(artifactory_url='http://artifactory.company.com/artifactory',
                                                             repository='libs-releases',
                                                             group_id='com.company.sampleapp.web-module',
                                                             artifact_id='web',
                                                             version='1.1_RC8-SNAPSHOT')
        self.assertEqual(metadata['snapshot_versions']['war'], '1.1_RC8-20140418.150212-1')

    def test_artifact_metadata_url(self):
        metadata_url = artifactory._get_artifact_metadata_url(artifactory_url='http://artifactory.company.com/artifactory',
                                                             repository='libs-releases',
                                                             group_id='com.company.sampleapp.web-module',
                                                             artifact_id='web')

        self.assertEqual(metadata_url, "http://artifactory.company.com/artifactory/libs-releases/com/company/sampleapp/web-module/web/maven-metadata.xml")

    def test_snapshot_version_metadata_url(self):
        metadata_url = artifactory._get_snapshot_version_metadata_url(artifactory_url='http://artifactory.company.com/artifactory',
                                                             repository='libs-snapshots',
                                                             group_id='com.company.sampleapp.web-module',
                                                             artifact_id='web',
                                                             version='1.0_RC10-SNAPSHOT')

        self.assertEqual(metadata_url, "http://artifactory.company.com/artifactory/libs-snapshots/com/company/sampleapp/web-module/web/1.0_RC10-SNAPSHOT/maven-metadata.xml")

    def test_construct_url_for_released_version(self):
        artifact_url, file_name = artifactory._get_release_url(repository='libs-releases',
                                      group_id='com.company.sampleapp.web-module',
                                      artifact_id='web',
                                      packaging='war',
                                      artifactory_url='http://artifactory.company.com/artifactory',
                                      version='1.0_RC20')

        self.assertEqual(artifact_url, "http://artifactory.company.com/artifactory/libs-releases/com/company/sampleapp/web-module/web/1.0_RC20/web-1.0_RC20.war")
        self.assertEqual(file_name, "web-1.0_RC20.war")

    def test_construct_url_for_snapshot_version(self):
        prev_artifactory_get_snapshot_version_metadata = artifactory._get_snapshot_version_metadata
        artifactory._get_snapshot_version_metadata = MagicMock(return_value={'snapshot_versions': {'war': '1.0_RC10-20131127.105838-2'}})

        artifact_url, file_name = artifactory._get_snapshot_url(artifactory_url='http://artifactory.company.com/artifactory',
                                               repository='libs-snapshots',
                                               group_id='com.company.sampleapp.web-module',
                                               artifact_id='web',
                                               version='1.0_RC10-SNAPSHOT',
                                               packaging='war')

        self.assertEqual(artifact_url, "http://artifactory.company.com/artifactory/libs-snapshots/com/company/sampleapp/web-module/web/1.0_RC10-SNAPSHOT/web-1.0_RC10-20131127.105838-2.war")
        self.assertEqual(file_name, "web-1.0_RC10-20131127.105838-2.war")
<|endoftext|>


<commit_before>#!/usr/bin/env python

import copy
import math
import os
import pyPdf
import shutil
import subprocess as spc
import sys
import tempfile

def hsplit_pages(tmpd, src_fname, dst_fname):
    src_f = file(src_fname, "r+b")
    dst_f = file(dst_fname, "w+b")

    input = pyPdf.PdfFileReader(src_f)
    output = pyPdf.PdfFileWriter()

    for i in range(input.getNumPages()):
    #for i in range(26, 28):
        p = input.getPage(i)
        q = copy.copy(p)
        q.mediaBox = copy.copy(p.mediaBox)

        x1, y1 = p.mediaBox.lowerLeft
        x2, y2 = p.mediaBox.upperRight

        #print(x1, y1)
        #print(x2, y2)

        proc = spc.Popen(["png-halve-sheet-music",
            os.path.join(tmpd, str(i + 1) + ".png")],
            stdout=spc.PIPE)
        (y_new1, y_new2, _) = proc.stdout.read().split("\n")
        proc.wait()
        #print y_new1
        #print y_new2
        y_new1 = int(y_new1)
        y_new2 = int(y_new2)

        p.mediaBox.lowerLeft = (x1, y_new1)
        q.mediaBox.upperRight = (x2, y_new2)

        output.addPage(p)
        output.addPage(q)

    output.write(dst_f)
    src_f.close()
    dst_f.close()

def wrap(src_fname, dst_fname):
    tmpd = tempfile.mkdtemp()
    proc = spc.Popen(["mudraw", "-o", os.path.join(tmpd, "%d.png"), src_fname])
    proc.wait()
    hsplit_pages(tmpd, src_fname, dst_fname)
    shutil.rmtree(tmpd)

if __name__ == "__main__":
    if len(sys.argv) != 3:
        print "usage: halve-sheet-music.py bach.pdf output.pdf"
        exit(-1)
    (_, src_fname, dst_fname) = sys.argv
    if os.path.isdir(dst_fname):
        dst_fname = os.path.join(dst_fname, os.path.basename(src_fname))
    wrap(src_fname, dst_fname)
<commit_msg>debug option<commit_after>#!/usr/bin/env python

import copy
import math
import os
import pyPdf
import shutil
import subprocess as spc
import sys
import tempfile

def hsplit_pages(tmpd, src_fname, dst_fname):
    src_f = file(src_fname, "r+b")
    dst_f = file(dst_fname, "w+b")

    input = pyPdf.PdfFileReader(src_f)
    output = pyPdf.PdfFileWriter()

    for i in range(input.getNumPages()):
    #for i in range(26, 28):
        p = input.getPage(i)
        q = copy.copy(p)
        q.mediaBox = copy.copy(p.mediaBox)

        x1, y1 = p.mediaBox.lowerLeft
        x2, y2 = p.mediaBox.upperRight

        #print(x1, y1)
        #print(x2, y2)

        proc = spc.Popen(["png-halve-sheet-music",
            os.path.join(tmpd, str(i + 1) + ".png")],
            stdout=spc.PIPE)
        (y_new1, y_new2, _) = proc.stdout.read().split("\n")
        proc.wait()
        #print y_new1
        #print y_new2
        y_new1 = int(y_new1)
        y_new2 = int(y_new2)

        p.mediaBox.lowerLeft = (x1, y_new1)
        q.mediaBox.upperRight = (x2, y_new2)

        output.addPage(p)
        output.addPage(q)

    output.write(dst_f)
    src_f.close()
    dst_f.close()

def wrap(src_fname, dst_fname, debug):
    tmpd = tempfile.mkdtemp()
    proc = spc.Popen(["mudraw", "-o", os.path.join(tmpd, "%d.png"), src_fname])
    proc.wait()
    hsplit_pages(tmpd, src_fname, dst_fname)
    if debug:
        print(tmpd)
    else:
        shutil.rmtree(tmpd)

if __name__ == "__main__":
    debug = False
    args = sys.argv[1:]
    if args[0] == '--debug':
        debug = True
        args = args[1:]
    if len(args) != 2:
        print "usage: halve-sheet-music.py [--debug] bach.pdf output.pdf"
        exit(-1)
    (src_fname, dst_fname) = args
    if os.path.isdir(dst_fname):
        dst_fname = os.path.join(dst_fname, os.path.basename(src_fname))
    wrap(src_fname, dst_fname, debug)
<|endoftext|>


<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-

# Copyright (C) 2009 Adriano Monteiro Marques.
#
# Author: Bartosz SKOWRON <getxsick at gmail dot com>
#
# This library is free software; you can redistribute it and/or modify
# it under the terms of the GNU Lesser General Public License as published
# by the Free Software Foundation; either version 2.1 of the License, or
# (at your option) any later version.
#
# This library is distributed in the hope that it will be useful, but
# WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
# or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public
# License for more details.
#
# You should have received a copy of the GNU Lesser General Public License
# along with this library; if not, write to the Free Software Foundation,
# Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA

import os
import py.test
from umpa.utils.security import *

def foo():
    return 1

def bar(a):
    return a

class TestUtilSecurity(object):
    def setup_class(cls):
        if os.name == 'posix' and os.geteuid() != 0:
            py.test.skip('root-priviliges are needed')

    def teardown_method(self, method):
        if os.name == 'posix':
            os.seteuid(0)

    def test_drop(self):
        import pwd
        drop_priviliges()
        assert os.geteuid() == pwd.getpwnam('nobody')[2]

    def test_drop_argument(self):
        drop_priviliges(1)
        assert os.geteuid() == 1

    def test_super_euid(self):
        euid = os.geteuid()
        super_priviliges(foo)
        assert euid == os.geteuid()

    def test_super_fun_call(self):
        result = super_priviliges(foo)
        assert result == 1

    def test_super_fun_call_arg(self):
        result = super_priviliges(bar, "foobar")
        assert result == "foobar"
<commit_msg>same as r4654 but better<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-

# Copyright (C) 2009 Adriano Monteiro Marques.
#
# Author: Bartosz SKOWRON <getxsick at gmail dot com>
#
# This library is free software; you can redistribute it and/or modify
# it under the terms of the GNU Lesser General Public License as published
# by the Free Software Foundation; either version 2.1 of the License, or
# (at your option) any later version.
#
# This library is distributed in the hope that it will be useful, but
# WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
# or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public
# License for more details.
#
# You should have received a copy of the GNU Lesser General Public License
# along with this library; if not, write to the Free Software Foundation,
# Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA

import os
import py.test
from umpa.utils.security import *

def foo():
    return 1

def bar(a):
    return a

class TestUtilSecurity(object):
    def setup_class(cls):
        if os.name == 'posix' and os.geteuid() != 0:
            py.test.skip('root-priviliges are needed')
        if os.name != 'posix':
            py.test.skip('only for POSIX platforms')

    def teardown_method(self, method):
        os.seteuid(0)

    def test_drop(self):
        import pwd
        drop_priviliges()
        assert os.geteuid() == pwd.getpwnam('nobody')[2]

    def test_drop_argument(self):
        drop_priviliges(1)
        assert os.geteuid() == 1

    def test_super_euid(self):
        euid = os.geteuid()
        super_priviliges(foo)
        assert euid == os.geteuid()

    def test_super_fun_call(self):
        result = super_priviliges(foo)
        assert result == 1

    def test_super_fun_call_arg(self):
        result = super_priviliges(bar, "foobar")
        assert result == "foobar"
<|endoftext|>


<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-

# Copyright 2020 EMBL - European Bioinformatics Institute
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import os
from unittest.mock import patch
import pytest

from emgapianns.management.lib import sanity_check  # noqa: E402
from emgapianns.management.lib.uploader_exceptions import (NoAnnotationsFoundException,
                                                           UnexpectedLibraryStrategyException, QCNotPassedException,
                                                           CoverageCheckException)

class TestSanityCheck:

    @staticmethod
    def run_init_tests(actual, expected_accession, expected_result_dir, expected_prefix):
        assert expected_accession == actual.accession
        assert expected_result_dir == actual.dir
        assert expected_prefix == actual.prefix
        config = actual.config
        assert config is not None
        assert isinstance(config, list)
        assert len(config) > 0
        file = config[0]
        assert isinstance(file, dict)
        assert len(file) > 0
        assert "_required" in file
        assert "blah" not in file

    @patch("os.path.basename")
    @pytest.mark.parametrize("expected_accession, experiment_type, version, expected_result_dir", [
        ("ERRXXXXXX", "amplicon", "4.1", "/tmp"),
        ("ERRXXXXXX", "amplicon", "5.0", "/tmp"),
        ("ERZXXXXXX", "assembly", "4.1", "/tmp"),
        ("ERZXXXXXX", "assembly", "5.0", "/tmp"),
        ("ERRXXXXXX", "wgs", "4.1", "/tmp"),
        ("ERRXXXXXX", "wgs", "5.0", "/tmp"),
        ("ERRXXXXXX", "rna-seq", "4.1", "/tmp"),
        ("ERRXXXXXX", "rna-seq", "5.0", "/tmp"),
        ("ERZXXXXXX", "ASSEMBLY", "4.1", "/tmp"),
        ("ERRXXXXXX", "WGS", "4.1", "/tmp"),
        ("ERRXXXXXX", "AMPLICON", "4.1", "/tmp")
    ])
    def test_check_initialisations(self, mock_dir, expected_accession, experiment_type, version,
                                   expected_result_dir):
        mock_dir.return_value = "test"
        test_instance = sanity_check.SanityCheck(expected_accession, expected_result_dir, experiment_type, version)
        self.run_init_tests(test_instance, expected_accession, expected_result_dir, mock_dir.return_value)

    @patch("os.path.basename")
    @pytest.mark.parametrize("accession, experiment_type, version, result_folder", [
        ("ERR2985769", "wgs", "1.0", "/tmp"),
        ("ERR2985769", "wgs", "2.0", "/tmp"),
        ("ERR2985769", "wgs", "3.0", "/tmp"),
        ("ERR2985769", "rna-seq", "1.0", "/tmp"),
        ("ERR2985769", "rna-seq", "2.0", "/tmp"),
        ("ERR2985769", "rna-seq", "3.0", "/tmp"),
        ("ERR2985769", "amplicon", "1.0", "/tmp"),
        ("ERR2985769", "amplicon", "2.0", "/tmp"),
        ("ERR2985769", "amplicon", "3.0", "/tmp"),
        ("ERR2985769", "assembly", "1.0", "/tmp"),
        ("ERR2985769", "assembly", "2.0", "/tmp"),
        ("ERR2985769", "assembly", "3.0", "/tmp")
    ])
    def test_check_initialisation_should_raise_exception(self, mock_dir, accession, experiment_type, version,
                                                         result_folder):
        mock_dir.return_value = "test"
        with pytest.raises(FileNotFoundError):
            sanity_check.SanityCheck(accession, result_folder, experiment_type, version)

    @patch("os.path.basename")
    @pytest.mark.parametrize("accession, experiment_type, version, result_folder", [
        ("ERR2985769", "other", "4.1", "/tmp"),
        ("ERR2985769", "WGA", "4.1", "/tmp"),
    ])
    def test_check_init_should_raise_unexpected_library_strategy_exception(self, mock_dir, accession,
                                                                           experiment_type, version,
                                                                           result_folder):
        mock_dir.return_value = "test"
        with pytest.raises(UnexpectedLibraryStrategyException):
            sanity_check.SanityCheck(accession, result_folder, experiment_type, version)

    @pytest.mark.parametrize("accession, experiment_type, amplicon_type, version, result_folder", [
        ("ERR3506537", "amplicon", "SSU", "4.1",
         "results/2019/09/ERP117125/version_4.1/ERR350/007/ERR3506537_MERGED_FASTQ"),
        ("ERR2237853", "amplicon", "ITS", "5.0",
         "results/2018/01/ERP106131/version_5.0/ERR223/003/ERR2237853_MERGED_FASTQ")
    ])
    def test_check_amplicons_v4_v5_results_succeeds(self, accession, experiment_type, amplicon_type, version,
                                                    result_folder):
        root_dir = os.path.join(os.path.dirname(__file__), "test_data")
        result_dir = os.path.join(root_dir, result_folder)
        test_instance = sanity_check.SanityCheck(accession, result_dir, experiment_type, version)
        test_instance.check_file_existence()

    def test_check_amplicon_ssu_v5_results_succeeds(self):
        accession = "ERR2237853"
        experiment_type = "amplicon"
        version = "5.0"
        root_dir = os.path.join(os.path.dirname(__file__), "test_data")
        result_dir = os.path.join(root_dir,
                                  "results/2018/01/ERP106131/version_{}/ERR223/003/{}_MERGED_FASTQ".format(version,
                                                                                accession))
        test_instance = sanity_check.SanityCheck(accession, result_dir, experiment_type, version)
        test_instance.check_file_existence()

    @pytest.mark.parametrize("accession, experiment_type, version, result_folder", [
        ("ERR2985769", "amplicon", "4.1", "no-results")
    ])
    def test_check_amplicon_v4_results_raise_exception(self, accession, experiment_type, version, result_folder):
        root_dir = os.path.join(os.path.dirname(__file__), "test_data")
        result_dir = os.path.join(root_dir, result_folder)
        test_instance = sanity_check.SanityCheck(accession, result_dir, experiment_type, version)
        with pytest.raises(FileNotFoundError):
            test_instance.check_file_existence()

    @pytest.mark.parametrize("accession, experiment_type, version, result_folder", [
        ("ERZ477576", "assembly", "4.1",
         "results/2017/11/ERP104174/version_4.1/ERZ477/006/ERZ477576_FASTA"),
        ("ERZ782882", "assembly", "5.0",
         "sanity_check/version_5.0/assembly/ERZ782882_FASTA"),
        ("ERZ782883", "assembly", "5.0",
         "sanity_check/version_5.0/assembly/ERZ782883_FASTA")
    ])
    def test_check_assemblies_v4_v5_results_succeeds(self, accession, experiment_type, version, result_folder):
        root_dir = os.path.join(os.path.dirname(__file__), "test_data")
        result_dir = os.path.join(root_dir, result_folder)
        test_instance = sanity_check.SanityCheck(accession, result_dir, experiment_type, version)
        test_instance.check_file_existence()

    @pytest.mark.parametrize("accession, experiment_type, version, result_folder", [
        ("ERR1913139", "wgs", "4.1", "sanity_check/version_4.1/wgs/ERR1913139_FASTQ"),
        ("ERR1697182", "wgs", "5.0", "sanity_check/version_5.0/wgs/ERR1697182_MERGED_FASTQ")
    ])
    def test_check_file_existence_wgs_v4_v5_results_succeeds(self, accession, experiment_type, version, result_folder):
        root_dir = os.path.join(os.path.dirname(__file__), "test_data")
        result_dir = os.path.join(root_dir, result_folder)
        test_instance = sanity_check.SanityCheck(accession, result_dir, experiment_type, version)
        test_instance.check_file_existence()

    @pytest.mark.parametrize("accession, experiment_type, version, result_folder", [
        ("ERR1864826", "amplicon", "4.1",
         "results/2019/02/ERP021864/version_4.1/ERR1864826_FASTQ")
    ])
    def test_check_qc_not_passed_raise_exception(self, accession, experiment_type, version, result_folder):
        root_dir = os.path.join(os.path.dirname(__file__), "test_data")
        result_dir = os.path.join(root_dir, result_folder)
        test_instance = sanity_check.SanityCheck(accession, result_dir, experiment_type, version)
        with pytest.raises(QCNotPassedException):
            test_instance.run_quality_control_check()

    @pytest.mark.parametrize("accession, experiment_type, version, result_folder", [
        ("ERR1913139", "wgs", "4.1",
         "sanity_check/version_4.1/wgs/ERR1913139_FASTQ")
    ])
    def test_check_qc_not_passed_do_not_raise_exception(self, accession, experiment_type, version, result_folder):
        root_dir = os.path.join(os.path.dirname(__file__), "test_data")
        result_dir = os.path.join(root_dir, result_folder)
        test_instance = sanity_check.SanityCheck(accession, result_dir, experiment_type, version)
        test_instance.run_quality_control_check()
        assert True

    @pytest.mark.parametrize("accession, experiment_type, version, result_folder", [
        ("ERR0000001", "amplicon", "4.1",
         "results/2019/02/ERP000001/version_4.1/ERR0000001_FASTQ"),
        ("ERR3506531", "amplicon", "4.1",
         "results/2019/09/ERP117125/version_4.1/ERR350/001/ERR3506531_MERGED_FASTQ"),
        ("ERR1697183", "wgs", "5.0", "sanity_check/version_5.0/wgs/ERR1697183_MERGED_FASTQ")
    ])
    def test_check_file_existence_should_raise_file_not_found_error(self, accession, experiment_type, version,
                                                                    result_folder):
        root_dir = os.path.join(os.path.dirname(__file__), "test_data")
        result_dir = os.path.join(root_dir, result_folder)
        test_instance = sanity_check.SanityCheck(accession, result_dir, experiment_type, version)
        with pytest.raises(FileNotFoundError):
            test_instance.check_file_existence()

    @pytest.mark.parametrize("accession, experiment_type, version, result_folder", [
        ("ERR3506537", "amplicon", "4.1",
         "results/2019/09/ERP117125/version_4.1/ERR350/007/ERR3506537_MERGED_FASTQ"),
        ("ERR2237853", "amplicon", "5.0",
         "results/2018/01/ERP106131/version_5.0/ERR223/003/ERR2237853_MERGED_FASTQ"),
        ("ERZ782882", "assembly", "5.0",
         "sanity_check/version_5.0/assembly/ERZ782882_FASTA"),
        ("ERR3506532", "wgs", "4.1",
         "results/2019/09/ERP117125/version_4.1/ERR350/002/ERR3506532_MERGED_FASTQ"),
        ("ERR1697182", "wgs", "5.0", "sanity_check/version_5.0/wgs/ERR1697182_MERGED_FASTQ")
    ])
    def test_coverage_check_succeeds(self, accession, experiment_type, version, result_folder):
        root_dir = os.path.join(os.path.dirname(__file__), "test_data")
        result_dir = os.path.join(root_dir, result_folder)
        test_instance = sanity_check.SanityCheck(accession, result_dir, experiment_type, version)
        test_instance.run_coverage_check()
        assert True

    @pytest.mark.parametrize("accession, experiment_type, version, result_folder", [
        ("ERR3506531", "amplicon", "4.1",
         "results/2019/09/ERP117125/version_4.1/ERR350/001/ERR3506531_MERGED_FASTQ"),
        ("ERR1913139", "wgs", "4.1",
         "sanity_check/version_4.1/wgs/ERR1913139_FASTQ"),
        ("ERZ782883", "assembly", "5.0",
         "sanity_check/version_5.0/assembly/ERZ782883_FASTA"),
        ("ERR1864826", "amplicon", "4.1",
         "results/2019/02/ERP021864/version_4.1/ERR1864826_FASTQ"),
        ("ERR1697183", "wgs", "5.0", "sanity_check/version_5.0/wgs/ERR1697183_MERGED_FASTQ")
    ])
    def test_coverage_check_should_raise_coverage_check_exception(self, accession, experiment_type, version,
                                                                    result_folder):
        root_dir = os.path.join(os.path.dirname(__file__), "test_data")
        result_dir = os.path.join(root_dir, result_folder)
        test_instance = sanity_check.SanityCheck(accession, result_dir, experiment_type, version)
        with pytest.raises(CoverageCheckException):
            test_instance.run_coverage_check()
<commit_msg>set result status<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-

# Copyright 2020 EMBL - European Bioinformatics Institute
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import os
from unittest.mock import patch
import pytest

from emgapianns.management.lib import sanity_check  # noqa: E402
from emgapianns.management.lib.uploader_exceptions import (NoAnnotationsFoundException,
                                                           UnexpectedLibraryStrategyException, QCNotPassedException,
                                                           CoverageCheckException)

class TestSanityCheck:

    @staticmethod
    def run_init_tests(actual, expected_accession, expected_result_dir, expected_prefix):
        assert expected_accession == actual.accession
        assert expected_result_dir == actual.dir
        assert expected_prefix == actual.prefix
        config = actual.config
        assert config is not None
        assert isinstance(config, list)
        assert len(config) > 0
        file = config[0]
        assert isinstance(file, dict)
        assert len(file) > 0
        assert "_required" in file
        assert "blah" not in file

    @patch("os.path.basename")
    @pytest.mark.parametrize("expected_accession, experiment_type, version, expected_result_dir", [
        ("ERRXXXXXX", "amplicon", "4.1", "/tmp"),
        ("ERRXXXXXX", "amplicon", "5.0", "/tmp"),
        ("ERZXXXXXX", "assembly", "4.1", "/tmp"),
        ("ERZXXXXXX", "assembly", "5.0", "/tmp"),
        ("ERRXXXXXX", "wgs", "4.1", "/tmp"),
        ("ERRXXXXXX", "wgs", "5.0", "/tmp"),
        ("ERRXXXXXX", "rna-seq", "4.1", "/tmp"),
        ("ERRXXXXXX", "rna-seq", "5.0", "/tmp"),
        ("ERZXXXXXX", "ASSEMBLY", "4.1", "/tmp"),
        ("ERRXXXXXX", "WGS", "4.1", "/tmp"),
        ("ERRXXXXXX", "AMPLICON", "4.1", "/tmp")
    ])
    def test_check_initialisations(self, mock_dir, expected_accession, experiment_type, version,
                                   expected_result_dir):
        mock_dir.return_value = "test"
        sanity_check.SanityCheck.result_status = 'full'
        test_instance = sanity_check.SanityCheck(expected_accession, expected_result_dir, experiment_type, version)
        self.run_init_tests(test_instance, expected_accession, expected_result_dir, mock_dir.return_value)

    @patch("os.path.basename")
    @pytest.mark.parametrize("accession, experiment_type, version, result_folder", [
        ("ERR2985769", "wgs", "1.0", "/tmp"),
        ("ERR2985769", "wgs", "2.0", "/tmp"),
        ("ERR2985769", "wgs", "3.0", "/tmp"),
        ("ERR2985769", "rna-seq", "1.0", "/tmp"),
        ("ERR2985769", "rna-seq", "2.0", "/tmp"),
        ("ERR2985769", "rna-seq", "3.0", "/tmp"),
        ("ERR2985769", "amplicon", "1.0", "/tmp"),
        ("ERR2985769", "amplicon", "2.0", "/tmp"),
        ("ERR2985769", "amplicon", "3.0", "/tmp"),
        ("ERR2985769", "assembly", "1.0", "/tmp"),
        ("ERR2985769", "assembly", "2.0", "/tmp"),
        ("ERR2985769", "assembly", "3.0", "/tmp")
    ])
    def test_check_initialisation_should_raise_exception(self, mock_dir, accession, experiment_type, version,
                                                         result_folder):
        mock_dir.return_value = "test"
        with pytest.raises(FileNotFoundError):
            sanity_check.SanityCheck(accession, result_folder, experiment_type, version)

    @patch("os.path.basename")
    @pytest.mark.parametrize("accession, experiment_type, version, result_folder", [
        ("ERR2985769", "other", "4.1", "/tmp"),
        ("ERR2985769", "WGA", "4.1", "/tmp"),
    ])
    def test_check_init_should_raise_unexpected_library_strategy_exception(self, mock_dir, accession,
                                                                           experiment_type, version,
                                                                           result_folder):
        mock_dir.return_value = "test"
        with pytest.raises(UnexpectedLibraryStrategyException):
            sanity_check.SanityCheck(accession, result_folder, experiment_type, version)

    @pytest.mark.parametrize("accession, experiment_type, amplicon_type, version, result_folder", [
        ("ERR3506537", "amplicon", "SSU", "4.1",
         "results/2019/09/ERP117125/version_4.1/ERR350/007/ERR3506537_MERGED_FASTQ"),
        ("ERR2237853", "amplicon", "ITS", "5.0",
         "results/2018/01/ERP106131/version_5.0/ERR223/003/ERR2237853_MERGED_FASTQ")
    ])
    def test_check_amplicons_v4_v5_results_succeeds(self, accession, experiment_type, amplicon_type, version,
                                                    result_folder):
        root_dir = os.path.join(os.path.dirname(__file__), "test_data")
        result_dir = os.path.join(root_dir, result_folder)
        test_instance = sanity_check.SanityCheck(accession, result_dir, experiment_type, version)
        test_instance.check_file_existence()

    def test_check_amplicon_ssu_v5_results_succeeds(self):
        accession = "ERR2237853"
        experiment_type = "amplicon"
        version = "5.0"
        root_dir = os.path.join(os.path.dirname(__file__), "test_data")
        result_dir = os.path.join(root_dir,
                                  "results/2018/01/ERP106131/version_{}/ERR223/003/{}_MERGED_FASTQ".format(version,
                                                                                accession))
        test_instance = sanity_check.SanityCheck(accession, result_dir, experiment_type, version)
        test_instance.check_file_existence()

    @pytest.mark.parametrize("accession, experiment_type, version, result_folder", [
        ("ERR2985769", "amplicon", "4.1", "no-results")
    ])
    def test_check_amplicon_v4_results_raise_exception(self, accession, experiment_type, version, result_folder):
        root_dir = os.path.join(os.path.dirname(__file__), "test_data")
        result_dir = os.path.join(root_dir, result_folder)
        test_instance = sanity_check.SanityCheck(accession, result_dir, experiment_type, version)
        with pytest.raises(FileNotFoundError):
            test_instance.check_file_existence()

    @pytest.mark.parametrize("accession, experiment_type, version, result_folder", [
        ("ERZ477576", "assembly", "4.1",
         "results/2017/11/ERP104174/version_4.1/ERZ477/006/ERZ477576_FASTA"),
        ("ERZ782882", "assembly", "5.0",
         "sanity_check/version_5.0/assembly/ERZ782882_FASTA"),
        ("ERZ782883", "assembly", "5.0",
         "sanity_check/version_5.0/assembly/ERZ782883_FASTA")
    ])
    def test_check_assemblies_v4_v5_results_succeeds(self, accession, experiment_type, version, result_folder):
        root_dir = os.path.join(os.path.dirname(__file__), "test_data")
        result_dir = os.path.join(root_dir, result_folder)
        test_instance = sanity_check.SanityCheck(accession, result_dir, experiment_type, version)
        test_instance.check_file_existence()

    @pytest.mark.parametrize("accession, experiment_type, version, result_folder", [
        ("ERR1913139", "wgs", "4.1", "sanity_check/version_4.1/wgs/ERR1913139_FASTQ"),
        ("ERR1697182", "wgs", "5.0", "sanity_check/version_5.0/wgs/ERR1697182_MERGED_FASTQ")
    ])
    def test_check_file_existence_wgs_v4_v5_results_succeeds(self, accession, experiment_type, version, result_folder):
        root_dir = os.path.join(os.path.dirname(__file__), "test_data")
        result_dir = os.path.join(root_dir, result_folder)
        test_instance = sanity_check.SanityCheck(accession, result_dir, experiment_type, version)
        test_instance.check_file_existence()

    @pytest.mark.parametrize("accession, experiment_type, version, result_folder", [
        ("ERR1864826", "amplicon", "4.1",
         "results/2019/02/ERP021864/version_4.1/ERR1864826_FASTQ")
    ])
    def test_check_qc_not_passed_raise_exception(self, accession, experiment_type, version, result_folder):
        root_dir = os.path.join(os.path.dirname(__file__), "test_data")
        result_dir = os.path.join(root_dir, result_folder)
        test_instance = sanity_check.SanityCheck(accession, result_dir, experiment_type, version)
        with pytest.raises(QCNotPassedException):
            test_instance.run_quality_control_check()

    @pytest.mark.parametrize("accession, experiment_type, version, result_folder", [
        ("ERR1913139", "wgs", "4.1",
         "sanity_check/version_4.1/wgs/ERR1913139_FASTQ")
    ])
    def test_check_qc_not_passed_do_not_raise_exception(self, accession, experiment_type, version, result_folder):
        root_dir = os.path.join(os.path.dirname(__file__), "test_data")
        result_dir = os.path.join(root_dir, result_folder)
        test_instance = sanity_check.SanityCheck(accession, result_dir, experiment_type, version)
        test_instance.run_quality_control_check()
        assert True

    @pytest.mark.parametrize("accession, experiment_type, version, result_folder", [
        ("ERR0000001", "amplicon", "4.1",
         "results/2019/02/ERP000001/version_4.1/ERR0000001_FASTQ"),
        ("ERR3506531", "amplicon", "4.1",
         "results/2019/09/ERP117125/version_4.1/ERR350/001/ERR3506531_MERGED_FASTQ"),
        ("ERR1697183", "wgs", "5.0", "sanity_check/version_5.0/wgs/ERR1697183_MERGED_FASTQ")
    ])
    def test_check_file_existence_should_raise_file_not_found_error(self, accession, experiment_type, version,
                                                                    result_folder):
        root_dir = os.path.join(os.path.dirname(__file__), "test_data")
        result_dir = os.path.join(root_dir, result_folder)
        test_instance = sanity_check.SanityCheck(accession, result_dir, experiment_type, version)
        with pytest.raises(FileNotFoundError):
            test_instance.check_file_existence()

    @pytest.mark.parametrize("accession, experiment_type, version, result_folder", [
        ("ERR3506537", "amplicon", "4.1",
         "results/2019/09/ERP117125/version_4.1/ERR350/007/ERR3506537_MERGED_FASTQ"),
        ("ERR2237853", "amplicon", "5.0",
         "results/2018/01/ERP106131/version_5.0/ERR223/003/ERR2237853_MERGED_FASTQ"),
        ("ERZ782882", "assembly", "5.0",
         "sanity_check/version_5.0/assembly/ERZ782882_FASTA"),
        ("ERR3506532", "wgs", "4.1",
         "results/2019/09/ERP117125/version_4.1/ERR350/002/ERR3506532_MERGED_FASTQ"),
        ("ERR1697182", "wgs", "5.0", "sanity_check/version_5.0/wgs/ERR1697182_MERGED_FASTQ")
    ])
    def test_coverage_check_succeeds(self, accession, experiment_type, version, result_folder):
        root_dir = os.path.join(os.path.dirname(__file__), "test_data")
        result_dir = os.path.join(root_dir, result_folder)
        test_instance = sanity_check.SanityCheck(accession, result_dir, experiment_type, version)
        test_instance.run_coverage_check()
        assert True

    @pytest.mark.parametrize("accession, experiment_type, version, result_folder", [
        ("ERR3506531", "amplicon", "4.1",
         "results/2019/09/ERP117125/version_4.1/ERR350/001/ERR3506531_MERGED_FASTQ"),
        ("ERR1913139", "wgs", "4.1",
         "sanity_check/version_4.1/wgs/ERR1913139_FASTQ"),
        ("ERZ782883", "assembly", "5.0",
         "sanity_check/version_5.0/assembly/ERZ782883_FASTA"),
        ("ERR1864826", "amplicon", "4.1",
         "results/2019/02/ERP021864/version_4.1/ERR1864826_FASTQ"),
        ("ERR1697183", "wgs", "5.0", "sanity_check/version_5.0/wgs/ERR1697183_MERGED_FASTQ")
    ])
    def test_coverage_check_should_raise_coverage_check_exception(self, accession, experiment_type, version,
                                                                    result_folder):
        root_dir = os.path.join(os.path.dirname(__file__), "test_data")
        result_dir = os.path.join(root_dir, result_folder)
        test_instance = sanity_check.SanityCheck(accession, result_dir, experiment_type, version)
        with pytest.raises(CoverageCheckException):
            test_instance.run_coverage_check()
<|endoftext|>


<commit_before># MIT License
# Copyright (c) 2017 MassChallenge, Inc.

from datetime import datetime
import dateutil.parser
from pytz import utc

from django.core.exceptions import ObjectDoesNotExist
from django.db.models import Q
from django.utils.formats import get_format

DAWN_OF_TIME = utc.localize(datetime.strptime(
    "2010-01-01T00:00:00Z",
    "%Y-%m-%dT%H:%M:%SZ"))  # format based on browsable API output

def override_updated_at(instance, timestamp):
    """
    Workaround to set the `updated_at` field of the instance to the given
    `datetime`. Can't use `save()` because that triggers `auto_now`.
    """
    instance_as_qs = instance._meta.default_manager.filter(pk=instance.pk)
    assert instance_as_qs.count() == 1
    instance_as_qs.update(updated_at=timestamp)

def parse_date(date_str):
    for item in get_format('DATE_INPUT_FORMATS'):
        try:
            return dateutil.parser(date_str, item)
        except (ValueError, TypeError):
            continue
    if date_str:
        return dateutil.parser.parse(date_str)

def get_profile(user):
    try:
        user_type = user.baseprofile.user_type
        if user_type == "ENTREPRENEUR":
            return user.entrepreneurprofile
        if user_type == "EXPERT":
            return user.expertprofile
        return user.memberprofile
    except ObjectDoesNotExist:
        return None

def model_name_case(model, related_model):
    if related_model:
        return model + '_' + related_model
    return model

def compose_filter(key_pieces, value):
    return {"__".join(key_pieces): value}

def next_instance(instance, query):
    return _find_instance(instance, Q(id__gt=instance.id) & query, "id")

def previous_instance(instance, query):
    return _find_instance(instance, Q(id__lt=instance.id) & query, "-id")

def _find_instance(instance, query, order):
    return type(instance).objects.filter(query).order_by(order).first()

"""
fetch a program role assigned to an user and those assigned to the startups
the user belong to

Time, Space, Query Complexity
Query: amount to the query complexity of the two helper function that
access the DB (see functions for query comp analysis for each)

Time/Space amount to time and space complexity of the three helper functions
(see functions for time/space comp analysis for each)
    """
def get_user_program_roles(
    user, user_roles_of_interest=[], startup_roles_of_interest=[]):

        user_prg_roles = _get_user_prg_role_by_program_family(
            user, user_roles_of_interest)
        startup_prg_roles = _get_user_startup_prg_role_by_program_family(
           user, startup_roles_of_interest
        )
        return _combine_prg_roles(
            user_prg_roles=user_prg_roles, startup_prg_roles=startup_prg_roles
        )

"""
return a use's program role grants grouped by the program family
filter role grants by the provided user role or return
all program role grant the user has ever had

Time, Space Complexity
O(n): time and space where n is the number of item in the result list
"""
def _get_user_prg_role_by_program_family(user, user_roles=[]):
    query = user.programrolegrant_set.filter(
        program_role__user_role__isnull=False
    )
    if user_roles:
        query = query.filter(
            program_role__user_role__name__in=user_roles)
    result = query.values_list(
        'program_role__name', 'program_role__program__program_family__name')
    prg_group = {}
    for program_role_grant, program_family in result:
        if prg_group.get(program_family):
            prg_group[program_family].append(program_role_grant)
        else:
            prg_group[program_family] = [program_role_grant]
    return prg_group

"""
fetch a the program status for all startup a given user
belongs to

Time, Space, Query Complexity
Query: 0(n + 1) where n is the number of startup a user belongs to
since for every startup in that list we are fetching the program status.
and the +1 is for the query to fetch all startup for the user

O(nm)Time/Space where n is the number of startup and m is
the number of program_startup_status
"""
def _get_user_startup_prg_role_by_program_family(user, startup_roles=[]):
    startups = user.startup_set.all()
    startup_status_group = {}
    for startup in startups:
        query = startup.program_startup_statuses()
        if startup_roles:
            query = query.filter(
                startup_role__name__in=startup_roles
            )
        result = query.values_list("startup_status", "program__program_family__name")
        for startup_status, program_family in result:
            if startup_status_group.get(program_family):
                startup_status_group[program_family].append(startup_status)
            else:
                startup_status_group[program_family] = [startup_status]

    return startup_status_group

"""
collapse two dictory with list values into one where similar keys values
in both dictionaries are merged

Time, Space Complexity
O(n)Time/Space; where n is the number of items in the startup_prg_roles

"""
def _combine_prg_roles(user_prg_roles, startup_prg_roles):
    for key, value in startup_prg_roles.items():
        if user_prg_roles.get(key):
            user_prg_roles[key] = user_prg_roles[key] + value
        else:
            user_prg_roles[key] = value

    return user_prg_roles
<commit_msg>[AC-7327] refactored grouping of roles by program family<commit_after># MIT License
# Copyright (c) 2017 MassChallenge, Inc.

from datetime import datetime
import dateutil.parser
from pytz import utc

from django.core.exceptions import ObjectDoesNotExist
from django.db.models import Q
from django.utils.formats import get_format

DAWN_OF_TIME = utc.localize(datetime.strptime(
    "2010-01-01T00:00:00Z",
    "%Y-%m-%dT%H:%M:%SZ"))  # format based on browsable API output

def override_updated_at(instance, timestamp):
    """
    Workaround to set the `updated_at` field of the instance to the given
    `datetime`. Can't use `save()` because that triggers `auto_now`.
    """
    instance_as_qs = instance._meta.default_manager.filter(pk=instance.pk)
    assert instance_as_qs.count() == 1
    instance_as_qs.update(updated_at=timestamp)

def parse_date(date_str):
    for item in get_format('DATE_INPUT_FORMATS'):
        try:
            return dateutil.parser(date_str, item)
        except (ValueError, TypeError):
            continue
    if date_str:
        return dateutil.parser.parse(date_str)

def get_profile(user):
    try:
        user_type = user.baseprofile.user_type
        if user_type == "ENTREPRENEUR":
            return user.entrepreneurprofile
        if user_type == "EXPERT":
            return user.expertprofile
        return user.memberprofile
    except ObjectDoesNotExist:
        return None

def model_name_case(model, related_model):
    if related_model:
        return model + '_' + related_model
    return model

def compose_filter(key_pieces, value):
    return {"__".join(key_pieces): value}

def next_instance(instance, query):
    return _find_instance(instance, Q(id__gt=instance.id) & query, "id")

def previous_instance(instance, query):
    return _find_instance(instance, Q(id__lt=instance.id) & query, "-id")

def _find_instance(instance, query, order):
    return type(instance).objects.filter(query).order_by(order).first()

"""
fetch a program role assigned to an user and those assigned to the startups
the user belong to

Time, Space, Query Complexity
Query: amount to the query complexity of the two helper function that
access the DB (see functions for query comp analysis for each)

Time/Space amount to time and space complexity of the three helper functions
(see functions for time/space comp analysis for each)
    """
def get_user_program_roles(
    user, user_roles_of_interest=[], startup_roles_of_interest=[]):

        user_prg_roles = _get_user_prg_role_by_program_family(
            user, user_roles_of_interest)
        startup_prg_roles = _get_user_startup_prg_role_by_program_family(
           user, startup_roles_of_interest
        )
        return _combine_prg_roles(
            user_prg_roles=user_prg_roles, startup_prg_roles=startup_prg_roles
        )

"""
return a use's program role grants grouped by the program family
filter role grants by the provided user role or return
all program role grant the user has ever had

Time, Space Complexity
O(n): time and space where n is the number of item in the result list
"""
def _get_user_prg_role_by_program_family(user, user_roles=[]):
    query = user.programrolegrant_set.filter(
        program_role__user_role__isnull=False
    )
    if user_roles:
        query = query.filter(
            program_role__user_role__name__in=user_roles)
    result = query.values_list(
        'program_role__name', 'program_role__program__program_family__name')
    return _group_by_program_family(result)

"""
fetch a the program status for all startup a given user
belongs to

Time, Space, Query Complexity
Query: 0(n + 1) where n is the number of startup a user belongs to
since for every startup in that list we are fetching the program status.
and the +1 is for the query to fetch all startup for the user

O(nm)Time/Space where n is the number of startup and m is
the number of program_startup_status
"""
def _get_user_startup_prg_role_by_program_family(user, startup_roles=[]):
    startups = user.startup_set.all()

    for startup in startups:
        query = startup.program_startup_statuses()
        if startup_roles:
            query = query.filter(
                startup_role__name__in=startup_roles
            )
        result = query.values_list("startup_status", "program__program_family__name")

    return _group_by_program_family(result)

"""
O(n)Time/Space where n is the number of items in the array
"""
def _group_by_program_family(array):
    by_program_family = {}
    for program_role, program_family in array:
        if by_program_family.get(program_family):
            by_program_family[program_family].append(program_role)
        else:
            by_program_family[program_family] = [program_role]
    return by_program_family

"""
collapse two dictory with list values into one where similar keys values
in both dictionaries are merged

Time, Space Complexity
O(n)Time/Space; where n is the number of items in the startup_prg_roles

"""
def _combine_prg_roles(user_prg_roles, startup_prg_roles):
    for key, value in startup_prg_roles.items():
        if user_prg_roles.get(key):
            user_prg_roles[key] = user_prg_roles[key] + value
        else:
            user_prg_roles[key] = value

    return user_prg_roles
<|endoftext|>


<commit_before># -*- python -*-
#
# Git Version: @git@

#-----------------------------------------------------------------------
# XALT: A tool that tracks users jobs and environments on a cluster.
# Copyright (C) 2013-2014 University of Texas at Austin
# Copyright (C) 2013-2014 University of Tennessee
#
# This library is free software; you can redistribute it and/or modify
# it under the terms of the GNU Lesser General Public License as
# published by the Free Software Foundation; either version 2.1 of
# the License, or (at your option) any later version.
#
# This library is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
# Lesser  General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public
# License along with this library; if not, write to the Free
# Software Foundation, Inc., 59 Temple Place, Suite 330,
# Boston, MA 02111-1307 USA
#-----------------------------------------------------------------------

from __future__             import print_function
import os, sys

dirNm, execName = os.path.split(os.path.realpath(sys.argv[0]))
sys.path.insert(1,os.path.abspath(os.path.join(dirNm, "../libexec")))

from xalt_parse_mpirun_args import find_exec

ignoreT = {
  'env'              : True,
  'time'             : True,
}

argT = {
  '--basepath'                  : 1,
  '--branchcount'               : 1,
  '--configfiles'               : 1,
  '--debuglvl'                  : 1,
  '--hybrid'                    : 1,
  '--ignoreconfigfiles'         : 1,
  '--mpirunoptions'             : 1,
  '--order'                     : 1,
  '--overridepin'               : 1,
  '--qlogicmpi-quiescencecheck' : 1,
  '--schedtype'                 : 1,
  '--stats'                     : 1,
  '--variablesprefix'           : 1,
  '-S'                          : 1,
  '-h'                          : 1,
}

npT = {
  }

def main():
  """
  Find name of executable when using ibrun.
  """

  print(find_exec(ignoreT, argT, npT, None, sys.argv[1:], dot=True))

if ( __name__ == '__main__'): main()
<commit_msg>no dot in path for mympirun<commit_after># -*- python -*-
#
# Git Version: @git@

#-----------------------------------------------------------------------
# XALT: A tool that tracks users jobs and environments on a cluster.
# Copyright (C) 2013-2014 University of Texas at Austin
# Copyright (C) 2013-2014 University of Tennessee
#
# This library is free software; you can redistribute it and/or modify
# it under the terms of the GNU Lesser General Public License as
# published by the Free Software Foundation; either version 2.1 of
# the License, or (at your option) any later version.
#
# This library is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
# Lesser  General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public
# License along with this library; if not, write to the Free
# Software Foundation, Inc., 59 Temple Place, Suite 330,
# Boston, MA 02111-1307 USA
#-----------------------------------------------------------------------

from __future__             import print_function
import os, sys

dirNm, execName = os.path.split(os.path.realpath(sys.argv[0]))
sys.path.insert(1,os.path.abspath(os.path.join(dirNm, "../libexec")))

from xalt_parse_mpirun_args import find_exec

ignoreT = {
  'env'              : True,
  'time'             : True,
}

argT = {
  '--basepath'                  : 1,
  '--branchcount'               : 1,
  '--configfiles'               : 1,
  '--debuglvl'                  : 1,
  '--hybrid'                    : 1,
  '--ignoreconfigfiles'         : 1,
  '--mpirunoptions'             : 1,
  '--order'                     : 1,
  '--overridepin'               : 1,
  '--qlogicmpi-quiescencecheck' : 1,
  '--schedtype'                 : 1,
  '--stats'                     : 1,
  '--variablesprefix'           : 1,
  '-S'                          : 1,
  '-h'                          : 1,
}

npT = {
  }

def main():
  """
  Find name of executable when using ibrun.
  """

  print(find_exec(ignoreT, argT, npT, None, sys.argv[1:]))

if ( __name__ == '__main__'): main()
<|endoftext|>


<commit_before>import requests
import urllib
import traceback,sys
import json
import rest
import re
from elasticsearch import RequestError
from flask import Response

class ConjunctiveQueryProcessor(object):
    def __init__(self,request,project_name,config_fields,project_root_name,es):
        self.myargs = request.args
        self.preprocess()
        self.field_names = self.myargs.get("_fields",None)
        self.num_results =  self.myargs.get("_size",20)
        self.ordering = self.myargs.get("_order-by",None)
        self.fr = self.myargs.get("_from",0)
        self.verbosity = self.myargs.get("_verbosity","es")
        self.response_format = self.myargs.get("_format","json")
        self.config_fields = config_fields.keys()
        self.config = config_fields
        self.es = es
        self.project_root_name = project_root_name
        self.project_name = project_name
        self.nested_query = self.myargs.get("_dereference",None)
        self.KG_PREFIX = 'knowledge_graph'
        self.SOURCE = '_source'
        self.group_by=self.myargs.get("_group-by",None)
        self.aggregation = self.myargs.get("_aggregation",None)
        self.aggregation_field = self.myargs.get("_aggregation-field",None)
        if self.aggregation_field is not None and self.aggregation is None:
            self.aggregation = "sum"
        self.interval = self.myargs.get("_interval","day")
        self.intervals = ["day","month","week","year","quarter","hour","minute","second"]
        self.aggregations = ["min","max","avg","count","sum"]

    def preprocess(self):
        for arg in self.myargs:
            arg = urllib.unquote(arg)
        return

    def process(self):
        '''
        This is the main function in this class. This calls several functions to validate input, set match clauses, set filter clauses
        set sort clauses and finally resolve any nested documents if needed. Finally this function returns the data as a json or json_lines
        joined by a '\n'
        '''
        valid_input = self.validate_input()
        if not valid_input:
            err_json = {}
            err_json['message'] = "Please enter valid query params. Fields must exist for the given project. If not sure, please access http://mydigurl/projects/<project_name>/fields API for reference"
            return rest.bad_request(err_json)
        query = self._build_query("must")
        res = None
        if self.num_results+self.fr > 10000:
            res = self.es.es_search(self.project_name, self.project_root_name ,query,True, ignore_no_index=True)
        else:
            res = self.es.es_search(self.project_name, self.project_root_name ,query,False, ignore_no_index=True)
        if type(res) == RequestError:
            return rest.bad_request(str(res))
        res_filtered = self.filter_response(res,self.field_names)
        resp={}
        print query
        if self.nested_query is not None and len(res_filtered['hits']['hits']) > 0:
            res_filtered = self.setNestedDocuments(res_filtered)
        if self.group_by is None:
            if self.verbosity == "minimal":
                if self.field_names is None:
                    self.field_names = ','.join(self.config_fields)
                resp = self.minify_response(res_filtered,self.field_names)
            elif self.verbosity == "full":
                resp = res_filtered['hits']['hits']
            else:
                resp = res_filtered
        else:
            resp = res_filtered
        if self.response_format =="json_lines":
            return Response(self.create_json_lines_response(resp),mimetype='application/x-jsonlines')
        return rest.ok(resp)

    def create_json_lines_response(self,resp):
        docs = resp if self.verbosity is not None and self.verbosity =="minimal" else resp['hits']['hits']
        json_lines = '\n'.join([json.dumps(x) for x in docs])
        return json_lines

    def setNestedDocuments(self,resp):
        '''
            This function is to query for the nested documents and retrieve them
            It attaches only the KG component of the nested document into the parent document. Expects a json response,
            returns a json resposne after dereferencing the required set of child documents. Can dereference multiple children
            documents.
            See readme for exact usage and details.
        '''
        list_of_fields = self.nested_query.split(',')
        ids_to_query = {}
        for field in list_of_fields:
            ids_to_query[field] = []
        for json_doc in resp['hits']['hits']:
            for field in list_of_fields:
                try:
                    for nest_doc in json_doc[self.SOURCE][self.KG_PREFIX][field]:
                        temp_id = nest_doc['value']
                        if temp_id not in ids_to_query[field]:
                            ids_to_query[field].append(temp_id)
                except Exception as e:
                    print e
                    pass
        result_map = self.executeNestedQuery(ids_to_query)
        if len(result_map.keys()) > 0:
            for json_doc in resp['hits']['hits']:
                for field in list_of_fields:
                    try:
                        for nest_doc in json_doc[self.SOURCE][self.KG_PREFIX][field]:
                            temp_id = nest_doc['value']
                            if temp_id in result_map:
                                nest_doc['knowledge_graph'] = result_map[temp_id][self.SOURCE][self.KG_PREFIX]
                    except Exception as e:
                        print e
                        pass
        return resp

    def executeNestedQuery(self,idMap):
        '''
            A function to perform the mget search and retrieve the nested documents
        '''
        result_map = {}
        for key in idMap:
            query = {
            "ids" : idMap[key]
            }
            resp = self.es.mget(index=self.project_name,body=query,doc_type=self.project_root_name)
            if resp is not None and len(resp['docs']) > 0:
                for json_doc in resp['docs']:
                    result_map[json_doc['_source']['document_id']] = json_doc
        return result_map

    def validate_input(self):
        for key in self.myargs.keys():
            if not key.startswith("_"):
                if "/" in key:
                    if key.split('/')[0] not in self.config_fields:
                        return False
                elif "$" in key:
                    if key.split('$')[0] not in self.config_fields:
                        return False
                elif "." in key:
                    continue
                elif key not in self.config_fields:
                    return False
        if self.interval is not None:
            gp = re.search(r"(\d+d|\d+m|\d+s|\d+h)",self.interval)
            if gp is None and self.interval not in self.intervals:
                return False
        elif self.group_by is not None and "." not in self.group_by and self.group_by not in self.config_fields:
            return False
        elif self.aggregation_field is not None and "." not in self.aggregation_field and self.aggregation_field not in self.config_fields:
            return False
        elif self.aggregation is not None and self.aggregation not in self.aggregations:
            return False

        return True

    def minify_response(self,response,myargs):
        '''
            This function takes in the response in json format and minimizes the KG Component down to simple key,value pairs
            Also minimizes the KG component inside a particular nested field as well.
        '''
        fields = myargs.split(',')
        docs = response['hits']['hits']
        minified_docs = []
        for json_doc in docs:
            minidoc = {}
            for field in fields:
                try:
                    new_list = []
                    for element in json_doc[self.SOURCE][self.KG_PREFIX][field]:
                        if self.KG_PREFIX in json_doc[self.SOURCE][self.KG_PREFIX][field][0].keys():
                            nested_doc = json.loads(json.dumps(json_doc[self.SOURCE][self.KG_PREFIX][field][0]))
                            nested_kg = nested_doc[self.KG_PREFIX]
                            min_nested_kg = {}
                            for inner_field in nested_kg.keys():
                                nest_list = []
                                for nested_element in nested_kg[inner_field]:
                                    if 'data' in nested_element.keys():
                                        nest_list.append(nested_element['data'])
                                    else:
                                        nest_list.append(nested_element['value'])
                                nested_kg[inner_field] = nest_list
                            nested_doc_id = []
                            nested_doc_id.append(nested_doc['value'])
                            nested_kg['doc_id'] = nested_doc_id
                            new_list.append(nested_kg)
                        elif 'data' in element.keys():
                           new_list.append(element['data'])
                        else:
                           new_list.append(element['value'])
                    minidoc[field] = new_list
                except Exception as e:
                    # exc_type, exc_value, exc_traceback = sys.exc_info()
                    # lines = traceback.format_exception(exc_type, exc_value, exc_traceback)
                    # lines = ''.join(lines)
                    # print lines
                    pass
            doc_id = []
            doc_id.append(json_doc[self.SOURCE]['document_id'])
            minidoc['doc_id'] = doc_id
            minified_docs.append(minidoc)
        return minified_docs

    def generate_match_clause(self,term,args):
        '''
            This function generates match clauses which are inserted into the must part of the conjunctive query
        '''
        extraction = dict()
        must_clause = dict()
        if "/" in term:
            extraction['field_name'],rest = term.split('/')
            extraction['valueorkey'] = rest
        elif "." in term:
            extraction['field_name'],rest = term.split('.')
            extraction['valueorkey'] = rest
        else:
            extraction['field_name'] = term
            extraction['valueorkey'] = "value"
        if "/" in term:
            must_clause = {
                "match": {
                    "knowledge_graph." + extraction['field_name'] + "." + extraction['valueorkey']: urllib.unquote(args[term])
                }
            }
        elif "." in term:
            must_clause = {
                "match": {
                    "knowledge_graph." + extraction['field_name'] + "__" + extraction['valueorkey']: urllib.unquote(args[term])
                }
            }
        else:
            must_clause = {
                "match": {
                    "knowledge_graph." + extraction['field_name'] + "." + extraction['valueorkey']: urllib.unquote(args[term])
                }
            }
        return must_clause

    def get_sort_order(self):
        '''
        This function generates order clauses for the query. It uses the member variables set in class to generate
        any order by clauses if needed. It looks for fields like field_name$desc or field_name$asc.
        '''
        sort_clauses = []
        field_prefix = "knowledge_graph."
        field_suffix = ".value"
        for order in self.ordering.split(','):
            order_key,order_val = order.split('$')
            sort_clause =  { field_prefix + order_key + field_suffix : {"order" : order_val}}
            sort_clauses.append(sort_clause)
        return sort_clauses

    def generate_range_clause(self,term,args):
        '''
        This function converts filter operators such as field.name$desc into a sort clause,
        which can be used in our Elastic search query
        @input a single term of the form field.value$asc or field.value$desc and set of all input argument mappings.
        @output a filter_clause with the field.value and filter type.

        '''
        conversions = { "less-than": "lt", "less-equal-than" : "lte", "greater-than": "gt","greater-equal-than": "gte"}
        extracted_term = term.split('$')
        range_clause = {
                    "range" : {
                         "knowledge_graph."+extracted_term[0]+".value" : {
                                 conversions[extracted_term[1]] : args[term]
                }
            }
        }
        return range_clause

    def _build_query(self,querytype):
        """
        Builds an ElasticSearch query from a simple spec of DIG fields and constraints.
        @param field_query_terms: List of field:value pairs.
        The field can end with "/value" or "/key" to specify where to do the term query.
        @type field_query_terms:
        @return: JSON object with a bool term query in the ElasticSearch DSL.
        @rtype: dict
        """
        self.num_results = int(self.num_results)
        self.fr = int(self.fr)
        clause_list = []
        for query_term in self.myargs:
            if not query_term.startswith("_") and "$" not in query_term:
                clause_list.append(self.generate_match_clause(query_term,self.myargs))
            elif not query_term.startswith("_"):
                clause_list.append(self.generate_range_clause(query_term,self.myargs))
        full_query = {}
        if len(clause_list) > 0:
            full_query['query'] = {
                "bool": {
                    querytype: clause_list
                }
            }

        full_query['size'] = self.num_results
        full_query['from'] = self.fr
        if self.ordering is not None:
            full_query['sort'] = self.get_sort_order()
        if self.group_by is not None:
            full_query['size'] = 0
            if "." not in self.group_by and self.config[self.group_by]['type'] == "date":
                query = self._addDateClause()
                full_query['aggs'] = query
            else:
                query = self._addGroupByClause()
                full_query['aggs'] = query
        return full_query

    def filter_response(self,resp,fields):
        '''
        This function takes a response from elasticSearch and keeps only the specified fields in the 'knowledge_graph' component
        @input : response from ES and fields specified in query
        @output : filter fields in _source.knowledge_graph basis fields specified by user
        '''
        if fields is None:
            return resp
        else:
            docs = resp['hits']['hits']
            for json_doc in docs:
                for field in json_doc[self.SOURCE][self.KG_PREFIX].keys():
                    if field not in fields:
                        del json_doc[self.SOURCE][self.KG_PREFIX][field]
            resp['hits']['hits'] = docs
            return resp

    def _addGroupByClause(self):
        if "." in self.group_by:
            params = self.group_by.split('.')
            self.group_by = params[0] + "__" + params[1]
        full_clause =  {
                  self.group_by: {
                   "terms": {
                    "field":  'knowledge_graph.'+self.group_by+'.key'
                }
            }
        }
        if self.aggregation_field is not None:
            if "." in self.aggregation_field:
                params = self.aggregation_field.split('.')
                self.aggregation_field = params[0] + "__" + params[1]
            agg_clause = {
                    self.aggregation_field: {
                     self.aggregation : {
                      "field": 'knowledge_graph.'+self.aggregation_field+'.key'
                    }
                }
            }
            full_clause['aggs'] = agg_clause
        return full_clause

    def _addDateClause(self):
        date_clause = {
            self.group_by : {
                "date_histogram" : {
                    "field" : 'knowledge_graph.'+self.group_by+'.key',
                    "interval" : self.interval
                }
            }
        }
        if self.aggregation_field is not None:
            if "." in self.aggregation_field:
                params = self.aggregation_field.split('.')
                self.aggregation_field = params[0] + "__" + params[1]
            agg_clause = {
                    self.aggregation_field: {
                     self.aggregation : {
                      "field": 'knowledge_graph.'+self.aggregation_field+'.key'
                    }
                }
            }
            date_clause[self.group_by]['aggs'] = agg_clause
        return date_clause
<commit_msg>added . to __ conversion for order by and filtering flows<commit_after>import requests
import urllib
import traceback,sys
import json
import rest
import re
from elasticsearch import RequestError
from flask import Response

class ConjunctiveQueryProcessor(object):
    def __init__(self,request,project_name,config_fields,project_root_name,es):
        self.myargs = request.args
        self.preprocess()
        self.field_names = self.myargs.get("_fields",None)
        self.num_results =  self.myargs.get("_size",20)
        self.ordering = self.myargs.get("_order-by",None)
        self.fr = self.myargs.get("_from",0)
        self.verbosity = self.myargs.get("_verbosity","es")
        self.response_format = self.myargs.get("_format","json")
        self.config_fields = config_fields.keys()
        self.config = config_fields
        self.es = es
        self.project_root_name = project_root_name
        self.project_name = project_name
        self.nested_query = self.myargs.get("_dereference",None)
        self.KG_PREFIX = 'knowledge_graph'
        self.SOURCE = '_source'
        self.group_by=self.myargs.get("_group-by",None)
        self.aggregation = self.myargs.get("_aggregation",None)
        self.aggregation_field = self.myargs.get("_aggregation-field",None)
        if self.aggregation_field is not None and self.aggregation is None:
            self.aggregation = "sum"
        self.interval = self.myargs.get("_interval","day")
        self.intervals = ["day","month","week","year","quarter","hour","minute","second"]
        self.aggregations = ["min","max","avg","count","sum"]

    def preprocess(self):
        for arg in self.myargs:
            arg = urllib.unquote(arg)
        return

    def process(self):
        '''
        This is the main function in this class. This calls several functions to validate input, set match clauses, set filter clauses
        set sort clauses and finally resolve any nested documents if needed. Finally this function returns the data as a json or json_lines
        joined by a '\n'
        '''
        valid_input = self.validate_input()
        if not valid_input:
            err_json = {}
            err_json['message'] = "Please enter valid query params. Fields must exist for the given project. If not sure, please access http://mydigurl/projects/<project_name>/fields API for reference"
            return rest.bad_request(err_json)
        query = self._build_query("must")
        res = None
        print query
        if self.num_results+self.fr > 10000:
            res = self.es.es_search(self.project_name, self.project_root_name ,query,True, ignore_no_index=True)
        else:
            res = self.es.es_search(self.project_name, self.project_root_name ,query,False, ignore_no_index=True)
        if type(res) == RequestError:
            return rest.bad_request(str(res))
        res_filtered = self.filter_response(res,self.field_names)
        resp={}
        if self.nested_query is not None and len(res_filtered['hits']['hits']) > 0:
            res_filtered = self.setNestedDocuments(res_filtered)
        if self.group_by is None:
            if self.verbosity == "minimal":
                if self.field_names is None:
                    self.field_names = ','.join(self.config_fields)
                resp = self.minify_response(res_filtered,self.field_names)
            elif self.verbosity == "full":
                resp = res_filtered['hits']['hits']
            else:
                resp = res_filtered
        else:
            resp = res_filtered
        if self.response_format =="json_lines":
            return Response(self.create_json_lines_response(resp),mimetype='application/x-jsonlines')
        return rest.ok(resp)

    def create_json_lines_response(self,resp):
        docs = resp if self.verbosity is not None and self.verbosity =="minimal" else resp['hits']['hits']
        json_lines = '\n'.join([json.dumps(x) for x in docs])
        return json_lines

    def setNestedDocuments(self,resp):
        '''
            This function is to query for the nested documents and retrieve them
            It attaches only the KG component of the nested document into the parent document. Expects a json response,
            returns a json resposne after dereferencing the required set of child documents. Can dereference multiple children
            documents.
            See readme for exact usage and details.
        '''
        list_of_fields = self.nested_query.split(',')
        ids_to_query = {}
        for field in list_of_fields:
            ids_to_query[field] = []
        for json_doc in resp['hits']['hits']:
            for field in list_of_fields:
                try:
                    for nest_doc in json_doc[self.SOURCE][self.KG_PREFIX][field]:
                        temp_id = nest_doc['value']
                        if temp_id not in ids_to_query[field]:
                            ids_to_query[field].append(temp_id)
                except Exception as e:
                    print e
                    pass
        result_map = self.executeNestedQuery(ids_to_query)
        if len(result_map.keys()) > 0:
            for json_doc in resp['hits']['hits']:
                for field in list_of_fields:
                    try:
                        for nest_doc in json_doc[self.SOURCE][self.KG_PREFIX][field]:
                            temp_id = nest_doc['value']
                            if temp_id in result_map:
                                nest_doc['knowledge_graph'] = result_map[temp_id][self.SOURCE][self.KG_PREFIX]
                    except Exception as e:
                        print e
                        pass
        return resp

    def executeNestedQuery(self,idMap):
        '''
            A function to perform the mget search and retrieve the nested documents
        '''
        result_map = {}
        for key in idMap:
            query = {
            "ids" : idMap[key]
            }
            resp = self.es.mget(index=self.project_name,body=query,doc_type=self.project_root_name)
            if resp is not None and len(resp['docs']) > 0:
                for json_doc in resp['docs']:
                    result_map[json_doc['_source']['document_id']] = json_doc
        return result_map

    def validate_input(self):
        for key in self.myargs.keys():
            if not key.startswith("_"):
                if "/" in key:
                    if key.split('/')[0] not in self.config_fields:
                        return False
                elif "." in key:
                    continue
                elif "$" in key:
                    if key.split('$')[0] not in self.config_fields:
                        return False
                elif key not in self.config_fields:
                    return False
        if self.interval is not None:
            gp = re.search(r"(\d+d|\d+m|\d+s|\d+h)",self.interval)
            if gp is None and self.interval not in self.intervals:
                return False
        elif self.group_by is not None and "." not in self.group_by and self.group_by not in self.config_fields:
            return False
        elif self.aggregation_field is not None and "." not in self.aggregation_field and self.aggregation_field not in self.config_fields:
            return False
        elif self.aggregation is not None and self.aggregation not in self.aggregations:
            return False

        return True

    def minify_response(self,response,myargs):
        '''
            This function takes in the response in json format and minimizes the KG Component down to simple key,value pairs
            Also minimizes the KG component inside a particular nested field as well.
        '''
        fields = myargs.split(',')
        docs = response['hits']['hits']
        minified_docs = []
        for json_doc in docs:
            minidoc = {}
            for field in fields:
                try:
                    new_list = []
                    for element in json_doc[self.SOURCE][self.KG_PREFIX][field]:
                        if self.KG_PREFIX in json_doc[self.SOURCE][self.KG_PREFIX][field][0].keys():
                            nested_doc = json.loads(json.dumps(json_doc[self.SOURCE][self.KG_PREFIX][field][0]))
                            nested_kg = nested_doc[self.KG_PREFIX]
                            min_nested_kg = {}
                            for inner_field in nested_kg.keys():
                                nest_list = []
                                for nested_element in nested_kg[inner_field]:
                                    if 'data' in nested_element.keys():
                                        nest_list.append(nested_element['data'])
                                    else:
                                        nest_list.append(nested_element['value'])
                                nested_kg[inner_field] = nest_list
                            nested_doc_id = []
                            nested_doc_id.append(nested_doc['value'])
                            nested_kg['doc_id'] = nested_doc_id
                            new_list.append(nested_kg)
                        elif 'data' in element.keys():
                           new_list.append(element['data'])
                        else:
                           new_list.append(element['value'])
                    minidoc[field] = new_list
                except Exception as e:
                    # exc_type, exc_value, exc_traceback = sys.exc_info()
                    # lines = traceback.format_exception(exc_type, exc_value, exc_traceback)
                    # lines = ''.join(lines)
                    # print lines
                    pass
            doc_id = []
            doc_id.append(json_doc[self.SOURCE]['document_id'])
            minidoc['doc_id'] = doc_id
            minified_docs.append(minidoc)
        return minified_docs

    def generate_match_clause(self,term,args):
        '''
            This function generates match clauses which are inserted into the must part of the conjunctive query
        '''
        extraction = dict()
        must_clause = dict()
        if "/" in term:
            extraction['field_name'],rest = term.split('/')
            extraction['valueorkey'] = rest
        elif "." in term:
            temp,rest = term.split('.')
            extraction['field_name'] = temp + '__' + rest
            extraction['valueorkey'] = "value"
        else:
            extraction['field_name'] = term
            extraction['valueorkey'] = "value"
        if "/" in term or "." in term:
            must_clause = {
                "match": {
                    "knowledge_graph." + extraction['field_name'] + "." + extraction['valueorkey']: urllib.unquote(args[term])
                }
            }
        else:
            must_clause = {
                "match": {
                    "knowledge_graph." + extraction['field_name'] + "." + extraction['valueorkey']: urllib.unquote(args[term])
                }
            }
        return must_clause

    def get_sort_order(self):
        '''
        This function generates order clauses for the query. It uses the member variables set in class to generate
        any order by clauses if needed. It looks for fields like field_name$desc or field_name$asc.
        '''
        sort_clauses = []
        field_prefix = "knowledge_graph."
        field_suffix = ".value"
        for order in self.ordering.split(','):
            order_key,order_val = order.split('$')
            if "." in order_key:
                order_key = order_key.replace(".","__")
            sort_clause =  { field_prefix + order_key + field_suffix : {"order" : order_val}}
            sort_clauses.append(sort_clause)
        return sort_clauses

    def generate_range_clause(self,term,args):
        '''
        This function converts filter operators such as field.name$desc into a sort clause,
        which can be used in our Elastic search query
        @input a single term of the form field.value$asc or field.value$desc and set of all input argument mappings.
        @output a filter_clause with the field.value and filter type.

        '''
        conversions = { "less-than": "lt", "less-equal-than" : "lte", "greater-than": "gt","greater-equal-than": "gte"}
        extracted_term = term.split('$')
        if "." in extracted_term[0]:
            extracted_term[0] = extracted_term[0].replace(".","__")
        range_clause = {
                    "range" : {
                         "knowledge_graph."+extracted_term[0]+".value" : {
                                 conversions[extracted_term[1]] : args[term]
                }
            }
        }
        return range_clause

    def _build_query(self,querytype):
        """
        Builds an ElasticSearch query from a simple spec of DIG fields and constraints.
        @param field_query_terms: List of field:value pairs.
        The field can end with "/value" or "/key" to specify where to do the term query.
        @type field_query_terms:
        @return: JSON object with a bool term query in the ElasticSearch DSL.
        @rtype: dict
        """
        self.num_results = int(self.num_results)
        self.fr = int(self.fr)
        clause_list = []
        for query_term in self.myargs:
            if not query_term.startswith("_") and "$" not in query_term:
                clause_list.append(self.generate_match_clause(query_term,self.myargs))
            elif not query_term.startswith("_"):
                clause_list.append(self.generate_range_clause(query_term,self.myargs))
        full_query = {}
        if len(clause_list) > 0:
            full_query['query'] = {
                "bool": {
                    querytype: clause_list
                }
            }

        full_query['size'] = self.num_results
        full_query['from'] = self.fr
        if self.ordering is not None:
            full_query['sort'] = self.get_sort_order()
        if self.group_by is not None:
            full_query['size'] = 0
            if "." not in self.group_by and self.config[self.group_by]['type'] == "date":
                query = self._addDateClause()
                full_query['aggs'] = query
            else:
                query = self._addGroupByClause()
                full_query['aggs'] = query
        return full_query

    def filter_response(self,resp,fields):
        '''
        This function takes a response from elasticSearch and keeps only the specified fields in the 'knowledge_graph' component
        @input : response from ES and fields specified in query
        @output : filter fields in _source.knowledge_graph basis fields specified by user
        '''
        if fields is None:
            return resp
        else:
            docs = resp['hits']['hits']
            for json_doc in docs:
                for field in json_doc[self.SOURCE][self.KG_PREFIX].keys():
                    if field not in fields:
                        del json_doc[self.SOURCE][self.KG_PREFIX][field]
            resp['hits']['hits'] = docs
            return resp

    def _addGroupByClause(self):
        if "." in self.group_by:
            params = self.group_by.split('.')
            self.group_by = params[0] + "__" + params[1]
        full_clause =  {
                  self.group_by: {
                   "terms": {
                    "field":  'knowledge_graph.'+self.group_by+'.key'
                }
            }
        }
        if self.aggregation_field is not None:
            if "." in self.aggregation_field:
                params = self.aggregation_field.split('.')
                self.aggregation_field = params[0] + "__" + params[1]
            agg_clause = {
                    self.aggregation_field: {
                     self.aggregation : {
                      "field": 'knowledge_graph.'+self.aggregation_field+'.key'
                    }
                }
            }
            full_clause['aggs'] = agg_clause
        return full_clause

    def _addDateClause(self):
        date_clause = {
            self.group_by : {
                "date_histogram" : {
                    "field" : 'knowledge_graph.'+self.group_by+'.key',
                    "interval" : self.interval
                }
            }
        }
        if self.aggregation_field is not None:
            if "." in self.aggregation_field:
                params = self.aggregation_field.split('.')
                self.aggregation_field = params[0] + "__" + params[1]
            agg_clause = {
                    self.aggregation_field: {
                     self.aggregation : {
                      "field": 'knowledge_graph.'+self.aggregation_field+'.key'
                    }
                }
            }
            date_clause[self.group_by]['aggs'] = agg_clause
        return date_clause
<|endoftext|>