<commit_before>"""
    isort.py

    Exposes a simple library to sort through imports within Python code

    usage:
        SortImports(file_name)
    or:
        sorted = SortImports(file_contents=file_contents).output

    Copyright (C) 2013  Timothy Edmund Crosley

    Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated
    documentation files (the "Software"), to deal in the Software without restriction, including without limitation
    the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and
    to permit persons to whom the Software is furnished to do so, subject to the following conditions:

    The above copyright notice and this permission notice shall be included in all copies or
    substantial portions of the Software.

    THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED
    TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
    THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF
    CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
    OTHER DEALINGS IN THE SOFTWARE.
"""

from __future__ import absolute_import, division, print_function, unicode_literals

import copy
import os
from sys import path as PYTHONPATH, stderr

from natsort import natsorted
from pies import *

from . import settings

class Sections(object):
    FUTURE = 1
    STDLIB = 2
    THIRDPARTY = 3
    FIRSTPARTY = 4
    ALL = (FUTURE, STDLIB, THIRDPARTY, FIRSTPARTY)

class SortImports(object):
    config = settings.default

    def __init__(self, file_path=None, file_contents=None, **setting_overrides):
        if setting_overrides:
            self.config = settings.default.copy()
            self.config.update(setting_overrides)

        file_name = file_path
        if file_path:
            if "/" in file_name:
                file_name = file_name[file_name.rfind('/') + 1:]
            if file_name in self.config['skip']:
                print("WARNING: {0} was skipped as it's listed in 'skip' setting".format(file_path), file=stderr)
            self.file_path = file_path
            with open(file_path) as file:
                file_contents = file.read()

        if file_contents is None:
            return

        self.in_lines = file_contents.split("\n")
        self.number_of_lines = len(self.in_lines)

        self.out_lines = []
        self.imports = {}
        self.as_map = {}
        for section in Sections.ALL:
            self.imports[section] = {'straight':set(), 'from':{}}

        self.index = 0
        self.import_index = -1
        self._parse()
        if self.import_index != -1:
            self._add_formatted_imports()

        self.output = "\n".join(self.out_lines)
        if file_name:
            with open(file_name, "w") as outputFile:
                outputFile.write(self.output)

    def place_module(self, moduleName):
        """Trys to determine if a module is a python std import,
           third party import, or project code:
           if it can't determine - it assumes it is project code
        """
        index = moduleName.find('.')
        if index:
            firstPart = moduleName[:index]
        else:
            firstPart = None

        if moduleName == "__future__" or (firstPart == "__future__"):
            return Sections.FUTURE
        elif moduleName in self.config['known_standard_library'] or \
                (firstPart in self.config['known_standard_library']):
            return Sections.STDLIB
        elif moduleName in self.config['known_third_party'] or (firstPart in self.config['known_third_party']):
            return Sections.THIRDPARTY
        elif moduleName in self.config['known_first_party'] or (firstPart in self.config['known_first_party']):
            return Sections.FIRSTPARTY

        for prefix in PYTHONPATH:
            fixed_module_name = moduleName.replace('.', '/')
            base_path = prefix + "/" + fixed_module_name
            if os.path.exists(base_path + ".py") or os.path.exists(base_path) or os.path.exists(base_path + ".so"):
                if "site-packages" in prefix or "dist-packages" in prefix:
                    return Sections.THIRDPARTY
                elif "python2" in prefix.lower() or "python3" in prefix.lower():
                    return Sections.STDLIB
                else:
                    return Sections.FIRSTPARTY

        return Sections.FIRSTPARTY

    def _get_line(self):
        """ Returns the current line from the file while
            incrementing the index
        """
        line = self.in_lines[self.index]
        self.index += 1
        return line

    @staticmethod
    def _import_type(line):
        """ If the current line is an import line it will
            return its type (from or straight)
        """
        if "isort:skip" in line:
            return
        elif line.startswith('import '):
            return "straight"
        elif line.startswith('from ') and "import" in line:
            return "from"

    def _at_end(self):
        """ returns True if we are at the end of the file """
        return self.index == self.number_of_lines

    @staticmethod
    def _module_key(module_name, config):
        module_name = str(module_name).lower()
        return "{0}{1}".format(module_name in config['force_to_top'] and "A" or "B",
                               config['length_sort'] and len(module_name) or module_name)

    def _add_formatted_imports(self):
        """ Adds the imports back to the file
            (at the index of the first import)
            sorted alphabetically and split between groups
        """
        output = []
        for section in Sections.ALL:
            straight_modules = list(self.imports[section]['straight'])
            straight_modules.sort(key=lambda key: self._module_key(key, self.config))

            for module in straight_modules:
                if module in self.as_map:
                    output.append("import {0} as {1}".format(module, self.as_map[module]))
                else:
                    output.append("import {0}".format(module))

            from_modules = list(self.imports[section]['from'].keys())
            from_modules = natsorted(from_modules, key=lambda key: self._module_key(key, self.config))
            for module in from_modules:
                import_start = "from {0} import ".format(module)
                from_imports = list(self.imports[section]['from'][module])
                from_imports = natsorted(from_imports, key=lambda key: self._module_key(key, self.config))
                for from_import in copy.copy(from_imports):
                    import_as = self.as_map.get(module + "." + from_import, False)
                    if import_as:
                        output.append(import_start + "{0} as {1}".format(from_import, import_as))
                        from_imports.remove(from_import)

                if from_imports:
                    if "*" in from_imports:
                        import_statement = "{0}*".format(import_start)
                    else:
                        import_statement = import_start + (", ").join(from_imports)
                        if len(import_statement) > self.config['line_length'] and len(from_imports) > 1:
                            import_statement = import_start
                            if self.config['multi_line_output'] != settings.MultiLineOutput.HANGING_INDENT:
                                import_statement += "("
                                white_space = " " * len(import_statement)
                            if self.config['multi_line_output'] == settings.MultiLineOutput.GRID:
                                import_statement += from_imports.pop(0)
                                while from_imports:
                                    next_import = from_imports.pop(0)
                                    next_statement = import_statement + ", " + next_import
                                    if len(next_statement.split("\n")[-1]) + 1 > self.config['line_length']:
                                        next_statement = "{0},\n{1}{2}".format(import_statement, white_space,
                                                                               next_import)
                                    import_statement = next_statement
                                import_statement += ")"
                            elif self.config['multi_line_output'] == settings.MultiLineOutput.VERTICAL:
                                import_statement += (",\n" + white_space).join(from_imports)
                                import_statement += ")"
                            elif self.config['multi_line_output'] == settings.MultiLineOutput.VERTICAL_HANGING_INDENT:
                                import_statement += "\n" + self.config['indent']
                                import_statement += (",\n" + self.config['indent']).join(from_imports)
                                import_statement += "\n)"
                            elif self.config['multi_line_output'] == settings.MultiLineOutput.HANGING_INDENT:
                                import_statement += " " + from_imports.pop(0)
                                while from_imports:
                                    next_import = from_imports.pop(0)
                                    next_statement = import_statement + ", " + next_import
                                    if len(next_statement.split("\n")[-1]) + 3 > self.config['line_length']:
                                        next_statement = "{0}, \\\n{1}{2}".format(import_statement,
                                                                                self.config['indent'],
                                                                                next_import)
                                    import_statement = next_statement

                    output.append(import_statement)

            if straight_modules or from_modules:
                output.append("")

        while output[-1:] == [""]:
            output.pop()

        while self.import_index + 2 < len(self.out_lines) and self.out_lines[self.import_index + 1] == "":
            self.out_lines.pop(self.import_index + 1)

        if len(self.out_lines) > self.import_index + 1:
            next_construct = self.out_lines[self.import_index + 1]
            if next_construct.startswith("def") or next_construct.startswith("class") or \
               next_construct.startswith("@"):
                output += ["", ""]
            else:
                output += [""]

        self.out_lines[self.import_index:1] = output

    @staticmethod
    def _strip_comments(line):
        """
            Removes comments from import line.
        """
        comment_start = line.find("#")
        if comment_start != -1:
            print("Removing comment(%s) so imports can be sorted correctly" % line[comment_start:], file=stderr)
            line = line[:comment_start]

        return line

    def _parse(self):
        """
            Parses a python file taking out and categorizing imports
        """
        while True:
            if self._at_end():
                return None

            line = self._get_line()
            import_type = self._import_type(line)
            if import_type:
                if self.import_index == -1:
                    self.import_index = self.index - 1

                import_string = self._strip_comments(line)
                if "(" in line and not self._at_end():
                    while not line.strip().endswith(")") and not self._at_end():
                        line = self._strip_comments(self._get_line())
                        import_string += "\n" + line
                else:
                    while line.strip().endswith("\\"):
                        line = self._strip_comments(self._get_line())
                        import_string += "\n" + line

                import_string = import_string.replace("_import", "[[i]]")
                for remove_syntax in ['\\', '(', ')', ",", 'from ', 'import ']:
                    import_string = import_string.replace(remove_syntax, " ")
                import_string = import_string.replace("[[i]]", "_import")

                imports = import_string.split()
                if "as" in imports and import_type != 'from':
                    while True:
                        try:
                            index = imports.index('as')
                        except:
                            break
                        self.as_map[imports[index - 1]] = imports[index + 1]
                        from_import = imports[index - 1]
                        module_placment = self.place_module(from_import)
                        self.imports[module_placment][import_type].update([from_import])
                        del imports[index -1:index + 2]
                elif import_type == 'from' and "as" in imports:
                    while True:
                        try:
                            index = imports.index('as')
                        except:
                            break
                        from_import = imports[0]
                        self.as_map[from_import] = imports[index + 1]
                        module_placment = self.place_module(from_import)
                        imports = ["{0} as {1}".format(imports[index - 1], imports[index + 1])]
                        self.imports[module_placment][import_type].setdefault(from_import, set()).update(imports)
                        del imports[index -1:index + 1]
                if import_type == "from":
                    impot_from = imports.pop(0)
                    root = self.imports[self.place_module(impot_from)][import_type]
                    if root.get(impot_from, False):
                        root[impot_from].update(imports)
                    else:
                        root[impot_from] = set(imports)
                else:
                    for module in imports:
                        self.imports[self.place_module(module)][import_type].add(module)

                if self._at_end():
                    print(self.file_path + ": Either you have an import at the end of your file, or something"
                                           " went horribly wrong!",
                          file=stderr)
                    sys.exit(1)

            else:
                self.out_lines.append(line)
<commit_msg>Write to absolute file path<commit_after>"""
    isort.py

    Exposes a simple library to sort through imports within Python code

    usage:
        SortImports(file_name)
    or:
        sorted = SortImports(file_contents=file_contents).output

    Copyright (C) 2013  Timothy Edmund Crosley

    Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated
    documentation files (the "Software"), to deal in the Software without restriction, including without limitation
    the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and
    to permit persons to whom the Software is furnished to do so, subject to the following conditions:

    The above copyright notice and this permission notice shall be included in all copies or
    substantial portions of the Software.

    THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED
    TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
    THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF
    CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
    OTHER DEALINGS IN THE SOFTWARE.
"""

from __future__ import absolute_import, division, print_function, unicode_literals

import copy
import os
from sys import path as PYTHONPATH, stderr

from natsort import natsorted
from pies import *

from . import settings

class Sections(object):
    FUTURE = 1
    STDLIB = 2
    THIRDPARTY = 3
    FIRSTPARTY = 4
    ALL = (FUTURE, STDLIB, THIRDPARTY, FIRSTPARTY)

class SortImports(object):
    config = settings.default

    def __init__(self, file_path=None, file_contents=None, **setting_overrides):
        if setting_overrides:
            self.config = settings.default.copy()
            self.config.update(setting_overrides)

        file_name = file_path
        if file_path:
            file_path = os.path.abspath(file_path)
            if "/" in file_name:
                file_name = file_name[file_name.rfind('/') + 1:]
            if file_name in self.config['skip']:
                print("WARNING: {0} was skipped as it's listed in 'skip' setting".format(file_path), file=stderr)
            self.file_path = file_path
            with open(file_path) as file:
                file_contents = file.read()

        if file_contents is None:
            return

        self.in_lines = file_contents.split("\n")
        self.number_of_lines = len(self.in_lines)

        self.out_lines = []
        self.imports = {}
        self.as_map = {}
        for section in Sections.ALL:
            self.imports[section] = {'straight':set(), 'from':{}}

        self.index = 0
        self.import_index = -1
        self._parse()
        if self.import_index != -1:
            self._add_formatted_imports()

        self.output = "\n".join(self.out_lines)
        if file_name:
            with open(self.file_path, "w") as outputFile:
                outputFile.write(self.output)

    def place_module(self, moduleName):
        """Trys to determine if a module is a python std import,
           third party import, or project code:
           if it can't determine - it assumes it is project code
        """
        index = moduleName.find('.')
        if index:
            firstPart = moduleName[:index]
        else:
            firstPart = None

        if moduleName == "__future__" or (firstPart == "__future__"):
            return Sections.FUTURE
        elif moduleName in self.config['known_standard_library'] or \
                (firstPart in self.config['known_standard_library']):
            return Sections.STDLIB
        elif moduleName in self.config['known_third_party'] or (firstPart in self.config['known_third_party']):
            return Sections.THIRDPARTY
        elif moduleName in self.config['known_first_party'] or (firstPart in self.config['known_first_party']):
            return Sections.FIRSTPARTY

        for prefix in PYTHONPATH:
            fixed_module_name = moduleName.replace('.', '/')
            base_path = prefix + "/" + fixed_module_name
            if os.path.exists(base_path + ".py") or os.path.exists(base_path) or os.path.exists(base_path + ".so"):
                if "site-packages" in prefix or "dist-packages" in prefix:
                    return Sections.THIRDPARTY
                elif "python2" in prefix.lower() or "python3" in prefix.lower():
                    return Sections.STDLIB
                else:
                    return Sections.FIRSTPARTY

        return Sections.FIRSTPARTY

    def _get_line(self):
        """ Returns the current line from the file while
            incrementing the index
        """
        line = self.in_lines[self.index]
        self.index += 1
        return line

    @staticmethod
    def _import_type(line):
        """ If the current line is an import line it will
            return its type (from or straight)
        """
        if "isort:skip" in line:
            return
        elif line.startswith('import '):
            return "straight"
        elif line.startswith('from ') and "import" in line:
            return "from"

    def _at_end(self):
        """ returns True if we are at the end of the file """
        return self.index == self.number_of_lines

    @staticmethod
    def _module_key(module_name, config):
        module_name = str(module_name).lower()
        return "{0}{1}".format(module_name in config['force_to_top'] and "A" or "B",
                               config['length_sort'] and len(module_name) or module_name)

    def _add_formatted_imports(self):
        """ Adds the imports back to the file
            (at the index of the first import)
            sorted alphabetically and split between groups
        """
        output = []
        for section in Sections.ALL:
            straight_modules = list(self.imports[section]['straight'])
            straight_modules.sort(key=lambda key: self._module_key(key, self.config))

            for module in straight_modules:
                if module in self.as_map:
                    output.append("import {0} as {1}".format(module, self.as_map[module]))
                else:
                    output.append("import {0}".format(module))

            from_modules = list(self.imports[section]['from'].keys())
            from_modules = natsorted(from_modules, key=lambda key: self._module_key(key, self.config))
            for module in from_modules:
                import_start = "from {0} import ".format(module)
                from_imports = list(self.imports[section]['from'][module])
                from_imports = natsorted(from_imports, key=lambda key: self._module_key(key, self.config))
                for from_import in copy.copy(from_imports):
                    import_as = self.as_map.get(module + "." + from_import, False)
                    if import_as:
                        output.append(import_start + "{0} as {1}".format(from_import, import_as))
                        from_imports.remove(from_import)

                if from_imports:
                    if "*" in from_imports:
                        import_statement = "{0}*".format(import_start)
                    else:
                        import_statement = import_start + (", ").join(from_imports)
                        if len(import_statement) > self.config['line_length'] and len(from_imports) > 1:
                            import_statement = import_start
                            if self.config['multi_line_output'] != settings.MultiLineOutput.HANGING_INDENT:
                                import_statement += "("
                                white_space = " " * len(import_statement)
                            if self.config['multi_line_output'] == settings.MultiLineOutput.GRID:
                                import_statement += from_imports.pop(0)
                                while from_imports:
                                    next_import = from_imports.pop(0)
                                    next_statement = import_statement + ", " + next_import
                                    if len(next_statement.split("\n")[-1]) + 1 > self.config['line_length']:
                                        next_statement = "{0},\n{1}{2}".format(import_statement, white_space,
                                                                               next_import)
                                    import_statement = next_statement
                                import_statement += ")"
                            elif self.config['multi_line_output'] == settings.MultiLineOutput.VERTICAL:
                                import_statement += (",\n" + white_space).join(from_imports)
                                import_statement += ")"
                            elif self.config['multi_line_output'] == settings.MultiLineOutput.VERTICAL_HANGING_INDENT:
                                import_statement += "\n" + self.config['indent']
                                import_statement += (",\n" + self.config['indent']).join(from_imports)
                                import_statement += "\n)"
                            elif self.config['multi_line_output'] == settings.MultiLineOutput.HANGING_INDENT:
                                import_statement += " " + from_imports.pop(0)
                                while from_imports:
                                    next_import = from_imports.pop(0)
                                    next_statement = import_statement + ", " + next_import
                                    if len(next_statement.split("\n")[-1]) + 3 > self.config['line_length']:
                                        next_statement = "{0}, \\\n{1}{2}".format(import_statement,
                                                                                self.config['indent'],
                                                                                next_import)
                                    import_statement = next_statement

                    output.append(import_statement)

            if straight_modules or from_modules:
                output.append("")

        while output[-1:] == [""]:
            output.pop()

        while self.import_index + 2 < len(self.out_lines) and self.out_lines[self.import_index + 1] == "":
            self.out_lines.pop(self.import_index + 1)

        if len(self.out_lines) > self.import_index + 1:
            next_construct = self.out_lines[self.import_index + 1]
            if next_construct.startswith("def") or next_construct.startswith("class") or \
               next_construct.startswith("@"):
                output += ["", ""]
            else:
                output += [""]

        self.out_lines[self.import_index:1] = output

    @staticmethod
    def _strip_comments(line):
        """
            Removes comments from import line.
        """
        comment_start = line.find("#")
        if comment_start != -1:
            print("Removing comment(%s) so imports can be sorted correctly" % line[comment_start:], file=stderr)
            line = line[:comment_start]

        return line

    def _parse(self):
        """
            Parses a python file taking out and categorizing imports
        """
        while True:
            if self._at_end():
                return None

            line = self._get_line()
            import_type = self._import_type(line)
            if import_type:
                if self.import_index == -1:
                    self.import_index = self.index - 1

                import_string = self._strip_comments(line)
                if "(" in line and not self._at_end():
                    while not line.strip().endswith(")") and not self._at_end():
                        line = self._strip_comments(self._get_line())
                        import_string += "\n" + line
                else:
                    while line.strip().endswith("\\"):
                        line = self._strip_comments(self._get_line())
                        import_string += "\n" + line

                import_string = import_string.replace("_import", "[[i]]")
                for remove_syntax in ['\\', '(', ')', ",", 'from ', 'import ']:
                    import_string = import_string.replace(remove_syntax, " ")
                import_string = import_string.replace("[[i]]", "_import")

                imports = import_string.split()
                if "as" in imports and import_type != 'from':
                    while True:
                        try:
                            index = imports.index('as')
                        except:
                            break
                        self.as_map[imports[index - 1]] = imports[index + 1]
                        from_import = imports[index - 1]
                        module_placment = self.place_module(from_import)
                        self.imports[module_placment][import_type].update([from_import])
                        del imports[index -1:index + 2]
                elif import_type == 'from' and "as" in imports:
                    while True:
                        try:
                            index = imports.index('as')
                        except:
                            break
                        from_import = imports[0]
                        self.as_map[from_import] = imports[index + 1]
                        module_placment = self.place_module(from_import)
                        imports = ["{0} as {1}".format(imports[index - 1], imports[index + 1])]
                        self.imports[module_placment][import_type].setdefault(from_import, set()).update(imports)
                        del imports[index -1:index + 1]
                if import_type == "from":
                    impot_from = imports.pop(0)
                    root = self.imports[self.place_module(impot_from)][import_type]
                    if root.get(impot_from, False):
                        root[impot_from].update(imports)
                    else:
                        root[impot_from] = set(imports)
                else:
                    for module in imports:
                        self.imports[self.place_module(module)][import_type].add(module)

                if self._at_end():
                    print(self.file_path + ": Either you have an import at the end of your file, or something"
                                           " went horribly wrong!",
                          file=stderr)
                    sys.exit(1)

            else:
                self.out_lines.append(line)
<|endoftext|>


<commit_before>__author__ = 'sallai'
from __future__ import division
from copy import deepcopy
from atom import Atom

import numpy as np

from compound import Compound
from bond import Bond
from box import Box
from coordinate_transform import equivalence_transform, translate
from periodic_kdtree import PeriodicCKDTree

def apply_mask(host, guest, mask, guest_port_name="port"):
    """ """
    box = host.boundingbox(excludeG=False)

    mask = mask * box.lengths + box.mins

    n_ports = len(host.referenced_ports())
    assert(n_ports >= mask.shape[0])

    port_pos = np.empty((n_ports,3))
    port_list = []
    for pidx, port in enumerate(host.referenced_ports()):
        port_pos[pidx, :] = port.middle.pos
        port_list.append(port)

    for mp in mask:
        closest_point_idx = np.argmin(host.min_periodic_distance(mp, port_pos))
        closest_port = port_list[closest_point_idx]
        brush = deepcopy(guest)
        equivalence_transform(brush, brush.labels[guest_port_name], closest_port)
        host.add(brush)
        port_pos[closest_point_idx,:] = np.array([np.inf, np.inf, np.inf])

def random_mask_3d(num_sites):
    """ """
    mask = np.random.random((num_sites, 3))
    return mask

def random_mask_2d(num_sites):
    """ """
    mask = random_mask_3d(num_sites)
    mask[:, 2] = 0
    return mask

def grid_mask_2d(n, m):
    """ """
    mask = np.zeros(shape=(n*m, 3), dtype=float)
    for i in range(n):
        for j in range(m):
            mask[i*m + j, 0] = i / n
            mask[i*m + j, 1] = j / m
    return mask

def grid_mask_3d(n, m, l):
    """ """
    mask = np.zeros(shape=(n*m*l, 3), dtype=float)
    for i in range(n):
        for j in range(m):
            for k in range(l):
                mask[i*m*l + j*l + k, 0] = i / n
                mask[i*m*l + j*l + k, 1] = j / m
                mask[i*m*l + j*l + k, 2] = k / l
    return mask

def sphere_mask(N):
    """Generate N evenly distributed points on the unit sphere.

    Sphere is centered at the origin. Alrgorithm based onthe 'Golden Spiral'.

    Code by Chris Colbert  numpy-discussion list:
    http://mail.scipy.org/pipermail/numpy-discussion/2009-July/043811.html

    """
    phi = (1 + np.sqrt(5)) / 2  # the golden ratio
    long_incr = 2*np.pi / phi   # how much to increment the longitude

    dz = 2.0 / float(N)         # a unit sphere has diameter 2
    bands = np.arange(N)        # each band will have one point placed on it
    z = bands * dz - 1 + (dz/2) # the height z of each band/point
    r = np.sqrt(1 - z*z)        # project onto xy-plane
    az = bands * long_incr      # azimuthal angle of point modulo 2 pi
    x = r * np.cos(az)
    y = r * np.sin(az)
    return np.column_stack((x, y, z))

def disk_mask(n):
    """ """
    radius = np.sqrt(np.arange(n) / float(n))

    golden_angle = np.pi * (3 - np.sqrt(5))
    theta = golden_angle * np.arange(n)

    points = np.zeros((n, 2))
    points[:,0] = np.cos(theta)
    points[:,1] = np.sin(theta)
    points *= radius.reshape((n, 1))

    return points

def vdw_radius(atomic_number):
    return 1.5

def solvate(host_compound, guest_compound, host_bounds, guest_bounds):
    """Solvate a Compound in a Box.

    Typical usage of this function would be to use a pre-equilibrated solvent
    system as the guest which will then be replicated to fill the host's Box.

    Args:
        host_compound (Compound):
        guest_compound (Compound):
        host_bounds (Box):
        guest_bounds (Box):

    """
    assert isinstance(host_bounds, Box)
    assert isinstance(guest_bounds, Box)
    assert np.all(guest_compound.periodicity == 0)

    # TODO: we may want to make sure that the axes of the two boxes line up

    # Replicate the guest so that it fills or overfills the host box
    host_atom_list = [atom for atom in host_compound.atoms() if atom.kind != 'G']
    host_atom_pos_list = [atom.pos for atom in host_atom_list]
    kdtree = PeriodicCKDTree(host_atom_pos_list)

    num_replicas = np.ceil(host_bounds.lengths / guest_bounds.lengths)
    num_replicas = num_replicas.astype('int')
    for xi in range(0,num_replicas[0]):
        for yi in range(0,num_replicas[0]):
            for zi in range(0,num_replicas[0]):
                guest = deepcopy(guest_compound)
                translate(guest, -guest_bounds.mins + host_bounds.mins + np.array([xi, yi, zi])*guest_bounds.lengths)

                # Remove atoms outside the host's box and anything bonded to them.
                guest_atoms = guest.atom_list_by_kind('*')
                atoms_to_remove = set()
                guest_atom_pos_list = [atom.pos for atom in guest_compound.atoms()]
                atom_indicies = np.where(np.logical_or(np.any(guest_atom_pos_list < host_bounds.mins, axis=1),
                        np.any(guest_atom_pos_list > host_bounds.maxes, axis=1)))[0]
                for ai in atom_indicies:
                    atoms_to_remove.add(guest_atoms[ai])
                    atoms_to_remove.update(guest_atoms[ai].bonded_atoms())
                guest.remove(atoms_to_remove)

                # Remove overlapping atoms and anything bonded to them.
                atoms_to_remove = set()
                for guest_atom in guest.atoms():
                    _, neighbors = kdtree.query(guest_atom.pos, k=10)
                    for host_atom_idx in neighbors:
                        if host_atom_idx < len(host_atom_list):
                            host_atom = host_atom_list[host_atom_idx]
                            if host_compound.min_periodic_distance(host_atom, guest_atom) < (vdw_radius(host_atom) + vdw_radius(guest_atom)):
                                atoms_to_remove.add(guest_atom)
                                atoms_to_remove.update(guest_atom.bonded_atoms())

                guest.remove(atoms_to_remove)
                host_compound.add(guest, "guest_{0}_{1}_{2}".format(xi,yi,zi))

def add_bond(compound, type_A, type_B, dmin, dmax):
    """Ai-Bj distance is in [dmin, dmax] => add bond A1xB(Ai,Bj) (symmetric)."""
    for a1 in compound.atom_list_by_kind(type_A):
        nearest = compound.atoms_in_range(a1.pos, dmax, kind=type_B)
        for b1 in nearest:
            if (b1.kind==type_B) and (dmin <= compound.min_periodic_distance(b1.pos, a1.pos) <= dmax):
                compound.add(Bond(a1, b1))

if __name__ == "__main__":
    print "hello"

<commit_msg>typos<commit_after>__author__ = 'sallai'
from __future__ import division
from copy import deepcopy
from atom import Atom

import numpy as np

from compound import Compound
from bond import Bond
from box import Box
from coordinate_transform import equivalence_transform, translate
from periodic_kdtree import PeriodicCKDTree

def apply_mask(host, guest, mask, guest_port_name="port"):
    """ """
    box = host.boundingbox(excludeG=False)

    mask = mask * box.lengths + box.mins

    n_ports = len(host.referenced_ports())
    assert(n_ports >= mask.shape[0])

    port_pos = np.empty((n_ports,3))
    port_list = []
    for pidx, port in enumerate(host.referenced_ports()):
        port_pos[pidx, :] = port.middle.pos
        port_list.append(port)

    for mp in mask:
        closest_point_idx = np.argmin(host.min_periodic_distance(mp, port_pos))
        closest_port = port_list[closest_point_idx]
        brush = deepcopy(guest)
        equivalence_transform(brush, brush.labels[guest_port_name], closest_port)
        host.add(brush)
        port_pos[closest_point_idx,:] = np.array([np.inf, np.inf, np.inf])

def random_mask_3d(num_sites):
    """ """
    mask = np.random.random((num_sites, 3))
    return mask

def random_mask_2d(num_sites):
    """ """
    mask = random_mask_3d(num_sites)
    mask[:, 2] = 0
    return mask

def grid_mask_2d(n, m):
    """ """
    mask = np.zeros(shape=(n*m, 3), dtype=float)
    for i in range(n):
        for j in range(m):
            mask[i*m + j, 0] = i / n
            mask[i*m + j, 1] = j / m
    return mask

def grid_mask_3d(n, m, l):
    """ """
    mask = np.zeros(shape=(n*m*l, 3), dtype=float)
    for i in range(n):
        for j in range(m):
            for k in range(l):
                mask[i*m*l + j*l + k, 0] = i / n
                mask[i*m*l + j*l + k, 1] = j / m
                mask[i*m*l + j*l + k, 2] = k / l
    return mask

def sphere_mask(N):
    """Generate N evenly distributed points on the unit sphere.

    Sphere is centered at the origin. Alrgorithm based on the 'Golden Spiral'.

    Code by Chris Colbert from the numpy-discussion list:
    http://mail.scipy.org/pipermail/numpy-discussion/2009-July/043811.html

    """
    phi = (1 + np.sqrt(5)) / 2  # the golden ratio
    long_incr = 2*np.pi / phi   # how much to increment the longitude

    dz = 2.0 / float(N)         # a unit sphere has diameter 2
    bands = np.arange(N)        # each band will have one point placed on it
    z = bands * dz - 1 + (dz/2) # the height z of each band/point
    r = np.sqrt(1 - z*z)        # project onto xy-plane
    az = bands * long_incr      # azimuthal angle of point modulo 2 pi
    x = r * np.cos(az)
    y = r * np.sin(az)
    return np.column_stack((x, y, z))

def disk_mask(n):
    """ """
    radius = np.sqrt(np.arange(n) / float(n))

    golden_angle = np.pi * (3 - np.sqrt(5))
    theta = golden_angle * np.arange(n)

    points = np.zeros((n, 2))
    points[:,0] = np.cos(theta)
    points[:,1] = np.sin(theta)
    points *= radius.reshape((n, 1))

    return points

def vdw_radius(atomic_number):
    return 1.5

def solvate(host_compound, guest_compound, host_bounds, guest_bounds):
    """Solvate a Compound in a Box.

    Typical usage of this function would be to use a pre-equilibrated solvent
    system as the guest which will then be replicated to fill the host's Box.
    All Compounds that have overlapping Atoms with the host are removed.

    Args:
        host_compound (Compound):
        guest_compound (Compound):
        host_bounds (Box):
        guest_bounds (Box):

    """
    assert isinstance(host_bounds, Box)
    assert isinstance(guest_bounds, Box)
    assert np.all(guest_compound.periodicity == 0)

    # TODO: we may want to make sure that the axes of the two boxes line up

    # Replicate the guest so that it fills or overfills the host box.
    host_atom_list = [atom for atom in host_compound.atoms() if atom.kind != 'G']
    host_atom_pos_list = [atom.pos for atom in host_atom_list]
    kdtree = PeriodicCKDTree(host_atom_pos_list)

    num_replicas = np.ceil(host_bounds.lengths / guest_bounds.lengths)
    num_replicas = num_replicas.astype('int')
    for xi in range(0,num_replicas[0]):
        for yi in range(0,num_replicas[0]):
            for zi in range(0,num_replicas[0]):
                guest = deepcopy(guest_compound)
                translate(guest, -guest_bounds.mins + host_bounds.mins + np.array([xi, yi, zi])*guest_bounds.lengths)

                # Remove atoms outside the host's box and anything bonded to them.
                guest_atoms = guest.atom_list_by_kind('*')
                atoms_to_remove = set()
                guest_atom_pos_list = [atom.pos for atom in guest_compound.atoms()]
                atom_indicies = np.where(np.logical_or(np.any(guest_atom_pos_list < host_bounds.mins, axis=1),
                        np.any(guest_atom_pos_list > host_bounds.maxes, axis=1)))[0]
                for ai in atom_indicies:
                    atoms_to_remove.add(guest_atoms[ai])
                    atoms_to_remove.update(guest_atoms[ai].bonded_atoms())
                guest.remove(atoms_to_remove)

                # Remove overlapping atoms and anything bonded to them.
                atoms_to_remove = set()
                for guest_atom in guest.atoms():
                    _, neighbors = kdtree.query(guest_atom.pos, k=10)
                    for host_atom_idx in neighbors:
                        if host_atom_idx < len(host_atom_list):
                            host_atom = host_atom_list[host_atom_idx]
                            if host_compound.min_periodic_distance(host_atom, guest_atom) < (vdw_radius(host_atom) + vdw_radius(guest_atom)):
                                atoms_to_remove.add(guest_atom)
                                atoms_to_remove.update(guest_atom.bonded_atoms())

                guest.remove(atoms_to_remove)
                host_compound.add(guest, "guest_{0}_{1}_{2}".format(xi,yi,zi))

def add_bond(compound, type_A, type_B, dmin, dmax):
    """Ai-Bj distance is in [dmin, dmax] => add bond A1xB(Ai,Bj) (symmetric)."""
    for a1 in compound.atom_list_by_kind(type_A):
        nearest = compound.atoms_in_range(a1.pos, dmax, kind=type_B)
        for b1 in nearest:
            if (b1.kind==type_B) and (dmin <= compound.min_periodic_distance(b1.pos, a1.pos) <= dmax):
                compound.add(Bond(a1, b1))

if __name__ == "__main__":
    print "hello"

<|endoftext|>


<commit_before>from __future__ import unicode_literals
from fig.project import Project
from .testcases import DockerClientTestCase

class ProjectTest(DockerClientTestCase):
    def test_from_dict(self):
        project = Project.from_dicts('figtest', [
            {
                'name': 'web',
                'image': 'ubuntu'
            },
            {
                'name': 'db',
                'image': 'ubuntu'
            }
        ], self.client)
        self.assertEqual(len(project.services), 2)
        self.assertEqual(project.get_service('web').name, 'web')
        self.assertEqual(project.get_service('web').options['image'], 'ubuntu')
        self.assertEqual(project.get_service('db').name, 'db')
        self.assertEqual(project.get_service('db').options['image'], 'ubuntu')

    def test_from_dict_sorts_in_dependency_order(self):
        project = Project.from_dicts('figtest', [
            {
                'name': 'web',
                'image': 'ubuntu',
                'links': ['db'],
            },
            {
                'name': 'db',
                'image': 'ubuntu'
            }
        ], self.client)

        self.assertEqual(project.services[0].name, 'db')
        self.assertEqual(project.services[1].name, 'web')

    def test_get_service(self):
        web = self.create_service('web')
        project = Project('test', [web], self.client)
        self.assertEqual(project.get_service('web'), web)

    def test_recreate_containers(self):
        web = self.create_service('web')
        db = self.create_service('db')
        project = Project('test', [web, db], self.client)

        old_web_container = web.create_container()
        self.assertEqual(len(web.containers(stopped=True)), 1)
        self.assertEqual(len(db.containers(stopped=True)), 0)

        (old, new) = project.recreate_containers()
        self.assertEqual(old, [(web, old_web_container)])
        self.assertEqual(len(new), 2)
        self.assertEqual(new[0][0], web)
        self.assertEqual(new[1][0], db)

        self.assertEqual(len(web.containers(stopped=True)), 2)
        self.assertEqual(len(db.containers(stopped=True)), 1)

    def test_start_stop_kill_remove(self):
        web = self.create_service('web')
        db = self.create_service('db')
        project = Project('figtest', [web, db], self.client)

        project.start()

        self.assertEqual(len(web.containers()), 0)
        self.assertEqual(len(db.containers()), 0)

        web_container_1 = web.create_container()
        web_container_2 = web.create_container()
        db_container = db.create_container()

        project.start(service_names=['web'])
        self.assertEqual(set(c.name for c in project.containers()), set([web_container_1.name, web_container_2.name]))

        project.start()
        self.assertEqual(set(c.name for c in project.containers()), set([web_container_1.name, web_container_2.name, db_container.name]))

        project.stop(service_names=['web'], timeout=1)
        self.assertEqual(set(c.name for c in project.containers()), set([db_container.name]))

        project.kill(service_names=['db'])
        self.assertEqual(len(project.containers()), 0)
        self.assertEqual(len(project.containers(stopped=True)), 3)

        project.remove_stopped(service_names=['web'])
        self.assertEqual(len(project.containers(stopped=True)), 1)

        project.remove_stopped()
        self.assertEqual(len(project.containers(stopped=True)), 0)
<commit_msg>Fix test regression<commit_after>from __future__ import unicode_literals
from fig.project import Project
from .testcases import DockerClientTestCase

class ProjectTest(DockerClientTestCase):
    def test_from_dict(self):
        project = Project.from_dicts('figtest', [
            {
                'name': 'web',
                'image': 'ubuntu'
            },
            {
                'name': 'db',
                'image': 'ubuntu'
            }
        ], self.client)
        self.assertEqual(len(project.services), 2)
        self.assertEqual(project.get_service('web').name, 'web')
        self.assertEqual(project.get_service('web').options['image'], 'ubuntu')
        self.assertEqual(project.get_service('db').name, 'db')
        self.assertEqual(project.get_service('db').options['image'], 'ubuntu')

    def test_from_dict_sorts_in_dependency_order(self):
        project = Project.from_dicts('figtest', [
            {
                'name': 'web',
                'image': 'ubuntu',
                'links': ['db'],
            },
            {
                'name': 'db',
                'image': 'ubuntu'
            }
        ], self.client)

        self.assertEqual(project.services[0].name, 'db')
        self.assertEqual(project.services[1].name, 'web')

    def test_get_service(self):
        web = self.create_service('web')
        project = Project('test', [web], self.client)
        self.assertEqual(project.get_service('web'), web)

    def test_recreate_containers(self):
        web = self.create_service('web')
        db = self.create_service('db')
        project = Project('test', [web, db], self.client)

        old_web_container = web.create_container()
        self.assertEqual(len(web.containers(stopped=True)), 1)
        self.assertEqual(len(db.containers(stopped=True)), 0)

        (old, new) = project.recreate_containers()
        self.assertEqual(len(old), 1)
        self.assertEqual(old[0][0], web)
        self.assertEqual(len(new), 2)
        self.assertEqual(new[0][0], web)
        self.assertEqual(new[1][0], db)

        self.assertEqual(len(web.containers(stopped=True)), 1)
        self.assertEqual(len(db.containers(stopped=True)), 1)

    def test_start_stop_kill_remove(self):
        web = self.create_service('web')
        db = self.create_service('db')
        project = Project('figtest', [web, db], self.client)

        project.start()

        self.assertEqual(len(web.containers()), 0)
        self.assertEqual(len(db.containers()), 0)

        web_container_1 = web.create_container()
        web_container_2 = web.create_container()
        db_container = db.create_container()

        project.start(service_names=['web'])
        self.assertEqual(set(c.name for c in project.containers()), set([web_container_1.name, web_container_2.name]))

        project.start()
        self.assertEqual(set(c.name for c in project.containers()), set([web_container_1.name, web_container_2.name, db_container.name]))

        project.stop(service_names=['web'], timeout=1)
        self.assertEqual(set(c.name for c in project.containers()), set([db_container.name]))

        project.kill(service_names=['db'])
        self.assertEqual(len(project.containers()), 0)
        self.assertEqual(len(project.containers(stopped=True)), 3)

        project.remove_stopped(service_names=['web'])
        self.assertEqual(len(project.containers(stopped=True)), 1)

        project.remove_stopped()
        self.assertEqual(len(project.containers(stopped=True)), 0)
<|endoftext|>


<commit_before>import sys
from pymongo import MongoClient
import numpy as np
from tqdm import tqdm
from dateutil import parser
import json

def averageWorldDistribution ():
	client = MongoClient()
	unloc = client.twitterdb.stream_nongeo_coordinates

	cursor = unloc.find({},{ 'user.utc_offset': 1 })
	tweets = []
	times = []

	print("getting tweets from database")
	for tweet in tqdm(cursor):
		#print (json.dumps(tweet, indent=4, sort_keys=True))
		tweets.append(tweet['user']['utc_offset'])

   	tzone_dict = {w: 0 for w in tweets}

   	print('made a dictionary. Now counting tweets by time zones')
   	for tweet in tqdm(tweets):
   		tzone_dict[tweet] += 1

   	for zone, value in tzone_dict.items():
   		print(float(zone)/(60*60), value)

averageWorldDistribution()<commit_msg>world distribution<commit_after>import sys
from pymongo import MongoClient
import numpy as np
from tqdm import tqdm
from dateutil import parser
import json

def averageWorldDistribution ():
	client = MongoClient()
	unloc = client.twitterdb.stream_nongeo_coordinates

	cursor = unloc.find({},{ 'user.utc_offset': 1, 'user.time_zone': 1 })
	tweets = []
	times = []

	print("getting tweets from database")
	for tweet in tqdm(cursor):
		#print (json.dumps(tweet, indent=4, sort_keys=True))
		tweets.append((tweet['user']['utc_offset'], tweet['user']['time_zone']))

   	tzone2_dict = {w:tz for w, tz in tweets}
   	tzone_dict = {w: 0 for w, tz in tweets}

   	print('made a dictionary. Now counting tweets by time zones')
   	for tweet in tqdm(tweets):
   		tzone_dict[tweet] += 1

   	for zone, value in tzone_dict.items():
   		print(float(zone)/(60*60), value, tzone2_dict[zone])

averageWorldDistribution()<|endoftext|>


<commit_before>"""Python unit tests for gapi.errors"""

import json

import unittest
from unittest.mock import patch

import googleapiclient.errors
from gapi import errors

def create_simple_http_error(status, reason, message):
  content = errors._create_http_error_dict(status, reason, message)
  return create_http_error(status, content)

def create_http_error(status, content):
  response = {
      'status': status,
      'content-type': 'application/json',
  }
  content_as_bytes = json.dumps(content).encode('UTF-8')
  return googleapiclient.errors.HttpError(response, content_as_bytes)

class ErrorsTest(unittest.TestCase):

  def test_get_gapi_error_detail_quota_exceeded(self):
    # TODO: Add test logic once the opening ValueError exception case has a
    # repro case (i.e. an Exception type/format that will cause it to raise).
    pass

  def test_get_gapi_error_detail_invalid_domain(self):
    # TODO: Add test logic once the opening ValueError exception case has a
    # repro case (i.e. an Exception type/format that will cause it to raise).
    pass

  def test_get_gapi_error_detail_invalid_signing_key(self):
    # TODO: Add test logic once the opening ValueError exception case has a
    # repro case (i.e. an Exception type/format that will cause it to raise).
    pass

  def test_get_gapi_error_detail_unknown_error(self):
    # TODO: Add test logic once the opening ValueError exception case has a
    # repro case (i.e. an Exception type/format that will cause it to raise).
    pass

  def test_get_gapi_error_retry_http_error(self):
    # TODO: Add test logic once the opening ValueError exception case has a
    # repro case (i.e. an Exception type/format that will cause it to raise).
    pass

  def test_get_gapi_error_prints_soft_errors(self):
    # TODO: Add test logic once the opening ValueError exception case has a
    # repro case (i.e. an Exception type/format that will cause it to raise).
    pass

  def test_get_gapi_error_exits_on_unrecoverable_errors(self):
    # TODO: Add test logic once the opening ValueError exception case has a
    # repro case (i.e. an Exception type/format that will cause it to raise).
    pass

  def test_get_gapi_error_quota_exceeded_for_current_request(self):
    # TODO: Add test logic once the opening ValueError exception case has a
    # repro case (i.e. an Exception type/format that will cause it to raise).
    pass

  def test_get_gapi_error_quota_exceeded_high_request_rate(self):
    # TODO: Add test logic once the opening ValueError exception case has a
    # repro case (i.e. an Exception type/format that will cause it to raise).
    pass

  def test_get_gapi_error_extracts_user_not_found(self):
    err = create_simple_http_error(404, 'notFound',
                                   'Resource Not Found: userKey.')
    http_status, reason, message = errors.get_gapi_error_detail(err)
    self.assertEqual(http_status, 404)
    self.assertEqual(reason, errors.ErrorReason.USER_NOT_FOUND.value)
    self.assertEqual(message, 'Resource Not Found: userKey.')

  def test_get_gapi_error_extracts_group_not_found(self):
    err = create_simple_http_error(404, 'notFound',
                                   'Resource Not Found: groupKey.')
    http_status, reason, message = errors.get_gapi_error_detail(err)
    self.assertEqual(http_status, 404)
    self.assertEqual(reason, errors.ErrorReason.GROUP_NOT_FOUND.value)
    self.assertEqual(message, 'Resource Not Found: groupKey.')

  def test_get_gapi_error_extracts_member_not_found(self):
    err = create_simple_http_error(404, 'notFound',
                                   'Resource Not Found: memberKey.')
    http_status, reason, message = errors.get_gapi_error_detail(err)
    self.assertEqual(http_status, 404)
    self.assertEqual(reason, errors.ErrorReason.MEMBER_NOT_FOUND.value)
    self.assertEqual(message, 'Resource Not Found: memberKey.')

  def test_get_gapi_error_extracts_domain_not_found(self):
    err = create_simple_http_error(404, 'notFound', 'Domain not found.')
    http_status, reason, message = errors.get_gapi_error_detail(err)
    self.assertEqual(http_status, 404)
    self.assertEqual(reason, errors.ErrorReason.DOMAIN_NOT_FOUND.value)
    self.assertEqual(message, 'Domain not found.')

  def test_get_gapi_error_extracts_generic_resource_not_found(self):
    err = create_simple_http_error(404, 'notFound',
                                   'Resource Not Found: unknownResource.')
    http_status, reason, message = errors.get_gapi_error_detail(err)
    self.assertEqual(http_status, 404)
    self.assertEqual(reason, errors.ErrorReason.RESOURCE_NOT_FOUND.value)
    self.assertEqual(message, 'Resource Not Found: unknownResource.')

  def test_get_gapi_error_extracts_invalid_userid(self):
    err = create_simple_http_error(400, 'invalid', 'Invalid Input: userId')
    http_status, reason, message = errors.get_gapi_error_detail(err)
    self.assertEqual(http_status, 400)
    self.assertEqual(reason, errors.ErrorReason.USER_NOT_FOUND.value)
    self.assertEqual(message, 'Invalid Input: userId')

  def test_get_gapi_error_extracts_invalid_member(self):
    err = create_simple_http_error(400, 'invalid', 'Invalid Input: memberKey')
    http_status, reason, message = errors.get_gapi_error_detail(err)
    self.assertEqual(http_status, 400)
    self.assertEqual(reason, errors.ErrorReason.INVALID_MEMBER.value)
    self.assertEqual(message, 'Invalid Input: memberKey')

  def test_get_gapi_error_extracts_bad_request(self):
    err = create_simple_http_error(400, 'failedPrecondition', 'Bad Request')
    http_status, reason, message = errors.get_gapi_error_detail(err)
    self.assertEqual(http_status, 400)
    self.assertEqual(reason, errors.ErrorReason.BAD_REQUEST.value)
    self.assertEqual(message, 'Bad Request')

  def test_get_gapi_error_extracts_service_not_available(self):
    err = create_simple_http_error(400, 'failedPrecondition',
                                   'Mail service not enabled')
    http_status, reason, message = errors.get_gapi_error_detail(err)
    self.assertEqual(http_status, 400)
    self.assertEqual(reason, errors.ErrorReason.SERVICE_NOT_AVAILABLE.value)
    self.assertEqual(message, 'Mail service not enabled')

  def test_get_gapi_error_extracts_required_member_not_found(self):
    err = create_simple_http_error(400, 'required',
                                   'Missing required field: memberKey')
    http_status, reason, message = errors.get_gapi_error_detail(err)
    self.assertEqual(http_status, 400)
    self.assertEqual(reason, errors.ErrorReason.MEMBER_NOT_FOUND.value)
    self.assertEqual(message, 'Missing required field: memberKey')

  def test_get_gapi_error_extracts_cyclic_memberships_error(self):
    err = create_simple_http_error(400, 'conditionNotMet',
                                   'Cyclic memberships not allowed')
    http_status, reason, message = errors.get_gapi_error_detail(err)
    self.assertEqual(http_status, 400)
    self.assertEqual(reason,
                     errors.ErrorReason.CYCLIC_MEMBERSHIPS_NOT_ALLOWED.value)
    self.assertEqual(message, 'Cyclic memberships not allowed')

  def test_get_gapi_error_extracts_single_error_with_message(self):
    status_code = 999
    response = {'status': status_code}
    # This error does not have an "errors" key describing each error.
    content = {'error': {'code': status_code, 'message': 'unknown error'}}
    content_as_bytes = json.dumps(content).encode('UTF-8')
    err = errors.googleapiclient.errors.HttpError(response, content_as_bytes)

    http_status, reason, message = errors.get_gapi_error_detail(err)
    self.assertEqual(http_status, status_code)
    self.assertEqual(reason, str(status_code))
    self.assertEqual(message, content['error']['message'])

  def test_get_gapi_error_exits_code_4_on_malformed_error_with_unknown_description(
      self):
    status_code = 999
    response = {'status': status_code}
    # This error only has an error_description_field and an unknown description.
    content = {'error_description': 'something errored'}
    content_as_bytes = json.dumps(content).encode('UTF-8')
    err = errors.googleapiclient.errors.HttpError(response, content_as_bytes)

    with self.assertRaises(SystemExit) as context:
      errors.get_gapi_error_detail(err)
    self.assertEqual(4, context.exception.code)

  def test_get_gapi_error_exits_on_invalid_error_description(self):
    status_code = 400
    response = {'status': status_code}
    content = {'error_description': 'Invalid Value'}
    content_as_bytes = json.dumps(content).encode('UTF-8')
    err = errors.googleapiclient.errors.HttpError(response, content_as_bytes)

    http_status, reason, message = errors.get_gapi_error_detail(err)
    self.assertEqual(http_status, status_code)
    self.assertEqual(reason, errors.ErrorReason.INVALID.value)
    self.assertEqual(message, 'Invalid Value')

  def test_get_gapi_error_exits_code_4_on_unexpected_error_contents(self):
    status_code = 900
    response = {'status': status_code}
    content = {'notErrorContentThatIsExpected': 'foo'}
    content_as_bytes = json.dumps(content).encode('UTF-8')
    err = errors.googleapiclient.errors.HttpError(response, content_as_bytes)

    with self.assertRaises(SystemExit) as context:
      errors.get_gapi_error_detail(err)
    self.assertEqual(4, context.exception.code)

if __name__ == '__main__':
  unittest.main()
<commit_msg>modify gapi error tests<commit_after>"""Python unit tests for gapi.errors"""

import json

import unittest
from unittest.mock import patch

import googleapiclient.errors
from gapi import errors

def create_simple_http_error(status, reason, message):
  content = errors._create_http_error_dict(status, reason, message)
  return create_http_error(status, content)

def create_http_error(status, content):
  response = {
      'status': status,
      'content-type': 'application/json',
  }
  content_as_bytes = json.dumps(content).encode('UTF-8')
  return googleapiclient.errors.HttpError(response, content_as_bytes)

class ErrorsTest(unittest.TestCase):

  def test_get_gapi_error_detail_quota_exceeded(self):
    # TODO: Add test logic once the opening ValueError exception case has a
    # repro case (i.e. an Exception type/format that will cause it to raise).
    pass

  def test_get_gapi_error_detail_invalid_domain(self):
    # TODO: Add test logic once the opening ValueError exception case has a
    # repro case (i.e. an Exception type/format that will cause it to raise).
    pass

  def test_get_gapi_error_detail_invalid_signing_key(self):
    # TODO: Add test logic once the opening ValueError exception case has a
    # repro case (i.e. an Exception type/format that will cause it to raise).
    pass

  def test_get_gapi_error_detail_unknown_error(self):
    # TODO: Add test logic once the opening ValueError exception case has a
    # repro case (i.e. an Exception type/format that will cause it to raise).
    pass

  def test_get_gapi_error_retry_http_error(self):
    # TODO: Add test logic once the opening ValueError exception case has a
    # repro case (i.e. an Exception type/format that will cause it to raise).
    pass

  def test_get_gapi_error_prints_soft_errors(self):
    # TODO: Add test logic once the opening ValueError exception case has a
    # repro case (i.e. an Exception type/format that will cause it to raise).
    pass

  def test_get_gapi_error_exits_on_unrecoverable_errors(self):
    # TODO: Add test logic once the opening ValueError exception case has a
    # repro case (i.e. an Exception type/format that will cause it to raise).
    pass

  def test_get_gapi_error_quota_exceeded_for_current_request(self):
    # TODO: Add test logic once the opening ValueError exception case has a
    # repro case (i.e. an Exception type/format that will cause it to raise).
    pass

  def test_get_gapi_error_quota_exceeded_high_request_rate(self):
    # TODO: Add test logic once the opening ValueError exception case has a
    # repro case (i.e. an Exception type/format that will cause it to raise).
    pass

  def test_get_gapi_error_extracts_user_not_found(self):
    err = create_simple_http_error(404, 'notFound',
                                   'Resource Not Found: userKey.')
    http_status, reason, message = errors.get_gapi_error_detail(err)
    self.assertEqual(http_status, 404)
    self.assertEqual(reason, errors.ErrorReason.USER_NOT_FOUND.value)
    self.assertEqual(message, 'Resource Not Found: userKey.')

  def test_get_gapi_error_extracts_group_not_found(self):
    err = create_simple_http_error(404, 'notFound',
                                   'Resource Not Found: groupKey.')
    http_status, reason, message = errors.get_gapi_error_detail(err)
    self.assertEqual(http_status, 404)
    self.assertEqual(reason, errors.ErrorReason.GROUP_NOT_FOUND.value)
    self.assertEqual(message, 'Resource Not Found: groupKey.')

  def test_get_gapi_error_extracts_member_not_found(self):
    err = create_simple_http_error(404, 'notFound',
                                   'Resource Not Found: memberKey.')
    http_status, reason, message = errors.get_gapi_error_detail(err)
    self.assertEqual(http_status, 404)
    self.assertEqual(reason, errors.ErrorReason.MEMBER_NOT_FOUND.value)
    self.assertEqual(message, 'Resource Not Found: memberKey.')

  def test_get_gapi_error_extracts_domain_not_found(self):
    err = create_simple_http_error(404, 'notFound', 'Domain not found.')
    http_status, reason, message = errors.get_gapi_error_detail(err)
    self.assertEqual(http_status, 404)
    self.assertEqual(reason, errors.ErrorReason.DOMAIN_NOT_FOUND.value)
    self.assertEqual(message, 'Domain not found.')

  def test_get_gapi_error_extracts_generic_resource_not_found(self):
    err = create_simple_http_error(404, 'notFound',
                                   'Resource Not Found: unknownResource.')
    http_status, reason, message = errors.get_gapi_error_detail(err)
    self.assertEqual(http_status, 404)
    self.assertEqual(reason, errors.ErrorReason.RESOURCE_NOT_FOUND.value)
    self.assertEqual(message, 'Resource Not Found: unknownResource.')

  def test_get_gapi_error_extracts_invalid_userid(self):
    err = create_simple_http_error(400, 'invalid', 'Invalid Input: userId')
    http_status, reason, message = errors.get_gapi_error_detail(err)
    self.assertEqual(http_status, 400)
    self.assertEqual(reason, errors.ErrorReason.USER_NOT_FOUND.value)
    self.assertEqual(message, 'Invalid Input: userId')

  def test_get_gapi_error_extracts_invalid_member(self):
    err = create_simple_http_error(400, 'invalid', 'Invalid Input: memberKey')
    http_status, reason, message = errors.get_gapi_error_detail(err)
    self.assertEqual(http_status, 400)
    self.assertEqual(reason, errors.ErrorReason.INVALID_MEMBER.value)
    self.assertEqual(message, 'Invalid Input: memberKey')

  def test_get_gapi_error_extracts_bad_request(self):
    err = create_simple_http_error(400, 'failedPrecondition', 'Bad Request')
    http_status, reason, message = errors.get_gapi_error_detail(err)
    self.assertEqual(http_status, 400)
    self.assertEqual(reason, errors.ErrorReason.BAD_REQUEST.value)
    self.assertEqual(message, 'Bad Request')

  def test_get_gapi_error_extracts_service_not_available(self):
    err = create_simple_http_error(400, 'failedPrecondition',
                                   'Mail service not enabled')
    http_status, reason, message = errors.get_gapi_error_detail(err)
    self.assertEqual(http_status, 400)
    self.assertEqual(reason, errors.ErrorReason.SERVICE_NOT_AVAILABLE.value)
    self.assertEqual(message, 'Mail service not enabled')

  def test_get_gapi_error_extracts_required_member_not_found(self):
    err = create_simple_http_error(400, 'required',
                                   'Missing required field: memberKey')
    http_status, reason, message = errors.get_gapi_error_detail(err)
    self.assertEqual(http_status, 400)
    self.assertEqual(reason, errors.ErrorReason.MEMBER_NOT_FOUND.value)
    self.assertEqual(message, 'Missing required field: memberKey')

  def test_get_gapi_error_extracts_cyclic_memberships_error(self):
    err = create_simple_http_error(400, 'conditionNotMet',
                                   'Cyclic memberships not allowed')
    http_status, reason, message = errors.get_gapi_error_detail(err)
    self.assertEqual(http_status, 400)
    self.assertEqual(reason,
                     errors.ErrorReason.CYCLIC_MEMBERSHIPS_NOT_ALLOWED.value)
    self.assertEqual(message, 'Cyclic memberships not allowed')

  def test_get_gapi_error_extracts_single_error_with_message(self):
    status_code = 999
    response = {'status': status_code}
    # This error does not have an "errors" key describing each error.
    content = {'error': {'code': status_code, 'message': 'unknown error'}}
    content_as_bytes = json.dumps(content).encode('UTF-8')
    err = googleapiclient.errors.HttpError(response, content_as_bytes)

    http_status, reason, message = errors.get_gapi_error_detail(err)
    self.assertEqual(http_status, status_code)
    self.assertEqual(reason, str(status_code))
    self.assertEqual(message, content['error']['message'])

  def test_get_gapi_error_exits_code_4_on_malformed_error_with_unknown_description(
      self):
    status_code = 999
    response = {'status': status_code}
    # This error only has an error_description_field and an unknown description.
    content = {'error_description': 'something errored'}
    content_as_bytes = json.dumps(content).encode('UTF-8')
    err = googleapiclient.errors.HttpError(response, content_as_bytes)

    with self.assertRaises(SystemExit) as context:
      errors.get_gapi_error_detail(err)
    self.assertEqual(4, context.exception.code)

  def test_get_gapi_error_exits_on_invalid_error_description(self):
    status_code = 400
    response = {'status': status_code}
    content = {'error_description': 'Invalid Value'}
    content_as_bytes = json.dumps(content).encode('UTF-8')
    err = googleapiclient.errors.HttpError(response, content_as_bytes)

    http_status, reason, message = errors.get_gapi_error_detail(err)
    self.assertEqual(http_status, status_code)
    self.assertEqual(reason, errors.ErrorReason.INVALID.value)
    self.assertEqual(message, 'Invalid Value')

  def test_get_gapi_error_exits_code_4_on_unexpected_error_contents(self):
    status_code = 900
    response = {'status': status_code}
    content = {'notErrorContentThatIsExpected': 'foo'}
    content_as_bytes = json.dumps(content).encode('UTF-8')
    err = googleapiclient.errors.HttpError(response, content_as_bytes)

    with self.assertRaises(SystemExit) as context:
      errors.get_gapi_error_detail(err)
    self.assertEqual(4, context.exception.code)

if __name__ == '__main__':
  unittest.main()
<|endoftext|>


<commit_before>class EmitterError(Exception):
  pass

def emit_hostmask(nickname, username, host):
  return "{}!{}@{}".format(nickname, username, host)

def emit_message(prefix, command, params):
  buf = []

  if prefix is not None:
    buf.append(":" + prefix)

  buf.append(command)

  if params:
    *init, last = params

    for param in init:
      if " " in param:
        raise EmitterError("malformed message")

    if " " in last or last[0] == ":":
      buf.extend(init)
      buf.append(":" + last)
    else:
      buf.extend(params)

  return " ".join(buf)
<commit_msg>handle awkward edge case<commit_after>class EmitterError(Exception):
  pass

def emit_hostmask(nickname, username, host):
  return "{}!{}@{}".format(nickname, username, host)

def emit_message(prefix, command, params):
  buf = []

  if prefix is not None:
    buf.append(":" + prefix)

  buf.append(command)

  if params:
    *init, last = params

    for param in init:
      if " " in param:
        raise EmitterError("malformed message")

    if not last or " " in last or last[0] == ":":
      buf.extend(init)
      buf.append(":" + last)
    else:
      buf.extend(params)

  return " ".join(buf)
<|endoftext|>


<commit_before>from datetime import datetime, timedelta
import re
import logging

from bs4 import BeautifulSoup
from core.observables import Ip, Hostname, Hash, Url
from core.feed import Feed
from core.errors import ObservableValidationError
import requests

class FeodoTracker(Feed):
    descriptions = {
        'A':
            "Hosted on compromised webservers running an nginx proxy on port 8080 TCP forwarding all botnet traffic to a tier 2 proxy node. Botnet traffic usually directly hits these hosts on port 8080 TCP without using a domain name.",
        'B':
            "Hosted on servers rented and operated by cybercriminals for the exclusive purpose of hosting a Feodo botnet controller. Usually taking advantage of a domain name within ccTLD .ru. Botnet traffic usually hits these domain names using port 80 TCP.",
        'C':
            "Successor of Feodo, completely different code. Hosted on the same botnet infrastructure as Version A (compromised webservers, nginx on port 8080 TCP or port 7779 TCP, no domain names) but using a different URL structure. This Version is also known as Geodo.",
        'D':
            "Successor of Cridex. This version is also known as Dridex",
        'E':
            "Successor of Geodo / Emotet (Version C) called Heodo. First appeared in March 2017."
    }

    variants = {
        'A': "Feodo",
        'B': "Feodo",
        'C': "Geodo",
        'D': "Dridex",
        'E': "Heodo"
    }

    default_values = {
        "frequency":
            timedelta(hours=1),
        "name":
            "FeodoTracker",
        "source":
            "https://feodotracker.abuse.ch/feodotracker.rss",
        "description":
            "Feodo Tracker RSS Feed. This feed shows the latest twenty Feodo C2 servers which Feodo Tracker has identified.",
    }

    def update(self):
        for item in self.update_xml(
                'item', ["title", "link", "description", "guid"]):
            self.analyze(item)

    def analyze(self, item):
        context = item
        date_string = re.search(
            r"\((?P<datetime>[\d\- :]+)\)", context['title']).group('datetime')
        try:
            context['date_added'] = datetime.strptime(
                date_string, "%Y-%m-%d %H:%M:%S")
        except ValueError:
            pass

        g = re.match(
            r'^Host: (?P<host>.+), Version: (?P<version>\w)',
            context['description'])
        g = g.groupdict()
        context['version'] = g['version']
        context['description'] = FeodoTracker.descriptions[g['version']]
        context['subfamily'] = FeodoTracker.variants[g['version']]
        context['source'] = self.name
        del context['title']
        new = None
        variant_tag = FeodoTracker.variants[g['version']].lower()
        try:
            if re.search(r"[0-9]{1,3}.[0-9]{1,3}.[0-9]{1,3}.[0-9]{1,3}",
                         g['host']):
                new = Ip.get_or_create(value=g['host'])
            else:
                new = Hostname.get_or_create(value=g['host'])
            new.add_context(context)
            new.add_source("feed")
            new.tag([variant_tag, 'malware', 'crimeware', 'banker', 'c2'])

        except ObservableValidationError as e:
            logging.error(e)

        try:
            url_fedeo = context['guid']
            r = requests.get(url_fedeo)
            if r.status_code == 200:
                if r.status_code == 200:
                    html_source = r.text

                    soup = BeautifulSoup(html_source, 'html.parser')
                    tab = soup.find('table', attrs='sortable')

                    all_tr = tab.find_all('tr')
                    results = []
                    for tr in all_tr:
                        all_td = tr.find_all('td')
                        if all_td and len(all_td) == 7:
                            results.append({
                                'timestamp': all_td[0].text,
                                'md5_hash': all_td[1].text,
                                'filesize': all_td[2].text,
                                'VT': all_td[3].text,
                                'Host': all_td[4].text,
                                'Port': all_td[5].text,
                                'SSL Certif or method': all_td[6]
                            })

                        for r in results:
                            new_hash = Hash.get_or_create(value=r['md5_hash'])
                            new_hash.add_context(context)
                            new_hash.add_source('feed')
                            new_hash.tag([
                                variant_tag, 'malware', 'crimeware', 'banker', 'payload'
                            ])
                            new_hash.active_link_to(
                                new, 'c2', self.name, clean_old=False)
                            host = Url.get_or_create(
                                value='https://%s:%s' % (g['host'], r['Port']))
                            host.add_source('feed')
                            host.add_context(context)
                            host.tag(
                                [variant_tag, 'malware', 'crimeware', 'banker', 'c2'])
                            new_hash.active_link_to(
                                host, 'c2', self.name, clean_old=False)

        except ObservableValidationError as e:
            logging.error(e)
<commit_msg>ident code and remove useless code<commit_after>from datetime import datetime, timedelta
import re
import logging

from bs4 import BeautifulSoup
from core.observables import Ip, Hostname, Hash, Url
from core.feed import Feed
from core.errors import ObservableValidationError
import requests

class FeodoTracker(Feed):
    descriptions = {
        'A':
            "Hosted on compromised webservers running an nginx proxy on port 8080 TCP forwarding all botnet traffic to a tier 2 proxy node. Botnet traffic usually directly hits these hosts on port 8080 TCP without using a domain name.",
        'B':
            "Hosted on servers rented and operated by cybercriminals for the exclusive purpose of hosting a Feodo botnet controller. Usually taking advantage of a domain name within ccTLD .ru. Botnet traffic usually hits these domain names using port 80 TCP.",
        'C':
            "Successor of Feodo, completely different code. Hosted on the same botnet infrastructure as Version A (compromised webservers, nginx on port 8080 TCP or port 7779 TCP, no domain names) but using a different URL structure. This Version is also known as Geodo.",
        'D':
            "Successor of Cridex. This version is also known as Dridex",
        'E':
            "Successor of Geodo / Emotet (Version C) called Heodo. First appeared in March 2017."
    }

    variants = {
        'A': "Feodo",
        'B': "Feodo",
        'C': "Geodo",
        'D': "Dridex",
        'E': "Heodo"
    }

    default_values = {
        "frequency":
            timedelta(hours=1),
        "name":
            "FeodoTracker",
        "source":
            "https://feodotracker.abuse.ch/feodotracker.rss",
        "description":
            "Feodo Tracker RSS Feed. This feed shows the latest twenty Feodo C2 servers which Feodo Tracker has identified.",
    }

    def update(self):
        for item in self.update_xml(
                'item', ["title", "link", "description", "guid"]):
            self.analyze(item)

    def analyze(self, item):
        context = item
        date_string = re.search(
            r"\((?P<datetime>[\d\- :]+)\)", context['title']).group('datetime')
        try:
            context['date_added'] = datetime.strptime(
                date_string, "%Y-%m-%d %H:%M:%S")
        except ValueError:
            pass

        g = re.match(
            r'^Host: (?P<host>.+), Version: (?P<version>\w)',
            context['description'])
        g = g.groupdict()
        context['version'] = g['version']
        context['description'] = FeodoTracker.descriptions[g['version']]
        context['subfamily'] = FeodoTracker.variants[g['version']]
        context['source'] = self.name
        del context['title']
        new = None
        variant_tag = FeodoTracker.variants[g['version']].lower()
        try:
            if re.search(r"[0-9]{1,3}.[0-9]{1,3}.[0-9]{1,3}.[0-9]{1,3}",
                         g['host']):
                new = Ip.get_or_create(value=g['host'])
            else:
                new = Hostname.get_or_create(value=g['host'])
            new.add_context(context)
            new.add_source("feed")
            new.tag([variant_tag, 'malware', 'crimeware', 'banker', 'c2'])

        except ObservableValidationError as e:
            logging.error(e)

        try:
            url_fedeo = context['guid']
            r = requests.get(url_fedeo)
            if r.status_code == 200:

                html_source = r.text

                soup = BeautifulSoup(html_source, 'html.parser')
                tab = soup.find('table', attrs='sortable')

                all_tr = tab.find_all('tr')
                results = []
                for tr in all_tr:
                    all_td = tr.find_all('td')
                    if all_td and len(all_td) == 7:
                        results.append({
                            'timestamp': all_td[0].text,
                            'md5_hash': all_td[1].text,
                            'filesize': all_td[2].text,
                            'VT': all_td[3].text,
                            'Host': all_td[4].text,
                            'Port': all_td[5].text,
                            'SSL Certif or method': all_td[6]
                        })

                for r in results:
                    new_hash = Hash.get_or_create(value=r['md5_hash'])
                    new_hash.add_context(context)
                    new_hash.add_source('feed')
                    new_hash.tag([
                        variant_tag, 'malware', 'crimeware', 'banker', 'payload'
                    ])
                    new_hash.active_link_to(
                        new, 'c2', self.name, clean_old=False)
                    host = Url.get_or_create(
                        value='https://%s:%s' % (g['host'], r['Port']))
                    host.add_source('feed')
                    host.add_context(context)
                    host.tag(
                        [variant_tag, 'malware', 'crimeware', 'banker', 'c2'])
                    new_hash.active_link_to(
                        host, 'c2', self.name, clean_old=False)

        except ObservableValidationError as e:
            logging.error(e)
<|endoftext|>