<commit_before>################################### LICENSE ####################################
#                            Copyright 2016 Morphux                            #
#                                                                              #
#        Licensed under the Apache License, Version 2.0 (the "License");       #
#        you may not use this file except in compliance with the License.      #
#                  You may obtain a copy of the License at                     #
#                                                                              #
#                 http://www.apache.org/licenses/LICENSE-2.0                   #
#                                                                              #
#      Unless required by applicable law or agreed to in writing, software     #
#       distributed under the License is distributed on an "AS IS" BASIS,      #
#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.  #
#        See the License for the specific language governing permissions and   #
#                       limitations under the License.                         #
################################################################################

##
# binutils2_p1.py
# Created: 04/12/2016
# By: Louis Solofrizzo <louis@morphux.org>
##

import      os

class   Binutils2_P1:

    conf_lst = {}
    e = False
    root_dir = ""

    def     init(self, c_lst, ex, root_dir):
        self.conf_lst = c_lst
        self.e = ex
        self.root_dir = root_dir
        self.config = {
            "name": "binutils2", # Name of the package
            "version": "2.27", # Version of the package
            "size": 533, # Size of the installed package (MB)
            "archive": "binutils-2.27.tar.bz2", # Archive name
            "SBU": 1.1, # SBU (Compilation time)
            "tmp_install": True, # Is this package part of the temporary install
            "next": "gcc2", # Next package to install
            "chdir": False,
            "urls": [ # Url to download the package. The first one must be morphux servers
                "https://install.morphux.org/packages/binutils-2.27.tar.bz2"
            ]
        }
        return self.config

    def     before(self):
        os.chdir("binutils-2.27")
        res = self.e(["rm", "-rf", "build"])
        res = self.e(["mkdir", "-vp", "build"])
        os.chdir("build")
        return res

    def     configure(self):
        os.environ["CC"] = self.conf_lst["target"] + "-gcc"
        os.environ["AR"] = self.conf_lst["target"] + "-ar"
        os.environ["RANLIB"] = self.conf_lst["target"] + "-ranlib"
        return self.e(["../configure",
                "--prefix=/tools",
                "--disable-nls",
                "--disable-werror",
                "--with-lib-path=/tools/lib",
                "--with-sysroot"
            ])

    def     make(self):
        return self.e(["make", "-j", self.conf_lst["cpus"]])

    def     install(self):
        return self.e(["make", "install"])

    def     after(self):
        self.e(["make", "-C", "ld", "clean"])
        self.e(["make", "-C", "ld", "LIBPATH=/usr/lib:/lib"], shell=True)
        return self.e(["cp", "-v", "ld/ld-new", "/tools/bin"])
<commit_msg>Fix(Binutils Phase 2): Forget a _ in a compilation option<commit_after>################################### LICENSE ####################################
#                            Copyright 2016 Morphux                            #
#                                                                              #
#        Licensed under the Apache License, Version 2.0 (the "License");       #
#        you may not use this file except in compliance with the License.      #
#                  You may obtain a copy of the License at                     #
#                                                                              #
#                 http://www.apache.org/licenses/LICENSE-2.0                   #
#                                                                              #
#      Unless required by applicable law or agreed to in writing, software     #
#       distributed under the License is distributed on an "AS IS" BASIS,      #
#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.  #
#        See the License for the specific language governing permissions and   #
#                       limitations under the License.                         #
################################################################################

##
# binutils2_p1.py
# Created: 04/12/2016
# By: Louis Solofrizzo <louis@morphux.org>
##

import      os

class   Binutils2_P1:

    conf_lst = {}
    e = False
    root_dir = ""

    def     init(self, c_lst, ex, root_dir):
        self.conf_lst = c_lst
        self.e = ex
        self.root_dir = root_dir
        self.config = {
            "name": "binutils2", # Name of the package
            "version": "2.27", # Version of the package
            "size": 533, # Size of the installed package (MB)
            "archive": "binutils-2.27.tar.bz2", # Archive name
            "SBU": 1.1, # SBU (Compilation time)
            "tmp_install": True, # Is this package part of the temporary install
            "next": "gcc2", # Next package to install
            "chdir": False,
            "urls": [ # Url to download the package. The first one must be morphux servers
                "https://install.morphux.org/packages/binutils-2.27.tar.bz2"
            ]
        }
        return self.config

    def     before(self):
        os.chdir("binutils-2.27")
        res = self.e(["rm", "-rf", "build"])
        res = self.e(["mkdir", "-vp", "build"])
        os.chdir("build")
        return res

    def     configure(self):
        os.environ["CC"] = self.conf_lst["target"] + "-gcc"
        os.environ["AR"] = self.conf_lst["target"] + "-ar"
        os.environ["RANLIB"] = self.conf_lst["target"] + "-ranlib"
        return self.e(["../configure",
                "--prefix=/tools",
                "--disable-nls",
                "--disable-werror",
                "--with-lib-path=/tools/lib",
                "--with-sysroot"
            ])

    def     make(self):
        return self.e(["make", "-j", self.conf_lst["cpus"]])

    def     install(self):
        return self.e(["make", "install"])

    def     after(self):
        self.e(["make", "-C", "ld", "clean"])
        self.e(["make", "-C", "ld", "LIB_PATH=/usr/lib:/lib"], shell=True)
        return self.e(["cp", "-v", "ld/ld-new", "/tools/bin"])
<|endoftext|>


<commit_before>"""
This file is part of RAPD

Copyright (C) 2009-2016, Cornell University
All rights reserved.

RAPD is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as published by
the Free Software Foundation, version 3.

RAPD is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
GNU Affero General Public License for more details.

You should have received a copy of the GNU Affero General Public License
along with this program.  If not, see <http://www.gnu.org/licenses/>.
"""

__created__ = "2009-07-10"
__maintainer__ = "Frank Murphy"
__email__ = "fmurphy@anl.gov"
__status__ = "Production"

# Standard imports
import argparse
import importlib
import logging
import logging.handlers
import socket
import sys
import time

# RAPD imports
import utils.commandline
from utils.lock import file_lock
import utils.log
from utils.modules import load_module
import utils.site
import utils.text as text

# from rapd_database import Database

BUFFER_SIZE = 8192
database = None

class Launcher(object):
    """
    Runs a socket server and spawns new threads when connections are received
    """

    database = None
    adapter = None
    address = None
    ip_address = None
    tag = None
    port = None
    job_types = None
    adapter_file = None
    launcher = None

    def __init__(self, site, tag=""):
        """
        The main server thread
        """

        # Get the logger Instance
        self.logger = logging.getLogger("RAPDLogger")
        self.logger.debug("__init__")

        # Save passed-in variables
        self.site = site
        self.tag = tag

        # Retrieve settings for this Launcher
        self.get_settings()

        # Load the adapter
        self.load_adapter()

        # Set up connection to the control database - not used yet!
        # self.connect_to_database()

        # Start listening for commands
        self.run()

    def run(self):
        """
        The core process of the Launcher instance
        """

        # Create socket to listen for commands
        _socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
        _socket.settimeout(5)
        _socket.bind(("", self.specifications["port"]))

        # This is the server portion of the code
        while 1:
            try:
                _socket.listen(5)
                conn, addr = _socket.accept()

                # Read the message from the socket
                message = ""
                while not message.endswith("<rapd_end>"):
                    try:
                        data = conn.recv(BUFFER_SIZE)
                        message += data
                    except:
                        pass
                    time.sleep(0.01)

                # Close the connection
                conn.close()

                # Handle the message
                self.handle_message(message)

            except socket.timeout:
                print "5 seconds up"

        # If we exit...
        _socket.close()

    def handle_message(self, message):
        """
        Handle an incoming message
        """

        self.logger.debug("Message received: %s", message)

        # Strip the message of its delivery tags
        message = message.rstrip().replace("<rapd_start>", "").replace("<rapd_end>", "")

        # Use the adapter to launch
        self.adapter(self.site, message, self.specifications)

    def get_settings(self):
        """
        Get the settings for this Launcher based on ip address and tag
        """

        # Save typing
        launchers = self.site.LAUNCHER_SETTINGS["LAUNCHER_REGISTER"]

        # Get IP Address
        self.ip_address = utils.site.get_ip_address()
        self.logger.debug("Found ip address to be %s", self.ip_address)

        # Look for the launcher matching this ip_address and the input tag
        possible_tags = []
        for launcher in launchers:
            if launcher[0] == self.ip_address and launcher[1] == self.tag:
                self.launcher = launcher
                break
            elif launcher[0] == self.ip_address:
                possible_tags.append(launcher[1])

        # No launcher adapter
        if self.launcher is None:

            # No launchers for this IP address
            if len(possible_tags) == 0:
                print "  There are no launcher adapters registered for this ip address"
            # IP Address in launchers, but not the input tag
            else:
                print text.error + "There is a launcher adapter registered for thi\
s IP address (%s), but not for the input tag (%s)" % (self.ip_address, self.tag)
                print "  Available tags for this IP address:"
                for t in possible_tags:
                    print "    %s" % t
                print text.stop

            # Exit in error state
            sys.exit(9)
        else:
            # Unpack address
            self.ip_address, self.tag, self.launcher_id = self.launcher
            self.specifications = self.site.LAUNCHER_SETTINGS["LAUNCHER_SPECIFICATIONS"][self.launcher_id]
            # Tag launcher in self.site
            self.site.LAUNCHER_ID = self.launcher_id

    def load_adapter(self):
        """Find and load the adapter"""

        # Import the database adapter as database module
        self.adapter = load_module(
            seek_module=self.specifications["adapter"],
            directories=self.site.LAUNCHER_SETTINGS["RAPD_LAUNCHER_ADAPTER_DIRECTORIES"]).LauncherAdapter

        self.logger.debug(self.adapter)

    def connect_to_database(self):
        """Set up database connection"""

        # Import the database adapter as database module
        global database
        database = importlib.import_module('database.rapd_%s_adapter' % self.site.CONTROL_DATABASE)

        # Instantiate the database connection
        self.database = database.Database(settings=self.site.CONTROL_DATABASE_SETTINGS)

def get_commandline():
    """Get the commandline variables and handle them"""

    # Parse the commandline arguments
    commandline_description = """The Launch process for handling calls for
    computation"""
    parser = argparse.ArgumentParser(parents=[utils.commandline.base_parser],
                                     description=commandline_description)

    # Add the possibility to tag the Launcher
    # This will make it possible to run multiple Launcher configurations
    # on one machine
    parser.add_argument("--tag", "-t",
                        action="store",
                        dest="tag",
                        default="",
                        help="Specify a tag for the Launcher")

    return parser.parse_args()

def main():
    """Run the main process"""

    # Get the commandline args
    commandline_args = get_commandline()

    # Get the environmental variables
    environmental_vars = utils.site.get_environmental_variables()

    # Determine the site
    site_file = utils.site.determine_site(site_arg=commandline_args.site)

    # Determine the tag - commandline wins
    if commandline_args.tag:
        tag = commandline_args.tag
    elif environmental_vars.has_key("RAPD_LAUNCHER_TAG"):
        tag = environmental_vars["RAPD_LAUNCHER_TAG"]
    else:
        tag = ""

    # Import the site settings
    SITE = importlib.import_module(site_file)

    # Single process lock?
    file_lock(SITE.LAUNCHER_LOCK_FILE)

    # Set up logging level
    if commandline_args.verbose:
        log_level = 10
    else:
        log_level = SITE.LOG_LEVEL

    # Instantiate the logger
    logger = utils.log.get_logger(logfile_dir=SITE.LOGFILE_DIR,
                                  logfile_id="rapd_launcher_"+SITE.ID,
                                  level=log_level)

    logger.debug("Commandline arguments:")
    for pair in commandline_args._get_kwargs():
        logger.debug("  arg:%s  val:%s" % pair)

    LAUNCHER = Launcher(site=SITE,
                        tag=tag)

if __name__ == "__main__":

    main()
<commit_msg>Overwatchify the rapd_launcher<commit_after>"""
This file is part of RAPD

Copyright (C) 2009-2016, Cornell University
All rights reserved.

RAPD is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as published by
the Free Software Foundation, version 3.

RAPD is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
GNU Affero General Public License for more details.

You should have received a copy of the GNU Affero General Public License
along with this program.  If not, see <http://www.gnu.org/licenses/>.
"""

__created__ = "2009-07-10"
__maintainer__ = "Frank Murphy"
__email__ = "fmurphy@anl.gov"
__status__ = "Production"

# Standard imports
import argparse
import importlib
import logging
import logging.handlers
import socket
import sys
import time

# RAPD imports
import utils.commandline
from utils.lock import file_lock
import utils.log
from utils.modules import load_module
import utils.site
import utils.text as text

# from rapd_database import Database

BUFFER_SIZE = 8192
database = None

class Launcher(object):
    """
    Runs a socket server and spawns new threads when connections are received
    """

    database = None
    adapter = None
    address = None
    ip_address = None
    tag = None
    port = None
    job_types = None
    adapter_file = None
    launcher = None

    def __init__(self, site, tag="", overwatcher_id=False):
        """
        The main server thread
        """

        # Get the logger Instance
        self.logger = logging.getLogger("RAPDLogger")
        self.logger.debug("__init__")

        # Save passed-in variables
        self.site = site
        self.tag = tag
        self.overwatcher_id = overwatcher_id

        # Retrieve settings for this Launcher
        self.get_settings()

        # Load the adapter
        self.load_adapter()

        # Set up connection to the control database - not used yet!
        # self.connect_to_database()

        # Start listening for commands
        self.run()

    def run(self):
        """
        The core process of the Launcher instance
        """

        # Set up overwatcher
        if self.overwatcher_id:
            self.ow_registrar = Registrar(site=self.site,
                                          ow_type="launcher",
                                          ow_id=self.overwatcher_id)
            self.ow_registrar.register({"site_id":self.site.ID})

        # Create socket to listen for commands
        _socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
        _socket.settimeout(5)
        _socket.bind(("", self.specifications["port"]))

        # This is the server portion of the code
        while 1:
            try:
                # Have Registrar update status
                self.ow_registrar.update({"site_id":self.site.ID})

                # Listen for connections
                _socket.listen(5)
                conn, addr = _socket.accept()

                # Read the message from the socket
                message = ""
                while not message.endswith("<rapd_end>"):
                    try:
                        data = conn.recv(BUFFER_SIZE)
                        message += data
                    except:
                        pass
                    time.sleep(0.01)

                # Close the connection
                conn.close()

                # Handle the message
                self.handle_message(message)

            except socket.timeout:
                print "5 seconds up"

        # If we exit...
        _socket.close()

    def handle_message(self, message):
        """
        Handle an incoming message
        """

        self.logger.debug("Message received: %s", message)

        # Strip the message of its delivery tags
        message = message.rstrip().replace("<rapd_start>", "").replace("<rapd_end>", "")

        # Use the adapter to launch
        self.adapter(self.site, message, self.specifications)

    def get_settings(self):
        """
        Get the settings for this Launcher based on ip address and tag
        """

        # Save typing
        launchers = self.site.LAUNCHER_SETTINGS["LAUNCHER_REGISTER"]

        # Get IP Address
        self.ip_address = utils.site.get_ip_address()
        self.logger.debug("Found ip address to be %s", self.ip_address)

        # Look for the launcher matching this ip_address and the input tag
        possible_tags = []
        for launcher in launchers:
            if launcher[0] == self.ip_address and launcher[1] == self.tag:
                self.launcher = launcher
                break
            elif launcher[0] == self.ip_address:
                possible_tags.append(launcher[1])

        # No launcher adapter
        if self.launcher is None:

            # No launchers for this IP address
            if len(possible_tags) == 0:
                print "  There are no launcher adapters registered for this ip address"
            # IP Address in launchers, but not the input tag
            else:
                print text.error + "There is a launcher adapter registered for thi\
s IP address (%s), but not for the input tag (%s)" % (self.ip_address, self.tag)
                print "  Available tags for this IP address:"
                for t in possible_tags:
                    print "    %s" % t
                print text.stop

            # Exit in error state
            sys.exit(9)
        else:
            # Unpack address
            self.ip_address, self.tag, self.launcher_id = self.launcher
            self.specifications = self.site.LAUNCHER_SETTINGS["LAUNCHER_SPECIFICATIONS"][self.launcher_id]
            # Tag launcher in self.site
            self.site.LAUNCHER_ID = self.launcher_id

    def load_adapter(self):
        """Find and load the adapter"""

        # Import the database adapter as database module
        self.adapter = load_module(
            seek_module=self.specifications["adapter"],
            directories=self.site.LAUNCHER_SETTINGS["RAPD_LAUNCHER_ADAPTER_DIRECTORIES"]).LauncherAdapter

        self.logger.debug(self.adapter)

    def connect_to_database(self):
        """Set up database connection"""

        # Import the database adapter as database module
        global database
        database = importlib.import_module('database.rapd_%s_adapter' % self.site.CONTROL_DATABASE)

        # Instantiate the database connection
        self.database = database.Database(settings=self.site.CONTROL_DATABASE_SETTINGS)

def get_commandline():
    """Get the commandline variables and handle them"""

    # Parse the commandline arguments
    commandline_description = """The Launch process for handling calls for
    computation"""
    parser = argparse.ArgumentParser(parents=[utils.commandline.base_parser],
                                     description=commandline_description)

    # Add the possibility to tag the Launcher
    # This will make it possible to run multiple Launcher configurations
    # on one machine
    parser.add_argument("--tag", "-t",
                        action="store",
                        dest="tag",
                        default="",
                        help="Specify a tag for the Launcher")

    return parser.parse_args()

def main():
    """Run the main process"""

    # Get the commandline args
    commandline_args = get_commandline()

    # Get the environmental variables
    environmental_vars = utils.site.get_environmental_variables()

    # Get site - commandline wins
    if commandline_args.site:
        site = commandline_args.site
    elif environmental_vars["RAPD_SITE"]:
        site = environmental_vars["RAPD_SITE"]

    # Determine the site
    site_file = utils.site.determine_site(site_arg=site)

    # Determine the tag - commandline wins
    if commandline_args.tag:
        tag = commandline_args.tag
    elif environmental_vars.has_key("RAPD_LAUNCHER_TAG"):
        tag = environmental_vars["RAPD_LAUNCHER_TAG"]
    else:
        tag = ""

    # Import the site settings
    SITE = importlib.import_module(site_file)

    # Single process lock?
    file_lock(SITE.LAUNCHER_LOCK_FILE)

    # Set up logging level
    if commandline_args.verbose:
        log_level = 10
    else:
        log_level = SITE.LOG_LEVEL

    # Instantiate the logger
    logger = utils.log.get_logger(logfile_dir=SITE.LOGFILE_DIR,
                                  logfile_id="rapd_launcher_"+SITE.ID,
                                  level=log_level)

    logger.debug("Commandline arguments:")
    for pair in commandline_args._get_kwargs():
        logger.debug("  arg:%s  val:%s" % pair)

    LAUNCHER = Launcher(site=SITE,
                        tag=tag,
                        overwatcher_id=commandline_args.overwatcher_id)

if __name__ == "__main__":

    main()
<|endoftext|>


<commit_before>import functools
import inspect

from .particle_class import Particle
from ..utils import (AtomicError,
                     InvalidParticleError,
                     InvalidElementError,
                     InvalidIonError,
                     InvalidIsotopeError,
                     ChargeError)

from typing import Callable, Union, Any, Set, List, Tuple

# TODO: make sure particle_input works with classes and class methods

def particle_input(wrapped_function: Callable = None,
                   must_be: Union[str, Set, List, Tuple] = set(),
                   cannot_be: Union[str, Set, List, Tuple] = set(),
                   any: bool = False,
                   **kwargs) -> Any:
    r"""A decorator to take arguments and keywords related to a
    particle, and pass through the Particle class to the callable instead.

    Parameters
    ----------

    wrapped_function : callable
        The function to be decorated.

    must_be : str, set, list, or tuple; optional
        The categories that the Particle must be in; otherwise the

    Notes
    -----

    This version of particle_input only works with functions, but in the future
    should be extended to work with classes and methods.

    """

    def _errmsg(argname: str, argval: str, Z: int = None,
                mass_numb: int = None, funcname: str = None) -> str:
        r"""Returns a string with an appropriate error message for an
        InvalidParticleError."""

        errmsg = f"In {funcname}, {argname} = {repr(argval)} "
        if mass_numb is not None or Z is not None:
            errmsg += "with "
        if mass_numb is not None:
            errmsg += f"mass_numb = {repr(mass_numb)} "
        if mass_numb is not None and Z is not None:
            errmsg += "and "
        if Z is not None:
            errmsg += f"integer charge Z = {repr(Z)} "
        errmsg += "does not correspond to a valid particle."
        return errmsg

    def decorator(wrapped_function: Callable):
        wrapped_signature = inspect.signature(wrapped_function)

        @functools.wraps(wrapped_function)
        def wrapper(*args, **kwargs):

            # The following couple of statements will likely need to be
            # modified in order to work with methods.

            bound_args = wrapped_signature.bind(*args, **kwargs)
            arguments = bound_args.arguments
            argnames = bound_args.arguments.keys()
            funcname = wrapped_function.__name__
            annotations = wrapped_function.__annotations__

            args_to_become_particles = [
                argname for argname in annotations.keys()
                if annotations[argname] is Particle
            ]

            if not args_to_become_particles:
                raise AtomicError(
                    f"None of the arguments or keywords to {funcname} "
                    f"have been annotated with Particle as required "
                    f"by the particle_input decorator.")
            elif len(args_to_become_particles) > 1:
                if 'Z' in argnames or 'mass_numb' in argnames:
                    raise AtomicError(
                        f"The arguments Z and mass_numb in {funcname} are not "
                        f"allowed when more than one argument or keyword is "
                        f"annotated with Particle in functions decorated "
                        f"with @particle_input.")

            # If the number of arguments and keywords annotated with
            # Particle is exactly one, then the Z and mass_numb keywords
            # can be used without potential for ambiguity.

            Z = arguments.get('Z', None)
            mass_numb = arguments.get('mass_numb', None)

            if Z is not None and not isinstance(Z, int):
                raise TypeError(
                    f"The argument Z = {repr(Z)} in {funcname} is not an "
                    f"integer.")

            if mass_numb is not None and not isinstance(mass_numb, int):
                raise TypeError(
                    f"The argument mass_numb = {repr(mass_numb)} in "
                    f"{funcname} is not an integer.")

            # Go through the argument names and check whether or not they are
            # annotated with Particle.  If they aren't, include the name and
            # value of the argument as an item in the new keyword arguments
            # dictionary unchanged.  If they are annotated with Particle, then
            # either convert the representation of a Particle to a Particle if
            # it is not already a Particle.

            new_kwargs = {}

            for argname in argnames:
                argval = arguments[argname]

                should_be_particle = argname in args_to_become_particles
                not_already_particle = not isinstance(argval, Particle)

                if should_be_particle and not_already_particle:

                    if not isinstance(argval, (int, str)):
                        raise TypeError

                    try:
                        particle = Particle(argval, Z=Z, mass_numb=mass_numb)
                    except InvalidParticleError as e:
                        raise InvalidParticleError(
                            _errmsg(argname, argval, Z, mass_numb, funcname)
                        ) from e

                    if argname == 'element' and not particle.element:
                        raise InvalidElementError
                    if argname == 'isotope' and not particle.isotope:
                        raise InvalidIsotopeError
                    if argname == 'ion' and not particle.ion:
                        raise InvalidIonError
                    if 'charged' in must_be and not particle._integer_charge:
                        raise ChargeError
                    if not particle.is_category(must_be, exclude=cannot_be):
                        raise AtomicError

                    new_kwargs[argname] = particle
                else:
                    new_kwargs[argname] = argval

            return wrapped_function(**new_kwargs)

        return wrapper

    # The following code helps allow the decorator to be used either with
    # or without arguments.  In particular, this helps allow us to invoke
    # the decorator either as `@particle_input` or as `@particle_input()`,
    # where the latter call allows the decorator to have keyword arguments.

    if wrapped_function is not None:
        return decorator(wrapped_function)
    else:
        return decorator
<commit_msg>Remove redundant type checking from particle_input<commit_after>import functools
import inspect

from .particle_class import Particle
from ..utils import (AtomicError,
                     InvalidParticleError,
                     InvalidElementError,
                     InvalidIonError,
                     InvalidIsotopeError,
                     ChargeError)

from typing import Callable, Union, Any, Set, List, Tuple

# TODO: make sure particle_input works with class methods

def particle_input(wrapped_function: Callable = None,
                   must_be: Union[str, Set, List, Tuple] = set(),
                   cannot_be: Union[str, Set, List, Tuple] = set(),
                   any: bool = False,
                   **kwargs) -> Any:
    r"""A decorator to take arguments and keywords related to a
    particle, and pass through the Particle class to the callable instead.

    Parameters
    ----------

    wrapped_function : callable
        The function to be decorated.

    must_be : str, set, list, or tuple; optional
        The categories that the Particle must be in; otherwise the

    Notes
    -----

    This version of particle_input only works with functions, but in the future
    should be extended to work with classes and methods.

    """

    def _errmsg(argname: str, argval: str, Z: int = None,
                mass_numb: int = None, funcname: str = None) -> str:
        r"""Returns a string with an appropriate error message for an
        InvalidParticleError."""

        errmsg = f"In {funcname}, {argname} = {repr(argval)} "
        if mass_numb is not None or Z is not None:
            errmsg += "with "
        if mass_numb is not None:
            errmsg += f"mass_numb = {repr(mass_numb)} "
        if mass_numb is not None and Z is not None:
            errmsg += "and "
        if Z is not None:
            errmsg += f"integer charge Z = {repr(Z)} "
        errmsg += "does not correspond to a valid particle."
        return errmsg

    def decorator(wrapped_function: Callable):
        wrapped_signature = inspect.signature(wrapped_function)

        @functools.wraps(wrapped_function)
        def wrapper(*args, **kwargs):

            # The following couple of statements will likely need to be
            # modified in order to work with methods.

            bound_args = wrapped_signature.bind(*args, **kwargs)
            arguments = bound_args.arguments
            argnames = bound_args.arguments.keys()
            funcname = wrapped_function.__name__
            annotations = wrapped_function.__annotations__

            args_to_become_particles = [
                argname for argname in annotations.keys()
                if annotations[argname] is Particle
            ]

            if not args_to_become_particles:
                raise AtomicError(
                    f"None of the arguments or keywords to {funcname} "
                    f"have been annotated with Particle as required "
                    f"by the particle_input decorator.")
            elif len(args_to_become_particles) > 1:
                if 'Z' in argnames or 'mass_numb' in argnames:
                    raise AtomicError(
                        f"The arguments Z and mass_numb in {funcname} are not "
                        f"allowed when more than one argument or keyword is "
                        f"annotated with Particle in functions decorated "
                        f"with @particle_input.")

            # If the number of arguments and keywords annotated with
            # Particle is exactly one, then the Z and mass_numb keywords
            # can be used without potential for ambiguity.

            Z = arguments.get('Z', None)
            mass_numb = arguments.get('mass_numb', None)

            # Go through the argument names and check whether or not they are
            # annotated with Particle.  If they aren't, include the name and
            # value of the argument as an item in the new keyword arguments
            # dictionary unchanged.  If they are annotated with Particle, then
            # either convert the representation of a Particle to a Particle if
            # it is not already a Particle.

            new_kwargs = {}

            for argname in argnames:
                argval = arguments[argname]

                should_be_particle = argname in args_to_become_particles
                not_already_particle = not isinstance(argval, Particle)

                if should_be_particle and not_already_particle:

                    if not isinstance(argval, (int, str)):
                        raise TypeError

                    try:
                        particle = Particle(argval, Z=Z, mass_numb=mass_numb)
                    except InvalidParticleError as e:
                        raise InvalidParticleError(
                            _errmsg(argname, argval, Z, mass_numb, funcname)
                        ) from e

                    if argname == 'element' and not particle.element:
                        raise InvalidElementError
                    if argname == 'isotope' and not particle.isotope:
                        raise InvalidIsotopeError
                    if argname == 'ion' and not particle.ion:
                        raise InvalidIonError
                    if 'charged' in must_be and not particle._integer_charge:
                        raise ChargeError
                    if not particle.is_category(must_be, exclude=cannot_be):
                        raise AtomicError

                    new_kwargs[argname] = particle
                else:
                    new_kwargs[argname] = argval

            return wrapped_function(**new_kwargs)

        return wrapper

    # The following code helps allow the decorator to be used either with
    # or without arguments.  In particular, this helps allow us to invoke
    # the decorator either as `@particle_input` or as `@particle_input()`,
    # where the latter call allows the decorator to have keyword arguments.

    if wrapped_function is not None:
        return decorator(wrapped_function)
    else:
        return decorator
<|endoftext|>


<commit_before># The MIT License (MIT)
#
# Copyright (c) 2016 PyBER
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in all
# copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
# SOFTWARE.

import os
import numpy as np
from PyQt4 import QtGui

def perf_reader(filename, s_values):

    # there are two different file formats, classic (lines begni with SNR) and new (lines begin with SNR value directly)
    # indices of the different parameter change, and the following array is used to convert classic indices to new
    convert_to_v1 = np.array([0, 4, 5, 10, 0, 3, 2])
    convert_to_v2 = np.array([0, 4, 5, 7 , 0, 3, 2])

    fin = open(filename, "r")

    perf_list = []

    for line in fin:

        if line.startswith("#"):
            continue

        # classic
        if line.startswith("SNR = "):
            line = line.replace("#", "")
            line = line.replace("=", "")
            line = line.replace("SNR", "")
            line = line.replace("BER", "")
            line = line.replace("FER", "")
            line = line.replace("BE/FE", "")
            line = line.replace("BPS", "")
            line = line.replace("BPS", "")
            line = line.replace("MATRICES", "")
            line = line.replace("BE", "")
            line = line.replace("FE", "")
            line = line.replace("RUNTIME", "")
            line = line.replace("ERR", "")
            line = line.replace(" ", "")
            line = line.split('|')
            perf_list.append(float(line[s_values]))
        # new
        else:
            line = line.replace("||", "|")
            line = line.split('|')
            try:
                float(line[0])  # in cases where the first item of the line is neither "SNR = " nor a float
            except ValueError:
                continue        # line is ignored

            if(len(line) == 14):
                perf_list.append(float(line[convert_to_v1[s_values]]))
            else:
                perf_list.append(float(line[convert_to_v2[s_values]]))

    return perf_list

def perf_snr_reader(filename):
    return perf_reader(filename, 0)

def perf_ber_reader(filename):
    return perf_reader(filename, 1)

def perf_fer_reader(filename):
    return perf_reader(filename, 2)

def perf_bps_reader(filename):
    return perf_reader(filename, 3)

def perf_fe_reader(filename):
    return perf_reader(filename, 5)

def perf_be_reader(filename):
    return perf_reader(filename, 6)

def perf_befe_reader(filename):
    a = []
    b = []
    a = perf_be_reader(filename)
    b = perf_fe_reader(filename)
    c = []
    if a and b:
        for i in range(0, len(a)):
            c.append(a[i] / b[i])
    # c = np.divide(a, b)
    return c

def buildLegendLayout(filename):
    # open the file in read mode
    f = open(filename, 'r')

    # read all the lines from the current file f
    lines = []
    for line in f:
        lines.append(line)

    # get the command to to run to reproduce this trace
    runCommand = ""
    if "Run command:" in lines[0]:
        runCommand = lines[1].strip()
    runCmd = QtGui.QLineEdit(str(runCommand))
    runCmd.setReadOnly(True)

    entries = []

    # get the main infos from the lines
    for l in lines:
        if len(l) > 3 and l[0] == '#' and l[2] == '*':
            entry = l.replace("# * ", "").replace("\n", "").split(" = ")
            entries.append(entry)
    f.close()

    layout = QtGui.QFormLayout()
    layout.addRow("File name: ", QtGui.QLabel(os.path.basename(filename)))

    for entry in entries:
        if len(entry) == 2:
            layout.addRow("" + entry[0] + ": ", QtGui.QLabel(entry[1]))

    if runCommand:
        layout.addRow("Run command: ", runCmd)

    return layout<commit_msg>Fix div by 0 in PyBER.<commit_after># The MIT License (MIT)
#
# Copyright (c) 2016 PyBER
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in all
# copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
# SOFTWARE.

import os
import numpy as np
from PyQt4 import QtGui

def perf_reader(filename, s_values):

    # there are two different file formats, classic (lines begni with SNR) and new (lines begin with SNR value directly)
    # indices of the different parameter change, and the following array is used to convert classic indices to new
    convert_to_v1 = np.array([0, 4, 5, 10, 0, 3, 2])
    convert_to_v2 = np.array([0, 4, 5, 7 , 0, 3, 2])

    fin = open(filename, "r")

    perf_list = []

    for line in fin:

        if line.startswith("#"):
            continue

        # classic
        if line.startswith("SNR = "):
            line = line.replace("#", "")
            line = line.replace("=", "")
            line = line.replace("SNR", "")
            line = line.replace("BER", "")
            line = line.replace("FER", "")
            line = line.replace("BE/FE", "")
            line = line.replace("BPS", "")
            line = line.replace("BPS", "")
            line = line.replace("MATRICES", "")
            line = line.replace("BE", "")
            line = line.replace("FE", "")
            line = line.replace("RUNTIME", "")
            line = line.replace("ERR", "")
            line = line.replace(" ", "")
            line = line.split('|')
            perf_list.append(float(line[s_values]))
        # new
        else:
            line = line.replace("||", "|")
            line = line.split('|')
            try:
                float(line[0])  # in cases where the first item of the line is neither "SNR = " nor a float
            except ValueError:
                continue        # line is ignored

            if(len(line) == 14):
                perf_list.append(float(line[convert_to_v1[s_values]]))
            else:
                perf_list.append(float(line[convert_to_v2[s_values]]))

    return perf_list

def perf_snr_reader(filename):
    return perf_reader(filename, 0)

def perf_ber_reader(filename):
    return perf_reader(filename, 1)

def perf_fer_reader(filename):
    return perf_reader(filename, 2)

def perf_bps_reader(filename):
    return perf_reader(filename, 3)

def perf_fe_reader(filename):
    return perf_reader(filename, 5)

def perf_be_reader(filename):
    return perf_reader(filename, 6)

def perf_befe_reader(filename):
    a = []
    b = []
    a = perf_be_reader(filename)
    b = perf_fe_reader(filename)
    c = []
    if a and b:
        for i in range(0, len(a)):
            if b[i] == 0:
                c.append(0.0)
            else :
                c.append(a[i] / b[i])
    # c = np.divide(a, b)
    return c

def buildLegendLayout(filename):
    # open the file in read mode
    f = open(filename, 'r')

    # read all the lines from the current file f
    lines = []
    for line in f:
        lines.append(line)

    # get the command to to run to reproduce this trace
    runCommand = ""
    if "Run command:" in lines[0]:
        runCommand = lines[1].strip()
    runCmd = QtGui.QLineEdit(str(runCommand))
    runCmd.setReadOnly(True)

    entries = []

    # get the main infos from the lines
    for l in lines:
        if len(l) > 3 and l[0] == '#' and l[2] == '*':
            entry = l.replace("# * ", "").replace("\n", "").split(" = ")
            entries.append(entry)
    f.close()

    layout = QtGui.QFormLayout()
    layout.addRow("File name: ", QtGui.QLabel(os.path.basename(filename)))

    for entry in entries:
        if len(entry) == 2:
            layout.addRow("" + entry[0] + ": ", QtGui.QLabel(entry[1]))

    if runCommand:
        layout.addRow("Run command: ", runCmd)

    return layout<|endoftext|>


<commit_before>#!/usr/bin/env python
#
# Load favorites for a Twitter user and output them to a file.
#

import csv
import time
import twitter

from time import sleep
from twitter.error import TwitterError
from sqlalchemy import desc
from sqlalchemy.orm.exc import NoResultFound

from myarchive.db.tables.twittertables import (
    CSVTweet, RawTweet, Tweet, TwitterUser)

from account_info import *

SLEEP_TIME = 60 * (60 / 15)

USER = "USER"
FAVORITES = "FAVORITES"

KEYS = [
    u'user',
    u'text',
    u'in_reply_to_screen_name',
    u'media',

    # Not that interesting, but saved.
    u'hashtags',
    u'id',
    u'in_reply_to_status_id',
    # Seems to be empty a lot. Put it at the end.
    u'urls',

    # Don't really care about these.
    # u'user_mentions',
    # u'source',
    # u'created_at',
    # u'id_str',
    # u'place',
    # u'in_reply_to_user_id',
    # u'lang',
    # u'possibly_sensitive',
    # u'favorited',
    # u'favorite_count',
    # u'retweeted',
    # u'retweet_count',
]

class BulkApi(twitter.Api):
    """API with an extra call."""

    def LookupStatuses(self, status_ids, trim_user=False,
                       include_entities=True):
        """
        Returns up to 100 status messages, specified by a list passed to
        status_ids.

        Args:
          status_ids:
            A list of numeric IDs of the statuses you are trying to retrieve.
          trim_user:
            When set to True, each tweet returned in a timeline will include
            a user object including only the status authors numerical ID.
            Omit this parameter to receive the complete user object. [Optional]
          include_entities:
            If False, the entities node will be disincluded.
            This node offers a variety of metadata about the tweet in a
            discreet structure, including: user_mentions, urls, and
            hashtags. [Optional]
        Returns:
          A twitter.Status instance representing that status message
        """
        url = '%s/statuses/lookup.json' % self.base_url

        parameters = dict()

        if not status_ids or len(status_ids) > 100:
            raise TwitterError(
                "status_ids must be between 1 and 100 in length.")
        # This takes a comma-separated list of up to 100 IDs.
        parameters['id'] = ",".join(status_ids)

        if trim_user:
            parameters['trim_user'] = True
        if include_entities:
            parameters['include_entities'] = True

        resp = self._RequestUrl(url, 'GET', data=parameters)
        data = self._ParseAndCheckTwitter(resp.content.decode('utf-8'))

        return [twitter.Status.NewFromJsonDict(x) for x in data]

def archive_tweets(username, db_session, types=(USER, FAVORITES)):
    """
    Archives several types of new tweets along with their associated content.
    """
    new_ids = []
    api = twitter.Api(
        CONSUMER_KEY, CONSUMER_SECRET, ACCESS_KEY, ACCESS_SECRET,
        sleep_on_rate_limit=True)

    for type_ in types:

        # For favorites, always do a full sweep. We can't guarantee an older
        # tweet wasn't recently favorited!
        if type_ == FAVORITES:
            since_id = None
        else:
            since_id = db_session.query(RawTweet.id).\
                filter(RawTweet.favorited_by_str.like("%%%s%%" % username)).\
                order_by(desc(RawTweet.id)).first()
        if since_id:
            since_id = since_id[0]
        print type_, since_id

        max_id = None
        early_termination = False
        while not early_termination:
            print ("Pulling 200 tweets from API starting with ID %s and "
                   "ending with ID %s..." % (since_id, max_id))
            if type_ == FAVORITES:
                statuses = api.GetFavorites(
                    screen_name=username,
                    count=200,
                    since_id=since_id,
                    max_id=max_id,
                    include_entities=True)
                # 15 requests per 15 minutes.
                sleep_time = 60
            elif type_ == USER:
                statuses = api.GetUserTimeline(
                    screen_name=username,
                    count=200,
                    since_id=since_id,
                    max_id=max_id)
                # 300 requests per 15 minutes.
                sleep_time = 3
            print "Found %s tweets this iteration..." % len(statuses)
            if not statuses:
                break

            # Format things the way we want and handle max_id changes.
            for status in statuses:
                status_dict = status.AsDict()
                status_id = int(status_dict["id"])
                if since_id is not None and status_id >= since_id:
                    early_termination = True
                    break
                else:
                    new_ids.append(status_id)
                try:
                    raw_tweet = db_session.query(RawTweet).\
                        filter_by(id=status_id).one()
                except NoResultFound:
                    raw_tweet = RawTweet(status_dict=status_dict)
                    db_session.add(raw_tweet)
                if type_ == FAVORITES:
                    raw_tweet.add_user_favorite(username)
                db_session.commit()
                # Capture new max_id
                if status_id < max_id or max_id is None:
                    max_id = status_id - 1

            # Twitter rate-limits us to 15 requests / 15 minutes, so
            # space this out a bit to avoid a super-long sleep at the
            # end which could lose the connection.
            if early_termination is False:
                print "Sleeping for %s seconds to ease up rate limit..." % (
                    sleep_time)
                sleep(sleep_time)
    return new_ids

def import_from_csv(db_session, csv_filepath, username):
    api = BulkApi(
        CONSUMER_KEY, CONSUMER_SECRET, ACCESS_KEY, ACCESS_SECRET,
        sleep_on_rate_limit=True)

    print "Importing into CSVTweets..."
    csv_rows_by_id = dict()
    with open(csv_filepath) as csvfile:
        reader = csv.DictReader(csvfile)
        for row in reader:
            tweet_id = int(row['tweet_id'])
            csv_rows_by_id[tweet_id] = row
            try:
                db_session.query(CSVTweet).filter_by(id=tweet_id).one()
            except NoResultFound:
                csv_tweet = CSVTweet(
                    id=tweet_id,
                    username=username,
                    in_reply_to_status_id=row["in_reply_to_status_id"],
                    in_reply_to_user_id=row["in_reply_to_user_id"],
                    timestamp=row["timestamp"],
                    text=row["text"],
                    retweeted_status_id=row["retweeted_status_id"],
                    retweeted_status_user_id=row["retweeted_status_user_id"],
                    retweeted_status_timestamp=row["retweeted_status_timestamp"],
                    expanded_urls=row["expanded_urls"])
                db_session.add(csv_tweet)
                db_session.commit()
    csv_ids = list(csv_rows_by_id.keys())
    new_api_tweets = []

    print "Attempting API import..."
    index = 0
    sliced_ids = csv_ids[:100]
    while sliced_ids:
        start_time = time.time()
        new_ids = []
        for status_id in sliced_ids:
            try:
                # If the tweet is found, it's already been imported. Ignore it.
                db_session.query(RawTweet.id).filter_by(id=status_id).one()
                csv_tweet = db_session.query(CSVTweet).\
                    filter_by(id=status_id).one()
                csv_tweet.api_import_complete = True
                db_session.commit()
            except NoResultFound:
                new_ids.append(str(status_id))
        if new_ids:
            print "Attempting import of id %s to %s of %s..." % (
                index + 1, index + 100, len(csv_ids))
            try:
                statuses = api.LookupStatuses(
                    status_ids=new_ids, trim_user=False, include_entities=True)
                for status in statuses:
                    status_dict = status.AsDict()
                    raw_tweet = RawTweet(status_dict=status_dict)
                    db_session.add(raw_tweet)
                    db_session.commit()
                    # Mark CSV tweet appropriately.
                    csv_tweet = db_session.query(CSVTweet).\
                        filter_by(id=int(status_dict["id"])).one()
                    csv_tweet.api_import_complete = True
                    db_session.commit()
                    # Append to new list.
                    new_api_tweets.append(raw_tweet)

                # Sleep to not hit the rate limit.
                # 60 requests per 15 minutes.
                duration = time.time() - start_time
                if duration < 15:
                    sleep_duration = 15 - duration
                    print ("Sleeping for %s seconds to ease up on rate "
                           "limit..." % sleep_duration)
                    sleep(sleep_duration)
            except TwitterError as e:
                print e
        index += 100
        sliced_ids = csv_ids[index:100 + index]

    csv_only_tweets = db_session.query(CSVTweet.api_import_complete).\
        filter_by(api_import_complete=False).all()

    return new_api_tweets, csv_only_tweets

def parse_tweets(db_session, media_path, raw_tweets=None, csv_only_tweets=None,
                 parse_all_raw=False, download_files=False):

    if parse_all_raw is True:
        # Process all captured raw tweets.
        raw_tweets = db_session.query(RawTweet)

    for raw_tweet in raw_tweets:

        # Generate User objects.
        user_dict = raw_tweet.raw_status_dict["user"]
        user_id = int(user_dict["id"])
        twitter_user_ids = db_session.query(TwitterUser.id).all()
        if user_id not in twitter_user_ids:
            user = TwitterUser.add_from_user_dict(db_session, user_dict)
            if download_files is True:
                user.download_files(db_session, media_path)
        else:
            user = db_session.query(TwitterUser).filter_by(id=user_id).one()

        # Generate Tweet objects.
        status_dict = raw_tweet.raw_status_dict
        tweet_id = int(status_dict["id"])
        tweet_ids = db_session.query(Tweet.id).filter_by(user_id=user.id).all()
        if tweet_id not in tweet_ids:
            tweet = Tweet.make_from_raw(raw_tweet)
            db_session.add(tweet)
        else:
            tweet = db_session.query(Tweet).filter_by(id=tweet_id).one()
        user.tweets.append(tweet)
        db_session.commit()

    if csv_only_tweets:
        print len(csv_only_tweets)

def print_tweets(db_session):
    for raw_tweet in db_session.query(RawTweet).all():
        print raw_tweet
<commit_msg>Calculate sleeps so we can run a little faster.<commit_after>#!/usr/bin/env python
#
# Load favorites for a Twitter user and output them to a file.
#

import csv
import time
import twitter

from time import sleep
from twitter.error import TwitterError
from sqlalchemy import desc
from sqlalchemy.orm.exc import NoResultFound

from myarchive.db.tables.twittertables import (
    CSVTweet, RawTweet, Tweet, TwitterUser)

from account_info import *

SLEEP_TIME = 60 * (60 / 15)

USER = "USER"
FAVORITES = "FAVORITES"

KEYS = [
    u'user',
    u'text',
    u'in_reply_to_screen_name',
    u'media',

    # Not that interesting, but saved.
    u'hashtags',
    u'id',
    u'in_reply_to_status_id',
    # Seems to be empty a lot. Put it at the end.
    u'urls',

    # Don't really care about these.
    # u'user_mentions',
    # u'source',
    # u'created_at',
    # u'id_str',
    # u'place',
    # u'in_reply_to_user_id',
    # u'lang',
    # u'possibly_sensitive',
    # u'favorited',
    # u'favorite_count',
    # u'retweeted',
    # u'retweet_count',
]

class BulkApi(twitter.Api):
    """API with an extra call."""

    def LookupStatuses(self, status_ids, trim_user=False,
                       include_entities=True):
        """
        Returns up to 100 status messages, specified by a list passed to
        status_ids.

        Args:
          status_ids:
            A list of numeric IDs of the statuses you are trying to retrieve.
          trim_user:
            When set to True, each tweet returned in a timeline will include
            a user object including only the status authors numerical ID.
            Omit this parameter to receive the complete user object. [Optional]
          include_entities:
            If False, the entities node will be disincluded.
            This node offers a variety of metadata about the tweet in a
            discreet structure, including: user_mentions, urls, and
            hashtags. [Optional]
        Returns:
          A twitter.Status instance representing that status message
        """
        url = '%s/statuses/lookup.json' % self.base_url

        parameters = dict()

        if not status_ids or len(status_ids) > 100:
            raise TwitterError(
                "status_ids must be between 1 and 100 in length.")
        # This takes a comma-separated list of up to 100 IDs.
        parameters['id'] = ",".join(status_ids)

        if trim_user:
            parameters['trim_user'] = True
        if include_entities:
            parameters['include_entities'] = True

        resp = self._RequestUrl(url, 'GET', data=parameters)
        data = self._ParseAndCheckTwitter(resp.content.decode('utf-8'))

        return [twitter.Status.NewFromJsonDict(x) for x in data]

def archive_tweets(username, db_session, types=(USER, FAVORITES)):
    """
    Archives several types of new tweets along with their associated content.
    """
    new_ids = []
    api = twitter.Api(
        CONSUMER_KEY, CONSUMER_SECRET, ACCESS_KEY, ACCESS_SECRET,
        sleep_on_rate_limit=True)

    for type_ in types:

        # For favorites, always do a full sweep. We can't guarantee an older
        # tweet wasn't recently favorited!
        if type_ == FAVORITES:
            since_id = None
        else:
            since_id = db_session.query(RawTweet.id).\
                filter(RawTweet.favorited_by_str.like("%%%s%%" % username)).\
                order_by(desc(RawTweet.id)).first()
        if since_id:
            since_id = since_id[0]
        print type_, since_id

        max_id = None
        early_termination = False
        while not early_termination:
            print ("Pulling 200 tweets from API starting with ID %s and "
                   "ending with ID %s..." % (since_id, max_id))
            start_time = time.time()
            if type_ == FAVORITES:
                statuses = api.GetFavorites(
                    screen_name=username,
                    count=200,
                    since_id=since_id,
                    max_id=max_id,
                    include_entities=True)
                # 15 requests per 15 minutes.
                sleep_time = 60
            elif type_ == USER:
                statuses = api.GetUserTimeline(
                    screen_name=username,
                    count=200,
                    since_id=since_id,
                    max_id=max_id)
                # 300 requests per 15 minutes.
                sleep_time = 3
            print "Found %s tweets this iteration..." % len(statuses)
            if not statuses:
                break

            # Format things the way we want and handle max_id changes.
            for status in statuses:
                status_dict = status.AsDict()
                status_id = int(status_dict["id"])
                if since_id is not None and status_id >= since_id:
                    early_termination = True
                    break
                else:
                    new_ids.append(status_id)
                try:
                    raw_tweet = db_session.query(RawTweet).\
                        filter_by(id=status_id).one()
                except NoResultFound:
                    raw_tweet = RawTweet(status_dict=status_dict)
                    db_session.add(raw_tweet)
                if type_ == FAVORITES:
                    raw_tweet.add_user_favorite(username)
                db_session.commit()
                # Capture new max_id
                if status_id < max_id or max_id is None:
                    max_id = status_id - 1

            # Twitter rate-limits us to 15 requests / 15 minutes, so
            # space this out a bit to avoid a super-long sleep at the
            # end which could lose the connection.
            duration = time.time() - start_time
            if duration < sleep_time:
                sleep_duration = sleep_time - duration
                print ("Sleeping for %s seconds to ease up on rate "
                       "limit..." % sleep_duration)
                sleep(sleep_duration)
    return new_ids

def import_from_csv(db_session, csv_filepath, username):
    api = BulkApi(
        CONSUMER_KEY, CONSUMER_SECRET, ACCESS_KEY, ACCESS_SECRET,
        sleep_on_rate_limit=True)

    print "Importing into CSVTweets..."
    csv_rows_by_id = dict()
    with open(csv_filepath) as csvfile:
        reader = csv.DictReader(csvfile)
        for row in reader:
            tweet_id = int(row['tweet_id'])
            csv_rows_by_id[tweet_id] = row
            try:
                db_session.query(CSVTweet).filter_by(id=tweet_id).one()
            except NoResultFound:
                csv_tweet = CSVTweet(
                    id=tweet_id,
                    username=username,
                    in_reply_to_status_id=row["in_reply_to_status_id"],
                    in_reply_to_user_id=row["in_reply_to_user_id"],
                    timestamp=row["timestamp"],
                    text=row["text"],
                    retweeted_status_id=row["retweeted_status_id"],
                    retweeted_status_user_id=row["retweeted_status_user_id"],
                    retweeted_status_timestamp=row["retweeted_status_timestamp"],
                    expanded_urls=row["expanded_urls"])
                db_session.add(csv_tweet)
                db_session.commit()
    csv_ids = list(csv_rows_by_id.keys())
    new_api_tweets = []

    print "Attempting API import..."
    index = 0
    sliced_ids = csv_ids[:100]
    while sliced_ids:
        start_time = time.time()
        new_ids = []
        for status_id in sliced_ids:
            try:
                # If the tweet is found, it's already been imported. Ignore it.
                db_session.query(RawTweet.id).filter_by(id=status_id).one()
                csv_tweet = db_session.query(CSVTweet).\
                    filter_by(id=status_id).one()
                csv_tweet.api_import_complete = True
                db_session.commit()
            except NoResultFound:
                new_ids.append(str(status_id))
        if new_ids:
            print "Attempting import of id %s to %s of %s..." % (
                index + 1, index + 100, len(csv_ids))
            try:
                statuses = api.LookupStatuses(
                    status_ids=new_ids, trim_user=False, include_entities=True)
                for status in statuses:
                    status_dict = status.AsDict()
                    raw_tweet = RawTweet(status_dict=status_dict)
                    db_session.add(raw_tweet)
                    db_session.commit()
                    # Mark CSV tweet appropriately.
                    csv_tweet = db_session.query(CSVTweet).\
                        filter_by(id=int(status_dict["id"])).one()
                    csv_tweet.api_import_complete = True
                    db_session.commit()
                    # Append to new list.
                    new_api_tweets.append(raw_tweet)

                # Sleep to not hit the rate limit.
                # 60 requests per 15 minutes.
                duration = time.time() - start_time
                if duration < 15:
                    sleep_duration = 15 - duration
                    print ("Sleeping for %s seconds to ease up on rate "
                           "limit..." % sleep_duration)
                    sleep(sleep_duration)
            except TwitterError as e:
                print e
        index += 100
        sliced_ids = csv_ids[index:100 + index]

    csv_only_tweets = db_session.query(CSVTweet.api_import_complete).\
        filter_by(api_import_complete=False).all()

    return new_api_tweets, csv_only_tweets

def parse_tweets(db_session, media_path, raw_tweets=None, csv_only_tweets=None,
                 parse_all_raw=False, download_files=False):

    if parse_all_raw is True:
        # Process all captured raw tweets.
        raw_tweets = db_session.query(RawTweet)

    for raw_tweet in raw_tweets:

        # Generate User objects.
        user_dict = raw_tweet.raw_status_dict["user"]
        user_id = int(user_dict["id"])
        twitter_user_ids = db_session.query(TwitterUser.id).all()
        if user_id not in twitter_user_ids:
            user = TwitterUser.add_from_user_dict(db_session, user_dict)
            if download_files is True:
                user.download_files(db_session, media_path)
        else:
            user = db_session.query(TwitterUser).filter_by(id=user_id).one()

        # Generate Tweet objects.
        status_dict = raw_tweet.raw_status_dict
        tweet_id = int(status_dict["id"])
        tweet_ids = db_session.query(Tweet.id).filter_by(user_id=user.id).all()
        if tweet_id not in tweet_ids:
            tweet = Tweet.make_from_raw(raw_tweet)
            db_session.add(tweet)
        else:
            tweet = db_session.query(Tweet).filter_by(id=tweet_id).one()
        user.tweets.append(tweet)
        db_session.commit()

    if csv_only_tweets:
        print len(csv_only_tweets)

def print_tweets(db_session):
    for raw_tweet in db_session.query(RawTweet).all():
        print raw_tweet
<|endoftext|>


<commit_before>"""
Django settings for mysana project.

For more information on this file, see
https://docs.djangoproject.com/en/dev/topics/settings/

For the full list of settings and their values, see
https://docs.djangoproject.com/en/dev/ref/settings/
"""
from django.core.urlresolvers import reverse_lazy
from os.path import dirname, join, exists

# Build paths inside the project like this: join(BASE_DIR, "directory")
BASE_DIR = dirname(dirname(dirname(__file__)))
STATICFILES_DIRS = [join(BASE_DIR, 'static')]
MEDIA_ROOT = join(BASE_DIR, 'media')
MEDIA_URL = "/media/"

# Use Django templates using the new Django 1.8 TEMPLATES settings
TEMPLATES = [
    {
        'BACKEND': 'django.template.backends.django.DjangoTemplates',
        'DIRS': [
            join(BASE_DIR, 'templates'),
            # insert more TEMPLATE_DIRS here
        ],
        'APP_DIRS': True,
        'OPTIONS': {
            'context_processors': [
                # Insert your TEMPLATE_CONTEXT_PROCESSORS here or use this
                # list if you haven't customized them:
                'django.contrib.auth.context_processors.auth',
                'django.template.context_processors.debug',
                'django.template.context_processors.i18n',
                'django.template.context_processors.media',
                'django.template.context_processors.static',
                'django.template.context_processors.tz',
                'django.contrib.messages.context_processors.messages',
            ],
        },
    },
]

# Use 12factor inspired environment variables or from a file
import environ
env = environ.Env()

# Ideally move env file should be outside the git repo
# i.e. BASE_DIR.parent.parent
env_file = join(dirname(__file__), 'local.env')
if exists(env_file):
    environ.Env.read_env(str(env_file))

# Quick-start development settings - unsuitable for production
# See https://docs.djangoproject.com/en/dev/howto/deployment/checklist/

# SECURITY WARNING: keep the secret key used in production secret!
# Raises ImproperlyConfigured exception if SECRET_KEY not in os.environ
SECRET_KEY = env('SECRET_KEY')

ALLOWED_HOSTS = ['*']

# Application definition

INSTALLED_APPS = (
    'django.contrib.auth',
    'django_admin_bootstrapped',
    'django.contrib.admin',
    'django.contrib.contenttypes',
    'django.contrib.sessions',
    'django.contrib.messages',
    'django.contrib.staticfiles',

    'authtools',
    'crispy_forms',
    'easy_thumbnails',

    'profiles',
    'accounts',

)

MIDDLEWARE_CLASSES = (
    'django.contrib.sessions.middleware.SessionMiddleware',
    'django.middleware.common.CommonMiddleware',
    'django.middleware.csrf.CsrfViewMiddleware',
    'django.contrib.auth.middleware.AuthenticationMiddleware',
    'django.contrib.auth.middleware.SessionAuthenticationMiddleware',
    'django.contrib.messages.middleware.MessageMiddleware',
    'django.middleware.clickjacking.XFrameOptionsMiddleware',
)

ROOT_URLCONF = 'mysana.urls'

WSGI_APPLICATION = 'mysana.wsgi.application'

# Internationalization
# https://docs.djangoproject.com/en/dev/topics/i18n/

LANGUAGE_CODE = 'en-us'

TIME_ZONE = 'UTC'

USE_I18N = True

USE_L10N = True

USE_TZ = True

# Static files (CSS, JavaScript, Images)
# https://docs.djangoproject.com/en/dev/howto/static-files/

STATIC_URL = '/static/'

# Crispy Form Theme - Bootstrap 3
CRISPY_TEMPLATE_PACK = 'bootstrap3'

# For Bootstrap 3, change error alert to 'danger'
from django.contrib import messages
MESSAGE_TAGS = {
    messages.ERROR: 'danger'
}

# Authentication Settings
AUTH_USER_MODEL = 'authtools.User'
LOGIN_REDIRECT_URL = reverse_lazy("profiles:show_self")
LOGIN_URL = reverse_lazy("accounts:login")

THUMBNAIL_EXTENSION = 'png'     # Or any extn for your thumbnails
<commit_msg>goals app was added into the INSTALLED_APPS<commit_after>"""
Django settings for mysana project.

For more information on this file, see
https://docs.djangoproject.com/en/dev/topics/settings/

For the full list of settings and their values, see
https://docs.djangoproject.com/en/dev/ref/settings/
"""
from django.core.urlresolvers import reverse_lazy
from os.path import dirname, join, exists

# Build paths inside the project like this: join(BASE_DIR, "directory")
BASE_DIR = dirname(dirname(dirname(__file__)))
STATICFILES_DIRS = [join(BASE_DIR, 'static')]
MEDIA_ROOT = join(BASE_DIR, 'media')
MEDIA_URL = "/media/"

# Use Django templates using the new Django 1.8 TEMPLATES settings
TEMPLATES = [
    {
        'BACKEND': 'django.template.backends.django.DjangoTemplates',
        'DIRS': [
            join(BASE_DIR, 'templates'),
            # insert more TEMPLATE_DIRS here
        ],
        'APP_DIRS': True,
        'OPTIONS': {
            'context_processors': [
                # Insert your TEMPLATE_CONTEXT_PROCESSORS here or use this
                # list if you haven't customized them:
                'django.contrib.auth.context_processors.auth',
                'django.template.context_processors.debug',
                'django.template.context_processors.i18n',
                'django.template.context_processors.media',
                'django.template.context_processors.static',
                'django.template.context_processors.tz',
                'django.contrib.messages.context_processors.messages',
            ],
        },
    },
]

# Use 12factor inspired environment variables or from a file
import environ
env = environ.Env()

# Ideally move env file should be outside the git repo
# i.e. BASE_DIR.parent.parent
env_file = join(dirname(__file__), 'local.env')
if exists(env_file):
    environ.Env.read_env(str(env_file))

# Quick-start development settings - unsuitable for production
# See https://docs.djangoproject.com/en/dev/howto/deployment/checklist/

# SECURITY WARNING: keep the secret key used in production secret!
# Raises ImproperlyConfigured exception if SECRET_KEY not in os.environ
SECRET_KEY = env('SECRET_KEY')

ALLOWED_HOSTS = ['*']

# Application definition

INSTALLED_APPS = (
    'django.contrib.auth',
    'django_admin_bootstrapped',
    'django.contrib.admin',
    'django.contrib.contenttypes',
    'django.contrib.sessions',
    'django.contrib.messages',
    'django.contrib.staticfiles',

    'authtools',
    'crispy_forms',
    'easy_thumbnails',

    'profiles',
    'accounts',
    'goals',

)

MIDDLEWARE_CLASSES = (
    'django.contrib.sessions.middleware.SessionMiddleware',
    'django.middleware.common.CommonMiddleware',
    'django.middleware.csrf.CsrfViewMiddleware',
    'django.contrib.auth.middleware.AuthenticationMiddleware',
    'django.contrib.auth.middleware.SessionAuthenticationMiddleware',
    'django.contrib.messages.middleware.MessageMiddleware',
    'django.middleware.clickjacking.XFrameOptionsMiddleware',
)

ROOT_URLCONF = 'mysana.urls'

WSGI_APPLICATION = 'mysana.wsgi.application'

# Internationalization
# https://docs.djangoproject.com/en/dev/topics/i18n/

LANGUAGE_CODE = 'en-us'

TIME_ZONE = 'UTC'

USE_I18N = True

USE_L10N = True

USE_TZ = True

# Static files (CSS, JavaScript, Images)
# https://docs.djangoproject.com/en/dev/howto/static-files/

STATIC_URL = '/static/'

# Crispy Form Theme - Bootstrap 3
CRISPY_TEMPLATE_PACK = 'bootstrap3'

# For Bootstrap 3, change error alert to 'danger'
from django.contrib import messages
MESSAGE_TAGS = {
    messages.ERROR: 'danger'
}

# Authentication Settings
AUTH_USER_MODEL = 'authtools.User'
LOGIN_REDIRECT_URL = reverse_lazy("profiles:show_self")
LOGIN_URL = reverse_lazy("accounts:login")

THUMBNAIL_EXTENSION = 'png'     # Or any extn for your thumbnails
<|endoftext|>


<commit_before>#
# Copyright (c) 2010 Red Hat, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#           http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#

from ovirtcli.command.command import OvirtCommand
from cli.messages import Messages

class AddCommand(OvirtCommand):

    name = 'add'
    aliases = ('add',)
    description = 'creates a new object or adds existent'
    args_check = 1
    valid_options = [ ('*', str) ]

    helptext0 = """\
        == Usage ==

        add <type> [parent identifiers] [command options]

        == Description ==

        Creates a new object or adds existent

        == Supported Help formats ==

        - This help will list all available attribute options for given resource creation

          * format      - help add resource_name
          * example     - help add vm

        - This help will list all available attribute options for given subresource creation

          * format      - help add subresource_name --resource-identifier xxx
          * example     - help add disk --vm-identifier myvm

        == Available Types ==

        The following types of objects can be added:

          $types

        == Base Identifiers ==

        Some objects can only be created inside other objects. For example a
        disk can only be created inside a virtual machine. In this case, one
        or more base identifier options need to be given to identify the
        containing object. These options have the form --<type>id <id>.

        == Attribute Options ==

        Attribute options specify values for attributes of the to be created
        object.

        Attributes for the new object can be specified in one of two ways.

          * Using command-line options. For example, "--description foo"
            would add the object with a description of "foo".
          * By providing pre-formatted input, in the format specified by the
            'input_format' configuration variable. In this case the input
            needs to be provided using the '<<' input redirection operator.

        Type 'help add <type>' to see an overview of which attributes are
        available for a given type.

        == Examples ==

        - This example create a new virtual machine in the Default cluster based on the
          Blank template:

          $ add vm --name myvm --template-name iscsi_desktop_tmpl --cluster-name Default_iscsi

        - This example create vm nic:

          $ add nic --vm-identifier cli_vm3 --network-name engine --name test

        == Return Values ==

          $statuses
        """

    helptext1 = """\
        == Usage ==

        add <type> [parent identifiers] [command options]

        == Description ==

        Creates a new object or adds existent with type $type. See 'help add' for generic
        help on creating objects.

        == Attribute Options ==

        The following options are available for objects with type $type:

          $options

        == Collection based option format ==

          * [--x-y: collection]
            {
              [y.a: string]
              [y.b: string]
              [y.c: string]
            }

          e.g:

          --x-y "y.a=a1,y.b=b1,y.c=c1"
          --x-y "y.a=a2,y.b=b2,y.c=c2"
          ...

          where a, b, c are option properties and aN, bN, cN is actual user's data

        == Return Values ==

          $statuses
        """

    def execute(self):
        """Execute the "add" command."""
        args = self.arguments
        opts = self.options
        base = self.resolve_base(opts)
        typ = args[0] + 's'
        collection = None
        typs = self.get_singular_types(method='add', typ=args[0])

        if base:
            collection = getattr(base, typ)

        else:
            connection = self.check_connection()
            if hasattr(connection, typ):
                collection = getattr(connection, typ)

        if collection != None:
            result = self.execute_method(collection, 'add', opts)
            self.context.formatter.format(self.context, result)
        else:
            err_str = Messages.Error.CANNOT_CREATE
            if typs:
                err_str = err_str + \
                (Messages.Info.POSSIBALE_ARGUMENTS_COMBINATIONS % str(typs))
            self.error(
                   err_str % (args[0], typ)
            )

    def show_help(self):
        """Show help for "add"."""
        args = self.arguments
        opts = self.options
        types = self.get_singular_types(method='add')
        subst = {}
        if len(args) == 0:
            helptext = self.helptext0
            subst['types'] = self.format_map(types)
        elif len(args) == 1:
            if self.is_supported_type(types.keys(), args[0]):
                helptext = self.helptext1
                params_list = self.get_options(
                       method='add',
                       resource=args[0],
                       sub_resource=self.resolve_base(opts),
                       context_variants=types[args[0]]
                )
                subst['options'] = self.format_list(params_list, sort=False)
                subst['type'] = args[0]
        statuses = self.get_statuses()
        subst['statuses'] = self.format_list(statuses)
        helptext = self.format_help(helptext, subst)
        self.write(helptext)
<commit_msg>cli: Check existence of collection before adding<commit_after>#
# Copyright (c) 2010 Red Hat, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#           http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#

from ovirtcli.command.command import OvirtCommand
from cli.messages import Messages

class AddCommand(OvirtCommand):

    name = 'add'
    aliases = ('add',)
    description = 'creates a new object or adds existent'
    args_check = 1
    valid_options = [ ('*', str) ]

    helptext0 = """\
        == Usage ==

        add <type> [parent identifiers] [command options]

        == Description ==

        Creates a new object or adds existent

        == Supported Help formats ==

        - This help will list all available attribute options for given resource creation

          * format      - help add resource_name
          * example     - help add vm

        - This help will list all available attribute options for given subresource creation

          * format      - help add subresource_name --resource-identifier xxx
          * example     - help add disk --vm-identifier myvm

        == Available Types ==

        The following types of objects can be added:

          $types

        == Base Identifiers ==

        Some objects can only be created inside other objects. For example a
        disk can only be created inside a virtual machine. In this case, one
        or more base identifier options need to be given to identify the
        containing object. These options have the form --<type>id <id>.

        == Attribute Options ==

        Attribute options specify values for attributes of the to be created
        object.

        Attributes for the new object can be specified in one of two ways.

          * Using command-line options. For example, "--description foo"
            would add the object with a description of "foo".
          * By providing pre-formatted input, in the format specified by the
            'input_format' configuration variable. In this case the input
            needs to be provided using the '<<' input redirection operator.

        Type 'help add <type>' to see an overview of which attributes are
        available for a given type.

        == Examples ==

        - This example create a new virtual machine in the Default cluster based on the
          Blank template:

          $ add vm --name myvm --template-name iscsi_desktop_tmpl --cluster-name Default_iscsi

        - This example create vm nic:

          $ add nic --vm-identifier cli_vm3 --network-name engine --name test

        == Return Values ==

          $statuses
        """

    helptext1 = """\
        == Usage ==

        add <type> [parent identifiers] [command options]

        == Description ==

        Creates a new object or adds existent with type $type. See 'help add' for generic
        help on creating objects.

        == Attribute Options ==

        The following options are available for objects with type $type:

          $options

        == Collection based option format ==

          * [--x-y: collection]
            {
              [y.a: string]
              [y.b: string]
              [y.c: string]
            }

          e.g:

          --x-y "y.a=a1,y.b=b1,y.c=c1"
          --x-y "y.a=a2,y.b=b2,y.c=c2"
          ...

          where a, b, c are option properties and aN, bN, cN is actual user's data

        == Return Values ==

          $statuses
        """

    def execute(self):
        """Execute the "add" command."""
        args = self.arguments
        opts = self.options
        base = self.resolve_base(opts)
        typ = args[0] + 's'
        collection = None
        typs = self.get_singular_types(method='add', typ=args[0])

        if base:
            if hasattr(base, typ):
                collection = getattr(base, typ)
        else:
            connection = self.check_connection()
            if hasattr(connection, typ):
                collection = getattr(connection, typ)

        if collection != None:
            result = self.execute_method(collection, 'add', opts)
            self.context.formatter.format(self.context, result)
        else:
            err_str = Messages.Error.CANNOT_CREATE
            if typs:
                err_str = err_str + \
                (Messages.Info.POSSIBALE_ARGUMENTS_COMBINATIONS % str(typs))
            self.error(
                   err_str % (args[0], typ)
            )

    def show_help(self):
        """Show help for "add"."""
        args = self.arguments
        opts = self.options
        types = self.get_singular_types(method='add')
        subst = {}
        if len(args) == 0:
            helptext = self.helptext0
            subst['types'] = self.format_map(types)
        elif len(args) == 1:
            if self.is_supported_type(types.keys(), args[0]):
                helptext = self.helptext1
                params_list = self.get_options(
                       method='add',
                       resource=args[0],
                       sub_resource=self.resolve_base(opts),
                       context_variants=types[args[0]]
                )
                subst['options'] = self.format_list(params_list, sort=False)
                subst['type'] = args[0]
        statuses = self.get_statuses()
        subst['statuses'] = self.format_list(statuses)
        helptext = self.format_help(helptext, subst)
        self.write(helptext)
<|endoftext|>


<commit_before># Copyright 2014 Google Inc. All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are
# met:
#
#     * Redistributions of source code must retain the above copyright
# notice, this list of conditions and the following disclaimer.
#     * Redistributions in binary form must reproduce the above
# copyright notice, this list of conditions and the following disclaimer
# in the documentation and/or other materials provided with the
# distribution.
#     * Neither the name of Google Inc. nor the names of its
# contributors may be used to endorse or promote products derived from
# this software without specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

{
  'includes': [
    'processor_tools.gypi',
  ],
  'targets': [
    {
      'target_name': 'processor',
      'type': 'static_library',
      'sources': [
        'address_map-inl.h',
        'address_map.h',
        'basic_code_module.h',
        'basic_code_modules.cc',
        'basic_code_modules.h',
        'basic_source_line_resolver.cc',
        'basic_source_line_resolver_types.h',
        'binarystream.cc',
        'binarystream.h',
        'call_stack.cc',
        'cfi_frame_info-inl.h',
        'cfi_frame_info.cc',
        'cfi_frame_info.h',
        'contained_range_map-inl.h',
        'contained_range_map.h',
        'disassembler_x86.cc',
        'disassembler_x86.h',
        'exploitability.cc',
        'exploitability_linux.cc',
        'exploitability_linux.h',
        'exploitability_win.cc',
        'exploitability_win.h',
        'fast_source_line_resolver.cc',
        'fast_source_line_resolver_types.h',
        'linked_ptr.h',
        'logging.cc',
        'logging.h',
        'map_serializers-inl.h',
        'map_serializers.h',
        'minidump.cc',
        'minidump_processor.cc',
        'module_comparer.cc',
        'module_comparer.h',
        'module_factory.h',
        'module_serializer.cc',
        'module_serializer.h',
        'pathname_stripper.cc',
        'pathname_stripper.h',
        'postfix_evaluator-inl.h',
        'postfix_evaluator.h',
        'process_state.cc',
        'range_map-inl.h',
        'range_map.h',
        'simple_serializer-inl.h',
        'simple_serializer.h',
        'simple_symbol_supplier.cc',
        'simple_symbol_supplier.h',
        'source_line_resolver_base.cc',
        'source_line_resolver_base_types.h',
        'stack_frame_cpu.cc',
        'stack_frame_symbolizer.cc',
        'stackwalker.cc',
        'stackwalker_address_list.cc',
        'stackwalker_address_list.h',
        'stackwalker_amd64.cc',
        'stackwalker_amd64.h',
        'stackwalker_arm.cc',
        'stackwalker_arm.h',
        'stackwalker_arm64.cc',
        'stackwalker_arm64.h',
        'stackwalker_mips.cc',
        'stackwalker_mips.h',
        'stackwalker_ppc.cc',
        'stackwalker_ppc.h',
        'stackwalker_ppc64.cc',
        'stackwalker_ppc64.h',
        'stackwalker_selftest.cc',
        'stackwalker_sparc.cc',
        'stackwalker_sparc.h',
        'stackwalker_x86.cc',
        'stackwalker_x86.h',
        'static_address_map-inl.h',
        'static_address_map.h',
        'static_contained_range_map-inl.h',
        'static_contained_range_map.h',
        'static_map-inl.h',
        'static_map.h',
        'static_map_iterator-inl.h',
        'static_map_iterator.h',
        'static_range_map-inl.h',
        'static_range_map.h',
        'synth_minidump.cc',
        'synth_minidump.h',
        'tokenize.cc',
        'tokenize.h',
        'windows_frame_info.h',
      ],
      'include_dirs': [
        '..',
      ],
      'dependencies': [
        '../common/common.gyp:common',
        '../third_party/libdisasm/libdisasm.gyp:libdisasm',
      ],
    },
    {
      'target_name': 'processor_unittests',
      'type': 'executable',
      'sources': [
        'address_map_unittest.cc',
        'basic_source_line_resolver_unittest.cc',
        'binarystream_unittest.cc',
        'cfi_frame_info_unittest.cc',
        'contained_range_map_unittest.cc',
        'disassembler_x86_unittest.cc',
        'exploitability_unittest.cc',
        'fast_source_line_resolver_unittest.cc',
        'map_serializers_unittest.cc',
        'minidump_processor_unittest.cc',
        'minidump_unittest.cc',
        'pathname_stripper_unittest.cc',
        'postfix_evaluator_unittest.cc',
        'range_map_unittest.cc',
        'stackwalker_address_list_unittest.cc',
        'stackwalker_amd64_unittest.cc',
        'stackwalker_arm64_unittest.cc',
        'stackwalker_arm_unittest.cc',
        'stackwalker_mips_unittest.cc',
        'stackwalker_unittest_utils.h',
        'stackwalker_x86_unittest.cc',
        'static_address_map_unittest.cc',
        'static_contained_range_map_unittest.cc',
        'static_map_unittest.cc',
        'static_range_map_unittest.cc',
        'synth_minidump_unittest.cc',
        'synth_minidump_unittest_data.h',
      ],
      'include_dirs': [
        '..',
      ],
      'dependencies': [
        'processor',
        '../build/testing.gypi:gmock',
        '../build/testing.gypi:gtest',
      ],
    },
  ],
}
<commit_msg>Update processor.gyp file with new files added as part of microdump processing upcoming implementation.<commit_after># Copyright 2014 Google Inc. All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are
# met:
#
#     * Redistributions of source code must retain the above copyright
# notice, this list of conditions and the following disclaimer.
#     * Redistributions in binary form must reproduce the above
# copyright notice, this list of conditions and the following disclaimer
# in the documentation and/or other materials provided with the
# distribution.
#     * Neither the name of Google Inc. nor the names of its
# contributors may be used to endorse or promote products derived from
# this software without specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

{
  'includes': [
    'processor_tools.gypi',
  ],
  'targets': [
    {
      'target_name': 'processor',
      'type': 'static_library',
      'sources': [
        'address_map-inl.h',
        'address_map.h',
        'basic_code_module.h',
        'basic_code_modules.cc',
        'basic_code_modules.h',
        'basic_source_line_resolver.cc',
        'basic_source_line_resolver_types.h',
        'binarystream.cc',
        'binarystream.h',
        'call_stack.cc',
        'cfi_frame_info-inl.h',
        'cfi_frame_info.cc',
        'cfi_frame_info.h',
        'contained_range_map-inl.h',
        'contained_range_map.h',
        'disassembler_x86.cc',
        'disassembler_x86.h',
        'dump_context.cc',
        'dump_object.cc',
        'exploitability.cc',
        'exploitability_linux.cc',
        'exploitability_linux.h',
        'exploitability_win.cc',
        'exploitability_win.h',
        'fast_source_line_resolver.cc',
        'fast_source_line_resolver_types.h',
        'linked_ptr.h',
        'logging.cc',
        'logging.h',
        'map_serializers-inl.h',
        'map_serializers.h',
        'microdump_processor.cc',
        'minidump.cc',
        'minidump_processor.cc',
        'module_comparer.cc',
        'module_comparer.h',
        'module_factory.h',
        'module_serializer.cc',
        'module_serializer.h',
        'pathname_stripper.cc',
        'pathname_stripper.h',
        'postfix_evaluator-inl.h',
        'postfix_evaluator.h',
        'process_state.cc',
        'range_map-inl.h',
        'range_map.h',
        'simple_serializer-inl.h',
        'simple_serializer.h',
        'simple_symbol_supplier.cc',
        'simple_symbol_supplier.h',
        'source_line_resolver_base.cc',
        'source_line_resolver_base_types.h',
        'stack_frame_cpu.cc',
        'stack_frame_symbolizer.cc',
        'stackwalker.cc',
        'stackwalker_address_list.cc',
        'stackwalker_address_list.h',
        'stackwalker_amd64.cc',
        'stackwalker_amd64.h',
        'stackwalker_arm.cc',
        'stackwalker_arm.h',
        'stackwalker_arm64.cc',
        'stackwalker_arm64.h',
        'stackwalker_mips.cc',
        'stackwalker_mips.h',
        'stackwalker_ppc.cc',
        'stackwalker_ppc.h',
        'stackwalker_ppc64.cc',
        'stackwalker_ppc64.h',
        'stackwalker_selftest.cc',
        'stackwalker_sparc.cc',
        'stackwalker_sparc.h',
        'stackwalker_x86.cc',
        'stackwalker_x86.h',
        'static_address_map-inl.h',
        'static_address_map.h',
        'static_contained_range_map-inl.h',
        'static_contained_range_map.h',
        'static_map-inl.h',
        'static_map.h',
        'static_map_iterator-inl.h',
        'static_map_iterator.h',
        'static_range_map-inl.h',
        'static_range_map.h',
        'synth_minidump.cc',
        'synth_minidump.h',
        'tokenize.cc',
        'tokenize.h',
        'windows_frame_info.h',
      ],
      'include_dirs': [
        '..',
      ],
      'dependencies': [
        '../common/common.gyp:common',
        '../third_party/libdisasm/libdisasm.gyp:libdisasm',
      ],
    },
    {
      'target_name': 'processor_unittests',
      'type': 'executable',
      'sources': [
        'address_map_unittest.cc',
        'basic_source_line_resolver_unittest.cc',
        'binarystream_unittest.cc',
        'cfi_frame_info_unittest.cc',
        'contained_range_map_unittest.cc',
        'disassembler_x86_unittest.cc',
        'exploitability_unittest.cc',
        'fast_source_line_resolver_unittest.cc',
        'map_serializers_unittest.cc',
        'microdump_processor_unittest.cc',
        'minidump_processor_unittest.cc',
        'minidump_unittest.cc',
        'pathname_stripper_unittest.cc',
        'postfix_evaluator_unittest.cc',
        'range_map_unittest.cc',
        'stackwalker_address_list_unittest.cc',
        'stackwalker_amd64_unittest.cc',
        'stackwalker_arm64_unittest.cc',
        'stackwalker_arm_unittest.cc',
        'stackwalker_mips_unittest.cc',
        'stackwalker_unittest_utils.h',
        'stackwalker_x86_unittest.cc',
        'static_address_map_unittest.cc',
        'static_contained_range_map_unittest.cc',
        'static_map_unittest.cc',
        'static_range_map_unittest.cc',
        'synth_minidump_unittest.cc',
        'synth_minidump_unittest_data.h',
      ],
      'include_dirs': [
        '..',
      ],
      'dependencies': [
        'processor',
        '../build/testing.gypi:gmock',
        '../build/testing.gypi:gtest',
      ],
    },
  ],
}
<|endoftext|>