<commit_before>import webob

from fanstatic.config import convert_config
from fanstatic import ConfigurationError
from fanstatic import compat
from fanstatic.core import Bundle
from fanstatic import DEBUG, MINIFIED
import fanstatic

CONTENT_TYPES = ['text/html', 'text/xml', 'application/xhtml+xml']

class Injector(object):
    """Fanstatic injector WSGI framework component.

    This WSGI component takes care of injecting the proper resource
    inclusions into HTML when needed.

    This WSGI component is used automatically by the
    :py:func:`Fanstatic` WSGI framework component, but can also be
    used independently if you need more control.

    :param app: The WSGI app to wrap with the injector.

    :param ``**config``: Optional keyword arguments. These are passed
      to :py:class:`NeededResources` when it is constructed. It also
      makes sure that when initialized, it isn't given any
      configuration parameters that cannot be passed to
      ``NeededResources``.
    """
    def __init__(self, app, injector=None, **config):
        self.app = app

        # this is just to give useful feedback early on
        fanstatic.NeededResources(**config)

        self.config = config
        # BBB Backwards compatible: the default behavior was the top bottom
        # injector.
        if injector is None:
            injector = TopBottomInjector(config)
        self.injector = injector

    def __call__(self, environ, start_response):
        request = webob.Request(environ)
        # We only continue if the request method is appropriate.
        if not request.method in ['GET', 'POST']:
            return self.app(environ, start_response)

        # Initialize a needed resources object.
        # XXX this will set the needed on the thread local data, even
        # if the wrapped framework only gets the needed from the WSGI
        # environ.
        needed = fanstatic.init_needed(
            script_name=request.environ.get('SCRIPT_NAME'), **self.config)

        # Make sure the needed resource object is put in the WSGI
        # environment as well, for frameworks that choose to use it
        # from there.
        request.environ[fanstatic.NEEDED] = needed

        # Get the response from the wrapped application:
        response = request.get_response(self.app)
        # We only continue if the content-type is appropriate.
        if not (response.content_type and
                response.content_type.lower() in CONTENT_TYPES):
            # Clean up after our behinds.
            fanstatic.del_needed()
            return response(environ, start_response)

        # The wrapped application may have `needed` resources.
        if needed.has_resources():
            # Can't use response.text because there might not be any
            # charset. body is not unicode.
            result = self.injector(response.body, needed, request, response)
            # Reset the body...
            response.body = b''
            # Write will propely unfolder the previous application and
            # call close. Setting response.text or response.body won't do it.
            response.write(result)

        # Clean up after our behinds.
        fanstatic.del_needed()

        return response(environ, start_response)

class Inclusion(object):
    """Will be instantiated for every request.

    :param minified: If set to ``True``, Fanstatic will include all
      resources in ``minified`` form. If a Resource instance does not
      provide a ``minified`` mode, the "main" (non-named) mode is used.

    :param debug: If set to ``True``, Fanstatic will include all
      resources in ``debug`` form. If a Resource instance does not
      provide a ``debug`` mode, the "main" (non-named) mode is used.
      An exception is raised when both the ``debug`` and ``minified``
      parameters are ``True``.

    :param rollup: If set to True (default is False) rolled up
      combined resources will be served if they exist and supersede
      existing resources that are needed.

    :param bundle: If set to True, Fanstatic will attempt to bundle
      resources that fit together into larger Bundle objects. These
      can then be rendered as single URLs to these bundles.

    :param compile: XXX
    """
    def __init__(self, needed, resources=None,
            compile=False, bundle=False,
            mode=None, rollup=False):

        # Needed is basically the context object.
        self.needed = needed

        if resources is None:
            resources = needed.resources()

        if rollup:
            resources = rollup_resources(resources)

        if mode is not None:
            resources = [resource.mode(mode) for resource in resources]

        resources = sort_resources(resources)

        if compile:
            for resource in resources:
                resource.compile()

        if bundle:
            resources = bundle_resources(resources)

        self.resources = resources

    def __len__(self):
        return len(self.resources)

    def render(self):
        result = []
        for resource in self.resources:
            result.append(
                resource.render(
                    self.needed.library_url(resource.library)))
        return '\n'.join(result)

class InjectorPlugin(object):
    """Base class that can be use to write an injector plugin. It will
    take out from the configuration the common options that can be
    used in conjunction with an Inclusion.
    """

    def __init__(self, options):
        self._compile = options.pop('compile', False)
        self._bundle = options.pop('bundle', False)
        self._rollup = options.pop('rollup', False)
        debug = options.pop('debug', False)
        minified = options.pop('minified', False)
        self._mode = None
        if (debug and minified):
            raise ConfigurationError('Choose *one* of debug and minified')
        if debug is True:
            self._mode = DEBUG
        if minified is True:
            self._mode = MINIFIED

    def make_inclusion(self, needed, resources):
        """Helper to create an Inclusion passing all the options
        configured in the configuration file.
        """
        return Inclusion(needed, resources,
            compile=self._compile, bundle=self._bundle,
            mode=self._mode, rollup=self._rollup)

class TopBottomInjector(InjectorPlugin):

    name = 'topbottom'

    def __init__(self, options):
        """
        :param bottom: If set to ``True``, Fanstatic will include any
          resource that has been marked as "bottom safe" at the bottom of
          the web page, at the end of ````, as opposed to in the
          ```` section. This is useful for optimizing the load-time
          of Javascript resources.

        :param force_bottom: If set to ``True`` and ``bottom`` is set to
          ``True`` as well, all Javascript resources will be included at
          the bottom of a web page, even if they aren't marked bottom
          safe.

        """
        super(TopBottomInjector, self).__init__(options)
        self._bottom = options.pop('bottom', False)
        self._force_bottom = options.pop('force_bottom', False)

    def group(self, needed):
        """Return the top and bottom resources."""
        resources = needed.resources()
        if self._bottom:
            top_resources = []
            bottom_resources = []
            if not self._force_bottom:
                for resource in resources:
                    if resource.bottom:
                        bottom_resources.append(resource)
                    else:
                        top_resources.append(resource)
            else:
                for resource in resources:
                    if resource.ext == '.js':
                        bottom_resources.append(resource)
                    else:
                        top_resources.append(resource)
        else:
            top_resources = resources
            bottom_resources = []
        return (
            self.make_inclusion(needed, top_resources),
            self.make_inclusion(needed, bottom_resources)
        )

    def __call__(self, html, needed, request=None, response=None):
        # seperate inclusions in top and bottom inclusions if this is needed
        top, bottom = self.group(needed)
        if top:
            html = html.replace(
                compat.as_bytestring(''),
                compat.as_bytestring('%s' % top.render()), 1)
        if bottom:
            html = html.replace(
                compat.as_bytestring(''),
                compat.as_bytestring('%s' % bottom.render()), 1)
        return html

def bundle_resources(resources):
    """Bundle sorted resources together.

    resources is expected to be a list previously sorted by sorted_resources.

    Returns a list of renderable resources, which can include several
    resources bundled together into Bundles.
    """
    result = []
    bundle = Bundle()
    for resource in resources:
        if bundle.fits(resource):
            bundle.append(resource)
        else:
            # add the previous bundle to the list and create new bundle
            bundle.add_to_list(result)
            bundle = Bundle()
            if resource.dont_bundle:
                result.append(resource)
            else:
                bundle.append(resource)
    # add the last bundle to the list
    bundle.add_to_list(result)
    return result

def rollup_resources(resources):
    """Rollup resources together: if a resource include multiple
    separate ones (i.e. is a rollup) and all the separate ones are
    included the rollup will be used instead.
    """
    # keep track of rollups: rollup key -> set of resource keys
    potential_rollups = {}
    for resource in resources:
        for rollup in resource.rollups:
            s = potential_rollups.setdefault(
                (rollup.library, rollup.relpath), set())
            s.add((resource.library, resource.relpath))

    # now go through resources, replacing them with rollups if
    # conditions match
    result = set()
    for resource in resources:
        superseders = []
        for rollup in resource.rollups:
            s = potential_rollups[(rollup.library, rollup.relpath)]
            if len(s) == len(rollup.supersedes):
                superseders.append(rollup)
        if superseders:
            # use the exact superseder that rolls up the most
            superseders = sorted(superseders, key=lambda i: len(i.supersedes))
            result.add(superseders[-1])
        else:
            # nothing to supersede resource so use it directly
            result.add(resource)
    return result

def sort_resources(resources):
    """Sort resources for inclusion on web page.

    A number of rules are followed:

    * resources are always grouped per renderer (.js, .css, etc)
    * resources that depend on other resources are sorted later
    * resources are grouped by library, if the dependencies allow it
    * libraries are sorted by name, if dependencies allow it
    * resources are sorted by resource path if they both would be
      sorted the same otherwise.

    The only purpose of sorting on library is so we can
    group resources per library, so that bundles can later be created
    of them if bundling support is enabled.

    Note this sorting algorithm guarantees a consistent ordering, no
    matter in what order resources were needed.
    """
    for resource in resources:
        resource.library.init_library_nr()

    def key(resource):
        return (
            resource.order,
            resource.library.library_nr,
            resource.library.name,
            resource.dependency_nr,
            resource.relpath)
    return sorted(resources, key=key)

def make_injector(app, global_config, **local_config):
    local_config = convert_config(local_config)
    # Look up injector factory by name.
    injector_name = local_config.pop('injector', 'topbottom')
    injector_factory = fanstatic.registry.InjectorRegistry.instance().get(injector_name)
    if injector_factory is None:
        raise ConfigurationError(
            'No injector found for name %s' % injector_name)
    injector = injector_factory(local_config)
    return Injector(app, injector=injector, **local_config)
<commit_msg>Add an skeleton for the __call__ method in the injector base class.<commit_after>import webob

from fanstatic.config import convert_config
from fanstatic import ConfigurationError
from fanstatic import compat
from fanstatic.core import Bundle
from fanstatic import DEBUG, MINIFIED
import fanstatic

CONTENT_TYPES = ['text/html', 'text/xml', 'application/xhtml+xml']

class Injector(object):
    """Fanstatic injector WSGI framework component.

    This WSGI component takes care of injecting the proper resource
    inclusions into HTML when needed.

    This WSGI component is used automatically by the
    :py:func:`Fanstatic` WSGI framework component, but can also be
    used independently if you need more control.

    :param app: The WSGI app to wrap with the injector.

    :param ``**config``: Optional keyword arguments. These are passed
      to :py:class:`NeededResources` when it is constructed. It also
      makes sure that when initialized, it isn't given any
      configuration parameters that cannot be passed to
      ``NeededResources``.
    """
    def __init__(self, app, injector=None, **config):
        self.app = app

        # this is just to give useful feedback early on
        fanstatic.NeededResources(**config)

        self.config = config
        # BBB Backwards compatible: the default behavior was the top bottom
        # injector.
        if injector is None:
            injector = TopBottomInjector(config)
        self.injector = injector

    def __call__(self, environ, start_response):
        request = webob.Request(environ)
        # We only continue if the request method is appropriate.
        if not request.method in ['GET', 'POST']:
            return self.app(environ, start_response)

        # Initialize a needed resources object.
        # XXX this will set the needed on the thread local data, even
        # if the wrapped framework only gets the needed from the WSGI
        # environ.
        needed = fanstatic.init_needed(
            script_name=request.environ.get('SCRIPT_NAME'), **self.config)

        # Make sure the needed resource object is put in the WSGI
        # environment as well, for frameworks that choose to use it
        # from there.
        request.environ[fanstatic.NEEDED] = needed

        # Get the response from the wrapped application:
        response = request.get_response(self.app)
        # We only continue if the content-type is appropriate.
        if not (response.content_type and
                response.content_type.lower() in CONTENT_TYPES):
            # Clean up after our behinds.
            fanstatic.del_needed()
            return response(environ, start_response)

        # The wrapped application may have `needed` resources.
        if needed.has_resources():
            # Can't use response.text because there might not be any
            # charset. body is not unicode.
            result = self.injector(response.body, needed, request, response)
            # Reset the body...
            response.body = b''
            # Write will propely unfolder the previous application and
            # call close. Setting response.text or response.body won't do it.
            response.write(result)

        # Clean up after our behinds.
        fanstatic.del_needed()

        return response(environ, start_response)

class Inclusion(object):
    """Will be instantiated for every request.

    :param minified: If set to ``True``, Fanstatic will include all
      resources in ``minified`` form. If a Resource instance does not
      provide a ``minified`` mode, the "main" (non-named) mode is used.

    :param debug: If set to ``True``, Fanstatic will include all
      resources in ``debug`` form. If a Resource instance does not
      provide a ``debug`` mode, the "main" (non-named) mode is used.
      An exception is raised when both the ``debug`` and ``minified``
      parameters are ``True``.

    :param rollup: If set to True (default is False) rolled up
      combined resources will be served if they exist and supersede
      existing resources that are needed.

    :param bundle: If set to True, Fanstatic will attempt to bundle
      resources that fit together into larger Bundle objects. These
      can then be rendered as single URLs to these bundles.

    :param compile: XXX
    """
    def __init__(self, needed, resources=None,
            compile=False, bundle=False,
            mode=None, rollup=False):

        # Needed is basically the context object.
        self.needed = needed

        if resources is None:
            resources = needed.resources()

        if rollup:
            resources = rollup_resources(resources)

        if mode is not None:
            resources = [resource.mode(mode) for resource in resources]

        resources = sort_resources(resources)

        if compile:
            for resource in resources:
                resource.compile()

        if bundle:
            resources = bundle_resources(resources)

        self.resources = resources

    def __len__(self):
        return len(self.resources)

    def render(self):
        result = []
        for resource in self.resources:
            result.append(
                resource.render(
                    self.needed.library_url(resource.library)))
        return '\n'.join(result)

class InjectorPlugin(object):
    """Base class that can be use to write an injector plugin. It will
    take out from the configuration the common options that can be
    used in conjunction with an Inclusion.
    """

    def __init__(self, options):
        self._compile = options.pop('compile', False)
        self._bundle = options.pop('bundle', False)
        self._rollup = options.pop('rollup', False)
        debug = options.pop('debug', False)
        minified = options.pop('minified', False)
        self._mode = None
        if (debug and minified):
            raise ConfigurationError('Choose *one* of debug and minified')
        if debug is True:
            self._mode = DEBUG
        if minified is True:
            self._mode = MINIFIED

    def make_inclusion(self, needed, resources):
        """Helper to create an Inclusion passing all the options
        configured in the configuration file.
        """
        return Inclusion(needed, resources,
            compile=self._compile, bundle=self._bundle,
            mode=self._mode, rollup=self._rollup)

    def __call__(self, html, needed, request, response):
        raise NotImplementedError

class TopBottomInjector(InjectorPlugin):

    name = 'topbottom'

    def __init__(self, options):
        """
        :param bottom: If set to ``True``, Fanstatic will include any
          resource that has been marked as "bottom safe" at the bottom of
          the web page, at the end of ````, as opposed to in the
          ```` section. This is useful for optimizing the load-time
          of Javascript resources.

        :param force_bottom: If set to ``True`` and ``bottom`` is set to
          ``True`` as well, all Javascript resources will be included at
          the bottom of a web page, even if they aren't marked bottom
          safe.

        """
        super(TopBottomInjector, self).__init__(options)
        self._bottom = options.pop('bottom', False)
        self._force_bottom = options.pop('force_bottom', False)

    def group(self, needed):
        """Return the top and bottom resources."""
        resources = needed.resources()
        if self._bottom:
            top_resources = []
            bottom_resources = []
            if not self._force_bottom:
                for resource in resources:
                    if resource.bottom:
                        bottom_resources.append(resource)
                    else:
                        top_resources.append(resource)
            else:
                for resource in resources:
                    if resource.ext == '.js':
                        bottom_resources.append(resource)
                    else:
                        top_resources.append(resource)
        else:
            top_resources = resources
            bottom_resources = []
        return (
            self.make_inclusion(needed, top_resources),
            self.make_inclusion(needed, bottom_resources)
        )

    def __call__(self, html, needed, request=None, response=None):
        # seperate inclusions in top and bottom inclusions if this is needed
        top, bottom = self.group(needed)
        if top:
            html = html.replace(
                compat.as_bytestring(''),
                compat.as_bytestring('%s' % top.render()), 1)
        if bottom:
            html = html.replace(
                compat.as_bytestring(''),
                compat.as_bytestring('%s' % bottom.render()), 1)
        return html

def bundle_resources(resources):
    """Bundle sorted resources together.

    resources is expected to be a list previously sorted by sorted_resources.

    Returns a list of renderable resources, which can include several
    resources bundled together into Bundles.
    """
    result = []
    bundle = Bundle()
    for resource in resources:
        if bundle.fits(resource):
            bundle.append(resource)
        else:
            # add the previous bundle to the list and create new bundle
            bundle.add_to_list(result)
            bundle = Bundle()
            if resource.dont_bundle:
                result.append(resource)
            else:
                bundle.append(resource)
    # add the last bundle to the list
    bundle.add_to_list(result)
    return result

def rollup_resources(resources):
    """Rollup resources together: if a resource include multiple
    separate ones (i.e. is a rollup) and all the separate ones are
    included the rollup will be used instead.
    """
    # keep track of rollups: rollup key -> set of resource keys
    potential_rollups = {}
    for resource in resources:
        for rollup in resource.rollups:
            s = potential_rollups.setdefault(
                (rollup.library, rollup.relpath), set())
            s.add((resource.library, resource.relpath))

    # now go through resources, replacing them with rollups if
    # conditions match
    result = set()
    for resource in resources:
        superseders = []
        for rollup in resource.rollups:
            s = potential_rollups[(rollup.library, rollup.relpath)]
            if len(s) == len(rollup.supersedes):
                superseders.append(rollup)
        if superseders:
            # use the exact superseder that rolls up the most
            superseders = sorted(superseders, key=lambda i: len(i.supersedes))
            result.add(superseders[-1])
        else:
            # nothing to supersede resource so use it directly
            result.add(resource)
    return result

def sort_resources(resources):
    """Sort resources for inclusion on web page.

    A number of rules are followed:

    * resources are always grouped per renderer (.js, .css, etc)
    * resources that depend on other resources are sorted later
    * resources are grouped by library, if the dependencies allow it
    * libraries are sorted by name, if dependencies allow it
    * resources are sorted by resource path if they both would be
      sorted the same otherwise.

    The only purpose of sorting on library is so we can
    group resources per library, so that bundles can later be created
    of them if bundling support is enabled.

    Note this sorting algorithm guarantees a consistent ordering, no
    matter in what order resources were needed.
    """
    for resource in resources:
        resource.library.init_library_nr()

    def key(resource):
        return (
            resource.order,
            resource.library.library_nr,
            resource.library.name,
            resource.dependency_nr,
            resource.relpath)
    return sorted(resources, key=key)

def make_injector(app, global_config, **local_config):
    local_config = convert_config(local_config)
    # Look up injector factory by name.
    injector_name = local_config.pop('injector', 'topbottom')
    injector_factory = fanstatic.registry.InjectorRegistry.instance().get(injector_name)
    if injector_factory is None:
        raise ConfigurationError(
            'No injector found for name %s' % injector_name)
    injector = injector_factory(local_config)
    return Injector(app, injector=injector, **local_config)
<|endoftext|>


<commit_before>import os
import subprocess

ROOT = subprocess.check_output([
    'git', 'rev-parse', '--show-toplevel']).decode('ascii').strip()

VERSION_FILE = os.path.join(ROOT, 'cosmic_ray', 'version.py')

def git(*args):
    subprocess.check_call(('git',) + args)

def tags():
    result = [t.decode('ascii') for t in subprocess.check_output([
        'git', 'tag'
    ]).split(b"\n")]
    assert len(set(result)) == len(result)
    return set(result)

def create_tag_and_push(version):
    assert version not in tags()
    git('config', 'user.name', 'Travis CI on behalf of Austin Bingham')
    git('config', 'user.email', 'austin@sixty-north.com')
    git('config', 'core.sshCommand', 'ssh -i deploy_key')
    git(
        'remote', 'add', 'ssh-origin',
        'git@github.com:sixty-north/cosmic-ray.git'
    )
    git('tag', version)

    subprocess.check_call([
        'ssh-agent', 'sh', '-c',
        'chmod 0600 deploy_key && ' +
        'ssh-add deploy_key && ' +
        # 'git push ssh-origin HEAD:master &&'
        'git push ssh-origin --tags'
    ])

def read_version(version_file):
    vars = {}
    with open(version_file) as f:
        exec(f.read(), {}, vars)
    return (vars['__version__'], vars['__version_info__'])

if __name__ == '__main__':
    version, version_info = read_version(VERSION_FILE)
    create_tag_and_push(version)
<commit_msg>Added docstrings to the release script.<commit_after>import os
import subprocess

ROOT = subprocess.check_output([
    'git', 'rev-parse', '--show-toplevel']).decode('ascii').strip()

VERSION_FILE = os.path.join(ROOT, 'cosmic_ray', 'version.py')

def git(*args):
    "Run a git command."
    subprocess.check_call(('git',) + args)

def tags():
    "Get a set of tags for the current git repo."
    result = [t.decode('ascii') for t in subprocess.check_output([
        'git', 'tag'
    ]).split(b"\n")]
    assert len(set(result)) == len(result)
    return set(result)

def create_tag_and_push(version):
    "Create a git tag for `version` and push it to origin."
    assert version not in tags()
    git('config', 'user.name', 'Travis CI on behalf of Austin Bingham')
    git('config', 'user.email', 'austin@sixty-north.com')
    git('config', 'core.sshCommand', 'ssh -i deploy_key')
    git(
        'remote', 'add', 'ssh-origin',
        'git@github.com:sixty-north/cosmic-ray.git'
    )
    git('tag', version)

    subprocess.check_call([
        'ssh-agent', 'sh', '-c',
        'chmod 0600 deploy_key && ' +
        'ssh-add deploy_key && ' +
        # 'git push ssh-origin HEAD:master &&'
        'git push ssh-origin --tags'
    ])

def read_version(version_file):
    "Read the `(version-string, version-info)` from `version_file`."
    vars = {}
    with open(version_file) as f:
        exec(f.read(), {}, vars)
    return (vars['__version__'], vars['__version_info__'])

if __name__ == '__main__':
    version, version_info = read_version(VERSION_FILE)
    create_tag_and_push(version)
<|endoftext|>


<commit_before># Copyright 2018 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""This script is used to synthesize generated parts of this library."""

import synthtool as s
import synthtool.gcp as gcp

gapic = gcp.GAPICGenerator()
common_templates = gcp.CommonTemplates()

library = gapic.java_library(
    service='container',
    version='v1beta1',
    config_path='/google/devtools/containeranalysis/artman_containeranalysis_v1beta1.yaml',
    artman_output_name='')

s.copy(library / 'gapic-google-cloud-containeranalysis-v1beta1/src', 'src')
s.copy(library / 'grpc-google-cloud-containeranalysis-v1beta1/src', '../../google-api-grpc/grpc-google-cloud-containeranalysis-v1beta1/src')
s.copy(library / 'proto-google-cloud-containeranalysis-v1beta1/src', '../../google-api-grpc/proto-google-cloud-containeranalysis-v1beta1/src')
<commit_msg>Add lint test and format generated code (#4114)<commit_after># Copyright 2018 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""This script is used to synthesize generated parts of this library."""

import synthtool as s
import synthtool.gcp as gcp
import synthtool.languages.java as java

gapic = gcp.GAPICGenerator()

service = 'containeranalysis'
versions = ['v1beta1']
config_pattern = '/google/devtools/containeranalysis/artman_containeranalysis_{version}.yaml'

for version in versions:
  library = gapic.java_library(
      service=service,
      version=version,
      config_path=config_pattern.format(version=version),
      artman_output_name='')

  s.copy(library / f'gapic-google-cloud-{service}-{version}/src', 'src')
  s.copy(library / f'grpc-google-cloud-{service}-{version}/src', f'../../google-api-grpc/grpc-google-cloud-{service}-{version}/src')
  s.copy(library / f'proto-google-cloud-{service}-{version}/src', f'../../google-api-grpc/proto-google-cloud-{service}-{version}/src')

  java.format_code('./src')
  java.format_code(f'../../google-api-grpc/grpc-google-cloud-{service}-{version}/src')
  java.format_code(f'../../google-api-grpc/proto-google-cloud-{service}-{version}/src')
<|endoftext|>


<commit_before>#!/usr/bin/env python
"""A Python calculator"""
import sys

command = sys.argv[1]
numbers = [float(a) for a in sys.argv[2:]]
a = float(sys.argv[2])
b = float(sys.argv[3])

if command == 'add':
    print sum(numbers)

elif command == 'multiply':
    product = 1
    for num in numbers:
        product *= num
    print product
elif command == 'divide':
    div = 1
    if b != 0:
        div = a/b
        print div

print ':)'
<commit_msg>added error and line<commit_after>#!/usr/bin/env python
"""A Python calculator"""
import sys

command = sys.argv[1]
numbers = [float(a) for a in sys.argv[2:]]
a = float(sys.argv[2])
b = float(sys.argv[3])

if command == 'add':
    print sum(numbers)

elif command == 'multiply':
    product = 1
    for num in numbers:
        product *= num
    print product
elif command == 'divide':
    div = 1
    if b != 0:
        div = a/b
        print div
adding some errors here.
print ':)'

print "all good."
<|endoftext|>


<commit_before>import os
import sys
import logging
import logging.config

from copy import copy
from config import Config

from dashi import DashiConnection

DEFAULT_CONFIG_FILES = [
    'config/service.yml',
    'config/service.local.yml',
    ]
LOGGING_CONFIG_FILES = [
    'config/logging.yml',
    'config/logging.local.yml',
    ]

class Service(object):
    """Base class for services. Meant to be subclassed

    """

    topic = "service"

    def __init__(self, config_files=DEFAULT_CONFIG_FILES,
                 logging_config_files=LOGGING_CONFIG_FILES, *args, **kwargs):

        self.CFG = Config(config_files).data
        self.LOGGING_CFG = Config(logging_config_files).data

        self.CFG['cli_args'] = self._parse_argv()

        self.amqp_uri = "amqp://%s:%s@%s/%s" % (
                        self.CFG.server.amqp.username,
                        self.CFG.server.amqp.password,
                        self.CFG.server.amqp.host,
                        self.CFG.server.amqp.vhost,
                        )

        self.dashi = DashiConnection(self.topic, self.amqp_uri, self.topic)

    def get_logger(self, name=None):
        """set up logging for a service using the py 2.7 dictConfig
        """

        if not name:
            name = self.__class__.__name__

        return get_logger(name, self.LOGGING_CFG)

    def _parse_argv(self, argv=copy(sys.argv)):
        """return argv as a parsed dictionary, looks like the following:

        app --option1 likethis --option2 likethat --flag

        ->

        {'option1': 'likethis', 'option2': 'likethat', 'flag': True}
        """

        cli_args = {}
        while argv:
            arg = argv[0]
            try:
                maybe_val = argv[1]
            except IndexError:
                maybe_val = None

            if arg[0] == '-':
                key = arg.lstrip('-')
                if not maybe_val or maybe_val[0] == '-':
                    val = True
                    argv = argv[1:]
                else:
                    val = maybe_val
                    argv = argv[2:]
                cli_args[key] = val
            else:
                #skip arguments that aren't preceded with -
                argv = argv[1:]

        return cli_args

def get_logger(name, CFG=None):
    """set up logging for a service using the py 2.7 dictConfig
    """

    logger = logging.getLogger(name)

    if CFG:
        # Make log directory if it doesn't exist
        for handler in CFG.get('handlers', {}).itervalues():
            if 'filename' in handler:
                log_dir = os.path.dirname(handler['filename'])
                if not os.path.exists(log_dir):
                    os.makedirs(log_dir)
        try:
            #TODO: This requires python 2.7
            logging.config.dictConfig(CFG)
        except AttributeError:
            print >> sys.stderr, '"logging.config.dictConfig" doesn\'t seem to be supported in your python'
            raise

    return logger
<commit_msg>Set default exchange for Dashi connections in bootstrap<commit_after>import os
import sys
import logging
import logging.config

from copy import copy
from config import Config

from dashi import DashiConnection

DEFAULT_CONFIG_FILES = [
    'config/service.yml',
    'config/service.local.yml',
    ]
LOGGING_CONFIG_FILES = [
    'config/logging.yml',
    'config/logging.local.yml',
    ]

DEFAULT_EXCHANGE = "default_dashi_exchange"

class Service(object):
    """Base class for services. Meant to be subclassed

    """

    topic = "service"

    def __init__(self, config_files=DEFAULT_CONFIG_FILES,
                 logging_config_files=LOGGING_CONFIG_FILES, *args, **kwargs):

        self.CFG = Config(config_files).data
        self.LOGGING_CFG = Config(logging_config_files).data

        self.CFG['cli_args'] = self._parse_argv()

        self.amqp_uri = "amqp://%s:%s@%s/%s" % (
                        self.CFG.server.amqp.username,
                        self.CFG.server.amqp.password,
                        self.CFG.server.amqp.host,
                        self.CFG.server.amqp.vhost,
                        )
        try:
            self.dashi_exchange = self.CFG.dashi.exchange
        except NameError:
            self.dashi_exchange = DEFAULT_EXCHANGE

        self.dashi = DashiConnection(self.topic, self.amqp_uri, self.dashi_exchange)

    def get_logger(self, name=None):
        """set up logging for a service using the py 2.7 dictConfig
        """

        if not name:
            name = self.__class__.__name__

        return get_logger(name, self.LOGGING_CFG)

    def _parse_argv(self, argv=copy(sys.argv)):
        """return argv as a parsed dictionary, looks like the following:

        app --option1 likethis --option2 likethat --flag

        ->

        {'option1': 'likethis', 'option2': 'likethat', 'flag': True}
        """

        cli_args = {}
        while argv:
            arg = argv[0]
            try:
                maybe_val = argv[1]
            except IndexError:
                maybe_val = None

            if arg[0] == '-':
                key = arg.lstrip('-')
                if not maybe_val or maybe_val[0] == '-':
                    val = True
                    argv = argv[1:]
                else:
                    val = maybe_val
                    argv = argv[2:]
                cli_args[key] = val
            else:
                #skip arguments that aren't preceded with -
                argv = argv[1:]

        return cli_args

def get_logger(name, CFG=None):
    """set up logging for a service using the py 2.7 dictConfig
    """

    logger = logging.getLogger(name)

    if CFG:
        # Make log directory if it doesn't exist
        for handler in CFG.get('handlers', {}).itervalues():
            if 'filename' in handler:
                log_dir = os.path.dirname(handler['filename'])
                if not os.path.exists(log_dir):
                    os.makedirs(log_dir)
        try:
            #TODO: This requires python 2.7
            logging.config.dictConfig(CFG)
        except AttributeError:
            print >> sys.stderr, '"logging.config.dictConfig" doesn\'t seem to be supported in your python'
            raise

    return logger
<|endoftext|>


<commit_before>
# Copyright 2016 Battelle Energy Alliance, LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

from ci import models
import re
from django.core.urlresolvers import reverse

def find_in_output(output, key):
    """
    Find a key in the output and return its value.
    """
    matches = re.search("^%s=(.*)" % key, output, re.MULTILINE)
    if matches:
        return matches.groups()[0]
    return None

def get_output_by_position(job, position):
    """
    Utility function to get the output of a job step result by position
    """
    return job.step_results.get(position=position).output

def check_submodule_update(job, position):
    """
    Checks to see if certain submodules have been updated and post a comment to the PR if so.
    """
    output = get_output_by_position(job, position)
    modules = find_in_output(output, "CIVET_CLIENT_SUBMODULE_UPDATES")
    if not modules:
        return False
    if not job.event.pull_request or not job.event.pull_request.review_comments_url:
        return False
    for mod in modules.split():
        oauth_session = job.event.build_user.start_session()
        api = job.event.pull_request.repository.server().api()
        url = job.event.pull_request.review_comments_url
        sha = job.event.head.sha
        msg = "**Caution!** This contains a submodule update"
        # The 2 position will leave the message on the new submodule hash
        api.pr_review_comment(oauth_session, url, sha, mod, 2, msg)
        return True

def ensure_single_new_comment(oauth_session, api, builduser, url, msg, comment_re):
    """
    Adds a new comment and deletes any existing similar comments.
    The difference between this and edit_comment() is that this method will
    typically cause a new email to be sent.
    """
    comments = api.get_pr_comments(oauth_session, url, builduser.name, comment_re)
    for c in comments:
        api.remove_pr_comment(oauth_session, c)
    api.pr_comment(oauth_session, url, msg)

def edit_comment(oauth_session, api, builduser, url, msg, comment_re):
    """
    Replaces an existing comment with a new one. Removes any similar
    messages except the first one.
    """
    comments = api.get_pr_comments(oauth_session, url, builduser.name, comment_re)
    if comments:
        for c in comments[1:]:
            api.remove_pr_comment(oauth_session, c)
        api.edit_pr_comment(oauth_session, comments[0], msg)
    else:
        api.pr_comment(oauth_session, url, msg)

def check_post_comment(request, job, position, edit, delete):
    """
    Checks to see if we should post a message to the PR.
    """
    output = get_output_by_position(job, position)
    message = find_in_output(output, "CIVET_CLIENT_POST_MESSAGE")
    if not message:
        matches = re.search("^CIVET_CLIENT_START_POST_MESSAGE$\n(.*)\n^CIVET_CLIENT_END_POST_MESSAGE$", output, re.MULTILINE|re.DOTALL)
        if matches:
            message = matches.groups()[0]
    if message and job.event.comments_url:
        builduser = job.event.build_user
        repo = job.event.pull_request.repository
        oauth_session = builduser.start_session()
        abs_job_url = request.build_absolute_uri(reverse('ci:view_job', args=[job.pk]))
        msg = "Job [%s](%s) on %s wanted to post the following:\n\n%s" % (job.unique_name(), abs_job_url, job.event.head.short_sha(), message)
        api = repo.server().api()
        url = job.event.comments_url
        comment_re = r"^Job \[%s\]\(.*\) on \w+ wanted to post the following:" % job.unique_name()

        if edit:
            edit_comment(oauth_session, api, builduser, url, msg, comment_re)
        elif delete:
            ensure_single_new_comment(oauth_session, api, builduser, url, msg, comment_re)
        else:
            api.pr_comment(oauth_session, url, msg)
        return True
    return False

def process_commands(request, job):
    """
    See if we need to check for any commands on this job.
    Commands take the form of an environment variable set on the recipe to
    indicate that we need to check the output for certain key value pairs.
    """
    if job.event.cause != models.Event.PULL_REQUEST:
        return
    for step in job.recipe.steps.prefetch_related("step_environment").all():
        edit = False
        delete = False
        for step_env in step.step_environment.all():
            if step_env.name == "CIVET_SERVER_POST_REMOVE_OLD" and step_env.value == "1":
                delete = True
                break
            elif step_env.name == "CIVET_SERVER_POST_EDIT_EXISTING" and step_env.value == "1":
                edit = True
                break
        for step_env in step.step_environment.all():
            if step_env.name == "CIVET_SERVER_POST_ON_SUBMODULE_UPDATE" and step_env.value == "1":
                check_submodule_update(job, step.position)
                break
            elif step_env.name == "CIVET_SERVER_POST_COMMENT" and step_env.value == "1":
                check_post_comment(request, job, step.position, edit, delete)
                break
<commit_msg>Add a line to indicate that a comment will auto update<commit_after>
# Copyright 2016 Battelle Energy Alliance, LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

from ci import models
import re
from django.core.urlresolvers import reverse

def find_in_output(output, key):
    """
    Find a key in the output and return its value.
    """
    matches = re.search("^%s=(.*)" % key, output, re.MULTILINE)
    if matches:
        return matches.groups()[0]
    return None

def get_output_by_position(job, position):
    """
    Utility function to get the output of a job step result by position
    """
    return job.step_results.get(position=position).output

def check_submodule_update(job, position):
    """
    Checks to see if certain submodules have been updated and post a comment to the PR if so.
    """
    output = get_output_by_position(job, position)
    modules = find_in_output(output, "CIVET_CLIENT_SUBMODULE_UPDATES")
    if not modules:
        return False
    if not job.event.pull_request or not job.event.pull_request.review_comments_url:
        return False
    for mod in modules.split():
        oauth_session = job.event.build_user.start_session()
        api = job.event.pull_request.repository.server().api()
        url = job.event.pull_request.review_comments_url
        sha = job.event.head.sha
        msg = "**Caution!** This contains a submodule update"
        # The 2 position will leave the message on the new submodule hash
        api.pr_review_comment(oauth_session, url, sha, mod, 2, msg)
        return True

def ensure_single_new_comment(oauth_session, api, builduser, url, msg, comment_re):
    """
    Adds a new comment and deletes any existing similar comments.
    The difference between this and edit_comment() is that this method will
    typically cause a new email to be sent.
    """
    comments = api.get_pr_comments(oauth_session, url, builduser.name, comment_re)
    for c in comments:
        api.remove_pr_comment(oauth_session, c)
    api.pr_comment(oauth_session, url, msg)

def edit_comment(oauth_session, api, builduser, url, msg, comment_re):
    """
    Replaces an existing comment with a new one. Removes any similar
    messages except the first one.
    """
    comments = api.get_pr_comments(oauth_session, url, builduser.name, comment_re)
    if comments:
        for c in comments[1:]:
            api.remove_pr_comment(oauth_session, c)
        api.edit_pr_comment(oauth_session, comments[0], msg)
    else:
        api.pr_comment(oauth_session, url, msg)

def check_post_comment(request, job, position, edit, delete):
    """
    Checks to see if we should post a message to the PR.
    """
    output = get_output_by_position(job, position)
    message = find_in_output(output, "CIVET_CLIENT_POST_MESSAGE")
    if not message:
        matches = re.search("^CIVET_CLIENT_START_POST_MESSAGE$\n(.*)\n^CIVET_CLIENT_END_POST_MESSAGE$", output, re.MULTILINE|re.DOTALL)
        if matches:
            message = matches.groups()[0]
    if message and job.event.comments_url:
        builduser = job.event.build_user
        repo = job.event.pull_request.repository
        oauth_session = builduser.start_session()
        abs_job_url = request.build_absolute_uri(reverse('ci:view_job', args=[job.pk]))
        msg = "Job [%s](%s) on %s wanted to post the following:\n\n%s" % (job.unique_name(), abs_job_url, job.event.head.short_sha(), message)
        api = repo.server().api()
        url = job.event.comments_url
        comment_re = r"^Job \[%s\]\(.*\) on \w+ wanted to post the following:" % job.unique_name()

        if edit:
            msg = "%s\n\nThis comment will be updated on new commits." % msg
            edit_comment(oauth_session, api, builduser, url, msg, comment_re)
        elif delete:
            ensure_single_new_comment(oauth_session, api, builduser, url, msg, comment_re)
        else:
            api.pr_comment(oauth_session, url, msg)
        return True
    return False

def process_commands(request, job):
    """
    See if we need to check for any commands on this job.
    Commands take the form of an environment variable set on the recipe to
    indicate that we need to check the output for certain key value pairs.
    """
    if job.event.cause != models.Event.PULL_REQUEST:
        return
    for step in job.recipe.steps.prefetch_related("step_environment").all():
        edit = False
        delete = False
        for step_env in step.step_environment.all():
            if step_env.name == "CIVET_SERVER_POST_REMOVE_OLD" and step_env.value == "1":
                delete = True
                break
            elif step_env.name == "CIVET_SERVER_POST_EDIT_EXISTING" and step_env.value == "1":
                edit = True
                break
        for step_env in step.step_environment.all():
            if step_env.name == "CIVET_SERVER_POST_ON_SUBMODULE_UPDATE" and step_env.value == "1":
                check_submodule_update(job, step.position)
                break
            elif step_env.name == "CIVET_SERVER_POST_COMMENT" and step_env.value == "1":
                check_post_comment(request, job, step.position, edit, delete)
                break
<|endoftext|>


<commit_before># Copyright (C) 2010 Jamu Kakar <jkakar@kakar.ca>
# Licenced under the txaws licence available at /LICENSE in the txaws source.

"""
A L{Command} object makes an arbitrary EC2 API method call and displays the
response received from the backend cloud.
"""

import sys

from txaws.ec2.client import Query
from txaws.service import AWSServiceRegion

class Command(object):
    """
    An EC2 API method call command that can make a request and display the
    response received from the backend cloud.

    @param key: The AWS access key ID to use when making the method call.
    @param secret: The AWS secret key to sign the method call with.
    @param endpoint: The URL of the cloud to invoke the method on.
    @param parameters: A C{dict} with parameters to include with the method
        call.
    @param output: Optionally, a stream to write output to.  Defaults to
        C{sys.stdout}.
    @param query_factory: Optionally, a factory to create the L{Query} object
        used to invoke the method.  Defaults to returning a L{Query} instance.
    """

    def __init__(self, key, secret, endpoint, action, parameters, output=None,
                 query_factory=None):
        self.key = key
        self.secret = secret
        self.endpoint = endpoint
        self.action = action
        self.parameters = parameters
        if output is None:
            output = sys.stdout
        self.output = output
        if query_factory is None:
            query_factory = Query
        self.query_factory = query_factory

    def run(self):
        """
        Run the configured method and write the HTTP response status and text
        to the output stream.
        """
        region = AWSServiceRegion(access_key=self.key, secret_key=self.secret,
                                  uri=self.endpoint)
        query = self.query_factory(action=self.action, creds=region.creds,
                                   endpoint=region.ec2_endpoint,
                                   other_params=self.parameters)

        def write_response(response):
            print >>self.output, "HTTP status code: %s" % query.client.status
            print >>self.output
            print >>self.output, response

        def write_error(failure):
            message = failure.getErrorMessage()
            if message.startswith("Error Message: "):
                message = message[len("Error Message: "):]
            print >>self.output, "HTTP status code: %s" % query.client.status
            print >>self.output
            print >>self.output, message

        deferred = query.submit()
        deferred.addCallback(write_response)
        deferred.addErrback(write_error)
        return deferred
<commit_msg>- Print the full XML when an error occurs.<commit_after># Copyright (C) 2010 Jamu Kakar <jkakar@kakar.ca>
# Licenced under the txaws licence available at /LICENSE in the txaws source.

"""
A L{Command} object makes an arbitrary EC2 API method call and displays the
response received from the backend cloud.
"""

import sys

from txaws.ec2.client import Query
from txaws.exception import AWSError
from txaws.service import AWSServiceRegion

class Command(object):
    """
    An EC2 API method call command that can make a request and display the
    response received from the backend cloud.

    @param key: The AWS access key ID to use when making the method call.
    @param secret: The AWS secret key to sign the method call with.
    @param endpoint: The URL of the cloud to invoke the method on.
    @param parameters: A C{dict} with parameters to include with the method
        call.
    @param output: Optionally, a stream to write output to.  Defaults to
        C{sys.stdout}.
    @param query_factory: Optionally, a factory to create the L{Query} object
        used to invoke the method.  Defaults to returning a L{Query} instance.
    """

    def __init__(self, key, secret, endpoint, action, parameters, output=None,
                 query_factory=None):
        self.key = key
        self.secret = secret
        self.endpoint = endpoint
        self.action = action
        self.parameters = parameters
        if output is None:
            output = sys.stdout
        self.output = output
        if query_factory is None:
            query_factory = Query
        self.query_factory = query_factory

    def run(self):
        """
        Run the configured method and write the HTTP response status and text
        to the output stream.
        """
        region = AWSServiceRegion(access_key=self.key, secret_key=self.secret,
                                  uri=self.endpoint)
        query = self.query_factory(action=self.action, creds=region.creds,
                                   endpoint=region.ec2_endpoint,
                                   other_params=self.parameters)

        def write_response(response):
            print >>self.output, "HTTP status code: %s" % query.client.status
            print >>self.output
            print >>self.output, response

        def write_error(failure):
            if failure.check(AWSError):
                message = failure.value.original
            else:
                message = failure.getErrorMessage()
                if message.startswith("Error Message: "):
                    message = message[len("Error Message: "):]
            print >>self.output, "HTTP status code: %s" % query.client.status
            print >>self.output
            print >>self.output, message

        deferred = query.submit()
        deferred.addCallback(write_response)
        deferred.addErrback(write_error)
        return deferred
<|endoftext|>


<commit_before># Copyright 2017 The Forseti Security Authors. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""Forseti Server program."""

# pylint: disable=missing-type-doc,missing-param-doc
# pylint: disable=line-too-long,useless-suppression

import argparse

from abc import ABCMeta, abstractmethod
from multiprocessing.pool import ThreadPool
import time
from concurrent import futures
import grpc

from google.cloud.forseti.services.client import ClientComposition
from google.cloud.forseti.services import db
from google.cloud.forseti.services.dao import ModelManager, create_engine
from google.cloud.forseti.services.explain.service import GrpcExplainerFactory
from google.cloud.forseti.services.playground.service import GrpcPlaygrounderFactory
from google.cloud.forseti.services.inventory.service import GrpcInventoryFactory
from google.cloud.forseti.services.scanner.service import GrpcScannerFactory
from google.cloud.forseti.services.model.service import GrpcModellerFactory
from google.cloud.forseti.services.inventory.storage import Storage

STATIC_SERVICE_MAPPING = {
    'explain': GrpcExplainerFactory,
    'playground': GrpcPlaygrounderFactory,
    'inventory': GrpcInventoryFactory,
    'scanner': GrpcScannerFactory,
    'model': GrpcModellerFactory,
}

class AbstractServiceConfig(object):
    """Abstract base class for service configuration. This class
    is used to implement dependency injection for the gRPC services."""

    __metaclass__ = ABCMeta

    @abstractmethod
    def get_engine(self):
        """Get the database engine.

        Raises:
            NotImplementedError: Abstract.
        """

        raise NotImplementedError()

    @abstractmethod
    def scoped_session(self):
        """Get a scoped session.

        Raises:
            NotImplementedError: Abstract.
        """

        raise NotImplementedError()

    @abstractmethod
    def client(self):
        """Get an API client.

        Raises:
            NotImplementedError: Abstract.
        """

        raise NotImplementedError()

    @abstractmethod
    def run_in_background(self, function):
        """Runs a function in a thread pool in the background.

        Args:
            function (Function): Function to be executed.

        Raises:
            NotImplementedError: Abstract.
        """

        raise NotImplementedError()

    def get_storage_class(self):
        """Returns the class used for the inventory storage.

        Raises:
            NotImplementedError: Abstract.
        """

        raise NotImplementedError()

class AbstractInventoryConfig(dict):
    """Abstract base class for service configuration. This class
    is used to implement dependency injection for the gRPC services."""

    __metaclass__ = ABCMeta

    def get_root_resource_id(self):
        """Returns the root resource id.

        Raises:
            NotImplementedError: Abstract.
        """

        raise NotImplementedError()

    def get_gsuite_sa_path(self):
        """Returns gsuite service account path.

        Raises:
            NotImplementedError: Abstract.
        """

        raise NotImplementedError()

    def get_gsuite_admin_email(self):
        """Returns gsuite admin email.

        Raises:
            NotImplementedError: Abstract.
        """

        raise NotImplementedError()

    def get_service_config(self):
        """Returns the service config.

        Raises:
            NotImplementedError: Abstract.
        """

        raise NotImplementedError()

class InventoryConfig(AbstractInventoryConfig):
    """Implements composed dependency injection for the inventory."""

    def __init__(self,
                 root_resource_id,
                 gsuite_sa_path,
                 gsuite_admin_email,
                 *args,
                 **kwargs):

        super(InventoryConfig, self).__init__(*args, **kwargs)
        self.service_config = None
        self.root_resource_id = root_resource_id
        self.gsuite_sa_path = gsuite_sa_path
        self.gsuite_admin_email = gsuite_admin_email

    def get_root_resource_id(self):
        """Return the configured root resource id.

        Returns:
            str: Root resource id.
        """

        return self.root_resource_id

    def get_gsuite_sa_path(self):
        """Return the gsuite service account path.

        Returns:
            str: Gsuite service account path.
        """

        return self.gsuite_sa_path

    def get_gsuite_admin_email(self):
        """Return the gsuite admin email to use.

        Returns:
            str: Gsuite admin email.
        """

        return self.gsuite_admin_email

    def get_service_config(self):
        """Return the attached service configuration.

        Returns:
            object: Service configuration.
        """

        return self.service_config

    def set_service_config(self, service_config):
        """Attach a service configuration.

        Args:
            service_config (object): Service configuration.
        """

        self.service_config = service_config

class ServiceConfig(AbstractServiceConfig):
    """Implements composed dependency injection to Forseti Server services."""

    def __init__(self,
                 inventory_config,
                 explain_connect_string,
                 endpoint):

        super(ServiceConfig, self).__init__()
        self.thread_pool = ThreadPool()
        self.engine = create_engine(explain_connect_string, pool_recycle=3600)
        self.model_manager = ModelManager(self.engine)
        self.sessionmaker = db.create_scoped_sessionmaker(self.engine)
        self.endpoint = endpoint

        self.inventory_config = inventory_config
        self.inventory_config.set_service_config(self)

    def get_inventory_config(self):
        """Get the inventory config.

        Returns:
            object: Inventory config.
        """

        return self.inventory_config

    def get_engine(self):
        """Get the database engine.

        Returns:
            object: Database engine object.
        """

        return self.engine

    def scoped_session(self):
        """Get a scoped session.

        Returns:
            object: A scoped session.
        """

        return self.sessionmaker()

    def client(self):
        """Get an API client.

        Returns:
            object: API client to use against services.
        """

        return ClientComposition(self.endpoint)

    def run_in_background(self, function):
        """Runs a function in a thread pool in the background.

        Args:
            function (Function): Function to be executed.
        """

        self.thread_pool.apply_async(function)

    def get_storage_class(self):
        """Returns the storage class used to access the inventory.

        Returns:
            class: Type of a storage implementation.
        """

        return Storage

def serve(endpoint, services,
          explain_connect_string,
          gsuite_sa_path, gsuite_admin_email,
          root_resource_id, max_workers=32, wait_shutdown_secs=3):
    """Instantiate the services and serves them via gRPC.

    Raises:
        Exception: No services to start
    """

    factories = []
    for service in services:
        factories.append(STATIC_SERVICE_MAPPING[service])

    if not factories:
        raise Exception("No services to start")

    # Setting up configurations
    inventory_config = InventoryConfig(root_resource_id,
                                       gsuite_sa_path,
                                       gsuite_admin_email)
    config = ServiceConfig(inventory_config,
                           explain_connect_string,
                           endpoint)
    inventory_config.set_service_config(config)

    server = grpc.server(futures.ThreadPoolExecutor(max_workers))
    for factory in factories:
        factory(config).create_and_register_service(server)

    server.add_insecure_port(endpoint)
    server.start()
    while True:
        try:
            time.sleep(1)
        except KeyboardInterrupt:
            server.stop(wait_shutdown_secs).wait()
            return

def main():
    """Run."""
    parser = argparse.ArgumentParser()
    parser.add_argument(
        '--endpoint',
        default='[::]:50051',
        help='Server endpoint')
    parser.add_argument(
        '--forseti_db',
        help=('Forseti database string, formatted as '
              '"mysql://<db_user>@<db_host>:<db_port>/<db_name>"'))
    parser.add_argument(
        '--gsuite_private_keyfile',
        help='Path to G Suite service account private keyfile')
    parser.add_argument(
        '--gsuite_admin_email',
        help='G Suite admin email')
    parser.add_argument(
        '--root_resource_id',
        help=('Root resource to start crawling from, formatted as '
              '"<resource_type>/<resource_id>" '
              '(e.g. "organizations/12345677890")'))
    parser.add_argument(
        '--services',
        nargs='*',
        default=[],
        help='Forseti services')
    args = vars(parser.parse_args())

    serve(args['endpoint'], args['services'], args['forseti_db'],
          args['gsuite_private_keyfile'], args['gsuite_admin_email'],
          args['root_resource_id'])

if __name__ == "__main__":
    main()
<commit_msg>Allow users to specify the log level when starting the server<commit_after># Copyright 2017 The Forseti Security Authors. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""Forseti Server program."""

# pylint: disable=missing-type-doc,missing-param-doc
# pylint: disable=line-too-long,useless-suppression

import argparse

from abc import ABCMeta, abstractmethod
from multiprocessing.pool import ThreadPool
import time
from concurrent import futures
import grpc

from google.cloud.forseti.services.client import ClientComposition
from google.cloud.forseti.services import db
from google.cloud.forseti.services.dao import ModelManager, create_engine
from google.cloud.forseti.services.explain.service import GrpcExplainerFactory
from google.cloud.forseti.services.playground.service import GrpcPlaygrounderFactory
from google.cloud.forseti.services.inventory.service import GrpcInventoryFactory
from google.cloud.forseti.services.scanner.service import GrpcScannerFactory
from google.cloud.forseti.services.model.service import GrpcModellerFactory
from google.cloud.forseti.services.inventory.storage import Storage

from google.cloud.forseti.common.util import log_util

STATIC_SERVICE_MAPPING = {
    'explain': GrpcExplainerFactory,
    'playground': GrpcPlaygrounderFactory,
    'inventory': GrpcInventoryFactory,
    'scanner': GrpcScannerFactory,
    'model': GrpcModellerFactory,
}

class AbstractServiceConfig(object):
    """Abstract base class for service configuration. This class
    is used to implement dependency injection for the gRPC services."""

    __metaclass__ = ABCMeta

    @abstractmethod
    def get_engine(self):
        """Get the database engine.

        Raises:
            NotImplementedError: Abstract.
        """

        raise NotImplementedError()

    @abstractmethod
    def scoped_session(self):
        """Get a scoped session.

        Raises:
            NotImplementedError: Abstract.
        """

        raise NotImplementedError()

    @abstractmethod
    def client(self):
        """Get an API client.

        Raises:
            NotImplementedError: Abstract.
        """

        raise NotImplementedError()

    @abstractmethod
    def run_in_background(self, function):
        """Runs a function in a thread pool in the background.

        Args:
            function (Function): Function to be executed.

        Raises:
            NotImplementedError: Abstract.
        """

        raise NotImplementedError()

    def get_storage_class(self):
        """Returns the class used for the inventory storage.

        Raises:
            NotImplementedError: Abstract.
        """

        raise NotImplementedError()

class AbstractInventoryConfig(dict):
    """Abstract base class for service configuration. This class
    is used to implement dependency injection for the gRPC services."""

    __metaclass__ = ABCMeta

    def get_root_resource_id(self):
        """Returns the root resource id.

        Raises:
            NotImplementedError: Abstract.
        """

        raise NotImplementedError()

    def get_gsuite_sa_path(self):
        """Returns gsuite service account path.

        Raises:
            NotImplementedError: Abstract.
        """

        raise NotImplementedError()

    def get_gsuite_admin_email(self):
        """Returns gsuite admin email.

        Raises:
            NotImplementedError: Abstract.
        """

        raise NotImplementedError()

    def get_service_config(self):
        """Returns the service config.

        Raises:
            NotImplementedError: Abstract.
        """

        raise NotImplementedError()

class InventoryConfig(AbstractInventoryConfig):
    """Implements composed dependency injection for the inventory."""

    def __init__(self,
                 root_resource_id,
                 gsuite_sa_path,
                 gsuite_admin_email,
                 *args,
                 **kwargs):

        super(InventoryConfig, self).__init__(*args, **kwargs)
        self.service_config = None
        self.root_resource_id = root_resource_id
        self.gsuite_sa_path = gsuite_sa_path
        self.gsuite_admin_email = gsuite_admin_email

    def get_root_resource_id(self):
        """Return the configured root resource id.

        Returns:
            str: Root resource id.
        """

        return self.root_resource_id

    def get_gsuite_sa_path(self):
        """Return the gsuite service account path.

        Returns:
            str: Gsuite service account path.
        """

        return self.gsuite_sa_path

    def get_gsuite_admin_email(self):
        """Return the gsuite admin email to use.

        Returns:
            str: Gsuite admin email.
        """

        return self.gsuite_admin_email

    def get_service_config(self):
        """Return the attached service configuration.

        Returns:
            object: Service configuration.
        """

        return self.service_config

    def set_service_config(self, service_config):
        """Attach a service configuration.

        Args:
            service_config (object): Service configuration.
        """

        self.service_config = service_config

class ServiceConfig(AbstractServiceConfig):
    """Implements composed dependency injection to Forseti Server services."""

    def __init__(self,
                 inventory_config,
                 explain_connect_string,
                 endpoint):

        super(ServiceConfig, self).__init__()
        self.thread_pool = ThreadPool()
        self.engine = create_engine(explain_connect_string, pool_recycle=3600)
        self.model_manager = ModelManager(self.engine)
        self.sessionmaker = db.create_scoped_sessionmaker(self.engine)
        self.endpoint = endpoint

        self.inventory_config = inventory_config
        self.inventory_config.set_service_config(self)

    def get_inventory_config(self):
        """Get the inventory config.

        Returns:
            object: Inventory config.
        """

        return self.inventory_config

    def get_engine(self):
        """Get the database engine.

        Returns:
            object: Database engine object.
        """

        return self.engine

    def scoped_session(self):
        """Get a scoped session.

        Returns:
            object: A scoped session.
        """

        return self.sessionmaker()

    def client(self):
        """Get an API client.

        Returns:
            object: API client to use against services.
        """

        return ClientComposition(self.endpoint)

    def run_in_background(self, function):
        """Runs a function in a thread pool in the background.

        Args:
            function (Function): Function to be executed.
        """

        self.thread_pool.apply_async(function)

    def get_storage_class(self):
        """Returns the storage class used to access the inventory.

        Returns:
            class: Type of a storage implementation.
        """

        return Storage

def serve(endpoint, services,
          explain_connect_string,
          gsuite_sa_path, gsuite_admin_email,
          root_resource_id, log_level,
          max_workers=32, wait_shutdown_secs=3):
    """Instantiate the services and serves them via gRPC.

    Raises:
        Exception: No services to start
    """

    factories = []
    for service in services:
        factories.append(STATIC_SERVICE_MAPPING[service])

    if not factories:
        raise Exception("No services to start")

    # Configuring log level for the application
    log_util.set_logger_level_from_config(log_level)

    # Setting up configurations
    inventory_config = InventoryConfig(root_resource_id,
                                       gsuite_sa_path,
                                       gsuite_admin_email)
    config = ServiceConfig(inventory_config,
                           explain_connect_string,
                           endpoint)
    inventory_config.set_service_config(config)

    server = grpc.server(futures.ThreadPoolExecutor(max_workers))
    for factory in factories:
        factory(config).create_and_register_service(server)

    server.add_insecure_port(endpoint)
    server.start()
    while True:
        try:
            time.sleep(1)
        except KeyboardInterrupt:
            server.stop(wait_shutdown_secs).wait()
            return

def main():
    """Run."""
    parser = argparse.ArgumentParser()
    parser.add_argument(
        '--endpoint',
        default='[::]:50051',
        help='Server endpoint')
    parser.add_argument(
        '--forseti_db',
        help=('Forseti database string, formatted as '
              '"mysql://<db_user>@<db_host>:<db_port>/<db_name>"'))
    parser.add_argument(
        '--gsuite_private_keyfile',
        help='Path to G Suite service account private keyfile')
    parser.add_argument(
        '--gsuite_admin_email',
        help='G Suite admin email')
    parser.add_argument(
        '--root_resource_id',
        help=('Root resource to start crawling from, formatted as '
              '"<resource_type>/<resource_id>" '
              '(e.g. "organizations/12345677890")'))
    parser.add_argument(
        '--services',
        nargs='*',
        default=[],
        help='Forseti services')
    parser.add_argument(
        '--log_level',
        default='info',
        choices=['debug', 'info', 'warning', 'error'],
        help="Sets the threshold for Forseti's logger."
             " Logging messages which are less severe"
             " than the level you set will be ignored.")
    args = vars(parser.parse_args())

    serve(args['endpoint'], args['services'], args['forseti_db'],
          args['gsuite_private_keyfile'], args['gsuite_admin_email'],
          args['root_resource_id'], args['log_level'])

if __name__ == "__main__":
    main()
<|endoftext|>


<commit_before>from future.standard_library import install_aliases
install_aliases()  # NOQA
import base64
import codecs
import json
import netaddr
import os
try:
    import subprocess32 as subprocess  # Closes pipes on failure
except ImportError:
    import subprocess
from pyramid.config import Configurator
from pyramid.path import (
    AssetResolver,
    caller_package,
)
from pyramid.authorization import ACLAuthorizationPolicy
from pyramid.session import SignedCookieSessionFactory
from pyramid.settings import (
    aslist,
    asbool,
)
from sqlalchemy import engine_from_config
from webob.cookies import JSONSerializer
from snovault.json_renderer import json_renderer
from snovault.app import (
    STATIC_MAX_AGE,
    session,
    json_from_path,
    configure_dbsession,
    changelogs,
    json_asset
)
from dcicutils.log_utils import set_logging
from dcicutils.beanstalk_utils import whodaman
from encoded.commands.create_mapping_on_deploy import (
    ENV_WEBPROD,
    ENV_WEBPROD2,
    BEANSTALK_PROD_ENVS,
)
import structlog
import logging

# location of environment variables on elasticbeanstalk
BEANSTALK_ENV_PATH = "/opt/python/current/env"

def get_mirror_env(settings):
    """ Gets the mirror_env from whodaman instead of env variable """
    who_is_data, who_i_am = whodaman(), settings.get('env.name', '')
    if who_i_am not in BEANSTALK_PROD_ENVS:  # no mirror if we're not in prod
        return None
    if who_is_data == ENV_WEBPROD:
        return ENV_WEBPROD2
    return ENV_WEBPROD

def static_resources(config):
    from pkg_resources import resource_filename
    import mimetypes
    mimetypes.init()
    mimetypes.init([resource_filename('encoded', 'static/mime.types')])
    config.add_static_view('static', 'static', cache_max_age=STATIC_MAX_AGE)
    config.add_static_view('profiles', 'schemas', cache_max_age=STATIC_MAX_AGE)

    # Favicon
    favicon_path = '/static/img/favicon.ico'
    if config.route_prefix:
        favicon_path = '/%s%s' % (config.route_prefix, favicon_path)
    config.add_route('favicon.ico', 'favicon.ico')

    def favicon(request):
        subreq = request.copy()
        subreq.path_info = favicon_path
        response = request.invoke_subrequest(subreq)
        return response

    config.add_view(favicon, route_name='favicon.ico')

    # Robots.txt
    robots_txt_path = None
    if config.registry.settings.get('testing') in [True, 'true', 'True']:
        robots_txt_path = '/static/dev-robots.txt'
    else:
        robots_txt_path = '/static/robots.txt'

    if config.route_prefix:
        robots_txt_path = '/%s%s' % (config.route_prefix, robots_txt_path)

    config.add_route('robots.txt-conditional', '/robots.txt')

    def robots_txt(request):
        subreq = request.copy()
        subreq.path_info = robots_txt_path
        response = request.invoke_subrequest(subreq)
        return response

    config.add_view(robots_txt, route_name='robots.txt-conditional')

def load_workbook(app, workbook_filename, docsdir):
    from .loadxl import load_all
    from webtest import TestApp
    environ = {
        'HTTP_ACCEPT': 'application/json',
        'REMOTE_USER': 'IMPORT',
    }
    testapp = TestApp(app, environ)
    load_all(testapp, workbook_filename, docsdir)

def source_beanstalk_env_vars(config_file=BEANSTALK_ENV_PATH):
    """
    set environment variables if we are on Elastic Beanstalk
    AWS_ACCESS_KEY_ID is indicative of whether or not env vars are sourced

    Args:
        config_file (str): filepath to load env vars from
    """
    if os.path.exists(config_file) and not os.environ.get("AWS_ACCESS_KEY_ID"):
        command = ['bash', '-c', 'source ' + config_file + ' && env']
        proc = subprocess.Popen(command, stdout=subprocess.PIPE, universal_newlines=True)
        for line in proc.stdout:
            key, _, value = line.partition("=")
            os.environ[key] = value[:-1]
        proc.communicate()

def app_version(config):
    import hashlib
    if not config.registry.settings.get('snovault.app_version'):
        # we update version as part of deployment process `deploy_beanstalk.py`
        # but if we didn't check env then git
        version = os.environ.get("ENCODED_VERSION")
        if not version:
            try:
                version = subprocess.check_output(
                    ['git', '-C', os.path.dirname(__file__), 'describe']).decode('utf-8').strip()
                diff = subprocess.check_output(
                    ['git', '-C', os.path.dirname(__file__), 'diff', '--no-ext-diff'])
                if diff:
                    version += '-patch' + hashlib.sha1(diff).hexdigest()[:7]
            except:
                version = "test"

        config.registry.settings['snovault.app_version'] = version

    # GA Config
    ga_conf_file = config.registry.settings.get('ga_config_location')
    ga_conf_existing = config.registry.settings.get('ga_config')
    if ga_conf_file and not ga_conf_existing:
        ga_conf_file = os.path.normpath(
            os.path.join(
                os.path.dirname(os.path.abspath(__file__)), # Absolute loc. of this file
                "../../",                                   # Go back up to repo dir
                ga_conf_file
            )
        )
        if not os.path.exists(ga_conf_file):
            raise Exception(ga_conf_file + " does not exist in filesystem. Aborting.")
        with open(ga_conf_file) as json_file:
            config.registry.settings["ga_config"] = json.load(json_file)

def main(global_config, **local_config):
    """
    This function returns a Pyramid WSGI application.
    """

    settings = global_config
    settings.update(local_config)

    set_logging(in_prod=settings.get('production'))
    # set_logging(settings.get('elasticsearch.server'), settings.get('production'))

    # source environment variables on elastic beanstalk
    source_beanstalk_env_vars()

    settings['snovault.jsonld.namespaces'] = json_asset('encoded:schemas/namespaces.json')
    settings['snovault.jsonld.terms_namespace'] = 'https://www.encodeproject.org/terms/'
    settings['snovault.jsonld.terms_prefix'] = 'encode'
    # set auth0 keys
    settings['auth0.secret'] = os.environ.get("Auth0Secret")
    settings['auth0.client'] = os.environ.get("Auth0Client")
    # set google reCAPTCHA keys
    settings['g.recaptcha.key'] = os.environ.get('reCaptchaKey')
    settings['g.recaptcha.secret'] = os.environ.get('reCaptchaSecret')
    # set mirrored Elasticsearch location (for webprod/webprod2)
    import ipdb; ipdb.set_trace()
    settings['mirror.env.name'] = get_mirror_env(settings)
    config = Configurator(settings=settings)

    from snovault.elasticsearch import APP_FACTORY
    config.registry[APP_FACTORY] = main  # used by mp_indexer
    config.include(app_version)

    config.include('pyramid_multiauth')  # must be before calling set_authorization_policy
    from pyramid_localroles import LocalRolesAuthorizationPolicy
    # Override default authz policy set by pyramid_multiauth
    config.set_authorization_policy(LocalRolesAuthorizationPolicy())
    config.include(session)

    # must include, as tm.attempts was removed from pyramid_tm
    config.include('pyramid_retry')

    config.include(configure_dbsession)
    config.include('snovault')
    config.commit()  # commit so search can override listing

    # Render an HTML page to browsers and a JSON document for API clients
    #config.include(add_schemas_to_html_responses)
    config.include('.renderers')
    config.include('.authentication')
    config.include('.server_defaults')
    config.include('.root')
    config.include('.types')
    config.include('.batch_download')
    config.include('.loadxl')
    config.include('.visualization')

    if 'elasticsearch.server' in config.registry.settings:
        config.include('snovault.elasticsearch')
        config.include('.search')

    # this contains fall back url, so make sure it comes just before static_resoruces
    config.include('.types.page')
    config.include(static_resources)
    config.include(changelogs)

    # we are loading ontologies now as regular items
    # config.registry['ontology'] = json_from_path(settings.get('ontology_path'), {})
    aws_ip_ranges = json_from_path(settings.get('aws_ip_ranges_path'), {'prefixes': []})
    config.registry['aws_ipset'] = netaddr.IPSet(
        record['ip_prefix'] for record in aws_ip_ranges['prefixes'] if record['service'] == 'AMAZON')

    if asbool(settings.get('testing', False)):
        config.include('.tests.testing_views')

    # Load upgrades last so that all views (including testing views) are
    # registered.
    config.include('.upgrade')

    app = config.make_wsgi_app()

    workbook_filename = settings.get('load_workbook', '')
    load_test_only = asbool(settings.get('load_test_only', False))
    docsdir = settings.get('load_docsdir', None)
    if docsdir is not None:
        docsdir = [path.strip() for path in docsdir.strip().split('\n')]
    if workbook_filename:
        load_workbook(app, workbook_filename, docsdir)

    return app
<commit_msg>remove pdb<commit_after>from future.standard_library import install_aliases
install_aliases()  # NOQA
import base64
import codecs
import json
import netaddr
import os
try:
    import subprocess32 as subprocess  # Closes pipes on failure
except ImportError:
    import subprocess
from pyramid.config import Configurator
from pyramid.path import (
    AssetResolver,
    caller_package,
)
from pyramid.authorization import ACLAuthorizationPolicy
from pyramid.session import SignedCookieSessionFactory
from pyramid.settings import (
    aslist,
    asbool,
)
from sqlalchemy import engine_from_config
from webob.cookies import JSONSerializer
from snovault.json_renderer import json_renderer
from snovault.app import (
    STATIC_MAX_AGE,
    session,
    json_from_path,
    configure_dbsession,
    changelogs,
    json_asset
)
from dcicutils.log_utils import set_logging
from dcicutils.beanstalk_utils import whodaman
from encoded.commands.create_mapping_on_deploy import (
    ENV_WEBPROD,
    ENV_WEBPROD2,
    BEANSTALK_PROD_ENVS,
)
import structlog
import logging

# location of environment variables on elasticbeanstalk
BEANSTALK_ENV_PATH = "/opt/python/current/env"

def get_mirror_env(settings):
    """ Gets the mirror_env from whodaman instead of env variable """
    who_is_data, who_i_am = whodaman(), settings.get('env.name', '')
    if who_i_am not in BEANSTALK_PROD_ENVS:  # no mirror if we're not in prod
        return None
    if who_is_data == ENV_WEBPROD:
        return ENV_WEBPROD2
    return ENV_WEBPROD

def static_resources(config):
    from pkg_resources import resource_filename
    import mimetypes
    mimetypes.init()
    mimetypes.init([resource_filename('encoded', 'static/mime.types')])
    config.add_static_view('static', 'static', cache_max_age=STATIC_MAX_AGE)
    config.add_static_view('profiles', 'schemas', cache_max_age=STATIC_MAX_AGE)

    # Favicon
    favicon_path = '/static/img/favicon.ico'
    if config.route_prefix:
        favicon_path = '/%s%s' % (config.route_prefix, favicon_path)
    config.add_route('favicon.ico', 'favicon.ico')

    def favicon(request):
        subreq = request.copy()
        subreq.path_info = favicon_path
        response = request.invoke_subrequest(subreq)
        return response

    config.add_view(favicon, route_name='favicon.ico')

    # Robots.txt
    robots_txt_path = None
    if config.registry.settings.get('testing') in [True, 'true', 'True']:
        robots_txt_path = '/static/dev-robots.txt'
    else:
        robots_txt_path = '/static/robots.txt'

    if config.route_prefix:
        robots_txt_path = '/%s%s' % (config.route_prefix, robots_txt_path)

    config.add_route('robots.txt-conditional', '/robots.txt')

    def robots_txt(request):
        subreq = request.copy()
        subreq.path_info = robots_txt_path
        response = request.invoke_subrequest(subreq)
        return response

    config.add_view(robots_txt, route_name='robots.txt-conditional')

def load_workbook(app, workbook_filename, docsdir):
    from .loadxl import load_all
    from webtest import TestApp
    environ = {
        'HTTP_ACCEPT': 'application/json',
        'REMOTE_USER': 'IMPORT',
    }
    testapp = TestApp(app, environ)
    load_all(testapp, workbook_filename, docsdir)

def source_beanstalk_env_vars(config_file=BEANSTALK_ENV_PATH):
    """
    set environment variables if we are on Elastic Beanstalk
    AWS_ACCESS_KEY_ID is indicative of whether or not env vars are sourced

    Args:
        config_file (str): filepath to load env vars from
    """
    if os.path.exists(config_file) and not os.environ.get("AWS_ACCESS_KEY_ID"):
        command = ['bash', '-c', 'source ' + config_file + ' && env']
        proc = subprocess.Popen(command, stdout=subprocess.PIPE, universal_newlines=True)
        for line in proc.stdout:
            key, _, value = line.partition("=")
            os.environ[key] = value[:-1]
        proc.communicate()

def app_version(config):
    import hashlib
    if not config.registry.settings.get('snovault.app_version'):
        # we update version as part of deployment process `deploy_beanstalk.py`
        # but if we didn't check env then git
        version = os.environ.get("ENCODED_VERSION")
        if not version:
            try:
                version = subprocess.check_output(
                    ['git', '-C', os.path.dirname(__file__), 'describe']).decode('utf-8').strip()
                diff = subprocess.check_output(
                    ['git', '-C', os.path.dirname(__file__), 'diff', '--no-ext-diff'])
                if diff:
                    version += '-patch' + hashlib.sha1(diff).hexdigest()[:7]
            except:
                version = "test"

        config.registry.settings['snovault.app_version'] = version

    # GA Config
    ga_conf_file = config.registry.settings.get('ga_config_location')
    ga_conf_existing = config.registry.settings.get('ga_config')
    if ga_conf_file and not ga_conf_existing:
        ga_conf_file = os.path.normpath(
            os.path.join(
                os.path.dirname(os.path.abspath(__file__)), # Absolute loc. of this file
                "../../",                                   # Go back up to repo dir
                ga_conf_file
            )
        )
        if not os.path.exists(ga_conf_file):
            raise Exception(ga_conf_file + " does not exist in filesystem. Aborting.")
        with open(ga_conf_file) as json_file:
            config.registry.settings["ga_config"] = json.load(json_file)

def main(global_config, **local_config):
    """
    This function returns a Pyramid WSGI application.
    """

    settings = global_config
    settings.update(local_config)

    set_logging(in_prod=settings.get('production'))
    # set_logging(settings.get('elasticsearch.server'), settings.get('production'))

    # source environment variables on elastic beanstalk
    source_beanstalk_env_vars()

    settings['snovault.jsonld.namespaces'] = json_asset('encoded:schemas/namespaces.json')
    settings['snovault.jsonld.terms_namespace'] = 'https://www.encodeproject.org/terms/'
    settings['snovault.jsonld.terms_prefix'] = 'encode'
    # set auth0 keys
    settings['auth0.secret'] = os.environ.get("Auth0Secret")
    settings['auth0.client'] = os.environ.get("Auth0Client")
    # set google reCAPTCHA keys
    settings['g.recaptcha.key'] = os.environ.get('reCaptchaKey')
    settings['g.recaptcha.secret'] = os.environ.get('reCaptchaSecret')
    # set mirrored Elasticsearch location (for webprod/webprod2)
    settings['mirror.env.name'] = get_mirror_env(settings)
    config = Configurator(settings=settings)

    from snovault.elasticsearch import APP_FACTORY
    config.registry[APP_FACTORY] = main  # used by mp_indexer
    config.include(app_version)

    config.include('pyramid_multiauth')  # must be before calling set_authorization_policy
    from pyramid_localroles import LocalRolesAuthorizationPolicy
    # Override default authz policy set by pyramid_multiauth
    config.set_authorization_policy(LocalRolesAuthorizationPolicy())
    config.include(session)

    # must include, as tm.attempts was removed from pyramid_tm
    config.include('pyramid_retry')

    config.include(configure_dbsession)
    config.include('snovault')
    config.commit()  # commit so search can override listing

    # Render an HTML page to browsers and a JSON document for API clients
    #config.include(add_schemas_to_html_responses)
    config.include('.renderers')
    config.include('.authentication')
    config.include('.server_defaults')
    config.include('.root')
    config.include('.types')
    config.include('.batch_download')
    config.include('.loadxl')
    config.include('.visualization')

    if 'elasticsearch.server' in config.registry.settings:
        config.include('snovault.elasticsearch')
        config.include('.search')

    # this contains fall back url, so make sure it comes just before static_resoruces
    config.include('.types.page')
    config.include(static_resources)
    config.include(changelogs)

    # we are loading ontologies now as regular items
    # config.registry['ontology'] = json_from_path(settings.get('ontology_path'), {})
    aws_ip_ranges = json_from_path(settings.get('aws_ip_ranges_path'), {'prefixes': []})
    config.registry['aws_ipset'] = netaddr.IPSet(
        record['ip_prefix'] for record in aws_ip_ranges['prefixes'] if record['service'] == 'AMAZON')

    if asbool(settings.get('testing', False)):
        config.include('.tests.testing_views')

    # Load upgrades last so that all views (including testing views) are
    # registered.
    config.include('.upgrade')

    app = config.make_wsgi_app()

    workbook_filename = settings.get('load_workbook', '')
    load_test_only = asbool(settings.get('load_test_only', False))
    docsdir = settings.get('load_docsdir', None)
    if docsdir is not None:
        docsdir = [path.strip() for path in docsdir.strip().split('\n')]
    if workbook_filename:
        load_workbook(app, workbook_filename, docsdir)

    return app
<|endoftext|>