<commit_before># -*- coding: utf-8 -*-
'''
Manage basic template commands
'''

# Import python libs
import time
import os
import codecs
import logging
from cStringIO import StringIO as cStringIO
from StringIO import StringIO as pyStringIO

# Import salt libs
import salt.utils
from salt._compat import string_types

log = logging.getLogger(__name__)

def string_io(data=None):  # cStringIO can't handle unicode
    '''
    Pass data through to stringIO module and return result
    '''
    try:
        return cStringIO(bytes(data))
    except UnicodeEncodeError:
        return pyStringIO(data)

#FIXME: we should make the default encoding of a .sls file a configurable
#       option in the config, and default it to 'utf-8'.
#
SLS_ENCODING = 'utf-8'  # this one has no BOM.
SLS_ENCODER = codecs.getencoder(SLS_ENCODING)

def compile_template(template,
                     renderers,
                     default,
                     saltenv='base',
                     sls='',
                     **kwargs):
    '''
    Take the path to a template and return the high data structure
    derived from the template.
    '''

    # We "map" env to the same as saltenv until Boron is out in order to follow the same deprecation path
    kwargs.setdefault('env', saltenv)
    salt.utils.warn_until(
        'Boron',
        'We are only supporting \'env\' in the templating context until Boron comes out. '
        'Once this warning is shown, please remove the above mapping',
        _dont_call_warnings=True
    )

    # Template was specified incorrectly
    if not isinstance(template, string_types):
        return {}
    # Template does not exists
    if not os.path.isfile(template):
        return {}
    # Template is an empty file
    if salt.utils.is_empty(template):
        return {}

    # Get the list of render funcs in the render pipe line.
    render_pipe = template_shebang(template, renderers, default)

    with codecs.open(template, encoding=SLS_ENCODING) as ifile:
        # data input to the first render function in the pipe
        input_data = ifile.read()
        if not input_data.strip():
            # Template is nothing but whitespace
            return {}

    input_data = string_io(input_data)
    for render, argline in render_pipe:
        try:
            input_data.seek(0)
        except Exception:
            pass
        render_kwargs = dict(renderers=renderers, tmplpath=template)
        render_kwargs.update(kwargs)
        if argline:
            render_kwargs['argline'] = argline
        ret = render(input_data, saltenv, sls, **render_kwargs)
        if ret is None:
            # The file is empty or is being written elsewhere
            time.sleep(0.01)
            ret = render(input_data, saltenv, sls, **render_kwargs)
        input_data = ret
        if log.isEnabledFor(logging.GARBAGE):  # pylint: disable=no-member
            try:
                log.debug('Rendered data from file: {0}:\n{1}'.format(
                    template,
                    ret.read()))
                ret.seek(0)
            except Exception:
                pass
    return ret

def compile_template_str(template, renderers, default):
    '''
    Take template as a string and return the high data structure
    derived from the template.
    '''
    fn_ = salt.utils.mkstemp()
    with salt.utils.fopen(fn_, 'wb') as ofile:
        ofile.write(SLS_ENCODER(template)[0])
    return compile_template(fn_, renderers, default)

def template_shebang(template, renderers, default):
    '''
    Check the template shebang line and return the list of renderers specified
    in the pipe.

    Example shebang lines::

      #!yaml_jinja
      #!yaml_mako
      #!mako|yaml
      #!jinja|yaml
      #!jinja|mako|yaml
      #!mako|yaml|stateconf
      #!jinja|yaml|stateconf
      #!mako|yaml_odict
      #!mako|yaml_odict|stateconf

    '''
    render_pipe = []

    # Open up the first line of the sls template
    with salt.utils.fopen(template, 'r') as ifile:
        line = ifile.readline()

        # Check if it starts with a shebang and not a path
        if line.startswith('#!') and not line.startswith('#!/'):

            # pull out the shebang data
            render_pipe = check_render_pipe_str(line.strip()[2:], renderers)

    if not render_pipe:
        render_pipe = check_render_pipe_str(default, renderers)

    return render_pipe

# A dict of combined renderer (i.e., rend1_rend2_...) to
# render-pipe (i.e., rend1|rend2|...)
#
OLD_STYLE_RENDERERS = {}

for comb in '''
        yaml_jinja
        yaml_mako
        yaml_wempy
        json_jinja
        json_mako
        json_wempy
        yamlex_jinja
        yamlexyamlex_mako
        yamlexyamlex_wempy
        '''.strip().split():

    fmt, tmpl = comb.split('_')
    OLD_STYLE_RENDERERS[comb] = '{0}|{1}'.format(tmpl, fmt)

def check_render_pipe_str(pipestr, renderers):
    '''
    Check that all renderers specified in the pipe string are available.
    If so, return the list of render functions in the pipe as
    (render_func, arg_str) tuples; otherwise return [].
    '''
    parts = [r.strip() for r in pipestr.split('|')]
    # Note: currently, | is not allowed anywhere in the shebang line except
    #       as pipes between renderers.

    results = []
    try:
        if parts[0] == pipestr and pipestr in OLD_STYLE_RENDERERS:
            parts = OLD_STYLE_RENDERERS[pipestr].split('|')
        for part in parts:
            name, argline = (part + ' ').split(' ', 1)
            results.append((renderers[name], argline.strip()))
        return results
    except KeyError:
        log.error('The renderer "{0}" is not available'.format(pipestr))
        return []
<commit_msg>modernize template.py<commit_after># -*- coding: utf-8 -*-
'''
Manage basic template commands
'''

from __future__ import absolute_import

# Import python libs
import time
import os
import codecs
import logging

# Import salt libs
import salt.utils
from salt._compat import string_types, string_io

log = logging.getLogger(__name__)

#FIXME: we should make the default encoding of a .sls file a configurable
#       option in the config, and default it to 'utf-8'.
#
SLS_ENCODING = 'utf-8'  # this one has no BOM.
SLS_ENCODER = codecs.getencoder(SLS_ENCODING)

def compile_template(template,
                     renderers,
                     default,
                     saltenv='base',
                     sls='',
                     **kwargs):
    '''
    Take the path to a template and return the high data structure
    derived from the template.
    '''

    # We "map" env to the same as saltenv until Boron is out in order to follow the same deprecation path
    kwargs.setdefault('env', saltenv)
    salt.utils.warn_until(
        'Boron',
        'We are only supporting \'env\' in the templating context until Boron comes out. '
        'Once this warning is shown, please remove the above mapping',
        _dont_call_warnings=True
    )

    # Template was specified incorrectly
    if not isinstance(template, string_types):
        return {}
    # Template does not exists
    if not os.path.isfile(template):
        return {}
    # Template is an empty file
    if salt.utils.is_empty(template):
        return {}

    # Get the list of render funcs in the render pipe line.
    render_pipe = template_shebang(template, renderers, default)

    with codecs.open(template, encoding=SLS_ENCODING) as ifile:
        # data input to the first render function in the pipe
        input_data = ifile.read()
        if not input_data.strip():
            # Template is nothing but whitespace
            return {}

    input_data = string_io(input_data)
    for render, argline in render_pipe:
        try:
            input_data.seek(0)
        except Exception:
            pass
        render_kwargs = dict(renderers=renderers, tmplpath=template)
        render_kwargs.update(kwargs)
        if argline:
            render_kwargs['argline'] = argline
        ret = render(input_data, saltenv, sls, **render_kwargs)
        if ret is None:
            # The file is empty or is being written elsewhere
            time.sleep(0.01)
            ret = render(input_data, saltenv, sls, **render_kwargs)
        input_data = ret
        if log.isEnabledFor(logging.GARBAGE):  # pylint: disable=no-member
            try:
                log.debug('Rendered data from file: {0}:\n{1}'.format(
                    template,
                    ret.read()))
                ret.seek(0)
            except Exception:
                pass
    return ret

def compile_template_str(template, renderers, default):
    '''
    Take template as a string and return the high data structure
    derived from the template.
    '''
    fn_ = salt.utils.mkstemp()
    with salt.utils.fopen(fn_, 'wb') as ofile:
        ofile.write(SLS_ENCODER(template)[0])
    return compile_template(fn_, renderers, default)

def template_shebang(template, renderers, default):
    '''
    Check the template shebang line and return the list of renderers specified
    in the pipe.

    Example shebang lines::

      #!yaml_jinja
      #!yaml_mako
      #!mako|yaml
      #!jinja|yaml
      #!jinja|mako|yaml
      #!mako|yaml|stateconf
      #!jinja|yaml|stateconf
      #!mako|yaml_odict
      #!mako|yaml_odict|stateconf

    '''
    render_pipe = []

    # Open up the first line of the sls template
    with salt.utils.fopen(template, 'r') as ifile:
        line = ifile.readline()

        # Check if it starts with a shebang and not a path
        if line.startswith('#!') and not line.startswith('#!/'):

            # pull out the shebang data
            render_pipe = check_render_pipe_str(line.strip()[2:], renderers)

    if not render_pipe:
        render_pipe = check_render_pipe_str(default, renderers)

    return render_pipe

# A dict of combined renderer (i.e., rend1_rend2_...) to
# render-pipe (i.e., rend1|rend2|...)
#
OLD_STYLE_RENDERERS = {}

for comb in '''
        yaml_jinja
        yaml_mako
        yaml_wempy
        json_jinja
        json_mako
        json_wempy
        yamlex_jinja
        yamlexyamlex_mako
        yamlexyamlex_wempy
        '''.strip().split():

    fmt, tmpl = comb.split('_')
    OLD_STYLE_RENDERERS[comb] = '{0}|{1}'.format(tmpl, fmt)

def check_render_pipe_str(pipestr, renderers):
    '''
    Check that all renderers specified in the pipe string are available.
    If so, return the list of render functions in the pipe as
    (render_func, arg_str) tuples; otherwise return [].
    '''
    parts = [r.strip() for r in pipestr.split('|')]
    # Note: currently, | is not allowed anywhere in the shebang line except
    #       as pipes between renderers.

    results = []
    try:
        if parts[0] == pipestr and pipestr in OLD_STYLE_RENDERERS:
            parts = OLD_STYLE_RENDERERS[pipestr].split('|')
        for part in parts:
            name, argline = (part + ' ').split(' ', 1)
            results.append((renderers[name], argline.strip()))
        return results
    except KeyError:
        log.error('The renderer "{0}" is not available'.format(pipestr))
        return []
<|endoftext|>


<commit_before>'''
Primary interfaces for the salt-cloud system
'''
# Need to get data from 4 sources!
# CLI options
# salt cloud config - /etc/salt/cloud
# salt master config (for master integration)
# salt VM config, where VMs are defined - /etc/salt/cloud.profiles
#
# The cli, master and cloud configs will merge for opts
# the VM data will be in opts['vm']

# Import python libs
import optparse
import os

# Import salt libs
import saltcloud.config
import saltcloud.output
import salt.config
import salt.output

# Import saltcloud libs
from saltcloud.utils import parsers
from saltcloud.libcloudfuncs import libcloud_version

class SaltCloud(parsers.SaltCloudParser):
    def run(self):
        '''
        Execute the salt-cloud command line
        '''
        libcloud_version()

        # Parse shell arguments
        self.parse_args()

        # Setup log file logging
        self.setup_logfile_logger()

        # Late imports so logging works as expected
        import logging
        logging.getLogger(__name__).info('salt-cloud starting')
        import saltcloud.cloud
        mapper = saltcloud.cloud.Map(self.config)

        if self.options.update_bootstrap:
            import urllib
            branch = 'develop'
            url = ('https://raw.github.com/saltstack/salt-bootstrap/{0}/bootstrap-salt-minion.sh'.format(branch))
            req = urllib.urlopen(url)
            deploy_path = os.path.join(
                    os.path.dirname(os.path.dirname(__file__)),
                    'saltcloud', 'deploy', 'bootstrap-salt-minion.sh')
            print('Updating bootstrap-salt-minion.sh.'
                  '\n\tSource:      {0}'
                  '\n\tDestination: {1}'.format(url, deploy_path))
            f = open(deploy_path, 'w')
            f.write(req.read())
            f.close()

        if self.selected_query_option is not None:
            if self.options.map:
                try:
                    query_map = mapper.interpolated_map(
                        query=self.selected_query_option
                    )
                except Exception as exc:
                    print('There was an error: {0}'.format(exc))
            else:
                try:
                    query_map = mapper.map_providers(
                        query=self.selected_query_option
                    )
                except Exception as exc:
                    print('There was an error: {0}'.format(exc))
            salt.output.display_output(query_map, '', self.config)

        if self.options.list_locations is not None:
            try:
                saltcloud.output.double_layer(
                    mapper.location_list(self.options.list_locations)
                )
            except Exception as exc:
                print('There was an error: {0}'.format(exc))
            self.exit(0)

        if self.options.list_images is not None:
            try:
                saltcloud.output.double_layer(
                    mapper.image_list(self.options.list_images)
                )
            except Exception as exc:
                print('There was an error: {0}'.format(exc))
            self.exit(0)

        if self.options.list_sizes is not None:
            try:
                saltcloud.output.double_layer(
                    mapper.size_list(self.options.list_sizes)
                )
            except Exception as exc:
                print('There was an error: {0}'.format(exc))
            self.exit(0)

        if self.options.destroy and (self.config.get('names', None) or
                                     self.options.map):
            if self.options.map:
                names = mapper.delete_map(query='list_nodes')
            else:
                names = self.config.get('names', None)

            msg = 'The following virtual machines are set to be destroyed:\n'
            for name in names:
                msg += '  {0}\n'.format(name)

            try:
                if self.print_confirm(msg):
                    mapper.destroy(names)
            except Exception as exc:
                print('There was an error: {0}'.format(exc))
            self.exit(0)

        if self.options.action and (self.config.get('names', None) or
                                     self.options.map):
            if self.options.map:
                names = mapper.delete_map(query='list_nodes')
            else:
                names = self.config.get('names', None)

            kwargs = {}
            machines = []
            msg = 'The following virtual machines are set to be actioned with "{0}":\n'.format(self.options.action)
            for name in names:
                if '=' in name:
                    # This is obviously not a machine name, treat it as a kwarg
                    comps = name.split('=')
                    kwargs[comps[0]] = comps[1]
                else:
                    msg += '  {0}\n'.format(name)
                    machines.append(name)
            names = machines

            try:
                if self.print_confirm(msg):
                    mapper.do_action(names, kwargs)
            except Exception as exc:
                print('There was an error: {0}'.format(exc))
            self.exit(0)

        if self.options.profile and self.config.get('names', False):
            try:
                mapper.run_profile()
            except Exception as exc:
                print('There was an error: {0}'.format(exc))
            self.exit(0)

        if self.options.map and self.selected_query_option is None:
            if len(mapper.map) == 0:
                print('Nothing to do')
                self.exit(0)
            try:
                dmap = self.map_data()

                msg = 'The following virtual machines are set to be created:\n'
                for name in dmap['create']:
                    msg += '  {0}\n'.format(name)
                if 'destroy' in dmap:
                    msg += 'The following virtual machines are set to be destroyed:\n'
                    for name in dmap['destroy']:
                        msg += '  {0}\n'.format(name)

                if self.print_confirm(msg):
                    mapper.run_map(dmap)
            except Exception as exc:
                print('There was an error: {0}'.format(exc))
            self.exit(0)

    def print_confirm(self, msg):
        if self.options.assume_yes:
            return True
        print(msg)
        res = raw_input('Proceed? [N/y]')
        if not res.lower().startswith('y'):
            return False
        print('...proceeding')
        return True
<commit_msg>Making errors clearer<commit_after>'''
Primary interfaces for the salt-cloud system
'''
# Need to get data from 4 sources!
# CLI options
# salt cloud config - /etc/salt/cloud
# salt master config (for master integration)
# salt VM config, where VMs are defined - /etc/salt/cloud.profiles
#
# The cli, master and cloud configs will merge for opts
# the VM data will be in opts['vm']

# Import python libs
import optparse
import os

# Import salt libs
import saltcloud.config
import saltcloud.output
import salt.config
import salt.output

# Import saltcloud libs
from saltcloud.utils import parsers
from saltcloud.libcloudfuncs import libcloud_version

class SaltCloud(parsers.SaltCloudParser):
    def run(self):
        '''
        Execute the salt-cloud command line
        '''
        libcloud_version()

        # Parse shell arguments
        self.parse_args()

        # Setup log file logging
        self.setup_logfile_logger()

        # Late imports so logging works as expected
        import logging
        logging.getLogger(__name__).info('salt-cloud starting')
        import saltcloud.cloud
        mapper = saltcloud.cloud.Map(self.config)

        if self.options.update_bootstrap:
            import urllib
            branch = 'develop'
            url = ('https://raw.github.com/saltstack/salt-bootstrap/{0}/bootstrap-salt-minion.sh'.format(branch))
            req = urllib.urlopen(url)
            deploy_path = os.path.join(
                    os.path.dirname(os.path.dirname(__file__)),
                    'saltcloud', 'deploy', 'bootstrap-salt-minion.sh')
            print('Updating bootstrap-salt-minion.sh.'
                  '\n\tSource:      {0}'
                  '\n\tDestination: {1}'.format(url, deploy_path))
            f = open(deploy_path, 'w')
            f.write(req.read())
            f.close()

        if self.selected_query_option is not None:
            if self.options.map:
                try:
                    query_map = mapper.interpolated_map(
                        query=self.selected_query_option
                    )
                except Exception as exc:
                    print('There was an error with a custom map: {0}'.format(exc))
            else:
                try:
                    query_map = mapper.map_providers(
                        query=self.selected_query_option
                    )
                except Exception as exc:
                    print('There was an error with a map: {0}'.format(exc))
            salt.output.display_output(query_map, '', self.config)

        if self.options.list_locations is not None:
            try:
                saltcloud.output.double_layer(
                    mapper.location_list(self.options.list_locations)
                )
            except Exception as exc:
                print('There was an error listing locations: {0}'.format(exc))
            self.exit(0)

        if self.options.list_images is not None:
            try:
                saltcloud.output.double_layer(
                    mapper.image_list(self.options.list_images)
                )
            except Exception as exc:
                print('There was an error listing images: {0}'.format(exc))
            self.exit(0)

        if self.options.list_sizes is not None:
            try:
                saltcloud.output.double_layer(
                    mapper.size_list(self.options.list_sizes)
                )
            except Exception as exc:
                print('There was an error listing sizes: {0}'.format(exc))
            self.exit(0)

        if self.options.destroy and (self.config.get('names', None) or
                                     self.options.map):
            if self.options.map:
                names = mapper.delete_map(query='list_nodes')
            else:
                names = self.config.get('names', None)

            msg = 'The following virtual machines are set to be destroyed:\n'
            for name in names:
                msg += '  {0}\n'.format(name)

            try:
                if self.print_confirm(msg):
                    mapper.destroy(names)
            except Exception as exc:
                print('There was an error destroy machines: {0}'.format(exc))
            self.exit(0)

        if self.options.action and (self.config.get('names', None) or
                                     self.options.map):
            if self.options.map:
                names = mapper.delete_map(query='list_nodes')
            else:
                names = self.config.get('names', None)

            kwargs = {}
            machines = []
            msg = 'The following virtual machines are set to be actioned with "{0}":\n'.format(self.options.action)
            for name in names:
                if '=' in name:
                    # This is obviously not a machine name, treat it as a kwarg
                    comps = name.split('=')
                    kwargs[comps[0]] = comps[1]
                else:
                    msg += '  {0}\n'.format(name)
                    machines.append(name)
            names = machines

            try:
                if self.print_confirm(msg):
                    mapper.do_action(names, kwargs)
            except Exception as exc:
                print('There was an error actioning machines: {0}'.format(exc))
            self.exit(0)

        if self.options.profile and self.config.get('names', False):
            try:
                mapper.run_profile()
            except Exception as exc:
                print('There was a profile error: {0}'.format(exc))
            self.exit(0)

        if self.options.map and self.selected_query_option is None:
            if len(mapper.map) == 0:
                print('Nothing to do')
                self.exit(0)
            try:
                dmap = self.map_data()

                msg = 'The following virtual machines are set to be created:\n'
                for name in dmap['create']:
                    msg += '  {0}\n'.format(name)
                if 'destroy' in dmap:
                    msg += 'The following virtual machines are set to be destroyed:\n'
                    for name in dmap['destroy']:
                        msg += '  {0}\n'.format(name)

                if self.print_confirm(msg):
                    mapper.run_map(dmap)
            except Exception as exc:
                print('There was a query error: {0}'.format(exc))
            self.exit(0)

    def print_confirm(self, msg):
        if self.options.assume_yes:
            return True
        print(msg)
        res = raw_input('Proceed? [N/y]')
        if not res.lower().startswith('y'):
            return False
        print('...proceeding')
        return True
<|endoftext|>


<commit_before>{
  'settings': {
    'include_dirs': [
      '..',
    ],
    'vs_props': ['build/base.vsprops'],
  },
  'targets': [
    {
      'name': 'base',
      'type': 'static_library',
      'dependencies': [
        '../third_party/icu38/icu38.gyp:icudata',
        '../third_party/icu38/icu38.gyp:icui18n',
        '../third_party/icu38/icu38.gyp:icuuc',
      ],
      'sources': [
        'third_party/dmg_fp/dtoa.cc',
        'third_party/dmg_fp/g_fmt.cc',
        'third_party/nspr/prtime.cc',
        'third_party/nss/sha512.cc',
        'at_exit.cc',
        'base_paths.cc',
        'base_paths_linux.cc',
        'base_paths_mac.mm',
        'base_paths_win.cc',
        'base_switches.cc',
        'bzip2_error_handler.cc',
        'clipboard.cc',
        'clipboard_linux.cc',
        'clipboard_mac.mm',
        'clipboard_win.cc',
        'command_line.cc',
        'condition_variable_posix.cc',
        'condition_variable_win.cc',
        'debug_util.cc',
        'debug_util_posix.cc',
        'debug_util_win.cc',
        'directory_watcher_win.cc',
        'event_recorder_stubs.cc',
        'field_trial.cc',
        'file_path.cc',
        'file_util.cc',
        'file_util_linux.cc',
        'file_util_mac.mm',
        'file_util_posix.cc',
        'file_util_win.cc',
        'file_version_info_linux.cc',
        'file_version_info_mac.mm',
        'histogram.cc',
        'hmac_mac.cc',
        'hmac_win.cc',
        'icu_util.cc',
        'idle_timer.cc',
        'json_reader.cc',
        'json_writer.cc',
        'lazy_instance.cc',
        'lock.cc',
        'lock_impl_posix.cc',
        'lock_impl_win.cc',
        'logging.cc',
        'mac_util.mm',
        'md5.cc',
        'memory_debug.cc',
        'message_loop.cc',
        'message_pump_default.cc',
        'message_pump_mac.mm',
        'message_pump_win.cc',
        'non_thread_safe.cc',
        'path_service.cc',
        'pickle.cc',
        'platform_file_posix.cc',
        'platform_thread_mac.mm',
        'platform_thread_posix.cc',
        'platform_thread_win.cc',
        'process_posix.cc',
        'process_util_linux.cc',
        'process_util_mac.mm',
        'process_util_posix.cc',
        'process_util_win.cc',
        'rand_util.cc',
        'rand_util_posix.cc',
        'rand_util_win.cc',
        'ref_counted.cc',
        'revocable_store.cc',
        'scoped_bstr_win.cc',
        'scoped_clipboard_writer.cc',
        'scoped_nsautorelease_pool.mm',
        'sha2.cc',
        'shared_memory_posix.cc',
        'shared_memory_win.cc',
        'simple_thread.cc',
        'stats_table.cc',
        'string_escape.cc',
        'string_piece.cc',
        'string_util.cc',
        'string_util_icu.cc',
        'sys_info_posix.cc',
        'sys_info_win.cc',
        'sys_string_conversions_linux.cc',
        'sys_string_conversions_mac.mm',
        'sys_string_conversions_win.cc',
        'system_monitor.cc',
        'system_monitor_posix.cc',
        'system_monitor_win.cc',
        'test_file_util_linux.cc',
        'test_file_util_mac.cc',
        'test_file_util_win.cc',
        'thread.cc',
        'thread_collision_warner.cc',
        'thread_local_posix.cc',
        'thread_local_storage_posix.cc',
        'thread_local_storage_win.cc',
        'thread_local_win.cc',
        'time.cc',
        'time_format.cc',
        'time_mac.cc',
        'time_win.cc',
        'timer.cc',
        'trace_event.cc',
        'tracked.cc',
        'tracked_objects.cc',
        'values.cc',
        'waitable_event_posix.cc',
        'waitable_event_watcher_posix.cc',
        'waitable_event_watcher_win.cc',
        'waitable_event_win.cc',
        'watchdog.cc',
        'word_iterator.cc',
        'worker_pool_mac.mm',
      ],
      'conditions': [
        [ 'OS==win', {
            'source_patterns': [ ['exclude', '_(linux|mac|posix)\\.cc$'],
                                 ['exclude', '\\.mm?$' ],
                               ],
          },
        ],
        [ 'OS==linux', {
            'source_patterns': [ ['exclude', '_(mac|win)\\.cc$'],
                                 ['exclude', '\\.mm?$' ],
                               ],
            'dependencies': ['../third_party/libevent/libevent.gyp:libevent'],
          },
        ],
        [ 'OS==mac', {
            'source_patterns': [ ['exclude', '_(linux|win)\\.cc$'] ],
            'libraries': [
              '/System/Library/Frameworks/AppKit.framework',
              '/System/Library/Frameworks/Carbon.framework',
              '/System/Library/Frameworks/CoreFoundation.framework',
              '/System/Library/Frameworks/Foundation.framework',
            ],
            'dependencies': ['../third_party/libevent/libevent.gyp:libevent'],
            'sources': [
              'string16.cc',
              'message_pump_libevent.cc',
            ],
          },
        ],
      ],
    },
    {
      'name': 'base_gfx',
      'type': 'static_library',
      'sources': [
        'gfx/jpeg_codec.cc',
        'gfx/png_decoder.cc',
        'gfx/png_encoder.cc',
        'gfx/point.cc',
        'gfx/rect.cc',
        'gfx/size.cc',
      ],
      'xcode_framework_dirs': [
        '/System/Library/Frameworks/ApplicationServices.framework/Frameworks',
      ],
      'dependencies': [
        'base',
        '../skia/skia.gyp:skia',
        '../third_party/libjpeg/libjpeg.gyp:libjpeg',
        '../third_party/libpng/libpng.gyp:libpng',
      ],
    },
    {
      'name': 'base_unittests',
      'type': 'executable',
      'sources': [
        'gfx/jpeg_codec_unittest.cc',
        'gfx/png_codec_unittest.cc',
        'gfx/rect_unittest.cc',
        'at_exit_unittest.cc',
        'atomicops_unittest.cc',
        'clipboard_unittest.cc',
        'command_line_unittest.cc',
        'condition_variable_unittest.cc',
        'field_trial_unittest.cc',
        'file_path_unittest.cc',
        'file_util_unittest.cc',
        'file_version_info_unittest.cc',
        'histogram_unittest.cc',
        'hmac_unittest.cc',
        'idletimer_unittest.cc',
        'json_reader_unittest.cc',
        'json_writer_unittest.cc',
        'lazy_instance_unittest.cc',
        'linked_ptr_unittest.cc',
        'message_loop_unittest.cc',
        'observer_list_unittest.cc',
        'path_service_unittest.cc',
        'pickle_unittest.cc',
        'process_util_unittest.cc',
        'pr_time_unittest.cc',
        'rand_util_unittest.cc',
        'ref_counted_unittest.cc',
        'run_all_unittests.cc',
        'scoped_ptr_unittest.cc',
        'sha2_unittest.cc',
        'shared_memory_unittest.cc',
        'simple_thread_unittest.cc',
        'singleton_unittest.cc',
        'stack_container_unittest.cc',
        'string_escape_unittest.cc',
        'string_piece_unittest.cc',
        'string_tokenizer_unittest.cc',
        'string_util_unittest.cc',
        'sys_info_unittest.cc',
        'sys_string_conversions_unittest.cc',
        'thread_collision_warner_unittest.cc',
        'thread_local_storage_unittest.cc',
        'thread_local_unittest.cc',
        'thread_unittest.cc',
        'time_unittest.cc',
        'timer_unittest.cc',
        'tracked_objects_unittest.cc',
        'tuple_unittest.cc',
        'values_unittest.cc',
        'waitable_event_unittest.cc',
        'watchdog_unittest.cc',
        'word_iterator_unittest.cc',
        'worker_pool_unittest.cc',
      ],
      'include_dirs': [
        # word_iterator.h (used by word_iterator_unittest.cc) leaks an ICU
        # #include for unicode/uchar.h.  This should probably be cleaned up.
        '../third_party/icu38/public/common',
      ],
      'dependencies': [
        'base',
        'base_gfx',
        '../testing/gtest.gyp:gtest',
      ],
      'conditions': [
        [ 'OS==mac', {
            'sources': [
              'mac_util_unittest.cc',
            ],
          },
        ],
      ],
    },
  ],
}
<commit_msg>Keep all source files in "sources" sections, and in "conditions" sections, use "source_excludes" in preference to "sources".  This approach has the advantage that generators will be able to access lists of excluded files and add them to projects with compilation suppressed.<commit_after>{
  'settings': {
    'include_dirs': [
      '..',
    ],
    'vs_props': ['build/base.vsprops'],
  },
  'targets': [
    {
      'name': 'base',
      'type': 'static_library',
      'dependencies': [
        '../third_party/icu38/icu38.gyp:icudata',
        '../third_party/icu38/icu38.gyp:icui18n',
        '../third_party/icu38/icu38.gyp:icuuc',
      ],
      'sources': [
        'third_party/dmg_fp/dtoa.cc',
        'third_party/dmg_fp/g_fmt.cc',
        'third_party/nspr/prtime.cc',
        'third_party/nss/sha512.cc',
        'at_exit.cc',
        'base_paths.cc',
        'base_paths_linux.cc',
        'base_paths_mac.mm',
        'base_paths_win.cc',
        'base_switches.cc',
        'bzip2_error_handler.cc',
        'clipboard.cc',
        'clipboard_linux.cc',
        'clipboard_mac.mm',
        'clipboard_win.cc',
        'command_line.cc',
        'condition_variable_posix.cc',
        'condition_variable_win.cc',
        'debug_util.cc',
        'debug_util_posix.cc',
        'debug_util_win.cc',
        'directory_watcher_win.cc',
        'event_recorder_stubs.cc',
        'field_trial.cc',
        'file_path.cc',
        'file_util.cc',
        'file_util_linux.cc',
        'file_util_mac.mm',
        'file_util_posix.cc',
        'file_util_win.cc',
        'file_version_info_linux.cc',
        'file_version_info_mac.mm',
        'histogram.cc',
        'hmac_mac.cc',
        'hmac_win.cc',
        'icu_util.cc',
        'idle_timer.cc',
        'json_reader.cc',
        'json_writer.cc',
        'lazy_instance.cc',
        'lock.cc',
        'lock_impl_posix.cc',
        'lock_impl_win.cc',
        'logging.cc',
        'mac_util.mm',
        'md5.cc',
        'memory_debug.cc',
        'message_pump_libevent.cc',
        'message_loop.cc',
        'message_pump_default.cc',
        'message_pump_mac.mm',
        'message_pump_win.cc',
        'non_thread_safe.cc',
        'path_service.cc',
        'pickle.cc',
        'platform_file_posix.cc',
        'platform_thread_mac.mm',
        'platform_thread_posix.cc',
        'platform_thread_win.cc',
        'process_posix.cc',
        'process_util_linux.cc',
        'process_util_mac.mm',
        'process_util_posix.cc',
        'process_util_win.cc',
        'rand_util.cc',
        'rand_util_posix.cc',
        'rand_util_win.cc',
        'ref_counted.cc',
        'revocable_store.cc',
        'scoped_bstr_win.cc',
        'scoped_clipboard_writer.cc',
        'scoped_nsautorelease_pool.mm',
        'sha2.cc',
        'shared_memory_posix.cc',
        'shared_memory_win.cc',
        'simple_thread.cc',
        'stats_table.cc',
        'string16.cc',
        'string_escape.cc',
        'string_piece.cc',
        'string_util.cc',
        'string_util_icu.cc',
        'sys_info_posix.cc',
        'sys_info_win.cc',
        'sys_string_conversions_linux.cc',
        'sys_string_conversions_mac.mm',
        'sys_string_conversions_win.cc',
        'system_monitor.cc',
        'system_monitor_posix.cc',
        'system_monitor_win.cc',
        'test_file_util_linux.cc',
        'test_file_util_mac.cc',
        'test_file_util_win.cc',
        'thread.cc',
        'thread_collision_warner.cc',
        'thread_local_posix.cc',
        'thread_local_storage_posix.cc',
        'thread_local_storage_win.cc',
        'thread_local_win.cc',
        'time.cc',
        'time_format.cc',
        'time_mac.cc',
        'time_win.cc',
        'timer.cc',
        'trace_event.cc',
        'tracked.cc',
        'tracked_objects.cc',
        'values.cc',
        'waitable_event_posix.cc',
        'waitable_event_watcher_posix.cc',
        'waitable_event_watcher_win.cc',
        'waitable_event_win.cc',
        'watchdog.cc',
        'word_iterator.cc',
        'worker_pool_mac.mm',
      ],
      'conditions': [
        [ 'OS==win', {
            'source_patterns': [ ['exclude', '_(linux|mac|posix)\\.cc$'],
                                 ['exclude', '\\.mm?$' ],
                               ],
            'source_excludes': [
              'string16.cc',
            ],
          },
        ],
        [ 'OS==linux', {
            'source_patterns': [ ['exclude', '_(mac|win)\\.cc$'],
                                 ['exclude', '\\.mm?$' ],
                               ],
            'dependencies': ['../third_party/libevent/libevent.gyp:libevent'],
          },
        ],
        [ 'OS==mac', {
            'source_patterns': [ ['exclude', '_(linux|win)\\.cc$'] ],
            'libraries': [
              '/System/Library/Frameworks/AppKit.framework',
              '/System/Library/Frameworks/Carbon.framework',
              '/System/Library/Frameworks/CoreFoundation.framework',
              '/System/Library/Frameworks/Foundation.framework',
            ],
            'dependencies': ['../third_party/libevent/libevent.gyp:libevent'],
          },
        ],
        [ 'OS!=mac', {
            'source_excludes': [
              'message_pump_libevent.cc',
            ],
          },
        ],
      ],
    },
    {
      'name': 'base_gfx',
      'type': 'static_library',
      'sources': [
        'gfx/jpeg_codec.cc',
        'gfx/png_decoder.cc',
        'gfx/png_encoder.cc',
        'gfx/point.cc',
        'gfx/rect.cc',
        'gfx/size.cc',
      ],
      'xcode_framework_dirs': [
        '/System/Library/Frameworks/ApplicationServices.framework/Frameworks',
      ],
      'dependencies': [
        'base',
        '../skia/skia.gyp:skia',
        '../third_party/libjpeg/libjpeg.gyp:libjpeg',
        '../third_party/libpng/libpng.gyp:libpng',
      ],
    },
    {
      'name': 'base_unittests',
      'type': 'executable',
      'sources': [
        'gfx/jpeg_codec_unittest.cc',
        'gfx/png_codec_unittest.cc',
        'gfx/rect_unittest.cc',
        'at_exit_unittest.cc',
        'atomicops_unittest.cc',
        'clipboard_unittest.cc',
        'command_line_unittest.cc',
        'condition_variable_unittest.cc',
        'field_trial_unittest.cc',
        'file_path_unittest.cc',
        'file_util_unittest.cc',
        'file_version_info_unittest.cc',
        'histogram_unittest.cc',
        'hmac_unittest.cc',
        'idletimer_unittest.cc',
        'json_reader_unittest.cc',
        'json_writer_unittest.cc',
        'lazy_instance_unittest.cc',
        'linked_ptr_unittest.cc',
        'mac_util_unittest.cc',
        'message_loop_unittest.cc',
        'observer_list_unittest.cc',
        'path_service_unittest.cc',
        'pickle_unittest.cc',
        'process_util_unittest.cc',
        'pr_time_unittest.cc',
        'rand_util_unittest.cc',
        'ref_counted_unittest.cc',
        'run_all_unittests.cc',
        'scoped_ptr_unittest.cc',
        'sha2_unittest.cc',
        'shared_memory_unittest.cc',
        'simple_thread_unittest.cc',
        'singleton_unittest.cc',
        'stack_container_unittest.cc',
        'string_escape_unittest.cc',
        'string_piece_unittest.cc',
        'string_tokenizer_unittest.cc',
        'string_util_unittest.cc',
        'sys_info_unittest.cc',
        'sys_string_conversions_unittest.cc',
        'thread_collision_warner_unittest.cc',
        'thread_local_storage_unittest.cc',
        'thread_local_unittest.cc',
        'thread_unittest.cc',
        'time_unittest.cc',
        'timer_unittest.cc',
        'tracked_objects_unittest.cc',
        'tuple_unittest.cc',
        'values_unittest.cc',
        'waitable_event_unittest.cc',
        'watchdog_unittest.cc',
        'word_iterator_unittest.cc',
        'worker_pool_unittest.cc',
      ],
      'include_dirs': [
        # word_iterator.h (used by word_iterator_unittest.cc) leaks an ICU
        # #include for unicode/uchar.h.  This should probably be cleaned up.
        '../third_party/icu38/public/common',
      ],
      'dependencies': [
        'base',
        'base_gfx',
        '../testing/gtest.gyp:gtest',
      ],
      'conditions': [
        [ 'OS!=mac', {
            'source_excludes': [
              'mac_util_unittest.cc',
            ],
          },
        ],
      ],
    },
  ],
}
<|endoftext|>


<commit_before>"""Django settings for Sana project.

This file contains the application configuration variablesan is available with
all default values as: ::

    settings.py.tmpl

and should be renamed to settings.py prior to filling in local values. Once
updated, enter the following from the mds installation directory::

    $> ./manage.py syncdb

This will require root privileges.

:Authors: Sana Development Team
:Version: 1.1
"""

import os

DEBUG = True
''' Global debug level. Should be set to False in production environments. '''

TEMPLATE_DEBUG = DEBUG
''' Template debug level. Should be set to False in production environments. '''

ADMINS = (
    # ('Your Name', 'your_email@domain.com'),
)
''' Tuple of admin names and email addresses. '''

MANAGERS = ADMINS

ALLOWED_HOSTS = ['']

DATABASES = {
    'default': {
        # 'ENGINE': os.environ['DATABASE_ENGINE'],
        'ENGINE': 'django.db.backends.sqlite3',
        'NAME': 'dev.db',
        'USER': '',
        'PASSWORD': '',
        'HOST': '',
        'PORt': '',
    }
}

TIME_ZONE = 'America/Chicago'
"""Local time zone for this installation. Choices can be found here:

    http://en.wikipedia.org/wiki/List_of_tz_zones_by_name

although not all choices may be available on all operating systems.
If running in a Windows environment this must be set to the same as your
system time zone.
"""

LANGUAGE_CODE = 'en-us'
"""Language code for this installation. All choices can be found here:

    http://www.i18nguy.com/unicode/language-identifiers.html
"""

SITE_ID = 1
"""Don't touch this unless you know what you are doing."""

USE_I18N = True
"""If you set this to False, Django will make some optimizations so as not to
load the internationalization machinery."""

MEDIA_ROOT = ''
"""Absolute path to the directory that holds media. For a typical Sana
deployment use: "/opt/sana/media/"
"""

MEDIA_URL = ''
"""URL that handles the media served from MEDIA_ROOT. Make sure to use a
trailing slash if there is a path component (optional in other cases). For a
typical Sana deployment use: "/mds/media/". """

ADMIN_MEDIA_PREFIX = '/media/'
"""URL prefix for admin media -- CSS, JavaScript and images. Make sure to use a
trailing slash. Examples: "http://foo.com/media/", "/media/".
"""

SECRET_KEY = 'b#%x46e0f=jx%_#-a9b5(4bvxlfz-obm*gs4iu3i6k!034j(mx'
"""Make this unique, and don't share it with anybody. Seriously."""

TEMPLATE_LOADERS = [('django.template.loaders.cached.Loader', (
            'django.template.loaders.filesystem.Loader',
            'django.template.loaders.app_directories.Loader',
)),]
"""List of callables that know how to import templates from various sources."""

MIDDLEWARE_CLASSES = (
    'django.middleware.common.CommonMiddleware',
    'django.contrib.sessions.middleware.SessionMiddleware',
    'django.contrib.auth.middleware.AuthenticationMiddleware',
    'django.contrib.messages.middleware.MessageMiddleware',
    'sana.mrs.util.LoggingMiddleware',
)
"""Don't touch this unless you know what you are doing."""

ROOT_URLCONF = 'sana.urls'
"""Don't touch this unless you know what you are doing."""

TEMPLATE_DIRS = (
)
"""Put strings here, like "/home/html/django_templates" or
"C:/www/django/templates". Always use forward slashes, even on Windows. Don't
forget to use absolute paths, not relative paths.For a typical Sana
deployment use: "/opt/sana/templates/"."""

INSTALLED_APPS = (
    'sana.mrs',
    'django.contrib.auth',
    'django.contrib.contenttypes',
    'django.contrib.sessions',
    'django.contrib.sites',
    'django.contrib.admin',
    'south',
)
"""Don't touch this unless you know what you are doing."""

### OpenMRS settings. OpenMRS versions seem to have some difference in submitted
### date format.
OPENMRS_SERVER_URL = 'http://localhost/openmrs'
"""Change localhost when deployed."""
OPENMRS_DATE_FMT = '%m/%d/%Y %H:%M'
"""For OpenMRS ver. 1.6. Set to %d/%m/%Y when using OpenMRS 1.7."""
OPENMRS_REPLYTO = ''
"""Reply address for notifications from OpenMRS."""

### Clickatell Settings
CLICKATELL_URI = ''
"""Example 'http://api.clickatell.com/http/sendmsg?%s'"""
CLICKATELL_USER = ''
"""A valid  username."""
CLICKATELL_PASSWORD = ''
"""A valid  password."""
CLICKATELL_API = ''
"""Refer to Clickatell documentation for this value."""

### Kannel Settings
KANNEL_URI = ''
"""URI Example:  'http://127.0.0.1:12121/cgi-bin/sendsms?%s'"""
KANNEL_USER = ''
"""A valid  username."""
KANNEL_PASSWORD = ''

### ZniSMS Settings
ZNISMS_URL = ''
"""URI. Example: http://api.znisms.com/post/smsv3.asp?%s"""
ZNISMS_USER = ''
"""Consult ZniSMS documentation."""
ZNISMS_APIKEY = ''
"""Consult ZniSMS documentation."""
ZNISMS_SENDERID = ''
"""Consult ZniSMS documentation."""

### Email Configuration
EMAIL_HOST = ''
"""Outgoing mail server."""
EMAIL_HOST_USER = ''
"""Password for account used to send."""
EMAIL_HOST_PASSWORD = ''
"""Password for account used to send."""
EMAIL_PORT = ''
"""Check with mail host, i.e. gmail uses 587."""
EMAIL_USE_TLS = True
"""Check with mail host if encryption is supported"""

CONVERT_MEDIA = False
"""Set to True to convert media; i.e. if you are uploading audio or video."""

FLUSH_SAVEDPROCEDURE = False
"""Set to True to flush text data on successful send."""
FLUSH_BINARYRESOURCE = False
"""Set to True to flush file data on successful send."""
FLUSH_NOTIFICATION = False
# Set to True to flush notification data on successful send."""

<commit_msg>upgraded to django 1.5<commit_after>"""Django settings for Sana project.

This file contains the application configuration variablesan is available with
all default values as: ::

    settings.py.tmpl

and should be renamed to settings.py prior to filling in local values. Once
updated, enter the following from the mds installation directory::

    $> ./manage.py syncdb

This will require root privileges.

:Authors: Sana Development Team
:Version: 1.1
"""

import os

DEBUG = True
''' Global debug level. Should be set to False in production environments. '''

TEMPLATE_DEBUG = DEBUG
''' Template debug level. Should be set to False in production environments. '''

ADMINS = (
    # ('Your Name', 'your_email@domain.com'),
)
''' Tuple of admin names and email addresses. '''

MANAGERS = ADMINS

ALLOWED_HOSTS = ['']

DATABASES = {
    'default': {
        'ENGINE': os.environ['DATABASE_ENGINE'],
        # 'ENGINE': 'django.db.backends.sqlite3',
        'NAME': '',
        'USER': '',
        'PASSWORD': '',
        'HOST': '',
        'PORt': '',
    }
}

TIME_ZONE = 'America/Chicago'
"""Local time zone for this installation. Choices can be found here:

    http://en.wikipedia.org/wiki/List_of_tz_zones_by_name

although not all choices may be available on all operating systems.
If running in a Windows environment this must be set to the same as your
system time zone.
"""

LANGUAGE_CODE = 'en-us'
"""Language code for this installation. All choices can be found here:

    http://www.i18nguy.com/unicode/language-identifiers.html
"""

SITE_ID = 1
"""Don't touch this unless you know what you are doing."""

USE_I18N = True
"""If you set this to False, Django will make some optimizations so as not to
load the internationalization machinery."""

MEDIA_ROOT = ''
"""Absolute path to the directory that holds media. For a typical Sana
deployment use: "/opt/sana/media/"
"""

MEDIA_URL = ''
"""URL that handles the media served from MEDIA_ROOT. Make sure to use a
trailing slash if there is a path component (optional in other cases). For a
typical Sana deployment use: "/mds/media/". """

ADMIN_MEDIA_PREFIX = '/media/'
"""URL prefix for admin media -- CSS, JavaScript and images. Make sure to use a
trailing slash. Examples: "http://foo.com/media/", "/media/".
"""

SECRET_KEY = 'b#%x46e0f=jx%_#-a9b5(4bvxlfz-obm*gs4iu3i6k!034j(mx'
"""Make this unique, and don't share it with anybody. Seriously."""

TEMPLATE_LOADERS = [('django.template.loaders.cached.Loader', (
            'django.template.loaders.filesystem.Loader',
            'django.template.loaders.app_directories.Loader',
)),]
"""List of callables that know how to import templates from various sources."""

MIDDLEWARE_CLASSES = (
    'django.middleware.common.CommonMiddleware',
    'django.contrib.sessions.middleware.SessionMiddleware',
    'django.contrib.auth.middleware.AuthenticationMiddleware',
    'django.contrib.messages.middleware.MessageMiddleware',
    'sana.mrs.util.LoggingMiddleware',
)
"""Don't touch this unless you know what you are doing."""

ROOT_URLCONF = 'sana.urls'
"""Don't touch this unless you know what you are doing."""

TEMPLATE_DIRS = (
)
"""Put strings here, like "/home/html/django_templates" or
"C:/www/django/templates". Always use forward slashes, even on Windows. Don't
forget to use absolute paths, not relative paths.For a typical Sana
deployment use: "/opt/sana/templates/"."""

INSTALLED_APPS = (
    'sana.mrs',
    'django.contrib.auth',
    'django.contrib.contenttypes',
    'django.contrib.sessions',
    'django.contrib.sites',
    'django.contrib.admin',
    'south',
)
"""Don't touch this unless you know what you are doing."""

### OpenMRS settings. OpenMRS versions seem to have some difference in submitted
### date format.
OPENMRS_SERVER_URL = 'http://localhost/openmrs'
"""Change localhost when deployed."""
OPENMRS_DATE_FMT = '%m/%d/%Y %H:%M'
"""For OpenMRS ver. 1.6. Set to %d/%m/%Y when using OpenMRS 1.7."""
OPENMRS_REPLYTO = ''
"""Reply address for notifications from OpenMRS."""

### Clickatell Settings
CLICKATELL_URI = ''
"""Example 'http://api.clickatell.com/http/sendmsg?%s'"""
CLICKATELL_USER = ''
"""A valid  username."""
CLICKATELL_PASSWORD = ''
"""A valid  password."""
CLICKATELL_API = ''
"""Refer to Clickatell documentation for this value."""

### Kannel Settings
KANNEL_URI = ''
"""URI Example:  'http://127.0.0.1:12121/cgi-bin/sendsms?%s'"""
KANNEL_USER = ''
"""A valid  username."""
KANNEL_PASSWORD = ''

### ZniSMS Settings
ZNISMS_URL = ''
"""URI. Example: http://api.znisms.com/post/smsv3.asp?%s"""
ZNISMS_USER = ''
"""Consult ZniSMS documentation."""
ZNISMS_APIKEY = ''
"""Consult ZniSMS documentation."""
ZNISMS_SENDERID = ''
"""Consult ZniSMS documentation."""

### Email Configuration
EMAIL_HOST = ''
"""Outgoing mail server."""
EMAIL_HOST_USER = ''
"""Password for account used to send."""
EMAIL_HOST_PASSWORD = ''
"""Password for account used to send."""
EMAIL_PORT = ''
"""Check with mail host, i.e. gmail uses 587."""
EMAIL_USE_TLS = True
"""Check with mail host if encryption is supported"""

CONVERT_MEDIA = False
"""Set to True to convert media; i.e. if you are uploading audio or video."""

FLUSH_SAVEDPROCEDURE = False
"""Set to True to flush text data on successful send."""
FLUSH_BINARYRESOURCE = False
"""Set to True to flush file data on successful send."""
FLUSH_NOTIFICATION = False
# Set to True to flush notification data on successful send."""

<|endoftext|>


<commit_before>

def banner():
	print "[***]	sandbox detection p118	[***]"

class LASTINPUTINFO():

def get_last_input():

def get_key_presses():

def detect_sandbox():

<commit_msg>Work in Progress.  Continue on pg 119<commit_after>

def banner():
	print "[***]	sandbox detection p118	[***]"

class LASTINPUTINFO(ctypes.Structure):
	_fields_ = [("cbsize",ctypes.c_uint),("dwTime",ctypes.c_ulong)]

def get_last_input():
	struct_lastinputinfo = LASTINPUTINFO()
	struct_lastinputinfo.cbSize = ctypes.sizeof(LASTINPUTINFO)
	user32.GetLastInputInfo(ctypes.byref(struct_lastinputinfo))
	run_time = kernel32.GetTickCount()
	elapsed = run_time = sturct_lastinputinfo.dwTime

def get_key_presses():

def detect_sandbox():
<|endoftext|>


<commit_before>__version__ = '2.2.0'
<commit_msg>bump version for pypi<commit_after>__version__ = '2.2.1'
<|endoftext|>


<commit_before>#!/usr/bin/env python3
# -*- coding: utf-8 -*-

import sys
import os
import subprocess
import numpy
import theano
import theanolm
from filetypes import TextFileType

def add_arguments(parser):
    argument_group = parser.add_argument_group("files")
    argument_group.add_argument(
        'model_path', metavar='MODEL', type=str,
        help='path where the best model state will be saved in numpy .npz '
             'format')
    argument_group.add_argument(
        'input_file', metavar='INPUT', type=TextFileType('r'),
        help='text or .gz file containing text to be scored (one sentence per '
             'line)')
    argument_group.add_argument(
        'dictionary_file', metavar='DICTIONARY', type=TextFileType('r'),
        help='text or .gz file containing word list or class definitions')
    argument_group.add_argument(
        '--dictionary-format', metavar='FORMAT', type=str, default='words',
        help='dictionary format, one of "words" (one word per line, default), '
             '"classes" (word and class ID per line), "srilm-classes" (class '
             'name, membership probability, and word per line)')
    argument_group.add_argument(
        '--output-file', metavar='OUTPUT', type=TextFileType('w'), default='-',
        help='where to write the score or rescored n-best list (default '
             'stdout)')

    argument_group = parser.add_argument_group("scoring")
    argument_group.add_argument(
        '--output', metavar='DETAIL', type=str, default='text',
        help='what to output, one of "perplexity", "utterance-scores", '
             '"word-scores" (default "perplexity")')
    argument_group.add_argument(
        '--log-base', metavar='B', type=int, default=None,
        help='convert output log probabilities to base B (default is the '
             'natural logarithm)')

def score(args):
    print("Reading model state from %s." % args.model_path)
    state = numpy.load(args.model_path)

    print("Reading dictionary.")
    sys.stdout.flush()
    dictionary = theanolm.Dictionary(args.dictionary_file, args.dictionary_format)
    print("Number of words in vocabulary:", dictionary.num_words())
    print("Number of word classes:", dictionary.num_classes())

    print("Building neural network.")
    sys.stdout.flush()
    architecture = theanolm.Network.Architecture.from_state(state)
    print(architecture)
    network = theanolm.Network(dictionary, architecture)
    print("Restoring neural network state.")
    network.set_state(state)

    print("Building text scorer.")
    sys.stdout.flush()
    scorer = theanolm.TextScorer(network)

    print("Scoring text.")
    if args.output == 'perplexity':
        _score_text(args.input_file, dictionary, scorer, args.output_file,
                    args.log_base, False)
    elif args.output == 'word-scores':
        _score_text(args.input_file, dictionary, scorer, args.output_file,
                    args.log_base, True)
    elif args.output == 'utterance-scores':
        _score_utterances(args.input_file, dictionary, scorer, args.output_file,
                          args.log_base)

def _score_text(input_file, dictionary, scorer, output_file,
                log_base=None, word_level=False):
    """Reads text from ``input_file``, computes perplexity using
    ``scorer``, and writes to ``output_file``.

    :type input_file: file object
    :param input_file: a file that contains the input sentences in SRILM n-best
                       format

    :type dictionary: Dictionary
    :param dictionary: dictionary that provides mapping between words and word
                       IDs

    :type scorer: TextScorer
    :param scorer: a text scorer for rescoring the input sentences

    :type output_file: file object
    :param output_file: a file where to write the output n-best list in SRILM
                        format

    :type log_base: int
    :param log_base: if set to other than None, convert log probabilities to
                     this base

    :type word_level: bool
    :param word_level: if set to True, also writes word-level statistics
    """

    validation_iter = theanolm.BatchIterator(input_file, dictionary)
    base_conversion = 1 if log_base is None else numpy.log(log_base)

    total_logprob = 0
    num_sentences = 0
    num_words = 0
    num_probs = 0
    for word_ids, membership_probs, mask in validation_iter:
        logprobs = scorer.score_batch(word_ids, membership_probs, mask)
        for seq_index, seq_logprobs in enumerate(logprobs):
            seq_logprob = sum(seq_logprobs)
            num_probs += len(seq_logprobs)
            total_logprob += seq_logprob
            seq_word_ids = word_ids[:, seq_index]
            num_words += len(seq_word_ids)
            num_sentences += 1
            if not word_level:
                continue

            seq_logprobs = [x / base_conversion for x in seq_logprobs]
            seq_logprob /= base_conversion
            seq_class_names = dictionary.ids_to_names(seq_word_ids)
            output_file.write("# Sentence {0}\n".format(num_sentences))
            for word_index, word_logprob in enumerate(seq_logprobs):
                if word_index - 2 > 0:
                    history = seq_class_names[word_index:word_index - 3:-1]
                    history.append('...')
                else:
                    history = seq_class_names[word_index::-1]
                output_file.write("log(p({0} | {1})) = {2}\n".format(
                    seq_class_names[word_index + 1],
                    ', '.join(history),
                    seq_logprobs[word_index]))
            output_file.write("Sentence perplexity: {0}\n\n".format(
                numpy.exp(-seq_logprob / len(seq_logprobs))))

    output_file.write("Number of sentences: {0}\n".format(num_sentences))
    output_file.write("Number of words: {0}\n".format(num_words))
    output_file.write("Number of predicted probabilities: {0}\n".format(num_probs))
    if num_words > 0:
        cross_entropy = -total_logprob / num_probs
        perplexity = numpy.exp(cross_entropy)
        output_file.write("Cross entropy (base e): {0}\n".format(cross_entropy))
        if not log_base is None:
            cross_entropy /= base_conversion
            output_file.write("Cross entropy (base {1}): {0}\n".format(
                cross_entropy, log_base))
        output_file.write("Perplexity: {0}\n".format(perplexity))

def _score_utterances(input_file, dictionary, scorer, output_file,
                      log_base=None):
    """Reads utterances from ``input_file``, computes LM scores using
    ``scorer``, and writes one score per line to ``output_file``.

    :type input_file: file object
    :param input_file: a file that contains the input sentences in SRILM n-best
                       format

    :type dictionary: Dictionary
    :param dictionary: dictionary that provides mapping between words and word
                       IDs

    :type scorer: TextScorer
    :param scorer: a text scorer for rescoring the input sentences

    :type output_file: file object
    :param output_file: a file where to write the output n-best list in SRILM
                        format

    :type log_base: int
    :param log_base: if set to other than None, convert log probabilities to
                     this base
    """

    base_conversion = 1 if log_base is None else numpy.log(log_base)

    num_words = 0
    num_unks = 0
    for line_num, line in enumerate(input_file):
        words = line.split()
        if words[0] != '<s>':
            words.insert(0, '<s>')
        if words[-1] != '</s>':
            words.append('</s>')

        word_ids = dictionary.words_to_ids(words)
        num_words += len(word_ids)
        num_unks += word_ids.count(dictionary.unk_id)

        probs = dictionary.words_to_probs(words)

        lm_score = scorer.score_sequence(word_ids, probs)
        lm_score /= base_conversion
        output_file.write(str(lm_score) + '\n')

        if (line_num + 1) % 1000 == 0:
            print("%d sentences rescored.".format(line_num + 1))
        sys.stdout.flush()

    if num_words == 0:
        print("The input file contains no words.")
    else:
        print("%d words processed, including start-of-sentence and "
              "end-of-sentence tags, and %d (%.1f %%) out-of-vocabulary "
              "words".format(num_words, num_unks, num_unks / num_words))
<commit_msg>Fixed number formatting.<commit_after>#!/usr/bin/env python3
# -*- coding: utf-8 -*-

import sys
import os
import subprocess
import numpy
import theano
import theanolm
from filetypes import TextFileType

def add_arguments(parser):
    argument_group = parser.add_argument_group("files")
    argument_group.add_argument(
        'model_path', metavar='MODEL', type=str,
        help='path where the best model state will be saved in numpy .npz '
             'format')
    argument_group.add_argument(
        'input_file', metavar='INPUT', type=TextFileType('r'),
        help='text or .gz file containing text to be scored (one sentence per '
             'line)')
    argument_group.add_argument(
        'dictionary_file', metavar='DICTIONARY', type=TextFileType('r'),
        help='text or .gz file containing word list or class definitions')
    argument_group.add_argument(
        '--dictionary-format', metavar='FORMAT', type=str, default='words',
        help='dictionary format, one of "words" (one word per line, default), '
             '"classes" (word and class ID per line), "srilm-classes" (class '
             'name, membership probability, and word per line)')
    argument_group.add_argument(
        '--output-file', metavar='OUTPUT', type=TextFileType('w'), default='-',
        help='where to write the score or rescored n-best list (default '
             'stdout)')

    argument_group = parser.add_argument_group("scoring")
    argument_group.add_argument(
        '--output', metavar='DETAIL', type=str, default='text',
        help='what to output, one of "perplexity", "utterance-scores", '
             '"word-scores" (default "perplexity")')
    argument_group.add_argument(
        '--log-base', metavar='B', type=int, default=None,
        help='convert output log probabilities to base B (default is the '
             'natural logarithm)')

def score(args):
    print("Reading model state from %s." % args.model_path)
    state = numpy.load(args.model_path)

    print("Reading dictionary.")
    sys.stdout.flush()
    dictionary = theanolm.Dictionary(args.dictionary_file, args.dictionary_format)
    print("Number of words in vocabulary:", dictionary.num_words())
    print("Number of word classes:", dictionary.num_classes())

    print("Building neural network.")
    sys.stdout.flush()
    architecture = theanolm.Network.Architecture.from_state(state)
    print(architecture)
    network = theanolm.Network(dictionary, architecture)
    print("Restoring neural network state.")
    network.set_state(state)

    print("Building text scorer.")
    sys.stdout.flush()
    scorer = theanolm.TextScorer(network)

    print("Scoring text.")
    if args.output == 'perplexity':
        _score_text(args.input_file, dictionary, scorer, args.output_file,
                    args.log_base, False)
    elif args.output == 'word-scores':
        _score_text(args.input_file, dictionary, scorer, args.output_file,
                    args.log_base, True)
    elif args.output == 'utterance-scores':
        _score_utterances(args.input_file, dictionary, scorer, args.output_file,
                          args.log_base)

def _score_text(input_file, dictionary, scorer, output_file,
                log_base=None, word_level=False):
    """Reads text from ``input_file``, computes perplexity using
    ``scorer``, and writes to ``output_file``.

    :type input_file: file object
    :param input_file: a file that contains the input sentences in SRILM n-best
                       format

    :type dictionary: Dictionary
    :param dictionary: dictionary that provides mapping between words and word
                       IDs

    :type scorer: TextScorer
    :param scorer: a text scorer for rescoring the input sentences

    :type output_file: file object
    :param output_file: a file where to write the output n-best list in SRILM
                        format

    :type log_base: int
    :param log_base: if set to other than None, convert log probabilities to
                     this base

    :type word_level: bool
    :param word_level: if set to True, also writes word-level statistics
    """

    validation_iter = theanolm.BatchIterator(input_file, dictionary)
    base_conversion = 1 if log_base is None else numpy.log(log_base)

    total_logprob = 0
    num_sentences = 0
    num_words = 0
    num_probs = 0
    for word_ids, membership_probs, mask in validation_iter:
        logprobs = scorer.score_batch(word_ids, membership_probs, mask)
        for seq_index, seq_logprobs in enumerate(logprobs):
            seq_logprob = sum(seq_logprobs)
            num_probs += len(seq_logprobs)
            total_logprob += seq_logprob
            seq_word_ids = word_ids[:, seq_index]
            num_words += len(seq_word_ids)
            num_sentences += 1
            if not word_level:
                continue

            seq_logprobs = [x / base_conversion for x in seq_logprobs]
            seq_logprob /= base_conversion
            seq_class_names = dictionary.ids_to_names(seq_word_ids)
            output_file.write("# Sentence {0}\n".format(num_sentences))
            for word_index, word_logprob in enumerate(seq_logprobs):
                if word_index - 2 > 0:
                    history = seq_class_names[word_index:word_index - 3:-1]
                    history.append('...')
                else:
                    history = seq_class_names[word_index::-1]
                output_file.write("log(p({0} | {1})) = {2}\n".format(
                    seq_class_names[word_index + 1],
                    ', '.join(history),
                    seq_logprobs[word_index]))
            output_file.write("Sentence perplexity: {0}\n\n".format(
                numpy.exp(-seq_logprob / len(seq_logprobs))))

    output_file.write("Number of sentences: {0}\n".format(num_sentences))
    output_file.write("Number of words: {0}\n".format(num_words))
    output_file.write("Number of predicted probabilities: {0}\n".format(num_probs))
    if num_words > 0:
        cross_entropy = -total_logprob / num_probs
        perplexity = numpy.exp(cross_entropy)
        output_file.write("Cross entropy (base e): {0}\n".format(cross_entropy))
        if not log_base is None:
            cross_entropy /= base_conversion
            output_file.write("Cross entropy (base {1}): {0}\n".format(
                cross_entropy, log_base))
        output_file.write("Perplexity: {0}\n".format(perplexity))

def _score_utterances(input_file, dictionary, scorer, output_file,
                      log_base=None):
    """Reads utterances from ``input_file``, computes LM scores using
    ``scorer``, and writes one score per line to ``output_file``.

    :type input_file: file object
    :param input_file: a file that contains the input sentences in SRILM n-best
                       format

    :type dictionary: Dictionary
    :param dictionary: dictionary that provides mapping between words and word
                       IDs

    :type scorer: TextScorer
    :param scorer: a text scorer for rescoring the input sentences

    :type output_file: file object
    :param output_file: a file where to write the output n-best list in SRILM
                        format

    :type log_base: int
    :param log_base: if set to other than None, convert log probabilities to
                     this base
    """

    base_conversion = 1 if log_base is None else numpy.log(log_base)

    num_words = 0
    num_unks = 0
    for line_num, line in enumerate(input_file):
        words = line.split()
        if words[0] != '<s>':
            words.insert(0, '<s>')
        if words[-1] != '</s>':
            words.append('</s>')

        word_ids = dictionary.words_to_ids(words)
        num_words += len(word_ids)
        num_unks += word_ids.count(dictionary.unk_id)

        probs = dictionary.words_to_probs(words)

        lm_score = scorer.score_sequence(word_ids, probs)
        lm_score /= base_conversion
        output_file.write(str(lm_score) + '\n')

        if (line_num + 1) % 1000 == 0:
            print("%d sentences rescored.".format(line_num + 1))
        sys.stdout.flush()

    if num_words == 0:
        print("The input file contains no words.")
    else:
        print("{0} words processed, including start-of-sentence and "
              "end-of-sentence tags, and {1} ({2:.1f} %) out-of-vocabulary "
              "words".format(num_words, num_unks, num_unks / num_words))
<|endoftext|>


<commit_before>import asyncio
import websockets
import json

class WebsocketClient(object):

    URL = 'ws://192.168.0.7:5000/'

    def __init__(self, queue):
        self.queue = queue

    @asyncio.coroutine
    def run(self):
        self.websocket = yield from websockets.connect(self.URL)

        while True:
            message = yield from self.websocket.recv()
            if message is None:
                break
            message = json.loads(message)
            self.queue.put_nowait(message)

        yield from self.websocket.close()
<commit_msg>reconnect if connection lost<commit_after>import asyncio
import websockets
import json

class WebsocketClient(object):

    RECONNECT_WAIT = 3

    URL = 'ws://192.168.0.7:5000/'

    def __init__(self, queue):
        self.queue = queue

    @asyncio.coroutine
    def run(self):
        while True:
            try:
                print('Attempting to connect')
                self.websocket = yield from websockets.connect(self.URL)
                print('Connected')
                while True:
                    message = yield from self.websocket.recv()
                    if message is None:
                        break
                    message = json.loads(message)
                    self.queue.put_nowait(message)

                yield from self.websocket.close()
            except Exception as e:
                print('Caught %s' % e)
            print('Lost connection')
            yield from asyncio.sleep(self.RECONNECT_WAIT)
<|endoftext|>


<commit_before>import re
import json
import urlparse

from .exceptions import Http404

re_upload = re.compile(r'^/(?P<repo>[^/]+)$')

def parse_repo(env):
    m = re_upload.match(env['PATH_INFO'])

    if m is None:
        raise Http404()

    return m.group('repo')

def json_response(start_response, val, http_header='200 OK'):
    start_response(http_header, [('Content-Type', 'application/json')])

    return [json.dumps(val) + '\n']

def parse_querystring(env):
    result = {}

    for k, v in urlparse.parse_qsl(env.get('QUERY_STRING', '')):
        # Always use the last value
        result[k] = v

    return result
<commit_msg>Prevent curl --uploadfile blah.deb creating a *blah.deb* repo!<commit_after>import re
import json
import urlparse

from .exceptions import Http404

re_upload = re.compile(r'^/(?P<repo>\w+)$')

def parse_repo(env):
    m = re_upload.match(env['PATH_INFO'])

    if m is None:
        raise Http404()

    return m.group('repo')

def json_response(start_response, val, http_header='200 OK'):
    start_response(http_header, [('Content-Type', 'application/json')])

    return [json.dumps(val) + '\n']

def parse_querystring(env):
    result = {}

    for k, v in urlparse.parse_qsl(env.get('QUERY_STRING', '')):
        # Always use the last value
        result[k] = v

    return result
<|endoftext|>


<commit_before>import sys
import shutil
import os
import logging
import pickle
import subprocess
import re
import time
import json
import ConfigParser

from bottle import Bottle, run, urljoin, HTTPResponse, request

AGENT_CONFIG_SERVER = None
GATEWAY_IP          = None
GATEWAY_PORT        = None
BASE_PATH           = None

# agent pickle file path
pickle_path = '.bidders'
json_path   = '.config'

# agent base path
exec_base_path   = None
config_base_path = None
log_base_path    = None
bidders_config_base_path = None

# set up logging
logging.basicConfig(filename='agent_gateway.log',
        format='%(asctime)-15s %(levelname)s %(message)s',
        level=logging.DEBUG)
logger = logging.getLogger('agent_gateway')

# app bottle
app = Bottle()

# initialize bidder map
bidders = {}

def map_and_redirect(uri, name):
    """
        maps the name, sets the uri and raises a redirection
        otherwise returns the json result code
    """
    try :
        # try to map the name to the internal config name
        location = urljoin(
            AGENT_CONFIG_SERVER,
            uri % bidders[name]['agent_conf_name'])
    except :
        return  {
                'resultCode'        :    1,
                'resultDescription' :   'unable to map %s' % name
                }
        raise HTTPResponse(body=json.dumps(result), status=404,
                Content_Type='application/json')
    raise HTTPResponse("", status=302, Location=location)

@app.get('/v1/agents')
def get_agents():
    """
        returns the list of bidders
    """
    return HTTPResponse(json.dumps(bidders.keys()), status=200, Content_Type='application/json; charset=UTF8')

@app.get('/v1/agents/full')
def get_agents_full():
    """
        returns the list of bidders with full configs
    """
    return HTTPResponse(json.dumps(bidders), status=200, Content_Type='application/json; charset=UTF8')

@app.post('/v1/agents/<name>/config')
@app.get('/v1/agents/<name>/config')
def get_config(name):
    """
        redirects the call to the agent configuration service
        on /v1/agents/<name>/config for the given name
    """
    return map_and_redirect('/v1/agents/%s/config', name)

@app.post('/v1/agents/<name>/heartbeat')
def heartbeat(name):
    """
        redirects the call to the agent configuration service
        on /v1/agents/<name>/heartbeat for the given name
    """
    return map_and_redirect('/v1/agents/%s/heartbeat', name)

@app.get('/v1/agents/all')
def get_all():
    """
        redirects the call to the agent configuration service
        on /v1/agents/all
    """
    location = urljoin(AGENT_CONFIG_SERVER, '/v1/agents/all')
    raise HTTPResponse("", status=302, Location=location)

@app.post('/v1/agents/<name>/start')
def start_bidder(name):
    """
        Starts up a bidder using as the instance parameters
        the arguments passed in the query string
    """
    global _process_id
    result = {
            'resultCode'        :   0,
            'resultDescription' :   'ok'
    }

    if name in bidders :
        result['resultCode'] = 1
        result['resultDescription'] = 'agent already started'
        raise HTTPResponse(body=json.dumps(result), status=500,
                Content_Type='application/json')
    else :
        bidder = {}

    # save the executable name and external name
    bidder['bidder_name'] = name
    bidder['executable'] = request.query['executable']
    # save the params
    escape = lambda x : '"%s"' % x
    bidder['params'] = {
         k:escape(v) for k,v in request.query.iteritems()
            if k not in ('executable', )
    }

    # create a file with the json configuration
    conf_file_name = os.path.join(
        bidders_config_base_path, '%s.conf.json' % name)
    try :
        conf_file = open(conf_file_name, 'w')
        conf_file.write(json.dumps(request.json))
        conf_file.close()
    except :
        result['resultCode'] = 6
        result['resultDescription'] = 'unable to create config file'
        raise HTTPResponse(body=json.dumps(result), status=500,
                Content_Type='application/json')

    logger.info('bringing up bidder %s=%s' % (name, bidder))
    # set the args a list (popen expects them that way)
    arguments = []
    for k,v in bidder['params'].iteritems() :
        arguments.append('-%s' % k)
        arguments.append(v)

    exe = ['nohup']
    exe.append('./%s' % bidder['executable'])
    exe.extend(arguments)
    exe.append('-f')
    exe.append(conf_file_name)
    exe.append('&')
    logger.info('executing : %s' % ' '.join(exe))

    # check the log file
    log_file_name = 'agent_%s_%s.log' % (name, time.strftime('%d.%m.%Y_%H.%M.%S'))
    log_path = os.path.join(log_base_path, log_file_name)
    #try to unlink and then relink
    try :
        os.unlink(
            os.path.join(log_base_path, 'agent_%s.log' % name))
    except :
        pass
    os.symlink(log_file_name,
            os.path.join(log_base_path, 'agent_%s.log' % name))

    log_file = open(log_path, 'w')
    # bring the process up
    try :
        proc = subprocess.Popen(
            ' '.join(exe),
            cwd=exec_base_path,
            shell=True,
            close_fds=True,
            stdout=log_file)
    except :
        result['resultCode'] = 3
        result['resultDescription'] = 'error executing agent'
        raise HTTPResponse(body=json.dumps(result), status=500,
                Content_Type='application/json')

    # read the pid, the one that proc returns belongs to the shell
    pid = None
    # give it some time to make sure the pid is there
    time.sleep(1)
    with open(log_path, 'r') as f:
        for line in f:
            m = re.match('pid:(?P<pid>\d+)', line)
            if m is not None:
                pid = int(m.group('pid'))
                break
    f.close()
    if pid is None :
        # something is not ok
        logger.error('unable to find pid, are you printing it?')
        result['resultCode'] = 4
        result['resultDescription'] = 'unable to find pid, are you printing it?'
        raise HTTPResponse(body=json.dumps(result), status=500,
                Content_Type='application/json')
    # check if the pid still exists, sometimes a bidder starts and aborts
    # right away.
    if not os.path.exists('/proc/%d' % pid):
        # something is not ok
        logger.error('did the process aborted?')
        result['resultCode'] = 5
        result['resultDescription'] = 'process id %d lost' % pid
        raise HTTPResponse(body=json.dumps(result), status=500,
                Content_Type='application/json')

    # save the pid for the new bidder
    bidder['pid']  = pid
    logger.info('pid is : %d' % int(pid))
    # the key stored by the agent configuration service
    # is a concatenation of the bidder name passed and the
    # pid for for process
    bidder['agent_conf_name'] = \
        '%s_%s' % (name, bidder['pid'])
    logger.info('bidder %s got pid %d' % (name, bidder['pid']))

    # save it
    bidders[name] = bidder
    # great, let's pickle the data
    try :
        f = open(os.path.join(pickle_path, str(bidders[name]['pid'])), 'wb')
        pickle.dump(bidders[name], f)
        f.close()
    except :
        result['resultCode'] = 2
        result['resultDescription'] = 'unable to pickle configuration'
        raise HTTPResponse(body=json.dumps(result), status=500,
                Content_Type='application/json')

    result['pid'] = int(pid)
    return result

@app.post('/v1/agents/<name>/stop')
def stop_bidder(name):
    """
        Stops a running agent
    """
    result = {
            'resultCode'        :   0,
            'resultDescription' :   'ok'
    }

    if name not in bidders :
        result['resultCode'] = 1
        result['resultDescription'] = 'bidder not running'
        raise HTTPResponse(body=json.dumps(result), status=404,
                Content_Type='application/json')

    logger.info('stopping bidder %s=%s' % (name, bidders[name]))

    pid = bidders[name]['pid']
    try :
        signal = 9
        if 'signal' in request.query :
            signal = int(request.query['signal'])
        os.kill(pid, signal)
        logger.info('signal %d sent to process with pid %d' % (signal, pid))
    except :
        result['resultCode'] = 2
        result['resultDescription'] = 'unable to kill process %s' % pid
        raise HTTPResponse(body=json.dumps(result), status=500,
                Content_Type='application/json')

    logger.info('agent %s with pid %d stopped' % (name, pid))

    # clean up
    del bidders[name]
    try :
        os.remove(os.path.join(pickle_path, str(pid)))
    except :
        result = {
            'resultCode'        :   3,
            'resultDescription' :   'unable to delete pickled data'
        }
        raise HTTPResponse(body=json.dumps(result), status=500,
                Content_Type='application/json')
    return result

@app.get('/v1/agents/<name>/status')
def get_status(name):
    """
        retrieves the status of a given agent
    """
    result = {
            'resultCode'        :   1,
            'resultDescription' :   'down'
    }

    if not name in bidders:
        raise HTTPResponse(body=json.dumps(result), status=404,
                Content_Type='application/json')

    pid = bidders[name]['pid']
    if not os.path.exists('/proc/%d' % pid):
        # something is not ok, the bidder aborted
        logger.error('did the process %d aborted?' % pid)
        # clean up
        try :
            os.remove(os.path.join(pickle_path, str(pid)))
        except :
            pass
        del bidders[name]
        result['resultCode'] = 2
        result['resultDescription'] = 'process id %d lost' % pid
        raise HTTPResponse(body=json.dumps(result), status=500,
                Content_Type='application/json')

    result['resultCode'] = 0
    result['resultDescription'] = 'up'
    return result

class application:
    '''
        Main app exposed
    '''

    def __init__(self, config_file):
        '''
            init everything
        '''
        self.app = app

        # load configuration
        self.config = ConfigParser.ConfigParser()
        self.config.read(config_file)

        global AGENT_CONFIG_SERVER
        global GATEWAY_IP
        global GATEWAY_PORT
        global BASE_PATH

        AGENT_CONFIG_SERVER = self.config.get('global', 'AGENT_CONFIG_SERVER')
        GATEWAY_IP = self.config.get('global', 'GATEWAY_IP')
        GATEWAY_PORT = int(self.config.get('global', 'GATEWAY_PORT'))
        BASE_PATH = self.config.get('global', 'BASE_PATH')

        logger.warning('AGENT_CONFIG_SERVER: %s' % AGENT_CONFIG_SERVER)
        logger.warning('GATEWAY_IP: %s' % GATEWAY_IP)
        logger.warning('GATEWAY_PORT: %d' % GATEWAY_PORT)
        logger.warning('BASE_PATH: %s' % BASE_PATH)

        global exec_base_path
        global config_base_path
        global log_base_path
        global bidders_config_base_path

        exec_base_path   = os.path.join(BASE_PATH, 'build/x86_64/bin')
        config_base_path = BASE_PATH
        log_base_path    = os.path.join(BASE_PATH, 'logs')
        bidders_config_base_path = os.path.join(os.getcwd(), json_path)

        # check if the pickle_path exists
        if not os.path.exists(pickle_path):
              os.mkdir(pickle_path)
        # check if the json_path exists
        if not os.path.exists(json_path):
              os.mkdir(json_path)

        # for each pickled process reload the configuration
        for config in os.listdir(pickle_path):
            if config.isdigit():
                #ToDo: add check bidder name too. If process with same pid was re-run
                if not os.path.exists('/proc/%s' % config):
                    logger.error('look like prosess with %s pid no longer exists' % config)
                    os.unlink (os.path.join(pickle_path, config))
                else:
                    f = open(os.path.join(pickle_path, config), 'rb')
                    c = pickle.load(f)
                    bidders[c['bidder_name']] = c
                    f.close()
                    logger.warning('loaded agent %s=%s' % (c['bidder_name'], c))

    def run(self):
        '''
            run the development bottle server
        '''
        logger.warning('starting up server')
        run(self.app, host=GATEWAY_IP, port=GATEWAY_PORT, reloader=False)

<commit_msg>change command line format for correct json handling<commit_after>import sys
import shutil
import os
import logging
import pickle
import subprocess
import re
import time
import json
import ConfigParser

from bottle import Bottle, run, urljoin, HTTPResponse, request

AGENT_CONFIG_SERVER = None
GATEWAY_IP          = None
GATEWAY_PORT        = None
BASE_PATH           = None

# agent pickle file path
pickle_path = '.bidders'
json_path   = '.config'

# agent base path
exec_base_path   = None
config_base_path = None
log_base_path    = None
bidders_config_base_path = None

# set up logging
logging.basicConfig(filename='agent_gateway.log',
        format='%(asctime)-15s %(levelname)s %(message)s',
        level=logging.DEBUG)
logger = logging.getLogger('agent_gateway')

# app bottle
app = Bottle()

# initialize bidder map
bidders = {}

def map_and_redirect(uri, name):
    """
        maps the name, sets the uri and raises a redirection
        otherwise returns the json result code
    """
    try :
        # try to map the name to the internal config name
        location = urljoin(
            AGENT_CONFIG_SERVER,
            uri % bidders[name]['agent_conf_name'])
    except :
        return  {
                'resultCode'        :    1,
                'resultDescription' :   'unable to map %s' % name
                }
        raise HTTPResponse(body=json.dumps(result), status=404,
                Content_Type='application/json')
    raise HTTPResponse("", status=302, Location=location)

@app.get('/v1/agents')
def get_agents():
    """
        returns the list of bidders
    """
    return HTTPResponse(json.dumps(bidders.keys()), status=200, Content_Type='application/json; charset=UTF8')

@app.get('/v1/agents/full')
def get_agents_full():
    """
        returns the list of bidders with full configs
    """
    return HTTPResponse(json.dumps(bidders), status=200, Content_Type='application/json; charset=UTF8')

@app.post('/v1/agents/<name>/config')
@app.get('/v1/agents/<name>/config')
def get_config(name):
    """
        redirects the call to the agent configuration service
        on /v1/agents/<name>/config for the given name
    """
    return map_and_redirect('/v1/agents/%s/config', name)

@app.post('/v1/agents/<name>/heartbeat')
def heartbeat(name):
    """
        redirects the call to the agent configuration service
        on /v1/agents/<name>/heartbeat for the given name
    """
    return map_and_redirect('/v1/agents/%s/heartbeat', name)

@app.get('/v1/agents/all')
def get_all():
    """
        redirects the call to the agent configuration service
        on /v1/agents/all
    """
    location = urljoin(AGENT_CONFIG_SERVER, '/v1/agents/all')
    raise HTTPResponse("", status=302, Location=location)

@app.post('/v1/agents/<name>/start')
def start_bidder(name):
    """
        Starts up a bidder using as the instance parameters
        the arguments passed in the query string
    """
    global _process_id
    result = {
            'resultCode'        :   0,
            'resultDescription' :   'ok'
    }

    if name in bidders :
        result['resultCode'] = 1
        result['resultDescription'] = 'agent already started'
        raise HTTPResponse(body=json.dumps(result), status=500,
                Content_Type='application/json')
    else :
        bidder = {}

    # save the executable name and external name
    bidder['bidder_name'] = name
    bidder['executable'] = request.query['executable']
    bidder['tagid'] = request.query['tagid']
    # save the params
    escape = lambda x : '%s' % x
    bidder['params'] = {
         k:escape(v) for k,v in request.query.iteritems()
            if k not in ('executable', 'tagid', )
    }

    # create a file with the json configuration
    conf_file_name = os.path.join(
        bidders_config_base_path, '%s.conf.json' % name)
    try :
        conf_file = open(conf_file_name, 'w')
        conf_file.write(json.dumps(request.json))
        conf_file.close()
    except :
        result['resultCode'] = 6
        result['resultDescription'] = 'unable to create config file'
        raise HTTPResponse(body=json.dumps(result), status=500,
                Content_Type='application/json')

    logger.info('bringing up bidder %s=%s' % (name, bidder))
    # set the args a list (popen expects them that way)
    arguments = []
    for k,v in bidder['params'].iteritems() :
        arguments.append('-%s' % k)
        arguments.append("'%s'" % v)

    exe = ['nohup']
    exe.append('./%s' % bidder['executable'])
    exe.extend(arguments)
    exe.append('-f')
    exe.append(conf_file_name)
    exe.append('&')
    logger.info('executing : %s' % ' '.join(exe))

    # check the log file
    log_file_name = 'agents/agent_%s_%s.log' % (name, time.strftime('%d.%m.%Y_%H.%M.%S'))
    log_path = os.path.join(log_base_path, log_file_name)
    #try to unlink and then relink
    try :
        os.unlink(
            os.path.join(log_base_path, 'agent_%s.log' % name))
    except :
        pass
    os.symlink(log_file_name,
            os.path.join(log_base_path, 'agent_%s.log' % name))

    log_file = open(log_path, 'w')
    # bring the process up
    try :
        proc = subprocess.Popen(
            ' '.join(exe),
            cwd=exec_base_path,
            shell=True,
            close_fds=True,
            stdout=log_file)
    except :
        result['resultCode'] = 3
        result['resultDescription'] = 'error executing agent'
        raise HTTPResponse(body=json.dumps(result), status=500,
                Content_Type='application/json')

    # read the pid, the one that proc returns belongs to the shell
    pid = None
    # give it some time to make sure the pid is there
    time.sleep(1)
    with open(log_path, 'r') as f:
        for line in f:
            m = re.match('pid:(?P<pid>\d+)', line)
            if m is not None:
                pid = int(m.group('pid'))
                break
    f.close()
    if pid is None :
        # something is not ok
        logger.error('unable to find pid, are you printing it?')
        result['resultCode'] = 4
        result['resultDescription'] = 'unable to find pid, are you printing it?'
        raise HTTPResponse(body=json.dumps(result), status=500,
                Content_Type='application/json')
    # check if the pid still exists, sometimes a bidder starts and aborts
    # right away.
    if not os.path.exists('/proc/%d' % pid):
        # something is not ok
        logger.error('did the process aborted?')
        result['resultCode'] = 5
        result['resultDescription'] = 'process id %d lost' % pid
        raise HTTPResponse(body=json.dumps(result), status=500,
                Content_Type='application/json')

    # save the pid for the new bidder
    bidder['pid']  = pid
    logger.info('pid is : %d' % int(pid))
    # the key stored by the agent configuration service
    # is a concatenation of the bidder name passed and the
    # pid for for process
    bidder['agent_conf_name'] = \
        '%s_%s' % (name, bidder['pid'])
    logger.info('bidder %s got pid %d' % (name, bidder['pid']))

    # save it
    bidders[name] = bidder
    # great, let's pickle the data
    try :
        f = open(os.path.join(pickle_path, str(bidders[name]['pid'])), 'wb')
        pickle.dump(bidders[name], f)
        f.close()
    except :
        result['resultCode'] = 2
        result['resultDescription'] = 'unable to pickle configuration'
        raise HTTPResponse(body=json.dumps(result), status=500,
                Content_Type='application/json')

    result['pid'] = int(pid)
    return result

@app.post('/v1/agents/<name>/stop')
def stop_bidder(name):
    """
        Stops a running agent
    """
    result = {
            'resultCode'        :   0,
            'resultDescription' :   'ok'
    }

    if name not in bidders :
        result['resultCode'] = 1
        result['resultDescription'] = 'bidder not running'
        raise HTTPResponse(body=json.dumps(result), status=404,
                Content_Type='application/json')

    logger.info('stopping bidder %s=%s' % (name, bidders[name]))

    pid = bidders[name]['pid']
    try :
        signal = 9
        if 'signal' in request.query :
            signal = int(request.query['signal'])
        os.kill(pid, signal)
        logger.info('signal %d sent to process with pid %d' % (signal, pid))
    except :
        result['resultCode'] = 2
        result['resultDescription'] = 'unable to kill process %s' % pid
        raise HTTPResponse(body=json.dumps(result), status=500,
                Content_Type='application/json')

    logger.info('agent %s with pid %d stopped' % (name, pid))

    # clean up
    del bidders[name]
    try :
        os.remove(os.path.join(pickle_path, str(pid)))
    except :
        result = {
            'resultCode'        :   3,
            'resultDescription' :   'unable to delete pickled data'
        }
        raise HTTPResponse(body=json.dumps(result), status=500,
                Content_Type='application/json')
    return result

@app.get('/v1/agents/<name>/status')
def get_status(name):
    """
        retrieves the status of a given agent
    """
    result = {
            'resultCode'        :   1,
            'resultDescription' :   'down'
    }

    if not name in bidders:
        raise HTTPResponse(body=json.dumps(result), status=404,
                Content_Type='application/json')

    pid = bidders[name]['pid']
    if not os.path.exists('/proc/%d' % pid):
        # something is not ok, the bidder aborted
        logger.error('did the process %d aborted?' % pid)
        # clean up
        try :
            os.remove(os.path.join(pickle_path, str(pid)))
        except :
            pass
        del bidders[name]
        result['resultCode'] = 2
        result['resultDescription'] = 'process id %d lost' % pid
        raise HTTPResponse(body=json.dumps(result), status=500,
                Content_Type='application/json')

    result['resultCode'] = 0
    result['resultDescription'] = 'up'
    return result

class application:
    '''
        Main app exposed
    '''

    def __init__(self, config_file):
        '''
            init everything
        '''
        self.app = app

        # load configuration
        self.config = ConfigParser.ConfigParser()
        self.config.read(config_file)

        global AGENT_CONFIG_SERVER
        global GATEWAY_IP
        global GATEWAY_PORT
        global BASE_PATH

        AGENT_CONFIG_SERVER = self.config.get('global', 'AGENT_CONFIG_SERVER')
        GATEWAY_IP = self.config.get('global', 'GATEWAY_IP')
        GATEWAY_PORT = int(self.config.get('global', 'GATEWAY_PORT'))
        BASE_PATH = self.config.get('global', 'BASE_PATH')

        logger.warning('AGENT_CONFIG_SERVER: %s' % AGENT_CONFIG_SERVER)
        logger.warning('GATEWAY_IP: %s' % GATEWAY_IP)
        logger.warning('GATEWAY_PORT: %d' % GATEWAY_PORT)
        logger.warning('BASE_PATH: %s' % BASE_PATH)

        global exec_base_path
        global config_base_path
        global log_base_path
        global bidders_config_base_path

        exec_base_path   = os.path.join(BASE_PATH, 'build/x86_64/bin')
        config_base_path = BASE_PATH
        log_base_path    = os.path.join(BASE_PATH, 'logs')
        bidders_config_base_path = os.path.join(os.getcwd(), json_path)

        # check if the pickle_path exists
        if not os.path.exists(pickle_path):
              os.mkdir(pickle_path)
        # check if the json_path exists
        if not os.path.exists(json_path):
              os.mkdir(json_path)

        # check if folder for logs exists
        agent_logs = os.path.join(log_base_path, 'agents')
        if not os.path.exists(agent_logs):
              os.mkdir(agent_logs)

        # for each pickled process reload the configuration
        for config in os.listdir(pickle_path):
            if config.isdigit():
                #ToDo: add check bidder name too. If process with same pid was re-run
                if not os.path.exists('/proc/%s' % config):
                    logger.error('look like prosess with %s pid no longer exists' % config)
                    os.unlink (os.path.join(pickle_path, config))
                else:
                    f = open(os.path.join(pickle_path, config), 'rb')
                    c = pickle.load(f)
                    bidders[c['bidder_name']] = c
                    f.close()
                    logger.warning('loaded agent %s=%s' % (c['bidder_name'], c))

    def run(self):
        '''
            run the development bottle server
        '''
        logger.warning('starting up server')
        run(self.app, host=GATEWAY_IP, port=GATEWAY_PORT, reloader=False)

<|endoftext|>