<commit_before># Copyright (c) 2016 Intracom S.A. Telecom Solutions. All rights reserved.
#
# This program and the accompanying materials are made available under the
# terms of the Eclipse Public License v1.0 which accompanies this distribution,
# and is available at http://www.eclipse.org/legal/epl-v10.html

""" NB-Generator Class- All NB-Generator-related functionality is here"""

# import emulators.nb_generator
import gevent
import logging
import stress_test.nb_generator_exceptions
import sys
import time
import traceback
import util.netutil

class NBgen:
    """
    All NB-generator related functionality is here
    """

    def __init__(self, nb_gen_base_dir, test_config, controller, sbemu,
                 log_level="DEBUG"):
        """Create an NB-generator object. Options from JSON input file

        :param nb_gen_base_dir: emulator base directory
        :param test_config: JSON input configuration
        :param controller: object of the Controller class
        :param sbemu: object of the SBEmu subclass
        :param log_level: defines the logging level. (DEBUG, INFO, ERROR)
        :type nb_gen_base_dir: str
        :type test_config: JSON configuration dictionary
        :type controller: object
        :type sbemu: object
        :type log_level: str
        """
        self.controller = controller
        self.sbemu = sbemu
        self.name = test_config['nb_emulator_name']
        self.base_dir = nb_gen_base_dir
        self.traceback_enabled = True

        self.ip = test_config['nb_emulator_node_ip']
        self.ssh_port = test_config['nb_emulator_node_ssh_port']
        self.ssh_user = test_config['nb_emulator_node_username']
        self.ssh_pass = test_config['nb_emulator_node_password']
        self.build_hnd = (self.base_dir +
                          test_config['nb_emulator_build_handler'])
        self.clean_hnd = (self.base_dir +
                          test_config['nb_emulator_clean_handler'])
        self.run_hnd = (self.base_dir +
                        test_config['nb_emulator_run_handler'])
        self.get_oper_ds_flows_hnd = (
            self.base_dir + test_config['nb_emulator_get_oper_ds_handler'])

        self.status = 'UNKNOWN'
        self._ssh_conn = None

        self.flow_delete_flag = test_config['flow_delete_flag']
        self.flows_per_request = test_config['flows_per_request']
        self.log_level = log_level
        # The parameters initialized as None are dimensions of the test.
        # These values are passed outside, from the test in the main for loop.
        # ---------------------------------------------------------------------
        self.flow_workers = None
        self.total_flows = None
        self.flow_operations_delay_ms = None
        # ---------------------------------------------------------------------
        self.flows_ds_discovery_deadline = 240

        self.confirm_time = 0.0
        self.e2e_installation_time = 0.0
        self.discover_flows_on_switches_time = 0.0

        self.venv_hnd = self.base_dir + "bin/venv_handler.sh"

    def _error_handling(self, error_message, error_num=1):
        """
        Handles custom errors of nb_generator

        :param error_message: message of the handled error
        :param error_num: error number of the handled error, used to define
        subcases of raised errors.
        :type error_message: str
        :type error_num: int
        :raises nb_generator_exceptions.NBGenError: to terminate execution of
        test after error handling
        """
        exc_type, exc_obj, exc_tb = sys.exc_info()
        logging.error('{0} :::::::::: Exception :::::::::::'.
                      format(exc_obj))
        logging.error(error_message)
        logging.error('Error number: {0}'.format(error_num))
        logging.error('{0} - {1} Exception: {2}, {3}'.
                      format(exc_obj, self.name, exc_type, exc_tb.tb_lineno))
        if self.traceback_enabled:
            traceback.print_exc()
        raise(stress_test.nb_generator_exceptions.NBGenError)

    def init_ssh(self):
        """
        Initializes a new SSH client object, with the nb_generator node and \
            assigns it to the protected attribute _ssh_conn. If a connection \
            already exists it returns a new SSH client object to the  \
            controller node.

        :raises nb_generator_exceptions.NBGenNodeConnectionError: if ssh \
            connection establishment fails
        """
        logging.info(
            '[open_ssh_connection] Initiating SSH session with {0} node on '
            '{1} host.'.format(self.name, self.ip))
        try:
            try:
                if self._ssh_conn is None:
                    self._ssh_conn = util.netutil.ssh_connect_or_return(
                        self.ip, int(self.ssh_port), self.ssh_user,
                        self.ssh_pass, 10)
                else:
                    # Return a new client ssh object for the nb-generator node
                    return util.netutil.ssh_connect_or_return(
                        self.ip, int(self.ssh_port), self.ssh_user,
                        self.ssh_pass, 10)
            except:
                raise(stress_test.nb_generator_exceptions.NBGenNodeConnectionError)
        except stress_test.nb_generator_exceptions.NBGenError as e:
            self._error_handling(e.err_msg, e.err_code)

    def build(self):
        """
        Wrapper to the NB-Generator build handler

        :raises IOError: if the handler does not exist on the remote host
        :raises nb_generator_exceptions.NBGenBuildError: if build process fails
        """
        logging.info('[NB_generator] Building')
        self.status = 'BUILDING'
        try:
            try:
                if not util.netutil.isfile(self.ip, self.ssh_port,
                                           self.ssh_user, self.ssh_pass,
                                           [self.build_hnd]):
                    self.status = 'NOT_BUILT'
                    raise(IOError(
                        '{0} build handler does not exist'.
                        format('[nb_generator.build]')))
                else:
                    util.netutil.make_remote_file_executable(
                        self.ip, self.ssh_port, self.ssh_user, self.ssh_pass,
                        self.build_hnd)
                exit_status, cmd_output = util.netutil.ssh_run_command(
                    self._ssh_conn, ' '.join([self.build_hnd]),
                    '[NB_generator.build_handler]')
                if exit_status == 0:
                    self.status = 'BUILT'
                    logging.info("[NB_generator] Successful building")
                else:
                    self.status = 'NOT_BUILT'
                    raise(stress_test.nb_generator_exceptions.NBGenBuildError(
                        '[NB_generator] Failure during running. Build handler '
                        'exited with no zero exit status. \n '
                        'Handler output: {0}'.format(cmd_output), exit_status))
            except stress_test.nb_generator_exceptions.NBGenError as e:
                self._error_handling(e.err_msg, e.err_code)
            except:
                raise(stress_test.nb_generator_exceptions.NBGenBuildError(
                    '[NB_generator] Build handler was not executed at all. '
                    'Failure running the handler.'))
        except stress_test.nb_generator_exceptions.NBGenError as e:
            self._error_handling(e.err_msg, e.err_code)

    def clean(self):
        """
        Wrapper to the NB-Generator clean handler

        :raises IOError: if the handler does not exist on the remote host
        :raises nb_generator_exceptions.NBGenCleanError: if clean process fails
        """
        logging.info('[NB_generator] Cleaning')
        self.status = 'CLEANING'
        try:
            try:
                if not util.netutil.isfile(self.ip, self.ssh_port,
                                           self.ssh_user, self.ssh_pass,
                                           [self.clean_hnd]):
                    self.status = 'NOT_CLEANED'
                    raise(IOError(
                        '{0} clean handler does not exist'.
                        format('[nb_generator.clean]')))
                else:
                    util.netutil.make_remote_file_executable(
                        self.ip, self.ssh_port, self.ssh_user, self.ssh_pass,
                        self.clean_hnd)
                exit_status, cmd_output = util.netutil.ssh_run_command(
                    self._ssh_conn, self.clean_hnd,
                    '[NB_generator.clean_handler]')
                if exit_status == 0:
                    self.status = 'CLEANED'
                    logging.info("[NB_generator] Successful clean")
                else:
                    self.status = 'NOT_CLEANED'
                    raise(stress_test.nb_generator_exceptions.NBGenCleanError(
                        '[NB_generator] Failure during running. Clean handler '
                        'exited with no zero exit status. \n '
                        'Handler output: {0}'.format(cmd_output), exit_status))
            except stress_test.nb_generator_exceptions.NBGenError as e:
                self._error_handling(e.err_msg, e.err_code)
            except:
                raise(stress_test.nb_generator_exceptions.NBGenCleanError(
                    '[NB_generator] Clean handler was not executed at all. '
                    'Failure running the handler.'))
        except stress_test.nb_generator_exceptions.NBGenError as e:
            self._error_handling(e.err_msg, e.err_code)

    def run(self):
        """
        Wrapper to the NB-Generator run handler

        :returns: Returns the combined stdout - stderr of the executed command
        :rtype: str
        :raises IOError: if the handler does not exist on the remote host
        :raises nb_generator_exceptions.NBGenRunError: if running \
            nb_generator fails
        """
        logging.info("[NB_generator] Run handler")
        self.status = 'STARTED'
        try:
            try:
                if not util.netutil.isfile(self.ip, self.ssh_port,
                                           self.ssh_user, self.ssh_pass,
                                           [self.run_hnd]):
                    self.status = 'NB_GEN_NOT_RUNNING'
                    raise(IOError(
                        '{0} run handler does not exist'.
                        format('[nb_generator.run]')))
                else:
                    util.netutil.make_remote_file_executable(
                        self.ip, self.ssh_port, self.ssh_user, self.ssh_pass,
                        self.run_hnd)
                exit_status, cmd_output = \
                    util.netutil.ssh_run_command(
                        self._ssh_conn,
                        ' '.join([str(self.venv_hnd),
                                  str(self.base_dir),
                                  str(self.run_hnd),
                                  str(self.controller.ip),
                                  str(self.controller.restconf_port),
                                  str(self.total_flows),
                                  str(self.flow_workers),
                                  str(self.flow_operations_delay_ms),
                                  str(self.flow_delete_flag),
                                  str(self.controller.restconf_user),
                                  str(self.controller.restconf_pass),
                                  str(self.flows_per_request),
                                  str(self.log_level)]),
                        '[NB_generator] run_handler')
                if exit_status == 0:
                    self.status = 'NB_GEN_RUNNING'
                    logging.info("[NB_generator] up and running")
                else:
                    self.status = 'NB_GEN_NOT_RUNNING'
                    raise(stress_test.nb_generator_exceptions.NBGenRunError(
                        '[NB_generator] Failure during running. {0}'.
                        format(cmd_output), exit_status))
                return cmd_output
            except stress_test.nb_generator_exceptions.NBGenError as e:
                self._error_handling(e.err_msg, e.err_code)
            except:
                raise(stress_test.nb_generator_exceptions.NBGenRunError)
        except stress_test.nb_generator_exceptions.NBGenError as e:
            self._error_handling(e.err_msg, e.err_code)

    def __del__(self):
        """
        Method called when object is destroyed"""
        try:
            logging.info('Closing NB-Generator ssh connection.')
            self._ssh_conn.close()
        except:
            pass
<commit_msg>Add debug print command<commit_after># Copyright (c) 2016 Intracom S.A. Telecom Solutions. All rights reserved.
#
# This program and the accompanying materials are made available under the
# terms of the Eclipse Public License v1.0 which accompanies this distribution,
# and is available at http://www.eclipse.org/legal/epl-v10.html

""" NB-Generator Class- All NB-Generator-related functionality is here"""

# import emulators.nb_generator
import gevent
import logging
import stress_test.nb_generator_exceptions
import sys
import time
import traceback
import util.netutil

class NBgen:
    """
    All NB-generator related functionality is here
    """

    def __init__(self, nb_gen_base_dir, test_config, controller, sbemu,
                 log_level="DEBUG"):
        """Create an NB-generator object. Options from JSON input file

        :param nb_gen_base_dir: emulator base directory
        :param test_config: JSON input configuration
        :param controller: object of the Controller class
        :param sbemu: object of the SBEmu subclass
        :param log_level: defines the logging level. (DEBUG, INFO, ERROR)
        :type nb_gen_base_dir: str
        :type test_config: JSON configuration dictionary
        :type controller: object
        :type sbemu: object
        :type log_level: str
        """
        self.controller = controller
        self.sbemu = sbemu
        self.name = test_config['nb_emulator_name']
        self.base_dir = nb_gen_base_dir
        self.traceback_enabled = True

        self.ip = test_config['nb_emulator_node_ip']
        self.ssh_port = test_config['nb_emulator_node_ssh_port']
        self.ssh_user = test_config['nb_emulator_node_username']
        self.ssh_pass = test_config['nb_emulator_node_password']
        self.build_hnd = (self.base_dir +
                          test_config['nb_emulator_build_handler'])
        self.clean_hnd = (self.base_dir +
                          test_config['nb_emulator_clean_handler'])
        self.run_hnd = (self.base_dir +
                        test_config['nb_emulator_run_handler'])
        self.get_oper_ds_flows_hnd = (
            self.base_dir + test_config['nb_emulator_get_oper_ds_handler'])

        self.status = 'UNKNOWN'
        self._ssh_conn = None

        self.flow_delete_flag = test_config['flow_delete_flag']
        self.flows_per_request = test_config['flows_per_request']
        self.log_level = log_level
        # The parameters initialized as None are dimensions of the test.
        # These values are passed outside, from the test in the main for loop.
        # ---------------------------------------------------------------------
        self.flow_workers = None
        self.total_flows = None
        self.flow_operations_delay_ms = None
        # ---------------------------------------------------------------------
        self.flows_ds_discovery_deadline = 240

        self.confirm_time = 0.0
        self.e2e_installation_time = 0.0
        self.discover_flows_on_switches_time = 0.0

        self.venv_hnd = self.base_dir + "bin/venv_handler.sh"
        print('=====================================')
        print('=====================================')
        print('=====================================')
        print(nb_gen_base_dir)
        print('=====================================')
        print('=====================================')
        print('=====================================')

    def _error_handling(self, error_message, error_num=1):
        """
        Handles custom errors of nb_generator

        :param error_message: message of the handled error
        :param error_num: error number of the handled error, used to define
        subcases of raised errors.
        :type error_message: str
        :type error_num: int
        :raises nb_generator_exceptions.NBGenError: to terminate execution of
        test after error handling
        """
        exc_type, exc_obj, exc_tb = sys.exc_info()
        logging.error('{0} :::::::::: Exception :::::::::::'.
                      format(exc_obj))
        logging.error(error_message)
        logging.error('Error number: {0}'.format(error_num))
        logging.error('{0} - {1} Exception: {2}, {3}'.
                      format(exc_obj, self.name, exc_type, exc_tb.tb_lineno))
        if self.traceback_enabled:
            traceback.print_exc()
        raise(stress_test.nb_generator_exceptions.NBGenError)

    def init_ssh(self):
        """
        Initializes a new SSH client object, with the nb_generator node and \
            assigns it to the protected attribute _ssh_conn. If a connection \
            already exists it returns a new SSH client object to the  \
            controller node.

        :raises nb_generator_exceptions.NBGenNodeConnectionError: if ssh \
            connection establishment fails
        """
        logging.info(
            '[open_ssh_connection] Initiating SSH session with {0} node on '
            '{1} host.'.format(self.name, self.ip))
        try:
            try:
                if self._ssh_conn is None:
                    self._ssh_conn = util.netutil.ssh_connect_or_return(
                        self.ip, int(self.ssh_port), self.ssh_user,
                        self.ssh_pass, 10)
                else:
                    # Return a new client ssh object for the nb-generator node
                    return util.netutil.ssh_connect_or_return(
                        self.ip, int(self.ssh_port), self.ssh_user,
                        self.ssh_pass, 10)
            except:
                raise(stress_test.nb_generator_exceptions.NBGenNodeConnectionError)
        except stress_test.nb_generator_exceptions.NBGenError as e:
            self._error_handling(e.err_msg, e.err_code)

    def build(self):
        """
        Wrapper to the NB-Generator build handler

        :raises IOError: if the handler does not exist on the remote host
        :raises nb_generator_exceptions.NBGenBuildError: if build process fails
        """
        logging.info('[NB_generator] Building')
        self.status = 'BUILDING'
        try:
            try:
                if not util.netutil.isfile(self.ip, self.ssh_port,
                                           self.ssh_user, self.ssh_pass,
                                           [self.build_hnd]):
                    self.status = 'NOT_BUILT'
                    raise(IOError(
                        '{0} build handler does not exist'.
                        format('[nb_generator.build]')))
                else:
                    util.netutil.make_remote_file_executable(
                        self.ip, self.ssh_port, self.ssh_user, self.ssh_pass,
                        self.build_hnd)
                exit_status, cmd_output = util.netutil.ssh_run_command(
                    self._ssh_conn, ' '.join([self.build_hnd]),
                    '[NB_generator.build_handler]')
                if exit_status == 0:
                    self.status = 'BUILT'
                    logging.info("[NB_generator] Successful building")
                else:
                    self.status = 'NOT_BUILT'
                    raise(stress_test.nb_generator_exceptions.NBGenBuildError(
                        '[NB_generator] Failure during running. Build handler '
                        'exited with no zero exit status. \n '
                        'Handler output: {0}'.format(cmd_output), exit_status))
            except stress_test.nb_generator_exceptions.NBGenError as e:
                self._error_handling(e.err_msg, e.err_code)
            except:
                raise(stress_test.nb_generator_exceptions.NBGenBuildError(
                    '[NB_generator] Build handler was not executed at all. '
                    'Failure running the handler.'))
        except stress_test.nb_generator_exceptions.NBGenError as e:
            self._error_handling(e.err_msg, e.err_code)

    def clean(self):
        """
        Wrapper to the NB-Generator clean handler

        :raises IOError: if the handler does not exist on the remote host
        :raises nb_generator_exceptions.NBGenCleanError: if clean process fails
        """
        logging.info('[NB_generator] Cleaning')
        self.status = 'CLEANING'
        try:
            try:
                if not util.netutil.isfile(self.ip, self.ssh_port,
                                           self.ssh_user, self.ssh_pass,
                                           [self.clean_hnd]):
                    self.status = 'NOT_CLEANED'
                    raise(IOError(
                        '{0} clean handler does not exist'.
                        format('[nb_generator.clean]')))
                else:
                    util.netutil.make_remote_file_executable(
                        self.ip, self.ssh_port, self.ssh_user, self.ssh_pass,
                        self.clean_hnd)
                exit_status, cmd_output = util.netutil.ssh_run_command(
                    self._ssh_conn, self.clean_hnd,
                    '[NB_generator.clean_handler]')
                if exit_status == 0:
                    self.status = 'CLEANED'
                    logging.info("[NB_generator] Successful clean")
                else:
                    self.status = 'NOT_CLEANED'
                    raise(stress_test.nb_generator_exceptions.NBGenCleanError(
                        '[NB_generator] Failure during running. Clean handler '
                        'exited with no zero exit status. \n '
                        'Handler output: {0}'.format(cmd_output), exit_status))
            except stress_test.nb_generator_exceptions.NBGenError as e:
                self._error_handling(e.err_msg, e.err_code)
            except:
                raise(stress_test.nb_generator_exceptions.NBGenCleanError(
                    '[NB_generator] Clean handler was not executed at all. '
                    'Failure running the handler.'))
        except stress_test.nb_generator_exceptions.NBGenError as e:
            self._error_handling(e.err_msg, e.err_code)

    def run(self):
        """
        Wrapper to the NB-Generator run handler

        :returns: Returns the combined stdout - stderr of the executed command
        :rtype: str
        :raises IOError: if the handler does not exist on the remote host
        :raises nb_generator_exceptions.NBGenRunError: if running \
            nb_generator fails
        """
        logging.info("[NB_generator] Run handler")
        self.status = 'STARTED'
        try:
            try:
                if not util.netutil.isfile(self.ip, self.ssh_port,
                                           self.ssh_user, self.ssh_pass,
                                           [self.run_hnd]):
                    self.status = 'NB_GEN_NOT_RUNNING'
                    raise(IOError(
                        '{0} run handler does not exist'.
                        format('[nb_generator.run]')))
                else:
                    util.netutil.make_remote_file_executable(
                        self.ip, self.ssh_port, self.ssh_user, self.ssh_pass,
                        self.run_hnd)
                exit_status, cmd_output = \
                    util.netutil.ssh_run_command(
                        self._ssh_conn,
                        ' '.join([str(self.venv_hnd),
                                  str(self.base_dir),
                                  str(self.run_hnd),
                                  str(self.controller.ip),
                                  str(self.controller.restconf_port),
                                  str(self.total_flows),
                                  str(self.flow_workers),
                                  str(self.flow_operations_delay_ms),
                                  str(self.flow_delete_flag),
                                  str(self.controller.restconf_user),
                                  str(self.controller.restconf_pass),
                                  str(self.flows_per_request),
                                  str(self.log_level)]),
                        '[NB_generator] run_handler')
                if exit_status == 0:
                    self.status = 'NB_GEN_RUNNING'
                    logging.info("[NB_generator] up and running")
                else:
                    self.status = 'NB_GEN_NOT_RUNNING'
                    raise(stress_test.nb_generator_exceptions.NBGenRunError(
                        '[NB_generator] Failure during running. {0}'.
                        format(cmd_output), exit_status))
                return cmd_output
            except stress_test.nb_generator_exceptions.NBGenError as e:
                self._error_handling(e.err_msg, e.err_code)
            except:
                raise(stress_test.nb_generator_exceptions.NBGenRunError)
        except stress_test.nb_generator_exceptions.NBGenError as e:
            self._error_handling(e.err_msg, e.err_code)

    def __del__(self):
        """
        Method called when object is destroyed"""
        try:
            logging.info('Closing NB-Generator ssh connection.')
            self._ssh_conn.close()
        except:
            pass
<|endoftext|>


<commit_before># registry class is for generic framework components register function with configuration
class Registry:

    def __init__(self, registrar):
        self.registrar = registrar

    def get(self, key):
        try :
            layers = key.split('.')
            value = self.registrar
            for key in layers:
                value = value[key]
            return value
        except:
            return None

    def set(self, key, value):
        target = self.registrar
        for element in key.split(".")[:-1]:
            target = target.setdefault(element, dict())
        target[key.split(".")[-1]] = value

    def flush(self):
        self.registrar.clear()

    def update(self, registrar):
        self.registrar.update(registrar)

    def all(self):
        return self.registrar

    def __str__(self):
        return self.registrar

# Config class
class Config(Registry):
    pass

# Session class
class Session(Registry):
    pass

# Cookie Class
class Cookie(Registry):
    pass

class IoC:

    def __init__(self, instances = {}):
        self.instances = instances

    def bind(self, key, value):
        self.instances[key] = value

    def resolve(self, key):
        try:
            return self.instances[key]
        except:
            return None

# metaclass for facade class
class DeflectToInstance(type):

    def __getattr__(selfcls, a): # selfcls in order to make clear it is a class object (as we are a metaclass)
        try:
            # first, inquiry the class itself
            return super(DeflectToInstance, selfcls).__getattr__(a)
        except AttributeError:
            # Not found, so try to inquiry the instance attribute:
            return getattr(selfcls.instance, a)

# facade that is used to hold instances statically with boot method
class Facade:

    __metaclass__ = DeflectToInstance

    instance = None

    @classmethod
    def boot(cls, object, configuration = {}):
        if cls.instance is None:
            cls.instance = object(configuration)
<commit_msg>split comment line into two<commit_after># registry class is for generic framework components register function with configuration
class Registry:

    def __init__(self, registrar):
        self.registrar = registrar

    def get(self, key):
        try :
            layers = key.split('.')
            value = self.registrar
            for key in layers:
                value = value[key]
            return value
        except:
            return None

    def set(self, key, value):
        target = self.registrar
        for element in key.split(".")[:-1]:
            target = target.setdefault(element, dict())
        target[key.split(".")[-1]] = value

    def flush(self):
        self.registrar.clear()

    def update(self, registrar):
        self.registrar.update(registrar)

    def all(self):
        return self.registrar

    def __str__(self):
        return self.registrar

# Config class
class Config(Registry):
    pass

# Session class
class Session(Registry):
    pass

# Cookie Class
class Cookie(Registry):
    pass

class IoC:

    def __init__(self, instances = {}):
        self.instances = instances

    def bind(self, key, value):
        self.instances[key] = value

    def resolve(self, key):
        try:
            return self.instances[key]
        except:
            return None

# metaclass for facade class
class DeflectToInstance(type):

    # selfcls in order to make clear it is a class object (as we are a metaclass)
    def __getattr__(selfcls, a):
        try:
            # first, inquiry the class itself
            return super(DeflectToInstance, selfcls).__getattr__(a)
        except AttributeError:
            # Not found, so try to inquiry the instance attribute:
            return getattr(selfcls.instance, a)

# facade that is used to hold instances statically with boot method
class Facade:

    __metaclass__ = DeflectToInstance

    instance = None

    @classmethod
    def boot(cls, object, configuration = {}):
        if cls.instance is None:
            cls.instance = object(configuration)
<|endoftext|>


<commit_before>import user

lush = user.User('adampsyche')

dpv = []
venue = 257163

# iterate over all distinct beers, check for venue, if found, add to list
# iterate all at venue,

results = lush.GetDistinctBeers()
dpv.append(results)
while lush.GetDistinctBeers(params={'offset': 25}):
  dpv.append(results)<commit_msg>Pony Bar unique beer checkins YES<commit_after>import user

lush = user.User(user_name='adampsyche')

pony_checkins = []
venue = 257163

def GetPonyCheckins(checkin):
  if checkin.get('venue'):
    if checkin['venue']['venue_name'] == 'The Pony Bar':
      return checkin
  return None

checkins = lush.GetUserCheckins({'limit': 50})
for checkin in checkins:
  if GetPonyCheckins(checkin):
    pony_checkins.append(checkin)

marker_checkin = checkins[-1]['checkin_id']

while lush.GetUserCheckins({'limit': 50, 'max_id': marker_checkin}):
  checkins = lush.GetUserCheckins({'limit': 50, 'max_id': marker_checkin})
  if len(checkins) > 1:
    for checkin in checkins:
      if GetPonyCheckins(checkin):
        pony_checkins.append(checkin)
  marker_checkin = checkins[-1]['checkin_id']

pony_uniques = []

for pony_checkin in pony_checkins:
  if pony_checkin['beer']['beer_name'] not in pony_uniques:
    pony_uniques.append(pony_checkin['beer']['beer_name'])<|endoftext|>


<commit_before>import sublime

from project import Project

import functools
import os
import subprocess
import thread

class SbtRunner(object):

    runners = {}

    @classmethod
    def get_runner(cls, window):
        if window.id() not in cls.runners:
            cls.runners[window.id()] = SbtRunner(window)
        return cls.runners[window.id()]

    @classmethod
    def is_sbt_running_for(cls, window):
        return cls.get_runner(window).is_sbt_running()

    def __init__(self, window):
        self._project = Project.get_project(window)
        self._proc = None

    def project_root(self):
        return self._project.project_root()

    def sbt_command(self, command):
        cmdline = self._project.sbt_command()
        if command is not None:
            cmdline.append(command)
        return cmdline

    def start_sbt(self, command, on_start, on_stop, on_stdout, on_stderr):
        if self.project_root() and not self.is_sbt_running():
            self._start_sbt_proc(self.sbt_command(command))
            on_start()
            if self._proc.stdout:
                thread.start_new_thread(self._monitor_output, (self._proc.stdout, on_stdout))
            if self._proc.stderr:
                thread.start_new_thread(self._monitor_output, (self._proc.stderr, on_stderr))
            thread.start_new_thread(self._monitor_proc, (on_stop,))

    def stop_sbt(self):
        if self.is_sbt_running():
            self._proc.terminate()

    def kill_sbt(self):
        if self.is_sbt_running():
            self._proc.kill()

    def is_sbt_running(self):
        return (self._proc is not None) and (self._proc.returncode is None)

    def send_to_sbt(self, input):
        if self.is_sbt_running():
            self._proc.stdin.write(input)
            self._proc.stdin.flush()

    def _start_sbt_proc(self, cmdline):
        saved_cwd = os.getcwd()
        os.chdir(self.project_root())
        self._proc = subprocess.Popen(cmdline,
                                      stdin=subprocess.PIPE,
                                      stdout=subprocess.PIPE,
                                      stderr=subprocess.PIPE)
        os.chdir(saved_cwd)

    def _monitor_output(self, pipe, handle_output):
        while True:
            output = os.read(pipe.fileno(), 2 ** 15)
            if output != "":
                sublime.set_timeout(functools.partial(handle_output, output), 0)
            else:
                pipe.close()
                return

    def _monitor_proc(self, handle_stop):
        self._proc.wait()
        sublime.set_timeout(handle_stop, 0)
<commit_msg>Show an error message if the sbt command is not found.<commit_after>import sublime

from project import Project

import functools
import os
import subprocess
import thread

class SbtRunner(object):

    runners = {}

    @classmethod
    def get_runner(cls, window):
        if window.id() not in cls.runners:
            cls.runners[window.id()] = SbtRunner(window)
        return cls.runners[window.id()]

    @classmethod
    def is_sbt_running_for(cls, window):
        return cls.get_runner(window).is_sbt_running()

    def __init__(self, window):
        self._project = Project.get_project(window)
        self._proc = None

    def project_root(self):
        return self._project.project_root()

    def sbt_command(self, command):
        cmdline = self._project.sbt_command()
        if command is not None:
            cmdline.append(command)
        return cmdline

    def start_sbt(self, command, on_start, on_stop, on_stdout, on_stderr):
        if self.project_root() and not self.is_sbt_running():
            self._proc = self._try_start_sbt_proc(self.sbt_command(command))
            if self._proc is not None:
                on_start()
                if self._proc.stdout:
                    thread.start_new_thread(self._monitor_output,
                                            (self._proc.stdout, on_stdout))
                if self._proc.stderr:
                    thread.start_new_thread(self._monitor_output,
                                            (self._proc.stderr, on_stderr))
                thread.start_new_thread(self._monitor_proc, (on_stop,))

    def stop_sbt(self):
        if self.is_sbt_running():
            self._proc.terminate()

    def kill_sbt(self):
        if self.is_sbt_running():
            self._proc.kill()

    def is_sbt_running(self):
        return (self._proc is not None) and (self._proc.returncode is None)

    def send_to_sbt(self, input):
        if self.is_sbt_running():
            self._proc.stdin.write(input)
            self._proc.stdin.flush()

    def _try_start_sbt_proc(self, cmdline):
        try:
            return self._start_sbt_proc(cmdline)
        except OSError:
            msg = ('Unable to find "%s".\n\n'
                   'You may need to specify the full path to your sbt command.'
                   % cmdline[0])
            sublime.error_message(msg)

    def _start_sbt_proc(self, cmdline):
        saved_cwd = os.getcwd()
        os.chdir(self.project_root())
        proc = self._popen(cmdline)
        os.chdir(saved_cwd)
        return proc

    def _popen(self, cmdline):
        return subprocess.Popen(cmdline,
                                stdin=subprocess.PIPE,
                                stdout=subprocess.PIPE,
                                stderr=subprocess.PIPE)

    def _monitor_output(self, pipe, handle_output):
        while True:
            output = os.read(pipe.fileno(), 2 ** 15)
            if output != "":
                sublime.set_timeout(functools.partial(handle_output, output), 0)
            else:
                pipe.close()
                return

    def _monitor_proc(self, handle_stop):
        self._proc.wait()
        sublime.set_timeout(handle_stop, 0)
<|endoftext|>


<commit_before>import os
import sys
import struct
import signal
import string

class StoragePage:
	def __init__(self):
		self.offset = 0
		self.size = 0

class StorageHeaderPage(StoragePage):
	def __init__(self):
		self.chunk_id = 0
		self.min_log_segment_id = 0
		self.max_log_segment_id = 0
		self.max_log_command_id = 0
		self.num_keys = 0
		self.use_bloomfilter = False
		self.index_page_offset = 0
		self.index_page_size = 0
		self.bloom_page_offset = 0
		self.bloom_page_size = 0
		self.first_key = ""
		self.last_key = ""
		self.mid_point = ""

	def read(self, file):
		data = file.read(4096)
		if len(data) != 4096:
			return False
		pos = 12
		size, checksum, version = struct.unpack("<III", data[:pos])
		data = data[pos:]
		# skip text
		text = data[:64]
		data = data[64:]
		self.chunk_id, self.min_log_segment_id, self.max_log_segment_id, self.max_log_command_id = \
			struct.unpack("<QQQI", data[:28])
		data = data[28:]
		if data[0] == 'T':
			self.use_bloomfilter = True
		data = data[1:]
		self.num_keys, self.index_page_offset, self.index_page_size = struct.unpack("<QQI", data[:20])
		data = data[20:]
		if self.use_bloomfilter:
			self.bloom_page_offset, self.bloom_page_size = struct.unpack("<QI", data[:12])
			data = data[12:]

		length, = struct.unpack("<I", data[:4])
		data = data[4:]
		self.first_key = data[:length]
		data = data[length:]

		length, = struct.unpack("<I", data[:4])
		data = data[4:]
		self.last_key = data[:length]
		data = data[length:]

		length, = struct.unpack("<I", data[:4])
		data = data[4:]
		self.mid_point = data[:length]
		data = data[length:]

class StorageDataPage(StoragePage):
	def __init__(self, index):
		self.index = index
		self.size = 0
		self.offset = 0
		self.num_keys = 0
		self.keybuffer_length = 0
		self.checksum = 0
		self.key_values = {}

	def read(self, file):
		data = file.read(4096)
		self.size, self.checksum, self.keybuffer_length, self.num_keys = struct.unpack("<IIII", data[:16])
		if self.size > 4096:
			data += file.read(self.size - 4096)
		data = data[16:]
		print("-- Page index %d, Num keys: %d" % (self.index, self.num_keys))
		i = 0
		prev_key = ""
		while i < self.num_keys:
			type, keylen = struct.unpack("<cH", data[:3])
			data = data[3:]
			key = data[:keylen]
			if key <= prev_key:
				print("Error")
			prev_key = key
			data = data[keylen:]
			# TODO: values
			# if type == 's':
			# 	vallen, = struct.unpack("<I", data[:4])
			# 	data = data[4:]
			# 	value = data[:vallen]
			# 	data = data[vallen:]
			# 	if self.key_values.has_key(key):
			# 		print("Error")
			# 		print(file.tell())
			# 	self.key_values[key] = value
			# else:
			# 	print("Delete key: " + key)
			# 	if self.key_values.has_key(key):
			# 		self.key_values.pop(key)
			# print(key, value)
			print(key)
			i += 1
		return i

class StorageIndexPage(StoragePage):
	def __init__(self):
		self.size = 0
		self.offset = 0
		self.checksum = 0
		self.num_keys = 0

	def read(self, file):
		data = file.read(4096)
		self.size, self.checksum, self.num_keys = struct.unpack("<III", data[:12])
		if self.size > 4096:
			data += file.read(self.size - 4096)
		data = data[12:]
		i = 0
		while i < self.num_keys:
			key_offset, keylen = struct.unpack("<IH", data[:6])
			data = data[6:]
			key = data[:keylen]
			data = data[keylen:]
			i += 1
			print("Index key: %s" % (key))

def check_chunk_file(filename):
	f = open(filename, "rb")
	if os.path.getsize(filename) == 0:
		print("Empty chunk file")
		return
	header_page = StorageHeaderPage()
	header_page.read(f)
	print("num_keys = " + str(header_page.num_keys))
	print(str(header_page.__dict__))
	keys = 0
	index = 0
	while keys < header_page.num_keys:
		data_page = StorageDataPage(index)
		keys += data_page.read(f)
		index += 1
		#print("Keys: " + str(keys))
	index_page = StorageIndexPage()
	index_page.read(f)

if __name__ == "__main__":
	signal.signal(signal.SIGPIPE, signal.SIG_DFL)
	check_chunk_file(sys.argv[1])	<commit_msg>Added bloom page listing to Python db tool.<commit_after>import os
import sys
import struct
import signal
import string

class StoragePage:
	def __init__(self):
		self.offset = 0
		self.size = 0

class StorageHeaderPage(StoragePage):
	def __init__(self):
		self.chunk_id = 0
		self.min_log_segment_id = 0
		self.max_log_segment_id = 0
		self.max_log_command_id = 0
		self.num_keys = 0
		self.use_bloomfilter = False
		self.index_page_offset = 0
		self.index_page_size = 0
		self.bloom_page_offset = 0
		self.bloom_page_size = 0
		self.first_key = ""
		self.last_key = ""
		self.mid_point = ""

	def read(self, file):
		data = file.read(4096)
		if len(data) != 4096:
			return False
		pos = 12
		size, checksum, version = struct.unpack("<III", data[:pos])
		data = data[pos:]
		# skip text
		text = data[:64]
		data = data[64:]
		self.chunk_id, self.min_log_segment_id, self.max_log_segment_id, self.max_log_command_id = \
			struct.unpack("<QQQI", data[:28])
		data = data[28:]
		if data[0] == 'T':
			self.use_bloomfilter = True
		data = data[1:]
		self.num_keys, self.index_page_offset, self.index_page_size = struct.unpack("<QQI", data[:20])
		data = data[20:]
		if self.use_bloomfilter:
			self.bloom_page_offset, self.bloom_page_size = struct.unpack("<QI", data[:12])
			data = data[12:]

		length, = struct.unpack("<I", data[:4])
		data = data[4:]
		self.first_key = data[:length]
		data = data[length:]

		length, = struct.unpack("<I", data[:4])
		data = data[4:]
		self.last_key = data[:length]
		data = data[length:]

		length, = struct.unpack("<I", data[:4])
		data = data[4:]
		self.mid_point = data[:length]
		data = data[length:]

class StorageDataPage(StoragePage):
	def __init__(self, index):
		self.index = index
		self.size = 0
		self.offset = 0
		self.num_keys = 0
		self.keybuffer_length = 0
		self.checksum = 0
		self.key_values = {}

	def read(self, file):
		data = file.read(4096)
		self.size, self.checksum, self.keybuffer_length, self.num_keys = struct.unpack("<IIII", data[:16])
		if self.size > 4096:
			data += file.read(self.size - 4096)
		data = data[16:]
		# print("-- Page index %d, Num keys: %d" % (self.index, self.num_keys))
		i = 0
		prev_key = ""
		while i < self.num_keys:
			type, keylen = struct.unpack("<cH", data[:3])
			data = data[3:]
			key = data[:keylen]
			if key <= prev_key:
				print("Error")
			prev_key = key
			data = data[keylen:]
			# TODO: values
			# if type == 's':
			# 	vallen, = struct.unpack("<I", data[:4])
			# 	data = data[4:]
			# 	value = data[:vallen]
			# 	data = data[vallen:]
			# 	if self.key_values.has_key(key):
			# 		print("Error")
			# 		print(file.tell())
			# 	self.key_values[key] = value
			# else:
			# 	print("Delete key: " + key)
			# 	if self.key_values.has_key(key):
			# 		self.key_values.pop(key)
			# print(key, value)
			print(key)
			i += 1
		return i

class StorageIndexPage(StoragePage):
	def __init__(self):
		self.size = 0
		self.offset = 0
		self.checksum = 0
		self.num_keys = 0

	def read(self, file):
		data = file.read(4096)
		self.size, self.checksum, self.num_keys = struct.unpack("<III", data[:12])
		if self.size > 4096:
			data += file.read(self.size - 4096)
		data = data[12:]
		i = 0
		while i < self.num_keys:
			key_offset, keylen = struct.unpack("<IH", data[:6])
			data = data[6:]
			key = data[:keylen]
			data = data[keylen:]
			i += 1
			# print("Index key: %s" % (key))

class StorageBloomPage(StoragePage):
	def __init__(self):
		self.size = 0
		self.offset = 0
		self.checksum = 0

	def read(self, file):
		data = file.read(4096)
		self.size, self.checksum = struct.unpack("<II", data[:8])
		if self.size > 4096:
			data += file.read(self.size - 4096)
		data = data[8:]
		print("-- Bloom page size: %d" % (self.size))

def check_chunk_file(filename):
	f = open(filename, "rb")
	if os.path.getsize(filename) == 0:
		print("Empty chunk file")
		return
	header_page = StorageHeaderPage()
	header_page.read(f)
	print("num_keys = " + str(header_page.num_keys))
	print(str(header_page.__dict__))
	keys = 0
	index = 0
	while keys < header_page.num_keys:
		data_page = StorageDataPage(index)
		keys += data_page.read(f)
		index += 1
		#print("Keys: " + str(keys))
	index_page = StorageIndexPage()
	index_page.read(f)
	if header_page.use_bloomfilter:
		bloom_page = StorageBloomPage()
		bloom_page.read(f)

if __name__ == "__main__":
	signal.signal(signal.SIGPIPE, signal.SIG_DFL)
	check_chunk_file(sys.argv[1])	<|endoftext|>


<commit_before>import random
import urllib.parse
from datetime import datetime

import pymongo
from upol_crawler.settings import *
from upol_crawler.urls import parser, url_tools

# Global database connection
# client = pymongo.MongoClient('localhost', 27017, maxPoolSize=None)
# database = client[DATABASE_NAME]

def init(db):
    db['Urls'].create_index('visited')
    db['Urls'].create_index('queued')

def _universal_insert_url(url, collection, visited, queued, depth):
    url_object = {'_id': url_tools.hash(url),
                  'url': url,
                  'domain': url_tools.domain(url),
                  'depth': depth,
                  'visited': visited,
                  'queued': queued}

    url_object['progress'] = {}
    url_object['progress']['discovered'] = str(datetime.now())

    try:
        result = collection.insert_one(url_object).inserted_id
    except pymongo.errors.DuplicateKeyError as e:
        return False

    return result

def insert_url(db, url, visited, queued, depth):
    """Insert url into db"""
    return _universal_insert_url(url, db['Urls'], visited, queued, depth)

def delete_url(db, url):
    """Try to delete url from db, returns True if case of success"""
    result = db['Urls'].delete_one({'_id': url_tools.hash(url)})

    return result.deleted_count > 0

def exists_url(db, url):
    """Return if url is exists in db"""
    url_hash = url_tools.hash(url)

    result = db['Urls'].find_one({'_id': url_hash})

    return result is not None

# def get_unvisited_url(db):
#     """Return unvisited url from db"""
#     result = db["Urls"].find_one({'visited': False})
#
#     if result is not None:
#         return result['url'], result['depth']
#     else:
#         return None, None
#
#
# def get_random_unvisited_url(db):
#     """Return random unvisited url"""
#     result = list(db["Urls"].aggregate([{"$match": {'visited': False}}, {"$sample": {'size': 1}}]))
#     if len(result) != 0:
#         return result[0]['url'], result[0]['depth']
#     else:
#         return None, None

def get_url_for_crawl(db):
    """Return url from db which is ready for crawling - unvisited and unqueued"""
    result = db['Urls'].find_one({'$and': [
                                {'visited': False},
                                {'queued': False}
                              ]})

    if result is not None:
        return result['url'], result['depth']
    else:
        return None, None

def get_random_url_for_crawl(db):
    """Return random url from db which is ready for crawling - unvisited and unqueued"""
    result = list(db['Urls'].aggregate([{'$match':
                                        {'$and': [
                                          {'visited': False},
                                          {'queued': False}]}}, {'$sample': {'size': 1}}]))

    if len(result) != 0:
        return result[0]['url'], result[0]['depth']
    else:
        return None, None

def set_visited_url(db, url, response, html):
    """Try to set url to visited and update other important informations"""
    url_hash = url_tools.hash(url)

    url_addition = {}

    url_addition['visited'] = True
    url_addition['queued'] = False

    url_addition['progress.last_visited'] = str(datetime.now())

    url_addition['content.html'] = html
    url_addition['content.hashes.document'] = parser.hash_document(html)
    url_addition['content.encoding'] = response.encoding
    # Later detect language

    url_addition['response.elapsed'] = str(response.elapsed)
    url_addition['response.redirect'] = response.is_redirect
    url_addition['response.status_code'] = response.status_code
    url_addition['response.reason'] = response.reason

    for key, value in response.headers.items():
        url_addition['response.' + str(key)] = str(value)

    # url_addition = {'$set': {'visited': True,
    #                          'queued': False,
    #                          'progress.last_visited': str(datetime.now()),
    #                          'content.html': html,
    #                          'content.encoding': response.encoding,
    #                          'content.hashes.document': parser.hash_document(html),
    #                          'response.elapsed': str(response.elapsed),
    #                          'response.redirect': response.is_redirect,
    #                          'response.status_code': response.status_code,
    #                          'response.reason': response.reason}}

    result = db['Urls'].find_one_and_update({'_id': url_hash}, {'$set': url_addition})

    return result is not None

def set_queued_url(db, url):
    """Try to set url to queued"""
    url_hash = url_tools.hash(url)

    result = db['Urls'].find_one_and_update({'_id': url_hash}, {'$set': {'queued': True}})

    return result is not None

def set_timeout_url(db, url):
    """Try to set url as timouted"""
    url_hash = url_tools.hash(url)

    result = db['Urls'].find_one_and_update({'_id': url_hash}, {'$set': {'timeout.timeout': True, 'timeout.last_timeout': str(datetime.now())}})

    return result is not None

def is_visited_or_queued(db, url):
    """Check if url is visited"""
    result = db['Urls'].find_one({'$or': [
                                {'visited': True},
                                {'queued': True}
                              ]})

    if result is not None:
        return True

def should_crawler_wait(db):
    """Check if some url is in queue"""
    result = db['Urls'].find_one({'$or': [{'$and': [{'visited': False}, {'queued': True}]},
                                          {'$and': [{'visited': False}, {'queued': False}]}]})

    return not ((result is None) or (len(result) == 0))

def insert_crawler_start(db):
    """Save when crawler start into database"""
    result = db['CrawlerInfo'].update({'_id': 1}, {'$set': {'time.start': str(datetime.now())}}, upsert=True)

    return result is not None

def insert_crawler_end(db):
    """Save when crawler start into database"""
    result = db['CrawlerInfo'].update({'_id': 1}, {'$set': {'time.end': str(datetime.now())}}, upsert=True)

    return result is not None

def flush_db():
    """Delete everything from database"""
    return db['Urls'].drop()
<commit_msg>Set timout url as not queued to allow crawler to stop<commit_after>import random
import urllib.parse
from datetime import datetime

import pymongo
from upol_crawler.settings import *
from upol_crawler.urls import parser, url_tools

# Global database connection
# client = pymongo.MongoClient('localhost', 27017, maxPoolSize=None)
# database = client[DATABASE_NAME]

def init(db):
    db['Urls'].create_index('visited')
    db['Urls'].create_index('queued')
    db['Urls'].create_index('timeout')

def _universal_insert_url(url, collection, visited, queued, depth):
    url_object = {'_id': url_tools.hash(url),
                  'url': url,
                  'domain': url_tools.domain(url),
                  'depth': depth,
                  'visited': visited,
                  'queued': queued}

    url_object['progress'] = {}
    url_object['progress']['discovered'] = str(datetime.now())

    try:
        result = collection.insert_one(url_object).inserted_id
    except pymongo.errors.DuplicateKeyError as e:
        return False

    return result

def insert_url(db, url, visited, queued, depth):
    """Insert url into db"""
    return _universal_insert_url(url, db['Urls'], visited, queued, depth)

def delete_url(db, url):
    """Try to delete url from db, returns True if case of success"""
    result = db['Urls'].delete_one({'_id': url_tools.hash(url)})

    return result.deleted_count > 0

def exists_url(db, url):
    """Return if url is exists in db"""
    url_hash = url_tools.hash(url)

    result = db['Urls'].find_one({'_id': url_hash})

    return result is not None

# def get_unvisited_url(db):
#     """Return unvisited url from db"""
#     result = db["Urls"].find_one({'visited': False})
#
#     if result is not None:
#         return result['url'], result['depth']
#     else:
#         return None, None
#
#
# def get_random_unvisited_url(db):
#     """Return random unvisited url"""
#     result = list(db["Urls"].aggregate([{"$match": {'visited': False}}, {"$sample": {'size': 1}}]))
#     if len(result) != 0:
#         return result[0]['url'], result[0]['depth']
#     else:
#         return None, None

def get_url_for_crawl(db):
    """Return url from db which is ready for crawling - unvisited and unqueued"""
    result = db['Urls'].find_one({'$and': [
                                {'visited': False},
                                {'queued': False},
                                {'timeout': {'$exists': False}}
                              ]})

    if result is not None:
        return result['url'], result['depth']
    else:
        return None, None

def get_random_url_for_crawl(db):
    """Return random url from db which is ready for crawling - unvisited and unqueued"""
    result = list(db['Urls'].aggregate([{'$match':
                                        {'$and': [
                                          {'visited': False},
                                          {'queued': False},
                                          {'timeout': {'$exists': False}}]}}, {'$sample': {'size': 1}}]))

    if len(result) != 0:
        return result[0]['url'], result[0]['depth']
    else:
        return None, None

def set_visited_url(db, url, response, html):
    """Try to set url to visited and update other important informations"""
    url_hash = url_tools.hash(url)

    url_addition = {}

    url_addition['visited'] = True
    url_addition['queued'] = False

    url_addition['progress.last_visited'] = str(datetime.now())

    url_addition['content.html'] = html
    url_addition['content.hashes.document'] = parser.hash_document(html)
    url_addition['content.encoding'] = response.encoding
    # Later detect language

    url_addition['response.elapsed'] = str(response.elapsed)
    url_addition['response.redirect'] = response.is_redirect
    url_addition['response.status_code'] = response.status_code
    url_addition['response.reason'] = response.reason

    for key, value in response.headers.items():
        url_addition['response.' + str(key)] = str(value)

    # url_addition = {'$set': {'visited': True,
    #                          'queued': False,
    #                          'progress.last_visited': str(datetime.now()),
    #                          'content.html': html,
    #                          'content.encoding': response.encoding,
    #                          'content.hashes.document': parser.hash_document(html),
    #                          'response.elapsed': str(response.elapsed),
    #                          'response.redirect': response.is_redirect,
    #                          'response.status_code': response.status_code,
    #                          'response.reason': response.reason}}

    result = db['Urls'].find_one_and_update({'_id': url_hash}, {'$set': url_addition})

    return result is not None

def set_queued_url(db, url):
    """Try to set url to queued"""
    url_hash = url_tools.hash(url)

    result = db['Urls'].find_one_and_update({'_id': url_hash}, {'$set': {'queued': True}})

    return result is not None

def set_timeout_url(db, url):
    """Try to set url as timouted"""
    url_hash = url_tools.hash(url)

    result = db['Urls'].find_one_and_update({'_id': url_hash}, {'$set': {'queued': False,
                                                                         'timeout.timeout': True,
                                                                         'timeout.last_timeout': str(datetime.now())}})

    return result is not None

def is_visited_or_queued(db, url):
    """Check if url is visited"""
    result = db['Urls'].find_one({'$or': [
                                {'visited': True},
                                {'queued': True}
                              ]})

    if result is not None:
        return True

def should_crawler_wait(db):
    """Check if some url is in queue"""
    result = db['Urls'].find_one({'$or': [{'$and': [{'visited': False}, {'queued': True}]},
                                          {'$and': [{'visited': False}, {'queued': False}, {'timeout': {'$exists': False}}]}]})

    return not ((result is None) or (len(result) == 0))

def insert_crawler_start(db):
    """Save when crawler start into database"""
    result = db['CrawlerInfo'].update({'_id': 1}, {'$set': {'time.start': str(datetime.now())}}, upsert=True)

    return result is not None

def insert_crawler_end(db):
    """Save when crawler start into database"""
    result = db['CrawlerInfo'].update({'_id': 1}, {'$set': {'time.end': str(datetime.now())}}, upsert=True)

    return result is not None

def flush_db():
    """Delete everything from database"""
    return db['Urls'].drop()
<|endoftext|>


<commit_before># Scale Test
#
# This program is intended to describe a scaling test procedure for OpenDaylight NetVirt.
# It could be used for other network virtualization implementations as well.
#
# As is, it's mostly a simulation, but the various print statements could be replaced by
# actual calls to do the work.
#
# @author: Andre Fredette, Red Hat Inc.
#

NUM_SERVERS = 20
PORTS_PER_SERVER = 1
PORTS_PER_NETWORK = 2
NETWORKS_PER_ROUTER = 2
CONCURRENT_NETWORKS = 2
CONCURRENT_ROUTERS = 2
FLOATING_IP_PER_NUM_PORTS = 2

class Network:
    next_network_id = 0      #
    next_network_index = 0   # Index used to loop through concurrent networks.

    def __init__(self, max_ports):
        self.id = Network.next_network_id
        self.num_ports = 0
        self.router_port_added = False
        self.max_ports = max_ports
        Network.next_network_id += 1

    def re_init(self):
        Network.__init__(self, self.max_ports)

    def add_port(self, port_id):
        if self.max_ports > 0:
            if self.num_ports < self.max_ports:
                print "Adding port " + str(port_id) + " to Network " + str(self.id)
                self.num_ports += 1
            else:
                self.re_init()
                print "Creating Network " + str(self.id)
                print "Adding port " + str(port_id) + " to Network " + str(self.id)
                self.num_ports += 1

    def add_router_port(self):
        self.router_port_added = True

class Router:
    next_router_id = 0
    next_router_index = 0

    def __init__(self, max_ports):
        self.id = Router.next_router_id
        self.num_ports = 0
        self.max_ports = max_ports
        Router.next_router_id += 1

    def re_init(self):
        Router.__init__(self, self.max_ports)

    def add_port(self, network):
        if self.max_ports > 0 and network.router_port_added == False:
            if self.num_ports < self.max_ports:
                print "Adding Network " + str(network.id) + " to Router " + str(self.id)
                self.num_ports += 1
            else:
                self.re_init()
                print "Creating Router " + str(self.id)
                print "Adding Network " + str(network.id) + " to Router " + str(self.id)
                self.num_ports += 1
            network.add_router_port()

print "\nScale Test\n"

# Init Networks
networks = []
for network in range(0, CONCURRENT_NETWORKS):
    networks.append(Network(PORTS_PER_NETWORK))
    print "Creating Network " + str(networks[network].id)

next_network_index = 0

print ""

# Init Routers
routers = []
for router in range(0, CONCURRENT_ROUTERS):
    routers.append(Router(NETWORKS_PER_ROUTER))
    print "Creating Router " + str(routers[router].id)

next_router_index = 0

print ""

# Init External Network
if FLOATING_IP_PER_NUM_PORTS > 0:
    print "Creating External Network\n"

next_port_id = 0

for server in range(0, NUM_SERVERS):
    for port in range (0, PORTS_PER_SERVER):
        print "Creating port " + str(next_port_id) + " on Server " + str(server)

        # Add port to network, if necessary
        if CONCURRENT_NETWORKS > 0:
            networks[next_network_index].add_port(next_port_id)

            # Add network to router, if necessary
            if CONCURRENT_ROUTERS > 0:
                routers[next_router_index].add_port(networks[next_network_index])
                next_router_index = next_router_index % CONCURRENT_ROUTERS;

                # Create floating IP for port if necessary.
                if FLOATING_IP_PER_NUM_PORTS > 0 and next_port_id % FLOATING_IP_PER_NUM_PORTS == 0:
                    print "Creating a Floating IP for Port " + str(next_port_id)

            next_network_index = next_network_index % CONCURRENT_NETWORKS;

        next_port_id += 1
        print ""
<commit_msg>Convert main code into function.<commit_after># Scale Test
#
# This program is intended to describe a scaling test procedure for OpenDaylight NetVirt.
# It could be used for other network virtualization implementations as well.
#
# As is, it's mostly a simulation, but the various print statements could be replaced by
# actual calls to do the work.
#
# @author: Andre Fredette, Red Hat Inc.
#

NUM_SERVERS = 20
PORTS_PER_SERVER = 2
PORTS_PER_NETWORK = 2
NETWORKS_PER_ROUTER = 2
CONCURRENT_NETWORKS = 2
CONCURRENT_ROUTERS = 2
FLOATING_IP_PER_NUM_PORTS = 2

class Network:
    next_network_id = 0      #
    next_network_index = 0   # Index used to loop through concurrent networks.

    def __init__(self, max_ports):
        self.id = Network.next_network_id
        self.num_ports = 0
        self.router_port_added = False
        self.max_ports = max_ports
        Network.next_network_id += 1

    def re_init(self):
        Network.__init__(self, self.max_ports)

    def add_port(self, port_id):
        if self.max_ports > 0:
            if self.num_ports < self.max_ports:
                print "Adding port " + str(port_id) + " to Network " + str(self.id)
                self.num_ports += 1
            else:
                self.re_init()
                print "Creating Network " + str(self.id)
                print "Adding port " + str(port_id) + " to Network " + str(self.id)
                self.num_ports += 1

    def add_router_port(self):
        self.router_port_added = True

class Router:
    next_router_id = 0
    next_router_index = 0

    def __init__(self, max_ports):
        self.id = Router.next_router_id
        self.num_ports = 0
        self.max_ports = max_ports
        Router.next_router_id += 1

    def re_init(self):
        Router.__init__(self, self.max_ports)

    def add_port(self, network):
        if self.max_ports > 0 and network.router_port_added == False:
            if self.num_ports < self.max_ports:
                print "Adding Network " + str(network.id) + " to Router " + str(self.id)
                self.num_ports += 1
            else:
                self.re_init()
                print "Creating Router " + str(self.id)
                print "Adding Network " + str(network.id) + " to Router " + str(self.id)
                self.num_ports += 1
            network.add_router_port()

def run_scale_test(num_servers, ports_per_server, ports_per_network, networks_per_router,
                   concurrent_networks, concurrent_routers, floating_ip_per_num_ports):

    print "\nScale Test\n"

    # Init Networks
    networks = []
    for network in range(0, concurrent_networks):
        networks.append(Network(ports_per_network))
        print "Creating Network " + str(networks[network].id)

    next_network_index = 0

    print ""

    # Init Routers
    routers = []
    for router in range(0, concurrent_routers):
        routers.append(Router(networks_per_router))
        print "Creating Router " + str(routers[router].id)

    next_router_index = 0

    print ""

    # Init External Network
    if floating_ip_per_num_ports > 0:
        print "Creating External Network\n"

    next_port_id = 0

    for server in range(0, num_servers):
        for port in range (0, ports_per_server):
            print "Creating port " + str(next_port_id) + " on Server " + str(server)

            # Add port to network, if necessary
            if concurrent_networks > 0:
                networks[next_network_index].add_port(next_port_id)

                # Add network to router, if necessary
                if concurrent_routers > 0:
                    routers[next_router_index].add_port(networks[next_network_index])
                    next_router_index = (next_router_index +1) % concurrent_routers;

                    # Create floating IP for port if necessary.
                    if floating_ip_per_num_ports > 0 and next_port_id % floating_ip_per_num_ports == 0:
                        print "Creating a Floating IP for Port " + str(next_port_id)

                next_network_index = (next_network_index + 1) % concurrent_networks;

            next_port_id += 1
            print ""

run_scale_test(NUM_SERVERS, PORTS_PER_SERVER, PORTS_PER_NETWORK, NETWORKS_PER_ROUTER,
               CONCURRENT_NETWORKS, CONCURRENT_ROUTERS, FLOATING_IP_PER_NUM_PORTS)<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
from __future__ import unicode_literals

from django.db import models, migrations

class Migration(migrations.Migration):

    dependencies = [
        ('cms', '0001_initial'),
    ]

    operations = [
        migrations.CreateModel(
            name='GoogleMap',
            fields=[
                ('cmsplugin_ptr', models.OneToOneField(serialize=False, parent_link=True, auto_created=True, to='cms.CMSPlugin', primary_key=True)),
                ('title', models.CharField(verbose_name='map title', blank=True, null=True, max_length=100)),
                ('address', models.CharField(verbose_name='address', max_length=150)),
                ('zipcode', models.CharField(verbose_name='zip code', max_length=30)),
                ('city', models.CharField(verbose_name='city', max_length=100)),
                ('content', models.CharField(help_text='Displayed under address in the bubble.', blank=True, max_length=255, verbose_name='additional content')),
                ('zoom', models.PositiveSmallIntegerField(verbose_name='zoom level', default=13, choices=[(0, '0'), (1, '1'), (2, '2'), (3, '3'), (4, '4'), (5, '5'), (6, '6'), (7, '7'), (8, '8'), (9, '9'), (10, '10'), (11, '11'), (12, '12'), (13, '13'), (14, '14'), (15, '15'), (16, '16'), (17, '17'), (18, '18'), (19, '19'), (20, '20'), (21, '21')])),
                ('lat', models.DecimalField(help_text='Use latitude & longitude to fine tune the map position.', blank=True, max_digits=10, verbose_name='latitude', null=True, decimal_places=6)),
                ('lng', models.DecimalField(max_digits=10, verbose_name='longitude', blank=True, null=True, decimal_places=6)),
                ('route_planer_title', models.CharField(verbose_name='route planer title', blank=True, null=True, max_length=150, default='Calculate your fastest way to here')),
                ('route_planer', models.BooleanField(verbose_name='route planer', default=False)),
                ('width', models.CharField(help_text='Plugin width (in pixels or percent).', default='100%', max_length=6, verbose_name='width')),
                ('height', models.CharField(help_text='Plugin height (in pixels).', default='400px', max_length=6, verbose_name='height')),
                ('info_window', models.BooleanField(help_text='Show textbox over marker', default=True, verbose_name='info window')),
                ('scrollwheel', models.BooleanField(help_text='Enable scrollwheel zooming on the map', default=True, verbose_name='scrollwheel')),
                ('double_click_zoom', models.BooleanField(verbose_name='double click zoom', default=True)),
                ('draggable', models.BooleanField(verbose_name='draggable', default=True)),
                ('keyboard_shortcuts', models.BooleanField(verbose_name='keyboard shortcuts', default=True)),
                ('pan_control', models.BooleanField(verbose_name='Pan control', default=True)),
                ('zoom_control', models.BooleanField(verbose_name='zoom control', default=True)),
                ('street_view_control', models.BooleanField(verbose_name='Street View control', default=True)),
            ],
            options={
                'abstract': False,
            },
            bases=('cms.cmsplugin',),
        ),
    ]
<commit_msg>Change migration dependency<commit_after># -*- coding: utf-8 -*-
from __future__ import unicode_literals

from django.db import models, migrations

class Migration(migrations.Migration):

    dependencies = [
        ('cms', '__first__'),
    ]

    operations = [
        migrations.CreateModel(
            name='GoogleMap',
            fields=[
                ('cmsplugin_ptr', models.OneToOneField(serialize=False, parent_link=True, auto_created=True, to='cms.CMSPlugin', primary_key=True)),
                ('title', models.CharField(verbose_name='map title', blank=True, null=True, max_length=100)),
                ('address', models.CharField(verbose_name='address', max_length=150)),
                ('zipcode', models.CharField(verbose_name='zip code', max_length=30)),
                ('city', models.CharField(verbose_name='city', max_length=100)),
                ('content', models.CharField(help_text='Displayed under address in the bubble.', blank=True, max_length=255, verbose_name='additional content')),
                ('zoom', models.PositiveSmallIntegerField(verbose_name='zoom level', default=13, choices=[(0, '0'), (1, '1'), (2, '2'), (3, '3'), (4, '4'), (5, '5'), (6, '6'), (7, '7'), (8, '8'), (9, '9'), (10, '10'), (11, '11'), (12, '12'), (13, '13'), (14, '14'), (15, '15'), (16, '16'), (17, '17'), (18, '18'), (19, '19'), (20, '20'), (21, '21')])),
                ('lat', models.DecimalField(help_text='Use latitude & longitude to fine tune the map position.', blank=True, max_digits=10, verbose_name='latitude', null=True, decimal_places=6)),
                ('lng', models.DecimalField(max_digits=10, verbose_name='longitude', blank=True, null=True, decimal_places=6)),
                ('route_planer_title', models.CharField(verbose_name='route planer title', blank=True, null=True, max_length=150, default='Calculate your fastest way to here')),
                ('route_planer', models.BooleanField(verbose_name='route planer', default=False)),
                ('width', models.CharField(help_text='Plugin width (in pixels or percent).', default='100%', max_length=6, verbose_name='width')),
                ('height', models.CharField(help_text='Plugin height (in pixels).', default='400px', max_length=6, verbose_name='height')),
                ('info_window', models.BooleanField(help_text='Show textbox over marker', default=True, verbose_name='info window')),
                ('scrollwheel', models.BooleanField(help_text='Enable scrollwheel zooming on the map', default=True, verbose_name='scrollwheel')),
                ('double_click_zoom', models.BooleanField(verbose_name='double click zoom', default=True)),
                ('draggable', models.BooleanField(verbose_name='draggable', default=True)),
                ('keyboard_shortcuts', models.BooleanField(verbose_name='keyboard shortcuts', default=True)),
                ('pan_control', models.BooleanField(verbose_name='Pan control', default=True)),
                ('zoom_control', models.BooleanField(verbose_name='zoom control', default=True)),
                ('street_view_control', models.BooleanField(verbose_name='Street View control', default=True)),
            ],
            options={
                'abstract': False,
            },
            bases=('cms.cmsplugin',),
        ),
    ]
<|endoftext|>


<commit_before>import sys

# Add Python path two levels up
sys.path.append('../..')

from Workflow import *

#############################################################################
def main():
    Pipeline.SetCacheDirectory('/home/cory/tmp')

    # Set up a pipeline
    p = Pipeline()

    # Set up some workflow steps
    c1 = CommandLineExecutablePipelineStep('FirstStep')
    c1.ExecutableName = '/usr/bin/python'
    c1.Arguments = ['script1.py', 'script1-input.txt', 'script1-output.txt']
    c1.InputFiles = ['script1.py', 'script1-input.txt']
    c1.OutputFiles = ['script1-output.txt']
    p.AddStep(c1)

    c2 = CommandLineExecutablePipelineStep('SecondStep')
    c2.ExecutableName = '/usr/bin/python'
    c2.Arguments = ['script2.py', 'script1-output.txt']
    c2.InputFiles = c2.Arguments
    p.AddStep(c2)

    # Optional argument to force execution of all pipeline steps
    try:
        sys.argv.index('--force-execute')
        p.ClearCache()
    except:
        pass

    # Execute the pipeline
    p.Execute()

#############################################################################
if __name__ == '__main__':
    main()
<commit_msg>Added --verbose option<commit_after>import sys

# Add Python path two levels up
sys.path.append('../..')

from Workflow import *

#############################################################################
def main():
    Pipeline.SetCacheDirectory('/home/cory/tmp')

    # Set up a pipeline
    p = Pipeline()

    # Set up some workflow steps
    c1 = CommandLineExecutablePipelineStep('FirstStep')
    c1.ExecutableName = '/usr/bin/python'
    c1.Arguments = ['script1.py', 'script1-input.txt', 'script1-output.txt']
    c1.InputFiles = ['script1.py', 'script1-input.txt']
    c1.OutputFiles = ['script1-output.txt']
    p.AddStep(c1)

    c2 = CommandLineExecutablePipelineStep('SecondStep')
    c2.ExecutableName = '/usr/bin/python'
    c2.Arguments = ['script2.py', 'script1-output.txt']
    c2.InputFiles = c2.Arguments
    p.AddStep(c2)

    # Optional argument to force execution of all pipeline steps
    if ('--force-execute' in sys.argv):
        p.ClearCache()

    # Optional argument for verbose execution
    verbose = '--verbose' in sys.argv

    # Execute the pipeline
    p.Execute(verbose=verbose)

#############################################################################
if __name__ == '__main__':
    main()
<|endoftext|>


<commit_before>#!/usr/bin/python
# -*- coding: utf-8 -*-

'''
1876. Centipede's Morning
Time limit: 0.5 second
Memory limit: 64 MB

[Description]
A centipede has 40 left feet and 40 right feet. It keeps a left slippers and b
right slippers under its bed. Every morning the centipede puts on the slippers.
It pokes its first left foot under the bed and puts on a random slipper, doing
it in one second. If the slipper is left, the centipede passes to shoeing the
second left foot. Otherwise, it takes off the slipper and puts it on any unshod
right foot, spending one more second, so that it takes two seconds altogether
to put on such a slipper. If there are no unshod right feet, the centipede
throws the slipper to a corner of the room, also in one second, so that two
seconds are spent altogether for this slipper. The process is continued until
all the left feet are in left slippers. Then the centipede starts shoeing its
right feet until all of them are shod.
Today the centipede has got out of bed on the wrong side, so it is preparing
for the worst. How many seconds will it need for shoeing?

[Input]
The only line contains the integers a and b (40 ≤ a, b ≤ 100).

[Output]
Output the number of seconds the centipede will need for shoeing in the worst
case.
'''

import sys;
import math;

def calc():
    a, b = sys.stdin.readline().strip('\r\n').split(' ')
    a = int(a)
    b = int(b)
    t = b * 2 + 40
    print t

if __name__ == '__main__':
    calc()

<commit_msg>Accepted 1876 (0.031	348 KB)<commit_after>#!/usr/bin/python
# -*- coding: utf-8 -*-

'''
1876. Centipede's Morning
Time limit: 0.5 second
Memory limit: 64 MB

[Description]
A centipede has 40 left feet and 40 right feet. It keeps a left slippers and b
right slippers under its bed. Every morning the centipede puts on the slippers.
It pokes its first left foot under the bed and puts on a random slipper, doing
it in one second. If the slipper is left, the centipede passes to shoeing the
second left foot. Otherwise, it takes off the slipper and puts it on any unshod
right foot, spending one more second, so that it takes two seconds altogether
to put on such a slipper. If there are no unshod right feet, the centipede
throws the slipper to a corner of the room, also in one second, so that two
seconds are spent altogether for this slipper. The process is continued until
all the left feet are in left slippers. Then the centipede starts shoeing its
right feet until all of them are shod.
Today the centipede has got out of bed on the wrong side, so it is preparing
for the worst. How many seconds will it need for shoeing?

[Input]
The only line contains the integers a and b (40 ≤ a, b ≤ 100).

[Output]
Output the number of seconds the centipede will need for shoeing in the worst
case.
'''

import sys;
import math;

def calc():
    a, b = sys.stdin.readline().strip('\r\n').split(' ')
    a = int(a)
    b = int(b)
    t = max(b * 2 + 40, 39 * 2 + 40 + (a - 40) * 2 + 1)
    print t

if __name__ == '__main__':
    calc()

<|endoftext|>


<commit_before>#!/usr/bin/env python
import os
import sys
import shutil
from glob import glob
try:
    from lib2to3.main import main
except ImportError:
    print('Error: you need the 2to3 tool to run this script.')
os.chdir(os.path.join(os.path.dirname(__file__), '..'))
try:
    os.unlink('src/version.py')
except OSError:
    pass
try:
    shutil.rmtree('py3k')
except OSError:
    pass
os.mkdir('py3k')
for dirname in ('locales', 'docs', 'plugins', 'src', 'test'):
    shutil.copytree(dirname, os.path.join('py3k', dirname))
for filename in ('setup.py',):
    shutil.copyfile(filename, os.path.join('py3k', filename))
os.chdir('py3k')

files = ['run.py', 'src', 'plugins', 'test', 'setup.py'] + glob('scripts/*')
args = ['-wn']
fixers = []
for fix in ['all', 'def_iteritems', 'def_itervalues', 'def_iterkeys', 'reload']:
    fixers += ['-f', fix]
sys.argv = files + args + fixers + sys.argv
sys.argc = len(sys.argv)

import fix_def_iteritems, fix_def_itervalues, fix_def_iterkeys, fix_reload

# Hacks
sys.modules['lib2to3.fixes.fix_def_iteritems'] = fix_def_iteritems
sys.modules['lib2to3.fixes.fix_def_itervalues'] = fix_def_itervalues
sys.modules['lib2to3.fixes.fix_def_iterkeys'] = fix_def_iterkeys
sys.modules['lib2to3.fixes.fix_reload'] = fix_reload

sys.exit(main("lib2to3.fixes"))
<commit_msg>Fix previous commit.<commit_after>#!/usr/bin/env python
import os
import sys
import shutil
from glob import glob
try:
    from lib2to3.main import main
except ImportError:
    print('Error: you need the 2to3 tool to run this script.')
os.chdir(os.path.join(os.path.dirname(__file__), '..'))
try:
    os.unlink('src/version.py')
except OSError:
    pass
try:
    shutil.rmtree('py3k')
except OSError:
    pass
os.mkdir('py3k')
for dirname in ('locales', 'docs', 'plugins', 'src', 'test', 'scripts'):
    shutil.copytree(dirname, os.path.join('py3k', dirname))
for filename in ('setup.py',):
    shutil.copyfile(filename, os.path.join('py3k', filename))
os.chdir('py3k')

files = ['run.py', 'src', 'plugins', 'test', 'setup.py'] + glob('scripts/*')
args = ['-wn']
fixers = []
for fix in ['all', 'def_iteritems', 'def_itervalues', 'def_iterkeys', 'reload']:
    fixers += ['-f', fix]
sys.argv = files + args + fixers + sys.argv
sys.argc = len(sys.argv)

import fix_def_iteritems, fix_def_itervalues, fix_def_iterkeys, fix_reload

# Hacks
sys.modules['lib2to3.fixes.fix_def_iteritems'] = fix_def_iteritems
sys.modules['lib2to3.fixes.fix_def_itervalues'] = fix_def_itervalues
sys.modules['lib2to3.fixes.fix_def_iterkeys'] = fix_def_iterkeys
sys.modules['lib2to3.fixes.fix_reload'] = fix_reload

sys.exit(main("lib2to3.fixes"))
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
import base64
print(base64.b64encode(b'binary\x00string'))
print(base64.b64decode(b'YmluYXJ5AHN0cmluZw=='))
#这是加密字符串，和解密字符串，前面b'代表Byte，即以byte形式识别字符串
#由于标准的Base64编码后可能出现字符+和/，在URL中就不能直接作为参数，所以又有一种"url safe"的base64编码，其实就是把字符+和/分别变成-和_：
print(base64.b64encode(b'i\xb7\x1d\xfb\xef\xff'))
#转换的内容含有斜杠，不能再URL当中使用
print(base64.urlsafe_b64encode(b'i\xb7\x1d\xfb\xef\xff'))
#可以在URL当中加密内容，比如中文
print(base64.urlsafe_b64decode(b'abcd--__'))
<commit_msg>20170317<commit_after># -*- coding: utf-8 -*-
import base64
print(base64.b64encode(b'binary\x00string'))
print(base64.b64decode(b'YmluYXJ5AHN0cmluZw=='))
#这是加密字符串，和解密字符串，前面b'代表Byte，即以byte形式识别字符串
#由于标准的Base64编码后可能出现字符+和/，在URL中就不能直接作为参数，所以又有一种"url safe"的base64编码，其实就是把字符+和/分别变成-和_：
print(base64.b64encode(b'i\xb7\x1d\xfb\xef\xff'))
#转换的内容含有斜杠，不能再URL当中使用
print(base64.urlsafe_b64encode(b'i\xb7\x1d\xfb\xef\xff'))
#可以在URL当中加密内容，比如中文
print(base64.urlsafe_b64decode(b'abcd--__'))
print(base64.urlsafe_b64encode(b'zhongqi.zhang'))
<|endoftext|>


<commit_before># coding=utf-8
"""
Automation server
=================

Right now, just import this file from your main.py, if you want to use the automateclient.py.
No configuration needed.
"""

import socket
import struct
import threading
import json
import base64
import os
from time import time

COMMAND_SIZE = struct.calcsize("I")
COMMANDS = {}

def command(f):
    COMMANDS[f.__name__] = f

@command
def screenshot(*args):
    """Do a screenshot of the window, and return it as a PNG:
    (width, height, "RGBA", base64-data)
    """
    from kivy.clock import mainthread
    _result = []
    _event = threading.Event()

    @mainthread
    def _real_screenshot():
        import tempfile
        from kivy.core.window import Window
        try:
            fd = tempfile.NamedTemporaryFile(suffix=".png", delete=False)
            # XXX hack, Window.screenshot doesn't respect our filename
            screenshot_fn = fd.name.rsplit(".")[-2] + "0001.png"
            fd.close()
            width, height = Window.size
            Window.screenshot(name=fd.name)
            with open(screenshot_fn, "rb") as fd:
                data = fd.read()
            _result[:] = [width, height, base64.encodestring(data)]
        except:
            import traceback
            traceback.print_exc()
        finally:
            try:
                os.unlink(screenshot_fn)
                os.unlink(fd.name)
            except:
                pass
            _event.set()

    _real_screenshot()
    _event.wait()
    return _result

@command
def execute(cmd):
    """Execute a random string in the app context
    """
    from kivy.clock import mainthread
    _result = [None]
    _event = threading.Event()

    @mainthread
    def _real_execute():
        from kivy.app import App
        app = App.get_running_app()
        idmap = {"app": app}
        try:
            exec cmd in idmap, idmap
        except Exception as e:
            _result[:] = [u"{}".format(e)]
        _event.set()

    _real_execute()
    _event.wait()
    return _result[0]

@command
def wait(cond, timeout=-1):
    """Wait a condition unless timeout happen
    """
    from kivy.app import App
    from kivy.clock import Clock
    _result = [None]
    _event = threading.Event()
    _start = time()

    def _real_wait(*args):
        try:
            app = App.get_running_app()
            idmap = {"app": app}
            if eval(cond, idmap):
                _result[:] = [True]
                _event.set()
                return False
        except:
            pass

        if timeout == -1:
            return

        if time() - _start > timeout:
            _result[:] = [False]
            _event.set()
            return False

    Clock.schedule_interval(_real_wait, .1)
    _event_timeout = None if timeout == -1 else timeout
    try:
        _event.wait(_event_timeout)
    except:
        pass
    Clock.unschedule(_real_wait)
    return _result[0]

@command
def ping():
    return True

@command
def quit():
    from kivy.app import App
    from kivy.clock import Clock

    def _real_quit(*args):
        App.get_running_app().stop()

    Clock.schedule_once(_real_quit, 0)
    return True

def recvall(sock, size):
    msg = ""
    while len(msg) < size:
        print size, len(msg)
        packet = sock.recv(size - len(msg))
        if not packet:
            return None
        msg += packet
    return msg

def run_automate_socket():
    netloc = ("0.0.0.0", 7777)
    servsock = socket.socket()
    servsock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, True)
    servsock.bind(netloc)
    servsock.listen(5)

    while True:
        sock, _ = servsock.accept()

        # size of the command
        try:
            size = struct.unpack("I", recvall(sock, struct.calcsize("I")))[0]
            data = recvall(sock, size)
            args = json.loads(data)

            cmd, args = args[0], args[1:]
            result = COMMANDS[cmd](*args)
            data = json.dumps(result)

            sock.sendall(struct.pack("I", len(data)))
            sock.sendall(data)

        except:
            import traceback
            traceback.print_exc()
        finally:
            sock.close()

automate_thread = threading.Thread(target=run_automate_socket)
automate_thread.daemon = True
automate_thread.start()
<commit_msg>fix screenshot issue for iOS<commit_after># coding=utf-8
"""
Automation server
=================

Right now, just import this file from your main.py, if you want to use the automateclient.py.
No configuration needed.
"""

import socket
import struct
import threading
import json
import base64
import os
from time import time

COMMAND_SIZE = struct.calcsize("I")
COMMANDS = {}

def command(f):
    COMMANDS[f.__name__] = f

@command
def screenshot(*args):
    """Do a screenshot of the window, and return it as a PNG:
    (width, height, "RGBA", base64-data)
    """
    from kivy.clock import mainthread
    _result = []
    _event = threading.Event()

    @mainthread
    def _real_screenshot():
        import tempfile
        from kivy.core.window import Window
        from kivy.utils import platform
        try:
            fd = tempfile.NamedTemporaryFile(suffix=".png", delete=False)
            # XXX hack, Window.screenshot doesn't respect our filename
            screenshot_fn = fd.name.rsplit(".")[-2] + "0001.png"
            fd.close()

            if platform == "ios":
                from kivy.graphics.opengl import glReadPixels, GL_RGBA, GL_UNSIGNED_BYTE
                from kivy.core.image import ImageLoader
                # hacky, as Window don't have correct system size (why, i don't
                # know.)
                width, height = Window._win._get_gl_size()
                Window.dispatch("on_draw")
                data = glReadPixels(0, 0, width, height, GL_RGBA, GL_UNSIGNED_BYTE)
                # save using standard imageloader, but in rgba
                loader = [x for x in ImageLoader.loaders if x.can_save()][0]
                loader.save(screenshot_fn, width, height, "rgba", data, True)
                """
                # strides issue
                data = list(data)
                del data[3::4]
                data = "".join(data)
                Window._win.save_bytes_in_png(screenshot_fn, data, width,
                        height)
                """
            else:
                Window.screenshot(name=fd.name)
            with open(screenshot_fn, "rb") as fd:
                data = fd.read()
            _result[:] = [width, height, base64.encodestring(data)]
        except:
            import traceback
            traceback.print_exc()
        finally:
            try:
                os.unlink(screenshot_fn)
                os.unlink(fd.name)
            except:
                pass
            _event.set()

    _real_screenshot()
    _event.wait()
    return _result

@command
def execute(cmd):
    """Execute a random string in the app context
    """
    from kivy.clock import mainthread
    _result = [None]
    _event = threading.Event()

    @mainthread
    def _real_execute():
        from kivy.app import App
        app = App.get_running_app()
        idmap = {"app": app}
        try:
            exec cmd in idmap, idmap
        except Exception as e:
            _result[:] = [u"{}".format(e)]
        _event.set()

    _real_execute()
    _event.wait()
    return _result[0]

@command
def wait(cond, timeout=-1):
    """Wait a condition unless timeout happen
    """
    from kivy.app import App
    from kivy.clock import Clock
    _result = [None]
    _event = threading.Event()
    _start = time()

    def _real_wait(*args):
        try:
            app = App.get_running_app()
            idmap = {"app": app}
            if eval(cond, idmap):
                _result[:] = [True]
                _event.set()
                return False
        except:
            pass

        if timeout == -1:
            return

        if time() - _start > timeout:
            _result[:] = [False]
            _event.set()
            return False

    Clock.schedule_interval(_real_wait, .1)
    _event_timeout = None if timeout == -1 else timeout
    try:
        _event.wait(_event_timeout)
    except:
        pass
    Clock.unschedule(_real_wait)
    return _result[0]

@command
def ping():
    return True

@command
def quit():
    from kivy.app import App
    from kivy.clock import Clock

    def _real_quit(*args):
        App.get_running_app().stop()

    Clock.schedule_once(_real_quit, 0)
    return True

def recvall(sock, size):
    msg = ""
    while len(msg) < size:
        print size, len(msg)
        packet = sock.recv(size - len(msg))
        if not packet:
            return None
        msg += packet
    return msg

def run_automate_socket():
    netloc = ("0.0.0.0", 7777)
    servsock = socket.socket()
    servsock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, True)
    servsock.bind(netloc)
    servsock.listen(5)

    while True:
        sock, _ = servsock.accept()

        # size of the command
        try:
            size = struct.unpack("I", recvall(sock, struct.calcsize("I")))[0]
            data = recvall(sock, size)
            args = json.loads(data)

            cmd, args = args[0], args[1:]
            result = COMMANDS[cmd](*args)
            data = json.dumps(result)

            sock.sendall(struct.pack("I", len(data)))
            sock.sendall(data)

        except:
            import traceback
            traceback.print_exc()
        finally:
            sock.close()

automate_thread = threading.Thread(target=run_automate_socket)
automate_thread.daemon = True
automate_thread.start()
<|endoftext|>