<commit_before>
import json
import terms as v
from Links import Links

class HypermediaResource(object) :

    def __init__(self):

        self._contentFormats = {}
        LinkFormatHandler(self)
        LinkFormatPatchHandler(self)
        PlainTextHandler(self)
        SenmlValueHandler(self)

        self._value = {v._v:None}
        self._linkArray = Links({v._href: "", v._rel: v._self, v._rt: "resource"})

    def registerContentHandler(self, contentFormat, handler):
        self._contentFormats[contentFormat] = handler

    def handleRequest(self, request):
        if request[v.contentFormat] in self._contentFormats:
            self._contentFormats[request[v.contentFormat]](request)
        else:
            request[v.response][v.status] = v.UnsupportedType

class LinkFormatHandler:

    def __init__(self, resource):
        self._resource = resource
        self._resource.registerContentHandler(v.linkFormatJsonType, self._handleRequest)

    def _handleRequest(self, request):
        if request[v.method] == v.get:
            request[v.response][v.payload] = json.dumps(self._resource._linkArray.get(request[v.uriQuery]))
            request[v.response][v.status] = v.Success
        elif request[v.method] == v.post:
            self._resource._linkArray.add(json.loads(request[v.payload]))
            request[v.response][v.status] = v.Success
        else:
            request[v.response][v.status] = v.MethodNotAllowed

class LinkFormatPatchHandler:

    def __init__(self, resource):
        self._resource = resource
        self._resource.registerContentHandler(v.linkFormatMergeType, self._handleRequest)

    def _handleRequest(self, request):
        if request[v.method] == v.patch:
            self._resource._linkArray.selectMerge( request[v.uriQuery], json.loads(request[v.payload]) )
            request[v.response][v.status] = v.Success
        else:
            request[v.response][v.status] = v.MethodNotAllowed

class PlainTextHandler:

    def __init__(self, resource):
        self._resource = resource
        self._resource.registerContentHandler(v.plainTextType, self._handleRequest)

    def _handleRequest(self, request):
        if request[v.method] == v.get:
            for key in self._resource._value:
                if key in (v._v, v._bv, v._sv, v._ov):
                    request[v.response][v.payload] = json.dumps(self._resource._value[key])
            request[v.response][v.status] = v.Success

        elif request[v.method] == v.put:
            self._newValue = (json.loads(request[v.payload]))
            if isinstance(self._newValue, str) or isinstance(self._newValue, unicode):
                self._resource._value[v._sv] = self._newValue
            elif isinstance(self._newValue, bool):
                self._resource._value[v._bv] = self._newValue
            elif isinstance(self._newValue, list) or isinstance(self._newValue, dict):
                self._resource._value[v._ov] = self._newValue
            else:
                self._resource._value[v._v] = self._newValue
            request[v.response][v.status] = v.Success
        else:
            request[v.response][v.status] = v.MethodNotAllowed

class SenmlValueHandler:

    def __init__(self, resource):
        self._resource = resource
        self._resource.registerContentHandler(v.senmlType, self._handleRequest)

    def _handleRequest(self, request):
        if request[v.method] == v.get:
            request[v.response][v.payload] = json.dumps(self._resource._value)
            request[v.response][v.status] = v.Success
        elif request[v.method] == v.put:
            self._resource._value = json.loads(request[v.payload])
            request[v.response][v.status] = v.Success
        else:
            request[v.response][v.status] = v.MethodNotAllowed

<commit_msg>content handler<commit_after>
import json
import terms as v
from Links import Links

class HypermediaResource(object) :

    def __init__(self):

        self._contentFormats = {}
        LinkFormatHandler(self)
        LinkFormatPatchHandler(self)
        PlainTextHandler(self)
        SenmlValueHandler(self)

        self._value = {v._v:None}
        self._linkArray = Links({v._href: "", v._rel: v._self, v._rt: "resource"})

    def registerContentHandler(self, contentFormat, handler):
        self._contentFormats[contentFormat] = handler

    def handleRequest(self, request):
        if request[v.contentFormat] in self._contentFormats:
            self._contentFormats[request[v.contentFormat]](request)
        else:
            request[v.response][v.status] = v.UnsupportedType

class ContentHandler:

    _contentFormat = None

    def __init__(self,resource):
        self._resource = resource
        self._resource.registerContentHandler(self._contentFormat, self._handleRequest)

    def _handleRequest(self, request):
        self._preProcess(request)
        self._processRequest(request)
        self._postProcess(request)
        return

    def _preProcess(self, request):
        pass

    def _processRequest(self, request):
        request[v.response][v.status] = v.ServerError

    def _postProcess(self, request):
        pass

class LinkFormatHandler(ContentHandler):

    _contentFormat = v.linkFormatJsonType

    def _processRequest(self, request):
        if request[v.method] == v.get:
            request[v.response][v.payload] = json.dumps(self._resource._linkArray.get(request[v.uriQuery]))
            request[v.response][v.status] = v.Success
        elif request[v.method] == v.post:
            self._resource._linkArray.add(json.loads(request[v.payload]))
            request[v.response][v.status] = v.Success
        else:
            request[v.response][v.status] = v.MethodNotAllowed

class LinkFormatPatchHandler(ContentHandler):

    _contentFormat = v.linkFormatMergeType

    def _processRequest(self, request):
        if request[v.method] == v.patch:
            self._resource._linkArray.selectMerge( request[v.uriQuery], json.loads(request[v.payload]) )
            request[v.response][v.status] = v.Success
        else:
            request[v.response][v.status] = v.MethodNotAllowed

class PlainTextHandler(ContentHandler):

    _contentFormat = v.plainTextType

    def _processRequest(self, request):
        if request[v.method] == v.get:
            for key in self._resource._value:
                if key in (v._v, v._bv, v._sv, v._ov):
                    request[v.response][v.payload] = json.dumps(self._resource._value[key])
            request[v.response][v.status] = v.Success

        elif request[v.method] == v.put:
            self._newValue = (json.loads(request[v.payload]))
            if isinstance(self._newValue, str) or isinstance(self._newValue, unicode):
                self._resource._value[v._sv] = self._newValue
            elif isinstance(self._newValue, bool):
                self._resource._value[v._bv] = self._newValue
            elif isinstance(self._newValue, list) or isinstance(self._newValue, dict):
                self._resource._value[v._ov] = self._newValue
            else:
                self._resource._value[v._v] = self._newValue
            request[v.response][v.status] = v.Success
        else:
            request[v.response][v.status] = v.MethodNotAllowed

class SenmlValueHandler(ContentHandler):

    _contentFormat = v.senmlType

    def _processRequest(self, request):
        if request[v.method] == v.get:
            request[v.response][v.payload] = json.dumps(self._resource._value)
            request[v.response][v.status] = v.Success
        elif request[v.method] == v.put:
            self._resource._value = json.loads(request[v.payload])
            request[v.response][v.status] = v.Success
        else:
            request[v.response][v.status] = v.MethodNotAllowed

<|endoftext|>


<commit_before>import argparse

from mosfit.fitter import Fitter

def main():
    """First, parse command line arguments.
    """

    parser = argparse.ArgumentParser(
        prog='MOSFiT',
        description='Fit astrophysical light curves using AstroCats data.')

    parser.add_argument(
        '--events',
        '-e',
        dest='events',
        default=['mosfit/tests/SN2006le.json'],
        nargs='+')

    parser.add_argument(
        '--models', '-m', dest='models', default=['default'], nargs='+')

    parser.add_argument(
        '--parameter-paths',
        '-P',
        dest='parameter_paths',
        default=[''],
        nargs='+')

    parser.add_argument('--plot-points', dest='plot_points', default=100)

    parser.add_argument(
        '--iterations', '-i', dest='iterations', type=int, default=10)

    parser.add_argument(
        '--num-walkers', '-N', dest='num_walkers', type=int, default=100)

    parser.add_argument(
        '--num-temps', '-T', dest='num_temps', type=int, default=2)

    parser.add_argument(
        '--no-fracking', dest='fracking', default=True, action='store_false')

    parser.add_argument(
        '--frack-step', '-f', dest='frack_step', type=int, default=100)

    parser.add_argument(
        '--post-burn', '-p', dest='post_burn', type=int, default=500)

    parser.add_argument(
        '--travis', dest='travis', default=False, action='store_true')

    args = parser.parse_args()
    """Then, fit the listed events with the listed models.
    """
    Fitter.fit_events(args.events, args.models, args.plot_points,
                      args.iterations, args.num_walkers, args.num_temps,
                      args.parameter_paths, args.fracking, args.frack_step,
                      args.travis, args.post_burn)

if __name__ == "__main__":
    main()
<commit_msg>ENH: Added help messages to most command line arguments<commit_after>import argparse

from mosfit.fitter import Fitter

def main():
    """First, parse command line arguments.
    """

    parser = argparse.ArgumentParser(
        prog='MOSFiT',
        description='Fit astrophysical light curves using AstroCats data.')

    parser.add_argument(
        '--events',
        '-e',
        dest='events',
        default=['mosfit/tests/SN2006le.json'],
        nargs='+',
        help=("List of event names to be fit, delimited by spaces. If an "
              "event name contains a space, enclose the event's name in "
              "double quote marks, e.g. \"SDSS-II SN 5944\"."))

    parser.add_argument(
        '--models',
        '-m',
        dest='models',
        default=['default'],
        nargs='+',
        help=("List of models to use to fit against the listed events. The "
              "model can either be a name of a model included with MOSFiT, or "
              "a path to a custom model JSON file generated by the user."))

    parser.add_argument(
        '--parameter-paths',
        '-P',
        dest='parameter_paths',
        default=[''],
        nargs='+',
        help=("Paths to parameter files corresponding to each model file; "
              "length of this list should be equal to the length of the list "
              "of models"))

    parser.add_argument('--plot-points', dest='plot_points', default=100)

    parser.add_argument(
        '--iterations', '-i', dest='iterations', type=int, default=1000,
        help=("Number of iterations to run emcee for, including burn-in and "
              "post-burn iterations."))

    parser.add_argument(
        '--num-walkers', '-N', dest='num_walkers', type=int, default=50,
        help=("Number of walkers to use in emcee, must be at least twice the "
              "total number of free parameters within the model."))

    parser.add_argument(
        '--num-temps', '-T', dest='num_temps', type=int, default=2,
        help=("Number of temperatures to use in the parallel-tempered emcee "
              "sampler. `-T 1` is equivalent to the standard "
              "EnsembleSampler."))

    parser.add_argument(
        '--no-fracking', dest='fracking', default=True, action='store_false',
        help=("Setting this flag will skip the `fracking` step of the "
              "optimization process."))

    parser.add_argument(
        '--frack-step', '-f', dest='frack_step', type=int, default=20,
        help=("Perform `fracking` every this number of steps while in the "
              "burn-in phase of the fitting process."))

    parser.add_argument(
        '--post-burn', '-p', dest='post_burn', type=int, default=500,
        help=("Run emcee this many more iterations after the burn-in phase. "
              "The burn-in phase will thus be run for (i - p) iterations, "
              "where i is the total number of iterations set with `-i` and "
              "p is the value of this parameter."))

    parser.add_argument(
        '--travis', dest='travis', default=False, action='store_true',
        help=("Alters the printing of output messages such that a new line is "
              "generated with each message. Users are unlikely to need this "
              "parameter; it is included as Travis requires new lines to be "
              "produed to detected program output."))

    args = parser.parse_args()
    """Then, fit the listed events with the listed models.
    """
    Fitter.fit_events(args.events, args.models, args.plot_points,
                      args.iterations, args.num_walkers, args.num_temps,
                      args.parameter_paths, args.fracking, args.frack_step,
                      args.travis, args.post_burn)

if __name__ == "__main__":
    main()
<|endoftext|>


<commit_before># Django settings for lot project.
from madrona.common.default_settings import *

BASE_DIR = os.path.dirname(os.path.abspath(__file__))
TIME_ZONE = 'America/Vancouver'
ROOT_URLCONF = 'urls'
LOGIN_REDIRECT_URL = '/visualize'

DATABASES = {
    'default': {
        'ENGINE': 'django.contrib.gis.db.backends.postgis',
        'NAME': 'marco',
        'USER': 'vagrant',
    }
}

LOG_FILE =  os.path.realpath(os.path.join(os.path.dirname(__file__), '..', 'mp.log'))

INSTALLED_APPS += (
                    'django_extensions',

                    'general',
                    'data_manager',
                    'mp_settings',
                    'explore',
                    'visualize',
                    'django.contrib.humanize',
                    'flatblocks',
                    'mp_proxy',
                    'map_proxy'
                  )

GEOMETRY_DB_SRID = 99996
GEOMETRY_CLIENT_SRID = 3857 #for latlon
GEOJSON_SRID = 3857

APP_NAME = "Marine Planner Data Portal"
SERVER_ADMIN = 'edwin@pointnineseven.com'
DEFAULT_FROM_EMAIL = 'USVI <developers@pointnineseven.com>'
SERVER_EMAIL = DEFAULT_FROM_EMAIL
MANAGERS = ADMINS
EMAIL_SUBJECT_PREFIX = 'Marine Planner'
ADMINS = (
    ('Edwin Knuth', 'edwin@pointnineseven.com'),
    ('Scott Fletcher', 'scott@pointnineseven.com'),
)

#FEEDBACK_RECIPIENT = "Marine Planning Team <mp-team@marineplanner.org>"
#HELP_EMAIL = "mp-team@marineplanner.org"
#DEFAULT_FROM_EMAIL = "Marine Planning Team <mp-team@marineplanner.org>"

# url for socket.io printing
#SOCKET_URL = 'http://dev.marco.marineplanning.org:8080'
SOCKET_URL = False

# Change the following line to True, to display the 'under maintenance' template
UNDER_MAINTENANCE_TEMPLATE = False

TEMPLATE_DIRS = ( os.path.realpath(os.path.join(os.path.dirname(__file__), 'templates').replace('\\','/')), )

MIDDLEWARE_CLASSES += (
    'django.middleware.common.CommonMiddleware',
    'django.contrib.sessions.middleware.SessionMiddleware',
    'django.middleware.csrf.CsrfViewMiddleware',
    'django.contrib.auth.middleware.AuthenticationMiddleware',
    'django.contrib.messages.middleware.MessageMiddleware',
    'django.middleware.gzip.GZipMiddleware'

    # Uncomment the next line for simple clickjacking protection:
    # 'django.middleware.clickjacking.XFrameOptionsMiddleware',
)
LOGGING = {
    'version': 1,
    'disable_existing_loggers': False,
    'formatters': {
        'main_formatter': {
            'format': '%(levelname)s:%(name)s: %(message)s '
                     '(%(asctime)s; %(filename)s:%(lineno)d)',
            'datefmt': "%Y-%m-%d %H:%M:%S",
        },
    },
    'handlers': {
        'mail_admins': {
            'level': 'ERROR',
            'filters': ['require_debug_false'],
            'class': 'django.utils.log.AdminEmailHandler'
        },
        'console':{
            'level': 'DEBUG',
            # 'filters': ['require_debug_true'],
            'class': 'logging.StreamHandler',
            'formatter': 'main_formatter',
        },
        'production_file':{
            'level' : 'INFO',
            'class' : 'logging.handlers.RotatingFileHandler',
            'filename' : 'logs/main.log',
            'maxBytes': 1024*1024*5, # 5 MB
            'backupCount' : 7,
            'formatter': 'main_formatter',
            # 'filters': ['require_debug_false'],
        },
        'debug_file':{
            'level' : 'DEBUG',
            'class' : 'logging.handlers.RotatingFileHandler',
            'filename' : 'logs/main_debug.log',
            'maxBytes': 1024*1024*5, # 5 MB
            'backupCount' : 7,
            'formatter': 'main_formatter',
            # 'filters': ['require_debug_true'],
        },
        'null': {
            "class": 'django.utils.log.NullHandler',
        }
    },
    'loggers': {
        'django.request': {
            'handlers': ['mail_admins', 'console', 'production_file',],
            'level': 'ERROR',
            'propagate': True,
        },
        'django': {
            'handlers': ['null', ],
        },
        'py.warnings': {
            'handlers': ['null', ],
        },
        'apps': {
            'handlers': ['console', 'production_file', 'debug_file'],
            'level': "DEBUG",
        },
        'tracekit': {
            'handlers': ['console', 'production_file', 'debug_file'],
            'level': "DEBUG",
        },
    }
}

import logging
logging.getLogger('django.db.backends').setLevel(logging.ERROR)

from settings_local import *

<commit_msg>last filter<commit_after># Django settings for lot project.
from madrona.common.default_settings import *

BASE_DIR = os.path.dirname(os.path.abspath(__file__))
TIME_ZONE = 'America/Vancouver'
ROOT_URLCONF = 'urls'
LOGIN_REDIRECT_URL = '/visualize'

DATABASES = {
    'default': {
        'ENGINE': 'django.contrib.gis.db.backends.postgis',
        'NAME': 'marco',
        'USER': 'vagrant',
    }
}

LOG_FILE =  os.path.realpath(os.path.join(os.path.dirname(__file__), '..', 'mp.log'))

INSTALLED_APPS += (
                    'django_extensions',

                    'general',
                    'data_manager',
                    'mp_settings',
                    'explore',
                    'visualize',
                    'django.contrib.humanize',
                    'flatblocks',
                    'mp_proxy',
                    'map_proxy'
                  )

GEOMETRY_DB_SRID = 99996
GEOMETRY_CLIENT_SRID = 3857 #for latlon
GEOJSON_SRID = 3857

APP_NAME = "Marine Planner Data Portal"
SERVER_ADMIN = 'edwin@pointnineseven.com'
DEFAULT_FROM_EMAIL = 'USVI <developers@pointnineseven.com>'
SERVER_EMAIL = DEFAULT_FROM_EMAIL
MANAGERS = ADMINS
EMAIL_SUBJECT_PREFIX = 'Marine Planner'
ADMINS = (
    ('Edwin Knuth', 'edwin@pointnineseven.com'),
    ('Scott Fletcher', 'scott@pointnineseven.com'),
)

#FEEDBACK_RECIPIENT = "Marine Planning Team <mp-team@marineplanner.org>"
#HELP_EMAIL = "mp-team@marineplanner.org"
#DEFAULT_FROM_EMAIL = "Marine Planning Team <mp-team@marineplanner.org>"

# url for socket.io printing
#SOCKET_URL = 'http://dev.marco.marineplanning.org:8080'
SOCKET_URL = False

# Change the following line to True, to display the 'under maintenance' template
UNDER_MAINTENANCE_TEMPLATE = False

TEMPLATE_DIRS = ( os.path.realpath(os.path.join(os.path.dirname(__file__), 'templates').replace('\\','/')), )

MIDDLEWARE_CLASSES += (
    'django.middleware.common.CommonMiddleware',
    'django.contrib.sessions.middleware.SessionMiddleware',
    'django.middleware.csrf.CsrfViewMiddleware',
    'django.contrib.auth.middleware.AuthenticationMiddleware',
    'django.contrib.messages.middleware.MessageMiddleware',
    'django.middleware.gzip.GZipMiddleware'

    # Uncomment the next line for simple clickjacking protection:
    # 'django.middleware.clickjacking.XFrameOptionsMiddleware',
)
LOGGING = {
    'version': 1,
    'disable_existing_loggers': False,
    'formatters': {
        'main_formatter': {
            'format': '%(levelname)s:%(name)s: %(message)s '
                     '(%(asctime)s; %(filename)s:%(lineno)d)',
            'datefmt': "%Y-%m-%d %H:%M:%S",
        },
    },
    'handlers': {
        'mail_admins': {
            'level': 'ERROR',
            # 'filters': ['require_debug_false'],
            'class': 'django.utils.log.AdminEmailHandler'
        },
        'console':{
            'level': 'DEBUG',
            # 'filters': ['require_debug_true'],
            'class': 'logging.StreamHandler',
            'formatter': 'main_formatter',
        },
        'production_file':{
            'level' : 'INFO',
            'class' : 'logging.handlers.RotatingFileHandler',
            'filename' : 'logs/main.log',
            'maxBytes': 1024*1024*5, # 5 MB
            'backupCount' : 7,
            'formatter': 'main_formatter',
            # 'filters': ['require_debug_false'],
        },
        'debug_file':{
            'level' : 'DEBUG',
            'class' : 'logging.handlers.RotatingFileHandler',
            'filename' : 'logs/main_debug.log',
            'maxBytes': 1024*1024*5, # 5 MB
            'backupCount' : 7,
            'formatter': 'main_formatter',
            # 'filters': ['require_debug_true'],
        },
        'null': {
            "class": 'django.utils.log.NullHandler',
        }
    },
    'loggers': {
        'django.request': {
            'handlers': ['mail_admins', 'console', 'production_file',],
            'level': 'ERROR',
            'propagate': True,
        },
        'django': {
            'handlers': ['null', ],
        },
        'py.warnings': {
            'handlers': ['null', ],
        },
        'apps': {
            'handlers': ['console', 'production_file', 'debug_file'],
            'level': "DEBUG",
        },
        'tracekit': {
            'handlers': ['console', 'production_file', 'debug_file'],
            'level': "DEBUG",
        },
    }
}

import logging
logging.getLogger('django.db.backends').setLevel(logging.ERROR)

from settings_local import *

<|endoftext|>


<commit_before>#!/usr/bin/env python
# -*- coding: iso-8859-1 -*-

__author__ = "Kyle Gordon"
__copyright__ = "Copyright (C) Kyle Gordon"

import os
import logging
import signal
import socket
import time
import sys
import csv

import mosquitto
import ConfigParser
import setproctitle

from datetime import datetime, timedelta

from zbxsend import Metric, send_to_zabbix

# Read the config file
config = ConfigParser.RawConfigParser()
config.read("/etc/mqtt-zabbix/mqtt-zabbix.cfg")

# Use ConfigParser to pick out the settings
DEBUG = config.getboolean("global", "debug")
LOGFILE = config.get("global", "logfile")
MQTT_HOST = config.get("global", "mqtt_host")
MQTT_PORT = config.getint("global", "mqtt_port")
MQTT_TOPIC = config.get("global", "mqtt_topic")

KEYFILE = config.get("global", "keyfile")
KEYHOST = config.get("global", "keyhost")
ZBXSERVER = config.get("global", "zabbix_server")
ZBXPORT = config.getint("global", "zabbix_port")

APPNAME = "mqtt-zabbix"
PRESENCETOPIC = "clients/" + socket.getfqdn() + "/" + APPNAME + "/state"
setproctitle.setproctitle(APPNAME)
client_id = APPNAME + "_%d" % os.getpid()
mqttc = mosquitto.Mosquitto(client_id)

LOGFORMAT = "%(asctime)-15s %(message)s"

if DEBUG:
    logging.basicConfig(filename=LOGFILE,
                        level=logging.DEBUG,
                        format=LOGFORMAT)
else:
    logging.basicConfig(filename=LOGFILE,
                        level=logging.INFO,
                        format=LOGFORMAT)

logging.info("Starting " + APPNAME)
logging.info("INFO MODE")
logging.debug("DEBUG MODE")

# All the MQTT callbacks start here

def on_publish(mosq, obj, mid):
    """
    What to do when a message is published
    """
    logging.debug("MID " + str(mid) + " published.")

def on_subscribe(mosq, obj, mid, qos_list):
    """
    What to do in the event of subscribing to a topic"
    """
    logging.debug("Subscribe with mid " + str(mid) + " received.")

def on_unsubscribe(mosq, obj, mid):
    """
    What to do in the event of unsubscribing from a topic
    """
    logging.debug("Unsubscribe with mid " + str(mid) + " received.")

def on_connect(mosq, obj, result_code):
    """
    Handle connections (or failures) to the broker.
    This is called after the client has received a CONNACK message
    from the broker in response to calling connect().
    The parameter rc is an integer giving the return code:

    0: Success
    1: Refused – unacceptable protocol version
    2: Refused – identifier rejected
    3: Refused – server unavailable
    4: Refused – bad user name or password (MQTT v3.1 broker only)
    5: Refused – not authorised (MQTT v3.1 broker only)
    """
    logging.debug("on_connect RC: " + str(result_code))
    if result_code == 0:
        logging.info("Connected to %s:%s", MQTT_HOST, MQTT_PORT)
        # Publish retained LWT as per
        # http://stackoverflow.com/q/97694
        # See also the will_set function in connect() below
        mqttc.publish(PRESENCETOPIC, "1", retain=True)
        process_connection()
    elif result_code == 1:
        logging.info("Connection refused - unacceptable protocol version")
        cleanup()
    elif result_code == 2:
        logging.info("Connection refused - identifier rejected")
        cleanup()
    elif result_code == 3:
        logging.info("Connection refused - server unavailable")
        logging.info("Retrying in 30 seconds")
        time.sleep(30)
    elif result_code == 4:
        logging.info("Connection refused - bad user name or password")
        cleanup()
    elif result_code == 5:
        logging.info("Connection refused - not authorised")
        cleanup()
    else:
        logging.warning("Something went wrong. RC:" + str(result_code))
        cleanup()

def on_disconnect(mosq, obj, result_code):
    """
    Handle disconnections from the broker
    """
    if result_code == 0:
        logging.info("Clean disconnection")
    else:
        logging.info("Unexpected disconnection! Reconnecting in 5 seconds")
        logging.debug("Result code: %s", result_code)
        time.sleep(5)

def on_message(mosq, obj, msg):
    """
    What to do when the client recieves a message from the broker
    """
    logging.debug("Received: " + msg.payload +
                  " received on topic " + msg.topic +
                  " with QoS " + str(msg.qos))
    process_message(msg)

def on_log(mosq, obj, level, string):
    """
    What to do with debug log output from the MQTT library
    """
    logging.debug(string)

# End of MQTT callbacks

def process_connection():
    """
    What to do when a new connection is established
    """
    logging.debug("Processing connection")
    mqttc.subscribe(MQTT_TOPIC, 2)

def process_message(msg):
    """
    What to do with the message that's arrived.
    Looks up the topic in the KeyMap dictionary, and forwards
    the message onto Zabbix using the associated Zabbix key
    """
    logging.debug("Processing : " + msg.topic)
    if msg.topic in KeyMap.mapdict:
        logging.info("Sending %s %s to Zabbix key %s",
                      msg.topic,
                      msg.payload,
                      KeyMap.mapdict[msg.topic])
        # Zabbix can also accept text and character data...
        # should we sanitize input or just accept it as is?
        send_to_zabbix([Metric(KEYHOST,
                        KeyMap.mapdict[msg.topic],
                        msg.payload)],
                        ZBXSERVER,
                        ZBXPORT)
    else:
        # Received something with a /raw/ topic,
        # but it didn't match anything. Log it, and discard it
        logging.debug("Unknown: %s", msg.topic)

def cleanup(signum, frame):
    """
    Signal handler to ensure we disconnect cleanly
    in the event of a SIGTERM or SIGINT.
    """
    logging.info("Disconnecting from broker")
    # Publish a retained message to state that this client is offline
    mqttc.publish(PRESENCETOPIC, "0", retain=True)
    mqttc.disconnect()
    logging.info("Exiting on signal %d", signum)
    sys.exit(signum)

def connect():
    """
    Connect to the broker, define the callbacks, and subscribe
    This will also set the Last Will and Testament (LWT)
    The LWT will be published in the event of an unclean or
    unexpected disconnection.
    """
    logging.debug("Connecting to %s:%s", MQTT_HOST, MQTT_PORT)
    # Set the Last Will and Testament (LWT) *before* connecting
    mqttc.will_set(PRESENCETOPIC, "0", qos=0, retain=True)
    result = mqttc.connect(MQTT_HOST, MQTT_PORT, 60, True)
    if result != 0:
        logging.info("Connection failed with error code %s. Retrying", result)
        time.sleep(10)
        connect()

    # Define the callbacks
    mqttc.on_connect = on_connect
    mqttc.on_disconnect = on_disconnect
    mqttc.on_publish = on_publish
    mqttc.on_subscribe = on_subscribe
    mqttc.on_unsubscribe = on_unsubscribe
    mqttc.on_message = on_message
    if DEBUG:
        mqttc.on_log = on_log

class KeyMap:
    """
    Read the topics and keys into a dictionary for internal lookups
    """
    logging.debug("Loading map")
    with open(KEYFILE, mode="r") as inputfile:
        reader = csv.reader(inputfile)
        mapdict = dict((rows[0], rows[1]) for rows in reader)

# Use the signal module to handle signals
signal.signal(signal.SIGTERM, cleanup)
signal.signal(signal.SIGINT, cleanup)

# Connect to the broker
connect()

# Try to loop_forever until interrupted
try:
    mqttc.loop_forever()
except KeyboardInterrupt:
    logging.info("Interrupted by keypress")
    sys.exit(0)
<commit_msg>send integer timestamp, for compatibility to Zabbix 3<commit_after>#!/usr/bin/env python
# -*- coding: iso-8859-1 -*-

__author__ = "Kyle Gordon"
__copyright__ = "Copyright (C) Kyle Gordon"

import os
import logging
import signal
import socket
import time
import sys
import csv

import mosquitto
import ConfigParser
import setproctitle

from datetime import datetime, timedelta

from zbxsend import Metric, send_to_zabbix

# Read the config file
config = ConfigParser.RawConfigParser()
config.read("/etc/mqtt-zabbix/mqtt-zabbix.cfg")

# Use ConfigParser to pick out the settings
DEBUG = config.getboolean("global", "debug")
LOGFILE = config.get("global", "logfile")
MQTT_HOST = config.get("global", "mqtt_host")
MQTT_PORT = config.getint("global", "mqtt_port")
MQTT_TOPIC = config.get("global", "mqtt_topic")

KEYFILE = config.get("global", "keyfile")
KEYHOST = config.get("global", "keyhost")
ZBXSERVER = config.get("global", "zabbix_server")
ZBXPORT = config.getint("global", "zabbix_port")

APPNAME = "mqtt-zabbix"
PRESENCETOPIC = "clients/" + socket.getfqdn() + "/" + APPNAME + "/state"
setproctitle.setproctitle(APPNAME)
client_id = APPNAME + "_%d" % os.getpid()
mqttc = mosquitto.Mosquitto(client_id)

LOGFORMAT = "%(asctime)-15s %(message)s"

if DEBUG:
    logging.basicConfig(filename=LOGFILE,
                        level=logging.DEBUG,
                        format=LOGFORMAT)
else:
    logging.basicConfig(filename=LOGFILE,
                        level=logging.INFO,
                        format=LOGFORMAT)

logging.info("Starting " + APPNAME)
logging.info("INFO MODE")
logging.debug("DEBUG MODE")

# All the MQTT callbacks start here

def on_publish(mosq, obj, mid):
    """
    What to do when a message is published
    """
    logging.debug("MID " + str(mid) + " published.")

def on_subscribe(mosq, obj, mid, qos_list):
    """
    What to do in the event of subscribing to a topic"
    """
    logging.debug("Subscribe with mid " + str(mid) + " received.")

def on_unsubscribe(mosq, obj, mid):
    """
    What to do in the event of unsubscribing from a topic
    """
    logging.debug("Unsubscribe with mid " + str(mid) + " received.")

def on_connect(mosq, obj, result_code):
    """
    Handle connections (or failures) to the broker.
    This is called after the client has received a CONNACK message
    from the broker in response to calling connect().
    The parameter rc is an integer giving the return code:

    0: Success
    1: Refused – unacceptable protocol version
    2: Refused – identifier rejected
    3: Refused – server unavailable
    4: Refused – bad user name or password (MQTT v3.1 broker only)
    5: Refused – not authorised (MQTT v3.1 broker only)
    """
    logging.debug("on_connect RC: " + str(result_code))
    if result_code == 0:
        logging.info("Connected to %s:%s", MQTT_HOST, MQTT_PORT)
        # Publish retained LWT as per
        # http://stackoverflow.com/q/97694
        # See also the will_set function in connect() below
        mqttc.publish(PRESENCETOPIC, "1", retain=True)
        process_connection()
    elif result_code == 1:
        logging.info("Connection refused - unacceptable protocol version")
        cleanup()
    elif result_code == 2:
        logging.info("Connection refused - identifier rejected")
        cleanup()
    elif result_code == 3:
        logging.info("Connection refused - server unavailable")
        logging.info("Retrying in 30 seconds")
        time.sleep(30)
    elif result_code == 4:
        logging.info("Connection refused - bad user name or password")
        cleanup()
    elif result_code == 5:
        logging.info("Connection refused - not authorised")
        cleanup()
    else:
        logging.warning("Something went wrong. RC:" + str(result_code))
        cleanup()

def on_disconnect(mosq, obj, result_code):
    """
    Handle disconnections from the broker
    """
    if result_code == 0:
        logging.info("Clean disconnection")
    else:
        logging.info("Unexpected disconnection! Reconnecting in 5 seconds")
        logging.debug("Result code: %s", result_code)
        time.sleep(5)

def on_message(mosq, obj, msg):
    """
    What to do when the client recieves a message from the broker
    """
    logging.debug("Received: " + msg.payload +
                  " received on topic " + msg.topic +
                  " with QoS " + str(msg.qos))
    process_message(msg)

def on_log(mosq, obj, level, string):
    """
    What to do with debug log output from the MQTT library
    """
    logging.debug(string)

# End of MQTT callbacks

def process_connection():
    """
    What to do when a new connection is established
    """
    logging.debug("Processing connection")
    mqttc.subscribe(MQTT_TOPIC, 2)

def process_message(msg):
    """
    What to do with the message that's arrived.
    Looks up the topic in the KeyMap dictionary, and forwards
    the message onto Zabbix using the associated Zabbix key
    """
    logging.debug("Processing : " + msg.topic)
    if msg.topic in KeyMap.mapdict:
        logging.info("Sending %s %s to Zabbix key %s",
                      msg.topic,
                      msg.payload,
                      KeyMap.mapdict[msg.topic])
        # Zabbix can also accept text and character data...
        # should we sanitize input or just accept it as is?
        send_to_zabbix([Metric(KEYHOST,
                        KeyMap.mapdict[msg.topic],
                        msg.payload,
                        time.strftime("%s"))],
                        ZBXSERVER,
                        ZBXPORT)
    else:
        # Received something with a /raw/ topic,
        # but it didn't match anything. Log it, and discard it
        logging.debug("Unknown: %s", msg.topic)

def cleanup(signum, frame):
    """
    Signal handler to ensure we disconnect cleanly
    in the event of a SIGTERM or SIGINT.
    """
    logging.info("Disconnecting from broker")
    # Publish a retained message to state that this client is offline
    mqttc.publish(PRESENCETOPIC, "0", retain=True)
    mqttc.disconnect()
    logging.info("Exiting on signal %d", signum)
    sys.exit(signum)

def connect():
    """
    Connect to the broker, define the callbacks, and subscribe
    This will also set the Last Will and Testament (LWT)
    The LWT will be published in the event of an unclean or
    unexpected disconnection.
    """
    logging.debug("Connecting to %s:%s", MQTT_HOST, MQTT_PORT)
    # Set the Last Will and Testament (LWT) *before* connecting
    mqttc.will_set(PRESENCETOPIC, "0", qos=0, retain=True)
    result = mqttc.connect(MQTT_HOST, MQTT_PORT, 60, True)
    if result != 0:
        logging.info("Connection failed with error code %s. Retrying", result)
        time.sleep(10)
        connect()

    # Define the callbacks
    mqttc.on_connect = on_connect
    mqttc.on_disconnect = on_disconnect
    mqttc.on_publish = on_publish
    mqttc.on_subscribe = on_subscribe
    mqttc.on_unsubscribe = on_unsubscribe
    mqttc.on_message = on_message
    if DEBUG:
        mqttc.on_log = on_log

class KeyMap:
    """
    Read the topics and keys into a dictionary for internal lookups
    """
    logging.debug("Loading map")
    with open(KEYFILE, mode="r") as inputfile:
        reader = csv.reader(inputfile)
        mapdict = dict((rows[0], rows[1]) for rows in reader)

# Use the signal module to handle signals
signal.signal(signal.SIGTERM, cleanup)
signal.signal(signal.SIGINT, cleanup)

# Connect to the broker
connect()

# Try to loop_forever until interrupted
try:
    mqttc.loop_forever()
except KeyboardInterrupt:
    logging.info("Interrupted by keypress")
    sys.exit(0)
<|endoftext|>


<commit_before>#built-in
import time
#downloaded
import praw
#local
import keys
import file_uploader
import form_handler

def main():
    global queue, timer, blocked
    subreddit = r.subreddit('test')
    #iterate through all comments in given subreddit
    for submission in subreddit.stream.submissions():
        #reveal all hiding comments
        if praw.models.comment_forest.MoreComments in submission.comments:
            submission.comments.replace_more(limit=None)
        for comment in submission.comments.list():
            if check_condition(comment):
                print('comment found!')
                #check if bot response recorded
                #need up update to use postgreSQL
                if not keys.comment_replied(comment.id):
                    already_commented = False
                    for reply in comment.replies:
                        print("checking replies")
                        #check if bot commented
                        if str(reply.author) == keys.reddit_user:
                            print("already commented")
                            keys.add_comment(comment.id)
                            already_commented = True
                            break
                else:
                    already_commented = True
                    print("already commented")
                #last check if reply found before activating bot
                if not already_commented:
                    bot_action(comment)
            if len(queue) > 0 and (not blocked or time.time() > timer + 60 * 10):
                print("submitting %d comments"%len(queue))
                blocked = False
                submit_comments();

def check_condition(comment):
    text = comment.body
    called = '{' in text and '}' in text
    if called:
        start = text.index('{')
        end = text.index('}')
    return called and '|' in text[start+1:end]

def bot_action(comment):
    print("enter bot_action")
    keys.add_comment(comment.id)
    text = comment.body
    start = text.index('{')
    end = text.index('}')
    notation,title,description = parse_notation(text[start+1:end])
    temp_res = form_handler.convert_from_abc(notation)
    if temp_res is None:
        print('unable to convert from abc notation')
        return
    hosted_res = list()
    i = 0
    for r_img,r_mus in temp_res:
        imglink = file_uploader.upload_image(r_img,title[i],description[i])
        muslink = file_uploader.upload_music(r_mus,comment,title[i],description[i])
        hosted_res.append((imglink,muslink))
        i += 1
    msg = "Hi, I\'m MusicalTextBot.\n\nI converted your abc notation to"
    for img,mus in hosted_res:
        msg = msg+"\n\n[score](%s) and [performed](%s)"%(img,mus)
    msg = msg+"\n\n^^the ^^music ^^link ^^will ^^expire ^^in ^^roughly ^^1 ^^hour"
    reply = (comment,msg)
    queue.append(reply)

def submit_comments():
    response = queue[0]
    comment,msg = response
    try:
        comment.reply(msg)
        queue.pop(0)
        print(msg)
    except:
        print('blocked, looking for more comments')
        blocked = True
        timer = time.time;

def parse_notation(notation):
    tokens = notation.split('\n')
    prefix = []
    titles = []
    alternate = False
    descriptions = []
    songs = []
    new_tags = True
    new_music = True
    new_songs = []
    return_count = -1 #first song => new song
    song_num = -1 #nothing allocated
    output = ''

    while tokens[0] == '': #remove excess leading newlines
        tokens.remove('')

    for token in tokens:
        if token == '' and return_count > -1: #check number of
            return_count += 1
            if(return_count > 1):
                return_count = -1
        elif ':' == token[1]:
            if new_tags or token[0] == 'X':
                song_num += 1
                if return_count == -1 or token[0] == 'X': #set new song
                    new_songs.append(song_num)
                    titles.append('')
                    descriptions.append('')
                    alternate = False
                new_tags = False
                prefix.append('') #allocate new position in list
            return_count = 0
            new_music = True
            if 'T:' in token: #start letter parsing
                if not alternate:
                    titles[len(new_songs)-1] = token[2:]
                    alternate = True
                else:
                    descriptions[len(new_songs)-1] += token[2:]+'\n'
            if 'N:' in token:
                descriptions[len(new_songs)-1] += token[2:]+'\n'
            if 'X:' in token: #don't trust user's numbering
                prefix[song_num] += token[0:2]+'%d\n'%song_num
            elif 'K:' in token and token.endswith(':'): #scale can't be missing
                prefix[song_num] += token+'C\n'
            else:
                prefix[song_num] += token+'\n'
        else:
            if return_count == -1 and new_tags: #set new song, empty tags
                song_num += 1
                new_songs.append(song_num)
                prefix.append('')
                titles.append('')
                descriptions.append('')
                songs.append('')
            if new_music: #following creation of tags
                new_music = False
                songs.append('') #allocate new position in list
            new_tags = True
            return_count = 0
            songs[song_num] += token
    for i in range(song_num+1):
        if 'X:' not in prefix[i] and i in new_songs: #starting tag
            prefix[i] = 'X:%d\n'%i + prefix[i]
        if 'K:' not in prefix[i] and i in new_songs: #ending tag
            prefix[i] += 'K:C\n'
        if len(songs[i]) > 0: #need music, can't process only tags
            output += prefix[i] + songs[i] + '\n'
    return output,titles,descriptions

r = praw.Reddit('redditBot')
queue = list()
blocked = False
timer = time.time()
main()
<commit_msg>revert back to comment stream search<commit_after>#built-in
import time
#downloaded
import praw
#local
import keys
import file_uploader
import form_handler

def main():
    global queue, timer, blocked
    subreddit = r.subreddit('test')
    #iterate through all comments in given subreddit
    for comment in subreddit.stream.comments():
        if check_condition(comment):
            print('comment found!')
            #check if bot response recorded
            #need up update to use postgreSQL
            if not keys.comment_replied(comment.id):
                already_commented = False
                for reply in comment.replies:
                    print("checking replies")
                    #check if bot commented
                    if str(reply.author) == keys.reddit_user:
                        print("already commented")
                        keys.add_comment(comment.id)
                        already_commented = True
                        break
            else:
                already_commented = True
                print("already commented")
            #last check if reply found before activating bot
            if not already_commented:
                bot_action(comment)
        if len(queue) > 0 and (not blocked or time.time() > timer + 60 * 10):
            print("submitting %d comment(s)"%len(queue))
            blocked = False
            #submit_comments();

def check_condition(comment):
    text = comment.body
    called = '{' in text and '}' in text
    if called:
        start = text.index('{')
        end = text.index('}')
    return called and '|' in text[start+1:end]

def bot_action(comment):
    print("enter bot_action")
    keys.add_comment(comment.id)
    text = comment.body
    start = text.index('{')
    end = text.index('}')
    notation,title,description = parse_notation(text[start+1:end])
    temp_res = form_handler.convert_from_abc(notation)
    if temp_res is None:
        print('unable to convert from abc notation')
        return
    hosted_res = list()
    i = 0
    for r_img,r_mus in temp_res:
        imglink = file_uploader.upload_image(r_img,title[i],description[i])
        muslink = file_uploader.upload_music(r_mus,comment,title[i],description[i])
        hosted_res.append((imglink,muslink))
        i += 1
    msg = "Hi, I\'m MusicalTextBot.\n\nI converted your abc notation to"
    for img,mus in hosted_res:
        msg = msg+"\n\n[score](%s) and [performed](%s)"%(img,mus)
    msg = msg+"\n\n^^the ^^music ^^link ^^will ^^expire ^^in ^^roughly ^^1 ^^hour"
    reply = (comment,msg)
    queue.append(reply)

def submit_comments():
    response = queue[0]
    comment,msg = response
    try:
        comment.reply(msg)
        queue.pop(0)
        print(msg)
    except:
        print('blocked, looking for more comments')
        blocked = True
        timer = time.time;

def parse_notation(notation):
    tokens = notation.split('\n')
    prefix = []
    titles = []
    alternate = False
    descriptions = []
    songs = []
    new_tags = True
    new_music = True
    new_songs = []
    return_count = -1 #first song => new song
    song_num = -1 #nothing allocated
    output = ''

    while tokens[0] == '': #remove excess leading newlines
        tokens.remove('')

    for token in tokens:
        if token == '' and return_count > -1: #check number of
            return_count += 1
            if(return_count > 1):
                return_count = -1
        elif ':' == token[1]:
            if new_tags or token[0] == 'X':
                song_num += 1
                if return_count == -1 or token[0] == 'X': #set new song
                    new_songs.append(song_num)
                    titles.append('')
                    descriptions.append('')
                    alternate = False
                new_tags = False
                prefix.append('') #allocate new position in list
            return_count = 0
            new_music = True
            if 'T:' in token: #start letter parsing
                if not alternate:
                    titles[len(new_songs)-1] = token[2:]
                    alternate = True
                else:
                    descriptions[len(new_songs)-1] += token[2:]+'\n'
            if 'N:' in token:
                descriptions[len(new_songs)-1] += token[2:]+'\n'
            if 'X:' in token: #don't trust user's numbering
                prefix[song_num] += token[0:2]+'%d\n'%song_num
            elif 'K:' in token and token.endswith(':'): #scale can't be missing
                prefix[song_num] += token+'C\n'
            else:
                prefix[song_num] += token+'\n'
        else:
            if return_count == -1 and new_tags: #set new song, empty tags
                song_num += 1
                new_songs.append(song_num)
                prefix.append('')
                titles.append('')
                descriptions.append('')
                songs.append('')
            if new_music: #following creation of tags
                new_music = False
                songs.append('') #allocate new position in list
            new_tags = True
            return_count = 0
            songs[song_num] += token
    for i in range(song_num+1):
        if 'X:' not in prefix[i] and i in new_songs: #starting tag
            prefix[i] = 'X:%d\n'%i + prefix[i]
        if 'K:' not in prefix[i] and i in new_songs: #ending tag
            prefix[i] += 'K:C\n'
        if len(songs[i]) > 0: #need music, can't process only tags
            output += prefix[i] + songs[i] + '\n'
    return output,titles,descriptions

r = praw.Reddit('redditBot')
queue = list()
blocked = False
timer = time.time()
main()
<|endoftext|>


<commit_before>from __future__ import print_function
from cement.core import controller
from collections import OrderedDict
from common.functions import template
from common import template
from concurrent.futures import ThreadPoolExecutor
from copy import deepcopy
from plugins.internal.base_plugin import BasePlugin
from plugins.internal.base_plugin_internal import BasePluginInternal
import common
import common.functions as f
import common.plugins_util as pu
import common.versions as v
import sys
import traceback

class Scan(BasePlugin):

    IDENTIFY_BATCH_SIZE = 1000

    class Meta:
        label = 'scan'
        description = 'cms scanning functionality.'
        stacked_on = 'base'
        stacked_type = 'nested'

        epilog = "\n"

        argument_formatter = common.SmartFormatter
        epilog = template("help_epilog.mustache")

        arguments = [
                (['-u', '--url'], dict(action='store', help='')),
                (['-U', '--url-file'], dict(action='store', help='''A file which
                    contains a list of URLs.''')),

                (['--enumerate', '-e'], dict(action='store', help='R|' +
                    common.template('help_enumerate.mustache'),
                    choices=common.enum_list(common.Enumerate), default='a')),
                (['--method'], dict(action='store', help='R|' +
                    common.template('help_method.mustache'), choices=common.enum_list(common.ScanningMethod))),
                (['--verb'], dict(action='store', help="""The HTTP verb to use;
                    the default option is head, except for version enumeration
                    requests, which are always get because we need to get the hash
                    from the file's contents""", default='head',
                    choices=common.enum_list(common.Verb))),
                (['--number', '-n'], dict(action='store', help='''Number of
                    words to attempt from the plugin/theme dictionary. Default
                    is 1000. Use -n 'all' to use all available.''',
                    default=BasePluginInternal.NUMBER_DEFAULT)),
                (['--plugins-base-url'], dict(action='store', help="""Location
                    where the plugins are stored by the CMS. Default is the CMS'
                    default location. First %%s in string will be replaced with
                    the url, and the second one will be replaced with the module
                    name. E.g. '%%ssites/all/modules/%%s/'""")),
                (['--themes-base-url'], dict(action='store', help='''Same as
                    above, but for themes.''')),
                (['--timeout'], dict(action='store', help="""How long to wait
                    for an HTTP response before timing out (in seconds).""",
                    default=45, type=int)),
                (['--timeout-host'], dict(action='store', help="""Maximum time
                    to spend per host (in seconds).""", default=1800, type=int)),
                (['--no-follow-redirects'], dict(action='store_false', help="""Prevent
                    the following of redirects.""", dest="follow_redirects", default=True)),
                (['--host'], dict(action='store', help="""Override host header
                    with this value.""", default=None)),
                (['--massscan-override'], dict(action='store_true',
                    help="""Overrides defaults with defaults convenient for
                    mass-scanning of hosts.""", default=False)),

                (['--threads', '-t'], dict(action='store', help='''Number of
                    threads. Default 4.''', default=4, type=int)),
                (['--threads-identify'], dict(action='store', help='''Number of
                    threads used for CMS identification.''', default=None, type=int)),
                (['--threads-scan'], dict(action='store', help='''Threads used
                    for mass scanning.''', default=None, type=int)),
                (['--threads-enumerate'], dict(action='store', help='''Threads
                    used for plugin enumeration.''', default=None, type=int)),

                (['--output', '-o'], dict(action='store', help='Output format',
                    choices=common.enum_list(common.ValidOutputs), default='standard')),
                (['--debug-requests'], dict(action='store_true', help="""Prints every
                    HTTP request made and the response returned from the server
                    for debugging purposes. Disables threading and loading
                    bars.""", default=False)),
                (['--error-log'], dict(action='store', help='''A file to store the
                    errors on.''', default=None)),
            ]

    @controller.expose(hide=True)
    def default(self):
        plugins = pu.plugins_base_get()
        opts = self._options(self.app.pargs)
        url_file_input = 'url_file' in opts
        self._general_init(opts)
        instances = self._instances_get(opts, plugins, url_file_input)

        follow_redirects = opts['follow_redirects']
        opts['follow_redirects'] = False

        if url_file_input:
            self.out.debug('scan.default -> url_file')
            self._process_scan_url_file(opts['url_file'], follow_redirects)
        else:
            self.out.debug('scan.default -> url')
            url = opts['url']
            if not url:
                self.out.fatal("--url parameter is blank.")

            cms_name, scan_out = self._process_cms_identify(url, opts, instances,
                    follow_redirects)

            url, host_header = scan_out

            inst_dict = instances[cms_name]
            inst = inst_dict['inst']

            opts['url'] = url
            opts['headers'] = self._generate_headers(host_header)

            inst.process_url(opts, **inst_dict['kwargs'])

    def _process_scan_url_file(self, file_location, follow_redirects):
        self.out.debug('scan._process_scan_url_file')
        with open(file_location) as url_file:
            i = 0
            urls = []
            for url in url_file:
                if i % self.IDENTIFY_BATCH_SIZE == 0 and i != 0:
                    plugins = pu.plugins_base_get()
                    opts = self._options(self.app.pargs)
                    executor = ThreadPoolExecutor(max_workers=opts['threads_identify'])
                    instances = self._instances_get(opts, plugins,
                            url_file_input=True)

                    self._process_generate_futures(urls, executor, opts,
                        instances, follow_redirects)

                    executor.shutdown()
                    del executor
                    del plugins
                    del opts
                    del instances
                else:
                    urls.append(url)

                i += 1

            if len(urls) > 0:
                plugins = pu.plugins_base_get()
                opts = self._options(self.app.pargs)
                executor = ThreadPoolExecutor(max_workers=opts['threads_identify'])
                instances = self._instances_get(opts, plugins,
                        url_file_input=True)

                self._process_generate_futures(urls, executor, opts,
                    instances, follow_redirects)

    def _process_generate_futures(self, urls, executor, opts, instances,
            follow_redirects):

        futures = []
        for url in urls:
            url = url.strip()
            future = executor.submit(self._process_cms_identify, url,
                    opts, instances, follow_redirects)

            futures.append({
                'url': url,
                'future': future
            })

        self._process_identify_futures(futures, opts, instances)

    def _process_cms_identify(self, url, opts, instances, follow_redirects):
        self.out.debug('scan._process_cms_identify -> %s' % url)
        try:
            url, host_header = url, opts['headers']['Host']
        except:
            url, host_header = self._process_host_line(url)

        url = f.repair_url(url, self.out)

        if follow_redirects:
            url, host_header = self.determine_redirect(url, host_header, opts)

        found = False
        for cms_name in instances:
            inst_dict = instances[cms_name]
            inst = inst_dict['inst']
            vf = inst_dict['vf']

            if inst.cms_identify(vf, url, opts['timeout'], self._generate_headers(host_header)) == True:
                found = True
                break

        if not found:
            return None, None
        else:
            return cms_name, (url, host_header)

    def _process_identify_futures(self, futures, opts, instances):
        self.out.debug('scan._process_identify_futures')
        to_scan = {}
        for future_dict in futures:
            future = future_dict['future']

            try:
                cms_name, result_tuple = future.result(timeout=opts['timeout_host'])

                if cms_name != None:
                    if cms_name not in to_scan:
                        to_scan[cms_name] = []

                    to_scan[cms_name].append(result_tuple)
            except:
                f.exc_handle(future_dict['url'], self.out, self.app.testing)

        if to_scan:
            self._process_scan(opts, instances, to_scan)
            to_scan = {}

    def _process_scan(self, opts, instances, to_scan):
        self.out.debug('scan._process_scan')
        for cms_name in to_scan:
            inst_dict = instances[cms_name]
            cms_urls = to_scan[cms_name]

            if len(cms_urls) > 0:
                inst_dict['inst'].process_url_iterable(cms_urls, opts, **inst_dict['kwargs'])

    def _instances_get(self, opts, plugins, url_file_input):
        """
        Creates and returns an ordered dictionary containing instances for all available
        scanning plugins, sort of ordered by popularity.
        @param opts: options as returned by self._options.
        @param plugins: plugins as returned by plugins_util.plugins_base_get.
        @param url_file_input: boolean value which indicates whether we are
            scanning an individual URL or a file. This is used to determine
            kwargs required.
        """
        instances = OrderedDict()
        preferred_order = ['wordpress', 'joomla', 'drupal']

        for cms_name in preferred_order:
            for plugin in plugins:
                plugin_name = plugin.__name__.lower()

                if cms_name == plugin_name:
                    instances[plugin_name] = self._instance_get(plugin, opts,
                            url_file_input)

        for plugin in plugins:
            plugin_name = plugin.__name__.lower()
            if plugin_name not in preferred_order:
                instances[plugin_name] = self._instance_get(plugin, opts,
                        url_file_input)

        return instances

    def _instance_get(self, plugin, opts, url_file_input):
        """
        Return an instance dictionary for an individual plugin.
        @see Scan._instances_get.
        """
        inst = plugin()
        hp, func, enabled_func = inst._general_init(opts)
        name = inst._meta.label
        vf = v.VersionsFile(inst.versions_file)

        kwargs = {
            'hide_progressbar': hp,
            'functionality': func,
            'enabled_functionality': enabled_func
        }

        if url_file_input:
            del kwargs['hide_progressbar']

        return {
            'inst': inst,
            'vf': vf,
            'kwargs': kwargs
        }

<commit_msg>Re-plan number of threads.<commit_after>from __future__ import print_function
from cement.core import controller
from collections import OrderedDict
from common.functions import template
from common import template
from concurrent.futures import ThreadPoolExecutor
from copy import deepcopy
from plugins.internal.base_plugin import BasePlugin
from plugins.internal.base_plugin_internal import BasePluginInternal
import common
import common.functions as f
import common.plugins_util as pu
import common.versions as v
import sys
import traceback

class Scan(BasePlugin):

    class Meta:
        label = 'scan'
        description = 'cms scanning functionality.'
        stacked_on = 'base'
        stacked_type = 'nested'

        epilog = "\n"

        argument_formatter = common.SmartFormatter
        epilog = template("help_epilog.mustache")

        arguments = [
                (['-u', '--url'], dict(action='store', help='')),
                (['-U', '--url-file'], dict(action='store', help='''A file which
                    contains a list of URLs.''')),

                (['--enumerate', '-e'], dict(action='store', help='R|' +
                    common.template('help_enumerate.mustache'),
                    choices=common.enum_list(common.Enumerate), default='a')),
                (['--method'], dict(action='store', help='R|' +
                    common.template('help_method.mustache'), choices=common.enum_list(common.ScanningMethod))),
                (['--verb'], dict(action='store', help="""The HTTP verb to use;
                    the default option is head, except for version enumeration
                    requests, which are always get because we need to get the hash
                    from the file's contents""", default='head',
                    choices=common.enum_list(common.Verb))),
                (['--number', '-n'], dict(action='store', help='''Number of
                    words to attempt from the plugin/theme dictionary. Default
                    is 1000. Use -n 'all' to use all available.''',
                    default=BasePluginInternal.NUMBER_DEFAULT)),
                (['--plugins-base-url'], dict(action='store', help="""Location
                    where the plugins are stored by the CMS. Default is the CMS'
                    default location. First %%s in string will be replaced with
                    the url, and the second one will be replaced with the module
                    name. E.g. '%%ssites/all/modules/%%s/'""")),
                (['--themes-base-url'], dict(action='store', help='''Same as
                    above, but for themes.''')),
                (['--timeout'], dict(action='store', help="""How long to wait
                    for an HTTP response before timing out (in seconds).""",
                    default=45, type=int)),
                (['--timeout-host'], dict(action='store', help="""Maximum time
                    to spend per host (in seconds).""", default=1800, type=int)),
                (['--no-follow-redirects'], dict(action='store_false', help="""Prevent
                    the following of redirects.""", dest="follow_redirects", default=True)),
                (['--host'], dict(action='store', help="""Override host header
                    with this value.""", default=None)),
                (['--massscan-override'], dict(action='store_true',
                    help="""Overrides defaults with defaults convenient for
                    mass-scanning of hosts.""", default=False)),

                (['--threads', '-t'], dict(action='store', help='''Number of
                    threads. Default 4.''', default=4, type=int)),
                (['--threads-identify'], dict(action='store', help='''Number of
                    threads used for CMS identification.''', default=None, type=int)),
                (['--threads-scan'], dict(action='store', help='''Threads used
                    for mass scanning.''', default=None, type=int)),
                (['--threads-enumerate'], dict(action='store', help='''Threads
                    used for plugin enumeration.''', default=None, type=int)),

                (['--output', '-o'], dict(action='store', help='Output format',
                    choices=common.enum_list(common.ValidOutputs), default='standard')),
                (['--debug-requests'], dict(action='store_true', help="""Prints every
                    HTTP request made and the response returned from the server
                    for debugging purposes. Disables threading and loading
                    bars.""", default=False)),
                (['--error-log'], dict(action='store', help='''A file to store the
                    errors on.''', default=None)),
            ]

    @controller.expose(hide=True)
    def default(self):
        plugins = pu.plugins_base_get()
        opts = self._options(self.app.pargs)
        url_file_input = 'url_file' in opts
        self._general_init(opts)
        instances = self._instances_get(opts, plugins, url_file_input)

        follow_redirects = opts['follow_redirects']
        opts['follow_redirects'] = False

        if url_file_input:
            self.out.debug('scan.default -> url_file')
            self._process_scan_url_file(opts['url_file'],
                opts['threads_identify'], follow_redirects)
        else:
            self.out.debug('scan.default -> url')
            url = opts['url']
            if not url:
                self.out.fatal("--url parameter is blank.")

            cms_name, scan_out = self._process_cms_identify(url, opts, instances,
                    follow_redirects)

            url, host_header = scan_out

            inst_dict = instances[cms_name]
            inst = inst_dict['inst']

            opts['url'] = url
            opts['headers'] = self._generate_headers(host_header)

            inst.process_url(opts, **inst_dict['kwargs'])

    def _process_scan_url_file(self, file_location, num_threads_identify, follow_redirects):
        self.out.debug('scan._process_scan_url_file')
        with open(file_location) as url_file:
            i = 0
            urls = []
            for url in url_file:
                if i % num_threads_identify == 0 and i != 0:
                    plugins = pu.plugins_base_get()
                    opts = self._options(self.app.pargs)
                    executor = ThreadPoolExecutor(max_workers=opts['threads_identify'])
                    instances = self._instances_get(opts, plugins,
                            url_file_input=True)

                    self._process_generate_futures(urls, executor, opts,
                        instances, follow_redirects)

                    executor.shutdown()
                    del executor
                    del plugins
                    del opts
                    del instances
                else:
                    urls.append(url)

                i += 1

            if len(urls) > 0:
                plugins = pu.plugins_base_get()
                opts = self._options(self.app.pargs)
                executor = ThreadPoolExecutor(max_workers=opts['threads_identify'])
                instances = self._instances_get(opts, plugins,
                        url_file_input=True)

                self._process_generate_futures(urls, executor, opts,
                    instances, follow_redirects)

    def _process_generate_futures(self, urls, executor, opts, instances,
            follow_redirects):

        futures = []
        for url in urls:
            url = url.strip()
            future = executor.submit(self._process_cms_identify, url,
                    opts, instances, follow_redirects)

            futures.append({
                'url': url,
                'future': future
            })

        self._process_identify_futures(futures, opts, instances)

    def _process_cms_identify(self, url, opts, instances, follow_redirects):
        self.out.debug('scan._process_cms_identify -> %s' % url)
        try:
            url, host_header = url, opts['headers']['Host']
        except:
            url, host_header = self._process_host_line(url)

        url = f.repair_url(url, self.out)

        if follow_redirects:
            url, host_header = self.determine_redirect(url, host_header, opts)

        found = False
        for cms_name in instances:
            inst_dict = instances[cms_name]
            inst = inst_dict['inst']
            vf = inst_dict['vf']

            if inst.cms_identify(vf, url, opts['timeout'], self._generate_headers(host_header)) == True:
                found = True
                break

        if not found:
            return None, None
        else:
            return cms_name, (url, host_header)

    def _process_identify_futures(self, futures, opts, instances):
        self.out.debug('scan._process_identify_futures')
        to_scan = {}
        for future_dict in futures:
            future = future_dict['future']

            try:
                cms_name, result_tuple = future.result(timeout=opts['timeout_host'])

                if cms_name != None:
                    if cms_name not in to_scan:
                        to_scan[cms_name] = []

                    to_scan[cms_name].append(result_tuple)
            except:
                f.exc_handle(future_dict['url'], self.out, self.app.testing)

        if to_scan:
            self._process_scan(opts, instances, to_scan)
            to_scan = {}

    def _process_scan(self, opts, instances, to_scan):
        self.out.debug('scan._process_scan')
        for cms_name in to_scan:
            inst_dict = instances[cms_name]
            cms_urls = to_scan[cms_name]

            if len(cms_urls) > 0:
                inst_dict['inst'].process_url_iterable(cms_urls, opts, **inst_dict['kwargs'])

    def _instances_get(self, opts, plugins, url_file_input):
        """
        Creates and returns an ordered dictionary containing instances for all available
        scanning plugins, sort of ordered by popularity.
        @param opts: options as returned by self._options.
        @param plugins: plugins as returned by plugins_util.plugins_base_get.
        @param url_file_input: boolean value which indicates whether we are
            scanning an individual URL or a file. This is used to determine
            kwargs required.
        """
        instances = OrderedDict()
        preferred_order = ['wordpress', 'joomla', 'drupal']

        for cms_name in preferred_order:
            for plugin in plugins:
                plugin_name = plugin.__name__.lower()

                if cms_name == plugin_name:
                    instances[plugin_name] = self._instance_get(plugin, opts,
                            url_file_input)

        for plugin in plugins:
            plugin_name = plugin.__name__.lower()
            if plugin_name not in preferred_order:
                instances[plugin_name] = self._instance_get(plugin, opts,
                        url_file_input)

        return instances

    def _instance_get(self, plugin, opts, url_file_input):
        """
        Return an instance dictionary for an individual plugin.
        @see Scan._instances_get.
        """
        inst = plugin()
        hp, func, enabled_func = inst._general_init(opts)
        name = inst._meta.label
        vf = v.VersionsFile(inst.versions_file)

        kwargs = {
            'hide_progressbar': hp,
            'functionality': func,
            'enabled_functionality': enabled_func
        }

        if url_file_input:
            del kwargs['hide_progressbar']

        return {
            'inst': inst,
            'vf': vf,
            'kwargs': kwargs
        }

<|endoftext|>


<commit_before>#!/usr/bin/python
# -*- coding: utf-8 -*-

import time
import re
import mimetypes
import binascii
import hashlib
from string import Template
from collections import namedtuple

#Evernote API:
from evernote.api.client import EvernoteClient
import evernote.edam.type.ttypes as Types
import evernote.edam.error.ttypes as Errors
from evernote.edam.notestore import NoteStore

import common

logger = common.logger.getChild('my-evernote')

note_link_re = re.compile('evernote\:\/\/\/view\/(?P<uid>\d+)/(?P<sid>s\d+)\/'
                          '(?P<note_id>[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-'
                          '[0-9a-f]{4}-[0-9a-f]{12})\/(?P=note_id)\/')

def ratelimit_wait_and_retry(func):
    def runner(*args, **kwargs):
        while True:
            try:
                return func(*args, **kwargs)
            except Errors.EDAMSystemException, e:
                # TODO: more flexible error handling? callbacks?
                if e.errorCode == Errors.EDAMErrorCode.RATE_LIMIT_REACHED:
                    wait_time = e.rateLimitDuration + 5
                    logger.warn(u'Evernote rate limit reached :-( '
                                u'Waiting %d seconds before retrying' %
                                (wait_time))
                    time.sleep(wait_time)
                    logger.debug(u'Finished waiting for rate limit reset.')
    return runner

class EvernoteApiWrapper():

    @classmethod
    def noteTemplate(cls):
        return Template('\r\n'.join([
          '<?xml version="1.0" encoding="UTF-8"?>',
          '<!DOCTYPE en-note SYSTEM "http://xml.evernote.com/pub/enml2.dtd">',
          '',
          '<en-note style="word-wrap: break-word; -webkit-nbsp-mode: space; '
          '-webkit-line-break: after-white-space;">',
          '${content}',
          '</en-note>',
          ]))

    @classmethod
    def makeNote(cls, title, content, in_notebook_guid=None, resources=None,
                 tags=[], created=None, updated=None):
        note = Types.Note(title=title, tagNames=tags)
        note.content = cls.noteTemplate().safe_substitute({'content': content})
        note.resources = resources
        if in_notebook_guid:
            note.notebookGuid = in_notebook_guid
        note.created = created
        note.updated = updated
        return note

    @classmethod
    def makeResource(cls, src_file, filename, mime=None):
        if not mime:
            mime = mimetypes.guess_type(filename)[0]
        if not mime:
            mime = u'application/octet-stream'
            logger.warning(u'Failed guessing mimetype for "%s" '
                            '(defaulting to "%s")' % (filename, mime))
        # mimetype workarounds:
        if 'image/x-png' == mime:
            # seems like Evernote (Windows) will display
            #  the image inline in the note only this way.
            mime = 'image/png'
        if 'image/pjpeg' == mime:
            # seems like Evernote (Windows) will display
            #  the image inline in the note only this way.
            mime = 'image/jpeg'
        body = src_file.read()
        data = Types.Data(body=body, size=len(body),
                          bodyHash=hashlib.md5(body).digest())
        attr = Types.ResourceAttributes(fileName=filename.encode('utf-8'))
        resource = Types.Resource(data=data, mime=mime, attributes=attr)
        resource_tag = '<en-media type="%s" hash="%s" />' % \
                       (mime, binascii.hexlify(data.bodyHash))
        return resource, resource_tag.encode('utf-8')

    @classmethod
    def parseNoteLinkUrl(cls, url):
        """Returns parsed link object.
        Ref: http://dev.evernote.com/doc/articles/note_links.php
        Currently supporting only notes in synced notebooks, and not linked.
        (e.g. no `client specific id` and `linked notebook guid`)
        """
        link = namedtuple('EvernoteLink', ['user_id', 'shard_id', 'noteGuid',])
        note_link_match = note_link_re.match(url)
        if note_link_match:
            d = note_link_match.groupdict()
            link.user_id = d['uid']
            link.shard_id = d['sid']
            link.noteGuid = d['note_id']
            return link
        else:
            logger.error('Failed parsing Evernote note link %s', url)
            raise RuntimeError()

    def __init__(self, token, sandbox=False):
        self.cached_notebook = None
        self._init_en_client(token, sandbox)
        self._notes_metadata_page_size = 100
        self._notebook_list = None

    @ratelimit_wait_and_retry
    def _listNotebooks(self):
        return self._note_store.listNotebooks()

    def _get_notebook(self, notebook_name):
        if not self._notebook_list:
            self._notebook_list = self._listNotebooks()
        for nb in self._notebook_list:
            if nb.name == notebook_name:
                return nb
        logger.warning(u'Could not find notebook "%s"', notebook_name)

    @property
    def notes_metadata_page_size(self):
        return self._notes_metadata_page_size
    @notes_metadata_page_size.setter
    def notes_metadata_page_size(self, value):
        self._notes_metadata_page_size = value

    def _init_en_client(self, token, sandbox):
        # Client initialization code in dedicated function
        #  to simplify mocking for unit tests.
        self._client = EvernoteClient(token=token, sandbox=sandbox)
        self._note_store = self._client.get_note_store()

    @ratelimit_wait_and_retry
    def _findNotesMetadata(self, *args, **kwargs):
        return self._note_store.findNotesMetadata(*args, **kwargs)

    def _notes_metadata_generator(self, note_filter, spec,
                                  start_offset=0, page_size=None):
        # API call wrapped in generator to simplify pagination and mocking.
        offset = start_offset
        if not page_size:
            page_size = self.notes_metadata_page_size()
        while True:
            notes_metadata = self._findNotesMetadata(self._client.token,
                                                     note_filter,
                                                     offset, page_size,
                                                     spec)
            for note_offset, note in enumerate(notes_metadata.notes,
                                               offset):
                # yield also note offset in query,
                #  to allow efficient re-entry in case of rate limit.
                yield note_offset, note
            if notes_metadata.startIndex + page_size >= \
                    notes_metadata.totalNotes:
                break
            offset += page_size

    def getNotesByTitle(self, title, in_notebook=None, page_size=None):
        notebook = in_notebook and self._get_notebook(in_notebook)
        # remove occurrences of '"' because Evernote ignores it in search
        query = 'intitle:"%s"' % (title.replace('"', '').encode('utf-8'))
        note_filter = NoteStore.NoteFilter(words=query,
                                       notebookGuid=notebook and notebook.guid)
        spec = NoteStore.NotesMetadataResultSpec(includeTitle=True,
                                                 includeUpdated=True)
        return self._notes_metadata_generator(note_filter, spec,
                                              page_size=page_size)

    def getSingleNoteByTitle(self, title, in_notebook=None):
        ret_note = None
        for offset, note_metadata in self.getNotesByTitle(title,
                                                          in_notebook, 2):
            if 0 == offset:
                ret_note = note_metadata
            else:
                logger.warn('More than 1 note matches query')
                break
        return ret_note

    @ratelimit_wait_and_retry
    def _createNote(self, note):
        self._note_store.createNote(self._client.token, note)

    def saveNoteToNotebook(self, note, in_notebook=None):
        # If no notebook specified - default notebook is used
        notebook = in_notebook and self._get_notebook(in_notebook)
        if notebook:
            note.notebookGuid = notebook.guid
        logger.info('Saving note "%s" to Evernote' % (note.title))
        self._createNote(note)

    @ratelimit_wait_and_retry
    def updateNote(self, note):
        self._note_store.updateNote(self._client.token, note)

    """Get Evernote Note object by GUID

    @param `note_guid`: The requested note GUID
    @param `with_content_flag`: If `True`, includes note content in response
    """
    @ratelimit_wait_and_retry
    def getNote(self, note_guid, with_content_flag=True):
        return self._note_store.getNote(self._client.token,
                                        note_guid, with_content_flag,
                                        False, False, False)
<commit_msg>docstring styling<commit_after>#!/usr/bin/python
# -*- coding: utf-8 -*-

import time
import re
import mimetypes
import binascii
import hashlib
from string import Template
from collections import namedtuple

#Evernote API:
from evernote.api.client import EvernoteClient
import evernote.edam.type.ttypes as Types
import evernote.edam.error.ttypes as Errors
from evernote.edam.notestore import NoteStore

import common

logger = common.logger.getChild('my-evernote')

note_link_re = re.compile('evernote\:\/\/\/view\/(?P<uid>\d+)/(?P<sid>s\d+)\/'
                          '(?P<note_id>[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-'
                          '[0-9a-f]{4}-[0-9a-f]{12})\/(?P=note_id)\/')

def ratelimit_wait_and_retry(func):
    def runner(*args, **kwargs):
        while True:
            try:
                return func(*args, **kwargs)
            except Errors.EDAMSystemException, e:
                # TODO: more flexible error handling? callbacks?
                if e.errorCode == Errors.EDAMErrorCode.RATE_LIMIT_REACHED:
                    wait_time = e.rateLimitDuration + 5
                    logger.warn(u'Evernote rate limit reached :-( '
                                u'Waiting %d seconds before retrying' %
                                (wait_time))
                    time.sleep(wait_time)
                    logger.debug(u'Finished waiting for rate limit reset.')
    return runner

class EvernoteApiWrapper():

    @classmethod
    def noteTemplate(cls):
        return Template('\r\n'.join([
          '<?xml version="1.0" encoding="UTF-8"?>',
          '<!DOCTYPE en-note SYSTEM "http://xml.evernote.com/pub/enml2.dtd">',
          '',
          '<en-note style="word-wrap: break-word; -webkit-nbsp-mode: space; '
          '-webkit-line-break: after-white-space;">',
          '${content}',
          '</en-note>',
          ]))

    @classmethod
    def makeNote(cls, title, content, in_notebook_guid=None, resources=None,
                 tags=[], created=None, updated=None):
        note = Types.Note(title=title, tagNames=tags)
        note.content = cls.noteTemplate().safe_substitute({'content': content})
        note.resources = resources
        if in_notebook_guid:
            note.notebookGuid = in_notebook_guid
        note.created = created
        note.updated = updated
        return note

    @classmethod
    def makeResource(cls, src_file, filename, mime=None):
        if not mime:
            mime = mimetypes.guess_type(filename)[0]
        if not mime:
            mime = u'application/octet-stream'
            logger.warning(u'Failed guessing mimetype for "%s" '
                            '(defaulting to "%s")' % (filename, mime))
        # mimetype workarounds:
        if 'image/x-png' == mime:
            # seems like Evernote (Windows) will display
            #  the image inline in the note only this way.
            mime = 'image/png'
        if 'image/pjpeg' == mime:
            # seems like Evernote (Windows) will display
            #  the image inline in the note only this way.
            mime = 'image/jpeg'
        body = src_file.read()
        data = Types.Data(body=body, size=len(body),
                          bodyHash=hashlib.md5(body).digest())
        attr = Types.ResourceAttributes(fileName=filename.encode('utf-8'))
        resource = Types.Resource(data=data, mime=mime, attributes=attr)
        resource_tag = '<en-media type="%s" hash="%s" />' % \
                       (mime, binascii.hexlify(data.bodyHash))
        return resource, resource_tag.encode('utf-8')

    @classmethod
    def parseNoteLinkUrl(cls, url):
        """Returns parsed link object.
        Ref: http://dev.evernote.com/doc/articles/note_links.php
        Currently supporting only notes in synced notebooks, and not linked.
        (e.g. no `client specific id` and `linked notebook guid`)
        """
        link = namedtuple('EvernoteLink', ['user_id', 'shard_id', 'noteGuid',])
        note_link_match = note_link_re.match(url)
        if note_link_match:
            d = note_link_match.groupdict()
            link.user_id = d['uid']
            link.shard_id = d['sid']
            link.noteGuid = d['note_id']
            return link
        else:
            logger.error('Failed parsing Evernote note link %s', url)
            raise RuntimeError()

    def __init__(self, token, sandbox=False):
        self.cached_notebook = None
        self._init_en_client(token, sandbox)
        self._notes_metadata_page_size = 100
        self._notebook_list = None

    @ratelimit_wait_and_retry
    def _listNotebooks(self):
        return self._note_store.listNotebooks()

    def _get_notebook(self, notebook_name):
        if not self._notebook_list:
            self._notebook_list = self._listNotebooks()
        for nb in self._notebook_list:
            if nb.name == notebook_name:
                return nb
        logger.warning(u'Could not find notebook "%s"', notebook_name)

    @property
    def notes_metadata_page_size(self):
        return self._notes_metadata_page_size
    @notes_metadata_page_size.setter
    def notes_metadata_page_size(self, value):
        self._notes_metadata_page_size = value

    def _init_en_client(self, token, sandbox):
        # Client initialization code in dedicated function
        #  to simplify mocking for unit tests.
        self._client = EvernoteClient(token=token, sandbox=sandbox)
        self._note_store = self._client.get_note_store()

    @ratelimit_wait_and_retry
    def _findNotesMetadata(self, *args, **kwargs):
        return self._note_store.findNotesMetadata(*args, **kwargs)

    def _notes_metadata_generator(self, note_filter, spec,
                                  start_offset=0, page_size=None):
        # API call wrapped in generator to simplify pagination and mocking.
        offset = start_offset
        if not page_size:
            page_size = self.notes_metadata_page_size()
        while True:
            notes_metadata = self._findNotesMetadata(self._client.token,
                                                     note_filter,
                                                     offset, page_size,
                                                     spec)
            for note_offset, note in enumerate(notes_metadata.notes,
                                               offset):
                # yield also note offset in query,
                #  to allow efficient re-entry in case of rate limit.
                yield note_offset, note
            if notes_metadata.startIndex + page_size >= \
                    notes_metadata.totalNotes:
                break
            offset += page_size

    def getNotesByTitle(self, title, in_notebook=None, page_size=None):
        notebook = in_notebook and self._get_notebook(in_notebook)
        # remove occurrences of '"' because Evernote ignores it in search
        query = 'intitle:"%s"' % (title.replace('"', '').encode('utf-8'))
        note_filter = NoteStore.NoteFilter(words=query,
                                       notebookGuid=notebook and notebook.guid)
        spec = NoteStore.NotesMetadataResultSpec(includeTitle=True,
                                                 includeUpdated=True)
        return self._notes_metadata_generator(note_filter, spec,
                                              page_size=page_size)

    def getSingleNoteByTitle(self, title, in_notebook=None):
        ret_note = None
        for offset, note_metadata in self.getNotesByTitle(title,
                                                          in_notebook, 2):
            if 0 == offset:
                ret_note = note_metadata
            else:
                logger.warn('More than 1 note matches query')
                break
        return ret_note

    @ratelimit_wait_and_retry
    def _createNote(self, note):
        self._note_store.createNote(self._client.token, note)

    def saveNoteToNotebook(self, note, in_notebook=None):
        # If no notebook specified - default notebook is used
        notebook = in_notebook and self._get_notebook(in_notebook)
        if notebook:
            note.notebookGuid = notebook.guid
        logger.info('Saving note "%s" to Evernote' % (note.title))
        self._createNote(note)

    @ratelimit_wait_and_retry
    def updateNote(self, note):
        """Update a note in the Evernote note store.

        :param note: The note to update.
        :type note: Types.Note
        """
        self._note_store.updateNote(self._client.token, note)

    @ratelimit_wait_and_retry
    def getNote(self, note_guid, with_content_flag=True):
        """Get Evernote Note object by GUID.

        :param note_guid: The requested note GUID
        :param with_content_flag: If `True`, includes note content in response
        """
        return self._note_store.getNote(self._client.token,
                                        note_guid, with_content_flag,
                                        False, False, False)
<|endoftext|>


<commit_before>import os
import json
import requests
from downstream_cherry_picker import get_sha_range

TESTS_DIR = os.path.dirname(os.path.abspath(__file__))
FIXTURES_DIR = os.path.join(TESTS_DIR, 'fixtures')

# https://help.github.com/articles/why-are-my-commits-in-the-wrong-order/

class MockResponse(object):
    def raise_for_status(self):
        pass

    def json(self):
        # Return data from our static fixture file
        # (eg from https://api.github.com/repos/ceph/ceph/pulls/12917)
        # (or from https://api.github.com/repos/ceph/ceph/pulls/12917/commits)
        path = self.url.replace('https://api.github.com/repos/ceph/ceph',
                                FIXTURES_DIR)
        filename = path + '.json'
        with open(filename) as fp:
            return json.load(fp)

def mock_get(url):
    m = MockResponse()
    m.url = url
    return m

class TestGetShaRange(object):

    def test_get_sha_range(self, monkeypatch):
        monkeypatch.delattr('requests.sessions.Session.request')
        monkeypatch.setattr(requests, 'get', mock_get)
        (first, last) = get_sha_range('ceph', 'ceph', 12917)
        assert first == '386640865dee30d38f17e55fc87535e419bc3cb5'
        assert last == '14a6aabe22f68436ea3297ce0851700f86ee5b12'
<commit_msg>tests: parameterize test_get_sha_range<commit_after>import pytest
import os
import json
import requests
from downstream_cherry_picker import get_sha_range

TESTS_DIR = os.path.dirname(os.path.abspath(__file__))
FIXTURES_DIR = os.path.join(TESTS_DIR, 'fixtures')

# https://help.github.com/articles/why-are-my-commits-in-the-wrong-order/

class MockResponse(object):
    def raise_for_status(self):
        pass

    def json(self):
        # Return data from our static fixture file
        # (eg from https://api.github.com/repos/ceph/ceph/pulls/12917)
        # (or from https://api.github.com/repos/ceph/ceph/pulls/12917/commits)
        path = self.url.replace('https://api.github.com/repos/ceph/ceph',
                                FIXTURES_DIR)
        filename = path + '.json'
        with open(filename) as fp:
            return json.load(fp)

def mock_get(url):
    m = MockResponse()
    m.url = url
    return m

class TestGetShaRange(object):

    @pytest.mark.parametrize('pr,first,last', [
        (12917, '386640865dee30d38f17e55fc87535e419bc3cb5',
                '14a6aabe22f68436ea3297ce0851700f86ee5b12'),
    ])
    def test_get_sha_range(self, monkeypatch, pr, first, last):
        monkeypatch.delattr('requests.sessions.Session.request')
        monkeypatch.setattr(requests, 'get', mock_get)
        (got_first, got_last) = get_sha_range('ceph', 'ceph', pr)
        assert got_first == first
        assert got_last == last
<|endoftext|>


<commit_before># coding=utf-8
import random

import matplotlib.pyplot as plt
import numpy as np
import pandas as pd
from sklearn.linear_model import LinearRegression
from sklearn.model_selection import train_test_split

# 构造训练数据
original_data = pd.read_csv(
        "pima-indians-diabetes.data.csv",
        sep=',',
        engine='python').values

input_data, test_data, target_data, test_target = train_test_split(
    original_data[:, :-1], original_data[:, -1:], train_size=0.75)
m = min(len(input_data[0]), random.randint(50, 256))  # 训练数据点数目
features = len(input_data[0])

# 两种终止条件
loop_max = 10000  # 最大迭代次数(防止死循环)
epsilon = 1e-3

# 初始化权值
np.random.seed(0)
w = np.random.randn(features)

alpha = 0.0005  # 步长
diff = 0.
error = np.zeros(features)
count = 0  # 循环次数
finish = 0  # 终止标志
error_list = []

# -------------------------------------------mini-BGD----------------------------------------------------------
while count < loop_max:
    count += 1

    # 遍历训练数据集，不断更新权值
    sum_m = np.zeros(features)

    for i in range(m):
        diff = (np.dot(w, input_data[i]) - target_data[i]) * target_data[i]  # 训练集代入,计算误差值
        # 采用min-BGD,对部分样例求和
        sum_m = sum_m + diff

    w = w - alpha * (sum_m / m)
    error_list.append(1.0 / (2 * m) * np.sum(sum_m)**2)
    if np.linalg.norm(w - error) < epsilon:     # 终止条件：前后两次计算出的权向量的绝对误差充分小
        finish = 1
        break
    else:
        error = w

# ----------------------------------------------------------------------------------------------------------------------

regressor = LinearRegression()
regressor.fit(input_data, target_data)

print regressor.score(test_data, test_target)

plt.plot(range(len(error_list[0:100])), error_list[0:100])
plt.show()
<commit_msg>add bias to input data<commit_after># coding=utf-8
import random

import matplotlib.pyplot as plt
import numpy as np
import pandas as pd
from sklearn.linear_model import LinearRegression
from sklearn.model_selection import train_test_split

# 构造训练数据
original_data = pd.read_csv(
        "pima-indians-diabetes.data.csv",
        sep=',',
        engine='python').values

train_data, test_data, target_data, test_target = train_test_split(
    original_data[:, :-1], original_data[:, -1:], train_size=0.75)
m = min(len(train_data[0]), random.randint(50, 256))  # 训练数据点数目
features = len(train_data[0])

input_data = np.c_[np.ones(features), train_data]

# 两种终止条件
loop_max = 10000  # 最大迭代次数(防止死循环)
epsilon = 1e-3

# 初始化权值
np.random.seed(0)
w = np.random.randn(features)

alpha = 0.0005  # 步长
diff = 0.
error = np.zeros(features)
count = 0  # 循环次数
finish = 0  # 终止标志
error_list = []

# -------------------------------------------mini-BGD----------------------------------------------------------
while count < loop_max:
    count += 1

    # 遍历训练数据集，不断更新权值
    sum_m = np.zeros(features)

    for i in range(m):
        diff = (np.dot(w, input_data[i]) - target_data[i]) * target_data[i]  # 训练集代入,计算误差值
        # 采用min-BGD,对部分样例求和
        sum_m = sum_m + diff

    w = w - alpha * (sum_m / m)
    error_list.append(1.0 / (2 * m) * np.sum(sum_m)**2)
    if np.linalg.norm(w - error) < epsilon:     # 终止条件：前后两次计算出的权向量的绝对误差充分小
        finish = 1
        break
    else:
        error = w

# ----------------------------------------------------------------------------------------------------------------------

regressor = LinearRegression()
regressor.fit(train_data, target_data)

print regressor.score(test_data, test_target)

plt.plot(range(len(error_list[0:100])), error_list[0:100])
plt.show()
<|endoftext|>