<commit_before>import csv
import gzip
import pymongo
import os
from collections import defaultdict

ADMINS = (
    ('Brett Thomas', 'brettpthomas@gmail.com'),
    ('Ben Weisburd', 'weisburd@broadinstitute.org'),
)

MANAGERS = ADMINS

TIME_ZONE = 'America/New_York'

LANGUAGE_CODE = 'en-us'

SITE_ID = 1

USE_I18N = True

USE_L10N = True

USE_TZ = True

STATICFILES_FINDERS = (
    'django.contrib.staticfiles.finders.FileSystemFinder',
    'django.contrib.staticfiles.finders.AppDirectoriesFinder',
    'compressor.finders.CompressorFinder',
)

TEMPLATE_LOADERS = (
    'django.template.loaders.filesystem.Loader',
    'django.template.loaders.app_directories.Loader',
)

MIDDLEWARE_CLASSES = (
    'django.middleware.common.CommonMiddleware',
    'django.contrib.sessions.middleware.SessionMiddleware',
    'django.middleware.csrf.CsrfViewMiddleware',
    'django.contrib.auth.middleware.AuthenticationMiddleware',
    'django.contrib.messages.middleware.MessageMiddleware',
    'django.middleware.clickjacking.XFrameOptionsMiddleware',
)

ROOT_URLCONF = 'xbrowse_server.urls'

WSGI_APPLICATION = 'wsgi.application'

TEMPLATE_DIRS = (
    os.path.dirname(os.path.realpath(__file__)) + '/xbrowse_server/templates/',
)

CRISPY_TEMPLATE_PACK = 'bootstrap3'

INSTALLED_APPS = (
    'django.contrib.auth',
    'django.contrib.contenttypes',
    'django.contrib.sessions',
    'django.contrib.sites',
    'django.contrib.messages',
    'django.contrib.staticfiles',

    'django.contrib.admin',
    'django.contrib.admindocs',

    'django_extensions',
    'compressor',
    'crispy_forms',

    'datasets',

    'xbrowse_server.base.apps.XBrowseBaseConfig',
    'xbrowse_server.api',
    'xbrowse_server.staff',
    'xbrowse_server.gene_lists',
    'xbrowse_server.search_cache',

    )

LOGGING = {
    'version': 1,
    'disable_existing_loggers': False,
    'filters': {
        'require_debug_false': {
            '()': 'django.utils.log.RequireDebugFalse'
        }
    },
    'handlers': {
        'mail_admins': {
            'level': 'ERROR',
            'filters': ['require_debug_false'],
            'class': 'django.utils.log.AdminEmailHandler'
        }
    },
    'loggers': {
        'django.request': {
            'handlers': ['mail_admins'],
            'level': 'ERROR',
            'propagate': True,
        },
        'xbrowse_server': {
            'handlers': ['mail_admins'],
            'level': 'ERROR',
            'propagate': True,
        },
    }
}

TEMPLATE_CONTEXT_PROCESSORS = (
    "django.contrib.auth.context_processors.auth",
    "django.core.context_processors.debug",
    "django.core.context_processors.i18n",
    "django.core.context_processors.media",
    "django.core.context_processors.static",
    "django.core.context_processors.tz",
    "django.contrib.messages.context_processors.messages",

    "xbrowse_server.base.context_processors.custom_processor",
)

TEST_RUNNER = 'django.test.runner.DiscoverRunner'

SESSION_COOKIE_NAME = "xsessionid"

AUTH_PROFILE_MODULE = 'base.UserProfile'

LOGGING_DB = pymongo.Connection().logging

UTILS_DB = pymongo.Connection().xbrowse_server_utils

FROM_EMAIL = "\"xBrowse\" <xbrowse@broadinstitute.org>"

XBROWSE_VERSION = 0.1

DOCS_DIR = os.path.dirname(os.path.realpath(__file__)) + '/xbrowse_server/user_docs/'

SHELL_PLUS_POST_IMPORTS = (
    ('xbrowse_server.shell_helpers', 'getproj'),
    ('xbrowse_server', 'mall'),
)

FAMILY_LOAD_BATCH_SIZE = 25000

ANNOTATION_BATCH_SIZE = 25000

# defaults for optional local settings
CONSTRUCTION_TEMPLATE = None
CLINVAR_TSV = None

from local_settings import *
#
# These are all settings that require the stuff in local_settings.py
#

ANNOTATOR_REFERENCE_POPULATIONS = ANNOTATOR_SETTINGS.reference_populations
ANNOTATOR_REFERENCE_POPULATION_SLUGS = [pop['slug'] for pop in ANNOTATOR_SETTINGS.reference_populations]

TEMPLATE_DEBUG = DEBUG

MEDIA_URL = URL_PREFIX + 'media/'

STATIC_URL = URL_PREFIX + 'static/'

LOGIN_URL = BASE_URL + 'login'

LOGOUT_URL = BASE_URL + 'logout'

CSRF_COOKIE_PATH = URL_PREFIX.rstrip('/')
SESSION_COOKIE_PATH = URL_PREFIX.rstrip('/')

# If supported by the browser, using the HttpOnly flag
# when generating a cookie helps mitigate the risk of client side script accessing the protected cookie. If a browser that supports HttpOnly
# detects a cookie containing the HttpOnly flag, and client side script code attempts to read the cookie, the browser returns an empty
# string as the result. This causes the attack to fail by preventing the malicious (usually XSS) code from sending the data to an attacker's website.
SESSION_COOKIE_HTTPONLY = True
CSRF_COOKIE_HTTPONLY = True
SESSION_COOKIE_SECURE = True
CSRF_COOKIE_SECURE = True
# SESSION_EXPIRE_AT_BROWSER_CLOSE=True

CLINVAR_VARIANTS = {} # maps (xpos, ref, alt) to a 2-tuple containing (measureset_id, clinical_significance)
if CLINVAR_TSV and os.path.isfile(CLINVAR_TSV):
    from xbrowse.core.genomeloc import get_xpos
    header = None
    pathogenicity_values_counter = defaultdict(int)
    #print("Reading Clinvar data into memory: " + CLINVAR_TSV)
    for line in open(CLINVAR_TSV):
        line = line.strip()
        if line.startswith("#"):
            continue
        fields = line.split("\t")
        if header is None:
            header = fields
        else:
            line_dict = dict(zip(header, fields))
            chrom = line_dict["chrom"]
            pos = int(line_dict["pos"])
            ref = line_dict["ref"]
            alt = line_dict["alt"]
            if "M" in chrom:
                continue   # because get_xpos doesn't support chrMT.
            clinical_significance = line_dict["clinical_significance"].lower()
            if clinical_significance in ["not provided", "other", "association"]:
                continue
            else:
                for c in clinical_significance.split(";"):
                    pathogenicity_values_counter[c] += 1
            xpos = get_xpos(chrom, pos)
            CLINVAR_VARIANTS[(xpos, ref, alt)] = (line_dict["measureset_id"], clinical_significance)
    #for k in sorted(pathogenicity_values_counter.keys(), key=lambda k: -pathogenicity_values_counter[k]):
    #    print("     %5d  %s"  % (pathogenicity_values_counter[k], k))

# set the secret key
if os.access("/etc/xbrowse_django_secret_key", os.R_OK):
    with open("/etc/xbrowse_django_secret_key") as f:
        SECRET_KEY = f.read().strip()
else:
    print("Warning: could not access /etc/xbrowse_django_secret_key. Falling back on insecure hard-coded SECRET_KEY")
    SECRET_KEY = "~~~ this key string is FOR DEVELOPMENT USE ONLY ~~~"

<commit_msg>hotfix: only set SESSION_COOKIE_SECURE=True and CSRF_COOKIE_SECURE=True on servers with HTTPS to avoid CSRF error<commit_after>import csv
import gzip
import pymongo
import os
from collections import defaultdict

ADMINS = (
    ('Brett Thomas', 'brettpthomas@gmail.com'),
    ('Ben Weisburd', 'weisburd@broadinstitute.org'),
)

MANAGERS = ADMINS

TIME_ZONE = 'America/New_York'

LANGUAGE_CODE = 'en-us'

SITE_ID = 1

USE_I18N = True

USE_L10N = True

USE_TZ = True

STATICFILES_FINDERS = (
    'django.contrib.staticfiles.finders.FileSystemFinder',
    'django.contrib.staticfiles.finders.AppDirectoriesFinder',
    'compressor.finders.CompressorFinder',
)

TEMPLATE_LOADERS = (
    'django.template.loaders.filesystem.Loader',
    'django.template.loaders.app_directories.Loader',
)

MIDDLEWARE_CLASSES = (
    'django.middleware.common.CommonMiddleware',
    'django.contrib.sessions.middleware.SessionMiddleware',
    'django.middleware.csrf.CsrfViewMiddleware',
    'django.contrib.auth.middleware.AuthenticationMiddleware',
    'django.contrib.messages.middleware.MessageMiddleware',
    'django.middleware.clickjacking.XFrameOptionsMiddleware',
)

ROOT_URLCONF = 'xbrowse_server.urls'

WSGI_APPLICATION = 'wsgi.application'

TEMPLATE_DIRS = (
    os.path.dirname(os.path.realpath(__file__)) + '/xbrowse_server/templates/',
)

CRISPY_TEMPLATE_PACK = 'bootstrap3'

INSTALLED_APPS = (
    'django.contrib.auth',
    'django.contrib.contenttypes',
    'django.contrib.sessions',
    'django.contrib.sites',
    'django.contrib.messages',
    'django.contrib.staticfiles',

    'django.contrib.admin',
    'django.contrib.admindocs',

    'django_extensions',
    'compressor',
    'crispy_forms',

    'datasets',

    'xbrowse_server.base.apps.XBrowseBaseConfig',
    'xbrowse_server.api',
    'xbrowse_server.staff',
    'xbrowse_server.gene_lists',
    'xbrowse_server.search_cache',

    )

LOGGING = {
    'version': 1,
    'disable_existing_loggers': False,
    'filters': {
        'require_debug_false': {
            '()': 'django.utils.log.RequireDebugFalse'
        }
    },
    'handlers': {
        'mail_admins': {
            'level': 'ERROR',
            'filters': ['require_debug_false'],
            'class': 'django.utils.log.AdminEmailHandler'
        }
    },
    'loggers': {
        'django.request': {
            'handlers': ['mail_admins'],
            'level': 'ERROR',
            'propagate': True,
        },
        'xbrowse_server': {
            'handlers': ['mail_admins'],
            'level': 'ERROR',
            'propagate': True,
        },
    }
}

TEMPLATE_CONTEXT_PROCESSORS = (
    "django.contrib.auth.context_processors.auth",
    "django.core.context_processors.debug",
    "django.core.context_processors.i18n",
    "django.core.context_processors.media",
    "django.core.context_processors.static",
    "django.core.context_processors.tz",
    "django.contrib.messages.context_processors.messages",

    "xbrowse_server.base.context_processors.custom_processor",
)

TEST_RUNNER = 'django.test.runner.DiscoverRunner'

SESSION_COOKIE_NAME = "xsessionid"

AUTH_PROFILE_MODULE = 'base.UserProfile'

LOGGING_DB = pymongo.Connection().logging

UTILS_DB = pymongo.Connection().xbrowse_server_utils

FROM_EMAIL = "\"xBrowse\" <xbrowse@broadinstitute.org>"

XBROWSE_VERSION = 0.1

DOCS_DIR = os.path.dirname(os.path.realpath(__file__)) + '/xbrowse_server/user_docs/'

SHELL_PLUS_POST_IMPORTS = (
    ('xbrowse_server.shell_helpers', 'getproj'),
    ('xbrowse_server', 'mall'),
)

FAMILY_LOAD_BATCH_SIZE = 25000

ANNOTATION_BATCH_SIZE = 25000

# defaults for optional local settings
CONSTRUCTION_TEMPLATE = None
CLINVAR_TSV = None

from local_settings import *
#
# These are all settings that require the stuff in local_settings.py
#

ANNOTATOR_REFERENCE_POPULATIONS = ANNOTATOR_SETTINGS.reference_populations
ANNOTATOR_REFERENCE_POPULATION_SLUGS = [pop['slug'] for pop in ANNOTATOR_SETTINGS.reference_populations]

TEMPLATE_DEBUG = DEBUG

MEDIA_URL = URL_PREFIX + 'media/'

STATIC_URL = URL_PREFIX + 'static/'

LOGIN_URL = BASE_URL + 'login'

LOGOUT_URL = BASE_URL + 'logout'

CSRF_COOKIE_PATH = URL_PREFIX.rstrip('/')
SESSION_COOKIE_PATH = URL_PREFIX.rstrip('/')

# If supported by the browser, using the HttpOnly flag
# when generating a cookie helps mitigate the risk of client side script accessing the protected cookie. If a browser that supports HttpOnly
# detects a cookie containing the HttpOnly flag, and client side script code attempts to read the cookie, the browser returns an empty
# string as the result. This causes the attack to fail by preventing the malicious (usually XSS) code from sending the data to an attacker's website.
SESSION_COOKIE_HTTPONLY = True
CSRF_COOKIE_HTTPONLY = True
# SESSION_EXPIRE_AT_BROWSER_CLOSE=True

CLINVAR_VARIANTS = {} # maps (xpos, ref, alt) to a 2-tuple containing (measureset_id, clinical_significance)
if CLINVAR_TSV and os.path.isfile(CLINVAR_TSV):
    from xbrowse.core.genomeloc import get_xpos
    header = None
    pathogenicity_values_counter = defaultdict(int)
    #print("Reading Clinvar data into memory: " + CLINVAR_TSV)
    for line in open(CLINVAR_TSV):
        line = line.strip()
        if line.startswith("#"):
            continue
        fields = line.split("\t")
        if header is None:
            header = fields
        else:
            line_dict = dict(zip(header, fields))
            chrom = line_dict["chrom"]
            pos = int(line_dict["pos"])
            ref = line_dict["ref"]
            alt = line_dict["alt"]
            if "M" in chrom:
                continue   # because get_xpos doesn't support chrMT.
            clinical_significance = line_dict["clinical_significance"].lower()
            if clinical_significance in ["not provided", "other", "association"]:
                continue
            else:
                for c in clinical_significance.split(";"):
                    pathogenicity_values_counter[c] += 1
            xpos = get_xpos(chrom, pos)
            CLINVAR_VARIANTS[(xpos, ref, alt)] = (line_dict["measureset_id"], clinical_significance)
    #for k in sorted(pathogenicity_values_counter.keys(), key=lambda k: -pathogenicity_values_counter[k]):
    #    print("     %5d  %s"  % (pathogenicity_values_counter[k], k))

# set the secret key
if os.access("/etc/xbrowse_django_secret_key", os.R_OK):
    with open("/etc/xbrowse_django_secret_key") as f:
        SECRET_KEY = f.read().strip()

    SESSION_COOKIE_SECURE = True
    CSRF_COOKIE_SECURE = True

else:
    print("Warning: could not access /etc/xbrowse_django_secret_key. Falling back on insecure hard-coded SECRET_KEY")
    SECRET_KEY = "~~~ this key string is FOR DEVELOPMENT USE ONLY ~~~"
<|endoftext|>


<commit_before>"""
Installs and configures MySQL
"""

import uuid
import logging

import packstack.installer.engine_validators as validate
from packstack.installer import basedefs
import packstack.installer.common_utils as utils

from packstack.modules.ospluginutils import getManifestTemplate, appendManifestFile

# Controller object will be initialized from main flow
controller = None

# Plugin name
PLUGIN_NAME = "OS-MySQL"
PLUGIN_NAME_COLORED = utils.getColoredText(PLUGIN_NAME, basedefs.BLUE)

logging.debug("plugin %s loaded", __name__)

def initConfig(controllerObject):
    global controller
    controller = controllerObject
    logging.debug("Adding MySQL Openstack configuration")
    paramsList = [
                  {"CMD_OPTION"      : "mysql-host",
                   "USAGE"           : "The IP address of the server on which to install MySQL",
                   "PROMPT"          : "The IP address of the server on which to install MySQL",
                   "OPTION_LIST"     : [],
                   "VALIDATION_FUNC" : validate.validatePing,
                   "DEFAULT_VALUE"   : "127.0.0.1",
                   "MASK_INPUT"      : False,
                   "LOOSE_VALIDATION": True,
                   "CONF_NAME"       : "CONFIG_MYSQL_HOST",
                   "USE_DEFAULT"     : False,
                   "NEED_CONFIRM"    : False,
                   "CONDITION"       : False },
                  {"CMD_OPTION"      : "mysql-user",
                   "USAGE"           : "Username of the MySQL admin user",
                   "PROMPT"          : "Username of the MySQL admin user",
                   "OPTION_LIST"     : [],
                   "VALIDATION_FUNC" : validate.validateStringNotEmpty,
                   "DEFAULT_VALUE"   : "root",
                   "MASK_INPUT"      : False,
                   "LOOSE_VALIDATION": False,
                   "CONF_NAME"       : "CONFIG_MYSQL_USER",
                   "USE_DEFAULT"     : True,
                   "NEED_CONFIRM"    : False,
                   "CONDITION"       : False },
                  {"CMD_OPTION"      : "mysql-pw",
                   "USAGE"           : "Password for the MySQL admin user",
                   "PROMPT"          : "Password for the MySQL admin user",
                   "OPTION_LIST"     : [],
                   "VALIDATION_FUNC" : validate.validateStringNotEmpty,
                   "DEFAULT_VALUE"   : uuid.uuid4().hex[:6],
                   "MASK_INPUT"      : True,
                   "LOOSE_VALIDATION": True,
                   "CONF_NAME"       : "CONFIG_MYSQL_PW",
                   "USE_DEFAULT"     : True,
                   "NEED_CONFIRM"    : True,
                   "CONDITION"       : False },
                 ]

    groupDict = { "GROUP_NAME"            : "MYSQL",
                  "DESCRIPTION"           : "MySQL Config paramaters",
                  "PRE_CONDITION"         : utils.returnYes,
                  "PRE_CONDITION_MATCH"   : "yes",
                  "POST_CONDITION"        : False,
                  "POST_CONDITION_MATCH"  : True}

    controller.addGroup(groupDict, paramsList)

def initSequences(controller):
    mysqlsteps = [
             {'title': 'Create MySQL Manifest',
              'functions':[createmanifest]}
    ]
    controller.addSequence("Installing MySQL", [], [], mysqlsteps)

def createmanifest():
    manifestfile = "%s_mysql.pp"%controller.CONF['CONFIG_MYSQL_HOST']
    manifestdata = getManifestTemplate("mysql.pp")
    appendManifestFile(manifestfile, manifestdata, 'pre')
<commit_msg>Added hostname processor for MySQL plugin<commit_after>"""
Installs and configures MySQL
"""

import uuid
import logging

import packstack.installer.engine_validators as validate
import packstack.installer.engine_processors as process
from packstack.installer import basedefs
import packstack.installer.common_utils as utils

from packstack.modules.ospluginutils import getManifestTemplate, appendManifestFile

# Controller object will be initialized from main flow
controller = None

# Plugin name
PLUGIN_NAME = "OS-MySQL"
PLUGIN_NAME_COLORED = utils.getColoredText(PLUGIN_NAME, basedefs.BLUE)

logging.debug("plugin %s loaded", __name__)

def initConfig(controllerObject):
    global controller
    controller = controllerObject
    logging.debug("Adding MySQL Openstack configuration")
    paramsList = [
                  {"CMD_OPTION"      : "mysql-host",
                   "USAGE"           : "The IP address of the server on which to install MySQL",
                   "PROMPT"          : "The IP address of the server on which to install MySQL",
                   "OPTION_LIST"     : [],
                   "VALIDATION_FUNC" : validate.validatePing,
                   "DEFAULT_VALUE"   : "127.0.0.1",
                   "PROCESSOR_ARGS"  : {"allow_localhost": True},
                   "PROCESSOR_FUNC"  : process.processHost,
                   "PROCESSOR_MSG"   : "WARN_VAL_IS_HOSTNAME",
                   "MASK_INPUT"      : False,
                   "LOOSE_VALIDATION": True,
                   "CONF_NAME"       : "CONFIG_MYSQL_HOST",
                   "USE_DEFAULT"     : False,
                   "NEED_CONFIRM"    : False,
                   "CONDITION"       : False },
                  {"CMD_OPTION"      : "mysql-user",
                   "USAGE"           : "Username of the MySQL admin user",
                   "PROMPT"          : "Username of the MySQL admin user",
                   "OPTION_LIST"     : [],
                   "VALIDATION_FUNC" : validate.validateStringNotEmpty,
                   "DEFAULT_VALUE"   : "root",
                   "MASK_INPUT"      : False,
                   "LOOSE_VALIDATION": False,
                   "CONF_NAME"       : "CONFIG_MYSQL_USER",
                   "USE_DEFAULT"     : True,
                   "NEED_CONFIRM"    : False,
                   "CONDITION"       : False },
                  {"CMD_OPTION"      : "mysql-pw",
                   "USAGE"           : "Password for the MySQL admin user",
                   "PROMPT"          : "Password for the MySQL admin user",
                   "OPTION_LIST"     : [],
                   "VALIDATION_FUNC" : validate.validateStringNotEmpty,
                   "DEFAULT_VALUE"   : uuid.uuid4().hex[:6],
                   "MASK_INPUT"      : True,
                   "LOOSE_VALIDATION": True,
                   "CONF_NAME"       : "CONFIG_MYSQL_PW",
                   "USE_DEFAULT"     : True,
                   "NEED_CONFIRM"    : True,
                   "CONDITION"       : False },
                 ]

    groupDict = { "GROUP_NAME"            : "MYSQL",
                  "DESCRIPTION"           : "MySQL Config paramaters",
                  "PRE_CONDITION"         : utils.returnYes,
                  "PRE_CONDITION_MATCH"   : "yes",
                  "POST_CONDITION"        : False,
                  "POST_CONDITION_MATCH"  : True}

    controller.addGroup(groupDict, paramsList)

def initSequences(controller):
    mysqlsteps = [
             {'title': 'Create MySQL Manifest',
              'functions':[createmanifest]}
    ]
    controller.addSequence("Installing MySQL", [], [], mysqlsteps)

def createmanifest():
    manifestfile = "%s_mysql.pp"%controller.CONF['CONFIG_MYSQL_HOST']
    manifestdata = getManifestTemplate("mysql.pp")
    appendManifestFile(manifestfile, manifestdata, 'pre')
<|endoftext|>


<commit_before>###############################################################################
#
# NOTE: If you don't need this file, feel free to delete it! It only serves to
#   be a demo of how plugins should be made.
#
#
# Hey there!
#
# This is dummy.py, or the whirlwind tour of what JshBot 0.3.x is about, and
#   how it can be used.
#
# The file demonsrates how to create new commands with standard syntax,
#   as well as shortcuts and help (manual entry).
#
# There is also a demonstration on how get_response() should be used, and how
#   to read a parsed result.
#
# Lastly, there is a small amount of demo code to show off that plugins can
#   define standard events from discord.Client, and they will be called
#   appropriately.
#
###############################################################################

import asyncio

from jshbot import utilities, configurations
from jshbot.commands import Command, SubCommands, Shortcuts
from jshbot.exceptions import BotException

__version__ = '0.1.0'
EXCEPTION = 'Dummy'
uses_configuration = False

def get_commands():
    """Returns a list of commands associated with this plugin."""
    commands = []

    commands.append(Command(
        'mycommand', SubCommands(
            ('myoption', 'myoption', 'This is a simple command with a single '
             'required option.'),
            ('?custom ?attached:', '(custom) (attached <"attached argument">)',
             'This has two different optional options, one without an '
             'attached argument, and the other requiring an attached argument '
             'if the option \'attached\' is specified.'),
            ('trailing ::+', 'trailing <"arg 1"> <"arg 2"> <"arg 3"> '
             '("arg 4") ("arg 5") (...)', 'This command requires lot of '
             'trailing arguments.'),
            ('grouped ^', 'grouped <grouped arguments>', 'This will group all '
             'given arguments as a single string.'),
            ('complex: ?other: :#', 'complex <"attached"> (other <"also '
             'required">) <"arg 1"> ("arg 2") ("arg 3") (...)', 'The complex '
             'option has a required positional argument, and the \'other\' '
             'option has one too if it is specified. Additionally, there will '
             'be a requirement of at least 1 trailing arguent.'),
            ('marquee ^', 'marquee <text>', 'Create a marquee that loops '
             'three times.')),
        shortcuts=Shortcuts(
            ('complex', 'complex {} other {} {} {}', ':::#',
             '<"attached"> other <"other"> <"arg 1"> ("arg 2") (...)',
             '<"attached"> <"other"> <"arg 1"> ("arg 2") (...)'),
            ('marquee', 'marquee {}', '^',
             'marquee <text>', '<text>')),
        description='Your command description here.',
        other='This text is optional - it just shows up after everything '
              'else. Quick note, all of the commands here can only be used by '
              'bot moderators or above, as indicated by elevated_level. A '
              'level of 2 would mean only server owners or above can use the '
              'command, and a level of 3 would restrict the command to only '
              'the bot owners.',
        elevated_level=1, group='demo'))

    commands.append(Command(
        'myothercommand', SubCommands(
            ('&', '<text>', ''),
            ('order matters', 'order matters', 'It is impossible to access '
             'this command because the first subcommand will always be '
             'satisfied first. Order of subcommands matters!'),
            ('sample foo bar', 'sample foo bar', 'Also impossible to access. '
             'This subcommand just adds some keywords to the command.')),
        description='Only bot owners can see this text!',
        other='Note that no shortcuts were defined. They, too, are optional. '
              'Also, this command is hidden, which means that only the bot '
              'owners can see this command listed from the help command. '
              'However, unless the command is configured with an elevated '
              'permissions level, any user can still execute the command. '
              'Users still will not be able to see the specific help for this '
              'command, though. Lastly, this command is disabled in DMs.',
        hidden=True, allow_direct=False, group='demo'))

    commands.append(Command(
        'notify', SubCommands(('^', '<text>', 'Notify the owners!')),
        other='This command uses a custom function. It is called with the '
              'same arguments as get_response. The command will show up to '
              'all users in the help command, but can only be used by server '
              'owners, as it is disallowed in direct messages.',
        elevated_level=2, allow_direct=False, function=custom_notify))

    return commands

async def get_response(
        bot, message, base, blueprint_index, options, arguments,
        keywords, cleaned_content):
    """Gets a response given the parsed input.

    Arguments:
    bot -- A reference to the bot itself.
    message -- The discord.message object obtained from on_message.
    base -- The base command name that immediately follows the invoker.
    blueprint_index -- The index of the subcommand of the given base command.
    options -- A dictionary representing the options and potential positional
        arguments that are attached to them.
    arguments -- A list of strings that follow the syntax of the blueprint
        index for arguments following the options.
    keywords -- Another list of strings that holds all option keywords. These
        can be used to prevent database conflicts with user commands.
    cleaned_content -- Simply the message content without the invoker.
    """

    # This is what the bot will say when it returns from this function.
    response = ''

    # Set to True if you want your message read with /tts (not recommended).
    tts = False

    # The message type dictates how the bot handles your returned message.
    #
    #   0 - Regular message. This message can be edited, and the bot will
    #       attempt to get a new response and replace the given message.
    #       This is the default recommended behavior.
    #
    #   1 - Permanent message. This message cannot be edited by a user changing
    #       their command. All types past this point are also non-editable.
    #
    #   2 - Terminal message. This message will self destruct in a defined
    #       number of seconds based on the 'extra' variable. For example, if
    #       message_type is set to 2, and extra is set to 10, the message will
    #       be displayed for 10 seconds, then be deleted. If the 'extra'
    #       variable is not set, it will default to 10 seconds.
    #
    #   3 - Active message. This message will be passed back to the plugin for
    #       extra processing and editing. The function it will call is
    #       handle_active_message(). See the comments for
    #       handle_active_message() for more information.
    #
    message_type = 0

    # The extra variable is used for the second and third message types.
    extra = None

    # Initially, check to make sure that you've matched the proper command.
    # If there is only one command specified, this may not be necessary.
    if base == 'mycommand':

        # Then, the blueprint_index will tell you which command syntax was
        #   satisfied. The order is the same as was specified initially.
        if blueprint_index == 0:  # myoption
            response = "You called the first subcommand!"
            # Do other stuff...

        elif blueprint_index == 1:  # custom/attached
            # To see if an optional option was included in the command, use:
            if 'custom' in options:
                response += "You included the \"custom\" flag!\n"
                # Do stuff relevant to this flag here...

            # To get the argument attached to an option, simply access it from
            #   the options dictionary.
            if 'attached' in options:
                response += "The attached argument: {}\n".format(
                    options['attached'])

            # In case somebody was looking for the help...
            if len(options) == 0:
                invoker = utilities.get_invoker(bot, server=message.server)
                response += "You didn't use either flag...\n"
                response += "For help, try `{}help mycommand`".format(invoker)

        elif blueprint_index == 2:  # trailing arguments
            # If arguments are specified as trailing, they will be in a list.
            response += "The list of trailing arguments: {}".format(arguments)

        elif blueprint_index == 3:  # grouped arguments
            # All arguments are grouped together as the first element
            message_type = 1
            response = ("You can't edit your command here.\n"
                        "Single grouped argument: {}").format(arguments[0])

        elif blueprint_index == 4:  # complex
            # This mixes elements of both examples seen above.
            response = ("The argument attached to the complex "
                        "option: {}\n").format(options['complex'])
            if 'other' in options:
                response += "The other option has attached: {}\n".format(
                    options['other'])
            response += "Lastly, the trailing arguments: {}".format(arguments)

        elif blueprint_index == 5:  # (Very slow) marquee
            # This demonstrates the active message type. Check
            #   handle_active_message to see how it works.
            text = arguments[0]
            if not text or len(text) > 100 or '\n' in text:
                response = ("Must have text 1-100 characters long, and must "
                            "not have any new lines.")
            else:
                message_type = 3  # active
                extra = ('marquee', text)
                response = "Setting up marquee..."  # This will be shown first

    # Here's another command base.
    elif base == 'myothercommand':

        if blueprint_index == 0:  # keyword checker
            text = arguments[0]
            if not text:
                response = "You didn't say anything...\n"
            else:
                response = "This is your input: {}\n".format(text)
                if text in keywords:
                    response += "Your input was in the list of keywords!\n"
                else:
                    response += ("Your input was not in the list of keywords. "
                                 "They are: {}\n").format(keywords)

            message_type = 2  # Self-destruct
            extra = 15  # 15 seconds
            response += "This message will self destruct in 15 seconds."

        else:  # impossible command???
            raise BotException(
                EXCEPTION, "This is a bug! You should never see this message.")

    return (response, tts, message_type, extra)

async def custom_notify(bot, message, *args):
    """This is only called with the notify command.

    This function is called with the same arguments as get_response.
    """
    response, tts, message_type, extra = ('', False, 0, None)

    notify_message = '{0} from server {1} is sending you: {2}'.format(
        message.author.display_name, message.server.name, args[3][0])
    await bot.notify_owners(notify_message)
    response = "Notified the owners with your message!"

    return (response, tts, message_type, extra)

async def handle_active_message(bot, message_reference, extra):
    """This is called if the given message was marked as active.

    (message_type of 3).
    """
    if extra[0] == 'marquee':  # Handle the marquee active message

        # Set text expanded with whitespace
        total_length = 40 + len(extra[1])
        text = '{0: ^{1}}'.format(extra[1], total_length)

        # Loop through the text three times
        for it in range(3):
            for move in range(total_length - 20):
                moving_text = '`|{:.20}|`'.format(text[move:])
                await asyncio.sleep(1)  # Don't get rate limited!
                await bot.edit_message(message_reference, moving_text)

        # When the marquee is done, just display the text
        await asyncio.sleep(1)
        await bot.edit_message(message_reference, extra[1])

# If necessary, discord.Client events can be defined here, and they will be
#   called appropriately. Be sure to include the bot argument first!

async def on_ready(bot):
    print("on_ready was called from dummy.py!")

async def on_message_edit(bot, before, after):
    if (before.author != bot.user and
            configurations.get(bot, __name__, key='show_edited_messages')):
        print("Somebody edited their message from '{0}' to '{1}'.".format(
            before.content, after.content))

async def bot_on_ready_boot(bot):
    """This is called only once every time the bot is started (or reloaded)."""
    # Use this to set up additonal permissions for the plugin
    permissions = {
        'read_messages': "This is a dummy additional permission.",
        'change_nickname': "This allows the bot to change its own nickname."
    }
    utilities.add_bot_permissions(bot, __name__, **permissions)
<commit_msg>Interactive message demo<commit_after>###############################################################################
#
# NOTE: If you don't need this file, feel free to delete it! It only serves to
#   be a demo of how plugins should be made.
#
#
# Hey there!
#
# This is dummy.py, or the whirlwind tour of what JshBot 0.3.x is about, and
#   how it can be used.
#
# The file demonsrates how to create new commands with standard syntax,
#   as well as shortcuts and help (manual entry).
#
# There is also a demonstration on how get_response() should be used, and how
#   to read a parsed result.
#
# Lastly, there is a small amount of demo code to show off that plugins can
#   define standard events from discord.Client, and they will be called
#   appropriately.
#
###############################################################################

import asyncio

from jshbot import utilities, configurations
from jshbot.commands import Command, SubCommands, Shortcuts
from jshbot.exceptions import BotException

__version__ = '0.1.0'
EXCEPTION = 'Dummy'
uses_configuration = False

def get_commands():
    """Returns a list of commands associated with this plugin."""
    commands = []

    commands.append(Command(
        'mycommand', SubCommands(
            ('myoption', 'myoption', 'This is a simple command with a single '
             'required option.'),
            ('?custom ?attached:', '(custom) (attached <"attached argument">)',
             'This has two different optional options, one without an '
             'attached argument, and the other requiring an attached argument '
             'if the option \'attached\' is specified.'),
            ('trailing ::+', 'trailing <"arg 1"> <"arg 2"> <"arg 3"> '
             '("arg 4") ("arg 5") (...)', 'This command requires lot of '
             'trailing arguments.'),
            ('grouped ^', 'grouped <grouped arguments>', 'This will group all '
             'given arguments as a single string.'),
            ('complex: ?other: :#', 'complex <"attached"> (other <"also '
             'required">) <"arg 1"> ("arg 2") ("arg 3") (...)', 'The complex '
             'option has a required positional argument, and the \'other\' '
             'option has one too if it is specified. Additionally, there will '
             'be a requirement of at least 1 trailing arguent.'),
            ('marquee ^', 'marquee <text>', 'Create a marquee that loops '
             'three times.')),
        shortcuts=Shortcuts(
            ('complex', 'complex {} other {} {} {}', ':::#',
             '<"attached"> other <"other"> <"arg 1"> ("arg 2") (...)',
             '<"attached"> <"other"> <"arg 1"> ("arg 2") (...)'),
            ('marquee', 'marquee {}', '^',
             'marquee <text>', '<text>')),
        description='Your command description here.',
        other='This text is optional - it just shows up after everything '
              'else. Quick note, all of the commands here can only be used by '
              'bot moderators or above, as indicated by elevated_level. A '
              'level of 2 would mean only server owners or above can use the '
              'command, and a level of 3 would restrict the command to only '
              'the bot owners.',
        elevated_level=1, group='demo'))

    commands.append(Command(
        'myothercommand', SubCommands(
            ('&', '<text>', ''),
            ('order matters', 'order matters', 'It is impossible to access '
             'this command because the first subcommand will always be '
             'satisfied first. Order of subcommands matters!'),
            ('sample foo bar', 'sample foo bar', 'Also impossible to access. '
             'This subcommand just adds some keywords to the command.')),
        description='Only bot owners can see this text!',
        other='Note that no shortcuts were defined. They, too, are optional. '
              'Also, this command is hidden, which means that only the bot '
              'owners can see this command listed from the help command. '
              'However, unless the command is configured with an elevated '
              'permissions level, any user can still execute the command. '
              'Users still will not be able to see the specific help for this '
              'command, though. Lastly, this command is disabled in DMs.',
        hidden=True, allow_direct=False, group='demo'))

    commands.append(Command(
        'notify', SubCommands(('^', '<text>', 'Notify the owners!')),
        other='This command uses a custom function. It is called with the '
              'same arguments as get_response. The command will show up to '
              'all users in the help command, but can only be used by server '
              'owners, as it is disallowed in direct messages.',
        elevated_level=2, allow_direct=False, function=custom_notify,
        group='demo'))

    commands.append(Command(
        'interact', SubCommands(('', '', '')),
        other='Use this command to demo the wait_for_message functionality.',
        group='demo'))

    return commands

async def get_response(
        bot, message, base, blueprint_index, options, arguments,
        keywords, cleaned_content):
    """Gets a response given the parsed input.

    Arguments:
    bot -- A reference to the bot itself.
    message -- The discord.message object obtained from on_message.
    base -- The base command name that immediately follows the invoker.
    blueprint_index -- The index of the subcommand of the given base command.
    options -- A dictionary representing the options and potential positional
        arguments that are attached to them.
    arguments -- A list of strings that follow the syntax of the blueprint
        index for arguments following the options.
    keywords -- Another list of strings that holds all option keywords. These
        can be used to prevent database conflicts with user commands.
    cleaned_content -- Simply the message content without the invoker.
    """

    # This is what the bot will say when it returns from this function.
    response = ''

    # Set to True if you want your message read with /tts (not recommended).
    tts = False

    # The message type dictates how the bot handles your returned message.
    #
    #   0 - Regular message. This message can be edited, and the bot will
    #       attempt to get a new response and replace the given message.
    #       This is the default recommended behavior.
    #
    #   1 - Permanent message. This message cannot be edited by a user changing
    #       their command. All types past this point are also non-editable.
    #
    #   2 - Terminal message. This message will self destruct in a defined
    #       number of seconds based on the 'extra' variable. For example, if
    #       message_type is set to 2, and extra is set to 10, the message will
    #       be displayed for 10 seconds, then be deleted. If the 'extra'
    #       variable is not set, it will default to 10 seconds.
    #
    #   3 - Active message. This message will be passed back to the plugin for
    #       extra processing and editing. The function it will call is
    #       handle_active_message(). See the comments for
    #       handle_active_message() for more information.
    #
    message_type = 0

    # The extra variable is used for the second and third message types.
    extra = None

    # Initially, check to make sure that you've matched the proper command.
    # If there is only one command specified, this may not be necessary.
    if base == 'mycommand':

        # Then, the blueprint_index will tell you which command syntax was
        #   satisfied. The order is the same as was specified initially.
        if blueprint_index == 0:  # myoption
            response = "You called the first subcommand!"
            # Do other stuff...

        elif blueprint_index == 1:  # custom/attached
            # To see if an optional option was included in the command, use:
            if 'custom' in options:
                response += "You included the \"custom\" flag!\n"
                # Do stuff relevant to this flag here...

            # To get the argument attached to an option, simply access it from
            #   the options dictionary.
            if 'attached' in options:
                response += "The attached argument: {}\n".format(
                    options['attached'])

            # In case somebody was looking for the help...
            if len(options) == 0:
                invoker = utilities.get_invoker(bot, server=message.server)
                response += "You didn't use either flag...\n"
                response += "For help, try `{}help mycommand`".format(invoker)

        elif blueprint_index == 2:  # trailing arguments
            # If arguments are specified as trailing, they will be in a list.
            response += "The list of trailing arguments: {}".format(arguments)

        elif blueprint_index == 3:  # grouped arguments
            # All arguments are grouped together as the first element
            message_type = 1
            response = ("You can't edit your command here.\n"
                        "Single grouped argument: {}").format(arguments[0])

        elif blueprint_index == 4:  # complex
            # This mixes elements of both examples seen above.
            response = ("The argument attached to the complex "
                        "option: {}\n").format(options['complex'])
            if 'other' in options:
                response += "The other option has attached: {}\n".format(
                    options['other'])
            response += "Lastly, the trailing arguments: {}".format(arguments)

        elif blueprint_index == 5:  # (Very slow) marquee
            # This demonstrates the active message type. Check
            #   handle_active_message to see how it works.
            text = arguments[0]
            if not text or len(text) > 100 or '\n' in text:
                response = ("Must have text 1-100 characters long, and must "
                            "not have any new lines.")
            else:
                message_type = 3  # active
                extra = ('marquee', text)
                response = "Setting up marquee..."  # This will be shown first

    # Here's another command base.
    elif base == 'myothercommand':

        if blueprint_index == 0:  # keyword checker
            text = arguments[0]
            if not text:
                response = "You didn't say anything...\n"
            else:
                response = "This is your input: {}\n".format(text)
                if text in keywords:
                    response += "Your input was in the list of keywords!\n"
                else:
                    response += ("Your input was not in the list of keywords. "
                                 "They are: {}\n").format(keywords)

            message_type = 2  # Self-destruct
            extra = 15  # 15 seconds
            response += "This message will self destruct in 15 seconds."

        else:  # impossible command???
            raise BotException(
                EXCEPTION, "This is a bug! You should never see this message.")

    elif base == 'interact':
        message_type = 6  # Use wait_for_message
        # The extra argument should consist of a 2 element tuple with the first
        #   element being the callback function, and the second being the
        #   keyword arguments passed into wait_for_message
        extra = (
            custom_interaction,
            {'timeout': 10, 'author': message.author},
            None
        )
        response = "Say something, {}".format(message.author)

    return (response, tts, message_type, extra)

async def custom_notify(bot, message, *args):
    """This is only called with the notify command.

    This function is called with the same arguments as get_response.
    """
    response, tts, message_type, extra = ('', False, 0, None)

    notify_message = '{0} from server {1} is sending you: {2}'.format(
        message.author.display_name, message.server.name, args[3][0])
    await bot.notify_owners(notify_message)
    response = "Notified the owners with your message!"

    return (response, tts, message_type, extra)

async def custom_interaction(bot, message_reference, reply, extra):
    """This is called when the message_type is 6.

    message_reference is the original message that can be edited.
    The reply argument is the return value of bot.wait_for_message.
    If the reply argument is None, the wait timed out.
    """
    if reply is None:
        edit = 'You took too long to respond...'
    elif reply.content:
        edit = 'You replied with "{}"'.format(reply.content[:100])
    else:
        edit = 'You did not reply with any text!'
    await bot.edit_message(message_reference, edit)

async def handle_active_message(bot, message_reference, extra):
    """This is called if the given message was marked as active.

    (message_type of 3).
    """
    if extra[0] == 'marquee':  # Handle the marquee active message

        # Set text expanded with whitespace
        total_length = 40 + len(extra[1])
        text = '{0: ^{1}}'.format(extra[1], total_length)

        # Loop through the text three times
        for it in range(3):
            for move in range(total_length - 20):
                moving_text = '`|{:.20}|`'.format(text[move:])
                await asyncio.sleep(1)  # Don't get rate limited!
                await bot.edit_message(message_reference, moving_text)

        # When the marquee is done, just display the text
        await asyncio.sleep(1)
        await bot.edit_message(message_reference, extra[1])

# If necessary, discord.Client events can be defined here, and they will be
#   called appropriately. Be sure to include the bot argument first!

async def on_ready(bot):
    print("on_ready was called from dummy.py!")

async def on_message_edit(bot, before, after):
    if (before.author != bot.user and
            configurations.get(bot, __name__, key='show_edited_messages')):
        print("Somebody edited their message from '{0}' to '{1}'.".format(
            before.content, after.content))

async def bot_on_ready_boot(bot):
    """This is called only once every time the bot is started (or reloaded)."""
    # Use this to set up additonal permissions for the plugin
    permissions = {
        'read_messages': "This is a dummy additional permission.",
        'change_nickname': "This allows the bot to change its own nickname."
    }
    utilities.add_bot_permissions(bot, __name__, **permissions)
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-

import logging
import json
import yaml
import re
import urlparse
import traceback
import lepl.apps.rfc3696

from pylons import config
import ckan.plugins as plugins
import ckan.plugins.toolkit as tk
from ckan import model
from ckan.common import request
from routes import url_for

log = logging.getLogger(__name__)

def create_updateInterval():
    '''Create update interval vocab and tags, if they don't exist already.'''
    user = tk.get_action('get_site_user')({'ignore_auth': True}, {})
    context = {'user': user['name']}
    try:
        data = {'id': 'updateInterval'}
        tk.get_action('vocabulary_show')(context, data)
        log.info("Update interval vocabulary already exists, skipping.")
    except tk.ObjectNotFound:
        log.info("Creating vocab 'updateInterval'")
        data = {'name': 'updateInterval'}
        vocab = tk.get_action('vocabulary_create')(context, data)
        for tag in (
            u'   ',
            u'laufend',
            u'jaehrlich',
            u'halbjaehrlich',
            u'quartalsweise',
            u'monatlich',
            u'woechentlich',
            u'taeglich',
            u'stuendlich',
            u'Echtzeit',
            u'sporadisch oder unregelmaessig',
            u'keines'
        ):
            log.info(
                "Adding tag {0} to vocab 'updateInterval'".format(tag))
            data = {'name': tag, 'vocabulary_id': vocab['id']}
            tk.get_action('tag_create')(context, data)

def updateInterval():
    '''Return the list of intervals from the updateInterval vocabulary.'''
    create_updateInterval()
    try:
        updateInterval = tk.get_action('tag_list')(
            data_dict={'vocabulary_id': 'updateInterval'})
        return updateInterval
    except tk.ObjectNotFound:
        return None

def create_dataType():
    '''Create update interval vocab and tags, if they don't exist already.'''
    user = tk.get_action('get_site_user')({'ignore_auth': True}, {})
    context = {'user': user['name']}
    try:
        data = {'id': 'dataType'}
        tk.get_action('vocabulary_show')(context, data)
        log.info("Data type vocabulary already exists, skipping.")
    except tk.ObjectNotFound:
        log.info("Creating vocab 'dataType'")
        data = {'name': 'dataType'}
        vocab = tk.get_action('vocabulary_create')(context, data)
        for tag in (
            u'   ',
            u'Bilddatei',
            u'Einzeldaten',
            u'Datenaggregat',
            u'Web-Service'
        ):
            log.info(
                "Adding tag {0} to vocab 'dataType'".format(tag))
            data = {'name': tag, 'vocabulary_id': vocab['id']}
            tk.get_action('tag_create')(context, data)

def dataType():
    '''Return the list of intervals from the dataType vocabulary.'''
    create_dataType()
    try:
        dataType = tk.get_action('tag_list')(
            data_dict={'vocabulary_id': 'dataType'})
        return dataType
    except tk.ObjectNotFound:
        return None

def groups():
    user = tk.get_action('get_site_user')({}, {})
    context = {'user': user['name']}
    data_dict = {
        'all_fields': True,
    }
    return tk.get_action('group_list')(context, data_dict)

def biggest_groups(n):
    '''
    Returns the n biggest groups, to display on start page.
    '''
    user = tk.get_action('get_site_user')({'ignore_auth': True},{})
    context = {'user': user['name']}
    data_dict = {
        'all_fields': True,
    }
    groups = tk.get_action('group_list')(context, data_dict)
    if len(groups) > n:
        return sorted(groups, key=lambda group: group.get('packages', 0))[-1:-(n+1):-1]
    else:
        return sorted(groups, key=lambda group: group.get('packages', 0))[::-1]

def package_has_group(group_name, groups):
    for group in groups:
        if group_name == group['name']:
            return True
    return False

def load_json(json_data):
    try:
        for kv_pair in json.loads(json_data):
            if not isinstance(kv_pair, list) and len(kv_pair) != 2:
                return False
        return json.loads(json_data)
    except:
        return False

def get_tag_vocab_values(package_dict):
    try:
        extras = package_dict['extras']
        results = {
            'dataType': '   ',
            'updateInterval': '   '
        }
        for extra in extras:
            if extra['key'] == u'dataType':
                results['dataType'] = extra['value']
            if extra['key'] == u'updateInterval':
                results['updateInterval'] = extra['value']
        return results
    except KeyError:
        return results

def get_package_dict(datasetID):
    user = tk.get_action('get_site_user')({}, {})
    context = {'user': user['name']}
    return tk.get_action('package_show')(context, {'id': datasetID})

def get_organization_dict(org=None):
    if org is None:
        return {}
    try:
        return tk.get_action('organization_show')({}, {'id': org})
    except Exception:
        return {}

def is_url(*args, **kw):
    '''
    Returns True if argument parses as a http, https or ftp URL
    '''
    if not args:
        return False
    try:
        url = urlparse.urlparse(args[0])
    except ValueError:
        return False

    valid_schemes = ('http', 'https', 'ftp')
    return url.scheme in valid_schemes

def full_external_url():
    ''' Returns the fully qualified current external url (eg http://...) useful
    for sharing etc '''
    return url_for(
        request.environ['CKAN_CURRENT_URL'],
        host=get_site_host(),
        protocol=get_site_protocol(),
        qualified=True
    )

def get_site_protocol():
    site_url = config.get('ckan.site_url', 'https://data.stadt-zuerich.ch')
    parsed_url = urlparse.urlparse(site_url)
    return parsed_url.scheme.encode('utf-8')

def get_site_host():
    site_url = config.get('ckan.site_url', 'https://data.stadt-zuerich.ch')
    parsed_url = urlparse.urlparse(site_url)
    return parsed_url.netloc.encode('utf-8')

def validate_date(datestring):
    m = re.match('^[0-9]{2}\.[0-9]{2}\.[0-9]{4}(, [0-9]{2}:[0-9]{2})?$', datestring)
    if m:
        return datestring
    else:
        return False

def validate_email(email):
    email_validator = lepl.apps.rfc3696.Email()
    if email_validator(email):
        return email
    else:
        return ''

class IFacetPlugin(plugins.SingletonPlugin):

    plugins.implements(plugins.IFacets, inherit=True)

    def dataset_facets(self, facets_dict, dataset_type):

        facets_dict = {'extras_updateInterval': 'Update Interval', 'tags': 'Keywords', 'organization': 'Organizations', 'res_format': ('File Format')}

        return facets_dict

class StadtzhThemePlugin(plugins.SingletonPlugin,
                         tk.DefaultDatasetForm):

    plugins.implements(plugins.IConfigurer, inherit=False)
    plugins.implements(plugins.IDatasetForm, inherit=False)
    plugins.implements(plugins.ITemplateHelpers, inherit=False)
    plugins.implements(plugins.IPackageController, inherit=True)

    def update_config(self, config):
        try:
            with open(config.get('ckanext.stadtzh-theme.descr_file'), 'r') as descr_file:
                self.descr_config = yaml.load(descr_file)
        except IOError:
            self.descr_config = {}

        tk.add_template_directory(config, 'templates')
        tk.add_public_directory(config, 'public')
        tk.add_resource('fanstatic', 'stadtzhtheme')

        config['ckan.site_logo'] = '/logo.png'

    def get_descr_config(self):
        return self.descr_config

    def get_helpers(self):
        return {
            'updateInterval': updateInterval,
            'dataType': dataType,
            'load_json': load_json,
            'groups': groups,
            'biggest_groups': biggest_groups,
            'package_has_group': package_has_group,
            'get_tag_vocab_values': get_tag_vocab_values,
            'get_package_dict': get_package_dict,
            'validate_date': validate_date,
            'validate_email': validate_email,
            'get_organization_dict': get_organization_dict,
            'full_external_url': full_external_url,
            'is_url': is_url,
            'get_site_protocol': get_site_protocol,
            'get_site_host': get_site_host,
            'get_descr_config': self.get_descr_config,
        }

    def is_fallback(self):
        # Return True to register this plugin as the default handler for
        # package types not handled by any other IDatasetForm plugin.
        return True

    def package_types(self):
        # This plugin doesn't handle any special package types, it just
        # registers itself as the default (above).
        return []

    def _modify_package_schema(self, schema):

        # Add spatial relationship as extra field
        schema.update({
            'spatialRelationship': [tk.get_validator('ignore_missing'),
                                    tk.get_converter('convert_to_extras')]
        })

        # Add date first published as extra field
        schema.update({
            'dateFirstPublished': [tk.get_validator('ignore_missing'),
                                   tk.get_converter('convert_to_extras')]
        })

        # Add time range as extra field
        schema.update({
            'timeRange': [tk.get_validator('ignore_missing'),
                          tk.get_converter('convert_to_extras')]
        })

        # Add update interval as extra field
        schema.update({
            'updateInterval': [tk.get_validator('ignore_missing'),
                               tk.get_converter('convert_to_tags')('updateInterval')]
        })

        # Add version as extra field
        schema.update({
            'version': [tk.get_validator('ignore_missing'),
                        tk.get_converter('convert_to_extras')]
        })

        # Add date last updated as extra field
        schema.update({
            'dateLastUpdated': [tk.get_validator('ignore_missing'),
                                tk.get_converter('convert_to_extras')]
        })

        # Add legal information as extra field
        schema.update({
            'legalInformation': [tk.get_validator('ignore_missing'),
                                 tk.get_converter('convert_to_extras')]
        })

        # Add comments as extra field
        schema.update({
            'sszBemerkungen': [tk.get_validator('ignore_missing'),
                         tk.get_converter('convert_to_extras')]
        })

        # Add update interval as extra field
        schema.update({
            'dataType': [tk.get_validator('ignore_missing'),
                         tk.get_converter('convert_to_tags')('dataType')]
        })

        # Add attributes as extra field
        schema.update({
            'sszFields': [tk.get_validator('ignore_missing'),
                           tk.get_converter('convert_to_extras')]
        })

        # Add data quality as extra field
        schema.update({
            'dataQuality': [tk.get_validator('ignore_missing'),
                            tk.get_converter('convert_to_extras')]
        })

        # Add groups as extra field
        schema.update({
            'group': [tk.get_validator('ignore_missing')]
        })

        return schema

    def create_package_schema(self):
        schema = super(StadtzhThemePlugin, self).create_package_schema()
        schema = self._modify_package_schema(schema)
        return schema

    def update_package_schema(self):
        schema = super(StadtzhThemePlugin, self).update_package_schema()
        schema = self._modify_package_schema(schema)
        return schema

    def show_package_schema(self):
        schema = super(StadtzhThemePlugin, self).show_package_schema()

        # Don't show vocab tags xed in with normal 'free' tags
        # (e.g. on dataset pages, or on the search page)
        schema['tags']['__extras'].append(tk.get_converter('free_tags_only'))

        # Add our spatialRelationship field to the dataset schema.
        schema.update({
            'spatialRelationship': [tk.get_converter('convert_from_extras'),
                                    tk.get_validator('ignore_missing')]
        })

        # Add our dateFirstPublished field to the dataset schema.
        schema.update({
            'dateFirstPublished': [tk.get_converter('convert_from_extras'),
                                   tk.get_validator('ignore_missing')]
        })

        # Add our timeRange field to the dataset schema.
        schema.update({
            'timeRange': [tk.get_converter('convert_from_extras'),
                          tk.get_validator('ignore_missing')]
        })

        # Add our updateInterval field to the dataset schema.
        schema.update({
            'updateInterval': [tk.get_converter('convert_from_tags')('updateInterval'),
                               tk.get_validator('ignore_missing')]
        })

        # Add our version field to the dataset schema.
        schema.update({
            'version': [tk.get_converter('convert_from_extras'),
                        tk.get_validator('ignore_missing')]
        })

        # Add our dateLastUpdated field to the dataset schema.
        schema.update({
            'dateLastUpdated': [tk.get_converter('convert_from_extras'),
                                tk.get_validator('ignore_missing')]
        })

        # Add our comments field to the dataset schema.
        schema.update({
            'sszBemerkungen': [tk.get_converter('convert_from_extras'),
                         tk.get_validator('ignore_missing')]
        })

        # Add our dataType field to the dataset schema.
        schema.update({
            'dataType': [tk.get_converter('convert_from_tags')('dataType'),
                         tk.get_validator('ignore_missing')]
        })

        # Add our legalInformation field to the dataset schema.
        schema.update({
            'legalInformation': [tk.get_converter('convert_from_extras'),
                                 tk.get_validator('ignore_missing')]
        })

        # Add our attributes field to the dataset schema.
        schema.update({
            'sszFields': [tk.get_converter('convert_from_extras'),
                           tk.get_validator('ignore_missing')]
        })

        # Add our data quality field to the dataset schema.
        schema.update({
            'dataQuality': [tk.get_converter('convert_from_extras'),
                            tk.get_validator('ignore_missing')]
        })

        # Add our groups field to the dataset schema.
        schema.update({
            'group': [tk.get_validator('ignore_missing')]
        })

        return schema

    def setup_template_variables(self, context, data_dict):
        context['descr_config'] = self.descr_config
        return super(StadtzhThemePlugin, self).setup_template_variables(
            context, data_dict)

    # IPackageController

    def after_show(self, context, pkg_dict):
        # set value of new field data_publisher with value of url
        pkg_dict['data_publisher'] = pkg_dict['url']
<commit_msg>Catch exceptions if groups/packages etc. cannot be found<commit_after># -*- coding: utf-8 -*-

import logging
import json
import yaml
import re
import urlparse
import traceback
import lepl.apps.rfc3696

from pylons import config
import ckan.plugins as plugins
import ckan.plugins.toolkit as tk
from ckan import model
from ckan.common import request
from routes import url_for

log = logging.getLogger(__name__)

def create_updateInterval():
    '''Create update interval vocab and tags, if they don't exist already.'''
    user = tk.get_action('get_site_user')({'ignore_auth': True}, {})
    context = {'user': user['name']}
    try:
        data = {'id': 'updateInterval'}
        tk.get_action('vocabulary_show')(context, data)
        log.info("Update interval vocabulary already exists, skipping.")
    except tk.ObjectNotFound:
        log.info("Creating vocab 'updateInterval'")
        data = {'name': 'updateInterval'}
        vocab = tk.get_action('vocabulary_create')(context, data)
        for tag in (
            u'   ',
            u'laufend',
            u'jaehrlich',
            u'halbjaehrlich',
            u'quartalsweise',
            u'monatlich',
            u'woechentlich',
            u'taeglich',
            u'stuendlich',
            u'Echtzeit',
            u'sporadisch oder unregelmaessig',
            u'keines'
        ):
            log.info(
                "Adding tag {0} to vocab 'updateInterval'".format(tag))
            data = {'name': tag, 'vocabulary_id': vocab['id']}
            tk.get_action('tag_create')(context, data)

def updateInterval():
    '''Return the list of intervals from the updateInterval vocabulary.'''
    create_updateInterval()
    try:
        updateInterval = tk.get_action('tag_list')(
            data_dict={'vocabulary_id': 'updateInterval'})
        return updateInterval
    except tk.ObjectNotFound:
        return None

def create_dataType():
    '''Create update interval vocab and tags, if they don't exist already.'''
    user = tk.get_action('get_site_user')({'ignore_auth': True}, {})
    context = {'user': user['name']}
    try:
        data = {'id': 'dataType'}
        tk.get_action('vocabulary_show')(context, data)
        log.info("Data type vocabulary already exists, skipping.")
    except tk.ObjectNotFound:
        log.info("Creating vocab 'dataType'")
        data = {'name': 'dataType'}
        vocab = tk.get_action('vocabulary_create')(context, data)
        for tag in (
            u'   ',
            u'Bilddatei',
            u'Einzeldaten',
            u'Datenaggregat',
            u'Web-Service'
        ):
            log.info(
                "Adding tag {0} to vocab 'dataType'".format(tag))
            data = {'name': tag, 'vocabulary_id': vocab['id']}
            tk.get_action('tag_create')(context, data)

def dataType():
    '''Return the list of intervals from the dataType vocabulary.'''
    create_dataType()
    try:
        dataType = tk.get_action('tag_list')(
            data_dict={'vocabulary_id': 'dataType'})
        return dataType
    except tk.ObjectNotFound:
        return None

def groups():
    user = tk.get_action('get_site_user')({}, {})
    context = {'user': user['name']}
    data_dict = {
        'all_fields': True,
    }
    try:
        return tk.get_action('group_list')(context, data_dict)
    except tk.ObjectNotFound:
        return None

def biggest_groups(n):
    '''
    Returns the n biggest groups, to display on start page.
    '''
    user = tk.get_action('get_site_user')({'ignore_auth': True},{})
    context = {'user': user['name']}
    data_dict = {
        'all_fields': True,
    }
    groups = tk.get_action('group_list')(context, data_dict)
    if len(groups) > n:
        return sorted(groups, key=lambda group: group.get('packages', 0))[-1:-(n+1):-1]
    else:
        return sorted(groups, key=lambda group: group.get('packages', 0))[::-1]

def package_has_group(group_name, groups):
    for group in groups:
        if group_name == group['name']:
            return True
    return False

def load_json(json_data):
    try:
        for kv_pair in json.loads(json_data):
            if not isinstance(kv_pair, list) and len(kv_pair) != 2:
                return False
        return json.loads(json_data)
    except:
        return False

def get_tag_vocab_values(package_dict):
    try:
        return {
            'dataType': package_dict['dataType'],
            'updateInterval': package_dict['updateInterval']
        }
    except KeyError:
        return {
            'dataType': '   ',
            'updateInterval': '   ',
        }

def get_package_dict(datasetID):
    user = tk.get_action('get_site_user')({}, {})
    context = {'user': user['name']}
    try:
        return tk.get_action('package_show')(context, {'id': datasetID})
    except:
        return {}

def get_organization_dict(org=None):
    if org is None:
        return {}
    try:
        return tk.get_action('organization_show')({}, {'id': org})
    except tk.ObjectNotFound:
        return {}

def is_url(*args, **kw):
    '''
    Returns True if argument parses as a http, https or ftp URL
    '''
    if not args:
        return False
    try:
        url = urlparse.urlparse(args[0])
    except ValueError:
        return False

    valid_schemes = ('http', 'https', 'ftp')
    return url.scheme in valid_schemes

def full_external_url():
    ''' Returns the fully qualified current external url (eg http://...) useful
    for sharing etc '''
    return url_for(
        request.environ['CKAN_CURRENT_URL'],
        host=get_site_host(),
        protocol=get_site_protocol(),
        qualified=True
    )

def get_site_protocol():
    site_url = config.get('ckan.site_url', 'https://data.stadt-zuerich.ch')
    parsed_url = urlparse.urlparse(site_url)
    return parsed_url.scheme.encode('utf-8')

def get_site_host():
    site_url = config.get('ckan.site_url', 'https://data.stadt-zuerich.ch')
    parsed_url = urlparse.urlparse(site_url)
    return parsed_url.netloc.encode('utf-8')

def validate_date(datestring):
    m = re.match('^[0-9]{2}\.[0-9]{2}\.[0-9]{4}(, [0-9]{2}:[0-9]{2})?$', datestring)
    if m:
        return datestring
    else:
        return False

def validate_email(email):
    email_validator = lepl.apps.rfc3696.Email()
    if email_validator(email):
        return email
    else:
        return ''

class IFacetPlugin(plugins.SingletonPlugin):

    plugins.implements(plugins.IFacets, inherit=True)

    def dataset_facets(self, facets_dict, dataset_type):

        facets_dict = {'extras_updateInterval': 'Update Interval', 'tags': 'Keywords', 'organization': 'Organizations', 'res_format': ('File Format')}

        return facets_dict

class StadtzhThemePlugin(plugins.SingletonPlugin,
                         tk.DefaultDatasetForm):

    plugins.implements(plugins.IConfigurer, inherit=False)
    plugins.implements(plugins.IDatasetForm, inherit=False)
    plugins.implements(plugins.ITemplateHelpers, inherit=False)
    plugins.implements(plugins.IPackageController, inherit=True)

    def update_config(self, config):
        try:
            with open(config.get('ckanext.stadtzh-theme.descr_file'), 'r') as descr_file:
                self.descr_config = yaml.load(descr_file)
        except IOError:
            self.descr_config = {}

        tk.add_template_directory(config, 'templates')
        tk.add_public_directory(config, 'public')
        tk.add_resource('fanstatic', 'stadtzhtheme')

        config['ckan.site_logo'] = '/logo.png'

    def get_descr_config(self):
        return self.descr_config

    def get_helpers(self):
        return {
            'updateInterval': updateInterval,
            'dataType': dataType,
            'load_json': load_json,
            'groups': groups,
            'biggest_groups': biggest_groups,
            'package_has_group': package_has_group,
            'get_tag_vocab_values': get_tag_vocab_values,
            'get_package_dict': get_package_dict,
            'validate_date': validate_date,
            'validate_email': validate_email,
            'get_organization_dict': get_organization_dict,
            'full_external_url': full_external_url,
            'is_url': is_url,
            'get_site_protocol': get_site_protocol,
            'get_site_host': get_site_host,
            'get_descr_config': self.get_descr_config,
        }

    def is_fallback(self):
        # Return True to register this plugin as the default handler for
        # package types not handled by any other IDatasetForm plugin.
        return True

    def package_types(self):
        # This plugin doesn't handle any special package types, it just
        # registers itself as the default (above).
        return []

    def _modify_package_schema(self, schema):

        # Add spatial relationship as extra field
        schema.update({
            'spatialRelationship': [tk.get_validator('ignore_missing'),
                                    tk.get_converter('convert_to_extras')]
        })

        # Add date first published as extra field
        schema.update({
            'dateFirstPublished': [tk.get_validator('ignore_missing'),
                                   tk.get_converter('convert_to_extras')]
        })

        # Add time range as extra field
        schema.update({
            'timeRange': [tk.get_validator('ignore_missing'),
                          tk.get_converter('convert_to_extras')]
        })

        # Add update interval as extra field
        schema.update({
            'updateInterval': [tk.get_validator('ignore_missing'),
                               tk.get_converter('convert_to_tags')('updateInterval')]
        })

        # Add version as extra field
        schema.update({
            'version': [tk.get_validator('ignore_missing'),
                        tk.get_converter('convert_to_extras')]
        })

        # Add date last updated as extra field
        schema.update({
            'dateLastUpdated': [tk.get_validator('ignore_missing'),
                                tk.get_converter('convert_to_extras')]
        })

        # Add legal information as extra field
        schema.update({
            'legalInformation': [tk.get_validator('ignore_missing'),
                                 tk.get_converter('convert_to_extras')]
        })

        # Add comments as extra field
        schema.update({
            'sszBemerkungen': [tk.get_validator('ignore_missing'),
                         tk.get_converter('convert_to_extras')]
        })

        # Add update interval as extra field
        schema.update({
            'dataType': [tk.get_validator('ignore_missing'),
                         tk.get_converter('convert_to_tags')('dataType')]
        })

        # Add attributes as extra field
        schema.update({
            'sszFields': [tk.get_validator('ignore_missing'),
                           tk.get_converter('convert_to_extras')]
        })

        # Add data quality as extra field
        schema.update({
            'dataQuality': [tk.get_validator('ignore_missing'),
                            tk.get_converter('convert_to_extras')]
        })

        # Add groups as extra field
        schema.update({
            'group': [tk.get_validator('ignore_missing')]
        })

        return schema

    def create_package_schema(self):
        schema = super(StadtzhThemePlugin, self).create_package_schema()
        schema = self._modify_package_schema(schema)
        return schema

    def update_package_schema(self):
        schema = super(StadtzhThemePlugin, self).update_package_schema()
        schema = self._modify_package_schema(schema)
        return schema

    def show_package_schema(self):
        schema = super(StadtzhThemePlugin, self).show_package_schema()

        # Don't show vocab tags xed in with normal 'free' tags
        # (e.g. on dataset pages, or on the search page)
        schema['tags']['__extras'].append(tk.get_converter('free_tags_only'))

        # Add our spatialRelationship field to the dataset schema.
        schema.update({
            'spatialRelationship': [tk.get_converter('convert_from_extras'),
                                    tk.get_validator('ignore_missing')]
        })

        # Add our dateFirstPublished field to the dataset schema.
        schema.update({
            'dateFirstPublished': [tk.get_converter('convert_from_extras'),
                                   tk.get_validator('ignore_missing')]
        })

        # Add our timeRange field to the dataset schema.
        schema.update({
            'timeRange': [tk.get_converter('convert_from_extras'),
                          tk.get_validator('ignore_missing')]
        })

        # Add our updateInterval field to the dataset schema.
        schema.update({
            'updateInterval': [tk.get_converter('convert_from_tags')('updateInterval'),
                               tk.get_validator('ignore_missing')]
        })

        # Add our version field to the dataset schema.
        schema.update({
            'version': [tk.get_converter('convert_from_extras'),
                        tk.get_validator('ignore_missing')]
        })

        # Add our dateLastUpdated field to the dataset schema.
        schema.update({
            'dateLastUpdated': [tk.get_converter('convert_from_extras'),
                                tk.get_validator('ignore_missing')]
        })

        # Add our comments field to the dataset schema.
        schema.update({
            'sszBemerkungen': [tk.get_converter('convert_from_extras'),
                         tk.get_validator('ignore_missing')]
        })

        # Add our dataType field to the dataset schema.
        schema.update({
            'dataType': [tk.get_converter('convert_from_tags')('dataType'),
                         tk.get_validator('ignore_missing')]
        })

        # Add our legalInformation field to the dataset schema.
        schema.update({
            'legalInformation': [tk.get_converter('convert_from_extras'),
                                 tk.get_validator('ignore_missing')]
        })

        # Add our attributes field to the dataset schema.
        schema.update({
            'sszFields': [tk.get_converter('convert_from_extras'),
                           tk.get_validator('ignore_missing')]
        })

        # Add our data quality field to the dataset schema.
        schema.update({
            'dataQuality': [tk.get_converter('convert_from_extras'),
                            tk.get_validator('ignore_missing')]
        })

        # Add our groups field to the dataset schema.
        schema.update({
            'group': [tk.get_validator('ignore_missing')]
        })

        return schema

    def setup_template_variables(self, context, data_dict):
        context['descr_config'] = self.descr_config
        return super(StadtzhThemePlugin, self).setup_template_variables(
            context, data_dict)

    # IPackageController

    def after_show(self, context, pkg_dict):
        # set value of new field data_publisher with value of url
        pkg_dict['data_publisher'] = pkg_dict['url']
<|endoftext|>


<commit_before>"""
    proclib.response
    ~~~~~~~~~~~~~~~~
    Implements the Response class.
"""

from .helpers import cached_property

class Response(object):
    """
    A Response object represents the result of running
    a process. The parameters supplied are stored and
    available as attributes as well.

    :param command: A command in the form of a list.
    :param process: A `subprocess.Popen` object.
    """

    def __init__(self, command, process):
        self.history = []
        self.command = command
        self.process = process

        self.pid = process.pid
        self.stdout = process.stdout
        self.stderr = process.stderr

    @cached_property
    def out(self):
        """
        Reads the entire stdout of the Popen instance
        and then returns it. Not recommended for long
        running processes.
        """
        with self.stdout:
            return self.stdout.read()

    @cached_property
    def err(self):
        """
        Similar to ``out``, reads the entirety of
        ``stderr``.
        """
        with self.stderr:
            return self.stderr.read()

    @property
    def status_code(self):
        """
        Returns the exit code of the process.
        """
        return self.process.poll()

    @property
    def finished(self):
        """
        Returns a boolean stating if the process has
        completed or not.
        """
        return self.process.poll() is not None

    def close(self):
        """
        If possible, close the stdout and stderr file
        handles.
        """
        if self.stdout: self.stdout.close()
        if self.stderr: self.stderr.close()

    @property
    def ok(self):
        """
        Returns a boolean depending on whether the
        `returncode` attribute equals zero.
        """
        return self.status_code == 0

    def wait(self):
        """
        Block until the process to complete.
        """
        self.process.wait()

    def terminate(self):
        """
        Terminate the process if it is not finished.
        """
        if not self.finished:
            self.process.terminate()

    def __repr__(self):
        return '<Response [%s]>' % self.command[0]

    def __enter__(self):
        return self

    def __exit__(self, *_):
        self.close()
<commit_msg>change semantics of Response.finished<commit_after>"""
    proclib.response
    ~~~~~~~~~~~~~~~~
    Implements the Response class.
"""

from .helpers import cached_property

class Response(object):
    """
    A Response object represents the result of running
    a process. The parameters supplied are stored and
    available as attributes as well.

    :param command: A command in the form of a list.
    :param process: A `subprocess.Popen` object.
    """

    def __init__(self, command, process):
        self.history = []
        self.command = command
        self.process = process

        self.pid = process.pid
        self.stdout = process.stdout
        self.stderr = process.stderr

    @cached_property
    def out(self):
        """
        Reads the entire stdout of the Popen instance
        and then returns it. Not recommended for long
        running processes.
        """
        with self.stdout:
            return self.stdout.read()

    @cached_property
    def err(self):
        """
        Similar to ``out``, reads the entirety of
        ``stderr``.
        """
        with self.stderr:
            return self.stderr.read()

    @property
    def status_code(self):
        """
        Returns the exit code of the process, None
        if the process hasn't completed.
        """
        return self.process.poll()

    @property
    def finished(self):
        """
        Returns a boolean stating if the process has
        completed or not. Internally this calls
        `status_code` to determine the exit code
        of the process.
        """
        return self.status_code is not None

    def close(self):
        """
        If possible, close the stdout and stderr file
        handles.
        """
        if self.stdout: self.stdout.close()
        if self.stderr: self.stderr.close()

    @property
    def ok(self):
        """
        Returns a boolean depending on whether the
        `returncode` attribute equals zero.
        """
        return self.status_code == 0

    def wait(self):
        """
        Block until the process to complete.
        """
        self.process.wait()

    def terminate(self):
        """
        Terminate the process if it is not finished.
        """
        if not self.finished:
            self.process.terminate()

    def __repr__(self):
        return '<Response [%s]>' % self.command[0]

    def __enter__(self):
        return self

    def __exit__(self, *_):
        self.close()
<|endoftext|>


<commit_before>from crawl_lib import *

# Takes in a soup and sends the product to the db (including price, url, name, image)
def _send_product(prod, cat_name):
    data = {}
    name = ''
    for span in prod.find_all('span'):
        if 'js-product-entry' in str(span):
            name = name + strip_name(str(span), '>', '</')

    if name == '':
        return # name was not found for this product
    data['name'] = name

    for img in prod.find_all('img', limit=1):
        data['image'] = img['src']

    price = 0
    for span in prod.find_all('span'):
        if 'Regular Price:' in str(span):
            price = strip_name(span['aria-label'], 'Regular Price:', None).strip()
            break

    if price == 0:
        return # price was not found for this product
    data['price'] = '%.2f' % float(price)

    for link in prod.find_all('a', limit=1):
        data['url'] = 'https://www.realcanadiansuperstore.ca' + link['href']

    data['store'] = 'Superstore'

    send_to_db(cat_name, data)

    return

# Parses starting from the base_url and sends the data to the db
def parse():
    base_url = 'https://www.realcanadiansuperstore.ca'
    soup = get_soup(base_url)

    set_links = set()
    for cat in soup.find_all('li'):
        if "data-level" in cat.attrs and cat['data-level'] in ['2', '3']:
            for cat_link in cat.find_all('a', limit=1):
                set_links.add(base_url + cat_link['href'])

    for link in set_links:
        cat_soup = get_soup(link)
        for prod in cat_soup.find_all('div', 'product-page-hotspot'):
            cat_name = strip_name(link, 'superstore.ca', '/plp').split('/c/')[0].split('/')
            cat_name = cat_name[len(cat_name) - 2].replace('%26', '&')
            _send_product(prod, cat_name)

    return

if __name__ == '__main__':
    parse()

    # todo:
    # send timestamp along with json doc to server
<commit_msg>catch superstore html exception<commit_after>from crawl_lib import *

# Takes in a soup and sends the product to the db (including price, url, name, image)
def _send_product(prod, cat_name):
    data = {}
    name = ''
    for span in prod.find_all('span'):
        if 'js-product-entry' in str(span):
            name = name + strip_name(str(span), '>', '</')

    if name == '':
        return # name was not found for this product
    data['name'] = name

    for img in prod.find_all('img', limit=1):
        data['image'] = img['src']

    price = 0
    for span in prod.find_all('span'):
        if 'Regular Price:' in str(span):
            price = strip_name(span['aria-label'], 'Regular Price:', None).strip()
            break

    if price == 0:
        return # price was not found for this product
    data['price'] = '%.2f' % float(price)

    for link in prod.find_all('a', limit=1):
        data['url'] = 'https://www.realcanadiansuperstore.ca' + link['href']

    data['store'] = 'Superstore'

    send_to_db(cat_name, data)

    return

# Parses starting from the base_url and sends the data to the db
def parse():
    base_url = 'https://www.realcanadiansuperstore.ca'
    soup = get_soup(base_url)

    set_links = set()
    for cat in soup.find_all('li'):
        if "data-level" in cat.attrs and cat['data-level'] in ['2', '3']:
            for cat_link in cat.find_all('a', limit=1):
                set_links.add(base_url + cat_link['href'])

    for link in set_links:
        print(link)
        try:
            cat_soup = get_soup(link)
        except Exception:
            traceback.print_exc()
            continue
        for prod in cat_soup.find_all('div', 'product-page-hotspot'):
            cat_name = strip_name(link, 'superstore.ca', '/plp').split('/c/')[0].split('/')
            cat_name = cat_name[len(cat_name) - 2].replace('%26', '&')
            _send_product(prod, cat_name)

    return

if __name__ == '__main__':
    parse()

    # todo:
    # send timestamp along with json doc to server
<|endoftext|>


<commit_before>from __future__ import unicode_literals

import logging
import os
import sys
import time

import colorama
from packaging import version

from dvc import __version__
from dvc.lock import Lock
from dvc.lock import LockError
from dvc.utils import boxify
from dvc.utils import env2bool
from dvc.utils.pkg import PKG

logger = logging.getLogger(__name__)

class Updater(object):  # pragma: no cover
    URL = "https://updater.dvc.org"
    UPDATER_FILE = "updater"
    TIMEOUT = 24 * 60 * 60  # every day
    TIMEOUT_GET = 10

    def __init__(self, dvc_dir):
        self.dvc_dir = dvc_dir
        self.updater_file = os.path.join(dvc_dir, self.UPDATER_FILE)
        self.lock = Lock(
            self.updater_file + ".lock", tmp_dir=os.path.join(dvc_dir, "tmp")
        )
        self.current = version.parse(__version__).base_version

    def _is_outdated_file(self):
        ctime = os.path.getmtime(self.updater_file)
        outdated = time.time() - ctime >= self.TIMEOUT
        if outdated:
            logger.debug("'{}' is outdated(".format(self.updater_file))
        return outdated

    def _with_lock(self, func, action):
        try:
            with self.lock:
                func()
        except LockError:
            msg = "Failed to acquire '{}' before {} updates"
            logger.debug(msg.format(self.lock.lockfile, action))

    def check(self):
        if os.getenv("CI") or env2bool("DVC_TEST") or PKG == "snap":
            return

        self._with_lock(self._check, "checking")

    def _check(self):
        if not os.path.exists(self.updater_file) or self._is_outdated_file():
            self.fetch()
            return

        with open(self.updater_file, "r") as fobj:
            import json

            try:
                info = json.load(fobj)
                self.latest = info["version"]
            except Exception as exc:
                msg = "'{}' is not a valid json: {}"
                logger.debug(msg.format(self.updater_file, exc))
                self.fetch()
                return

        if self._is_outdated():
            self._notify()

    def fetch(self, detach=True):
        from dvc.daemon import daemon

        if detach:
            daemon(["updater"])
            return

        self._with_lock(self._get_latest_version, "fetching")

    def _get_latest_version(self):
        import json
        import requests

        try:
            r = requests.get(self.URL, timeout=self.TIMEOUT_GET)
            info = r.json()
        except requests.exceptions.RequestException as exc:
            msg = "Failed to retrieve latest version: {}"
            logger.debug(msg.format(exc))
            return

        with open(self.updater_file, "w+") as fobj:
            json.dump(info, fobj)

    def _is_outdated(self):
        return version.parse(self.current) < version.parse(self.latest)

    def _notify(self):
        if not sys.stdout.isatty():
            return

        message = (
            "Update available {red}{current}{reset} -> {green}{latest}{reset}"
            + "\n"
            + self._get_update_instructions()
        ).format(
            red=colorama.Fore.RED,
            reset=colorama.Fore.RESET,
            green=colorama.Fore.GREEN,
            yellow=colorama.Fore.YELLOW,
            blue=colorama.Fore.BLUE,
            current=self.current,
            latest=self.latest,
        )

        logger.info(boxify(message, border_color="yellow"))

    def _get_update_instructions(self):
        instructions = {
            "pip": "Run {yellow}pip{reset} install dvc {blue}--upgrade{reset}",
            "rpm": "Run {yellow}yum{reset} update dvc",
            "brew": "Run {yellow}brew{reset} upgrade dvc",
            "deb": (
                "Run {yellow}apt-get{reset} install"
                " {blue}--only-upgrade{reset} dvc"
            ),
            "binary": (
                "To upgrade follow these steps:\n"
                "1. Uninstall dvc binary\n"
                "2. Go to {blue}https://dvc.org{reset}\n"
                "3. Download and install new binary"
            ),
            "conda": "Run {yellow}conda{reset} update dvc",
            None: (
                "Find the latest release at\n{blue}"
                "https://github.com/iterative/dvc/releases/latest"
                "{reset}"
            ),
        }

        package_manager = PKG
        if package_manager in ("osxpkg", "exe"):
            package_manager = "binary"

        return instructions[package_manager]
<commit_msg>updater: add an entry for chocolatey package<commit_after>from __future__ import unicode_literals

import logging
import os
import sys
import time

import colorama
from packaging import version

from dvc import __version__
from dvc.lock import Lock
from dvc.lock import LockError
from dvc.utils import boxify
from dvc.utils import env2bool
from dvc.utils.pkg import PKG

logger = logging.getLogger(__name__)

class Updater(object):  # pragma: no cover
    URL = "https://updater.dvc.org"
    UPDATER_FILE = "updater"
    TIMEOUT = 24 * 60 * 60  # every day
    TIMEOUT_GET = 10

    def __init__(self, dvc_dir):
        self.dvc_dir = dvc_dir
        self.updater_file = os.path.join(dvc_dir, self.UPDATER_FILE)
        self.lock = Lock(
            self.updater_file + ".lock", tmp_dir=os.path.join(dvc_dir, "tmp")
        )
        self.current = version.parse(__version__).base_version

    def _is_outdated_file(self):
        ctime = os.path.getmtime(self.updater_file)
        outdated = time.time() - ctime >= self.TIMEOUT
        if outdated:
            logger.debug("'{}' is outdated(".format(self.updater_file))
        return outdated

    def _with_lock(self, func, action):
        try:
            with self.lock:
                func()
        except LockError:
            msg = "Failed to acquire '{}' before {} updates"
            logger.debug(msg.format(self.lock.lockfile, action))

    def check(self):
        if os.getenv("CI") or env2bool("DVC_TEST") or PKG == "snap":
            return

        self._with_lock(self._check, "checking")

    def _check(self):
        if not os.path.exists(self.updater_file) or self._is_outdated_file():
            self.fetch()
            return

        with open(self.updater_file, "r") as fobj:
            import json

            try:
                info = json.load(fobj)
                self.latest = info["version"]
            except Exception as exc:
                msg = "'{}' is not a valid json: {}"
                logger.debug(msg.format(self.updater_file, exc))
                self.fetch()
                return

        if self._is_outdated():
            self._notify()

    def fetch(self, detach=True):
        from dvc.daemon import daemon

        if detach:
            daemon(["updater"])
            return

        self._with_lock(self._get_latest_version, "fetching")

    def _get_latest_version(self):
        import json
        import requests

        try:
            r = requests.get(self.URL, timeout=self.TIMEOUT_GET)
            info = r.json()
        except requests.exceptions.RequestException as exc:
            msg = "Failed to retrieve latest version: {}"
            logger.debug(msg.format(exc))
            return

        with open(self.updater_file, "w+") as fobj:
            json.dump(info, fobj)

    def _is_outdated(self):
        return version.parse(self.current) < version.parse(self.latest)

    def _notify(self):
        if not sys.stdout.isatty():
            return

        message = (
            "Update available {red}{current}{reset} -> {green}{latest}{reset}"
            + "\n"
            + self._get_update_instructions()
        ).format(
            red=colorama.Fore.RED,
            reset=colorama.Fore.RESET,
            green=colorama.Fore.GREEN,
            yellow=colorama.Fore.YELLOW,
            blue=colorama.Fore.BLUE,
            current=self.current,
            latest=self.latest,
        )

        logger.info(boxify(message, border_color="yellow"))

    def _get_update_instructions(self):
        instructions = {
            "pip": "Run {yellow}pip{reset} install dvc {blue}--upgrade{reset}",
            "rpm": "Run {yellow}yum{reset} update dvc",
            "brew": "Run {yellow}brew{reset} upgrade dvc",
            "deb": (
                "Run {yellow}apt-get{reset} install"
                " {blue}--only-upgrade{reset} dvc"
            ),
            "binary": (
                "To upgrade follow these steps:\n"
                "1. Uninstall dvc binary\n"
                "2. Go to {blue}https://dvc.org{reset}\n"
                "3. Download and install new binary"
            ),
            "conda": "Run {yellow}conda{reset} update dvc",
            "choco": "Run {yellow}choco{reset} upgrade dvc",
            None: (
                "Find the latest release at\n{blue}"
                "https://github.com/iterative/dvc/releases/latest"
                "{reset}"
            ),
        }

        package_manager = PKG
        if package_manager in ("osxpkg", "exe"):
            package_manager = "binary"

        return instructions[package_manager]
<|endoftext|>