<commit_before># coding: utf-8
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
from __future__ import unicode_literals
import os.path
import glob
from invoke import task, run

base_path = os.path.dirname(os.path.realpath(__file__))
pkg_dir = os.path.join(base_path, 'buncuts')
res_dir = os.path.join(pkg_dir, 'res')
ui_pkg = os.path.join(pkg_dir, 'ui')

# PyInstaller
exe_name = 'BunCuts'

runtime_hook = os.path.join(base_path, 'rthook_pyqt4.py')

app_file = os.path.join(base_path, 'app.pyw')
app_icon = os.path.join(res_dir, os.path.normpath('img/app.ico'))

spec_path = base_path
work_path = os.path.join(base_path, '_build')
dist_path = os.path.join(base_path, '_dist')

clean_paths = (work_path, dist_path,
               os.path.join(spec_path, '{}.spec'.format(exe_name)))

# pyrcc & pyuic
res_name = "app"
res_qrc = os.path.join(res_dir, res_name + '.qrc')
res_mod = os.path.join(ui_pkg, res_name + '_rc.py')
ui_glob = os.path.join(res_dir, os.path.normpath('ui/*.ui'))

@task
def test():
    """output test"""
    run('echo hello invoke',
        echo=True)

@task
def res():
    """compile resources to ui subpackage"""
    run('pyrcc4 -py2 -o "{}" "{}"'.format(res_mod, res_qrc),
        echo=True)

@task
def ui():
    """compile user interface to ui subpackage"""
    for ui_xml in glob.glob(ui_glob):
        name_noext = os.path.splitext(os.path.basename(ui_xml))[0]
        ui_mod = os.path.join(ui_pkg, name_noext + '.py')

        run('pyuic4 --from-imports -o "{}" "{}"'.format(
            ui_mod, ui_xml),
            echo=True)

@task
def app():
    """run the app"""
    run('python {}'.format(app_file),
        echo=True)

@task
def build():
    """build Windows executable with PyInstaller"""
    run(('pyinstaller --runtime-hook="{rthook}" --noconsole '
         '--specpath="{spec}" --workpath="{work}" --distpath="{dist}" '
         '--name={name} --icon="{icon}" "{script}"').format(
             rthook=runtime_hook,
             spec=spec_path,
             work=work_path,
             dist=dist_path,
             name=exe_name,
             icon=app_icon,
             script=app_file),
        echo=True)

@task
def clean():
    """clean the build files of PyInstaller"""
    for path in clean_paths:
        run('rm -rf "{}"'.format(path),
            echo=True)
<commit_msg>[tasks] updated<commit_after># coding: utf-8
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
from __future__ import unicode_literals
import os.path
import glob
from invoke import task, run

base_path = os.path.dirname(os.path.realpath(__file__))
pkg_dir = os.path.join(base_path, 'buncuts')
res_dir = os.path.join(pkg_dir, 'res')
ui_pkg = os.path.join(pkg_dir, 'ui')

# PyInstaller
exe_name = 'BunCuts'

runtime_hook = os.path.join(base_path, 'rthook_pyqt4.py')

app_file = os.path.join(base_path, 'app.pyw')
app_icon = os.path.join(res_dir, os.path.normpath('img/app.ico'))

spec_path = base_path
work_path = os.path.join(base_path, '_build')
dist_path = os.path.join(base_path, '_dist')

clean_paths = (work_path, dist_path,
               os.path.join(spec_path, '{}.spec'.format(exe_name)))

# pyrcc & pyuic
res_name = "app"
res_qrc = os.path.join(res_dir, res_name + '.qrc')
res_mod = os.path.join(ui_pkg, res_name + '_rc.py')
ui_glob = os.path.join(res_dir, os.path.normpath('ui/*.ui'))

@task
def res():
    """compile resources to ui subpackage"""
    run('pyrcc4 -py2 -o "{}" "{}"'.format(res_mod, res_qrc))

@task
def ui():
    """compile user interface to ui subpackage"""
    for ui_xml in glob.glob(ui_glob):
        name_noext = os.path.splitext(os.path.basename(ui_xml))[0]
        ui_mod = os.path.join(ui_pkg, name_noext + '.py')

        run('pyuic4 --from-imports -o "{}" "{}"'.format(
            ui_mod, ui_xml))

@task
def app():
    """run the app"""
    run('python {}'.format(app_file))

@task
def build():
    """build Windows executable with PyInstaller"""
    run(('pyinstaller --runtime-hook="{rthook}" --noconsole '
         '--specpath="{spec}" --workpath="{work}" --distpath="{dist}" '
         '--name={name} --icon="{icon}" "{script}"').format(
             rthook=runtime_hook,
             spec=spec_path,
             work=work_path,
             dist=dist_path,
             name=exe_name,
             icon=app_icon,
             script=app_file))

@task
def clean():
    """clean the build files of PyInstaller"""
    for path in clean_paths:
        run('rm -rf "{}"'.format(path))
<|endoftext|>


<commit_before>import os
import base64
import logging
import platform
from datetime import date, timedelta

from invoke import run, task
# from elasticsearch import helpers
# from dateutil.parser import parse
# from six.moves.urllib import parse as urllib_parse

# import scrapi.harvesters  # noqa
# from scrapi import linter
# from scrapi import registry
# from scrapi import settings

logger = logging.getLogger()

WHEELHOUSE_PATH = os.environ.get('WHEELHOUSE')

@task
def reindex(src, dest):
    ''' Reindexes documents from index {src} to index {dest}'''
    from elasticsearch import helpers
    from scrapi.processing.elasticsearch import DatabaseManager
    dm = DatabaseManager()
    dm.setup()

    helpers.reindex(dm.es, src, dest)
    dm.es.indices.delete(src)

@task
def alias(alias, index):
    ''' Creates an Elasticsearch index alias '''
    from scrapi.processing.elasticsearch import DatabaseManager
    dm = DatabaseManager()
    dm.setup()
    dm.es.indices.delete_alias(index=alias, name='_all', ignore=404)
    dm.es.indices.put_alias(alias, index)

@task
def migrate(migration, sources=None, kwargs_string=None, dry=True, async=False, group_size=1000):
    ''' Task to run a migration.

    :param migration: The migration function to run. This is passed in
    as a string then interpreted as a function by the invoke task.
    :type migration: str

    :param kwargs_string: parsed into an optional set of keyword
    arguments, so that the invoke migrate task can accept a variable
    number of arguments for each migration.
    The kwargs_string should be in the following format:
        'key:value, key2:value2'
    ...with the keys and values seperated by colons, and each kwarg seperated
    by commas.
    :type kwarg_string: str

    An example of usage renaming mit to mit 2 as a real run would be:
        inv migrate rename -s mit -k 'target:mit2' --no-dry
    An example of calling renormalize on two sources as an async dry run:
        inv migrate renormalize -s 'mit,asu' -a
    '''
    kwargs_string = kwargs_string or ':'
    sources = sources or ''

    from scrapi import migrations
    from scrapi.tasks import migrate

    kwargs = {}
    for key, val in map(lambda x: x.split(':'), kwargs_string.split(',')):
        key, val = key.strip(), val.strip()
        if key not in kwargs.keys():
            kwargs[key] = val
        elif isinstance(kwargs[key], list):
            kwargs[key].append(val)
        else:
            kwargs[key] = [kwargs[key], val]

    kwargs['dry'] = dry
    kwargs['async'] = async
    kwargs['group_size'] = group_size
    kwargs['sources'] = list(map(lambda x: x.strip(), sources.split(',')))

    if kwargs['sources'] == ['']:
        kwargs.pop('sources')

    migrate_func = migrations.__dict__[migration]

    migrate(migrate_func, **kwargs)

@task
def reset_search():
    ''' Restarts Elasticsearch '''
    run("curl -XPOST 'http://localhost:9200/_shutdown'")
    if platform.linux_distribution()[0] == 'Ubuntu':
        run("sudo service elasticsearch restart")
    elif platform.system() == 'Darwin':  # Mac OSX
        run('elasticsearch')

@task
def elasticsearch():
    '''Start a local elasticsearch server

    NOTE: Requires that elasticsearch is installed. See README for instructions
    '''
    if platform.linux_distribution()[0] == 'Ubuntu':
        run("sudo service elasticsearch restart")
    elif platform.system() == 'Darwin':  # Mac OSX
        run('elasticsearch')
    else:
        print(
            "Your system is not recognized, you will have to start elasticsearch manually")

@task
def test(cov=True, doctests=True, verbose=False, debug=False, pdb=False):
    """
    Runs all tests in the 'tests/' directory, and all doctests in the scrAPI directory
    """
    cmd = 'py.test scrapi tests'
    if doctests:
        cmd += ' --doctest-modules'
    if verbose:
        cmd += ' -v'
    if debug:
        cmd += ' -s'
    if cov:
        cmd += ' --cov-report term-missing --cov-config .coveragerc --cov scrapi --cov api'
    if pdb:
        cmd += ' --pdb'

    run(cmd, pty=True)

@task
def wheelhouse(develop=False):
    ''' Sets up the wheelhouse for requirements '''
    req_file = 'dev-requirements.txt' if develop else 'requirements.txt'
    cmd = 'pip wheel --find-links={} -r {} --wheel-dir={}'.format(WHEELHOUSE_PATH, req_file, WHEELHOUSE_PATH)
    run(cmd, pty=True)

@task
def requirements(develop=False, upgrade=False):
    ''' Installs all Python dependencies '''
    req_file = 'dev-requirements.txt' if develop else 'requirements.txt'
    cmd = 'pip install -r {}'.format(req_file)

    if upgrade:
        cmd += ' --upgrade'
    if WHEELHOUSE_PATH:
        cmd += ' --no-index --find-links={}'.format(WHEELHOUSE_PATH)
    run(cmd, pty=True)

@task
def beat(setup=True):
    ''' Runs the Celery Beat service '''
    from scrapi import registry
    from scrapi.tasks import app
    # Set up the provider map for elasticsearch
    if setup:
        provider_map(delete=True)

    app.conf['CELERYBEAT_SCHEDULE'] = registry.beat_schedule
    app.Beat().run()

@task
def worker(loglevel='INFO', hostname='%h', autoreload=False):
    ''' Runs the Celery worker service '''
    from scrapi.tasks import app
    command = ['worker']
    if loglevel:
        command.extend(['--loglevel', loglevel])
    if hostname:
        command.extend(['--hostname', hostname])
    if autoreload:
        command.extend(['--autoreload'])
    app.worker_main(command)

@task
def harvester(harvester_name, async=False, start=None, end=None):
    ''' Runs a specific harvester '''
    from scrapi import settings
    settings.CELERY_ALWAYS_EAGER = not async
    from scrapi import registry
    from scrapi.tasks import run_harvester
    from dateutil.parser import parse

    if not registry.get(harvester_name):
        raise ValueError('No such harvesters {}'.format(harvester_name))

    end = parse(end).date() if end else date.today()
    start = parse(start).date() if start else end - timedelta(settings.DAYS_BACK)

    run_harvester.delay(harvester_name, start_date=start, end_date=end)

@task
def harvesters(async=False, start=None, end=None):
    ''' Runs all harvesters '''
    from scrapi import settings
    settings.CELERY_ALWAYS_EAGER = not async
    from scrapi import registry
    from scrapi.tasks import run_harvester
    from dateutil.parser import parse

    start = parse(start).date() if start else date.today() - timedelta(settings.DAYS_BACK)
    end = parse(end).date() if end else date.today()

    exceptions = []
    for harvester_name in registry.keys():
        try:
            run_harvester.delay(harvester_name, start_date=start, end_date=end)
        except Exception as e:
            logger.exception(e)
            exceptions.append(e)

    logger.info("\n\nNumber of exceptions: {}".format(len(exceptions)))
    for exception in exceptions:
        logger.exception(e)

@task
def provider_map(delete=False):
    ''' Adds favicons and metadata for harvesters to Elasticsearch '''
    from six.moves.urllib import parse as urllib_parse
    from scrapi import registry
    from scrapi.processing.elasticsearch import DatabaseManager
    dm = DatabaseManager()
    dm.setup()
    es = dm.es
    if delete:
        es.indices.delete(index='share_providers', ignore=[404])

    for harvester_name, harvester in registry.items():
        with open("img/favicons/{}_favicon.ico".format(harvester.short_name), "rb") as f:
            favicon = urllib_parse.quote(base64.encodestring(f.read()))

        es.index(
            'share_providers',
            harvester.short_name,
            body={
                'favicon': 'data:image/png;base64,' + favicon,
                'short_name': harvester.short_name,
                'long_name': harvester.long_name,
                'url': harvester.url
            },
            id=harvester.short_name,
            refresh=True
        )
    print(es.count('share_providers', body={'query': {'match_all': {}}})['count'])

@task
def apiserver():
    ''' Runs the Django apiserver '''
    os.system('python manage.py runserver')

@task
def apidb():
    ''' Runs the Django migrations '''
    os.system('python manage.py migrate')

@task
def reset_all():
    ''' WARNING: Nukes all databases and sets them back up. Only run in development '''
    import sys
    from scrapi import settings

    if sys.version[0] == "3":
        raw_input = input

    if raw_input('Are you sure? y/N ') != 'y':
        return
    os.system('psql -c "DROP DATABASE scrapi;"')
    os.system('psql -c "CREATE DATABASE scrapi;"')
    os.system('python manage.py migrate')

    os.system("curl -XDELETE '{}/share*'".format(settings.ELASTIC_URI))
    os.system("invoke alias share share_v2")
    os.system("invoke provider_map")

@task(default=True)
def usage():
    ''' Lists the available commands '''
    run('invoke --list')
<commit_msg>add test for one harvester<commit_after>import os
import base64
import logging
import platform
from datetime import date, timedelta

from invoke import run, task
# from elasticsearch import helpers
# from dateutil.parser import parse
# from six.moves.urllib import parse as urllib_parse

# import scrapi.harvesters  # noqa
# from scrapi import linter
# from scrapi import registry
# from scrapi import settings

logger = logging.getLogger()

WHEELHOUSE_PATH = os.environ.get('WHEELHOUSE')

@task
def reindex(src, dest):
    ''' Reindexes documents from index {src} to index {dest}'''
    from elasticsearch import helpers
    from scrapi.processing.elasticsearch import DatabaseManager
    dm = DatabaseManager()
    dm.setup()

    helpers.reindex(dm.es, src, dest)
    dm.es.indices.delete(src)

@task
def alias(alias, index):
    ''' Creates an Elasticsearch index alias '''
    from scrapi.processing.elasticsearch import DatabaseManager
    dm = DatabaseManager()
    dm.setup()
    dm.es.indices.delete_alias(index=alias, name='_all', ignore=404)
    dm.es.indices.put_alias(alias, index)

@task
def migrate(migration, sources=None, kwargs_string=None, dry=True, async=False, group_size=1000):
    ''' Task to run a migration.

    :param migration: The migration function to run. This is passed in
    as a string then interpreted as a function by the invoke task.
    :type migration: str

    :param kwargs_string: parsed into an optional set of keyword
    arguments, so that the invoke migrate task can accept a variable
    number of arguments for each migration.
    The kwargs_string should be in the following format:
        'key:value, key2:value2'
    ...with the keys and values seperated by colons, and each kwarg seperated
    by commas.
    :type kwarg_string: str

    An example of usage renaming mit to mit 2 as a real run would be:
        inv migrate rename -s mit -k 'target:mit2' --no-dry
    An example of calling renormalize on two sources as an async dry run:
        inv migrate renormalize -s 'mit,asu' -a
    '''
    kwargs_string = kwargs_string or ':'
    sources = sources or ''

    from scrapi import migrations
    from scrapi.tasks import migrate

    kwargs = {}
    for key, val in map(lambda x: x.split(':'), kwargs_string.split(',')):
        key, val = key.strip(), val.strip()
        if key not in kwargs.keys():
            kwargs[key] = val
        elif isinstance(kwargs[key], list):
            kwargs[key].append(val)
        else:
            kwargs[key] = [kwargs[key], val]

    kwargs['dry'] = dry
    kwargs['async'] = async
    kwargs['group_size'] = group_size
    kwargs['sources'] = list(map(lambda x: x.strip(), sources.split(',')))

    if kwargs['sources'] == ['']:
        kwargs.pop('sources')

    migrate_func = migrations.__dict__[migration]

    migrate(migrate_func, **kwargs)

@task
def reset_search():
    ''' Restarts Elasticsearch '''
    run("curl -XPOST 'http://localhost:9200/_shutdown'")
    if platform.linux_distribution()[0] == 'Ubuntu':
        run("sudo service elasticsearch restart")
    elif platform.system() == 'Darwin':  # Mac OSX
        run('elasticsearch')

@task
def elasticsearch():
    '''Start a local elasticsearch server

    NOTE: Requires that elasticsearch is installed. See README for instructions
    '''
    if platform.linux_distribution()[0] == 'Ubuntu':
        run("sudo service elasticsearch restart")
    elif platform.system() == 'Darwin':  # Mac OSX
        run('elasticsearch')
    else:
        print(
            "Your system is not recognized, you will have to start elasticsearch manually")

@task
def test(cov=True, doctests=True, verbose=False, debug=False, pdb=False):
    """
    Runs all tests in the 'tests/' directory, and all doctests in the scrAPI directory
    """
    cmd = 'py.test scrapi tests'
    if doctests:
        cmd += ' --doctest-modules'
    if verbose:
        cmd += ' -v'
    if debug:
        cmd += ' -s'
    if cov:
        cmd += ' --cov-report term-missing --cov-config .coveragerc --cov scrapi --cov api'
    if pdb:
        cmd += ' --pdb'

    run(cmd, pty=True)

@task
def one_test(shortname):
    """
    Runs a test on a specific harvester
    """
    cmd = 'py.test tests/test_harvesters.py::test_harvester\[{}]'.format(shortname)

    run(cmd, pty=True)

@task
def wheelhouse(develop=False):
    ''' Sets up the wheelhouse for requirements '''
    req_file = 'dev-requirements.txt' if develop else 'requirements.txt'
    cmd = 'pip wheel --find-links={} -r {} --wheel-dir={}'.format(WHEELHOUSE_PATH, req_file, WHEELHOUSE_PATH)
    run(cmd, pty=True)

@task
def requirements(develop=False, upgrade=False):
    ''' Installs all Python dependencies '''
    req_file = 'dev-requirements.txt' if develop else 'requirements.txt'
    cmd = 'pip install -r {}'.format(req_file)

    if upgrade:
        cmd += ' --upgrade'
    if WHEELHOUSE_PATH:
        cmd += ' --no-index --find-links={}'.format(WHEELHOUSE_PATH)
    run(cmd, pty=True)

@task
def beat(setup=True):
    ''' Runs the Celery Beat service '''
    from scrapi import registry
    from scrapi.tasks import app
    # Set up the provider map for elasticsearch
    if setup:
        provider_map(delete=True)

    app.conf['CELERYBEAT_SCHEDULE'] = registry.beat_schedule
    app.Beat().run()

@task
def worker(loglevel='INFO', hostname='%h', autoreload=False):
    ''' Runs the Celery worker service '''
    from scrapi.tasks import app
    command = ['worker']
    if loglevel:
        command.extend(['--loglevel', loglevel])
    if hostname:
        command.extend(['--hostname', hostname])
    if autoreload:
        command.extend(['--autoreload'])
    app.worker_main(command)

@task
def harvester(harvester_name, async=False, start=None, end=None):
    ''' Runs a specific harvester '''
    from scrapi import settings
    settings.CELERY_ALWAYS_EAGER = not async
    from scrapi import registry
    from scrapi.tasks import run_harvester
    from dateutil.parser import parse

    if not registry.get(harvester_name):
        raise ValueError('No such harvesters {}'.format(harvester_name))

    end = parse(end).date() if end else date.today()
    start = parse(start).date() if start else end - timedelta(settings.DAYS_BACK)

    run_harvester.delay(harvester_name, start_date=start, end_date=end)

@task
def harvesters(async=False, start=None, end=None):
    ''' Runs all harvesters '''
    from scrapi import settings
    settings.CELERY_ALWAYS_EAGER = not async
    from scrapi import registry
    from scrapi.tasks import run_harvester
    from dateutil.parser import parse

    start = parse(start).date() if start else date.today() - timedelta(settings.DAYS_BACK)
    end = parse(end).date() if end else date.today()

    exceptions = []
    for harvester_name in registry.keys():
        try:
            run_harvester.delay(harvester_name, start_date=start, end_date=end)
        except Exception as e:
            logger.exception(e)
            exceptions.append(e)

    logger.info("\n\nNumber of exceptions: {}".format(len(exceptions)))
    for exception in exceptions:
        logger.exception(e)

@task
def provider_map(delete=False):
    ''' Adds favicons and metadata for harvesters to Elasticsearch '''
    from six.moves.urllib import parse as urllib_parse
    from scrapi import registry
    from scrapi.processing.elasticsearch import DatabaseManager
    dm = DatabaseManager()
    dm.setup()
    es = dm.es
    if delete:
        es.indices.delete(index='share_providers', ignore=[404])

    for harvester_name, harvester in registry.items():
        with open("img/favicons/{}_favicon.ico".format(harvester.short_name), "rb") as f:
            favicon = urllib_parse.quote(base64.encodestring(f.read()))

        es.index(
            'share_providers',
            harvester.short_name,
            body={
                'favicon': 'data:image/png;base64,' + favicon,
                'short_name': harvester.short_name,
                'long_name': harvester.long_name,
                'url': harvester.url
            },
            id=harvester.short_name,
            refresh=True
        )
    print(es.count('share_providers', body={'query': {'match_all': {}}})['count'])

@task
def apiserver():
    ''' Runs the Django apiserver '''
    os.system('python manage.py runserver')

@task
def apidb():
    ''' Runs the Django migrations '''
    os.system('python manage.py migrate')

@task
def reset_all():
    ''' WARNING: Nukes all databases and sets them back up. Only run in development '''
    import sys
    from scrapi import settings

    if sys.version[0] == "3":
        raw_input = input

    if raw_input('Are you sure? y/N ') != 'y':
        return
    os.system('psql -c "DROP DATABASE scrapi;"')
    os.system('psql -c "CREATE DATABASE scrapi;"')
    os.system('python manage.py migrate')

    os.system("curl -XDELETE '{}/share*'".format(settings.ELASTIC_URI))
    os.system("invoke alias share share_v2")
    os.system("invoke provider_map")

@task(default=True)
def usage():
    ''' Lists the available commands '''
    run('invoke --list')
<|endoftext|>


<commit_before>import socket

from celery import Celery

from ping import Ping
from download import Download

PING_COUNT = 5

whoami = socket.getfqdn()

app = Celery("tasks")
app.config_from_object("celeryconfig")

@app.task
def ping(host):
    p = Ping(PING_COUNT)
    return (p.time(host), whoami, host)

@app.task
def download(url):
    d = Download(url)
    return (d.time(url), whoami, url)
<commit_msg>Fixed download bug.<commit_after>import socket

from celery import Celery

from ping import Ping
from download import Download

PING_COUNT = 5

whoami = socket.getfqdn()

app = Celery("tasks")
app.config_from_object("celeryconfig")

@app.task
def ping(host):
    p = Ping(PING_COUNT)
    return (p.time(host), whoami, host)

@app.task
def download(url):
    d = Download()
    return (d.time(url), whoami, url)
<|endoftext|>


<commit_before>from __future__ import absolute_import, unicode_literals
from .celery import app
import halocelery.apputils as apputils
from celery.schedules import crontab
import os

events_hour = int(os.getenv("EVENT_EXPORT_HOUR", 21))
events_min = int(os.getenv("EVENT_EXPORT_MIN", 01))
scans_hour = int(os.getenv("SCAN_EXPORT_HOUR", 21))
scans_min = int(os.getenv("SCAN_EXPORT_MIN", 01))

@app.task
def list_all_groups_formatted():
    halo = apputils.Halo()
    return halo.list_all_groups_formatted()

@app.task
def list_all_servers_formatted():
    halo = apputils.Halo()
    return halo.list_all_servers_formatted()

@app.task
def report_group_formatted(target):
    halo = apputils.Halo()
    return halo.generate_group_report_formatted(target)

@app.task
def report_server_formatted(target):
    """Accepts a hostname or server_id"""
    halo = apputils.Halo()
    return halo.generate_server_report_formatted(target)

@app.task
def servers_in_group_formatted(target):
    """Accepts groupname or ID"""
    # !!! TODO: Need to print the group name at the top of the output!
    halo = apputils.Halo()
    return halo.list_servers_in_group_formatted(target)

@app.task
def report_group_firewall(target):
    """Accepts a hostname or server_id"""
    container = apputils.Containerized()
    return container.generate_firewall_graph(target)

@app.task
def report_ec2_halo_footprint_csv():
    """Return a base64-encoded CSV file listing EC2 resources without Halo."""
    runner = apputils.Containerized()
    return runner.halo_ec2_footprint_csv()

@app.task
def search_server_by_cve(target):
    halo = apputils.Halo()
    return halo.get_server_by_cve(target)

@app.task
def quarantine_server(server_id, quarantine_group_name):
    halo = apputils.Halo()
    quarantine_group_id = halo.get_id_for_group_target(quarantine_group_name)
    halo.move_server(server_id, quarantine_group_id)
    msg = ("Quarantined server %s in group %s\n" % (server_id,
                                                    quarantine_group_name))
    return msg

@app.task
def add_ip_to_list(ip_address, ip_zone_name):
    halo = apputils.Halo()
    return halo.add_ip_to_zone(ip_address, ip_zone_name)

@app.task
def remove_ip_from_list(ip_address, ip_zone_name):
    halo = apputils.Halo()
    return halo.remove_ip_from_zone(ip_address, ip_zone_name)

@app.task(bind=True)
def prior_day_scans_to_s3(self, s3_bucket_name):
    container = apputils.Containerized()
    target_date = apputils.Utility.iso8601_yesterday()
    try:
        container.scans_to_s3(target_date, s3_bucket_name)
    except Exception as e:
        "Exception encountered: %s" % e
        raise self.retry(countdown=120, exc=e, max_retries=5)

@app.task(bind=True)
def prior_day_events_to_s3(self, s3_bucket_name):
    container = apputils.Containerized()
    target_date = apputils.Utility.iso8601_yesterday()
    try:
        container.events_to_s3(target_date, s3_bucket_name)
    except Exception as e:
        "Exception encountered: %s" % e
        raise self.retry(countdown=120, exc=e, max_retries=5)

app.conf.beat_schedule = {
    'daily-events-export': {
        'task': 'halocelery.tasks.prior_day_events_to_s3',
        'schedule': crontab(hour=events_hour, minute=events_min),
        'args': (os.getenv("EVENTS_S3_BUCKET"), )},
    'daily-scans-export': {
        'task': 'halocelery.tasks.prior_day_scans_to_s3',
        'schedule': crontab(hour=scans_hour, minute=scans_min),
        'args': (os.getenv("SCANS_S3_BUCKET"), )}
    }
<commit_msg>chg: dev: pep-8 corrections in tasks.py. !minor<commit_after>from __future__ import absolute_import, unicode_literals
from .celery import app
import halocelery.apputils as apputils
from celery.schedules import crontab
import os

events_hour = int(os.getenv("EVENT_EXPORT_HOUR", 21))
events_min = int(os.getenv("EVENT_EXPORT_MIN", 01))
scans_hour = int(os.getenv("SCAN_EXPORT_HOUR", 21))
scans_min = int(os.getenv("SCAN_EXPORT_MIN", 01))

@app.task
def list_all_groups_formatted():
    halo = apputils.Halo()
    return halo.list_all_groups_formatted()

@app.task
def list_all_servers_formatted():
    halo = apputils.Halo()
    return halo.list_all_servers_formatted()

@app.task
def report_group_formatted(target):
    halo = apputils.Halo()
    return halo.generate_group_report_formatted(target)

@app.task
def report_server_formatted(target):
    """Accepts a hostname or server_id"""
    halo = apputils.Halo()
    return halo.generate_server_report_formatted(target)

@app.task
def servers_in_group_formatted(target):
    """Accepts groupname or ID"""
    # !!! TODO: Need to print the group name at the top of the output!
    halo = apputils.Halo()
    return halo.list_servers_in_group_formatted(target)

@app.task
def report_group_firewall(target):
    """Accepts a hostname or server_id"""
    container = apputils.Containerized()
    return container.generate_firewall_graph(target)

@app.task
def report_ec2_halo_footprint_csv():
    """Return a base64-encoded CSV file listing EC2 resources without Halo."""
    runner = apputils.Containerized()
    return runner.halo_ec2_footprint_csv()

@app.task
def search_server_by_cve(target):
    halo = apputils.Halo()
    return halo.get_server_by_cve(target)

@app.task
def quarantine_server(server_id, quarantine_group_name):
    halo = apputils.Halo()
    quarantine_group_id = halo.get_id_for_group_target(quarantine_group_name)
    halo.move_server(server_id, quarantine_group_id)
    msg = ("Quarantined server %s in group %s\n" % (server_id,
                                                    quarantine_group_name))
    return msg

@app.task
def add_ip_to_list(ip_address, ip_zone_name):
    halo = apputils.Halo()
    return halo.add_ip_to_zone(ip_address, ip_zone_name)

@app.task
def remove_ip_from_list(ip_address, ip_zone_name):
    halo = apputils.Halo()
    return halo.remove_ip_from_zone(ip_address, ip_zone_name)

@app.task(bind=True)
def prior_day_scans_to_s3(self, s3_bucket_name):
    container = apputils.Containerized()
    target_date = apputils.Utility.iso8601_yesterday()
    try:
        container.scans_to_s3(target_date, s3_bucket_name)
    except Exception as e:
        "Exception encountered: %s" % e
        raise self.retry(countdown=120, exc=e, max_retries=5)

@app.task(bind=True)
def prior_day_events_to_s3(self, s3_bucket_name):
    container = apputils.Containerized()
    target_date = apputils.Utility.iso8601_yesterday()
    try:
        container.events_to_s3(target_date, s3_bucket_name)
    except Exception as e:
        "Exception encountered: %s" % e
        raise self.retry(countdown=120, exc=e, max_retries=5)

app.conf.beat_schedule = {
    'daily-events-export': {
        'task': 'halocelery.tasks.prior_day_events_to_s3',
        'schedule': crontab(hour=events_hour, minute=events_min),
        'args': (os.getenv("EVENTS_S3_BUCKET"), )},
    'daily-scans-export': {
        'task': 'halocelery.tasks.prior_day_scans_to_s3',
        'schedule': crontab(hour=scans_hour, minute=scans_min),
        'args': (os.getenv("SCANS_S3_BUCKET"), )}
    }
<|endoftext|>


<commit_before># Copyright 2021 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

# Invoke tasks - a Python-equivelant of a Makefile or Rakefile.
# http://www.pyinvoke.org/

# LINTING NOTE: invoke doesn't support annotations in task signatures.
# https://github.com/pyinvoke/invoke/issues/777
# Workaround: add "  # noqa: ANN001, ANN201"

import os
import sys
from typing import List

from invoke import task

venv = "source ./venv/bin/activate"
GOOGLE_CLOUD_PROJECT = os.environ.get("GOOGLE_CLOUD_PROJECT")

@task
def require_project(c):  # noqa: ANN001, ANN201
    """(Check) Require GOOGLE_CLOUD_PROJECT be defined"""
    if GOOGLE_CLOUD_PROJECT is None:
        print("GOOGLE_CLOUD_PROJECT not defined. Required for task")
        sys.exit(1)

@task
def require_venv(c, test_requirements=False, quiet=True):  # noqa: ANN001, ANN201
    """(Check) Require that virtualenv is setup, requirements installed"""

    c.run("python -m venv venv")
    quiet_param = " -q" if quiet else ""

    with c.prefix(venv):
        c.run(f"pip install -r requirements.txt {quiet_param}")

        if test_requirements:
            c.run(f"pip install -r requirements-test.txt {quiet_param}")

@task
def require_venv_test(c):  # noqa: ANN001, ANN201
    """(Check) Require that virtualenv is setup, requirements (incl. test) installed"""
    require_venv(c, test_requirements=True)

@task
def setup_virtualenv(c):  # noqa: ANN001, ANN201
    """Create virtualenv, and install requirements, with output"""
    require_venv(c, test_requirements=True, quiet=False)

@task(pre=[require_venv])
def start(c):  # noqa: ANN001, ANN201
    """Start the web service"""
    with c.prefix(venv):
        c.run("python app.py")

@task(pre=[require_venv])
def dev(c):  # noqa: ANN001, ANN201
    """Start the web service in a development environment, with fast reload"""
    with c.prefix(venv):
        c.run("FLASK_ENV=development python app.py")

@task(pre=[require_venv])
def lint(c):  # noqa: ANN001, ANN201
    """Run linting checks"""
    with c.prefix(venv):
        local_names = _determine_local_import_names(".")
        c.run(
            "flake8 --exclude venv "
            "--max-line-length=88 "
            "--import-order-style=google "
            f"--application-import-names {','.join(local_names)} "
            "--ignore=E121,E123,E126,E203,E226,E24,E266,E501,E704,W503,W504,I202"
        )

def _determine_local_import_names(start_dir: str) -> List[str]:
    """Determines all import names that should be considered "local".
    This is used when running the linter to insure that import order is
    properly checked.
    """
    file_ext_pairs = [os.path.splitext(path) for path in os.listdir(start_dir)]
    return [
        basename
        for basename, extension in file_ext_pairs
        if extension == ".py"
        or os.path.isdir(os.path.join(start_dir, basename))
        and basename not in ("__pycache__")
    ]

@task(pre=[require_venv])
def fix(c):  # noqa: ANN001, ANN201
    """Apply linting fixes"""
    with c.prefix(venv):
        c.run("black *.py **/*.py --force-exclude venv")
        c.run("isort --profile google *.py **/*.py")

@task(pre=[require_project])
def build(c):  # noqa: ANN001, ANN201
    """Build the service into a container image"""
    c.run(
        f"gcloud builds submit --pack "
        f"image=us-central1-docker.pkg.dev/{GOOGLE_CLOUD_PROJECT}/samples/microservice-template:manual"
    )

@task(pre=[require_project])
def deploy(c):  # noqa: ANN001, ANN201
    """Deploy the container into Cloud Run (fully managed)"""
    c.run(
        "gcloud run deploy microservice-template "
        f"--image us-central1-docker.pkg.dev/{GOOGLE_CLOUD_PROJECT}/samples/microservice-template:manual "
        "--platform managed --region us-central1"
    )

@task(pre=[require_venv_test])
def test(c):  # noqa: ANN001, ANN201
    """Run unit tests"""
    with c.prefix(venv):
        c.run("pytest test/test_app.py")

@task(pre=[require_venv_test])
def system_test(c):  # noqa: ANN001, ANN201
    """Run system tests"""
    with c.prefix(venv):
        c.run("pytest test/test_system.py")
<commit_msg>refactor(tasks): Variabilize deployment region in tasks<commit_after># Copyright 2021 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

# Invoke tasks - a Python-equivelant of a Makefile or Rakefile.
# http://www.pyinvoke.org/

# LINTING NOTE: invoke doesn't support annotations in task signatures.
# https://github.com/pyinvoke/invoke/issues/777
# Workaround: add "  # noqa: ANN001, ANN201"

import os
import sys
from typing import List

from invoke import task

venv = "source ./venv/bin/activate"
GOOGLE_CLOUD_PROJECT = os.environ.get("GOOGLE_CLOUD_PROJECT")
REGION = os.environ.get("REGION", "us-central1")

@task
def require_project(c):  # noqa: ANN001, ANN201
    """(Check) Require GOOGLE_CLOUD_PROJECT be defined"""
    if GOOGLE_CLOUD_PROJECT is None:
        print("GOOGLE_CLOUD_PROJECT not defined. Required for task")
        sys.exit(1)

@task
def require_venv(c, test_requirements=False, quiet=True):  # noqa: ANN001, ANN201
    """(Check) Require that virtualenv is setup, requirements installed"""

    c.run("python -m venv venv")
    quiet_param = " -q" if quiet else ""

    with c.prefix(venv):
        c.run(f"pip install -r requirements.txt {quiet_param}")

        if test_requirements:
            c.run(f"pip install -r requirements-test.txt {quiet_param}")

@task
def require_venv_test(c):  # noqa: ANN001, ANN201
    """(Check) Require that virtualenv is setup, requirements (incl. test) installed"""
    require_venv(c, test_requirements=True)

@task
def setup_virtualenv(c):  # noqa: ANN001, ANN201
    """Create virtualenv, and install requirements, with output"""
    require_venv(c, test_requirements=True, quiet=False)

@task(pre=[require_venv])
def start(c):  # noqa: ANN001, ANN201
    """Start the web service"""
    with c.prefix(venv):
        c.run("python app.py")

@task(pre=[require_venv])
def dev(c):  # noqa: ANN001, ANN201
    """Start the web service in a development environment, with fast reload"""
    with c.prefix(venv):
        c.run("FLASK_ENV=development python app.py")

@task(pre=[require_venv])
def lint(c):  # noqa: ANN001, ANN201
    """Run linting checks"""
    with c.prefix(venv):
        local_names = _determine_local_import_names(".")
        c.run(
            "flake8 --exclude venv "
            "--max-line-length=88 "
            "--import-order-style=google "
            f"--application-import-names {','.join(local_names)} "
            "--ignore=E121,E123,E126,E203,E226,E24,E266,E501,E704,W503,W504,I202"
        )

def _determine_local_import_names(start_dir: str) -> List[str]:
    """Determines all import names that should be considered "local".
    This is used when running the linter to insure that import order is
    properly checked.
    """
    file_ext_pairs = [os.path.splitext(path) for path in os.listdir(start_dir)]
    return [
        basename
        for basename, extension in file_ext_pairs
        if extension == ".py"
        or os.path.isdir(os.path.join(start_dir, basename))
        and basename not in ("__pycache__")
    ]

@task(pre=[require_venv])
def fix(c):  # noqa: ANN001, ANN201
    """Apply linting fixes"""
    with c.prefix(venv):
        c.run("black *.py **/*.py --force-exclude venv")
        c.run("isort --profile google *.py **/*.py")

@task(pre=[require_project])
def build(c):  # noqa: ANN001, ANN201
    """Build the service into a container image"""
    c.run(
        f"gcloud builds submit --pack "
        f"image={REGION}-docker.pkg.dev/{GOOGLE_CLOUD_PROJECT}/samples/microservice-template:manual"
    )

@task(pre=[require_project])
def deploy(c):  # noqa: ANN001, ANN201
    """Deploy the container into Cloud Run (fully managed)"""
    c.run(
        "gcloud run deploy microservice-template "
        f"--image {REGION}-docker.pkg.dev/{GOOGLE_CLOUD_PROJECT}/samples/microservice-template:manual "
        f"--platform managed --region {REGION}"
    )

@task(pre=[require_venv_test])
def test(c):  # noqa: ANN001, ANN201
    """Run unit tests"""
    with c.prefix(venv):
        c.run("pytest test/test_app.py")

@task(pre=[require_venv_test])
def system_test(c):  # noqa: ANN001, ANN201
    """Run system tests"""
    with c.prefix(venv):
        c.run("pytest test/test_system.py")
<|endoftext|>


<commit_before>#App engine platform
import logging, webapp2, appengine_config

import GlobalUtilities as tools
import DataModels as models

from google.appengine.api import mail
from google.appengine.ext.webapp import template

class Confirmation(webapp2.RequestHandler):
    def post(self):
        donation_key = self.request.get("donation_key")
        d = tools.getKey(donation_key).get()

        logging.info("Retrying confirmation email through task queue for donation: " + donation_key)

        d.confirmation.email()

class MailchimpAdd(webapp2.RequestHandler):
    def post(self):
        email = self.request.get("email")
        name = self.request.get("name")

        settings_key = self.request.get("settings")
        s = tools.getKey(settings_key).get()

        s.mailchimp.add(email, name, True)

        logging.info("Retrying Mailchimp add through task queue for: " + email  + " under settings ID: " + settings_key)

class AnnualReport(webapp2.RequestHandler):
    def post(self):
        contact_key = self.request.get("contact_key")
        year = self.request.get("year")
        c = tools.getKey(contact_key).get()

        logging.info("AnnualReport handler hit with contact key " + contact_key + " and year " + year)

        if c.email:

            message = mail.EmailMessage()
            message.to = c.email
            message.sender = "Global Hope India <donate@globalhopeindia.org>"
            message.subject = "2012 Global Hope India Donations"

            donations = c.data.annual_donations(year)
            donation_total = tools.toDecimal(0)

            for d in donations:
                donation_total += d.confirmation_amount

            donation_total = "${:,.2f}".format(donation_total)

            template_variables = {"s":c.settings.get(), "c":c, "donations":donations, "year":year, "donation_total":str(donation_total)}

            message.html = template.render("pages/letters/donor_report.html", template_variables)
            message.send()

            logging.info("Annual report sent to:" + c.name)

        else:
            logging.info("Annual report not sent sent because " + c.name + "doesn't have an email.")

class IndexAll(webapp2.RequestHandler):
    def post(self):
        settings_key = self.request.get("settings_key")
        key = tools.getKey(settings_key)

        mode = self.request.get("mode")

        if mode == "contacts":
            contacts = models.Contact.gql("WHERE settings = :s", s=key)
            for c in contacts:
                c.search.index()

        elif mode == "deposits":
            deposits = models.DepositReceipt.gql("WHERE settings = :s", s=key)
            for de in deposits:
                de.search.index()

        elif mode == "donations":
            donations = models.Donation.gql("WHERE settings = :s", s=key)
            for d in donations:
                d.search.index()

        elif mode == "individuals":
            individuals = models.Individual.gql("WHERE settings = :s", s=key)
            for i in individuals:
                i.search.index()

        elif mode == "teams":
            teams = models.Team.gql("WHERE settings = :s", s=key)
            for t in teams:
                t.search.index()

class DelayIndexing(webapp2.RequestHandler):
    def post(self):
        entity_key = self.request.get("e")

        e = tools.getKey(entity_key).get()
        e.search.index()

app = webapp2.WSGIApplication([
       ('/tasks/confirmation', Confirmation),
       ('/tasks/annualreport', AnnualReport),
       ('/tasks/indexall', IndexAll),
       ('/tasks/delayindexing', DelayIndexing),
       ('/tasks/mailchimp', MailchimpAdd)],
       debug=True)
app = appengine_config.recording_add_wsgi_middleware(app)
<commit_msg>Add 500 error on delay indexing task queue<commit_after>#App engine platform
import logging, webapp2, appengine_config

import GlobalUtilities as tools
import DataModels as models

from google.appengine.api import mail
from google.appengine.ext.webapp import template

class Confirmation(webapp2.RequestHandler):
    def post(self):
        donation_key = self.request.get("donation_key")
        d = tools.getKey(donation_key).get()

        logging.info("Retrying confirmation email through task queue for donation: " + donation_key)

        d.confirmation.email()

class MailchimpAdd(webapp2.RequestHandler):
    def post(self):
        email = self.request.get("email")
        name = self.request.get("name")

        settings_key = self.request.get("settings")
        s = tools.getKey(settings_key).get()

        s.mailchimp.add(email, name, True)

        logging.info("Retrying Mailchimp add through task queue for: " + email  + " under settings ID: " + settings_key)

class AnnualReport(webapp2.RequestHandler):
    def post(self):
        contact_key = self.request.get("contact_key")
        year = self.request.get("year")
        c = tools.getKey(contact_key).get()

        logging.info("AnnualReport handler hit with contact key " + contact_key + " and year " + year)

        if c.email:

            message = mail.EmailMessage()
            message.to = c.email
            message.sender = "Global Hope India <donate@globalhopeindia.org>"
            message.subject = "2012 Global Hope India Donations"

            donations = c.data.annual_donations(year)
            donation_total = tools.toDecimal(0)

            for d in donations:
                donation_total += d.confirmation_amount

            donation_total = "${:,.2f}".format(donation_total)

            template_variables = {"s":c.settings.get(), "c":c, "donations":donations, "year":year, "donation_total":str(donation_total)}

            message.html = template.render("pages/letters/donor_report.html", template_variables)
            message.send()

            logging.info("Annual report sent to:" + c.name)

        else:
            logging.info("Annual report not sent sent because " + c.name + "doesn't have an email.")

class IndexAll(webapp2.RequestHandler):
    def post(self):
        settings_key = self.request.get("settings_key")
        key = tools.getKey(settings_key)

        mode = self.request.get("mode")

        if mode == "contacts":
            contacts = models.Contact.gql("WHERE settings = :s", s=key)
            for c in contacts:
                c.search.index()

        elif mode == "deposits":
            deposits = models.DepositReceipt.gql("WHERE settings = :s", s=key)
            for de in deposits:
                de.search.index()

        elif mode == "donations":
            donations = models.Donation.gql("WHERE settings = :s", s=key)
            for d in donations:
                d.search.index()

        elif mode == "individuals":
            individuals = models.Individual.gql("WHERE settings = :s", s=key)
            for i in individuals:
                i.search.index()

        elif mode == "teams":
            teams = models.Team.gql("WHERE settings = :s", s=key)
            for t in teams:
                t.search.index()

class DelayIndexing(webapp2.RequestHandler):
    def post(self):
        entity_key = self.request.get("e")

        try:
            e = tools.getKey(entity_key).get()
            e.search.index()

        except:
            self.error(500)

app = webapp2.WSGIApplication([
       ('/tasks/confirmation', Confirmation),
       ('/tasks/annualreport', AnnualReport),
       ('/tasks/indexall', IndexAll),
       ('/tasks/delayindexing', DelayIndexing),
       ('/tasks/mailchimp', MailchimpAdd)],
       debug=True)
app = appengine_config.recording_add_wsgi_middleware(app)
<|endoftext|>


<commit_before># coding: utf-8

import os
import errno
import shutil
import simplejson
from app import app
from models import *
from werkzeug.datastructures import FileStorage
from werkzeug.utils import secure_filename
from PIL import Image
import pysftp

DIST_ROOT = app.config['DATA_DIST']

ACTIVITY_TYPES = {
    Activity.ORDINARY: 'ordinary',
    Activity.TIME_LIMIT: 'time_limit',
    Activity.CHALLENGE: 'challenge'
}

QUESTION_TYPES = {
    Question.SINGLE: 'single',
    Question.MULTI: 'multi'
}

class UploadNotAllowed(Exception):
    pass

def save_welcome_img(storage, activity):
    if not isinstance(storage, FileStorage):
        raise TypeError("storage must be a werkzeug.FileStorage")

    allowed_extensions = ('.jpg', '.jpeg', '.png', '.gif', '.svg', '.bmp', '.webp')
    ext = os.path.splitext(storage.filename)[1]
    if not (ext.lower() in allowed_extensions):
        raise UploadNotAllowed(u'不支持的文件扩展名，只支持{}格式'
                               .format(','.join(allowed_extensions)))

    dist_path = os.path.join(app.config['MEDIA_PATH'], 'welcome_image')
    try:
        os.makedirs(dist_path, mode=0o775)
    except OSError as e:
        if e.errno != errno.EEXIST:
            raise

    image_file = secure_filename(activity.code + ext)
    target = os.path.join(dist_path, image_file)

    image = Image.open(storage)
    image.thumbnail((1024, 1024), Image.ANTIALIAS)
    image.save(target, image.format)
    return image_file

def chunks(l, n):
    for i in range(0, len(l), n):
        yield l[i:i + n]

def generate_json_files_for_activity(activity, welcome_img=None):
    config = {}
    dist_path = os.path.join(DIST_ROOT, activity.code)
    try:
        shutil.rmtree(dist_path)
    except OSError as e:
        if e.errno != errno.ENOENT:
            raise
    os.makedirs(dist_path, mode=0o775)

    file_counter = 1
    singles = []
    multiples = []

    for chunk in chunks(activity.book.questions.where(Question.type == Question.SINGLE), 20):
        filename = '%d.json' % file_counter
        full_path = os.path.join(dist_path, filename)
        file_counter += 1
        generate_json_for_questions(chunk, full_path)
        singles.append(filename)

    for chunk in chunks(activity.book.questions.where(Question.type == Question.MULTI), 20):
        filename = '%d.json' % file_counter
        full_path = os.path.join(dist_path, filename)
        file_counter += 1
        generate_json_for_questions(chunk, full_path)
        multiples.append(filename)

    config.update({
        "id": activity.id,
        "name": activity.name,
        "welcome": activity.welcome,
        "type": ACTIVITY_TYPES[activity.type],
        "max_chances": activity.chances,
        "count": {
            "single": activity.single,
            "multi": activity.multi
        },
        "question_files": {
            "single": singles,
            "multi": multiples
        },
        "question_points": {
            "single": activity.single_points,
            "multi": activity.multi_points
        },
        "info_fields": [],
        "start_at": activity.start_at.isoformat() + '+08:00',
        "end_at": activity.end_at.isoformat() + '+08:00',
        "show_answer": activity.show_answer
    })

    if activity.type in (Activity.ORDINARY, Activity.CHALLENGE):
        config['time_per_question'] = activity.time_limit
    if activity.type == Activity.TIME_LIMIT:
        config['time_per_quiz'] = activity.time_limit
        config['time_per_question'] = 0

    for i, field in enumerate(filter(None, [activity.info_field_1, activity.info_field_2, activity.info_field_3]),
                              start=1):
        config['info_fields'].append(['info_field_' + str(i), field])

    if activity.welcome_img:
        config['welcome_img'] = activity.welcome_img
        img_path = os.path.join(app.config['MEDIA_PATH'], 'welcome_image')
        img_file = os.path.join(img_path, activity.welcome_img)
        dist_img_file = os.path.join(dist_path, activity.welcome_img)
        shutil.copyfile(img_file, dist_img_file)

    config_filename = os.path.join(dist_path, 'config.json')
    with open(config_filename, 'w') as f:
        simplejson.dump(config, f)

    upload_files_for_activity(activity)

def generate_json_for_questions(questions, filename):
    objects = []
    for question in questions:
        question_options = filter(None, [question.option_A, question.option_B, question.option_C,
                                         question.option_D, question.option_E, question.option_F])
        options = [{'id': i, 'content': o} for i, o in enumerate(question_options, start=1)]
        objects.append({
            'id': question.id,
            'content': question.content,
            'options': options,
            'answer': map(int, question.correct_option.split(',')),
            'type': QUESTION_TYPES[question.type]
        })
    with open(filename, 'w') as f:
        simplejson.dump({'objects': objects}, f, indent=2, separators=(',', ': '))

def upload_files_for_activity(activity):
    dist_path = os.path.join(DIST_ROOT, activity.code)
    for node_config in app.config['QUIZ_NODES']:
        upload_data(node_config,
                    local=os.path.abspath(dist_path),
                    remote=activity.code)

def upload_data(sftp_config, local, remote):
    with pysftp.Connection(**sftp_config) as sftp:
        sftp.makedirs(remote, mode=755)
        sftp.put_d(local, remote)<commit_msg>refactor(task): refine file generation task<commit_after># coding: utf-8

import os
import errno
import shutil
import simplejson
from app import app
from models import *
from werkzeug.datastructures import FileStorage
from werkzeug.utils import secure_filename
from PIL import Image
import pysftp

DIST_ROOT = app.config['DATA_DIST']

ACTIVITY_TYPES = {
    Activity.ORDINARY: 'ordinary',
    Activity.TIME_LIMIT: 'time_limit',
    Activity.CHALLENGE: 'challenge'
}

QUESTION_TYPES = {
    Question.SINGLE: 'single',
    Question.MULTI: 'multi'
}

QUESTION_CHUNK_LENGTH = 20

class UploadNotAllowed(Exception):
    pass

def save_welcome_img(storage, activity):
    if not isinstance(storage, FileStorage):
        raise TypeError("storage must be a werkzeug.FileStorage")

    allowed_extensions = ('.jpg', '.jpeg', '.png', '.gif', '.svg', '.bmp', '.webp')
    ext = os.path.splitext(storage.filename)[1]
    if not (ext.lower() in allowed_extensions):
        raise UploadNotAllowed(u'不支持的文件扩展名，只支持{}格式'
                               .format(','.join(allowed_extensions)))

    dist_path = os.path.join(app.config['MEDIA_PATH'], 'welcome_image')
    try:
        os.makedirs(dist_path, mode=0o775)
    except OSError as e:
        if e.errno != errno.EEXIST:
            raise

    image_file = secure_filename(activity.code + ext)
    target = os.path.join(dist_path, image_file)

    image = Image.open(storage)
    image.thumbnail((1024, 1024), Image.ANTIALIAS)
    image.save(target, image.format)
    return image_file

def chunks(l, n):
    for i in range(0, len(l), n):
        yield l[i:i + n]

def generate_json_files_for_activity(activity):
    config = {}
    dist_path = os.path.join(DIST_ROOT, activity.code)
    try:
        shutil.rmtree(dist_path)
    except OSError as e:
        if e.errno != errno.ENOENT:
            raise
    os.makedirs(dist_path, mode=0o775)

    file_counter = 1
    singles = []
    multiples = []

    questions = activity.book.questions

    for chunk in chunks(questions.where(Question.type == Question.SINGLE),
                        QUESTION_CHUNK_LENGTH):
        filename = '%d.json' % file_counter
        full_path = os.path.join(dist_path, filename)
        file_counter += 1
        generate_json_for_questions(chunk, full_path)
        singles.append(filename)

    for chunk in chunks(questions.where(Question.type == Question.MULTI),
                        QUESTION_CHUNK_LENGTH):
        filename = '%d.json' % file_counter
        full_path = os.path.join(dist_path, filename)
        file_counter += 1
        generate_json_for_questions(chunk, full_path)
        multiples.append(filename)

    config.update({
        "id": activity.id,
        "name": activity.name,
        "welcome": activity.welcome,
        "type": ACTIVITY_TYPES[activity.type],
        "max_chances": activity.chances,
        "count": {
            "single": activity.single,
            "multi": activity.multi
        },
        "question_files": {
            "single": singles,
            "multi": multiples
        },
        "question_points": {
            "single": activity.single_points,
            "multi": activity.multi_points
        },
        "info_fields": [],
        "start_at": activity.start_at.isoformat() + '+08:00',
        "end_at": activity.end_at.isoformat() + '+08:00',
        "show_answer": activity.show_answer
    })

    if activity.type in (Activity.ORDINARY, Activity.CHALLENGE):
        config['time_per_question'] = activity.time_limit
    if activity.type == Activity.TIME_LIMIT:
        config['time_per_quiz'] = activity.time_limit
        config['time_per_question'] = 0

    for i, field in enumerate(filter(None, [activity.info_field_1, activity.info_field_2, activity.info_field_3]),
                              start=1):
        config['info_fields'].append(['info_field_' + str(i), field])

    if activity.welcome_img:
        config['welcome_img'] = activity.welcome_img
        img_path = os.path.join(app.config['MEDIA_PATH'], 'welcome_image')
        img_file = os.path.join(img_path, activity.welcome_img)
        dist_img_file = os.path.join(dist_path, activity.welcome_img)
        try:
            shutil.copyfile(img_file, dist_img_file)
        except OSError as e:
            app.logger.debug(e.message)

    config_filename = os.path.join(dist_path, 'config.json')
    with open(config_filename, 'w') as f:
        simplejson.dump(config, f)

    upload_files_for_activity(activity)

def generate_json_for_questions(questions, filename):
    objects = []
    for question in questions:
        question_options = filter(None, [question.option_A, question.option_B, question.option_C,
                                         question.option_D, question.option_E, question.option_F])
        options = [{'id': i, 'content': o} for i, o in enumerate(question_options, start=1)]
        objects.append({
            'id': question.id,
            'content': question.content,
            'options': options,
            'answer': map(int, question.correct_option.split(',')),
            'type': QUESTION_TYPES[question.type]
        })
    with open(filename, 'w') as f:
        simplejson.dump({'objects': objects}, f, indent=2, separators=(',', ': '))

def upload_files_for_activity(activity):
    dist_path = os.path.join(DIST_ROOT, activity.code)
    for node_config in app.config['QUIZ_NODES']:
        upload_data(node_config,
                    local=os.path.abspath(dist_path),
                    remote=activity.code)

def upload_data(sftp_config, local, remote):
    with pysftp.Connection(**sftp_config) as sftp:
        sftp.makedirs(remote, mode=755)
        sftp.put_d(local, remote)<|endoftext|>


<commit_before>
import json

import jsonschema

V2_SCHEMA_JSON = """

{
  "title": "Identity V2.0 Service Catalog",
  "type": "array",
  "items": {
    "type": "object",
    "properties": {
        "type": {"type": "string", "minlength": 1},
        "name": {"type": "string", "minlength": 1},
        "endpoints_links": {"type": "array", "maxItems": 0},
        "endpoints": {
          "type": "array",
          "items": {
            "type": "object",
            "properties": {
              "adminURL": {"type": "string"},
              "internalURL": {"type": "string"},
              "publicURL": {"type": "string"},
              "region": {"type": "string"},
              "id": {"type": "string"}
            },
            "required": [
              "adminURL", "internalURL", "publicURL", "region", "id"]
          }
        }
    },
    "required": ["type", "name", "endpoints_links", "endpoints"]
  }
}

"""

V2_SCHEMA = json.loads(V2_SCHEMA_JSON)

SAMPLE_V2_CATALOG_JSON = """

{
  "serviceCatalog": [
    {
      "type": "identity",
      "name": "keystone",
      "endpoints_links": [],
      "endpoints": [
        {
          "adminURL": "http://192.168.122.239:35357/v3",
          "internalURL": "http://192.168.122.239:5000/v3",
          "publicURL": "http://192.168.122.239:5000/v3",
          "region": "RegionOne",
          "id": "0e0b3d009aa04da0aee163e034dd6190"
        }
      ]
    }
  ]
}

"""

SAMPLE_V2_CATALOG = json.loads(SAMPLE_V2_CATALOG_JSON)

jsonschema.validate(SAMPLE_V2_CATALOG['serviceCatalog'], V2_SCHEMA)
<commit_msg>Start of V3 schema<commit_after>
import json

import jsonschema

V2_SCHEMA_JSON = """
{
  "title": "Identity V2.0 Service Catalog",
  "type": "array",
  "items": {
    "type": "object",
    "properties": {
        "type": {"type": "string", "minlength": 1},
        "name": {"type": "string", "minlength": 1},
        "endpoints_links": {"type": "array", "maxItems": 0},
        "endpoints": {
          "type": "array",
          "items": {
            "type": "object",
            "properties": {
              "adminURL": {"type": "string"},
              "internalURL": {"type": "string"},
              "publicURL": {"type": "string"},
              "region": {"type": "string"},
              "id": {"type": "string"}
            },
            "required": [
              "adminURL", "internalURL", "publicURL", "region", "id"]
          }
        }
    },
    "required": ["type", "name", "endpoints_links", "endpoints"]
  }
}
"""

V2_SCHEMA = json.loads(V2_SCHEMA_JSON)

SAMPLE_V2_CATALOG_JSON = """
{
  "serviceCatalog": [
    {
      "type": "identity",
      "name": "keystone",
      "endpoints_links": [],
      "endpoints": [
        {
          "adminURL": "http://192.168.122.239:35357/v3",
          "internalURL": "http://192.168.122.239:5000/v3",
          "publicURL": "http://192.168.122.239:5000/v3",
          "region": "RegionOne",
          "id": "0e0b3d009aa04da0aee163e034dd6190"
        }
      ]
    }
  ]
}
"""

SAMPLE_V2_CATALOG = json.loads(SAMPLE_V2_CATALOG_JSON)

jsonschema.validate(SAMPLE_V2_CATALOG['serviceCatalog'], V2_SCHEMA)

V3_SCHEMA_JSON = """
{
   "title": "Identity V3 Service Catalog"
}
"""

V3_SCHEMA = json.loads(V2_SCHEMA_JSON)
<|endoftext|>


<commit_before>import unittest
import todo_list
import todo_item

class TodoTests(unittest.TestCase):
	def test_todo_list_instantiation(self):
		test_list = todo_list.ToDoList()
		self.assertEqual(type(todo_list.ToDoList('My List', 'A list containing test to do items', [todo_item.ToDoItem('to do item 1')])), type(test_list))

	def test_todo_item_instantiation(self):
		test_item = todo_item.ToDoItem()
		self.assertEqual(type(todo_item.ToDoItem()), type(test_item))

	def test_todo_list_with_non_string_name(self):
		test_list = todo_list.ToDoList(12, 'A list containing test to do items', ['to do item 1', 'to do item 2'])
		self.assertEqual(test_list.name, 'Enter valid name')

	def test_todo_list_with_non_string_description(self):
		test_list = todo_list.ToDoList('My List', 22, ['to do item 1', 'to do item 2'])
		self.assertEqual(test_list.description, 'Enter valid description')

	def test_todo_list_items_attribute_type(self):
		test_list = todo_list.ToDoList('My List', 'A list containing test to do items', [todo_item.ToDoItem()])
		self.assertEqual(type(test_list.todo_items), type(todo_item.ToDoItem))

	def test_adding_of_item_with_none_string_content(self):
		test_list = todo_list.ToDoList('My List', 'A list containing test to do items', [todo_item.ToDoItem()])
		test_list.add_todo(23)
		self.assertEqual(test_list.todo_items[-1], 'Enter valid content')

	def test_adding_of_item_with_non_boolean_complete(self):
		test_list = todo_list.ToDoList('My List', 'A list containing test to do items', [todo_item.ToDoItem()])
		test_list.add_todo('Test to do item', 23)
		self.assertEqual(test_list.todo_items[-1].complete, False)

	def test_finish_item_out_of_range_index(self):
		test_list = todo_list.ToDoList('My List', 'A list containing test to do items', [todo_item.ToDoItem('to do item 1')])
		self.assertEqual(test_list.finish_item(5), 'That to do item does not exist')<commit_msg>add two tests for ToDoList instantiation<commit_after>import unittest
import todo_list
import todo_item

class TodoTests(unittest.TestCase):
	def test_todo_list_instantiation(self):
		test_list = todo_list.ToDoList()
		self.assertEqual(type(todo_list.ToDoList('My List', 'A list containing test to do items', [todo_item.ToDoItem('to do item 1')])), type(test_list))

	def test_todo_item_instantiation(self):
		test_item = todo_item.ToDoItem()
		self.assertEqual(type(todo_item.ToDoItem()), type(test_item))

	def test_todo_list_with_non_string_name(self):
		test_list = todo_list.ToDoList(12, 'A list containing test to do items', ['to do item 1', 'to do item 2'])
		self.assertEqual(test_list.name, 'Enter valid name')

	def test_todo_list_with_non_string_description(self):
		test_list = todo_list.ToDoList('My List', 22, ['to do item 1', 'to do item 2'])
		self.assertEqual(test_list.description, 'Enter valid description')

	def test_todo_list_items_attribute_type(self):
		test_list = todo_list.ToDoList('My List', 'A list containing test to do items', [todo_item.ToDoItem()])
		self.assertEqual(type(test_list.todo_items), type(todo_item.ToDoItem))

	def test_adding_of_item_with_none_string_content(self):
		test_list = todo_list.ToDoList('My List', 'A list containing test to do items', [todo_item.ToDoItem()])
		test_list.add_todo(23)
		self.assertEqual(test_list.todo_items[-1], 'Enter valid content')

	def test_adding_of_item_with_non_boolean_complete(self):
		test_list = todo_list.ToDoList('My List', 'A list containing test to do items', [todo_item.ToDoItem()])
		test_list.add_todo('Test to do item', 23)
		self.assertEqual(test_list.todo_items[-1].complete, False)

	def test_finish_item_out_of_range_index(self):
		test_list = todo_list.ToDoList('My List', 'A list containing test to do items', [todo_item.ToDoItem('to do item 1')])
		self.assertEqual(test_list.finish_item(5), 'That to do item does not exist')

	def test_todo_item_with_none_string_content(self):
		test_item = todo_item.ToDoItem(23)
		self.assertEqual(test_item.content, 'Invalid content')

	def test_none_boolean_test_item_complete_attribute(self):
		test_item = todo_item.ToDoItem('to do item 1', 'silly boolean')
		self.assertEqual(test_item.complete, False)<|endoftext|>


<commit_before>import os
import subprocess
import unittest

class BehaveDjangoTestCase(unittest.TestCase):

    def test_command_should_exit_zero_if_passing(self):
        FNULL = open(os.devnull, 'w')
        return_code = subprocess.call(
            ['python', 'manage.py', 'behave', '--tags', '~@failing'],
            stdout=FNULL,
            stderr=subprocess.STDOUT)
        self.assertEqual(return_code, 0)

    def test_command_should_exit_nonzero_if_failing(self):
        FNULL = open(os.devnull, 'w')
        return_code = subprocess.call(
            ['python', 'manage.py', 'behave', '--tags', '@failing'],
            stdout=FNULL,
            stderr=subprocess.STDOUT)
        self.assertNotEqual(return_code, 0)

if __name__ == '__main__':
    unittest.main()
<commit_msg>Temporarily deactivate stdout/stderr redirection<commit_after>import os
import subprocess
import unittest

class BehaveDjangoTestCase(unittest.TestCase):

    def test_command_should_exit_zero_if_passing(self):
        # FNULL = open(os.devnull, 'w')
        return_code = subprocess.call(
            ['python', 'manage.py', 'behave', '--tags', '~@failing'],
            # stdout=FNULL,
            # stderr=subprocess.STDOUT,
            )
        self.assertEqual(return_code, 0)

    def test_command_should_exit_nonzero_if_failing(self):
        # FNULL = open(os.devnull, 'w')
        return_code = subprocess.call(
            ['python', 'manage.py', 'behave', '--tags', '@failing'],
            # stdout=FNULL,
            # stderr=subprocess.STDOUT,
            )
        self.assertNotEqual(return_code, 0)

if __name__ == '__main__':
    unittest.main()
<|endoftext|>


<commit_before>import BasisSet as BS
import DIIS as DIIS
import HartreeFock as HF
import MolecularIntegrals as MI
import MPn as MP
import Properties as PROP
import Qfit as QFIT
import Utilityfunc as UF
import numpy as np

def test_N():
    check = 5.701643762839922
    calc = BS.N(1.0,1.0,1.0,1.0)
    assert abs(check-calc) < 10**-12

def test_gaussian_product_center():
    check = 3.5
    calc = MI.gaussian_product_center(1.0,2.0,3.0,4.0)
    assert abs(check - calc) < 10**-12

def test_magvec():
    check = 5.196152422706632
    calc = QFIT.magvec([1,2,3],[4,5,6])
    assert abs(check - calc) < 10**-12

def test_centerofcharge():
    check1 = 0.1020452034
    check2 = 0.162516435
    check3 = 0.0
    input1 = np.array([[10,0,0,0],[8.0, 0.0, 0.0, 0],[1.0, 1.70075339, 0.0, 0],[1.0, -0.68030136, 1.62516435, 0.0]])
    calc1, calc2, calc3 = QFIT.centerofcharge(input1)
    assert abs(check1 - calc1) < 10**-8
    assert abs(check2 - calc2) < 10**-8
    assert abs(check3 - calc3) < 10**-8

def test_solveFit():
    check = np.array([-4.0, 4.5])
    A = np.array([[1.0,2.0],[3.0,4.0]])
    B = np.array([5.0,6.0])
    calc = QFIT.solveFit(A,B)
    assert abs(check[0] - calc[0]) < 10**-12
    assert abs(check[1] - calc[1]) < 10**-12

def test_makepoints():
    check = np.array([[  4.92471737e-16,   0.00000000e+00,  -4.02133690e+00,
          0.00000000e+00,   0.00000000e+00],
       [  0.00000000e+00,   0.00000000e+00,   4.02133690e+00,
          0.00000000e+00,   0.00000000e+00]])
    settting = {'Griddensity':0.07, 'vdW scaling':1.4}
    input1 = np.array([[10,0,0,0],[8.0, 0.0, 0.0, 0],[1.0, 1.70075339, 0.0, 0],[1.0, -0.68030136, 1.62516435, 0.0]])
    calc = QFIT.makepoints(settting, input1)
    assert np.sum(np.abs(check-calc)) < 10**-8<commit_msg>added tests for basissets<commit_after>import BasisSet as BS
import DIIS as DIIS
import HartreeFock as HF
import MolecularIntegrals as MI
import MPn as MP
import Properties as PROP
import Qfit as QFIT
import Utilityfunc as UF
import numpy as np

def test_N():
    check = 5.701643762839922
    calc = BS.N(1.0,1.0,1.0,1.0)
    assert abs(check-calc) < 10**-12

def test_gaussian_product_center():
    check = 3.5
    calc = MI.gaussian_product_center(1.0,2.0,3.0,4.0)
    assert abs(check - calc) < 10**-12

def test_magvec():
    check = 5.196152422706632
    calc = QFIT.magvec([1,2,3],[4,5,6])
    assert abs(check - calc) < 10**-12

def test_centerofcharge():
    check1 = 0.1020452034
    check2 = 0.162516435
    check3 = 0.0
    input1 = np.array([[10,0,0,0],[8.0, 0.0, 0.0, 0],[1.0, 1.70075339, 0.0, 0],[1.0, -0.68030136, 1.62516435, 0.0]])
    calc1, calc2, calc3 = QFIT.centerofcharge(input1)
    assert abs(check1 - calc1) < 10**-8
    assert abs(check2 - calc2) < 10**-8
    assert abs(check3 - calc3) < 10**-8

def test_solveFit():
    check = np.array([-4.0, 4.5])
    A = np.array([[1.0,2.0],[3.0,4.0]])
    B = np.array([5.0,6.0])
    calc = QFIT.solveFit(A,B)
    assert abs(check[0] - calc[0]) < 10**-12
    assert abs(check[1] - calc[1]) < 10**-12

def test_makepoints():
    check = np.array([[  4.92471737e-16,   0.00000000e+00,  -4.02133690e+00,
          0.00000000e+00,   0.00000000e+00],
       [  0.00000000e+00,   0.00000000e+00,   4.02133690e+00,
          0.00000000e+00,   0.00000000e+00]])
    settting = {'Griddensity':0.07, 'vdW scaling':1.4}
    input1 = np.array([[10,0,0,0],[8.0, 0.0, 0.0, 0],[1.0, 1.70075339, 0.0, 0],[1.0, -0.68030136, 1.62516435, 0.0]])
    calc = QFIT.makepoints(settting, input1)
    assert np.sum(np.abs(check-calc)) < 10**-8

def test_basissets():
    Basis = ['STO3G','DZ','DZP']
    Atoms = {'STO3G':[1,6,7,8],'DZ':[1,8],'DZP':[1,8]}
    for i in Basis:
        Atom = Atoms[i]
        for j in Atom:
            A = BS.bassiset([[0,0,0,0],[j,0,0,0]], set={'basisset':i})
            sum1 = 0
            sum2 = 0
            sum3 = 0
            assert len(A[0][5]) == A[0][4]
            for k in range(0, len(A[0][5])):
                sum1 = sum1 + A[0][5][k][-1]
                sum2 = sum2 + A[0][5][k][-2]
                sum3 = sum3 + A[0][5][k][-3]
            assert sum1 == sum2
            assert sum1 == sum3

    <|endoftext|>


<commit_before>import os
import pytest

from click.testing import CliRunner
from mock import patch
from six.moves import input

from groceries.core import cli

del os.environ['GROCERIES_CONFIG']

mock_list = [
    dict(id=1, name='apples'),
    dict(id=2, name='bananas'),
    dict(id=3, name='citrus')]

class MockResponse(object):

    def __init__(self, status_code=200, json=None):
        self.status_code = status_code
        self.data = json if json else {}

    def json(self):
        return self.data

@pytest.fixture(scope='function')
def runner(request):
    return CliRunner()

@patch('requests.get', side_effect=[
    MockResponse(), MockResponse(json=dict(items=mock_list))])
@patch('requests.post', return_value=MockResponse(json=dict(token='secret')))
@patch('getpass.getpass', new=input)
def test_wizard(get, post, runner):
    with runner.isolated_filesystem():
        result = runner.invoke(
            cli, args=['list'], input='http://localhost\nuser\npass\n')
        assert result.exit_code == 0

@patch('requests.get', return_value=MockResponse(404))
def test_wizard_to_many_tries(get, runner):
    with runner.isolated_filesystem():
        result = runner.invoke(
            cli, args=['list'], input='http://localhost\n')
        args, kwargs = get.call_args
        print(args, kwargs)
        assert result.exit_code == 1

@patch('requests.get', side_effect=KeyboardInterrupt('Interrupt!'))
def test_wizard_keyboard_interrupt(get, runner):
    with runner.isolated_filesystem():
        result = runner.invoke(cli, args=['list'], input='http://localhost\n')
        assert result.exit_code == 1

@patch('groceries.config.Wizard.__call__')
@patch('requests.post', return_value=MockResponse())
def test_add(wizard, post, runner):
    result = runner.invoke(cli, args=['add', 'dates'])
    assert result.exit_code == 0
    assert result.output == ''

@patch('groceries.config.Wizard.__call__')
@patch('requests.post', return_value=MockResponse(500))
def test_add_failed(wizard, post, runner):
    result = runner.invoke(cli, args=['add', 'dates'])
    assert result.exit_code == 1
    assert 'dates' in result.output

@patch('groceries.config.Wizard.__call__')
@patch('requests.get', return_value=MockResponse(json=dict(items=mock_list)))
@patch('requests.put', return_value=MockResponse())
def test_buy(wizard, get, put, runner):
    result = runner.invoke(cli, args=['buy', 'citrus'])
    assert result.exit_code == 0
    assert result.output == ''

@patch('groceries.config.Wizard.__call__')
@patch('requests.get', return_value=MockResponse(json=dict(items=mock_list)))
def test_buy_failed(wizard, get, runner):
    result = runner.invoke(cli, args=['buy', 'cucumber'])
    assert result.exit_code == 1
    assert 'cucumber' in result.output

@patch('groceries.config.Wizard.__call__')
@patch('requests.get', return_value=MockResponse(json=dict(items=mock_list)))
def test_list(wizard, get, runner):
    result = runner.invoke(cli, args=['list'])
    assert result.exit_code == 0
    assert result.output == 'apples\nbananas\ncitrus\n'

@patch('groceries.config.Wizard.__call__')
@patch('requests.get', return_value=MockResponse(json=dict(items=mock_list)))
@patch('requests.delete', return_value=MockResponse())
def test_delete(wizard, get, delete, runner):
    result = runner.invoke(cli, args=['remove', 'citrus'])
    assert result.exit_code == 0
    assert result.output == ''

@patch('groceries.config.Wizard.__call__')
@patch('requests.get', return_value=MockResponse(json=dict(items=mock_list)))
def test_delete_failed(wizard, get, runner):
    result = runner.invoke(cli, args=['remove', 'cucumber'])
    assert result.exit_code == 1
    assert 'cucumber' in result.output
<commit_msg>check if env var exists before deleting it<commit_after>import os
import pytest

from click.testing import CliRunner
from mock import patch
from six.moves import input

from groceries.core import cli

if 'GROCERIES_CONFIG' in os.environ:
    del os.environ['GROCERIES_CONFIG']

mock_list = [
    dict(id=1, name='apples'),
    dict(id=2, name='bananas'),
    dict(id=3, name='citrus')]

class MockResponse(object):

    def __init__(self, status_code=200, json=None):
        self.status_code = status_code
        self.data = json if json else {}

    def json(self):
        return self.data

@pytest.fixture(scope='function')
def runner(request):
    return CliRunner()

@patch('requests.get', side_effect=[
    MockResponse(), MockResponse(json=dict(items=mock_list))])
@patch('requests.post', return_value=MockResponse(json=dict(token='secret')))
@patch('getpass.getpass', new=input)
def test_wizard(get, post, runner):
    with runner.isolated_filesystem():
        result = runner.invoke(
            cli, args=['list'], input='http://localhost\nuser\npass\n')
        assert result.exit_code == 0

@patch('requests.get', return_value=MockResponse(404))
def test_wizard_to_many_tries(get, runner):
    with runner.isolated_filesystem():
        result = runner.invoke(
            cli, args=['list'], input='http://localhost\n')
        args, kwargs = get.call_args
        print(args, kwargs)
        assert result.exit_code == 1

@patch('requests.get', side_effect=KeyboardInterrupt('Interrupt!'))
def test_wizard_keyboard_interrupt(get, runner):
    with runner.isolated_filesystem():
        result = runner.invoke(cli, args=['list'], input='http://localhost\n')
        assert result.exit_code == 1

@patch('groceries.config.Wizard.__call__')
@patch('requests.post', return_value=MockResponse())
def test_add(wizard, post, runner):
    result = runner.invoke(cli, args=['add', 'dates'])
    assert result.exit_code == 0
    assert result.output == ''

@patch('groceries.config.Wizard.__call__')
@patch('requests.post', return_value=MockResponse(500))
def test_add_failed(wizard, post, runner):
    result = runner.invoke(cli, args=['add', 'dates'])
    assert result.exit_code == 1
    assert 'dates' in result.output

@patch('groceries.config.Wizard.__call__')
@patch('requests.get', return_value=MockResponse(json=dict(items=mock_list)))
@patch('requests.put', return_value=MockResponse())
def test_buy(wizard, get, put, runner):
    result = runner.invoke(cli, args=['buy', 'citrus'])
    assert result.exit_code == 0
    assert result.output == ''

@patch('groceries.config.Wizard.__call__')
@patch('requests.get', return_value=MockResponse(json=dict(items=mock_list)))
def test_buy_failed(wizard, get, runner):
    result = runner.invoke(cli, args=['buy', 'cucumber'])
    assert result.exit_code == 1
    assert 'cucumber' in result.output

@patch('groceries.config.Wizard.__call__')
@patch('requests.get', return_value=MockResponse(json=dict(items=mock_list)))
def test_list(wizard, get, runner):
    result = runner.invoke(cli, args=['list'])
    assert result.exit_code == 0
    assert result.output == 'apples\nbananas\ncitrus\n'

@patch('groceries.config.Wizard.__call__')
@patch('requests.get', return_value=MockResponse(json=dict(items=mock_list)))
@patch('requests.delete', return_value=MockResponse())
def test_delete(wizard, get, delete, runner):
    result = runner.invoke(cli, args=['remove', 'citrus'])
    assert result.exit_code == 0
    assert result.output == ''

@patch('groceries.config.Wizard.__call__')
@patch('requests.get', return_value=MockResponse(json=dict(items=mock_list)))
def test_delete_failed(wizard, get, runner):
    result = runner.invoke(cli, args=['remove', 'cucumber'])
    assert result.exit_code == 1
    assert 'cucumber' in result.output
<|endoftext|>