<commit_before>'''
An extensible module for evaluation of arbitrary mathematical expressions
'''

import operator
import re

operations = [
    {
        '+': operator.add,
        '-': operator.sub,
    },
    {
        '*': operator.mul,
        '/': operator.truediv,
        '//': operator.floordiv,
        '%': operator.mod,
    },
    {
        '^': operator.pow,
    },
]

unary = {
    '+': operator.pos,
    '-': operator.neg,
    '~': operator.neg,
}

class EquationError (Exception):
    pass

def types(stack):
    '''
    Gets the types list from a token list
    '''
    return next(zip(*stack)) if stack else []

def tokenize(expression, operations=operations, unary=unary):
    '''
    Parses a mathematic expression into tokens
    '''
    def token(t):
        def callback(scanner, match):
            return t, match
        return callback
    operators = {op for ops in operations for op in ops}
    unary_operators = [op for op in unary if op not in operators]
    tokens = [(r'\s+', 'WHITESPACE')]
    tokens.extend((r'|'.join(map(re.escape, ops)), i) for i, ops in enumerate(operations))
    tokens.append((r'|'.join(map(re.escape, unary_operators)), 'UNARY'))
    tokens.extend([
        (r'-', len(operations)),  # ???
        (r'\d*\.\d+', 'FLOAT'),
        (r'\d+', 'INT'),
        (r'\(', 'PAREN_OPEN'),
        (r'\)', 'PAREN_CLOSE'),
    ])
    scanner = re.Scanner([(p, token(t)) for p, t in tokens])
    out, rest = scanner.scan(expression)
    if rest:
        raise ValueError('Could not parse equation from: {}'.format(rest))
    return [(t, m) for t, m in out if t != 'WHITESPACE']

def infix2postfix(expression, operations=operations, unary=unary):
    '''
    Converts an infix expression to a postfix token list
    '''
    equation = tokenize(expression, operations=operations, unary=unary)
    stack = []
    output = []

    prev_type = -1
    for item in equation:
        type, token = item
        if type in ('INT', 'FLOAT'):
            output.append(item)
        elif type == 'PAREN_OPEN':
            stack.append(item)
        elif type == 'PAREN_CLOSE':
            if 'PAREN_OPEN' not in types(stack):
                raise EquationError('Missing open parenthesis: {}'.format(expression))
            while stack[-1][0] != 'PAREN_OPEN':
                output.append(stack.pop())
            stack.pop()
        elif type == 'UNARY':
            stack.append(item)
        elif isinstance(type, int):  # operators
            if prev_type == 'PAREN_OPEN' or isinstance(prev_type, int):
                # unary operation
                stack.append(('UNARY', token))
            else:
                while stack and isinstance(stack[-1][0], int) and stack[-1][0] >= type:
                    output.append(stack.pop())
                stack.append(item)
        else:
            raise EquationError('Invalid token found: {} in {}'.format(token, equation))
        prev_type = type

    if 'PAREN_OPEN' in types(stack):
        raise EquationError('Missing closing parenthesis: {}'.format(expression))

    while stack:
        output.append(stack.pop())

    return output

def solve(expression, operations=operations, unary=unary):
    '''
    Solves an infix expression

    Operations is a list of operation dicts in reverse precedence order
        The keys of each dict should be the operator and the values should be
        a binary function to apply to the operands
        The functions should be able to take int or float arguments

    Unary is a dict of unary operations
        The keys of the dict should be the operator and the values should be
        a unary function to apply to the operands
        The functions should be able to take int or float arguments
    '''
    equation = infix2postfix(expression, operations=operations, unary=unary)
    stack = []

    for type, token in equation:
        # print(equation, stack, token, sep='\n')
        if type == 'INT':
            stack.append(int(token))
        elif type == 'FLOAT':
            stack.append(float(token))
        elif type == 'UNARY':
            if len(stack) < 1:
                raise EquationError('Not enough operands for unary {} in {}'.format(token, expression))
            else:
                stack.append(unary[token](stack.pop()))
        elif isinstance(type, int):
            if len(stack) < 2:
                raise EquationError('Not enough operands for {} in {}'.format(token, expression))
            else:
                b, a = stack.pop(), stack.pop()
                stack.append(operations[type][token](a, b))
        else:
            raise EquationError('Invalid token: {} in {}'.format(token, expression))

    if len(stack) != 1:
        raise EquationError('Too many operands for operators in {}'.format(expression))

    return stack[0]

if __name__ == '__main__':
    print(solve(input('Eq: ')))
<commit_msg>Changed UNARY to push directly onto output<commit_after>'''
An extensible module for evaluation of arbitrary mathematical expressions
'''

import operator
import re

operations = [
    {
        '+': operator.add,
        '-': operator.sub,
    },
    {
        '*': operator.mul,
        '/': operator.truediv,
        '//': operator.floordiv,
        '%': operator.mod,
    },
    {
        '^': operator.pow,
    },
]

unary = {
    '+': operator.pos,
    '-': operator.neg,
    '~': operator.neg,
}

class EquationError (Exception):
    pass

def types(stack):
    '''
    Gets the types list from a token list
    '''
    return next(zip(*stack)) if stack else []

def tokenize(expression, operations=operations, unary=unary):
    '''
    Parses a mathematic expression into tokens
    '''
    def token(t):
        def callback(scanner, match):
            return t, match
        return callback
    operators = {op for ops in operations for op in ops}
    unary_operators = [op for op in unary if op not in operators]
    tokens = [(r'\s+', 'WHITESPACE')]
    tokens.extend((r'|'.join(map(re.escape, ops)), i) for i, ops in enumerate(operations))
    tokens.append((r'|'.join(map(re.escape, unary_operators)), 'UNARY'))
    tokens.extend([
        (r'-', len(operations)),  # ???
        (r'\d*\.\d+', 'FLOAT'),
        (r'\d+', 'INT'),
        (r'\(', 'PAREN_OPEN'),
        (r'\)', 'PAREN_CLOSE'),
    ])
    scanner = re.Scanner([(p, token(t)) for p, t in tokens])
    out, rest = scanner.scan(expression)
    if rest:
        raise ValueError('Could not parse equation from: {}'.format(rest))
    return [(t, m) for t, m in out if t != 'WHITESPACE']

def infix2postfix(expression, operations=operations, unary=unary):
    '''
    Converts an infix expression to a postfix token list
    '''
    equation = tokenize(expression, operations=operations, unary=unary)
    stack = []
    output = []

    prev_type = -1
    for item in equation:
        type, token = item
        if type in ('INT', 'FLOAT'):
            output.append(item)
        elif type == 'PAREN_OPEN':
            stack.append(item)
        elif type == 'PAREN_CLOSE':
            if 'PAREN_OPEN' not in types(stack):
                raise EquationError('Missing open parenthesis: {}'.format(expression))
            while stack[-1][0] != 'PAREN_OPEN':
                output.append(stack.pop())
            stack.pop()
        elif type == 'UNARY':
            output.append(item)
        elif isinstance(type, int):  # operators
            if prev_type == 'PAREN_OPEN' or isinstance(prev_type, int):
                # unary operation
                output.append(('UNARY', token))
            else:
                while stack and isinstance(stack[-1][0], int) and stack[-1][0] >= type:
                    output.append(stack.pop())
                stack.append(item)
        else:
            raise EquationError('Invalid token found: {} in {}'.format(token, equation))
        prev_type = type

    if 'PAREN_OPEN' in types(stack):
        raise EquationError('Missing closing parenthesis: {}'.format(expression))

    while stack:
        output.append(stack.pop())

    return output

def solve(expression, operations=operations, unary=unary):
    '''
    Solves an infix expression

    Operations is a list of operation dicts in reverse precedence order (highest precedence last)
        The keys of each dict should be the operator and the values should be binary functions to apply to the operands
        The functions should be able to take int or float arguments

    Unary is a dict of unary operations
        The keys of the dict should be the operator and the values should be
        a unary function to apply to the operands
        The functions should be able to take int or float arguments
    '''
    equation = infix2postfix(expression, operations=operations, unary=unary)
    stack = []

    for type, token in equation:
        # print(equation, stack, token, sep='\n')
        if type == 'INT':
            stack.append(int(token))
        elif type == 'FLOAT':
            stack.append(float(token))
        elif type == 'UNARY':
            if len(stack) < 1:
                raise EquationError('Not enough operands for unary {} in {}'.format(token, expression))
            else:
                stack.append(unary[token](stack.pop()))
        elif isinstance(type, int):
            if len(stack) < 2:
                raise EquationError('Not enough operands for {} in {}'.format(token, expression))
            else:
                b, a = stack.pop(), stack.pop()
                stack.append(operations[type][token](a, b))
        else:
            raise EquationError('Invalid token: {} in {}'.format(token, expression))

    if len(stack) != 1:
        raise EquationError('Too many operands for operators in {}'.format(expression))

    return stack[0]

if __name__ == '__main__':
    print(solve(input('Eq: ')))
<|endoftext|>


<commit_before>#!/usr/bin/env python2.7
# -*- coding: utf-8 -*-
# Copyright (c) 2014 Asumi Kamikaze Inc.
# Licensed under the MIT License.
# Author: Alejandro M. Bernardis
# Email: alejandro (dot) bernardis (at) asumikamikaze (dot) com
# Created: 08/Oct/2014 10:22 AM

import os
import re
from fabric.api import *

rx_pid = re.compile(r'\d+')
BACKEND_PORT = 8000
BACKEND_API_PORT = 9000
ROOT_PATH = os.path.abspath(os.path.dirname(__file__))
LIBRARY_PATH = os.path.join(ROOT_PATH, '/Volumes/Projects/library/python/2.7')
DATA_LIB_PATH = os.path.join(ROOT_PATH, 'lib')
DATA_SRC_PATH = os.path.join(ROOT_PATH, 'src/backend')
PYTHON_PATH = '$PYTHONPATH:%s:%s:%s' \
              % (LIBRARY_PATH, DATA_LIB_PATH, DATA_SRC_PATH)

################################################################################
# helpers ----------------------------------------------------------------------

def _start_server(background=False, port=BACKEND_PORT):
    with lcd(DATA_SRC_PATH):
        with shell_env(PYTHONPATH=PYTHON_PATH):
            command = 'python2.7 -m server --port=%s' % port
            if background:
                command = "nohup {command} > /tmp/backend.{port}.log 2>&1 " \
                          "& echo $! > /tmp/backend.{port}.pid" \
                          .format(command=command, port=port)
            local(command)

def _stop_server(port=BACKEND_PORT):
    pid = rx_pid.findall(open('/tmp/backend.%s.pid' % port).read())[0]
    local('kill %s %s' % (pid, int(pid)+1))

def _tail_server(port=BACKEND_PORT):
    local('tail -f /tmp/backend.{port}.log'.format(port=port))

def _start_db():
    config_file = os.path.join(ROOT_PATH, 'etc/mongodb.conf')
    path_dir = '/tmp/mongodb'
    if not os.path.isdir(path_dir):
        local('mkdir -p {path_dir}'.format(path_dir=path_dir))
    local('mongod --config {config_file}'.format(config_file=config_file))

def _stop_db():
    pid = rx_pid.findall(open('/tmp/mongodb.pid').read())[0]
    local('kill {pid}'.format(pid=pid))

def _tail_db():
    local('tail -f /tmp/mongodb.log')

def _start_cache(server='memcached'):
    if server.lower() == 'redis':
        config_file = os.path.join(ROOT_PATH, 'etc/redis.conf')
        path_dir = '/tmp/redis'
        if not os.path.isdir(path_dir):
            local('mkdir -p {path_dir}'.format(path_dir=path_dir))
        local('redis-server {config_file}'.format(config_file=config_file))
    else:
        config_file = os.path.join(ROOT_PATH, 'etc/memcached.conf')
        local('memcached -d {config_file}'.format(config_file=config_file))

def _stop_cache(server='memcached'):
    if server.lower() == 'redis':
        pid = rx_pid.findall(open('/tmp/redis.pid').read())[0]
        local('kill {pid}'.format(pid=pid))
    else:
        local('killall memcached')

def _start_celery(background=False, worker='backend.background.server',
                  level='info'):
    with lcd(DATA_SRC_PATH):
        with shell_env(PYTHONPATH=PYTHON_PATH):
            command = 'celery -A {worker} worker --loglevel={level}' \
                      .format(worker=worker, level=level)
            if background:
                command = "nohup {command} > /tmp/celery.{worker}.log 2>&1 " \
                          "& echo $! > /tmp/celery.{worker}.pid" \
                          .format(command=command, worker=worker)
            local(command)

def _stop_celery(worker='backend.background.server'):
    pid = rx_pid.findall(open('/tmp/celery.{0}.pid'.format(worker)).read())[0]
    local('kill {pid}'.format(pid=pid))

def _tail_celery(worker='backend.background.server'):
    local('tail -f /tmp/celery.{worker}.log'.format(worker=worker))

def _start_broker(background=False, options=''):
    if background:
        options += ' -detached'
    local('rabbitmq-server{options}'.format(options=options))

def _stop_broker():
    pass

################################################################################
# servers ----------------------------------------------------------------------

@task
def start_backend(background=False):
    _start_server(background)

@task
def stop_backend():
    _stop_server()

@task
def tail_backend():
    _tail_server()

@task
def start_backend_api(background=False):
    _start_server(background, BACKEND_API_PORT)

@task
def stop_backend_api():
    _stop_server(BACKEND_API_PORT)

@task
def tail_backend_api():
    _tail_server(BACKEND_API_PORT)

# mongodb ----------------------------------------------------------------------

@task
def start_db():
    _start_db()

@task
def stop_db():
    _stop_db()

@task
def tail_db():
    _tail_db()

# redis/memcached --------------------------------------------------------------

@task
def start_cache():
    _start_cache('redis')

@task
def stop_cache():
    _stop_cache('redis')

# celery -----------------------------------------------------------------------

@task
def start_tasks(background=False):
    _start_broker()
    _start_celery(background)

@task
def stop_tasks():
    _stop_celery()
    _stop_broker()

@task
def tail_tasks():
    _tail_celery()

################################################################################<commit_msg>FIX // Remove memcached support (fabfile)<commit_after>#!/usr/bin/env python2.7
# -*- coding: utf-8 -*-
# Copyright (c) 2014 Asumi Kamikaze Inc.
# Licensed under the MIT License.
# Author: Alejandro M. Bernardis
# Email: alejandro (dot) bernardis (at) asumikamikaze (dot) com
# Created: 08/Oct/2014 10:22 AM

import os
import re
from fabric.api import *

rx_pid = re.compile(r'\d+')
BACKEND_PORT = 8000
BACKEND_API_PORT = 9000
ROOT_PATH = os.path.abspath(os.path.dirname(__file__))
LIBRARY_PATH = os.path.join(ROOT_PATH, '/Volumes/Projects/library/python/2.7')
DATA_LIB_PATH = os.path.join(ROOT_PATH, 'lib')
DATA_SRC_PATH = os.path.join(ROOT_PATH, 'src/backend')
PYTHON_PATH = '$PYTHONPATH:%s:%s:%s' \
              % (LIBRARY_PATH, DATA_LIB_PATH, DATA_SRC_PATH)

################################################################################
# helpers ----------------------------------------------------------------------

def _start_server(background=False, port=BACKEND_PORT):
    with lcd(DATA_SRC_PATH):
        with shell_env(PYTHONPATH=PYTHON_PATH):
            command = 'python2.7 -m server --port=%s' % port
            if background:
                command = "nohup {command} > /tmp/backend.{port}.log 2>&1 " \
                          "& echo $! > /tmp/backend.{port}.pid" \
                          .format(command=command, port=port)
            local(command)

def _stop_server(port=BACKEND_PORT):
    pid = rx_pid.findall(open('/tmp/backend.%s.pid' % port).read())[0]
    local('kill %s %s' % (pid, int(pid)+1))

def _tail_server(port=BACKEND_PORT):
    local('tail -f /tmp/backend.{port}.log'.format(port=port))

def _start_db():
    config_file = os.path.join(ROOT_PATH, 'etc/mongodb.conf')
    path_dir = '/tmp/mongodb'
    if not os.path.isdir(path_dir):
        local('mkdir -p {path_dir}'.format(path_dir=path_dir))
    local('mongod --config {config_file}'.format(config_file=config_file))

def _stop_db():
    pid = rx_pid.findall(open('/tmp/mongodb.pid').read())[0]
    local('kill {pid}'.format(pid=pid))

def _tail_db():
    local('tail -f /tmp/mongodb.log')

def _start_cache(server='memcached'):
    config_file = os.path.join(ROOT_PATH, 'etc/redis.conf')
    path_dir = '/tmp/redis'
    if not os.path.isdir(path_dir):
        local('mkdir -p {path_dir}'.format(path_dir=path_dir))
    local('redis-server {config_file}'.format(config_file=config_file))

def _stop_cache(server='memcached'):
    pid = rx_pid.findall(open('/tmp/redis.pid').read())[0]
    local('kill {pid}'.format(pid=pid))

def _start_celery(background=False, worker='backend.background.server',
                  level='info'):
    with lcd(DATA_SRC_PATH):
        with shell_env(PYTHONPATH=PYTHON_PATH):
            command = 'celery -A {worker} worker --loglevel={level}' \
                      .format(worker=worker, level=level)
            if background:
                command = "nohup {command} > /tmp/celery.{worker}.log 2>&1 " \
                          "& echo $! > /tmp/celery.{worker}.pid" \
                          .format(command=command, worker=worker)
            local(command)

def _stop_celery(worker='backend.background.server'):
    pid = rx_pid.findall(open('/tmp/celery.{0}.pid'.format(worker)).read())[0]
    local('kill {pid}'.format(pid=pid))

def _tail_celery(worker='backend.background.server'):
    local('tail -f /tmp/celery.{worker}.log'.format(worker=worker))

def _start_broker(background=False, options=''):
    if background:
        options += ' -detached'
    local('rabbitmq-server{options}'.format(options=options))

def _stop_broker():
    pass

################################################################################
# servers ----------------------------------------------------------------------

@task
def start_backend(background=False):
    _start_server(background)

@task
def stop_backend():
    _stop_server()

@task
def tail_backend():
    _tail_server()

@task
def start_backend_api(background=False):
    _start_server(background, BACKEND_API_PORT)

@task
def stop_backend_api():
    _stop_server(BACKEND_API_PORT)

@task
def tail_backend_api():
    _tail_server(BACKEND_API_PORT)

# mongodb ----------------------------------------------------------------------

@task
def start_db():
    _start_db()

@task
def stop_db():
    _stop_db()

@task
def tail_db():
    _tail_db()

# redis/memcached --------------------------------------------------------------

@task
def start_cache():
    _start_cache()

@task
def stop_cache():
    _stop_cache()

# celery -----------------------------------------------------------------------

@task
def start_tasks(background=False):
    _start_broker()
    _start_celery(background)

@task
def stop_tasks():
    _stop_celery()
    _stop_broker()

@task
def tail_tasks():
    _tail_celery()

################################################################################<|endoftext|>


<commit_before>from fabric.api import cd, sudo, env, puts

env.path = '/var/praekelt/vumi-go'

def deploy_go():
    with cd(env.path):
        sudo('git pull', user='vumi')
        sudo("find go -name '*.pyc' -delete")
        setup_js_env()
        _venv_command('./ve/bin/django-admin.py collectstatic --pythonpath=. '
                      '--settings=go.settings --noinput')

def setup_js_env():
    with cd(env.path):
        _venv_command('./utils/js-setup-env.sh')

def deploy_vumi():
    with cd('%s/ve/src/vumi/' % (env.path,)):
        sudo('git pull', user='vumi')
        sudo("find vumi -name '*.pyc' -delete")

def restart_celery():
    with cd(env.path):
        supervisorctl('restart vumi_celery:celery')

def restart_gunicorn():
    """
    Intentionally restart the gunicorns 1 by 1 so HAProxy is given
    time to load balance across gunicorns that have either already restarted
    or are waiting to be restarted
    """
    with cd(env.path):
        for i in range(1, 5):
            supervisorctl('restart vumi_web:goui_%s' % (i,))

def restart_all(group=None):
    """
    Restart all running processes one-by-one so that the system as a whole
    experiences no down-time during the restart. If a group is specified,
    restart only the processes within that group one by one.
    """
    processes = _supervisorctl_status()
    if group is not None:
        processes = [p for p in processes if p['group'] == group]
    processes = [p for p in processes if p['statename'] == 'RUNNING']
    for p in processes:
        p_name = '%s:%s' % (p['group'], p['name'])
        puts("Restarting %s ..." % (p_name,))
        supervisorctl("restart %s" % p_name)

def deploy_world(restart=True):
    """
    Update Vumi, Vumi Go and the node.js modules and then run restart_all.
    """
    deploy_vumi()
    deploy_go()
    if restart:
        restart_all()

def supervisorctl(command):
    return sudo('supervisorctl %s' % (command,))

def _venv_command(command, user='vumi'):
    return sudo('. ve/bin/activate && %s' % (command,), user=user)

def _supervisorctl_status():
    status = supervisorctl('status').splitlines()
    processes = []
    for line in status:
        parts = line.split()
        group, _, name = parts[0].partition(':')
        if not name:
            # single processes are in a group with the same
            # name as themselves
            name = group
        processes.append({
            'name': name,
            'group': group,
            'statename': parts[1],
        })
    return processes
<commit_msg>Fix reference to js-env-setup.sh (@hodgestar, @justinvdm).<commit_after>from fabric.api import cd, sudo, env, puts

env.path = '/var/praekelt/vumi-go'

def deploy_go():
    with cd(env.path):
        sudo('git pull', user='vumi')
        sudo("find go -name '*.pyc' -delete")
        setup_js_env()
        _venv_command('./ve/bin/django-admin.py collectstatic --pythonpath=. '
                      '--settings=go.settings --noinput')

def setup_js_env():
    with cd(env.path):
        _venv_command('./utils/js-env-setup.sh')

def deploy_vumi():
    with cd('%s/ve/src/vumi/' % (env.path,)):
        sudo('git pull', user='vumi')
        sudo("find vumi -name '*.pyc' -delete")

def restart_celery():
    with cd(env.path):
        supervisorctl('restart vumi_celery:celery')

def restart_gunicorn():
    """
    Intentionally restart the gunicorns 1 by 1 so HAProxy is given
    time to load balance across gunicorns that have either already restarted
    or are waiting to be restarted
    """
    with cd(env.path):
        for i in range(1, 5):
            supervisorctl('restart vumi_web:goui_%s' % (i,))

def restart_all(group=None):
    """
    Restart all running processes one-by-one so that the system as a whole
    experiences no down-time during the restart. If a group is specified,
    restart only the processes within that group one by one.
    """
    processes = _supervisorctl_status()
    if group is not None:
        processes = [p for p in processes if p['group'] == group]
    processes = [p for p in processes if p['statename'] == 'RUNNING']
    for p in processes:
        p_name = '%s:%s' % (p['group'], p['name'])
        puts("Restarting %s ..." % (p_name,))
        supervisorctl("restart %s" % p_name)

def deploy_world(restart=True):
    """
    Update Vumi, Vumi Go and the node.js modules and then run restart_all.
    """
    deploy_vumi()
    deploy_go()
    if restart:
        restart_all()

def supervisorctl(command):
    return sudo('supervisorctl %s' % (command,))

def _venv_command(command, user='vumi'):
    return sudo('. ve/bin/activate && %s' % (command,), user=user)

def _supervisorctl_status():
    status = supervisorctl('status').splitlines()
    processes = []
    for line in status:
        parts = line.split()
        group, _, name = parts[0].partition(':')
        if not name:
            # single processes are in a group with the same
            # name as themselves
            name = group
        processes.append({
            'name': name,
            'group': group,
            'statename': parts[1],
        })
    return processes
<|endoftext|>


<commit_before># Copyright 2019 D-Wave Systems Inc.
#
#    Licensed under the Apache License, Version 2.0 (the "License");
#    you may not use this file except in compliance with the License.
#    You may obtain a copy of the License at
#
#        http://www.apache.org/licenses/LICENSE-2.0
#
#    Unless required by applicable law or agreed to in writing, software
#    distributed under the License is distributed on an "AS IS" BASIS,
#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#    See the License for the specific language governing permissions and
#    limitations under the License.
#
# ============================================================================
"""
It is possible to construct samplers that handle binary polynomials - problems
that have binary variables but they are not constrainted to quadratic
interactions.

"""
import abc
import warnings

from six import add_metaclass

from dimod.core.composite import Composite
from dimod.higherorder.polynomial import BinaryPolynomial

__all__ = 'PolySampler', 'ComposedPolySampler'

@add_metaclass(abc.ABCMeta)
class PolySampler:
    """Sampler supports binary polynomials.

    Binary polynomials are an extension of binary quadratic models that allow
    higher-order interactions.

    """
    @abc.abstractproperty  # for python2 compatibility
    def parameters(self):
        """dict: A dict where keys are the keyword parameters accepted by the sampler
        methods and values are lists of the properties relevent to each parameter.
        """
        pass

    @abc.abstractproperty  # for python2 compatibility
    def properties(self):
        """dict: A dict containing any additional information about the sampler.
        """
        pass

    @abc.abstractmethod
    def sample_poly(self, polynomial, **kwargs):
        """Sample from a higher-order polynomial."""
        pass

    def sample_hising(self, h, J, **kwargs):
        """Sample from a higher-order Ising model.

        Convert the given higher-order Ising model to a :obj:`.BinaryPolynomial`
        and invoke `sample_poly`.

        Args:
            h (dict):
                The linear biases of the Ising problem.

            J (dict):
                The quadratic biases of the Ising problem.

            **kwargs:
                See `sample_poly` for additional keyword definitions.

        Returns:
            :obj:`.SampleSet`

        """
        return self.sample_poly(BinaryPolynomial.from_hising(h, J), **kwargs)

    def sample_hubo(self, H, **kwargs):
        """Sample from a higher-order unconstrainted binary optimization problem.

        Convert the given higher-order unconstrainted binary optimization
        problem to a :obj:`.BinaryPolynomial` and invoke `sample_poly`.

        Args:
            H (dict):
                The coefficients of the HUBO.

            **kwargs:
                See `sample_poly` for additional keyword definitions.

        Returns:
            :obj:`.SampleSet`

        """
        return self.sample_poly(BinaryPolynomial.from_hubo(H), **kwargs)

    def sample(self, bqm, *args, **kwargs):
        msg = ("PolySampler.sample is deprecated and will be removed in dimod "
               "0.9.0. In the future, when using PolySamplers, you should use "
               ".sample_poly")
        warnings.warn(msg, DeprecationWarning)
        return self.sample_hising(bqm.linear, bqm.quadratic, *args, **kwargs)

    def sample_ising(self, *args, **kwargs):
        msg = ("PolySampler.sample_ising is deprecated and will be removed in dimod "
               "0.9.0. In the future, when using PolySamplers, you should use "
               ".sample_hising")
        warnings.warn(msg, DeprecationWarning)
        return self.sample_hising(*args, **kwargs)

    def sample_qubo(self, *args, **kwargs):
        msg = ("PolySampler.sample_qubo is deprecated and will be removed in dimod "
               "0.9.0. In the future, when using PolySamplers, you should use "
               ".sample_hubo")
        warnings.warn(msg, DeprecationWarning)
        return self.sample_hubo(*args, **kwargs)

class ComposedPolySampler(PolySampler, Composite):
    """Abstract base class for dimod composed polynomial samplers.

    Inherits from :class:`.PolySampler` and :class:`.Composite`."""
    pass
<commit_msg>Fix typos and bug in polysampler.py<commit_after># Copyright 2019 D-Wave Systems Inc.
#
#    Licensed under the Apache License, Version 2.0 (the "License");
#    you may not use this file except in compliance with the License.
#    You may obtain a copy of the License at
#
#        http://www.apache.org/licenses/LICENSE-2.0
#
#    Unless required by applicable law or agreed to in writing, software
#    distributed under the License is distributed on an "AS IS" BASIS,
#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#    See the License for the specific language governing permissions and
#    limitations under the License.
#
# ============================================================================
"""
It is possible to construct samplers that handle binary polynomials - problems
that have binary variables but they are not constrained to quadratic
interactions.

"""
import abc
import warnings

from six import add_metaclass

from dimod.core.composite import Composite
from dimod.higherorder.polynomial import BinaryPolynomial

__all__ = 'PolySampler', 'ComposedPolySampler'

@add_metaclass(abc.ABCMeta)
class PolySampler:
    """Sampler supports binary polynomials.

    Binary polynomials are an extension of binary quadratic models that allow
    higher-order interactions.

    """
    @abc.abstractproperty  # for python2 compatibility
    def parameters(self):
        """dict: A dict where keys are the keyword parameters accepted by the sampler
        methods and values are lists of the properties relevant to each parameter.
        """
        pass

    @abc.abstractproperty  # for python2 compatibility
    def properties(self):
        """dict: A dict containing any additional information about the sampler.
        """
        pass

    @abc.abstractmethod
    def sample_poly(self, polynomial, **kwargs):
        """Sample from a higher-order polynomial."""
        pass

    def sample_hising(self, h, J, **kwargs):
        """Sample from a higher-order Ising model.

        Convert the given higher-order Ising model to a :obj:`.BinaryPolynomial`
        and invoke `sample_poly`.

        Args:
            h (dict):
                The linear biases of the Ising problem.

            J (dict):
                The quadratic biases of the Ising problem.

            **kwargs:
                See `sample_poly` for additional keyword definitions.

        Returns:
            :obj:`.SampleSet`

        """
        return self.sample_poly(BinaryPolynomial.from_hising(h, J), **kwargs)

    def sample_hubo(self, H, **kwargs):
        """Sample from a higher-order unconstrained binary optimization problem.

        Convert the given higher-order unconstrained binary optimization
        problem to a :obj:`.BinaryPolynomial` and invoke `sample_poly`.

        Args:
            H (dict):
                The coefficients of the HUBO.

            **kwargs:
                See `sample_poly` for additional keyword definitions.

        Returns:
            :obj:`.SampleSet`

        """
        return self.sample_poly(BinaryPolynomial.from_hubo(H), **kwargs)

    def sample(self, bqm, *args, **kwargs):
        msg = ("PolySampler.sample is deprecated and will be removed in dimod "
               "0.9.0. In the future, when using PolySamplers, you should use "
               ".sample_poly")
        warnings.warn(msg, DeprecationWarning)
        poly = BinaryPolynomial(bqm.quadratic)
        poly.update(((v,), bias) for v, bias in bqm.linear.items())
        return self.sample_poly(poly, *args, **kwargs)

    def sample_ising(self, *args, **kwargs):
        msg = ("PolySampler.sample_ising is deprecated and will be removed in dimod "
               "0.9.0. In the future, when using PolySamplers, you should use "
               ".sample_hising")
        warnings.warn(msg, DeprecationWarning)
        return self.sample_hising(*args, **kwargs)

    def sample_qubo(self, *args, **kwargs):
        msg = ("PolySampler.sample_qubo is deprecated and will be removed in dimod "
               "0.9.0. In the future, when using PolySamplers, you should use "
               ".sample_hubo")
        warnings.warn(msg, DeprecationWarning)
        return self.sample_hubo(*args, **kwargs)

class ComposedPolySampler(PolySampler, Composite):
    """Abstract base class for dimod composed polynomial samplers.

    Inherits from :class:`.PolySampler` and :class:`.Composite`."""
    pass
<|endoftext|>


<commit_before>import time
import traceback

import eventlet
import logging

import sys
from activitystreams import parse as parse_to_as

from eventlet.semaphore import Semaphore
from datetime import datetime as dt
from datetime import timedelta

from dino.config import ConfigKeys
from dino.endpoint.base import locked_method
from dino.environ import GNEnvironment
from dino.heartbeat import IHeartbeatManager

logger = logging.getLogger(__name__)

class HeartbeatManager(IHeartbeatManager):
    def __init__(self, env: GNEnvironment):
        self._lock = Semaphore(value=1)
        self.env = env
        self.to_check = dict()
        eventlet.spawn_after(func=self.loop, seconds=10)

    def loop(self):
        while True:
            try:
                expired = self.get_all_expired_user_ids()
                self.check_heartbeats(expired)
                time.sleep(20)
            except InterruptedError:
                logger.info('interrupted, exiting loop')
                break
            except Exception as e:
                logger.error('could not check heartbeat: {}'.format(str(e)))
                logger.exception(traceback.format_exc())
                self.env.capture_exception(sys.exc_info())
                time.sleep(1)

    def check_heartbeats(self, user_ids: list) -> None:
        for user_id in user_ids:
            still_online = self.env.cache.check_heartbeat(user_id)

            if still_online:
                self.add_heartbeat(user_id)
                continue

            hb_sid = 'hb-{}'.format(user_id)
            data = {
                'verb': 'disconnect',
                'actor': {
                    'id': user_id,
                    'content': hb_sid
                }
            }

            if not self.env.config.get(ConfigKeys.TESTING):
                # only used for single-session restrictions
                if self.env.connected_user_ids.get(user_id) == hb_sid:
                    del self.env.connected_user_ids[user_id]

            activity = parse_to_as(data)
            self.env.observer.emit('on_heartbeat_disconnect', (data, activity))

    @locked_method
    def has_heartbeat(self, user_id: str) -> bool:
        return user_id in self.to_check.keys()

    @locked_method
    def add_heartbeat(self, user_id: str) -> None:
        self.to_check[user_id] = dt.utcnow()

    @locked_method
    def get_all_expired_user_ids(self):
        expired = list()
        not_yet_expired = dict()
        now_time = dt.utcnow() + timedelta(seconds=100)

        for add_time, user_id in self.to_check.items():
            if add_time > now_time:
                expired.append(user_id)
            else:
                not_yet_expired[user_id] = add_time

        self.to_check = not_yet_expired.copy()
        return expired.copy()
<commit_msg>userid is the key, not the time<commit_after>import time
import traceback

import eventlet
import logging

import sys
from activitystreams import parse as parse_to_as

from eventlet.semaphore import Semaphore
from datetime import datetime as dt
from datetime import timedelta

from dino.config import ConfigKeys
from dino.endpoint.base import locked_method
from dino.environ import GNEnvironment
from dino.heartbeat import IHeartbeatManager

logger = logging.getLogger(__name__)

class HeartbeatManager(IHeartbeatManager):
    def __init__(self, env: GNEnvironment):
        self._lock = Semaphore(value=1)
        self.env = env
        self.to_check = dict()
        eventlet.spawn_after(func=self.loop, seconds=10)

    def loop(self):
        while True:
            try:
                expired = self.get_all_expired_user_ids()
                self.check_heartbeats(expired)
                time.sleep(20)
            except InterruptedError:
                logger.info('interrupted, exiting loop')
                break
            except Exception as e:
                logger.error('could not check heartbeat: {}'.format(str(e)))
                logger.exception(traceback.format_exc())
                self.env.capture_exception(sys.exc_info())
                time.sleep(1)

    def check_heartbeats(self, user_ids: list) -> None:
        for user_id in user_ids:
            still_online = self.env.cache.check_heartbeat(user_id)

            if still_online:
                self.add_heartbeat(user_id)
                continue

            hb_sid = 'hb-{}'.format(user_id)
            data = {
                'verb': 'disconnect',
                'actor': {
                    'id': user_id,
                    'content': hb_sid
                }
            }

            if not self.env.config.get(ConfigKeys.TESTING):
                # only used for single-session restrictions
                if self.env.connected_user_ids.get(user_id) == hb_sid:
                    del self.env.connected_user_ids[user_id]

            activity = parse_to_as(data)
            self.env.observer.emit('on_heartbeat_disconnect', (data, activity))

    @locked_method
    def has_heartbeat(self, user_id: str) -> bool:
        return user_id in self.to_check.keys()

    @locked_method
    def add_heartbeat(self, user_id: str) -> None:
        self.to_check[user_id] = dt.utcnow()

    @locked_method
    def get_all_expired_user_ids(self):
        expired = list()
        not_yet_expired = dict()
        now_time = dt.utcnow() + timedelta(seconds=100)

        for user_id, add_time in self.to_check.items():
            if add_time > now_time:
                expired.append(user_id)
            else:
                not_yet_expired[user_id] = add_time

        self.to_check = not_yet_expired.copy()
        return expired.copy()
<|endoftext|>


<commit_before>from fabric.api import roles, env, local
from fabric.context_managers import cd, prefix, lcd
from fabric.contrib.files import upload_template
from fabric.contrib import django

import shutil
import os
import sys
import zipfile
from tempfile import NamedTemporaryFile

from deployment.remote import exists, run, move, unzip, put, mkdir, phase_out, remove, which, normalize
from deployment.config import DeploymentConfig
from deployment.packaging import zipfiles, get_project_files, directory
from deployment.commands import Command

aptget = Command('apt-get', sudo=True)
python = Command('python2.7',
    sudo_install=lambda *_: aptget.install('-y', 'python-all')
)
pip = Command('pip-2.7',
    install=lambda *_: run('curl', 'https://raw.github.com/pypa/pip/master/contrib/get-pip.py', '|',
        python.path, '-')
)
virtualenv = Command('virtualenv',
    install=lambda *_: pip.install('virtualenv')
)

env.use_distribute = True
env.use_virtualenv = True
env.apache_restart = ('apache2', 'restart')

PROJECT_FILES = [
    directory('newapi', filter='*.py'),
    directory('courses', filter='*.py'),
    directory('scheduler', filter='*.py'),
    directory('lib', 'csp', filter='*.py'),
    directory('lib', 'rpi_courses', filter='*.py'),
    directory('static', filter='*.*'),
    directory('templates', filter='*.*'),
    directory('settings', filter='*.py'),
    '__init__.py',
    'manage.py',
    'requirements.txt',
    'urls.py',
]
TEMPLATES = {
    normalize('settings', 'overrides_template.py'): normalize('settings', 'overrides.py')
}

DEPLOY_FILE = os.path.join('deployment', 'deploy.json')

deploy_config = None

__all__ = (
    'staging',
    'production',
    'webfaction',
    'scss',
    'docs',
    'clean',
    'loc',
    'test',
    'generate_fixtures',
    'deploy',
    'restart',
    'update_courses',
    'update_cache',
)

def staging():
    "Performs actions on the staging servers, as defined in deploy.json."
    global deploy_config
    deploy_config = DeploymentConfig(DEPLOY_FILE, 'staging').assign_to_env(env)

def production():
    "Performs actions on the production servers, as defined in deploy.json."
    global deploy_config
    deploy_config = DeploymentConfig(DEPLOY_FILE, 'production').assign_to_env(env)

def webfaction():
    global pip
    wf_root = normalize(deploy_config.project_root, '..')
    pip = pip.extend(tail_args=[
        '--install-option="--install-scripts=%s/bin"' % wf_root,
        '--install-option="--install-lib=%s/lib/python2.7"' % wf_root,
    ])

    env.use_distribute = False
    env.use_virtualenv = False
    env.apache_restart = ('%s/apache2/bin/restart' % wf_root, )

    #mkdir('~/tmp', recursive=True)

def scss():
    "Watches sass files to convert to css"
    local('sass --watch static/global/scss:static/global/css')

def clean():
    "Removes all pyc cache files."
    for root, dirs, files in os.walk('.'):
        for f in files:
            if f.endswith('.pyc') or f.endswith('.pyo'):
                print " FILE", os.path.join(root, f)[2:]
                os.unlink(os.path.join(root, f))
        for d in dirs:
            if d == '__pycache__':
                print " DIR ", os.path.join(root, d)[2:]
                shutil.rmtree(os.path.join(root, d))

def docs():
    "Generates documentation."
    with lcd(os.path.join(os.path.abspath(os.path.dirname(__file__)), 'docs')):
        local('make html')

def generate_fixtures():
    "Generates various fixtures from the current data in the database."
    # Set up the Django Enviroment
    django.project('yacs')
    import sys
    sys.path.append('..')
    from yacs.courses import models

    def include(model, ids):
        return ' '.join('yacs.courses.%s[%d]' % (model, pk) for pk in ids)

    def include_all(*pairs):
        return ' '.join(include(*pair) for pair in pairs)

    def generate_fixture(output, models):
        local('python manage.py makefixture --indent=4 %s > %s.json' % (
            models, output
        ))
    def course_fixture(output, course_name):
        course_id = models.Course.objects.get(name=course_name).id
        sp_ids = models.SectionPeriod.objects.filter(section__course__id=course_id).values_list('id', flat=True)
        ss_ids = models.SemesterSection.objects.filter(section__course__id=course_id).values_list('id', flat=True)
        return generate_fixture(output,
            include_all(
                ('Course', [course_id]),
                ('SectionPeriod', sp_ids),
                ('SemesterSection', ss_ids),
            )
        )

    def semester_fixtures(output, course_names):
        ids = models.OfferedFor.objects.filter(course__name__in=course_names).values_list('id', flat=True)
        query = models.SemesterDepartment.objects.filter(department__code__in=('CSCI', 'MATH', 'ECSE'))
        return generate_fixture(output,
            include_all(
                ('Semester', models.Semester.objects.all().values_list('id', flat=True)),
                ('SemesterDepartment', query.values_list('id', flat=True)),
                ('OfferedFor', ids),
            )
        )

    course_fixture('courses/fixtures/calc1', 'CALCULUS I')
    course_fixture('courses/fixtures/calc2', 'CALCULUS II')
    course_fixture('courses/fixtures/data-structures', 'DATA STRUCTURES')
    course_fixture('courses/fixtures/intro-to-algorithms', 'INTRODUCTION TO ALGORITHMS')
    course_fixture('courses/fixtures/intro-to-cs', 'INTRO TO COMPUTER PROGRAMMING')
    print "Generating newapi fixtures"
    semester_fixtures('newapi/fixtures/semesters', ('CALCULUS I', 'DATA STRUCTURES', 'INTRO TO COMPUTER PROGRAMMING'))
    #course_fixture('newapi/fixtures/calc1', 'CALCULUS I')
    #course_fixture('newapi/fixtures/data-structures', 'DATA STRUCTURES')
    #course_fixture('newapi/fixtures/intro-to-cs', 'INTRO TO COMPUTER PROGRAMMING')

def test(apps=None):
    """Returns all tests in lib and custom django apps. Optionally accepts specific apps to test.
    If lib is provided as app, only the lib directory is tested.
    """
    if apps is None:
        apps = 'newapi courses scheduler lib'
    apps = apps.split(' ')
    test_lib = 'lib' in apps

    if test_lib:
        print
        print "=" * 50, "TESTING LIB"
        with lcd('lib'):
            local('nosetests -x')
    for app in apps:
        if app == 'lib':
            continue
        print
        print "=" * 50, "TESTING", app.upper()
        local('python manage.py test --failfast ' + app)

def loc():
    "Returns the number of lines of all source files, excluding migrations."
    local('find -E . -iregex ".+/[^0][^/]+\.(py|html|js|scss|txt)$" | xargs wc -l')

@roles('webservers')
def new_deploy():
    "Performs the set up of a first deployment, installed all the required software."
    pass

@roles('webservers')
def deploy(use_pip=False):
    "Performs updates to servers to update the code bases."
    upload_project()
    update_environment(use_pip)
    restart()

def activate_virtualenv_cmd():
    return 'source %s/bin/activate' % deploy_config.virtualenv_name

@roles('webservers')
def upload_project():
    "Uploads the django project to the server."
    project_files = get_project_files(os.path.abspath('.'), PROJECT_FILES)
    tmp_file = NamedTemporaryFile()

    # package for the server and upload it
    zipfiles(project_files, to=tmp_file.name, root=os.path.abspath('.'))

    if not exists(deploy_config.project_root, '..'):
        mkdir(deploy_config.project_root, '..')

    put(tmp_file.name, (deploy_config.project_root, '..', 'project.zip'))

    # extract into location.

    with phase_out(deploy_config.project_root) as backup:
        mkdir(deploy_config.project_root, recursive=True)

        move(
            (deploy_config.project_root, '..', 'project.zip'),
            (deploy_config.project_root, 'project.zip')
        )

        unzip(deploy_config.project_root, 'project.zip', delete=True)

        mkdir((deploy_config.project_root, 'logs'))

        # upload settings.py for staging?
        upload_templates()

        backup.delete()

@roles('webservers')
def upload_templates():
    with cd(deploy_config.project_root):
        for src, dest in TEMPLATES.items():
            upload_template(src, dest, context=deploy_config.deployment_settings, use_jinja=True, backup=False)

@roles('webservers')
def setup_environment():
    global virtualenv
    with cd(deploy_config.project_root):

        if env.use_distribute:
            if env.use_virtualenv:
                virtualenv = virtualenv.extend(['--distribute'])
            pip.install('distribute')

        if exists(deploy_config.virtualenv_name):
            remove(deploy_config.virtualenv_name, recursive=True, force=True)

        if env.use_virtualenv:
            virtualenv('--no-site-packages', deploy_config.virtualenv_name)

        # TODO: remove when not in debug
        if not exists('logs'):
            mkdir('logs')

        update_environment()

@roles('webservers')
def update_environment(use_pip=False):
    use_pip = bool(int(use_pip))
    with cd(deploy_config.project_root):
        if env.use_virtualenv:
            if use_pip:
                pip.install(r='requirements.txt', E=deploy_config.virtualenv_name)

            with prefix(activate_virtualenv_cmd()):
                run('python', 'manage.py', 'syncdb')
                run('python', 'manage.py', 'migrate')
                run('python', 'manage.py', 'collectstatic', '--noinput')
        else:
            if use_pip:
                pip.install(r='requirements.txt')

            manage_py = python.extend(['manage.py'])
            manage_py.syncdb()
            manage_py.migrate()
            manage_py.collectstatic('--noinput')

def run_manage_cmd(command, *args):
    with cd(deploy_config.project_root):
        if env.use_virtualenv:
            with prefix(activate_virtualenv_cmd()):
                run('python', 'manage.py', command, *args)
        else:
            python.extend(['manage.py'])(command, *args)

@roles('webservers')
def clear_course_data():
    run_manage_cmd('reset', 'sessions', '--noinput')
    run_manage_cmd('reset', 'courses', '--noinput')

@roles('webservers')
def update_courses(args=''):
    with cd(deploy_config.project_root):
        if env.use_virtualenv:
            with prefix(activate_virtualenv_cmd()):
                run('python', 'manage.py', 'import_course_data', args)
        else:
            python.extend(['manage.py']).import_course_data(args)

@roles('webservers')
def update_cache(args=''):
    with cd(deploy_config.project_root):
        if env.use_virtualenv:
            with prefix(activate_virtualenv_cmd()):
                run('python', 'manage.py', 'create_section_cache', args)
        else:
            python.extend(['manage.py']).create_section_cache(args)

@roles('webservers')
def restart():
    "Restarts the remote server's apache instance"
    run(*env.apache_restart)

<commit_msg>Added/update fab commands update cache & robots data.<commit_after>from fabric.api import roles, env, local
from fabric.context_managers import cd, prefix, lcd
from fabric.contrib.files import upload_template
from fabric.contrib import django

import shutil
import os
import sys
import zipfile
from tempfile import NamedTemporaryFile

from deployment.remote import exists, run, move, unzip, put, mkdir, phase_out, remove, which, normalize
from deployment.config import DeploymentConfig
from deployment.packaging import zipfiles, get_project_files, directory
from deployment.commands import Command

aptget = Command('apt-get', sudo=True)
python = Command('python2.7',
    sudo_install=lambda *_: aptget.install('-y', 'python-all')
)
pip = Command('pip-2.7',
    install=lambda *_: run('curl', 'https://raw.github.com/pypa/pip/master/contrib/get-pip.py', '|',
        python.path, '-')
)
virtualenv = Command('virtualenv',
    install=lambda *_: pip.install('virtualenv')
)

env.use_distribute = True
env.use_virtualenv = True
env.apache_restart = ('apache2', 'restart')

PROJECT_FILES = [
    directory('newapi', filter='*.py'),
    directory('courses', filter='*.py'),
    directory('scheduler', filter='*.py'),
    directory('lib', 'csp', filter='*.py'),
    directory('lib', 'rpi_courses', filter='*.py'),
    directory('static', filter='*.*'),
    directory('templates', filter='*.*'),
    directory('settings', filter='*.py'),
    '__init__.py',
    'manage.py',
    'requirements.txt',
    'urls.py',
]
TEMPLATES = {
    normalize('settings', 'overrides_template.py'): normalize('settings', 'overrides.py')
}

DEPLOY_FILE = os.path.join('deployment', 'deploy.json')

deploy_config = None

__all__ = (
    'staging',
    'production',
    'webfaction',
    'scss',
    'docs',
    'clean',
    'loc',
    'test',
    'generate_fixtures',
    'deploy',
    'restart',
    'update_courses',
    'update_cache',
    'update_cache',
    'create_robotstxt',
    'refresh_data',
)

def staging():
    "Performs actions on the staging servers, as defined in deploy.json."
    global deploy_config
    deploy_config = DeploymentConfig(DEPLOY_FILE, 'staging').assign_to_env(env)

def production():
    "Performs actions on the production servers, as defined in deploy.json."
    global deploy_config
    deploy_config = DeploymentConfig(DEPLOY_FILE, 'production').assign_to_env(env)

def webfaction():
    global pip
    wf_root = normalize(deploy_config.project_root, '..')
    pip = pip.extend(tail_args=[
        '--install-option="--install-scripts=%s/bin"' % wf_root,
        '--install-option="--install-lib=%s/lib/python2.7"' % wf_root,
    ])

    env.use_distribute = False
    env.use_virtualenv = False
    env.apache_restart = ('%s/apache2/bin/restart' % wf_root, )

    #mkdir('~/tmp', recursive=True)

def scss():
    "Watches sass files to convert to css"
    local('sass --watch static/global/scss:static/global/css')

def clean():
    "Removes all pyc cache files."
    for root, dirs, files in os.walk('.'):
        for f in files:
            if f.endswith('.pyc') or f.endswith('.pyo'):
                print " FILE", os.path.join(root, f)[2:]
                os.unlink(os.path.join(root, f))
        for d in dirs:
            if d == '__pycache__':
                print " DIR ", os.path.join(root, d)[2:]
                shutil.rmtree(os.path.join(root, d))

def docs():
    "Generates documentation."
    with lcd(os.path.join(os.path.abspath(os.path.dirname(__file__)), 'docs')):
        local('make html')

def generate_fixtures():
    "Generates various fixtures from the current data in the database."
    # Set up the Django Enviroment
    django.project('yacs')
    import sys
    sys.path.append('..')
    from yacs.courses import models

    def include(model, ids):
        return ' '.join('yacs.courses.%s[%d]' % (model, pk) for pk in ids)

    def include_all(*pairs):
        return ' '.join(include(*pair) for pair in pairs)

    def generate_fixture(output, models):
        local('python manage.py makefixture --indent=4 %s > %s.json' % (
            models, output
        ))
    def course_fixture(output, course_name):
        course_id = models.Course.objects.get(name=course_name).id
        sp_ids = models.SectionPeriod.objects.filter(section__course__id=course_id).values_list('id', flat=True)
        ss_ids = models.SemesterSection.objects.filter(section__course__id=course_id).values_list('id', flat=True)
        return generate_fixture(output,
            include_all(
                ('Course', [course_id]),
                ('SectionPeriod', sp_ids),
                ('SemesterSection', ss_ids),
            )
        )

    def semester_fixtures(output, course_names):
        ids = models.OfferedFor.objects.filter(course__name__in=course_names).values_list('id', flat=True)
        query = models.SemesterDepartment.objects.filter(department__code__in=('CSCI', 'MATH', 'ECSE'))
        return generate_fixture(output,
            include_all(
                ('Semester', models.Semester.objects.all().values_list('id', flat=True)),
                ('SemesterDepartment', query.values_list('id', flat=True)),
                ('OfferedFor', ids),
            )
        )

    course_fixture('courses/fixtures/calc1', 'CALCULUS I')
    course_fixture('courses/fixtures/calc2', 'CALCULUS II')
    course_fixture('courses/fixtures/data-structures', 'DATA STRUCTURES')
    course_fixture('courses/fixtures/intro-to-algorithms', 'INTRODUCTION TO ALGORITHMS')
    course_fixture('courses/fixtures/intro-to-cs', 'INTRO TO COMPUTER PROGRAMMING')
    print "Generating newapi fixtures"
    semester_fixtures('newapi/fixtures/semesters', ('CALCULUS I', 'DATA STRUCTURES', 'INTRO TO COMPUTER PROGRAMMING'))
    #course_fixture('newapi/fixtures/calc1', 'CALCULUS I')
    #course_fixture('newapi/fixtures/data-structures', 'DATA STRUCTURES')
    #course_fixture('newapi/fixtures/intro-to-cs', 'INTRO TO COMPUTER PROGRAMMING')

def test(apps=None):
    """Returns all tests in lib and custom django apps. Optionally accepts specific apps to test.
    If lib is provided as app, only the lib directory is tested.
    """
    if apps is None:
        apps = 'newapi courses scheduler lib'
    apps = apps.split(' ')
    test_lib = 'lib' in apps

    if test_lib:
        print
        print "=" * 50, "TESTING LIB"
        with lcd('lib'):
            local('nosetests -x')
    for app in apps:
        if app == 'lib':
            continue
        print
        print "=" * 50, "TESTING", app.upper()
        local('python manage.py test --failfast ' + app)

def loc():
    "Returns the number of lines of all source files, excluding migrations."
    local('find -E . -iregex ".+/[^0][^/]+\.(py|html|js|scss|txt)$" | xargs wc -l')

@roles('webservers')
def new_deploy():
    "Performs the set up of a first deployment, installed all the required software."
    pass

@roles('webservers')
def deploy(use_pip=False):
    "Performs updates to servers to update the code bases."
    upload_project()
    update_environment(use_pip)
    restart()

def activate_virtualenv_cmd():
    return 'source %s/bin/activate' % deploy_config.virtualenv_name

@roles('webservers')
def upload_project():
    "Uploads the django project to the server."
    project_files = get_project_files(os.path.abspath('.'), PROJECT_FILES)
    tmp_file = NamedTemporaryFile()

    # package for the server and upload it
    zipfiles(project_files, to=tmp_file.name, root=os.path.abspath('.'))

    if not exists(deploy_config.project_root, '..'):
        mkdir(deploy_config.project_root, '..')

    put(tmp_file.name, (deploy_config.project_root, '..', 'project.zip'))

    # extract into location.

    with phase_out(deploy_config.project_root) as backup:
        mkdir(deploy_config.project_root, recursive=True)

        move(
            (deploy_config.project_root, '..', 'project.zip'),
            (deploy_config.project_root, 'project.zip')
        )

        unzip(deploy_config.project_root, 'project.zip', delete=True)

        mkdir((deploy_config.project_root, 'logs'))

        # upload settings.py for staging?
        upload_templates()

        backup.delete()

@roles('webservers')
def upload_templates():
    with cd(deploy_config.project_root):
        for src, dest in TEMPLATES.items():
            upload_template(src, dest, context=deploy_config.deployment_settings, use_jinja=True, backup=False)

@roles('webservers')
def setup_environment():
    global virtualenv
    with cd(deploy_config.project_root):

        if env.use_distribute:
            if env.use_virtualenv:
                virtualenv = virtualenv.extend(['--distribute'])
            pip.install('distribute')

        if exists(deploy_config.virtualenv_name):
            remove(deploy_config.virtualenv_name, recursive=True, force=True)

        if env.use_virtualenv:
            virtualenv('--no-site-packages', deploy_config.virtualenv_name)

        # TODO: remove when not in debug
        if not exists('logs'):
            mkdir('logs')

        update_environment()

@roles('webservers')
def update_environment(use_pip=False):
    use_pip = bool(int(use_pip))
    with cd(deploy_config.project_root):
        if env.use_virtualenv:
            if use_pip:
                pip.install(r='requirements.txt', E=deploy_config.virtualenv_name)

            with prefix(activate_virtualenv_cmd()):
                run('python', 'manage.py', 'syncdb')
                run('python', 'manage.py', 'migrate')
                run('python', 'manage.py', 'collectstatic', '--noinput')
        else:
            if use_pip:
                pip.install(r='requirements.txt')

            manage_py = python.extend(['manage.py'])
            manage_py.syncdb()
            manage_py.migrate()
            manage_py.collectstatic('--noinput')

def run_manage_cmd(command, *args):
    with cd(deploy_config.project_root):
        if env.use_virtualenv:
            with prefix(activate_virtualenv_cmd()):
                run('python', 'manage.py', command, *args)
        else:
            python.extend(['manage.py'])(command, *args)

@roles('webserver')
def refresh_data():
    clear_course_data()
    update_courses()
    update_cache()
    create_robotstxt()

@roles('webservers')
def clear_course_data():
    run_manage_cmd('reset', 'sessions', '--noinput')
    run_manage_cmd('reset', 'courses', '--noinput')

@roles('webservers')
def update_courses(args=''):
    run_manage_cmd('import_course_data', args)

@roles('webservers')
def update_cache(args=''):
    run_manage_cmd('create_section_cache', args)

@roles('webservers')
def create_robotstxt():
    run_manage_cmd('sync_robots_data')

@roles('webservers')
def restart():
    "Restarts the remote server's apache instance"
    run(*env.apache_restart)

<|endoftext|>


<commit_before># -*- coding: utf-8 -*-

u"""
.. module:: fabfile

Be aware, that becaus fabric doesn't support py3k You need to execute this
particular script using Python 2.
"""

import contextlib
import os
import sys

from fabric.api import cd
from fabric.api import env
from fabric.api import execute
from fabric.api import prefix
from fabric.api import run
from fabric.contrib import files

NODE_VERSION = '9.3.0'

env.user = 'root'
if not env.hosts:
    env.hosts = ['dev.volontulo.pl']
env.forward_agent = True

env_vars = {
    'dev.volontulo.pl': {
        'git_branch': 'master',
        'django_settings': 'dev_volontulo_pl',
    },
    'rc.volontulo.pl': {
        'git_branch': 'rc',
        'django_settings': 'rc_volontulo_pl',
    },
    'volontulo.pl': {
        'git_branch': 'prod',
        'django_settings': 'volontulo_pl',
    },
}

def update():
    u"""Function defining all steps required to properly update application."""

    # Django app refresh:
    with cd('/var/www/volontulo'):
        run('git checkout -f {}'.format(env_vars[env.host_string]['git_branch']))
        run('git pull')

    with contextlib.nested(
        prefix('workon volontulo'),
        cd('/var/www/volontulo/backend'),
    ):
        run('pip install --upgrade -r requirements/base.txt')

    # Django site refresh:
    with contextlib.nested(
        cd('/var/www/volontulo/backend'),
        prefix('workon volontulo')
    ):
        run('python manage.py migrate --traceback')

    # Angular assets refresh:
    with contextlib.nested(
        prefix('nvm use {}'.format(NODE_VERSION)),
        cd('/var/www/volontulo/frontend'),
    ):
        run('npm install .')
        run('./node_modules/.bin/ng build --prod --env={}'.format(env.host_string))
        run('./node_modules/.bin/ng build --prod --env={} --app 1 --output-hashing=false'.format(env.host_string))
        run('./node_modules/.bin/webpack --config webpack.server.config.js --progress --colors')

    run('systemctl restart uwsgi.service')
    run('systemctl restart nginx')
    run('systemctl restart pm2-www-data.service')

def install():
    u"""Function defining all steps required to install application."""

    # ensure that we have secrets configured:
    sys.path.insert(0, os.path.dirname(__file__))
    try:
        from secrets import (
            CFP_ADMIN_PASSWORD,
            VOLONTULO_SENTRY_DSN,
            WRK_ADMIN_PASSWORD,
        )
    except ImportError:
        print("Missing secrets")
        raise

    # Sytem upgrade:
    run('apt-get update -y')
    run('apt-get -o Dpkg::Options::="--force-confold" upgrade -y')
    run('apt-get install -y python-minimal')  # <- old fronted npm install requires python 2 executable

    # Secrets:
    run('apt-get install -y pwgen')
    run('echo "export VOLONTULO_SECRET_KEY=\"`pwgen 64 1`\"" >> ~/.bash_profile')
    run('echo "export VOLONTULO_DB_PASS=\"`pwgen 64 1`\"" >> ~/.bash_profile')
    run('source ~/.bash_profile')

    # Databse:
    run('apt-get install -y postgresql postgresql-contrib')
    run('su - postgres -c "psql -c \'CREATE ROLE volontulo;\'"')
    run('su - postgres -c "psql -c \\\\"ALTER USER volontulo with password \'$VOLONTULO_DB_PASS\';\\\\""')
    run('su - postgres -c "psql -c \'ALTER ROLE volontulo WITH LOGIN;\'"')
    run('su - postgres -c "psql -c \\\\"CREATE DATABASE volontulo WITH  TEMPLATE=template0 ENCODING=\'utf-8\' owner volontulo;\\\\""')

    # Mail Transport Agent:
    run('debconf-set-selections <<< "postfix postfix/mailname string $HOSTNAME"')
    run('debconf-set-selections <<< "postfix postfix/main_mailer_type string \'Internet Site\"')
    run('apt-get install -y postfix')

    # Fetch Volontulo code:
    run('apt-get install -y nginx')
    with cd('/var/www'):
        run('git clone https://github.com/CodeForPoznan/volontulo.git')
    with cd('/var/www/volontulo'):
        run('git checkout -f {}'.format(env_vars[env.host_string]['git_branch']))

    # Install proper Node:
    run('echo "export NVM_DIR=\"/var/www/.nvm\"" >> ~/.bash_profile')
    run('source ~/.bash_profile')
    run('wget -qO- https://raw.githubusercontent.com/creationix/nvm/v0.33.8/install.sh | bash')
    run('echo \'[ -s "$NVM_DIR/nvm.sh" ] && . "$NVM_DIR/nvm.sh"\' >> ~/.bash_profile')
    run('echo \'[ -s "$NVM_DIR/bash_completion" ] && \. "$NVM_DIR/bash_completion"\' >> ~/.bash_profile')
    run('source ~/.bash_profile')
    run('nvm install {}'.format(NODE_VERSION))

    # Install pm2
    with prefix('nvm use {}'.format(NODE_VERSION)):
        run('npm install -g pm2')
    with cd('/var'):
        run('chown www-data:www-data www')
    with contextlib.nested(
        prefix('nvm use {}'.format(NODE_VERSION)),
        cd('/var/www/volontulo/frontend'),
    ):
        run('npm install .')
        run('./node_modules/.bin/ng build --prod --env={}'.format(env.host_string))
        run('./node_modules/.bin/ng build --prod --env={} --app 1 --output-hashing=false'.format(env.host_string))
        run('./node_modules/.bin/webpack --config webpack.server.config.js --progress --colors')
        run("sudo -u www-data bash -c 'export PATH=/var/www/.nvm/versions/node/v{}/bin:$PATH && export HOME=/var/www  && export VOLONTULO_PM2_HOST=127.0.0.1 && pm2 start dist/server && pm2 save'".format(NODE_VERSION))
        run("env PATH=$PATH:/var/www/.nvm/versions/node/v{}/bin /var/www/.nvm/versions/node/v{}/lib/node_modules/pm2/bin/pm2 startup ubuntu -u www-data --hp /var/www".format(NODE_VERSION, NODE_VERSION))

    # Install virtualenv:
    run('apt-get install -y python3-pip')
    run('pip3 install --upgrade pip')
    run('pip3 install virtualenv virtualenvwrapper')
    run('echo "export VIRTUALENVWRAPPER_PYTHON=/usr/bin/python3" >> ~/.bash_profile')
    run('echo "export WORKON_HOME=/var/www/virtualenvs" >> ~/.bash_profile')
    run('echo "source /usr/local/bin/virtualenvwrapper.sh" >> ~/.bash_profile')
    run('source ~/.bash_profile')
    run('mkvirtualenv volontulo')

    # Prepare Django application
    run('echo "export DJANGO_SETTINGS_MODULE=volontulo_org.settings.{}" >> ~/.bash_profile'.format(
        env_vars[env.host_string]['django_settings']))
    run('source ~/.bash_profile')
    with cd('/var/www/volontulo/backend'):
        run('mkdir media')
        run('chown www-data:www-data media')

    # Export Sentry DSN key
    run('echo "export VOLONTULO_SENTRY_DSN={}" >> ~/.bash_profile'.format(
        VOLONTULO_SENTRY_DSN))

    # Install uwsgi
    run('pip3 install uwsgi')
    run('mkdir -p /etc/uwsgi/sites')
    files.append('/etc/uwsgi/sites/volontulo.ini',
"""[uwsgi]
uid = www-data
chdir = /var/www/volontulo/backend
home = /var/www/virtualenvs/volontulo
module = volontulo_org.wsgi

env = DJANGO_SETTINGS_MODULE=volontulo_org.settings.{}""".format(
    env_vars[env.host_string]['django_settings']))
    run("echo 'env = VOLONTULO_SECRET_KEY='$VOLONTULO_SECRET_KEY >> /etc/uwsgi/sites/volontulo.ini")
    run("echo 'env = VOLONTULO_DB_PASS='$VOLONTULO_DB_PASS >> /etc/uwsgi/sites/volontulo.ini")
    run("echo 'env = VOLONTULO_SENTRY_DSN='$VOLONTULO_SENTRY_DSN >> /etc/uwsgi/sites/volontulo.ini")
    files.append('/etc/uwsgi/sites/volontulo.ini',
"""
socket = /run/uwsgi/volontulo.sock
chown-socket = www-data:www-data
chmod-socket = 660
vacuum = true
enable-threads = true
""")
    files.append('/etc/systemd/system/uwsgi.service',
"""[Unit]
Description=uWSGI Emperor service

[Service]
ExecStartPre=/bin/bash -c 'mkdir -p /run/uwsgi; chown www-data:www-data /run/uwsgi'
ExecStart=/usr/local/bin/uwsgi --emperor /etc/uwsgi/sites
Restart=always
KillSignal=SIGQUIT
Type=notify
NotifyAccess=all

[Install]
WantedBy=multi-user.target
""")
    run('systemctl start uwsgi')

    # Connect nginx with uwsgi:
    run('rm /etc/nginx/sites-enabled/default')
    files.append('/etc/nginx/sites-enabled/default',
"""server {{
    listen 80 default_server;
    listen [::]:80 default_server;

    if ($host != "{0}") {{
        return 301 https://{0}$request_uri;
    }}

}}

server {{
    charset utf-8;

    listen 80;
    listen [::]:80;

    listen 443 ssl;
    listen [::]:443 ssl;

    server_name {0};

    root /var/www/volontulo/frontend/dist/browser;
    index index.html;

    location /static/ {{
        root /var/www/volontulo/backend;
    }}

    location /media/ {{
        root /var/www/volontulo/backend;
    }}

    location = /o {{
        include uwsgi_params;
        uwsgi_pass unix:/run/uwsgi/volontulo.sock;
    }}

    location /o/ {{
        include uwsgi_params;
        uwsgi_pass unix:/run/uwsgi/volontulo.sock;
    }}

    location /api/ {{
        include uwsgi_params;
        uwsgi_pass unix:/run/uwsgi/volontulo.sock;
    }}

    location /admin {{
        include uwsgi_params;
        uwsgi_pass unix:/run/uwsgi/volontulo.sock;
    }}

    location / {{
        proxy_pass http://localhost:4200;
        proxy_http_version 1.1;
        proxy_set_header Upgrade $http_upgrade;
        proxy_set_header Connection 'upgrade';
        proxy_set_header Host $host;
        proxy_cache_bypass $http_upgrade;
    }}
}}
""".format(env.host_string))

    # Install SSL:
    run('add-apt-repository -y ppa:certbot/certbot')
    run('apt-get update -y')
    run('apt-get install -y python-certbot-nginx ')
    run('certbot --authenticator standalone --installer nginx -m hello@codeforpoznan.pl --agree-tos --no-eff-email -d {} --redirect --pre-hook "service nginx stop" --post-hook "service nginx start"'.format(env.host_string))
    run('(crontab -l 2>/dev/null; echo \'0 0 * * * certbot renew --post-hook "systemctl reload nginx"\') | crontab -')

    execute(update)

    # Create Django admin:
    with contextlib.nested(
        prefix('workon volontulo'),
        cd('/var/www/volontulo/backend'),
    ):
        run('python manage.py create_admin hello@codeforpoznan.pl {} --django-admin'.format(CFP_ADMIN_PASSWORD))
        run('python manage.py create_admin wolontariat@wrk.org.pl {}'.format(WRK_ADMIN_PASSWORD))
<commit_msg>Install DKIM for our Postfix-based MTA (#960)<commit_after># -*- coding: utf-8 -*-

u"""
.. module:: fabfile

Be aware, that becaus fabric doesn't support py3k You need to execute this
particular script using Python 2.
"""

import contextlib
import os
import sys

from fabric.api import cd
from fabric.api import env
from fabric.api import execute
from fabric.api import prefix
from fabric.api import run
from fabric.contrib import files

NODE_VERSION = '9.3.0'

env.user = 'root'
if not env.hosts:
    env.hosts = ['dev.volontulo.pl']
env.forward_agent = True

env_vars = {
    'dev.volontulo.pl': {
        'git_branch': 'master',
        'django_settings': 'dev_volontulo_pl',
    },
    'rc.volontulo.pl': {
        'git_branch': 'rc',
        'django_settings': 'rc_volontulo_pl',
    },
    'volontulo.pl': {
        'git_branch': 'prod',
        'django_settings': 'volontulo_pl',
    },
}

def update():
    u"""Function defining all steps required to properly update application."""

    # Django app refresh:
    with cd('/var/www/volontulo'):
        run('git checkout -f {}'.format(env_vars[env.host_string]['git_branch']))
        run('git pull')

    with contextlib.nested(
        prefix('workon volontulo'),
        cd('/var/www/volontulo/backend'),
    ):
        run('pip install --upgrade -r requirements/base.txt')

    # Django site refresh:
    with contextlib.nested(
        cd('/var/www/volontulo/backend'),
        prefix('workon volontulo')
    ):
        run('python manage.py migrate --traceback')

    # Angular assets refresh:
    with contextlib.nested(
        prefix('nvm use {}'.format(NODE_VERSION)),
        cd('/var/www/volontulo/frontend'),
    ):
        run('npm install .')
        run('./node_modules/.bin/ng build --prod --env={}'.format(env.host_string))
        run('./node_modules/.bin/ng build --prod --env={} --app 1 --output-hashing=false'.format(env.host_string))
        run('./node_modules/.bin/webpack --config webpack.server.config.js --progress --colors')

    run('systemctl restart uwsgi.service')
    run('systemctl restart nginx')
    run('systemctl restart pm2-www-data.service')

def install():
    u"""Function defining all steps required to install application."""

    # ensure that we have secrets configured:
    sys.path.insert(0, os.path.dirname(__file__))
    try:
        from secrets import (
            CFP_ADMIN_PASSWORD,
            DKIM_PRIVATE_KEY,
            VOLONTULO_SENTRY_DSN,
            WRK_ADMIN_PASSWORD,
        )
    except ImportError:
        print("Missing secrets")
        raise

    # Sytem upgrade:
    run('apt-get update -y')
    run('apt-get -o Dpkg::Options::="--force-confold" upgrade -y')
    run('apt-get install -y python-minimal')  # <- old fronted npm install requires python 2 executable

    # Secrets:
    run('apt-get install -y pwgen')
    run('echo "export VOLONTULO_SECRET_KEY=\"`pwgen 64 1`\"" >> ~/.bash_profile')
    run('echo "export VOLONTULO_DB_PASS=\"`pwgen 64 1`\"" >> ~/.bash_profile')
    run('source ~/.bash_profile')

    # Databse:
    run('apt-get install -y postgresql postgresql-contrib')
    run('su - postgres -c "psql -c \'CREATE ROLE volontulo;\'"')
    run('su - postgres -c "psql -c \\\\"ALTER USER volontulo with password \'$VOLONTULO_DB_PASS\';\\\\""')
    run('su - postgres -c "psql -c \'ALTER ROLE volontulo WITH LOGIN;\'"')
    run('su - postgres -c "psql -c \\\\"CREATE DATABASE volontulo WITH  TEMPLATE=template0 ENCODING=\'utf-8\' owner volontulo;\\\\""')

    # Mail Transport Agent:
    run('debconf-set-selections <<< "postfix postfix/mailname string $HOSTNAME"')
    run('debconf-set-selections <<< "postfix postfix/main_mailer_type string \'Internet Site\"')
    run('apt-get install -y postfix')

    # DKIM support for MTA:
    run('apt-get install -y opendkim opendkim-tools')
    files.append('/etc/opendkim.conf',
"""
AutoRestart             Yes
AutoRestartRate         10/1h
UMask                   002
Syslog                  yes
SyslogSuccess           Yes
LogWhy                  Yes

Canonicalization        relaxed/simple

ExternalIgnoreList      refile:/etc/opendkim/TrustedHosts
InternalHosts           refile:/etc/opendkim/TrustedHosts
KeyTable                refile:/etc/opendkim/KeyTable
SigningTable            refile:/etc/opendkim/SigningTable

Mode                    sv
PidFile                 /var/run/opendkim/opendkim.pid
SignatureAlgorithm      rsa-sha256

UserID                  opendkim:opendkim

Socket                  inet:12301@localhost
""")
    run('echo "SOCKET=\"inet:12301@localhost\"" >> /etc/default/opendkim')
    run('echo "milter_protocol = 2" >> /etc/postfix/main.cf')
    run('echo "milter_default_action = accept" >> /etc/postfix/main.cf')
    run('echo "smtpd_milters = inet:localhost:12301" >> /etc/postfix/main.cf')
    run('echo "non_smtpd_milters = inet:localhost:12301" >> /etc/postfix/main.cf')
    run('mkdir /etc/opendkim')
    run('mkdir /etc/opendkim/keys')
    run('echo "127.0.0.1/8" >> /etc/opendkim/TrustedHosts')
    run('echo "localhost" >> /etc/opendkim/TrustedHosts')
    run('echo "mail._domainkey.{} {}:mail:/etc/opendkim/keys/{}/mail.private" >> /etc/opendkim/KeyTable'.format(
        env.host_string
    ))
    run('echo "*@{} mail._domainkey.{}" >> /etc/opendkim/SigningTable'.format(env.host_string))
    run('mkdir /etc/opendkim/keys/{}'.format(env.host_string))
    files.append(
        '/etc/opendkim/keys/{}/mail.private'.format(env.host_string),
        DKIM_PRIVATE_KEY,
    )
    run('chown opendkim:opendkim /etc/opendkim/keys/{}/mail.private'.format(
        env.host_string
    ))
    run('service postfix restart')
    run('service opendkim restart')

    # Fetch Volontulo code:
    run('apt-get install -y nginx')
    with cd('/var/www'):
        run('git clone https://github.com/CodeForPoznan/volontulo.git')
    with cd('/var/www/volontulo'):
        run('git checkout -f {}'.format(env_vars[env.host_string]['git_branch']))

    # Install proper Node:
    run('echo "export NVM_DIR=\"/var/www/.nvm\"" >> ~/.bash_profile')
    run('source ~/.bash_profile')
    run('wget -qO- https://raw.githubusercontent.com/creationix/nvm/v0.33.8/install.sh | bash')
    run('echo \'[ -s "$NVM_DIR/nvm.sh" ] && . "$NVM_DIR/nvm.sh"\' >> ~/.bash_profile')
    run('echo \'[ -s "$NVM_DIR/bash_completion" ] && \. "$NVM_DIR/bash_completion"\' >> ~/.bash_profile')
    run('source ~/.bash_profile')
    run('nvm install {}'.format(NODE_VERSION))

    # Install pm2
    with prefix('nvm use {}'.format(NODE_VERSION)):
        run('npm install -g pm2')
    with cd('/var'):
        run('chown www-data:www-data www')
    with contextlib.nested(
        prefix('nvm use {}'.format(NODE_VERSION)),
        cd('/var/www/volontulo/frontend'),
    ):
        run('npm install .')
        run('./node_modules/.bin/ng build --prod --env={}'.format(env.host_string))
        run('./node_modules/.bin/ng build --prod --env={} --app 1 --output-hashing=false'.format(env.host_string))
        run('./node_modules/.bin/webpack --config webpack.server.config.js --progress --colors')
        run("sudo -u www-data bash -c 'export PATH=/var/www/.nvm/versions/node/v{}/bin:$PATH && export HOME=/var/www  && export VOLONTULO_PM2_HOST=127.0.0.1 && pm2 start dist/server && pm2 save'".format(NODE_VERSION))
        run("env PATH=$PATH:/var/www/.nvm/versions/node/v{}/bin /var/www/.nvm/versions/node/v{}/lib/node_modules/pm2/bin/pm2 startup ubuntu -u www-data --hp /var/www".format(NODE_VERSION, NODE_VERSION))

    # Install virtualenv:
    run('apt-get install -y python3-pip')
    run('pip3 install --upgrade pip')
    run('pip3 install virtualenv virtualenvwrapper')
    run('echo "export VIRTUALENVWRAPPER_PYTHON=/usr/bin/python3" >> ~/.bash_profile')
    run('echo "export WORKON_HOME=/var/www/virtualenvs" >> ~/.bash_profile')
    run('echo "source /usr/local/bin/virtualenvwrapper.sh" >> ~/.bash_profile')
    run('source ~/.bash_profile')
    run('mkvirtualenv volontulo')

    # Prepare Django application
    run('echo "export DJANGO_SETTINGS_MODULE=volontulo_org.settings.{}" >> ~/.bash_profile'.format(
        env_vars[env.host_string]['django_settings']))
    run('source ~/.bash_profile')
    with cd('/var/www/volontulo/backend'):
        run('mkdir media')
        run('chown www-data:www-data media')

    # Export Sentry DSN key
    run('echo "export VOLONTULO_SENTRY_DSN={}" >> ~/.bash_profile'.format(
        VOLONTULO_SENTRY_DSN))

    # Install uwsgi
    run('pip3 install uwsgi')
    run('mkdir -p /etc/uwsgi/sites')
    files.append('/etc/uwsgi/sites/volontulo.ini',
"""[uwsgi]
uid = www-data
chdir = /var/www/volontulo/backend
home = /var/www/virtualenvs/volontulo
module = volontulo_org.wsgi

env = DJANGO_SETTINGS_MODULE=volontulo_org.settings.{}""".format(
    env_vars[env.host_string]['django_settings']))
    run("echo 'env = VOLONTULO_SECRET_KEY='$VOLONTULO_SECRET_KEY >> /etc/uwsgi/sites/volontulo.ini")
    run("echo 'env = VOLONTULO_DB_PASS='$VOLONTULO_DB_PASS >> /etc/uwsgi/sites/volontulo.ini")
    run("echo 'env = VOLONTULO_SENTRY_DSN='$VOLONTULO_SENTRY_DSN >> /etc/uwsgi/sites/volontulo.ini")
    files.append('/etc/uwsgi/sites/volontulo.ini',
"""
socket = /run/uwsgi/volontulo.sock
chown-socket = www-data:www-data
chmod-socket = 660
vacuum = true
enable-threads = true
""")
    files.append('/etc/systemd/system/uwsgi.service',
"""[Unit]
Description=uWSGI Emperor service

[Service]
ExecStartPre=/bin/bash -c 'mkdir -p /run/uwsgi; chown www-data:www-data /run/uwsgi'
ExecStart=/usr/local/bin/uwsgi --emperor /etc/uwsgi/sites
Restart=always
KillSignal=SIGQUIT
Type=notify
NotifyAccess=all

[Install]
WantedBy=multi-user.target
""")
    run('systemctl start uwsgi')

    # Connect nginx with uwsgi:
    run('rm /etc/nginx/sites-enabled/default')
    files.append('/etc/nginx/sites-enabled/default',
"""server {{
    listen 80 default_server;
    listen [::]:80 default_server;

    if ($host != "{0}") {{
        return 301 https://{0}$request_uri;
    }}

}}

server {{
    charset utf-8;

    listen 80;
    listen [::]:80;

    listen 443 ssl;
    listen [::]:443 ssl;

    server_name {0};

    root /var/www/volontulo/frontend/dist/browser;
    index index.html;

    location /static/ {{
        root /var/www/volontulo/backend;
    }}

    location /media/ {{
        root /var/www/volontulo/backend;
    }}

    location = /o {{
        include uwsgi_params;
        uwsgi_pass unix:/run/uwsgi/volontulo.sock;
    }}

    location /o/ {{
        include uwsgi_params;
        uwsgi_pass unix:/run/uwsgi/volontulo.sock;
    }}

    location /api/ {{
        include uwsgi_params;
        uwsgi_pass unix:/run/uwsgi/volontulo.sock;
    }}

    location /admin {{
        include uwsgi_params;
        uwsgi_pass unix:/run/uwsgi/volontulo.sock;
    }}

    location / {{
        proxy_pass http://localhost:4200;
        proxy_http_version 1.1;
        proxy_set_header Upgrade $http_upgrade;
        proxy_set_header Connection 'upgrade';
        proxy_set_header Host $host;
        proxy_cache_bypass $http_upgrade;
    }}
}}
""".format(env.host_string))

    # Install SSL:
    run('add-apt-repository -y ppa:certbot/certbot')
    run('apt-get update -y')
    run('apt-get install -y python-certbot-nginx ')
    run('certbot --authenticator standalone --installer nginx -m hello@codeforpoznan.pl --agree-tos --no-eff-email -d {} --redirect --pre-hook "service nginx stop" --post-hook "service nginx start"'.format(env.host_string))
    run('(crontab -l 2>/dev/null; echo \'0 0 * * * certbot renew --post-hook "systemctl reload nginx"\') | crontab -')

    execute(update)

    # Create Django admin:
    with contextlib.nested(
        prefix('workon volontulo'),
        cd('/var/www/volontulo/backend'),
    ):
        run('python manage.py create_admin hello@codeforpoznan.pl {} --django-admin'.format(CFP_ADMIN_PASSWORD))
        run('python manage.py create_admin wolontariat@wrk.org.pl {}'.format(WRK_ADMIN_PASSWORD))

    print("""
Remember to:
 * enable Google monitoring for mail deliverability in Google Postmaster Tools
   (see: https://support.google.com/mail/answer/6258950)
 * add valid reverse DNS record for server IP address
   (see: https://www.ovh.com/world/dedicated-servers/reverse_personnalise.xml)
 * add DKIM public key to the DNS zone
   (see: https://www.digitalocean.com/community/tutorials/how-to-install-and-configure-dkim-with-postfix-on-debian-wheezy#add-the-public-key-to-the-domain-39-s-dns-records)
    """)
<|endoftext|>


<commit_before>from fabric.api import *

env.user = 'ptigas'
env.hosts = ['ptigas.com']

def deploy():
	with cd('repos/hocus'):
		run('git pull')

	with cd('repos/hocus/webapp'):
		run('composer install')

	with cd('repos/hocus'):
		run('cp -r webapp/* ~/Sites/hocus.io/')
		run('cp -r vendor ~/Sites/hocus.io/.')
		run('cp -r ~/Sites/hocus.io/settings.prod.php ~/Sites/hocus.io/settings.php')<commit_msg>changed domain<commit_after>from fabric.api import *

env.user = 'ptigas'
env.hosts = ['hocus.io']

def deploy():
	with cd('repos/hocus'):
		run('git pull')

	with cd('repos/hocus/webapp'):
		run('composer install')

	with cd('repos/hocus'):
		run('cp -r webapp/* ~/Sites/hocus.io/')
		run('cp -r vendor ~/Sites/hocus.io/.')
		run('cp -r ~/Sites/hocus.io/settings.prod.php ~/Sites/hocus.io/settings.php')
<|endoftext|>


<commit_before>import os
import sys
import json

from fabric.api import local, hosts, cd, env, prefix, run, sudo

def kobo_workon(venv_name):
    return prefix('kobo_workon %s' % venv_name)

DEPLOYMENTS = {}
IMPORTED_DEPLOYMENTS = {}
if os.path.exists('deployments.json'):
    with open('deployments.json', 'r') as f:
        IMPORTED_DEPLOYMENTS = json.load(f)

def exit_with_error(message):
    print message
    sys.exit(1)

def check_key_filename(deployment_configs):
    if 'key_filename' in deployment_configs and \
       not os.path.exists(deployment_configs['key_filename']):
        exit_with_error("Cannot find required permissions file: %s" %
                        deployment_configs['key_filename'])

def setup_env(deployment_name):
    deployment = DEPLOYMENTS.get(deployment_name, {})

    if 'shared' in IMPORTED_DEPLOYMENTS:
        deployment.update(IMPORTED_DEPLOYMENTS['shared'])

    if deployment_name in IMPORTED_DEPLOYMENTS:
        deployment.update(IMPORTED_DEPLOYMENTS[deployment_name])

    env.update(deployment)
    check_key_filename(deployment)

    env.virtualenv = os.path.join('/home', 'ubuntu', '.virtualenvs',
                                  env.kf_virtualenv_name, 'bin', 'activate')
    env.uwsgi_pidfile = os.path.join('/home', 'ubuntu', 'pids',
                                  'kobo-uwsgi-master.pid')
    env.kf_path = os.path.join(env.home, env.kf_path)
    env.pip_requirements_file = os.path.join(env.kf_path,
                                             'requirements.txt')

def deploy(deployment_name, branch='master'):
    setup_env(deployment_name)
    with cd(env.kf_path):
        run("git fetch origin")
        run("git checkout origin/%s" % branch)
        run('find . -name "*.pyc" -exec rm -rf {} \;')
        run('find . -type d -empty -delete')

    with kobo_workon(env.kf_virtualenv_name):
        run("pip install -r %s" % env.pip_requirements_file)

    with cd(env.kf_path):
        run("npm install")
        run("bower install")
        run("grunt build_all")

        with kobo_workon(env.kf_virtualenv_name):
            # run("echo 'from django.contrib.auth.models import User; print User.objects.count()' | python manage.py shell")
            run("python manage.py syncdb --all")
            run("python manage.py migrate")
            run("python manage.py compress")
            run("python manage.py collectstatic --noinput")

    run("sudo service uwsgi reload")
<commit_msg>pulling in configs from an env variable specified file<commit_after>import os
import sys
import json

from fabric.api import local, hosts, cd, env, prefix, run, sudo

def kobo_workon(venv_name):
    return prefix('kobo_workon %s' % venv_name)

DEPLOYMENTS = {}
IMPORTED_DEPLOYMENTS = {}
deployments_file = os.environ.get('DEPLOYMENTS_JSON', 'deployments.json')
if os.path.exists(deployments_file):
    with open(deployments_file, 'r') as f:
        IMPORTED_DEPLOYMENTS = json.load(f)

def exit_with_error(message):
    print message
    sys.exit(1)

def check_key_filename(deployment_configs):
    if 'key_filename' in deployment_configs and \
       not os.path.exists(deployment_configs['key_filename']):
        exit_with_error("Cannot find required permissions file: %s" %
                        deployment_configs['key_filename'])

def setup_env(deployment_name):
    deployment = DEPLOYMENTS.get(deployment_name, {})

    if 'shared' in IMPORTED_DEPLOYMENTS:
        deployment.update(IMPORTED_DEPLOYMENTS['shared'])

    if deployment_name in IMPORTED_DEPLOYMENTS:
        deployment.update(IMPORTED_DEPLOYMENTS[deployment_name])

    env.update(deployment)
    check_key_filename(deployment)

    env.virtualenv = os.path.join('/home', 'ubuntu', '.virtualenvs',
                                  env.kf_virtualenv_name, 'bin', 'activate')
    env.uwsgi_pidfile = os.path.join('/home', 'ubuntu', 'pids',
                                  'kobo-uwsgi-master.pid')
    env.kf_path = os.path.join(env.home, env.kf_path)
    env.pip_requirements_file = os.path.join(env.kf_path,
                                             'requirements.txt')

def deploy(deployment_name, branch='master'):
    setup_env(deployment_name)
    with cd(env.kf_path):
        run("git fetch origin")
        run("git checkout origin/%s" % branch)
        run('find . -name "*.pyc" -exec rm -rf {} \;')
        run('find . -type d -empty -delete')

    with kobo_workon(env.kf_virtualenv_name):
        run("pip install -r %s" % env.pip_requirements_file)

    with cd(env.kf_path):
        run("npm install")
        run("bower install")
        run("grunt build_all")

        with kobo_workon(env.kf_virtualenv_name):
            # run("echo 'from django.contrib.auth.models import User; print User.objects.count()' | python manage.py shell")
            run("python manage.py syncdb --all")
            run("python manage.py migrate")
            run("python manage.py compress")
            run("python manage.py collectstatic --noinput")

    run("sudo service uwsgi reload")
<|endoftext|>