<commit_before>#!/usr/bin/env python
#
#    Licensed under the Apache License, Version 2.0 (the "License"); you may
#    not use this file except in compliance with the License. You may obtain
#    a copy of the License at
#
#         http://www.apache.org/licenses/LICENSE-2.0
#
#    Unless required by applicable law or agreed to in writing, software
#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
#    License for the specific language governing permissions and limitations
#    under the License.

import json
import logging
import os
import subprocess
import sys

# Ideally this path would be /var/lib/heat-cfntools/cfn-init-data
# but this is where all boot metadata is stored
LAST_METADATA_DIR = os.environ.get('HEAT_CFN_INIT_LAST_METADATA_DIR',
                                   '/var/cache/heat-cfntools')

CFN_INIT_CMD = os.environ.get('HEAT_CFN_INIT_CMD',
                              '/opt/aws/bin/cfn-init')

def main(argv=sys.argv, stdin=sys.stdin, stdout=sys.stdout, stderr=sys.stderr):
    log = logging.getLogger('heat-config')
    handler = logging.StreamHandler(stderr)
    handler.setFormatter(
        logging.Formatter(
            '[%(asctime)s] (%(name)s) [%(levelname)s] %(message)s'))
    log.addHandler(handler)
    log.setLevel('DEBUG')

    c = json.load(stdin)

    config = c.get('config', {})
    if not isinstance(config, dict):
        config = json.loads(config)
    meta = {'AWS::CloudFormation::Init': config}

    if not os.path.isdir(LAST_METADATA_DIR):
        os.makedirs(LAST_METADATA_DIR, 0o700)

    fn = os.path.join(LAST_METADATA_DIR, 'last_metadata')
    with os.fdopen(os.open(fn, os.O_CREAT | os.O_WRONLY | os.O_TRUNC, 0o700),
                   'w') as f:
        json.dump(meta, f)

    log.debug('Running %s' % CFN_INIT_CMD)
    subproc = subprocess.Popen([CFN_INIT_CMD], stdout=subprocess.PIPE,
                               stderr=subprocess.PIPE)
    cstdout, cstderr = subproc.communicate()

    if cstdout:
        log.info(cstdout)
    if cstderr:
        log.info(cstderr)

    if subproc.returncode:
        log.error("Error running %s. [%s]\n" % (
            CFN_INIT_CMD, subproc.returncode))
    else:
        log.info('Completed %s' % CFN_INIT_CMD)

    response = {
        'deploy_stdout': cstdout,
        'deploy_stderr': cstderr,
        'deploy_status_code': subproc.returncode,
    }

    json.dump(response, stdout)

if __name__ == '__main__':
    sys.exit(main())
<commit_msg>Use relative path for cfn-init<commit_after>#!/usr/bin/env python
#
#    Licensed under the Apache License, Version 2.0 (the "License"); you may
#    not use this file except in compliance with the License. You may obtain
#    a copy of the License at
#
#         http://www.apache.org/licenses/LICENSE-2.0
#
#    Unless required by applicable law or agreed to in writing, software
#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
#    License for the specific language governing permissions and limitations
#    under the License.

import json
import logging
import os
import subprocess
import sys

# Ideally this path would be /var/lib/heat-cfntools/cfn-init-data
# but this is where all boot metadata is stored
LAST_METADATA_DIR = os.environ.get('HEAT_CFN_INIT_LAST_METADATA_DIR',
                                   '/var/cache/heat-cfntools')

CFN_INIT_CMD = os.environ.get('HEAT_CFN_INIT_CMD',
                              'cfn-init')

def main(argv=sys.argv, stdin=sys.stdin, stdout=sys.stdout, stderr=sys.stderr):
    log = logging.getLogger('heat-config')
    handler = logging.StreamHandler(stderr)
    handler.setFormatter(
        logging.Formatter(
            '[%(asctime)s] (%(name)s) [%(levelname)s] %(message)s'))
    log.addHandler(handler)
    log.setLevel('DEBUG')

    c = json.load(stdin)

    config = c.get('config', {})
    if not isinstance(config, dict):
        config = json.loads(config)
    meta = {'AWS::CloudFormation::Init': config}

    if not os.path.isdir(LAST_METADATA_DIR):
        os.makedirs(LAST_METADATA_DIR, 0o700)

    fn = os.path.join(LAST_METADATA_DIR, 'last_metadata')
    with os.fdopen(os.open(fn, os.O_CREAT | os.O_WRONLY | os.O_TRUNC, 0o700),
                   'w') as f:
        json.dump(meta, f)

    log.debug('Running %s' % CFN_INIT_CMD)
    subproc = subprocess.Popen([CFN_INIT_CMD], stdout=subprocess.PIPE,
                               stderr=subprocess.PIPE)
    cstdout, cstderr = subproc.communicate()

    if cstdout:
        log.info(cstdout)
    if cstderr:
        log.info(cstderr)

    if subproc.returncode:
        log.error("Error running %s. [%s]\n" % (
            CFN_INIT_CMD, subproc.returncode))
    else:
        log.info('Completed %s' % CFN_INIT_CMD)

    response = {
        'deploy_stdout': cstdout,
        'deploy_stderr': cstderr,
        'deploy_status_code': subproc.returncode,
    }

    json.dump(response, stdout)

if __name__ == '__main__':
    sys.exit(main())
<|endoftext|>


<commit_before>import logging
import cherrypy
import time
from sqlalchemy import event
from sqlalchemy.engine import Engine

def debug(msg):
    cherrypy.log.error(msg, context='timers', severity=logging.DEBUG)

@event.listens_for(Engine, "before_cursor_execute")
def before_cursor_execute(conn, cursor, statement, parameters, context, executemany):
    context._query_start_time = time.time()
    debug("start query: %s" % statement)

@event.listens_for(Engine, "after_cursor_execute")
def after_cursor_execute(conn, cursor, statement, parameters, context, executemany):
    total = time.time() - context._query_start_time

    if hasattr(cherrypy.request, '_timers_query_time'):
        cherrypy.request._timers_query_time += total

    if hasattr(cherrypy.request, '_timers_total_queries'):
        cherrypy.request._timers_total_queries += 1

    debug("end query, time: %f" % total)

def timers_start():
    cherrypy.request._timers_query_time = 0
    cherrypy.request._timers_total_queries = 0
    cherrypy.request._timers_total_time = time.time()

def timers_end():
    total_time = time.time() - cherrypy.request._timers_total_time
    total_queries = cherrypy.request._timers_total_queries
    query_time = cherrypy.request._timers_query_time

    debug("request ended: %f query time, %d queries, %f total time." %
        (query_time, total_queries, total_time))

timers_start_tool = cherrypy.Tool('on_start_resource', timers_start)
timers_end_tool = cherrypy.Tool('on_end_request', timers_end)

<commit_msg>pep8 fix<commit_after>import logging
import cherrypy
import time
from sqlalchemy import event
from sqlalchemy.engine import Engine

def debug(msg):
    cherrypy.log.error(msg, context='timers', severity=logging.DEBUG)

@event.listens_for(Engine, "before_cursor_execute")
def before_cursor_execute(conn, cursor, statement, parameters, context, executemany):
    context._query_start_time = time.time()
    debug("start query: %s" % statement)

@event.listens_for(Engine, "after_cursor_execute")
def after_cursor_execute(conn, cursor, statement, parameters, context, executemany):
    total = time.time() - context._query_start_time

    if hasattr(cherrypy.request, '_timers_query_time'):
        cherrypy.request._timers_query_time += total

    if hasattr(cherrypy.request, '_timers_total_queries'):
        cherrypy.request._timers_total_queries += 1

    debug("end query, time: %f" % total)

def timers_start():
    cherrypy.request._timers_query_time = 0
    cherrypy.request._timers_total_queries = 0
    cherrypy.request._timers_total_time = time.time()

def timers_end():
    total_time = time.time() - cherrypy.request._timers_total_time
    total_queries = cherrypy.request._timers_total_queries
    query_time = cherrypy.request._timers_query_time

    debug("request ended: %f query time, %d queries, %f total time." %
          (query_time, total_queries, total_time))

timers_start_tool = cherrypy.Tool('on_start_resource', timers_start)
timers_end_tool = cherrypy.Tool('on_end_request', timers_end)
<|endoftext|>


<commit_before># Copyright 2013-2020 Lawrence Livermore National Security, LLC and other
# Spack Project Developers. See the top-level COPYRIGHT file for details.
#
# SPDX-License-Identifier: (Apache-2.0 OR MIT)
from spack import *

class PyHumanfriendly(PythonPackage):
    """Human friendly output for text interfaces using Python"""

    homepage = "https://humanfriendly.readthedocs.io/"
    url      = "https://pypi.io/packages/source/h/humanfriendly/humanfriendly-8.1.tar.gz"

    version('8.1',  sha256='25c2108a45cfd1e8fbe9cdb30b825d34ef5d5675c8e11e4775c9aedbfb0bdee2')
    version('4.18', sha256='33ee8ceb63f1db61cce8b5c800c531e1a61023ac5488ccde2ba574a85be00a85')

    depends_on('python@2.7:2.8,3.5:', type=('build', 'run'))
    depends_on('py-setuptools', type='build')
    depends_on('py-monotonic', when='^python@:2', type=('build', 'run'))
<commit_msg>py-humanfriendly: add version 8.2 (#19073)<commit_after># Copyright 2013-2020 Lawrence Livermore National Security, LLC and other
# Spack Project Developers. See the top-level COPYRIGHT file for details.
#
# SPDX-License-Identifier: (Apache-2.0 OR MIT)
from spack import *

class PyHumanfriendly(PythonPackage):
    """Human friendly output for text interfaces using Python"""

    homepage = "https://humanfriendly.readthedocs.io/"
    url      = "https://pypi.io/packages/source/h/humanfriendly/humanfriendly-8.1.tar.gz"

    version('8.2',  sha256='bf52ec91244819c780341a3438d5d7b09f431d3f113a475147ac9b7b167a3d12')
    version('8.1',  sha256='25c2108a45cfd1e8fbe9cdb30b825d34ef5d5675c8e11e4775c9aedbfb0bdee2')
    version('4.18', sha256='33ee8ceb63f1db61cce8b5c800c531e1a61023ac5488ccde2ba574a85be00a85')

    depends_on('python@2.7:2.8,3.5:', type=('build', 'run'))
    depends_on('py-setuptools', type='build')
    depends_on('py-monotonic', when='^python@:2', type=('build', 'run'))
<|endoftext|>


<commit_before>import sys
reload(sys)
sys.setdefaultencoding('utf-8')

import re
import os
import json
import markdown
from datetime import date

from fabric.operations import prompt
from fabric.api import run
from fabric.colors import green as _green, yellow as _yellow, red as _red

from jinja2 import Environment, FileSystemLoader

def add():
    '''
    Add a new app
    '''
    print _green("--- Generating new app for Kippt App Gallery")

    name = prompt("App name:", default='', validate=r'^[\w\s-]+$')
    developer = prompt("Developer (your name/organization):", default='')
    platform = prompt("Platform (e.g. iOS):", default='')
    price = prompt("Price (e.g. $4):", default='')
    added = date.today().strftime('%B%e. %Y')
    link = prompt("Link (e.g. App Store or website):", default='')

    website = prompt("Website:", default='')
    twitter = prompt("Twitter (e.g. @getappname):", default='')

    data = {
        'name': name,
        'developer': developer,
        'platform': platform,
        'price': price,
        'added': added,
        'link': link,
        'website': website,
        'twitter': twitter,
    }

    # Create directory
    slug = re.sub('[^\w\s-]', '', name).strip().lower()
    slug = re.sub('[-\s]+', '-', slug)
    if not os.path.exists('apps/%s' % slug):
        os.makedirs('apps/%s' % slug)

    manifest = open("apps/%s/manifest.json" % slug, "w")
    manifest.write(json.dumps(data, indent=4))
    manifest.close()

    manifest = open("apps/%s/description.md" % slug, "w")
    manifest.write('This is %s.\n\nIt\'s time to add a markdown formatted description for the app.' % name)
    manifest.close()

    print _green('--- Saved to /apps/%s/' % slug)
    print _green('--- Remember to edit description.md and convert to html with')
    print _yellow('---     fab compile:%s' % slug)

def compile(slug):
    '''
    Compile app template into HTML
    '''
    if not os.path.exists('apps/%s' % slug):
        print _red('--- App with this slug can\'t be found(%s)' % slug)
        return

    env = Environment(loader=FileSystemLoader('templates'))
    template = env.get_template('apps/template.html')

    # Manifest
    data = ''
    with open ("apps/%s/manifest.json" % slug, "r") as manifest:
        data = manifest.read().replace('\n', '')
    manifest = json.loads(data)

    # Description
    description = ''
    with open ("apps/%s/description.md" % slug, "r") as desc_file:
        description = desc_file.read().replace('\n', '')

    context = manifest
    context['description']= markdown.markdown(description)

    # Write output
    if not os.path.exists('templates/apps/%s' % slug):
        os.makedirs('templates/apps/%s' % slug)
    else:
        # remove old files
        open("templates/apps/%s/index.html" % slug, "w").close()
    output = open("templates/apps/%s/index.html" % slug, "w")
    output.write('{% extends "base.html" %}\n{% block body %}\n')
    output.write(template.render(context))
    output.write('\n{% endblock %}')
    output.close()

    <commit_msg>Better app adding via Fabric<commit_after>import sys
reload(sys)
sys.setdefaultencoding('utf-8')

import re
import os
import json
import markdown
from datetime import date

from fabric.operations import prompt
from fabric.api import run
from fabric.colors import green as _green, yellow as _yellow, red as _red

from jinja2 import Environment, FileSystemLoader

def add():
    '''
    Add a new app
    '''
    print _green("--- Generating new app for Kippt App Gallery")

    name = prompt("App name:", default='', validate=r'^[\w\s-]+$')
    developer = prompt("Developer (your name/organization):", validate=r'^[\w\s-]+$')
    developer_website = prompt("(Optional) Developer website (or Twitter address):", default='')
    print 'Platform:\n[1] Web\n[2] iPhone\n[3] Android\n[4] Windows Phone\n[4] Desktop\n[5] Library\n[6] Other\n'
    platform = prompt("Platform:", default='1', validate=r'^[1-6]+$')
    price = prompt("(Optional) Price (e.g. $4):", default='')
    link = prompt("(Optional) Link (e.g. App Store or website):", default='')
    added = date.today().strftime('%B%e. %Y')

    website = prompt("(Optional) Website:", default='')
    twitter = prompt("(Optional) Twitter (e.g. @getappname):", default='')

    data = {
        'name': name,
        'developer': developer,
        'developer_website': developer_website,
        'platform': platform,
        'price': price,
        'added': added,
        'link': link,
        'website': website,
        'twitter': twitter,
    }

    # Create directory
    slug = re.sub('[^\w\s-]', '', name).strip().lower()
    slug = re.sub('[-\s]+', '-', slug)
    if not os.path.exists('apps/%s' % slug):
        os.makedirs('apps/%s' % slug)
        os.makedirs('apps/%s/images' % slug)

    manifest = open("apps/%s/manifest.json" % slug, "w")
    manifest.write(json.dumps(data, indent=4))
    manifest.close()

    manifest = open("apps/%s/description.md" % slug, "w")
    manifest.write('This is %s.\n\nIt\'s time to add a markdown formatted description for the app.' % name)
    manifest.close()

    print _green('--- Add app description to /apps/%s/description.md' % slug)
    prompt("Press enter to continue", default='')

    print _green('--- (Optional) Add logo (logo.png/logo.jpg, 256x256px) to /apps/%s/images/' % slug)
    prompt("Press enter to continue", default='')

    print _green('--- (Optional) Add screenshots (JPG/PNG, max. 1024x1024 & 1MB) to /apps/%s/images/' % slug)
    prompt("Press enter to continue", default='')

    print _green('--- Saved to /apps/%s/' % slug)
    print _yellow('---     fab compile:%s' % slug)

def compile(slug):
    '''
    Compile app template into HTML
    '''
    if not os.path.exists('apps/%s' % slug):
        print _red('--- App with this slug can\'t be found(%s)' % slug)
        return

    env = Environment(loader=FileSystemLoader('templates'))
    template = env.get_template('apps/template.html')

    # Manifest
    data = ''
    with open ("apps/%s/manifest.json" % slug, "r") as manifest:
        data = manifest.read().replace('\n', '')
    context = json.loads(data)

    # Description
    description = ''
    with open ("apps/%s/description.md" % slug, "r") as desc_file:
        description = desc_file.read()

    context['description']= markdown.markdown(description)

    # Write output
    if not os.path.exists('templates/apps/%s' % slug):
        os.makedirs('templates/apps/%s' % slug)
    else:
        # remove old files
        open("templates/apps/%s/index.html" % slug, "w").close()
    output = open("templates/apps/%s/index.html" % slug, "w")
    output.write('{% extends "base.html" %}\n{% block body %}\n')
    output.write(template.render(context))
    output.write('\n{% endblock %}')
    output.close()

    <|endoftext|>


<commit_before>from bs4 import BeautifulSoup
from collections import Counter
from time import sleep
import math
import matplotlib.pyplot as pyplot
import probability
import re
import requests
import random

def is_video(td):
    """It's a video if it has exactly one price label
    and if the stripped text inside that label starts with 'Video'"""
    pricelabels = td('span', 'pricelabel')
    return (len(pricelabels) == 1 and
            pricelabels[0].text.strip().startswith("video"))

def book_info(td):
    """Given a BeautifulSoup <td> representing a book,
    extract the book's details and return a dict"""
    title = td.find("div", "thumbheader").a.text
    by_author = td.find("div", "AuthorName").text
    authors = [x.strip() for x in re.sub("^By ", "", by_author).split(",")]
    isbn_link = td.find("div", "thumbheader").a.get("href")
    isbn = re.match("/product/(.*)\.do", isbn_link).groups()[0]
    date = td.find("span", "directorydate").text.strip()

    return {
        "title": title,
        "authors": authors,
        "isbn": isbn,
        "date": date,
    }

def scrape(num_pages=1):
    base_url = "http://shop.oreilly.com/category/browse-subjects/" + \
        "data.do?sortby=publicationDate&page="

    books = []
    for page_num in range(1, num_pages + 1):
        print "scraping page", page_num
        url = base_url + str(page_num)
        soup = BeautifulSoup(requests.get(url).text, 'html5lib')

        for td in soup('td', 'thumbtext'):
            if not is_video(td):
                books.append(book_info(td))
        sleep(30)

    return books

def get_year(book):
    """Format is 'November 1999' so
    split on space and take second element"""
    return int(book["date"].split()[-1])

def plot_years(books, asset="/tmp/oreilly-books.png"):
    year_counts = Counter(get_year(book) for book in books)

    years = sorted(year_counts)
    book_counts = [year_counts[year] for year in years]

    print "year_counts: {}".format(year_counts)
    print "book_counts: {}".format(book_counts)

    pyplot.bar([x - 0.5 for x in years], book_counts)
    pyplot.xlabel("years")
    pyplot.ylabel("# of data books")
    pyplot.title("Data grows up")
    pyplot.savefig(asset)

def bucketize(point, bucket_size):
    """floor the point to the next lower multiple of bucket size"""
    return bucket_size * math.floor(point / bucket_size)

def make_histogram(points, bucket_size):
    """buckets the points and counts how many in each bucket"""
    return Counter(bucketize(point, bucket_size) for point in points)

def plot_histogram(points, bucket_size, title="", asset="histogram.png"):
    """generate plot for bucketed data"""
    histogram = make_histogram(points, bucket_size)
    pyplot.bar(histogram.keys(), histogram.values(), width=bucket_size)
    pyplot.title(title)
    pyplot.savefig(asset)

if __name__ == '__main__':
    test = 'HISTOGRAM'

    if test == 'SCRAPE':
        books = scrape(6)
        for book in books:
            print "->{0}".format(book)
        plot_years(books)

    if test == 'HISTOGRAM':
        random.seed(0)
        uniform = [200 * random.random() - 100 for _ in range(10000)]
        normal = [57 * probability.inverse_normal_cdf(random.random())
                  for _ in range(10000)]

        plot_histogram(uniform, 10, 'Uniform Histogram', 'uniform.png')
        plot_histogram(normal, 10, 'Normal Histogram', 'normal.png')
<commit_msg>Can parse github, can scatter random_normal<commit_after>from bs4 import BeautifulSoup
from collections import Counter
from time import sleep
import math
import matplotlib.pyplot as pyplot
import probability
import re
import requests
import random
import json
import algebra
import stats

def is_video(td):
    """It's a video if it has exactly one price label
    and if the stripped text inside that label starts with 'Video'"""
    pricelabels = td('span', 'pricelabel')
    return (len(pricelabels) == 1 and
            pricelabels[0].text.strip().startswith("video"))

def book_info(td):
    """Given a BeautifulSoup <td> representing a book,
    extract the book's details and return a dict"""
    title = td.find("div", "thumbheader").a.text
    by_author = td.find("div", "AuthorName").text
    authors = [x.strip() for x in re.sub("^By ", "", by_author).split(",")]
    isbn_link = td.find("div", "thumbheader").a.get("href")
    isbn = re.match("/product/(.*)\.do", isbn_link).groups()[0]
    date = td.find("span", "directorydate").text.strip()

    return {
        "title": title,
        "authors": authors,
        "isbn": isbn,
        "date": date,
    }

def scrape(num_pages=1):
    base_url = "http://shop.oreilly.com/category/browse-subjects/" + \
        "data.do?sortby=publicationDate&page="

    books = []
    for page_num in range(1, num_pages + 1):
        print "scraping page", page_num
        url = base_url + str(page_num)
        soup = BeautifulSoup(requests.get(url).text, 'html5lib')

        for td in soup('td', 'thumbtext'):
            if not is_video(td):
                books.append(book_info(td))
        sleep(30)

    return books

def get_year(book):
    """Format is 'November 1999' so
    split on space and take second element"""
    return int(book["date"].split()[-1])

def plot_years(books, asset="/tmp/oreilly-books.png"):
    year_counts = Counter(get_year(book) for book in books)

    years = sorted(year_counts)
    book_counts = [year_counts[year] for year in years]

    print "year_counts: {}".format(year_counts)
    print "book_counts: {}".format(book_counts)

    pyplot.bar([x - 0.5 for x in years], book_counts)
    pyplot.xlabel("years")
    pyplot.ylabel("# of data books")
    pyplot.title("Data grows up")
    pyplot.savefig(asset)

def github_data(username="joelgrus"):
    endpoint = "https://api.github.com/users/{u}/repos".format(u=username)
    raw_json = requests.get(endpoint).text
    repos = json.loads(raw_json)
    return repos

def languages_for(repo_json, n=5):
    last_n_repos = sorted(repo_json,
                          key=lambda r: r['created_at'],
                          reverse=True)[0:n]
    languages = [repo['language'] for repo in last_n_repos]
    return languages

def bucketize(point, bucket_size):
    """floor the point to the next lower multiple of bucket size"""
    return bucket_size * math.floor(point / bucket_size)

def make_histogram(points, bucket_size):
    """buckets the points and counts how many in each bucket"""
    return Counter(bucketize(point, bucket_size) for point in points)

def plot_histogram(points, bucket_size, title="", asset="histogram.png"):
    """generate plot for bucketed data"""
    histogram = make_histogram(points, bucket_size)
    pyplot.bar(histogram.keys(), histogram.values(), width=bucket_size)
    pyplot.title(title)
    pyplot.savefig(asset)

def random_normal():
    """returns a random draw from a standard normal distribution"""
    return probability.inverse_normal_cdf(random.random())

def correlation_matrix(data):
    """returns the num_columns x num_columns matrix whose (i, j)th entry
    is the correlation between columns i and j of data"""
    _, num_columns = algebra.shape(data)

    def matrix_entry(i, j):
        return stats.correlation(algebra.get_column(data, i),
                                 algebra.get_column(data, j))

    return algebra.mk_matrix(num_columns, num_columns, matrix_entry)

if __name__ == '__main__':
    test = 'SCATTER'

    if test == 'SCRAPE':
        books = scrape(6)
        for book in books:
            print "->{0}".format(book)
        plot_years(books)

    if test == 'HISTOGRAM':
        random.seed(0)
        uniform = [200 * random.random() - 100 for _ in range(10000)]
        normal = [57 * probability.inverse_normal_cdf(random.random())
                  for _ in range(10000)]

        plot_histogram(uniform, 10, 'Uniform Histogram', 'uniform.png')
        plot_histogram(normal, 10, 'Normal Histogram', 'normal.png')

    if test == 'GITHUB':
        user = 'mjamesruggiero'
        repo = github_data(user)
        languages = languages_for(repo, 10)
        for l in languages:
            print "* {0}".format(l)

    if test == 'SCATTER':
        asset = 'scatter.png'
        xs = [random_normal() for _ in range(1000)]
        ys1 = [x + random_normal() / 2 for x in xs]
        ys2 = [-x + random_normal() / 2 for x in xs]

        pyplot.scatter(xs, ys1, marker='.', color='black', label='ys1')
        pyplot.scatter(xs, ys2, marker='.', color='gray', label='ys2')
        pyplot.xlabel('xs')
        pyplot.ylabel('ys')
        pyplot.legend(loc=9)
        pyplot.savefig(asset)
<|endoftext|>


<commit_before># Fabfile for deploying Project Bluebottle.
#
# This file is structured as follows:
#
#    1. Environment settings
#    2. Utility functions
#    3. Fabric tasks

from git import Repo
from fabric.api import env, roles, sudo, prefix, cd, task, require, run, local, put, prompt, abort
from fabric.contrib.console import confirm
from fabric.colors import green, red
from fabric.operations import get
from contextlib import contextmanager

# Configuration settings:
# If True, enables forwarding of your local SSH agent to the remote end.
env.forward_agent = True

# Assign the hosts to roles, for convenient reference and scaling
env.roledefs = {
    'production': ['production.onepercentclub.com'],
    'staging': ['staging.onepercentclub.com'],
    'testing': ['testing.onepercentclub.com'],
    'dev': ['dev.onepercentclub.com'],
    'backup': ['backups@bluebucket.onepercentclub.com']
}

# Admin user
env.user = 'onepercentadmin'

# User running the web service
env.web_user = 'onepercentsite'

# Directory (on the server) where our project will be running
env.directory = '/var/www/onepercentsite'

# Name of supervisor service
env.service_name = 'onepercentsite'

# Name of the database
env.database = 'onepercentsite'

# By default, confirm everywhere
env.noinput = False

# Utility functions:
@contextmanager
def virtualenv():
    """
    Make sure everything is executed from within the virtual environment.

    Example::

        with virtualenv():
            run('./manage.py collectstatic')

    """
    require('directory')

    with cd(env.directory):
        with prefix('source env/bin/activate'):
            yield

def set_django_settings():
    """ Environment-dependant Django settings. """
    require('host')
    environment = env.host.split('.', 1)[0]
    env.django_settings = 'onepercentclub.settings.%s' % environment

def run_web(*args, **kwargs):
    """ Run a command as the web user. """
    require('web_user')

    kwargs.setdefault('user', env.web_user)

    return sudo(*args, **kwargs)

def status_update(message):
    """ Print status update message. """

    print(green(message))

def get_commit_tags(commit):
    """ Get all tags for a commit. """

    r = Repo()

    tags = set()

    for tag in r.tags:
        if tag.commit == commit:
            tags.add(tag.name)

    return tags

def describe_commit(commit):
    """
    Return a verbose commit name based on shortened hash, tags and summary.
    """

    tags = ' '.join(get_commit_tags(commit))

    if tags:
        return '%s %s: %s' % (commit.hexsha[:7], tags, commit.summary)
    else:
        return '%s: %s' % (commit.hexsha[:7], commit.summary)

def get_commit(revspec):
    """
    Find the specified commit by revspec.
    """

    r = Repo()

    # Get the commit from the repo
    commit = r.commit(revspec)
    status_update('Deploying commit %s' % describe_commit(commit))

    return commit

def tag_commit(commit_id, tag):
    """
    Tag the specified commit and push it to the server.
    Overwriting old tags with the same name.
    """
    local('git tag %s %s' % (tag, commit_id))
    local('git push --tags origin %s' % tag)

def make_versioned_tag(tag, version):
    """ Generate a versioned version of a tag. """

    return '%s-%d' % (tag, version)

def find_latest_tag_version(tag):
    """ Find the latest version for the given tag. Returns an integer. """

    r = Repo()

    version = 1
    while make_versioned_tag(tag, version+1) in r.tags:
        version += 1

    status_update(
        'Latest version for tag %s is %d: %s' % \
            (tag, version, make_versioned_tag(tag, version))
    )

    return version

def find_available_tag(tag):
    """
    Find the latest available version for a versioned tag of the form tag-N.
    """

    latest_version = find_latest_tag_version(tag)

    new_tag = make_versioned_tag(tag, latest_version+1)

    return new_tag

def confirm_wrong_tag(commit, tag):
    """ Confirm deployment when commit does not have expected tag. """
    require('noinput')

    print(red('WARNING: This commit does not have the %s tag.' % tag))

    if not env.noinput:
        confirmed = confirm('Are you really sure you want to deploy %s?' % commit.hexsha, default=False)

        if not confirmed:
            abort('Confirmation required to continue.')

def prompt_role():
    """
    Query the user for the role (and thus host) to use and store results in env.
    """
    require('roledefs')

    if not env.host_string:
        role_options = ', '.join([role for role in env.roledefs.keys()])

        def validate(value):
            if not value in env.roledefs.keys():
                raise Exception('Role should be one of %s' % role_options)

            return value

        role = prompt('Please specify a role (one of %s): ' % role_options, validate=validate)

        # Warning: this is a rather hackish solution - Fabric doesn't seem
        # to like the hosts for a task to be changed in the middle of a
        # process. This works ok, but limits us to one host per role.
        assert len(env.roledefs[role]) == 1, \
            'This is a hackish solution that does not work for a role with multiple hosts.'

        env.host = env.roledefs[role][0]
        env.host_string = env.host

def assert_release_tag(commit, tag):
    """
    Make sure the given commit has the given tag and confirm if not so.
    """

    tags = get_commit_tags(commit)

    # Match a single string as to ignore versions in "tag-<VERSION>"
    if not tag in ' '.join(tags):
        # It's not found, confirm with user
        confirm_wrong_tag(commit, tag)

def git_fetch_local():
    """ Fetch local GIT updates. """
    local('git fetch -q')

def update_git(commit):
    """ Update the repo to given commit_id. """
    require('directory')

    status_update('Updating git repository to %s' % describe_commit(commit))

    with cd(env.directory):
        # Make sure only to fetch the required branch
        # This script should fail if we are updating to a non-deploy commit
        run('git fetch -q -p')
        run('git checkout -q %s' % commit.hexsha)

def update_tar(commit_id):
    """ Update the remote to given commit_id using tar. """
    require('directory')

    status_update('Transferring archive of commit %s.' % commit_id)

    filename = '%s.tbz2' % commit_id
    local('git archive %s | bzip2 -c > %s' % (commit_id, filename))
    put(filename, env.directory)

    with cd(env.directory):
        run('tar xjf %s' % filename)
        run('rm %s' % filename)

    local('rm -f %s' % filename)

def prune_unreferenced_files():
    """
    Prune unreferenced files.

    WARNING: This deletes cached thumbnails as well. This should be fixed and
    not used until.
    """
    require('directory')

    # Find unreferenced files
    with virtualenv():
        unreferenced_files = run_web('./manage.py unreferenced_files --settings=%s' % \
            env.django_settings
        ).splitlines()

    # If found, prompt for deletion
    if unreferenced_files:
        status_update('Pruning %d unreferenced files' % len(unreferenced_files))

        for unref_file in unreferenced_files:
            run_web('rm %s' % unref_file)

def add_git_commit():
    with cd(env.directory):
        run('echo "GIT_COMMIT = \'`git log --oneline | head -n1 | cut -c1-7`\'" >> onepercentclub/settings/base.py')

def prepare_django():
    """ Prepare a deployment. """
    set_django_settings()

    require('django_settings')

    status_update('Preparing deployment.')

    with virtualenv():
        # TODO: Filter out the following messages:
        # "Could not find a tag or branch '<commit_id>', assuming commit."
        run('pip install -q --allow-all-external --allow-unverified django-admin-tools -r requirements/requirements.txt')

        # Remove and compile the .pyc files.
        run('find . -name \*.pyc -delete')
        run('./manage.py compile_pyc --settings=%s' % env.django_settings)

        # Prepare the translations.
        run('./translations.sh compile %s' % env.django_settings)

        # Disabled until the following problem is fixed:
        # ERROR: test_site_profile_not_available (django.contrib.auth.tests.models.ProfileTestCase)
        #run_web('./manage.py test -v 0')

        # Make sure the web user can read and write the static media dir.
        run('chmod a+rw static/media')

        run_web('./manage.py syncdb --migrate --noinput --settings=%s' % env.django_settings)
        run_web('./manage.py collectstatic --clear -l -v 0 --noinput --settings=%s' % env.django_settings)

        # Disabled for now; it unjustly deletes cached thumbnails
        # prune_unreferenced_files()

def restart_site():
    """ Gracefully restart gunicorn using supervisor. """
    require('service_name')

    run('supervisorctl restart %s' % env.service_name)

def set_site_domain():
    """ Set the site domain dependent on env.host. """

    status_update('Updating domain for default Django Site to %s' % env.host)

    set_django_settings()
    require('django_settings')
    require('host')

    if 'production' in env.host:
        host = 'onepercentclub.com'
    else:
        host = env.host

    sites_command = (
        "from django.contrib.sites.models import Site; "
        "site = Site.objects.get(pk=1); "
        "site.domain = '%s'; "
        "site.name = 'onepercentclub.com'; "
        "site.save()"
    ) % host

    with virtualenv():
        run_web('echo "%s" | ./manage.py shell --plain --settings=%s' % (
            sites_command, env.django_settings
        ))

@roles('dev')
@task
def deploy_dev(revspec='origin/master'):
    """
    Update the dev server to the specified revspec, or HEAD of deploy branch.
    """
    # Don't ask for confirmation
    env.noinput = True

    # Update git locally
    git_fetch_local()

    # Find commit for revspec
    commit = get_commit(revspec)

    # Make sure the remote git repo is up to date
    update_git(commit)

    # Get Django ready
    prepare_django()

    # Update site domain for self-reference to work
    set_site_domain()

    # Add the current git commit hash to the settings file
    add_git_commit()

    # Restart app server
    restart_site()

@roles('testing')
@task
def deploy_testing(revspec='origin/master'):
    """
    Update the testing server to the specified revspec, or HEAD of deploy branch and optionally sync migrated data.
    """
    # Update git locally
    git_fetch_local()

    # Find commit for revspec
    commit = get_commit(revspec)
    tag = find_available_tag('testing')

    # Make sure the remote git repo is up to date
    update_git(commit)

    # Get Django ready
    prepare_django()

    # Update site domain for self-reference to work
    set_site_domain()

    # Add the current git commit hash to the settings file
    add_git_commit()

    # Restart app server
    restart_site()

    # Deploy complete, tag commit
    tag_commit(commit.hexsha, tag)

@roles('staging')
@task
def deploy_staging(revspec=None):
    """
    Update the staging server to the specified revspec, or the latest testing release and optionally sync migrated data.
    """

    # Update git locally
    git_fetch_local()

    # Find revspec or latest testing release
    if not revspec:
        version = find_latest_tag_version('testing')
        revspec = make_versioned_tag('testing', version)

    # Find commit for revspec
    commit = get_commit(revspec)

    # Find latest available staging version
    tag = find_available_tag('staging')

    # Check whether this commit has been tested
    assert_release_tag(commit, 'testing')

    # Update the code
    update_tar(commit.hexsha)

    # Get Django ready
    prepare_django()

    # Update site domain for self-reference to work
    set_site_domain()

    # Restart app server
    restart_site()

    # Deploy complete, tag commit
    tag_commit(commit.hexsha, tag)

@roles('production')
@task
def deploy_production(revspec=None):
    """ Update the production server to the specified revspec, or the latest staging release. """

    # Update git locally
    git_fetch_local()

    # Find revspec or latest testing release
    if not revspec:
        version = find_latest_tag_version('staging')
        revspec = make_versioned_tag('staging', version)

    # Find commit for revspec
    commit = get_commit(revspec)

    # Find latest available staging version
    tag = find_available_tag('production')

    # Check whether this commit has been staged.
    assert_release_tag(commit, 'staging')

    # Update the code
    update_tar(commit.hexsha)

    # Get Django ready
    prepare_django()

    # Update site domain for self-reference to work
    set_site_domain()

    # Restart app server
    restart_site()

    # Deploy complete, tag commit
    tag_commit(commit.hexsha, tag)

@roles('backup')
@task
def get_db():
    backup_dir = "/home/backups/onepercentclub-backups/onepercentsite/current"
    with cd(backup_dir):
        output = run("ls *.bz2")
        try:
            filename = output.split()[0]
        except IndexError:
            print "No database backup file found"

        if filename:
            get(remote_path="{0}/{1}".format(backup_dir, filename), local_path="./dump.sql.bz2")
            unpack_db()

def unpack_db(filename="dump.sql.bz2"):
    try:
        local("gunzip {0}".format(filename))
    except IndexError:
        print "No database file found"

<commit_msg>add git reset so that local changes don't prevent a deploy<commit_after># Fabfile for deploying Project Bluebottle.
#
# This file is structured as follows:
#
#    1. Environment settings
#    2. Utility functions
#    3. Fabric tasks

from git import Repo
from fabric.api import env, roles, sudo, prefix, cd, task, require, run, local, put, prompt, abort
from fabric.contrib.console import confirm
from fabric.colors import green, red
from fabric.operations import get
from contextlib import contextmanager

# Configuration settings:
# If True, enables forwarding of your local SSH agent to the remote end.
env.forward_agent = True

# Assign the hosts to roles, for convenient reference and scaling
env.roledefs = {
    'production': ['production.onepercentclub.com'],
    'staging': ['staging.onepercentclub.com'],
    'testing': ['testing.onepercentclub.com'],
    'dev': ['dev.onepercentclub.com'],
    'backup': ['backups@bluebucket.onepercentclub.com']
}

# Admin user
env.user = 'onepercentadmin'

# User running the web service
env.web_user = 'onepercentsite'

# Directory (on the server) where our project will be running
env.directory = '/var/www/onepercentsite'

# Name of supervisor service
env.service_name = 'onepercentsite'

# Name of the database
env.database = 'onepercentsite'

# By default, confirm everywhere
env.noinput = False

# Utility functions:
@contextmanager
def virtualenv():
    """
    Make sure everything is executed from within the virtual environment.

    Example::

        with virtualenv():
            run('./manage.py collectstatic')

    """
    require('directory')

    with cd(env.directory):
        with prefix('source env/bin/activate'):
            yield

def set_django_settings():
    """ Environment-dependant Django settings. """
    require('host')
    environment = env.host.split('.', 1)[0]
    env.django_settings = 'onepercentclub.settings.%s' % environment

def run_web(*args, **kwargs):
    """ Run a command as the web user. """
    require('web_user')

    kwargs.setdefault('user', env.web_user)

    return sudo(*args, **kwargs)

def status_update(message):
    """ Print status update message. """

    print(green(message))

def get_commit_tags(commit):
    """ Get all tags for a commit. """

    r = Repo()

    tags = set()

    for tag in r.tags:
        if tag.commit == commit:
            tags.add(tag.name)

    return tags

def describe_commit(commit):
    """
    Return a verbose commit name based on shortened hash, tags and summary.
    """

    tags = ' '.join(get_commit_tags(commit))

    if tags:
        return '%s %s: %s' % (commit.hexsha[:7], tags, commit.summary)
    else:
        return '%s: %s' % (commit.hexsha[:7], commit.summary)

def get_commit(revspec):
    """
    Find the specified commit by revspec.
    """

    r = Repo()

    # Get the commit from the repo
    commit = r.commit(revspec)
    status_update('Deploying commit %s' % describe_commit(commit))

    return commit

def tag_commit(commit_id, tag):
    """
    Tag the specified commit and push it to the server.
    Overwriting old tags with the same name.
    """
    local('git tag %s %s' % (tag, commit_id))
    local('git push --tags origin %s' % tag)

def make_versioned_tag(tag, version):
    """ Generate a versioned version of a tag. """

    return '%s-%d' % (tag, version)

def find_latest_tag_version(tag):
    """ Find the latest version for the given tag. Returns an integer. """

    r = Repo()

    version = 1
    while make_versioned_tag(tag, version+1) in r.tags:
        version += 1

    status_update(
        'Latest version for tag %s is %d: %s' % \
            (tag, version, make_versioned_tag(tag, version))
    )

    return version

def find_available_tag(tag):
    """
    Find the latest available version for a versioned tag of the form tag-N.
    """

    latest_version = find_latest_tag_version(tag)

    new_tag = make_versioned_tag(tag, latest_version+1)

    return new_tag

def confirm_wrong_tag(commit, tag):
    """ Confirm deployment when commit does not have expected tag. """
    require('noinput')

    print(red('WARNING: This commit does not have the %s tag.' % tag))

    if not env.noinput:
        confirmed = confirm('Are you really sure you want to deploy %s?' % commit.hexsha, default=False)

        if not confirmed:
            abort('Confirmation required to continue.')

def prompt_role():
    """
    Query the user for the role (and thus host) to use and store results in env.
    """
    require('roledefs')

    if not env.host_string:
        role_options = ', '.join([role for role in env.roledefs.keys()])

        def validate(value):
            if not value in env.roledefs.keys():
                raise Exception('Role should be one of %s' % role_options)

            return value

        role = prompt('Please specify a role (one of %s): ' % role_options, validate=validate)

        # Warning: this is a rather hackish solution - Fabric doesn't seem
        # to like the hosts for a task to be changed in the middle of a
        # process. This works ok, but limits us to one host per role.
        assert len(env.roledefs[role]) == 1, \
            'This is a hackish solution that does not work for a role with multiple hosts.'

        env.host = env.roledefs[role][0]
        env.host_string = env.host

def assert_release_tag(commit, tag):
    """
    Make sure the given commit has the given tag and confirm if not so.
    """

    tags = get_commit_tags(commit)

    # Match a single string as to ignore versions in "tag-<VERSION>"
    if not tag in ' '.join(tags):
        # It's not found, confirm with user
        confirm_wrong_tag(commit, tag)

def git_fetch_local():
    """ Fetch local GIT updates. """
    local('git fetch -q')

def update_git(commit):
    """ Update the repo to given commit_id. """
    require('directory')

    status_update('Updating git repository to %s' % describe_commit(commit))

    with cd(env.directory):
        # Make sure only to fetch the required branch
        # This script should fail if we are updating to a non-deploy commit
        run('git fetch -q -p')
        run('git reset --hard')
        run('git checkout -q %s' % commit.hexsha)

def update_tar(commit_id):
    """ Update the remote to given commit_id using tar. """
    require('directory')

    status_update('Transferring archive of commit %s.' % commit_id)

    filename = '%s.tbz2' % commit_id
    local('git archive %s | bzip2 -c > %s' % (commit_id, filename))
    put(filename, env.directory)

    with cd(env.directory):
        run('tar xjf %s' % filename)
        run('rm %s' % filename)

    local('rm -f %s' % filename)

def prune_unreferenced_files():
    """
    Prune unreferenced files.

    WARNING: This deletes cached thumbnails as well. This should be fixed and
    not used until.
    """
    require('directory')

    # Find unreferenced files
    with virtualenv():
        unreferenced_files = run_web('./manage.py unreferenced_files --settings=%s' % \
            env.django_settings
        ).splitlines()

    # If found, prompt for deletion
    if unreferenced_files:
        status_update('Pruning %d unreferenced files' % len(unreferenced_files))

        for unref_file in unreferenced_files:
            run_web('rm %s' % unref_file)

def add_git_commit():
    with cd(env.directory):
        run('echo "GIT_COMMIT = \'`git log --oneline | head -n1 | cut -c1-7`\'" >> onepercentclub/settings/base.py')

def prepare_django():
    """ Prepare a deployment. """
    set_django_settings()

    require('django_settings')

    status_update('Preparing deployment.')

    with virtualenv():
        # TODO: Filter out the following messages:
        # "Could not find a tag or branch '<commit_id>', assuming commit."
        run('pip install -q --allow-all-external --allow-unverified django-admin-tools -r requirements/requirements.txt')

        # Remove and compile the .pyc files.
        run('find . -name \*.pyc -delete')
        run('./manage.py compile_pyc --settings=%s' % env.django_settings)

        # Prepare the translations.
        run('./translations.sh compile %s' % env.django_settings)

        # Disabled until the following problem is fixed:
        # ERROR: test_site_profile_not_available (django.contrib.auth.tests.models.ProfileTestCase)
        #run_web('./manage.py test -v 0')

        # Make sure the web user can read and write the static media dir.
        run('chmod a+rw static/media')

        run_web('./manage.py syncdb --migrate --noinput --settings=%s' % env.django_settings)
        run_web('./manage.py collectstatic --clear -l -v 0 --noinput --settings=%s' % env.django_settings)

        # Disabled for now; it unjustly deletes cached thumbnails
        # prune_unreferenced_files()

def restart_site():
    """ Gracefully restart gunicorn using supervisor. """
    require('service_name')

    run('supervisorctl restart %s' % env.service_name)

def set_site_domain():
    """ Set the site domain dependent on env.host. """

    status_update('Updating domain for default Django Site to %s' % env.host)

    set_django_settings()
    require('django_settings')
    require('host')

    if 'production' in env.host:
        host = 'onepercentclub.com'
    else:
        host = env.host

    sites_command = (
        "from django.contrib.sites.models import Site; "
        "site = Site.objects.get(pk=1); "
        "site.domain = '%s'; "
        "site.name = 'onepercentclub.com'; "
        "site.save()"
    ) % host

    with virtualenv():
        run_web('echo "%s" | ./manage.py shell --plain --settings=%s' % (
            sites_command, env.django_settings
        ))

@roles('dev')
@task
def deploy_dev(revspec='origin/master'):
    """
    Update the dev server to the specified revspec, or HEAD of deploy branch.
    """
    # Don't ask for confirmation
    env.noinput = True

    # Update git locally
    git_fetch_local()

    # Find commit for revspec
    commit = get_commit(revspec)

    # Make sure the remote git repo is up to date
    update_git(commit)

    # Get Django ready
    prepare_django()

    # Update site domain for self-reference to work
    set_site_domain()

    # Add the current git commit hash to the settings file
    add_git_commit()

    # Restart app server
    restart_site()

@roles('testing')
@task
def deploy_testing(revspec='origin/master'):
    """
    Update the testing server to the specified revspec, or HEAD of deploy branch and optionally sync migrated data.
    """
    # Update git locally
    git_fetch_local()

    # Find commit for revspec
    commit = get_commit(revspec)
    tag = find_available_tag('testing')

    # Make sure the remote git repo is up to date
    update_git(commit)

    # Get Django ready
    prepare_django()

    # Update site domain for self-reference to work
    set_site_domain()

    # Add the current git commit hash to the settings file
    add_git_commit()

    # Restart app server
    restart_site()

    # Deploy complete, tag commit
    tag_commit(commit.hexsha, tag)

@roles('staging')
@task
def deploy_staging(revspec=None):
    """
    Update the staging server to the specified revspec, or the latest testing release and optionally sync migrated data.
    """

    # Update git locally
    git_fetch_local()

    # Find revspec or latest testing release
    if not revspec:
        version = find_latest_tag_version('testing')
        revspec = make_versioned_tag('testing', version)

    # Find commit for revspec
    commit = get_commit(revspec)

    # Find latest available staging version
    tag = find_available_tag('staging')

    # Check whether this commit has been tested
    assert_release_tag(commit, 'testing')

    # Update the code
    update_tar(commit.hexsha)

    # Get Django ready
    prepare_django()

    # Update site domain for self-reference to work
    set_site_domain()

    # Restart app server
    restart_site()

    # Deploy complete, tag commit
    tag_commit(commit.hexsha, tag)

@roles('production')
@task
def deploy_production(revspec=None):
    """ Update the production server to the specified revspec, or the latest staging release. """

    # Update git locally
    git_fetch_local()

    # Find revspec or latest testing release
    if not revspec:
        version = find_latest_tag_version('staging')
        revspec = make_versioned_tag('staging', version)

    # Find commit for revspec
    commit = get_commit(revspec)

    # Find latest available staging version
    tag = find_available_tag('production')

    # Check whether this commit has been staged.
    assert_release_tag(commit, 'staging')

    # Update the code
    update_tar(commit.hexsha)

    # Get Django ready
    prepare_django()

    # Update site domain for self-reference to work
    set_site_domain()

    # Restart app server
    restart_site()

    # Deploy complete, tag commit
    tag_commit(commit.hexsha, tag)

@roles('backup')
@task
def get_db():
    backup_dir = "/home/backups/onepercentclub-backups/onepercentsite/current"
    with cd(backup_dir):
        output = run("ls *.bz2")
        try:
            filename = output.split()[0]
        except IndexError:
            print "No database backup file found"

        if filename:
            get(remote_path="{0}/{1}".format(backup_dir, filename), local_path="./dump.sql.bz2")
            unpack_db()

def unpack_db(filename="dump.sql.bz2"):
    try:
        local("gunzip {0}".format(filename))
    except IndexError:
        print "No database file found"

<|endoftext|>


<commit_before>#!/usr/bin/python
import argparse

###basic parser for parent help statement###
def parentArgs():

	parser = argparse.ArgumentParser(formatter_class=argparse.RawDescriptionHelpFormatter,
		description='''\
	Suzanne's pipeline to identify somatic CNVs from single-cell whole-genome sequencing data
	=========================================================================================

	You must specify a function to perform:
	  *preprocess (trim fastq reads to the appropriate length)
	  *map (map fastq files to the hg38 or mm10 genome)
	  *count (count number of reads in 25,000 genomic bins)
	  *segment (run CBS -- requires Matlab!)
	  *interpret (perform QC assessment and removal of low-quality CNV calls)
	  [More functions coming soon...]
		''')

	parser.print_help()

	raise SystemExit

###interpret arguments needed to perform preprocessing of fastq files###
def preprocessArgs():

	parser = argparse.ArgumentParser(description='Trim fastq reads to the appropriate length')

	#required arguments#
	parser.add_argument('FastqDirectory',
		help = 'The path to the folder that contains fastq files to be processed')

	#optional arguments#
	parser.add_argument('-5', '--trim5', metavar='X', type=int, default=0,
		help = "Number of 5' bases to trim from fastq reads")
	parser.add_argument('-l', '--length', metavar='X', type=int, default=36,
		help = 'The desired read length')
	parser.add_argument('-r', '--remove', action='store_true',
		help = 'Set this flag if you want to delete the full length fastq files')
	parser.add_argument('-s', '--samples', metavar='/path/to/sample_list.txt', default=False,
		help='Path to a file containing a list of fastq files to be processed\n\tsample names only, no path or file extension needed')

	return parser

###interpret arguments needed to perform mapping of fastq files###
def mapArgs():

	parser = argparse.ArgumentParser(description='Map fastq files to the appropriate reference genome')

	#required arguments#
	parser.add_argument('FastqDirectory',
		help = 'The path to the folder that contains fastq files to be processed')
	parser.add_argument('species', choices=['hg38', 'mm10'],
		help = 'The genome build of the species being assessed')

	#optional arguments#
	parser.add_argument('-t', '--trim', metavar='X', nargs=2, type=int, default=[0, 0],
		help = "Number of 5' and 3' bases to trim from fastq reads during mapping")
	parser.add_argument('-o', '--output', metavar='/path/to/output_directory/', default=False,
		help = 'A filepath to the desired directory where you would like sam files saved, if not in the same parent directory as the fastq files')
	parser.add_argument('-x', '--statdir', metavar='/path/to/statistics_directory/', default=False,
		help = 'A filepath to the desired directory where you would like mapping statistics saved, if not in the same parent directory as the fastq files')
	parser.add_argument('-s', '--samples', metavar='/path/to/sample_list.txt', default=False,
		help='Path to a file containing a list of fastq files to be processed\n\tsample names only, no path or file extension needed')

	return parser

###interpret arguments needed to perform counting of unique.sam files###
def countArgs():

	parser = argparse.ArgumentParser(description='Count the reads per genomic bin from unique sam files')

	#required arguments#
	parser.add_argument('SamDirectory',
		help = 'The path to the folder that contains unique.sam files to be processed')
	parser.add_argument('species', choices=['hg38', 'mm10'],
		help = 'The genome build of the species being assessed')

	#optional arguments#
	parser.add_argument('-o', '--output', metavar='/path/to/output_directory/', default=False,
		help = 'A filepath to the desired directory where you would like bincount.txt files saved, if not in the same parent directory as the sam files')
	parser.add_argument('-x', '--statdir', metavar='/path/to/statistics_directory/', default=False,
		help = 'A filepath to the desired directory where you would like mapping statistics saved, if not in the same parent directory as the sam files')
	parser.add_argument('-s', '--samples', metavar='/path/to/sample_list.txt', default=False,
		help='Path to a file containing a list of unique.sam files to be processed\n\tsample names only, no path or file extension needed')

	return parser

###interpret arguments needed to perform normalization and segmentation of bincounts.txt files###
def segmentArgs():

	parser = argparse.ArgumentParser(description='Normalize and segment bincounts files to begin CNV identification process')

	#required arguments#
	parser.add_argument('CountDirectory',
		help = 'The path to the folder that contains bincounts.txt files to be processed')
	parser.add_argument('species', choices=['hg38', 'mm10'],
		help = 'The genome build of the species being assessed')

	#optional arguments#
	parser.add_argument('-o', '--output', metavar='/path/to/output_directory/', default=False,
		help = 'A filepath to the desired PARENT directory where you would like lowess.txt and segments.txt files saved, if not in the same parent directory as the bincounts files')
	parser.add_argument('-i', '--infofile', metavar='/path/to/sample.info.txt', default=False,
		help='Path to a .txt file containing information about the samples to be processed (unique name, amplification method, number of cells)\n\tIf not all are identical. This file should not have a header row')
	parser.add_argument('-c', '--columns', metavar='X X X', default=[0, 1, 2], type=int, nargs=3,
		help='The zero-indexed locations of the columns to import from the infofile in the order: name, method, cell number (if not the first 3 columns)')
	parser.add_argument('-g', '--gconly', action='store_true',
		help = 'Set this flag if you only want GC-correction to be performed during normalization')
	parser.add_argument('-n', '--normalizeonly', action='store_true',
		help = 'Set this flag if you do not want CBS to be performed')
	parser.add_argument('-s', '--samples', metavar='/path/to/sample_list.txt', default=False,
		help='Path to a file containing a list of bincounts.txt files to be processed\n\tsample names only, no path or file extension needed')

	return parser

###interpret arguments needed to perform QC and CNV analysis of each single cell sample###
def interpretArgs():

	parser = argparse.ArgumentParser(description='Assess sample quality, filter unreliable CNVs, and generate user-friendly output files')

	#required arguments#
	parser.add_argument('AnalysisDirectory',
		help = 'The path to the folder to save output files')
	parser.add_argument('species', choices=['hg38', 'mm10'],
		help = 'The genome build of the species being assessed')

	#optional arguments#
	parser.add_argument('-f', '--nofilter', action='store_true',
		help = 'Set this flag if you do not want to perform FUnC filtering of low-quality CNV calls')
	parser.add_argument('-i', '--infofile', metavar='/path/to/sample.info.txt', default=False,
		help='Path to a .txt file containing information about the samples to be processed (unique name, number of cells, group)\n\tIf not all are identical. This file should not have a header row')
	parser.add_argument('-c', '--columns', metavar='X X X', default=[0, 1, 2], type=int, nargs=3,
		help='The zero-indexed locations of the columns to import from the infofile in the order: name, cell number, group (if not the first 3 columns)')
	parser.add_argument('-l', '--lowess', metavar='/path/to/lowess.txt/files/', default=False,
		help = 'A filepath to the desired directory where all lowess.txt files are saved, if not AnalysisDirectory/Lowess/')
	parser.add_argument('-g', '--segments', metavar='/path/to/segments.txt/files/', default=False,
		help = 'A filepath to the desired directory where all segments.txt files are saved, if not AnalysisDirectory/Segments/')
	parser.add_argument('-r', '--countstats', metavar='/path/to/bincounts.stats.txt/files/', default=False,
		help = 'A filepath to the desired directory where all bincounts.stats.txt files are saved, if not AnalysisDirectory/PipelineStats/')
	parser.add_argument('-s', '--samples', metavar='/path/to/sample_list.txt', default=False,
		help='Path to a file containing a list of sample names to be processed\n\tno path or file extension needed')

	return parser

def fullParser(input):

	functionDict =	{
			'-h': parentArgs,
			'--help': parentArgs,
			'preprocess': preprocessArgs,
			'map': mapArgs,
			'count': countArgs,
			'segment': segmentArgs,
			'interpret': interpretArgs,
			}

	if input == []:
		parentArgs()

	if input[0] not in functionDict.keys():
		return input[0], False

	parser = functionDict[input[0]]()
	args = parser.parse_args(input[1:])

	return input[0], args

<commit_msg>Update arguments.py<commit_after>#!/usr/bin/python
import argparse

###basic parser for parent help statement###
def parentArgs():

	parser = argparse.ArgumentParser(formatter_class=argparse.RawDescriptionHelpFormatter,
		description='''\
	Suzanne's pipeline to identify somatic CNVs from single-cell whole-genome sequencing data
	=========================================================================================

	You must specify a function to perform:
	  *preprocess (trim fastq reads to the appropriate length)
	  *map (map fastq files to the hg38 or mm10 genome)
	  *count (count number of reads in 25,000 genomic bins)
	  *segment (run CBS -- requires Matlab!)
	  *interpret (perform QC assessment and removal of low-quality CNV calls)
	  [More functions coming soon...]
		''')

	parser.print_help()

	raise SystemExit

###interpret arguments needed to perform preprocessing of fastq files###
def preprocessArgs():

	parser = argparse.ArgumentParser(description='Trim fastq reads to the appropriate length')

	#required arguments#
	parser.add_argument('FastqDirectory',
		help = 'The path to the folder that contains fastq files to be processed')

	#optional arguments#
	parser.add_argument('-5', '--trim5', metavar='X', type=int, default=0,
		help = "Number of 5' bases to trim from fastq reads")
	parser.add_argument('-l', '--length', metavar='X', type=int, default=36,
		help = 'The desired read length')
	parser.add_argument('-r', '--remove', action='store_true',
		help = 'Set this flag if you want to delete the full length fastq files')
	parser.add_argument('-s', '--samples', metavar='/path/to/sample_list.txt', default=False,
		help='Path to a file containing a list of fastq files to be processed\n\tsample names only, no path or file extension needed')

	return parser

###interpret arguments needed to perform mapping of fastq files###
def mapArgs():

	parser = argparse.ArgumentParser(description='Map fastq files to the appropriate reference genome')

	#required arguments#
	parser.add_argument('FastqDirectory',
		help = 'The path to the folder that contains fastq files to be processed')
	parser.add_argument('MapIndex',
		help='The path to the bowtie (v1) mapping references, as you would input if running bowtie directly')
#	parser.add_argument('species', choices=['hg38', 'mm10'],
#		help = 'The genome build of the species being assessed')

	#optional arguments#
	parser.add_argument('-t', '--trim', metavar='X', nargs=2, type=int, default=[0, 0],
		help = "Number of 5' and 3' bases to trim from fastq reads during mapping")
	parser.add_argument('-o', '--output', metavar='/path/to/output_directory/', default=False,
		help = 'A filepath to the desired directory where you would like sam files saved, if not in the same parent directory as the fastq files')
	parser.add_argument('-x', '--statdir', metavar='/path/to/statistics_directory/', default=False,
		help = 'A filepath to the desired directory where you would like mapping statistics saved, if not in the same parent directory as the fastq files')
	parser.add_argument('-s', '--samples', metavar='/path/to/sample_list.txt', default=False,
		help='Path to a file containing a list of fastq files to be processed\n\tsample names only, no path or file extension needed')

	return parser

###interpret arguments needed to perform counting of unique.sam files###
def countArgs():

	parser = argparse.ArgumentParser(description='Count the reads per genomic bin from unique sam files')

	#required arguments#
	parser.add_argument('SamDirectory',
		help = 'The path to the folder that contains unique.sam files to be processed')
	parser.add_argument('species', choices=['hg38', 'mm10'],
		help = 'The genome build of the species being assessed')

	#optional arguments#
	parser.add_argument('-o', '--output', metavar='/path/to/output_directory/', default=False,
		help = 'A filepath to the desired directory where you would like bincount.txt files saved, if not in the same parent directory as the sam files')
	parser.add_argument('-x', '--statdir', metavar='/path/to/statistics_directory/', default=False,
		help = 'A filepath to the desired directory where you would like mapping statistics saved, if not in the same parent directory as the sam files')
	parser.add_argument('-s', '--samples', metavar='/path/to/sample_list.txt', default=False,
		help='Path to a file containing a list of unique.sam files to be processed\n\tsample names only, no path or file extension needed')

	return parser

###interpret arguments needed to perform normalization and segmentation of bincounts.txt files###
def segmentArgs():

	parser = argparse.ArgumentParser(description='Normalize and segment bincounts files to begin CNV identification process')

	#required arguments#
	parser.add_argument('CountDirectory',
		help = 'The path to the folder that contains bincounts.txt files to be processed')
	parser.add_argument('species', choices=['hg38', 'mm10'],
		help = 'The genome build of the species being assessed')

	#optional arguments#
	parser.add_argument('-o', '--output', metavar='/path/to/output_directory/', default=False,
		help = 'A filepath to the desired PARENT directory where you would like lowess.txt and segments.txt files saved, if not in the same parent directory as the bincounts files')
	parser.add_argument('-i', '--infofile', metavar='/path/to/sample.info.txt', default=False,
		help='Path to a .txt file containing information about the samples to be processed (unique name, amplification method, number of cells)\n\tIf not all are identical. This file should not have a header row')
	parser.add_argument('-c', '--columns', metavar='X X X', default=[0, 1, 2], type=int, nargs=3,
		help='The zero-indexed locations of the columns to import from the infofile in the order: name, method, cell number (if not the first 3 columns)')
	parser.add_argument('-g', '--gconly', action='store_true',
		help = 'Set this flag if you only want GC-correction to be performed during normalization')
	parser.add_argument('-n', '--normalizeonly', action='store_true',
		help = 'Set this flag if you do not want CBS to be performed')
	parser.add_argument('-s', '--samples', metavar='/path/to/sample_list.txt', default=False,
		help='Path to a file containing a list of bincounts.txt files to be processed\n\tsample names only, no path or file extension needed')

	return parser

###interpret arguments needed to perform QC and CNV analysis of each single cell sample###
def interpretArgs():

	parser = argparse.ArgumentParser(description='Assess sample quality, filter unreliable CNVs, and generate user-friendly output files')

	#required arguments#
	parser.add_argument('AnalysisDirectory',
		help = 'The path to the folder to save output files')
	parser.add_argument('species', choices=['hg38', 'mm10'],
		help = 'The genome build of the species being assessed')

	#optional arguments#
	parser.add_argument('-f', '--nofilter', action='store_true',
		help = 'Set this flag if you do not want to perform FUnC filtering of low-quality CNV calls')
	parser.add_argument('-i', '--infofile', metavar='/path/to/sample.info.txt', default=False,
		help='Path to a .txt file containing information about the samples to be processed (unique name, number of cells, group)\n\tIf not all are identical. This file should not have a header row')
	parser.add_argument('-c', '--columns', metavar='X X X', default=[0, 1, 2], type=int, nargs=3,
		help='The zero-indexed locations of the columns to import from the infofile in the order: name, cell number, group (if not the first 3 columns)')
	parser.add_argument('-l', '--lowess', metavar='/path/to/lowess.txt/files/', default=False,
		help = 'A filepath to the desired directory where all lowess.txt files are saved, if not AnalysisDirectory/Lowess/')
	parser.add_argument('-g', '--segments', metavar='/path/to/segments.txt/files/', default=False,
		help = 'A filepath to the desired directory where all segments.txt files are saved, if not AnalysisDirectory/Segments/')
	parser.add_argument('-r', '--countstats', metavar='/path/to/bincounts.stats.txt/files/', default=False,
		help = 'A filepath to the desired directory where all bincounts.stats.txt files are saved, if not AnalysisDirectory/PipelineStats/')
	parser.add_argument('-s', '--samples', metavar='/path/to/sample_list.txt', default=False,
		help='Path to a file containing a list of sample names to be processed\n\tno path or file extension needed')

	return parser

def fullParser(input):

	functionDict =	{
			'-h': parentArgs,
			'--help': parentArgs,
			'preprocess': preprocessArgs,
			'map': mapArgs,
			'count': countArgs,
			'segment': segmentArgs,
			'interpret': interpretArgs,
			}

	if input == []:
		parentArgs()

	if input[0] not in functionDict.keys():
		return input[0], False

	parser = functionDict[input[0]]()
	args = parser.parse_args(input[1:])

	return input[0], args

<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
"""
    1flow fabfile, which relies on sparks.

    Example calls:

        # migrate() must have the *named* `args` argument,
        # else it will conflicts with the implicit `remote_configuration`.
        fab local sdf.migrate:args='redisboard --fake'

        # Thus, this won't work:
        fab local sdf.migrate:'redisboard --fake'

        # copy model data from a DB to another:
        fab test sdf.getdata:landing
        fab local sdf.putdata

        # and then to production:
        fab production sdf.putdata

"""
import os

from fabric.api import env, task, roles, local as fablocal

from sparks.fabric import (with_remote_configuration,
                           set_roledefs_and_parallel,
                           worker_roles)
import sparks.django.fabfile as sdf

# Make the main deployment tasks immediately accessible
runable, deploy, fast_deploy = sdf.runable, sdf.deploy, sdf.fast_deploy
maintenance_mode, operational_mode = sdf.maintenance_mode, sdf.operational_mode
run_command, restart_services = sdf.run_command, sdf.restart_services
stop, start, status = sdf.stop_services, sdf.start_services, sdf.status_services
remove, pick, role = sdf.remove_services, sdf.pick, sdf.role

# The Django project name
env.project    = 'oneflow'
env.virtualenv = '1flow'
env.parallel   = True

# WARNING: don't set `env.user` here, it creates false-negatives when
# bare-connecting to servers manually from the current directory. Eg.:
#       paramiko.transport[DEBUG] userauth is OK
#       paramiko.transport[INFO] Authentication (publickey) failed.
# Whereas everything is OK from outside this directory.
# Conclusion: it seems `env.user` overrides even the ~/.ssh/config values.

# Where is the django project located
env.root         = '/home/1flow/www/src'
env.environment  = 'test'
env.repository   = 'git@dev.1flow.net:1flow.git'

def get_current_git_branch():
    return fablocal('git rev-parse --abbrev-ref HEAD',
                    capture=True).strip()

@task
def local():
    # NOTE: in theory a local environment doesn't need any roledef, because
    # we use Profile.development for running processes and all fabric
    # operations should be run via local(). BUT, in fact, they are run via
    # run() or sudo() (sparks doesn't make any difference, all tasks are
    # "remote"), that's why we populate roledefs, to benefit from all sparks
    # tasks.
    set_roledefs_and_parallel({
        'db': ['localhost'],
        'web': ['localhost'],
        'lang': ['localhost'],
        'beat': ['localhost'],
        'shell': ['localhost'],
        'flower': ['localhost'],
        'worker': ['localhost'],
    })

    # As of sparks 2.2+, it should not be needed to set env.host_string anymore,
    # but it's still comfortable to have it when lanching tasks that are not
    # fully roledefs-compatible (eg. sdf.compilemessages ran alone).
    env.host_string = 'localhost'
    env.root        = os.path.expanduser('~/sources/1flow')
    env.env_was_set = True

@task(alias='test')
def preview(branch=None):
    """ This is the default config, we don't need to set anything more.

        To create a new test environment:

        adapt .ssh/config
        ssh duncan
        lxc-clone
        in the LXC rootfs, clean /etc/supervisor/conf.d/*
        edit firewall / iptables
        start LXC

    """

    #
    # WARNING: when adding preview machine(s),
    # don't forget to add django settings files for them…
    #
    set_roledefs_and_parallel({
        'db': ['obi.1flow.io'],
        'web': ['obi.1flow.io'],
        'lang': ['obi.1flow.io'],
        'beat': ['obi.1flow.io'],
        'shell': ['worbi.1flow.io'],
        'flower': ['worbi.1flow.io'],
        'worker_high': ['obi.1flow.io'],
        'worker_medium': ['worbi.1flow.io'],
        'worker_low': ['worbi.1flow.io'],
    })

    if branch is None:
        env.branch = get_current_git_branch()
    # implicit: else: branch will be 'develop',
    # set directly from the sparks defaults.
    env.sparks_options = {
        'worker_concurrency': {
            'worker_low': 4,
            'worker_medium': 4,
            'worker_high': 6,
        }
    }

    # we force the user because we can login as standard user there
    env.user        = '1flow'
    env.env_was_set = True

@task(aliases=('work', ))
def workers():
    """ Just setup the workers roles so we can act easily on all of them. """

    #TODO: set worker_roles
    env.roles = worker_roles[:] + ['beat']

@task(aliases=('kill', ))
def reboot(more=False):
    run_command("ps ax | grep 'celery.*worker' | grep -v grep "
                "| awk '{print $1}' | sudo xargs -I% kill % || true")
    if more:
        run_command("ps ax | grep 'celery.*worker' | grep -v grep "
                    "| awk '{print $1}' | sudo xargs -I% kill -9 % || true")

@task
def count():
    run_command("ps ax | grep 'celery.*worker' | grep -v grep "
                "| wc -l")

@task(alias='prod')
def production():
    # we force the user because we can login as standard user there
    env.user        = '1flow'
    env.environment = 'production'
    set_roledefs_and_parallel({
        'db': ['1flow.io'],
        'web': ['1flow.io'],
        'beat': ['worker-01.1flow.io', ],
        'shell': ['worker-03.1flow.io', ],
        'flower': ['worker-01.1flow.io', ],
        'worker_high': ['worker-01.1flow.io', ],
        'worker_medium': ['worker-03.1flow.io', ],
        'worker_low': ['worker-05.1flow.io', ],
        'worker_fetch': ['worker-02.1flow.io',
                         'worker-03.1flow.io',
                         'worker-04.1flow.io',
                         'worker-05.1flow.io',
                         'worker-99.1flow.io', ],
    })
    env.sparks_options = {
        'repository': {
            # We need to patch this because worker-04 is an LXC on the same
            # physical host than dev.1flow.net, and getting from it directly
            # doesn't work because of my NAT configuration.
            'worker-04.1flow.io': 'git@10.0.3.110:1flow.git',
        },
        'worker_concurrency': {
            'worker_high': 8,
            'worker_medium': 10,
            'worker_low': 10,
            'worker_fetch@worker-03.1flow.io': 2,
            'worker_fetch@worker-05.1flow.io': 2,
            'worker-02.1flow.io': 8,
            'worker-04.1flow.io': 8,
            'worker-99.1flow.io': 16,
        },
    }
    env.env_was_set = True

@task
def zero(branch=None):
    """ A master clone, restarted from scratch everytime to test migrations. """

    set_roledefs_and_parallel({
        'db': ['zero.1flow.io'],
        'web': ['zero.1flow.io'],
        'beat': ['zero.1flow.io'],
        'shell': ['zero.1flow.io'],
        'worker': ['zero.1flow.io'],
        'flower': ['zero.1flow.io'],
    })

    if branch is None:
        env.branch = 'develop'
    else:
        env.branch = get_current_git_branch()

    env.host_string = 'zero.1flow.io'

    env.sparks_options = {
        'worker_concurrency': {
            '__all__': 2,
        }
    }

    # env.user is set via .ssh/config
    env.env_was_set = True

@task
def oneflowapp():
    """ 1flowapp.com parameters; must be used with test or production. """

    raise RuntimeError('As of 20130510, this environment '
                       'should not be used anymore.')

    if not hasattr(env, 'env_was_set'):
        # Without this, settings can be messed up.
        raise RuntimeError('Type `oneflowapp` *AFTER* test|production|local !')

    if env.environment == 'production':
        env.sparks_djsettings = '1flowapp_com'

    elif env.environment == 'test':
        if env.host_string == 'localhost':
            env.sparks_djsettings = 'chani_app'
        else:
            env.sparks_djsettings = 'obi_1flowapp_com'

    else:
        raise RuntimeError('environment not supported')

@task
@with_remote_configuration
def testapps(remote_configuration):

    project_apps = (app.split('.', 1)[1] for app
                    in remote_configuration.django_settings.INSTALLED_APPS
                    if app.startswith('{0}.'.format(env.project)))

    print(str(list(project_apps)))

@task
@roles('db')
def firstdata():
    sdf.putdata('./oneflow/landing/fixtures/landing_2013-05-14_final-before-beta-opening.json') # NOQA
    sdf.putdata('./oneflow/base/fixtures/base_2013-05-14_final-before-beta-opening.json') # NOQA
    sdf.putdata('./oneflow/fixtures/djcelery_2013-06-30_0001.json')

@task(aliases=('first', ))
def firstdeploy():
    deploy()
    firstdata()
<commit_msg>ensure duncan's 4 CPUs are correctly loaded; 16 workers make them idle sometimes (but 24 was too much, with some SoftTimeLimitExceeded).<commit_after># -*- coding: utf-8 -*-
"""
    1flow fabfile, which relies on sparks.

    Example calls:

        # migrate() must have the *named* `args` argument,
        # else it will conflicts with the implicit `remote_configuration`.
        fab local sdf.migrate:args='redisboard --fake'

        # Thus, this won't work:
        fab local sdf.migrate:'redisboard --fake'

        # copy model data from a DB to another:
        fab test sdf.getdata:landing
        fab local sdf.putdata

        # and then to production:
        fab production sdf.putdata

"""
import os

from fabric.api import env, task, roles, local as fablocal

from sparks.fabric import (with_remote_configuration,
                           set_roledefs_and_parallel,
                           worker_roles)
import sparks.django.fabfile as sdf

# Make the main deployment tasks immediately accessible
runable, deploy, fast_deploy = sdf.runable, sdf.deploy, sdf.fast_deploy
maintenance_mode, operational_mode = sdf.maintenance_mode, sdf.operational_mode
run_command, restart_services = sdf.run_command, sdf.restart_services
stop, start, status = sdf.stop_services, sdf.start_services, sdf.status_services
remove, pick, role = sdf.remove_services, sdf.pick, sdf.role

# The Django project name
env.project    = 'oneflow'
env.virtualenv = '1flow'
env.parallel   = True

# WARNING: don't set `env.user` here, it creates false-negatives when
# bare-connecting to servers manually from the current directory. Eg.:
#       paramiko.transport[DEBUG] userauth is OK
#       paramiko.transport[INFO] Authentication (publickey) failed.
# Whereas everything is OK from outside this directory.
# Conclusion: it seems `env.user` overrides even the ~/.ssh/config values.

# Where is the django project located
env.root         = '/home/1flow/www/src'
env.environment  = 'test'
env.repository   = 'git@dev.1flow.net:1flow.git'

def get_current_git_branch():
    return fablocal('git rev-parse --abbrev-ref HEAD',
                    capture=True).strip()

@task
def local():
    # NOTE: in theory a local environment doesn't need any roledef, because
    # we use Profile.development for running processes and all fabric
    # operations should be run via local(). BUT, in fact, they are run via
    # run() or sudo() (sparks doesn't make any difference, all tasks are
    # "remote"), that's why we populate roledefs, to benefit from all sparks
    # tasks.
    set_roledefs_and_parallel({
        'db': ['localhost'],
        'web': ['localhost'],
        'lang': ['localhost'],
        'beat': ['localhost'],
        'shell': ['localhost'],
        'flower': ['localhost'],
        'worker': ['localhost'],
    })

    # As of sparks 2.2+, it should not be needed to set env.host_string anymore,
    # but it's still comfortable to have it when lanching tasks that are not
    # fully roledefs-compatible (eg. sdf.compilemessages ran alone).
    env.host_string = 'localhost'
    env.root        = os.path.expanduser('~/sources/1flow')
    env.env_was_set = True

@task(alias='test')
def preview(branch=None):
    """ This is the default config, we don't need to set anything more.

        To create a new test environment:

        adapt .ssh/config
        ssh duncan
        lxc-clone
        in the LXC rootfs, clean /etc/supervisor/conf.d/*
        edit firewall / iptables
        start LXC

    """

    #
    # WARNING: when adding preview machine(s),
    # don't forget to add django settings files for them…
    #
    set_roledefs_and_parallel({
        'db': ['obi.1flow.io'],
        'web': ['obi.1flow.io'],
        'lang': ['obi.1flow.io'],
        'beat': ['obi.1flow.io'],
        'shell': ['worbi.1flow.io'],
        'flower': ['worbi.1flow.io'],
        'worker_high': ['obi.1flow.io'],
        'worker_medium': ['worbi.1flow.io'],
        'worker_low': ['worbi.1flow.io'],
    })

    if branch is None:
        env.branch = get_current_git_branch()
    # implicit: else: branch will be 'develop',
    # set directly from the sparks defaults.
    env.sparks_options = {
        'worker_concurrency': {
            'worker_low': 4,
            'worker_medium': 4,
            'worker_high': 6,
        }
    }

    # we force the user because we can login as standard user there
    env.user        = '1flow'
    env.env_was_set = True

@task(aliases=('work', ))
def workers():
    """ Just setup the workers roles so we can act easily on all of them. """

    #TODO: set worker_roles
    env.roles = worker_roles[:] + ['beat']

@task(aliases=('kill', ))
def reboot(more=False):
    run_command("ps ax | grep 'celery.*worker' | grep -v grep "
                "| awk '{print $1}' | sudo xargs -I% kill % || true")
    if more:
        run_command("ps ax | grep 'celery.*worker' | grep -v grep "
                    "| awk '{print $1}' | sudo xargs -I% kill -9 % || true")

@task
def count():
    run_command("ps ax | grep 'celery.*worker' | grep -v grep "
                "| wc -l")

@task(alias='prod')
def production():
    # we force the user because we can login as standard user there
    env.user        = '1flow'
    env.environment = 'production'
    set_roledefs_and_parallel({
        'db': ['1flow.io'],
        'web': ['1flow.io'],
        'beat': ['worker-01.1flow.io', ],
        'shell': ['worker-03.1flow.io', ],
        'flower': ['worker-01.1flow.io', ],
        'worker_high': ['worker-01.1flow.io', ],
        'worker_medium': ['worker-03.1flow.io', ],
        'worker_low': ['worker-05.1flow.io', ],
        'worker_fetch': ['worker-02.1flow.io',
                         'worker-03.1flow.io',
                         'worker-04.1flow.io',
                         'worker-05.1flow.io',
                         'worker-99.1flow.io', ],
    })
    env.sparks_options = {
        'repository': {
            # We need to patch this because worker-04 is an LXC on the same
            # physical host than dev.1flow.net, and getting from it directly
            # doesn't work because of my NAT configuration.
            'worker-04.1flow.io': 'git@10.0.3.110:1flow.git',
        },
        'worker_concurrency': {
            'worker_high': 8,
            'worker_medium': 10,
            'worker_low': 10,
            'worker_fetch@worker-03.1flow.io': 2,
            'worker_fetch@worker-05.1flow.io': 2,
            'worker-02.1flow.io': 8,
            'worker-04.1flow.io': 8,
            'worker-99.1flow.io': 24,
        },
    }
    env.env_was_set = True

@task
def zero(branch=None):
    """ A master clone, restarted from scratch everytime to test migrations. """

    set_roledefs_and_parallel({
        'db': ['zero.1flow.io'],
        'web': ['zero.1flow.io'],
        'beat': ['zero.1flow.io'],
        'shell': ['zero.1flow.io'],
        'worker': ['zero.1flow.io'],
        'flower': ['zero.1flow.io'],
    })

    if branch is None:
        env.branch = 'develop'
    else:
        env.branch = get_current_git_branch()

    env.host_string = 'zero.1flow.io'

    env.sparks_options = {
        'worker_concurrency': {
            '__all__': 2,
        }
    }

    # env.user is set via .ssh/config
    env.env_was_set = True

@task
def oneflowapp():
    """ 1flowapp.com parameters; must be used with test or production. """

    raise RuntimeError('As of 20130510, this environment '
                       'should not be used anymore.')

    if not hasattr(env, 'env_was_set'):
        # Without this, settings can be messed up.
        raise RuntimeError('Type `oneflowapp` *AFTER* test|production|local !')

    if env.environment == 'production':
        env.sparks_djsettings = '1flowapp_com'

    elif env.environment == 'test':
        if env.host_string == 'localhost':
            env.sparks_djsettings = 'chani_app'
        else:
            env.sparks_djsettings = 'obi_1flowapp_com'

    else:
        raise RuntimeError('environment not supported')

@task
@with_remote_configuration
def testapps(remote_configuration):

    project_apps = (app.split('.', 1)[1] for app
                    in remote_configuration.django_settings.INSTALLED_APPS
                    if app.startswith('{0}.'.format(env.project)))

    print(str(list(project_apps)))

@task
@roles('db')
def firstdata():
    sdf.putdata('./oneflow/landing/fixtures/landing_2013-05-14_final-before-beta-opening.json') # NOQA
    sdf.putdata('./oneflow/base/fixtures/base_2013-05-14_final-before-beta-opening.json') # NOQA
    sdf.putdata('./oneflow/fixtures/djcelery_2013-06-30_0001.json')

@task(aliases=('first', ))
def firstdeploy():
    deploy()
    firstdata()
<|endoftext|>


<commit_before>'''
Copyright (c) 2017, Jairus Martin.

Distributed under the terms of the MIT License.

The full license is in the file COPYING.txt, distributed with this software.

Created on May 20, 2017

@author: jrm
'''
from atom.api import (
    Typed, ForwardTyped, Unicode, observe
)

from enaml.core.declarative import d_

from .text_view import TextView, ProxyTextView

class ProxyAnalogClock(ProxyTextView):
    """ The abstract definition of a proxy AnalogClock object.

    """
    #: A reference to the Label declaration.
    declaration = ForwardTyped(lambda: AnalogClock)

class AnalogClock(TextView):
    """ A simple control for displaying an AnalogClock

    """

    #: A reference to the proxy object.
    proxy = Typed(ProxyAnalogClock)
<commit_msg>Use correct parent class for clock<commit_after>'''
Copyright (c) 2017, Jairus Martin.

Distributed under the terms of the MIT License.

The full license is in the file COPYING.txt, distributed with this software.

Created on May 20, 2017

@author: jrm
'''
from atom.api import (
    Typed, ForwardTyped, Unicode, observe
)

from enaml.core.declarative import d_

from .view import View, ProxyView

class ProxyAnalogClock(ProxyView):
    """ The abstract definition of a proxy AnalogClock object.

    """
    #: A reference to the Label declaration.
    declaration = ForwardTyped(lambda: AnalogClock)

class AnalogClock(View):
    """ A simple control for displaying an AnalogClock

    """

    #: A reference to the proxy object.
    proxy = Typed(ProxyAnalogClock)
<|endoftext|>