<commit_before>"""
This module contains the functionality to arrange the nodes of a
diagram on an abstract grid, and then to produce a graphical
representation of the grid.

The currently supported back-ends are Xy-pic [Xypic]

[Xypic] http://www.tug.org/applications/Xy-pic/
"""

from sympy.core import Basic

class _GrowableGrid:
    """
    Holds a growable grid of objects.

    It is possible to append or prepend a row or a column to the grid
    using the corresponding methods.  Prepending rows or columns has
    the effect of changing the coordinates of the already existing
    elements.

    This class currently represents a naive implementation of the
    functionality with little attempt at optimisation.
    """
    def __init__(self, width, height):
        self._width = width
        self._height = height

        self._array = [[None for j in xrange(width)] for i in xrange(height)]

    @property
    def width(self):
        return self._width

    @property
    def height(self):
        return self._height

    def __getitem__(self, (i, j)):
        """
        Returns the element located at in the i-th line and j-th
        column.
        """
        return self._array[i][j]

    def __setitem__(self, (i, j), newvalue):
        """
        Sets the element located at in the i-th line and j-th
        column.
        """
        self._array[i][j] = newvalue

    def append_row(self):
        """
        Appends an empty row to the grid.
        """
        self._height += 1
        self._array.append([None for j in xrange(self._width)])

    def append_column(self):
        """
        Appends an empty column to the grid.
        """
        self._width += 1
        for i in xrange(self._height):
            self._array[i].append(None)

    def prepend_row(self):
        """
        Prepends the grid with an empty row.
        """
        self._height += 1
        self._array.insert(0, [None for j in xrange(self._width)])

    def prepend_column(self):
        """
        Prepends the grid with an empty column.
        """
        self._width += 1
        for i in xrange(self._height):
            self._array[i].insert(0, None)

class DiagramGrid(Basic):
    """
    Constructs and holds the fitting of the diagram into a grid.

    The constructor of this class takes a :class:`Diagram` as an
    argument and fits it into a grid.  It is then possible to find the
    with and height of the grid using the properties ``width`` and
    ``height``, as well as accessing the objects located at certain
    coordinates and finding out which moprhisms connect the object
    with other objects in the grid.  TODO: Explain how to do that.
    """
    def __new__(cls, diagram):
        pass
<commit_msg>DiagramGrid: Simplify the premises and conclusions and merge them.<commit_after>"""
This module contains the functionality to arrange the nodes of a
diagram on an abstract grid, and then to produce a graphical
representation of the grid.

The currently supported back-ends are Xy-pic [Xypic]

[Xypic] http://www.tug.org/applications/Xy-pic/
"""

from sympy.core import Basic
from sympy.categories import CompositeMorphism, IdentityMorphism

class _GrowableGrid:
    """
    Holds a growable grid of objects.

    It is possible to append or prepend a row or a column to the grid
    using the corresponding methods.  Prepending rows or columns has
    the effect of changing the coordinates of the already existing
    elements.

    This class currently represents a naive implementation of the
    functionality with little attempt at optimisation.
    """
    def __init__(self, width, height):
        self._width = width
        self._height = height

        self._array = [[None for j in xrange(width)] for i in xrange(height)]

    @property
    def width(self):
        return self._width

    @property
    def height(self):
        return self._height

    def __getitem__(self, (i, j)):
        """
        Returns the element located at in the i-th line and j-th
        column.
        """
        return self._array[i][j]

    def __setitem__(self, (i, j), newvalue):
        """
        Sets the element located at in the i-th line and j-th
        column.
        """
        self._array[i][j] = newvalue

    def append_row(self):
        """
        Appends an empty row to the grid.
        """
        self._height += 1
        self._array.append([None for j in xrange(self._width)])

    def append_column(self):
        """
        Appends an empty column to the grid.
        """
        self._width += 1
        for i in xrange(self._height):
            self._array[i].append(None)

    def prepend_row(self):
        """
        Prepends the grid with an empty row.
        """
        self._height += 1
        self._array.insert(0, [None for j in xrange(self._width)])

    def prepend_column(self):
        """
        Prepends the grid with an empty column.
        """
        self._width += 1
        for i in xrange(self._height):
            self._array[i].insert(0, None)

class DiagramGrid(Basic):
    """
    Constructs and holds the fitting of the diagram into a grid.

    The constructor of this class takes a :class:`Diagram` as an
    argument and fits it into a grid.  It is then possible to find the
    with and height of the grid using the properties ``width`` and
    ``height``, as well as accessing the objects located at certain
    coordinates and finding out which moprhisms connect the object
    with other objects in the grid.  TODO: Explain how to do that.
    """
    @staticmethod
    def _simplify_morphisms(morphisms):
        """
        Given a dictionary mapping morphisms to their properties,
        returns a new dictionary in which there are no morphisms which
        do not have properties, and which are compositions of other
        morphisms included in the dictionary.  Identities are dropped
        as well.
        """
        newmorphisms = {}
        for morphism, props in morphisms.items():
            if isinstance(morphism, CompositeMorphism) and not props:
                continue
            elif isinstance(morphism, IdentityMorphism):
                continue
            else:
                newmorphisms[morphism] = props
        return newmorphisms

    @staticmethod
    def _merge_premises_conclusions(premises, conclusions):
        """
        Given two dictionaries of morphisms and their properties,
        produces a single dictionary which includes elements from both
        dictionaries.  If a morphism has some properties in premises
        and also in conclusions, the properties in conclusions take
        priority.
        """
        merged = premises
        for morphism, props in conclusions.items():
            merged[morphism] = props
        return merged

    def __new__(cls, diagram):
        premises = DiagramGrid._simplify_morphisms(diagram.premises)
        conclusions = DiagramGrid._simplify_morphisms(diagram.conclusions)
        merged_morphisms = DiagramGrid._merge_premises_conclusions(
            premises, conclusions)
<|endoftext|>


<commit_before>

class Options:
  '''
  Available options
  '''

  def __init__(self, options={}):
    if options is None:
      self.options = {}
    else:
      self.options = options

  def get_option(self, option):
    '''
    Gets an option if present, else return None
    '''
    if option in self.options:
      return self.options[option]
    return None

  NO_PUBLISH = 'no_publish'
  STOP_BEFORE = 'stop_before'
  STOP_AFTER = 'stop_after'
  FROMSCRATCH = 'fromscratch'
<commit_msg>fix option test when options is none<commit_after>

class Options:
  '''
  Available options
  '''

  def __init__(self, options=None):
      self.options = options

  def get_option(self, option):
    '''
    Gets an option if present, else return None
    '''
    if self.options is None:
      return None

    #if option in self.options:
    if hasattr(self.options, option):
      return self.options[option]

    return None

  NO_PUBLISH = 'no_publish'
  STOP_BEFORE = 'stop_before'
  STOP_AFTER = 'stop_after'
  FROMSCRATCH = 'fromscratch'
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-

from flask import render_template, redirect, flash, abort
from coaster.views import load_models, UrlForView, ModelView, route, render_with, requires_permission
from baseframe import _
from baseframe.forms import render_form, render_delete_sqla

from .. import app, funnelapp, lastuser
from ..models import db, Profile, Project, ProjectRedirect, Section
from ..forms import SectionForm

def section_data(section):
    return {
        'name': section.name,
        'title': section.title,
        'description': section.description,
        'url': None,
        'json_url': None
        }

@app.route('/<profile>/<project>/sections')
@funnelapp.route('/<project>/sections', subdomain='<profile>')
@lastuser.requires_login
@load_models(
    (Profile, {'name': 'profile'}, 'g.profile'),
    ((Project, ProjectRedirect), {'name': 'project', 'profile': 'profile'}, 'project'),
    permission='view-section')
def section_list(profile, project):
    sections = Section.query.filter_by(project=project).all()
    return render_template('sections.html.jinja2', project=project, sections=sections)

@app.route('/<profile>/<project>/sections/new', methods=['GET', 'POST'])
@funnelapp.route('/<project>/sections/new', methods=['GET', 'POST'], subdomain='<profile>')
@lastuser.requires_login
@load_models(
    (Profile, {'name': 'profile'}, 'g.profile'),
    ((Project, ProjectRedirect), {'name': 'project', 'profile': 'profile'}, 'project'),
    permission='new-section')
def section_new(profile, project):
    form = SectionForm(model=Section, parent=project)
    if form.validate_on_submit():
        section = Section(project=project)
        form.populate_obj(section)
        db.session.add(section)
        db.session.commit()
        flash(_("Your new section has been added"), 'info')
        return redirect(project.url_for('sections'), code=303)
    return render_form(form=form, title=_("New section"), submit=_("Create section"))

@route('/<profile>/<project>/sections/<section>')
class SectionView(UrlForView, ModelView):
    model = Section
    route_model_map = {'profile': 'project.profile.name', 'project': 'project.name', 'section': 'name'}
    __decorators__ = [lastuser.requires_login]

    def loader(self, profile, project, section):
        return self.model.query.join(Project).join(Profile).filter(
                Project.name == project, Profile.name == profile,
                Section.name == section
            ).first_or_404()

    @route('', methods=['GET'])
    @render_with('section.html.jinja2', json=True)
    @requires_permission('view-section')
    def view(self):
        return {'project': self.obj.project.current_access(), 'section': self.obj.current_access()}

    @route('edit', methods=['GET', 'POST'])
    @requires_permission('edit-section')
    def edit(self):
        form = SectionForm(obj=self.obj, model=Section, parent=self.obj.parent)
        if form.validate_on_submit():
            form.populate_obj(self.obj)
            db.session.commit()
            flash(_("Your section has been edited"), 'info')
            return redirect(self.obj.project.url_for('sections'), code=303)
        return render_form(form=form, title=_("Edit section"), submit=_("Save changes"))

    @route('delete', methods=['GET', 'POST'])
    @requires_permission('delete-section')
    def delete(self):
        return render_delete_sqla(self.obj, db, title=_(u"Confirm delete"),
            message=_(u"Do you really wish to delete section '{title}’?").format(title=self.obj.title),
            success=_("Your section has been deleted"),
            next=self.obj.project.url_for('sections'),
            cancel_url=self.obj.project.url_for('sections'))

@route('/<project>/sections/<section>', subdomain='<profile>')
class FunnelSectionView(SectionView):
    pass

SectionView.init_app(app)
FunnelSectionView.init_app(funnelapp)
<commit_msg>Flake8 fixes<commit_after># -*- coding: utf-8 -*-

from flask import flash, redirect, render_template
from coaster.views import ModelView, UrlForView, load_models, render_with, requires_permission, route
from baseframe import _
from baseframe.forms import render_delete_sqla, render_form

from .. import app, funnelapp, lastuser
from ..models import Profile, Project, ProjectRedirect, Section, db
from ..forms import SectionForm

def section_data(section):
    return {
        'name': section.name,
        'title': section.title,
        'description': section.description,
        'url': None,
        'json_url': None
        }

@app.route('/<profile>/<project>/sections')
@funnelapp.route('/<project>/sections', subdomain='<profile>')
@lastuser.requires_login
@load_models(
    (Profile, {'name': 'profile'}, 'g.profile'),
    ((Project, ProjectRedirect), {'name': 'project', 'profile': 'profile'}, 'project'),
    permission='view-section')
def section_list(profile, project):
    sections = Section.query.filter_by(project=project).all()
    return render_template('sections.html.jinja2', project=project, sections=sections)

@app.route('/<profile>/<project>/sections/new', methods=['GET', 'POST'])
@funnelapp.route('/<project>/sections/new', methods=['GET', 'POST'], subdomain='<profile>')
@lastuser.requires_login
@load_models(
    (Profile, {'name': 'profile'}, 'g.profile'),
    ((Project, ProjectRedirect), {'name': 'project', 'profile': 'profile'}, 'project'),
    permission='new-section')
def section_new(profile, project):
    form = SectionForm(model=Section, parent=project)
    if form.validate_on_submit():
        section = Section(project=project)
        form.populate_obj(section)
        db.session.add(section)
        db.session.commit()
        flash(_("Your new section has been added"), 'info')
        return redirect(project.url_for('sections'), code=303)
    return render_form(form=form, title=_("New section"), submit=_("Create section"))

@route('/<profile>/<project>/sections/<section>')
class SectionView(UrlForView, ModelView):
    model = Section
    route_model_map = {'profile': 'project.profile.name', 'project': 'project.name', 'section': 'name'}
    __decorators__ = [lastuser.requires_login]

    def loader(self, profile, project, section):
        return self.model.query.join(Project).join(Profile).filter(
            Project.name == project, Profile.name == profile,
            Section.name == section
            ).first_or_404()

    @route('', methods=['GET'])
    @render_with('section.html.jinja2', json=True)
    @requires_permission('view-section')
    def view(self):
        return {'project': self.obj.project.current_access(), 'section': self.obj.current_access()}

    @route('edit', methods=['GET', 'POST'])
    @requires_permission('edit-section')
    def edit(self):
        form = SectionForm(obj=self.obj, model=Section, parent=self.obj.parent)
        if form.validate_on_submit():
            form.populate_obj(self.obj)
            db.session.commit()
            flash(_("Your section has been edited"), 'info')
            return redirect(self.obj.project.url_for('sections'), code=303)
        return render_form(form=form, title=_("Edit section"), submit=_("Save changes"))

    @route('delete', methods=['GET', 'POST'])
    @requires_permission('delete-section')
    def delete(self):
        return render_delete_sqla(self.obj, db, title=_(u"Confirm delete"),
            message=_(u"Do you really wish to delete section '{title}’?").format(title=self.obj.title),
            success=_("Your section has been deleted"),
            next=self.obj.project.url_for('sections'),
            cancel_url=self.obj.project.url_for('sections'))

@route('/<project>/sections/<section>', subdomain='<profile>')
class FunnelSectionView(SectionView):
    pass

SectionView.init_app(app)
FunnelSectionView.init_app(funnelapp)
<|endoftext|>


<commit_before>import treq
import pylab
import sqlite3
import calendar
from matplotlib import dates
from datetime import date, datetime, timedelta
from twisted.internet import reactor

from candlechart_noqt import Candlestick

SECURE = True
SOURCEFORGE_STATS = "sourceforge.net/projects/bitcoin/files"

def year_month_day(date):
    return '%d-%02d-%02d' % (date.year, date.month, date.day)

pending_calls = [0]
def sf_stats(start_date, end_date):
    protocol = 'https' if SECURE else 'http'

    while start_date < end_date:
        pending_calls[0] += 1
        # Daily stats
        start = year_month_day(start_date)
        end = year_month_day(start_date)
        start_date += timedelta(days=1)

        d = treq.get('%s://%s/stats/json?start_date=%s&end_date=%s' % (protocol,
            SOURCEFORGE_STATS, start, end))
        d.addCallback(lambda response:
                treq.json_content(response).addCallback(sf_stats_json))

sf_daily_stats = {}
def sf_stats_json(data):
    pending_calls[0] -= 1
    print pending_calls[0]
    date = map(int, data['start_date'].split(' ')[0].split('-'))
    dtime = datetime(year=date[0], month=date[1], day=date[2])

    sf_daily_stats[dtime] = {'total': data['total']}

    for country, dlcount in data['countries']:
        sf_daily_stats[dtime][country] = dlcount

    if not pending_calls[0]:
        reactor.stop()

def ohlc_stats(start_date, end_date, dbname='mtgox_trades.db'):
    db = sqlite3.connect(dbname)
    table = 'btcusd'
    query = """SELECT price FROM [%s]
        WHERE timestamp >= ? AND timestamp < ?
        ORDER BY timestamp ASC""" % table

    while start_date < end_date:
        # Daily stats
        start = calendar.timegm(start_date.utctimetuple())
        start_date += timedelta(days=1)
        end = calendar.timegm(start_date.utctimetuple())

        result = db.execute(query, (start, end))
        first = result.fetchone()
        if first is None:
            print "Skipping", start_date
            continue
        first = float(first[0])
        o, h, l, c = first, first, first, first
        for trade in result:
            price = float(trade[0])
            c = price
            h = max(h, price)
            l = min(l, price)

        print start_date
        yield o, h, l, c

def do_the_plot(candle, sf_top):
    fig = pylab.figure()
    ax = fig.add_subplot(111)

    dl_factor = 100 # Download counts will be divided by this amount
    plot_sf_stats(sf_top, ax, fig, factor=dl_factor)
    plot_candles(ax, candle)
    xlim = ax.get_xlim()
    ax.set_xlim(xlim[0] - 1, xlim[1] + 1)

    ax.grid()

    fig.suptitle(
            u'Bitcoin-QT download stats (%g x) mixed with MtGox USD price' %
            dl_factor)

    box = ax.get_position()
    ax.set_position([box.x0, box.y0, box.width * 0.85, box.height])
    ax.legend(loc='upper left', bbox_to_anchor=(1, 1))

    pylab.show()

def plot_candles(ax, candle):
    cs = Candlestick(ax)
    for o, h, l, c in candle:
        cs.append_candle(o, h, l, c)

def plot_sf_stats(top, ax, fig, factor):
    factor = float(factor)

    x = []
    total = []
    other = [[] for _ in xrange(len(top))]
    for key in sorted(sf_daily_stats):
        val = sf_daily_stats[key]
        x.append(key)
        total.append(val['total'] / factor)
        for data, name in zip(other, top):
            data.append(val[name] / factor)

    ax.xaxis.set_major_formatter(dates.DateFormatter('%Y-%m-%d'))
    # Labels on mondays.
    ax.xaxis.set_major_locator(dates.WeekdayLocator(byweekday=dates.MO))

    #ax.set_ylabel(u'Download count (%g x)' % factor)

    ax.plot(x, total, label=u'Total DL')
    for data, name in zip(other, top):
        ax.plot(x, data, label=u'%s DL' % name)

    fig.autofmt_xdate()

if __name__ == "__main__":
    start_date = datetime(year=2013, month=2, day=1)
    end_date = datetime.fromordinal(date.today().toordinal())
    end_date += timedelta(days=1)

    sf_stats(start_date, end_date)
    reactor.run()

    # Find out the top two countries in terms of download count.
    from collections import defaultdict
    dl_country = defaultdict(int)
    for _, v in sf_daily_stats.iteritems():
        for country, dlcount in v.iteritems():
            if country == 'total':
                # Not really a country.
                continue
            dl_country[country] += dlcount
    n = 2
    top_n = sorted(((v, k) for k, v in dl_country.items()), reverse=True)[:n]
    print top_n
    top_n = [k for v, k in top_n]

    candle = ohlc_stats(start_date, end_date)

    # You might want to save sf_daily_stats
    #print sf_daily_stats
    do_the_plot(candle, top_n)
<commit_msg>A given country may not have download data for a given period.<commit_after>import treq
import pylab
import sqlite3
import calendar
from matplotlib import dates
from datetime import date, datetime, timedelta
from twisted.internet import reactor

from candlechart_noqt import Candlestick

SECURE = True
SOURCEFORGE_STATS = "sourceforge.net/projects/bitcoin/files"

def year_month_day(date):
    return '%d-%02d-%02d' % (date.year, date.month, date.day)

pending_calls = [0]
def sf_stats(start_date, end_date):
    protocol = 'https' if SECURE else 'http'

    while start_date < end_date:
        pending_calls[0] += 1
        # Daily stats
        start = year_month_day(start_date)
        end = year_month_day(start_date)
        start_date += timedelta(days=1)

        d = treq.get('%s://%s/stats/json?start_date=%s&end_date=%s' % (protocol,
            SOURCEFORGE_STATS, start, end))
        d.addCallback(lambda response:
                treq.json_content(response).addCallback(sf_stats_json))

sf_daily_stats = {}
def sf_stats_json(data):
    pending_calls[0] -= 1
    print pending_calls[0]
    date = map(int, data['start_date'].split(' ')[0].split('-'))
    dtime = datetime(year=date[0], month=date[1], day=date[2])

    sf_daily_stats[dtime] = {'total': data['total']}

    for country, dlcount in data['countries']:
        sf_daily_stats[dtime][country] = dlcount

    if not pending_calls[0]:
        reactor.stop()

def ohlc_stats(start_date, end_date, dbname='mtgox_trades.db'):
    db = sqlite3.connect(dbname)
    table = 'btcusd'
    query = """SELECT price FROM [%s]
        WHERE timestamp >= ? AND timestamp < ?
        ORDER BY timestamp ASC""" % table

    while start_date < end_date:
        # Daily stats
        start = calendar.timegm(start_date.utctimetuple())
        start_date += timedelta(days=1)
        end = calendar.timegm(start_date.utctimetuple())

        result = db.execute(query, (start, end))
        first = result.fetchone()
        if first is None:
            print "Skipping", start_date
            continue
        first = float(first[0])
        o, h, l, c = first, first, first, first
        for trade in result:
            price = float(trade[0])
            c = price
            h = max(h, price)
            l = min(l, price)

        print start_date
        yield o, h, l, c

def do_the_plot(candle, sf_top):
    fig = pylab.figure()
    ax = fig.add_subplot(111)

    dl_factor = 100 # Download counts will be divided by this amount
    plot_sf_stats(sf_top, ax, fig, factor=dl_factor)
    plot_candles(ax, candle)
    xlim = ax.get_xlim()
    ax.set_xlim(xlim[0] - 1, xlim[1] + 1)

    ax.grid()

    fig.suptitle(
            u'Bitcoin-QT download stats (%g x) mixed with MtGox USD price' %
            dl_factor)

    box = ax.get_position()
    ax.set_position([box.x0, box.y0, box.width * 0.85, box.height])
    ax.legend(loc='upper left', bbox_to_anchor=(1, 1))

    pylab.show()

def plot_candles(ax, candle):
    cs = Candlestick(ax)
    for o, h, l, c in candle:
        cs.append_candle(o, h, l, c)

def plot_sf_stats(top, ax, fig, factor):
    factor = float(factor)

    x = []
    total = []
    other = [[] for _ in xrange(len(top))]
    for key in sorted(sf_daily_stats):
        val = sf_daily_stats[key]
        x.append(key)
        total.append(val['total'] / factor)
        for data, name in zip(other, top):
            if name in val:
                data.append(val[name] / factor)
            else:
                print '%s missing at %s' % (name, key)
                data.append(0)

    ax.xaxis.set_major_formatter(dates.DateFormatter('%Y-%m-%d'))
    # Labels on mondays.
    ax.xaxis.set_major_locator(dates.WeekdayLocator(byweekday=dates.MO))

    #ax.set_ylabel(u'Download count (%g x)' % factor)

    ax.plot(x, total, label=u'Total DL')
    for data, name in zip(other, top):
        ax.plot(x, data, label=u'%s DL' % name)

    fig.autofmt_xdate()

if __name__ == "__main__":
    start_date = datetime(year=2013, month=2, day=1)
    end_date = datetime.fromordinal(date.today().toordinal())
    end_date += timedelta(days=1)

    sf_stats(start_date, end_date)
    reactor.run()

    # Find out the top two countries in terms of download count.
    from collections import defaultdict
    dl_country = defaultdict(int)
    for _, v in sf_daily_stats.iteritems():
        for country, dlcount in v.iteritems():
            if country == 'total':
                # Not really a country.
                continue
            dl_country[country] += dlcount
    n = 2
    top_n = sorted(((v, k) for k, v in dl_country.items()), reverse=True)[:n]
    print top_n
    top_n = [k for v, k in top_n]

    candle = ohlc_stats(start_date, end_date)

    # You might want to save sf_daily_stats
    #print sf_daily_stats
    do_the_plot(candle, top_n)
<|endoftext|>


<commit_before>from urllib.request import urlopen
from io import BytesIO
import json
import random
from PIL import Image

def get_url(url):
    '''get json page data using a specified eol API url'''
    response = urlopen(url)
    data = str(response.read().decode('utf-8'))
    page = json.loads(data)
    return page

class Question(object):
    def __init__(self, data):
        self.species = random.sample(data,3)
        self.answer = random.choice(self.species)
        self.picture = random.choice(self.answer.images_list)
        self.guess = None
        self.correct = None

    def make_guess(self, guess_species):
        if guess == None:
            self.guess = guess_species
        else:
            return

        if guess_species == self.answer:
            self.correct = True
        else:
            self.correct = False

class Species(object):
    '''Creates a new species object that stores scientific name, common name and images\
    from an eol page '''

    def __init__(self, scientific_name, common_name, images_list):
        self.scientific_name = scientific_name
        self.common_name = common_name
        self.images_list = images_list

    def show_image(self):
        response = urlopen(random.choice(self.images_list))
        img = Image.open(BytesIO(response.read()))
        img.show()

    @classmethod
    def from_eolid(cls, eolid):
        '''class method that creates a Species class from a provided eol ID'''

        url = 'http://eol.org/api/pages/1.0/{}.json?images=2&videos=0&sounds=0&maps=0&text=2&iucn=false&subjects=overview&licenses=all&details=true&common_names=true&synonyms=true&references=true&vetted=0&cache_ttl='.format(eolid)

        page = get_url(url)

        images_list = [objects.get('mediaURL') for objects in page['dataObjects']
                        if objects.get('mediaURL') is not None]

        scientific_name = page['scientificName']

        common_name = [name['vernacularName'] for name in page['vernacularNames']
                        if name['language'] == 'en' and name.get('eol_preferred') == True][0]

        return cls(scientific_name, common_name, images_list)

    def __repr__(self):
        return "{c} ({s})".format(c=self.common_name, s=self.scientific_name)

#search url for the eol 'hotlist' collection, first 500 pages sorted by richness
search_url = 'http://eol.org/api/collections/1.0/55422.json?page=1&per_page=500&filter=&sort_by=richness&sort_field=&cache_ttl='

#ping the API to get the json data for these pages
top500 = get_url(search_url)

#create a species list that contains the object ID for each species in the top500
species_ID_list = [item['object_id'] for item in top500['collection_items']]

#create a list of 3 random species IDs
random_list = random.sample(species_ID_list,3)
<commit_msg>adds game class<commit_after>from urllib.request import urlopen
from io import BytesIO
import json
import random
from PIL import Image

def get_url(url):
    '''get json page data using a specified eol API url'''
    response = urlopen(url)
    data = str(response.read().decode('utf-8'))
    page = json.loads(data)
    return page

class Question(object):
    def __init__(self, data):
        self.species = random.sample(data,3)
        self.answer = random.choice(self.species)
        self.picture = random.choice(self.answer.images_list)
        self.guess = None
        self.correct = None

    def verify(self, guess_species):
        if guess == None:
            self.guess = guess_species
        else:
            return
        self.correct = (guess_species == self.answer)

class Game(object):
    def __init__(self, data, total_questions):
        self.score = 0
        self.question_number = 0
        self.total_questions = total_questions
        self.questions = [Question(data) for i in range(self.total_questions)]

    def play(self):
        for question in self.questions:
            display_question(question)
            get_guess(question)
            input()
        display_final()

    def display_question(self, question):
        response = urlopen(question.picture)
        img = Image.open(BytesIO(response.read()))
        img.show()

        for choice_num, species in enumerate(question.species, start = 1):
            print("{}. {}".format(choice_num, species.scientific_name))

    def get_guess(self, question):
        input()

class Species(object):
    '''Creates a new species object that stores scientific name, common name and images\
    from an eol page '''

    def __init__(self, scientific_name, common_name, images_list):
        self.scientific_name = scientific_name
        self.common_name = common_name
        self.images_list = images_list

    def show_image(self):
        response = urlopen(random.choice(self.images_list))
        img = Image.open(BytesIO(response.read()))
        img.show()

    @classmethod
    def from_eolid(cls, eolid):
        '''class method that creates a Species class from a provided eol ID'''

        url = 'http://eol.org/api/pages/1.0/{}.json?images=2&videos=0&sounds=0&maps=0&text=2&iucn=false&subjects=overview&licenses=all&details=true&common_names=true&synonyms=true&references=true&vetted=0&cache_ttl='.format(eolid)

        page = get_url(url)

        images_list = [objects.get('mediaURL') for objects in page['dataObjects']
                        if objects.get('mediaURL') is not None]

        scientific_name = page['scientificName']

        common_name = [name['vernacularName'] for name in page['vernacularNames']
                        if name['language'] == 'en' and name.get('eol_preferred') == True][0]

        return cls(scientific_name, common_name, images_list)

    def __repr__(self):
        return "{c} ({s})".format(c=self.common_name, s=self.scientific_name)

#search url for the eol 'hotlist' collection, first 500 pages sorted by richness
search_url = 'http://eol.org/api/collections/1.0/55422.json?page=1&per_page=500&filter=&sort_by=richness&sort_field=&cache_ttl='

#ping the API to get the json data for these pages
top500 = get_url(search_url)

#create a species list that contains the object ID for each species in the top500
species_ID_list = [item['object_id'] for item in top500['collection_items']]

#create a list of 3 random species IDs
random_list = random.sample(species_ID_list,3)
<|endoftext|>


<commit_before>"""@author: Bryan Silverthorn <bcs@cargo-cult.org>"""

import os.path
import csv
import zlib
import base64
import cPickle as pickle
import numpy
import condor
import borg

logger = borg.get_logger(__name__, default_level = "INFO")

def run_solver_on(suite_path, solver_name, task_path, budget, store_answers):
    """Run a solver."""

    if condor.get_task() is not None:
        borg.statistics.set_prng_seeds(hash(condor.get_task().key))

    suite = borg.load_solvers(suite_path)

    with suite.domain.task_from_path(task_path) as task:
        with borg.accounting() as accountant:
            answer = suite.solvers[solver_name](task)(budget)

        succeeded = suite.domain.is_final(task, answer)

    cost = accountant.total.cpu_seconds

    logger.info(
        "%s %s in %.2f (of %.2f) on %s",
        solver_name,
        "succeeded" if succeeded else "failed",
        cost,
        budget,
        os.path.basename(task_path),
        )

    if store_answers:
        return (solver_name, budget, cost, succeeded, answer)
    else:
        return (solver_name, budget, cost, succeeded, None)

@borg.annotations(
    suite_path = ("path to the solvers suite", "positional", None, os.path.abspath),
    tasks_root = ("path to task files", "positional", None, os.path.abspath),
    budget = ("per-instance budget", "positional", None, float),
    only_missing = ("only make missing runs", "flag"),
    store_answers = ("store answers to instances", "flag"),
    only_solver = ("only make runs of one solver", "option"),
    runs = ("number of runs", "option", "r", int),
    suffix = ("runs file suffix", "option"),
    workers = ("submit jobs?", "option", "w", int),
    )
def main(
    suite_path,
    tasks_root,
    budget,
    only_missing = False,
    store_answers = False,
    only_solver = None,
    runs = 4,
    suffix = ".runs.csv",
    workers = 0,
    ):
    """Collect solver running-time data."""

    condor.defaults.condor_matching = \
        "InMastodon" \
        " && regexp(\"rhavan-.*\", ParallelSchedulingGroup)" \
        " && (Arch == \"X86_64\")" \
        " && (OpSys == \"LINUX\")" \
        " && (Memory > 1024)"

    def yield_runs():
        suite = borg.load_solvers(suite_path)

        logger.info("scanning paths under %s", tasks_root)

        paths = list(borg.util.files_under(tasks_root, suite.domain.extensions))

        if not paths:
            raise ValueError("no paths found under specified root")

        if only_solver is None:
            solver_names = suite.solvers.keys()
        else:
            solver_names = [only_solver]

        for path in paths:
            run_data = None

            if only_missing and os.path.exists(path + suffix):
                run_data = numpy.recfromcsv(path + suffix, usemask = True)

            for solver_name in solver_names:
                if only_missing and run_data is not None:
                    count = runs - numpy.sum(run_data.solver == solver_name)
                else:
                    count = runs

                logger.info("scheduling %i run(s) of %s on %s", count, solver_name, os.path.basename(path))

                for _ in xrange(count):
                    yield (run_solver_on, [suite_path, solver_name, path, budget, store_answers])

    for (task, row) in condor.do(yield_runs(), workers):
        # unpack run outcome
        (solver_name, budget, cost, succeeded, answer) = row

        if answer is None:
            answer_text = None
        else:
            answer_text = base64.b64encode(zlib.compress(pickle.dumps(answer)))

        # write it to disk
        (_, _, cnf_path, _, _) = task.args
        csv_path = cnf_path + suffix
        existed = os.path.exists(csv_path)

        with open(csv_path, "a") as csv_file:
            writer = csv.writer(csv_file)

            if not existed:
                writer.writerow(["solver", "budget", "cost", "succeeded", "answer"])

            writer.writerow([solver_name, budget, cost, succeeded, answer_text])

if __name__ == "__main__":
    borg.script(main)

<commit_msg>Fix output under -only_missing.<commit_after>"""@author: Bryan Silverthorn <bcs@cargo-cult.org>"""

import os.path
import csv
import zlib
import base64
import cPickle as pickle
import numpy
import condor
import borg

logger = borg.get_logger(__name__, default_level = "INFO")

def run_solver_on(suite_path, solver_name, task_path, budget, store_answers):
    """Run a solver."""

    if condor.get_task() is not None:
        borg.statistics.set_prng_seeds(hash(condor.get_task().key))

    suite = borg.load_solvers(suite_path)

    with suite.domain.task_from_path(task_path) as task:
        with borg.accounting() as accountant:
            answer = suite.solvers[solver_name](task)(budget)

        succeeded = suite.domain.is_final(task, answer)

    cost = accountant.total.cpu_seconds

    logger.info(
        "%s %s in %.2f (of %.2f) on %s",
        solver_name,
        "succeeded" if succeeded else "failed",
        cost,
        budget,
        os.path.basename(task_path),
        )

    if store_answers:
        return (solver_name, budget, cost, succeeded, answer)
    else:
        return (solver_name, budget, cost, succeeded, None)

@borg.annotations(
    suite_path = ("path to the solvers suite", "positional", None, os.path.abspath),
    tasks_root = ("path to task files", "positional", None, os.path.abspath),
    budget = ("per-instance budget", "positional", None, float),
    only_missing = ("only make missing runs", "flag"),
    store_answers = ("store answers to instances", "flag"),
    only_solver = ("only make runs of one solver", "option"),
    runs = ("number of runs", "option", "r", int),
    suffix = ("runs file suffix", "option"),
    workers = ("submit jobs?", "option", "w", int),
    )
def main(
    suite_path,
    tasks_root,
    budget,
    only_missing = False,
    store_answers = False,
    only_solver = None,
    runs = 4,
    suffix = ".runs.csv",
    workers = 0,
    ):
    """Collect solver running-time data."""

    condor.defaults.condor_matching = \
        "InMastodon" \
        " && regexp(\"rhavan-.*\", ParallelSchedulingGroup)" \
        " && (Arch == \"X86_64\")" \
        " && (OpSys == \"LINUX\")" \
        " && (Memory > 1024)"

    def yield_runs():
        suite = borg.load_solvers(suite_path)

        logger.info("scanning paths under %s", tasks_root)

        paths = list(borg.util.files_under(tasks_root, suite.domain.extensions))

        if not paths:
            raise ValueError("no paths found under specified root")

        if only_solver is None:
            solver_names = suite.solvers.keys()
        else:
            solver_names = [only_solver]

        for path in paths:
            run_data = None

            if only_missing and os.path.exists(path + suffix):
                run_data = numpy.recfromcsv(path + suffix, usemask = True)

            for solver_name in solver_names:
                if only_missing and run_data is not None:
                    count = max(0, runs - numpy.sum(run_data.solver == solver_name))
                else:
                    count = runs

                logger.info("scheduling %i run(s) of %s on %s", count, solver_name, os.path.basename(path))

                for _ in xrange(count):
                    yield (run_solver_on, [suite_path, solver_name, path, budget, store_answers])

    for (task, row) in condor.do(yield_runs(), workers):
        # unpack run outcome
        (solver_name, budget, cost, succeeded, answer) = row

        if answer is None:
            answer_text = None
        else:
            answer_text = base64.b64encode(zlib.compress(pickle.dumps(answer)))

        # write it to disk
        (_, _, cnf_path, _, _) = task.args
        csv_path = cnf_path + suffix
        existed = os.path.exists(csv_path)

        with open(csv_path, "a") as csv_file:
            writer = csv.writer(csv_file)

            if not existed:
                writer.writerow(["solver", "budget", "cost", "succeeded", "answer"])

            writer.writerow([solver_name, budget, cost, succeeded, answer_text])

if __name__ == "__main__":
    borg.script(main)

<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

import tensorflow as tf

def get_options(option_type):
  """
  option_type: string
    'training' or 'diplay' or 'visualize'
  """
  # name
  tf.app.flags.DEFINE_string("training_name","aux_test_causv1","name of next training in log")

  # Common
  tf.app.flags.DEFINE_string("env_type", "gym", "environment type (lab or gym or maze)")
  tf.app.flags.DEFINE_string("env_name", "CartPole-v1",  "environment name (for lab)")
  tf.app.flags.DEFINE_integer("env_max_steps", 400000, "max number of steps in environment")

  tf.app.flags.DEFINE_boolean("use_base", False, "whether to use base A3C for aux network")
  tf.app.flags.DEFINE_boolean("use_pixel_change", False, "whether to use pixel change")
  tf.app.flags.DEFINE_boolean("use_value_replay", False, "whether to use value function replay")
  tf.app.flags.DEFINE_boolean("use_reward_prediction", False, "whether to use reward prediction")
  tf.app.flags.DEFINE_boolean("use_temporal_coherence", False, "whether to use temporal coherence")
  tf.app.flags.DEFINE_boolean("use_proportionality", False, "whether to use proportionality")
  tf.app.flags.DEFINE_boolean("use_causality", True, "whether to use causality")
  tf.app.flags.DEFINE_string("checkpoint_dir", "/tmp/StRADRL/checkpoints", "checkpoint directory")

  # For training
  if option_type == 'training':
    tf.app.flags.DEFINE_string("temp_dir", "/tmp/StRADRL/tensorboard/", "base directory for tensorboard")
    tf.app.flags.DEFINE_string("log_dir", "/tmp/StRADRL/log/", "base directory for logs")
    tf.app.flags.DEFINE_integer("max_time_step", 10**8, "max time steps")
    tf.app.flags.DEFINE_integer("save_interval_step", 10**4, "saving interval steps")
    tf.app.flags.DEFINE_boolean("grad_norm_clip", 40.0, "gradient norm clipping")

    #base
    tf.app.flags.DEFINE_float("initial_learning_rate", 1e-4, "learning rate")
    tf.app.flags.DEFINE_float("gamma", 0.97, "discount factor for rewards")
    tf.app.flags.DEFINE_float("entropy_beta", 0.01, "entropy regurarlization constant")
    tf.app.flags.DEFINE_float("value_lambda", 0.01, "value ratio for base loss")
    tf.app.flags.DEFINE_float("base_lambda", 0.97, "generalized adv. est. lamba for short-long sight")

    # auxiliary
    tf.app.flags.DEFINE_integer("parallel_size", 1, "parallel thread size")
    tf.app.flags.DEFINE_float("aux_initial_learning_rate", 1e-4, "learning rate")
    tf.app.flags.DEFINE_float("aux_lambda", 0.0, "generalized adv. est. lamba for short-long sight (aux)")
    tf.app.flags.DEFINE_float("gamma_pc", 0.9, "discount factor for pixel control")
    tf.app.flags.DEFINE_float("pixel_change_lambda", 0.0001, "pixel change lambda") # 0.05, 0.01 ~ 0.1 for lab, 0.0001 ~ 0.01 for gym
    tf.app.flags.DEFINE_float("temporal_coherence_lambda", 1., "temporal coherence lambda")
    tf.app.flags.DEFINE_float("proportionality_lambda", 100., "proportionality lambda")
    tf.app.flags.DEFINE_float("causality_lambda", 1., "causality lambda")

    tf.app.flags.DEFINE_integer("experience_history_size", 100000, "experience replay buffer size")

    # queuer
    tf.app.flags.DEFINE_integer("local_t_max", 20, "repeat step size")
    tf.app.flags.DEFINE_integer("queue_length", 5, "max number of batches (of length local_t_max) in queue")
    tf.app.flags.DEFINE_integer("env_runner_sync", 1, "number of env episodes before sync to global")
    tf.app.flags.DEFINE_float("action_freq", 0,  "number of actions per second in env")

  # For display
  if option_type == 'display':
    tf.app.flags.DEFINE_string("frame_save_dir", "/tmp/StRADRL_frames", "frame save directory")
    tf.app.flags.DEFINE_boolean("recording", False, "whether to record movie")
    tf.app.flags.DEFINE_boolean("frame_saving", False, "whether to save frames")

  return tf.app.flags.FLAGS
<commit_msg>added causality prior and fixed confilct with laptop<commit_after># -*- coding: utf-8 -*-
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

import tensorflow as tf

def get_options(option_type):
  """
  option_type: string
    'training' or 'diplay' or 'visualize'
  """
  # name
  tf.app.flags.DEFINE_string("training_name","aux_test_causv1","name of next training in log")

  # Common
  tf.app.flags.DEFINE_string("env_type", "gym", "environment type (lab or gym or maze)")
  tf.app.flags.DEFINE_string("env_name", "CartPole-v1",  "environment name (for lab)")
  tf.app.flags.DEFINE_integer("env_max_steps", 400000, "max number of steps in environment")

  tf.app.flags.DEFINE_boolean("use_base", False, "whether to use base A3C for aux network")
  tf.app.flags.DEFINE_boolean("use_pixel_change", False, "whether to use pixel change")
  tf.app.flags.DEFINE_boolean("use_value_replay", False, "whether to use value function replay")
  tf.app.flags.DEFINE_boolean("use_reward_prediction", False, "whether to use reward prediction")
  tf.app.flags.DEFINE_boolean("use_temporal_coherence", False, "whether to use temporal coherence")
  tf.app.flags.DEFINE_boolean("use_proportionality", False, "whether to use proportionality")
  tf.app.flags.DEFINE_boolean("use_causality", True, "whether to use causality")
  tf.app.flags.DEFINE_string("checkpoint_dir", "/tmp/StRADRL/checkpoints", "checkpoint directory")

  # For training
  if option_type == 'training':
    tf.app.flags.DEFINE_string("temp_dir", "/tmp/StRADRL/tensorboard/", "base directory for tensorboard")
    tf.app.flags.DEFINE_string("log_dir", "/tmp/StRADRL/log/", "base directory for logs")
    tf.app.flags.DEFINE_integer("max_time_step", 10**8, "max time steps")
    tf.app.flags.DEFINE_integer("save_interval_step", 10**4, "saving interval steps")
    tf.app.flags.DEFINE_boolean("grad_norm_clip", 40.0, "gradient norm clipping")

    #base
    tf.app.flags.DEFINE_float("initial_learning_rate", 1e-4, "learning rate")
    tf.app.flags.DEFINE_float("gamma", 0.97, "discount factor for rewards")
    tf.app.flags.DEFINE_float("entropy_beta", 0.01, "entropy regurarlization constant")
    tf.app.flags.DEFINE_float("value_lambda", 0.01, "value ratio for base loss")
    tf.app.flags.DEFINE_float("base_lambda", 0.97, "generalized adv. est. lamba for short-long sight")

    # auxiliary
    tf.app.flags.DEFINE_integer("parallel_size", 1, "parallel thread size")
    tf.app.flags.DEFINE_float("aux_initial_learning_rate", 1e-4, "learning rate")
    tf.app.flags.DEFINE_float("aux_lambda", 0.0, "generalized adv. est. lamba for short-long sight (aux)")
    tf.app.flags.DEFINE_float("gamma_pc", 0.9, "discount factor for pixel control")
    tf.app.flags.DEFINE_float("pixel_change_lambda", 0.0001, "pixel change lambda") # 0.05, 0.01 ~ 0.1 for lab, 0.0001 ~ 0.01 for gym
    tf.app.flags.DEFINE_float("temporal_coherence_lambda", 1., "temporal coherence lambda")
    tf.app.flags.DEFINE_float("proportionality_lambda", 100., "proportionality lambda")
    tf.app.flags.DEFINE_float("causality_lambda", 1., "causality lambda")

    tf.app.flags.DEFINE_integer("experience_history_size", 100000, "experience replay buffer size")

    # queuer
    tf.app.flags.DEFINE_integer("local_t_max", 20, "repeat step size")
    tf.app.flags.DEFINE_integer("queue_length", 5, "max number of batches (of length local_t_max) in queue")
    tf.app.flags.DEFINE_integer("env_runner_sync", 1, "number of env episodes before sync to global")
    tf.app.flags.DEFINE_float("action_freq", 0,  "number of actions per second in env")

  # For display
  if option_type == 'display':
    tf.app.flags.DEFINE_string("frame_save_dir", "/tmp/StRADRL_frames", "frame save directory")
    tf.app.flags.DEFINE_boolean("recording", False, "whether to record movie")
    tf.app.flags.DEFINE_boolean("frame_saving", False, "whether to save frames")

  return tf.app.flags.FLAGS
<|endoftext|>


<commit_before>import logging
import os.path
import sys
import tarfile
import zipfile
import shutil

logger = logging.getLogger(__name__)

if sys.version_info[0] >= 3:
    import urllib.request as urllib
    from urllib.error import URLError
    from urllib.error import HTTPError
else:
    import urllib
    from urllib2 import URLError
    from urllib2 import HTTPError

from fusesoc.provider.provider import Provider

class Url(Provider):

    def _checkout(self, local_dir):
        url = self.config.get('url')
        logger.info("Downloading...")
        user_agent = self.config.get('user-agent')
        if user_agent:
            opener = urllib.build_opener()
            opener.addheaders = [('User-agent', user_agent)]
            urllib.install_opener(opener)
        try:
            (filename, headers) = urllib.urlretrieve(url)
        except (URLError, HTTPError) as e:
            raise RuntimeError("Failed to download '{}'. '{}'".format(url, e.reason))

        filetype = self.config.get('filetype')
        if filetype == 'tar':
            t = tarfile.open(filename)
            t.extractall(local_dir)
        elif filetype == 'zip':
            with zipfile.ZipFile(filename, "r") as z:
                z.extractall(local_dir)
        elif filetype == 'simple':
            _filename = url.rsplit('/', 1)[1]
            os.makedirs(local_dir)
            shutil.copy2(filename, os.path.join(local_dir, _filename))
        else:
            raise RuntimeError("Unknown file type '" + filetype + "' in [provider] section")
<commit_msg>Temporarily disable support for setting user-agent on python2<commit_after>import logging
import os.path
import sys
import tarfile
import zipfile
import shutil

logger = logging.getLogger(__name__)

if sys.version_info[0] >= 3:
    import urllib.request as urllib
    from urllib.error import URLError
    from urllib.error import HTTPError
else:
    import urllib
    from urllib2 import URLError
    from urllib2 import HTTPError

from fusesoc.provider.provider import Provider

class Url(Provider):

    def _checkout(self, local_dir):
        url = self.config.get('url')
        logger.info("Downloading...")
        user_agent = self.config.get('user-agent')
        if user_agent and sys.version_info[0] >= 3:
            opener = urllib.build_opener()
            opener.addheaders = [('User-agent', user_agent)]
            urllib.install_opener(opener)
        try:
            (filename, headers) = urllib.urlretrieve(url)
        except (URLError, HTTPError) as e:
            raise RuntimeError("Failed to download '{}'. '{}'".format(url, e.reason))

        filetype = self.config.get('filetype')
        if filetype == 'tar':
            t = tarfile.open(filename)
            t.extractall(local_dir)
        elif filetype == 'zip':
            with zipfile.ZipFile(filename, "r") as z:
                z.extractall(local_dir)
        elif filetype == 'simple':
            _filename = url.rsplit('/', 1)[1]
            os.makedirs(local_dir)
            shutil.copy2(filename, os.path.join(local_dir, _filename))
        else:
            raise RuntimeError("Unknown file type '" + filetype + "' in [provider] section")
<|endoftext|>


<commit_before>"""
This module provides access to thread pools to use in concurrent work.
Both a multiprocessing style ThreadPool and an Executor based pool
are available, Executors should be preferred for new work as it provides
a more modern interface.

To use these wrappers for single thread environment, set the following:

    synapseclient.config.single_threaded = True
"""

from concurrent.futures import Executor, Future, ThreadPoolExecutor
import multiprocessing
import multiprocessing.dummy

from . import config

DEFAULT_POOL_SIZE = 8

class SingleThreadPool:

    def map(self, func, iterable):
        for item in iterable:
            func(item)

    def terminate(self):
        pass

class SingleThreadExecutor(Executor):
    def __init__(self):
        self._shutdown = False

    def submit(self, fn, *args, **kwargs):
       if self._shutdown:
           raise RuntimeError('cannot schedule new futures after shutdown')

       f = Future()
       try:
           result = fn(*args, **kwargs)
       except BaseException as e:
           f.set_exception(e)
       else:
           f.set_result(result)

       return f

class FakeLock:

    def __enter__(self):
        pass

    def __exit__(self, type, value, traceback):
        pass

class SingleValue:

    value = None

    def __init__(self, type, value):
        self.value = value

    def get_lock(self):
        return FakeLock()

def get_pool():
    if config.single_threaded:
        return SingleThreadPool()
    else:
        return multiprocessing.dummy.Pool(DEFAULT_POOL_SIZE)

def get_executor(thread_count=DEFAULT_POOL_SIZE):
    """
    Provides an Executor as defined by the client config suitable
    for running tasks work as defined by the client config.

    :param thread_count:        number of concurrent threads

    :return: an Executor
    """
    if config.single_threaded:
        return SingleThreadExecutor()
    else:
        return ThreadPoolExecutor(max_workers=thread_count)

def get_value(type, value):
    if config.single_threaded:
        return SingleValue(type, value)
    else:
        return multiprocessing.Value(type, value)
<commit_msg>fix indent<commit_after>"""
This module provides access to thread pools to use in concurrent work.
Both a multiprocessing style ThreadPool and an Executor based pool
are available, Executors should be preferred for new work as it provides
a more modern interface.

To use these wrappers for single thread environment, set the following:

    synapseclient.config.single_threaded = True
"""

from concurrent.futures import Executor, Future, ThreadPoolExecutor
import multiprocessing
import multiprocessing.dummy

from . import config

DEFAULT_POOL_SIZE = 8

class SingleThreadPool:

    def map(self, func, iterable):
        for item in iterable:
            func(item)

    def terminate(self):
        pass

class SingleThreadExecutor(Executor):
    def __init__(self):
        self._shutdown = False

    def submit(self, fn, *args, **kwargs):
        if self._shutdown:
            raise RuntimeError('cannot schedule new futures after shutdown')

        f = Future()
        try:
            result = fn(*args, **kwargs)
        except BaseException as e:
            f.set_exception(e)
        else:
            f.set_result(result)

        return f

class FakeLock:

    def __enter__(self):
        pass

    def __exit__(self, type, value, traceback):
        pass

class SingleValue:

    value = None

    def __init__(self, type, value):
        self.value = value

    def get_lock(self):
        return FakeLock()

def get_pool():
    if config.single_threaded:
        return SingleThreadPool()
    else:
        return multiprocessing.dummy.Pool(DEFAULT_POOL_SIZE)

def get_executor(thread_count=DEFAULT_POOL_SIZE):
    """
    Provides an Executor as defined by the client config suitable
    for running tasks work as defined by the client config.

    :param thread_count:        number of concurrent threads

    :return: an Executor
    """
    if config.single_threaded:
        return SingleThreadExecutor()
    else:
        return ThreadPoolExecutor(max_workers=thread_count)

def get_value(type, value):
    if config.single_threaded:
        return SingleValue(type, value)
    else:
        return multiprocessing.Value(type, value)
<|endoftext|>


<commit_before>"""@author: Bryan Silverthorn <bcs@cargo-cult.org>"""

import os.path
import csv
import zlib
import base64
import cPickle as pickle
import numpy
import condor
import borg

logger = borg.get_logger(__name__)

def run_solver_on(suite_path, solver_name, task_path, budget):
    """Run a solver."""

    borg.statistics.set_prng_keys(hash(condor.get_task().key))

    suite = borg.load_solvers(suite_path)

    with suite.domain.task_from_path(task_path) as task:
        with borg.accounting() as accountant:
            answer = suite.solvers[solver_name](task)(budget)

        succeeded = suite.domain.is_final(task, answer)

    cost = accountant.total.cpu_seconds

    logger.info(
        "%s %s in %.2f (of %.2f) on %s",
        solver_name,
        "succeeded" if succeeded else "failed",
        cost,
        budget,
        os.path.basename(task_path),
        )

    return (solver_name, budget, cost, succeeded, answer)

def appender(items):
    def append(item):
        items.append(item)

        return items

    return append

@borg.annotations(
    suite_path = ("path to the solvers suite", "positional", None, os.path.abspath),
    tasks_root = ("path to task files", "positional", None, os.path.abspath),
    budget = ("per-instance budget", "positional", None, float),
    only_missing = ("only make missing runs", "flag"),
    only_solver = ("only make runs of one solver", "option"),
    runs = ("number of runs", "option", "r", int),
    suffix = ("runs file suffix", "option"),
    workers = ("submit jobs?", "option", "w", int),
    )
def main(
    suite_path,
    tasks_root,
    budget,
    only_missing = False,
    only_solver = None,
    runs = 4,
    suffix = ".runs.csv",
    workers = 0,
    ):
    """Collect solver running-time data."""

    condor.defaults.condor_matching = \
        "InMastodon" \
        " && regexp(\"rhavan-.*\", ParallelSchedulingGroup)" \
        " && (Arch == \"X86_64\")" \
        " && (OpSys == \"LINUX\")" \
        " && (Memory > 1024)"

    def yield_runs():
        suite = borg.load_solvers(suite_path)
        paths = list(borg.util.files_under(tasks_root, suite.domain.extensions))

        if not paths:
            raise ValueError("no paths found under specified root")

        if only_solver is None:
            solver_names = suite.solvers.keys()
        else:
            solver_names = [only_solver]

        for path in paths:
            run_data = None

            if only_missing and os.path.exists(path + suffix):
                run_data = numpy.recfromcsv(path + suffix, usemask = True)

            for solver_name in solver_names:
                if only_missing and run_data is not None:
                    count = runs - numpy.sum(run_data.solver == solver_name)
                else:
                    count = runs

                logger.info("scheduling %i run(s) of %s on %s", count, solver_name, os.path.basename(path))

                for _ in xrange(count):
                    yield (run_solver_on, [suite_path, solver_name, path, budget])

    def collect_run(task, row):
        # unpack run outcome
        (solver_name, budget, cost, succeeded, answer) = row

        if answer is None:
            answer_text = None
        else:
            answer_text = base64.b64encode(zlib.compress(pickle.dumps(answer)))

        # write it to disk
        (_, _, cnf_path, _) = task.args
        csv_path = cnf_path + suffix
        existed = os.path.exists(csv_path)

        with open(csv_path, "a") as csv_file:
            writer = csv.writer(csv_file)

            if not existed:
                writer.writerow(["solver", "budget", "cost", "succeeded", "answer"])

            writer.writerow([solver_name, budget, cost, succeeded, answer_text])

    condor.do(yield_runs(), workers, collect_run)

if __name__ == "__main__":
    borg.script(main)

<commit_msg>Fix run_solvers() after update.)<commit_after>"""@author: Bryan Silverthorn <bcs@cargo-cult.org>"""

import os.path
import csv
import zlib
import base64
import cPickle as pickle
import numpy
import condor
import borg

logger = borg.get_logger(__name__, default_level = "INFO")

def run_solver_on(suite_path, solver_name, task_path, budget):
    """Run a solver."""

    borg.statistics.set_prng_keys(hash(condor.get_task().key))

    suite = borg.load_solvers(suite_path)

    with suite.domain.task_from_path(task_path) as task:
        with borg.accounting() as accountant:
            answer = suite.solvers[solver_name](task)(budget)

        succeeded = suite.domain.is_final(task, answer)

    cost = accountant.total.cpu_seconds

    logger.info(
        "%s %s in %.2f (of %.2f) on %s",
        solver_name,
        "succeeded" if succeeded else "failed",
        cost,
        budget,
        os.path.basename(task_path),
        )

    return (solver_name, budget, cost, succeeded, answer)

def appender(items):
    def append(item):
        items.append(item)

        return items

    return append

@borg.annotations(
    suite_path = ("path to the solvers suite", "positional", None, os.path.abspath),
    tasks_root = ("path to task files", "positional", None, os.path.abspath),
    budget = ("per-instance budget", "positional", None, float),
    only_missing = ("only make missing runs", "flag"),
    only_solver = ("only make runs of one solver", "option"),
    runs = ("number of runs", "option", "r", int),
    suffix = ("runs file suffix", "option"),
    workers = ("submit jobs?", "option", "w", int),
    )
def main(
    suite_path,
    tasks_root,
    budget,
    only_missing = False,
    only_solver = None,
    runs = 4,
    suffix = ".runs.csv",
    workers = 0,
    ):
    """Collect solver running-time data."""

    condor.defaults.condor_matching = \
        "InMastodon" \
        " && regexp(\"rhavan-.*\", ParallelSchedulingGroup)" \
        " && (Arch == \"X86_64\")" \
        " && (OpSys == \"LINUX\")" \
        " && (Memory > 1024)"

    def yield_runs():
        suite = borg.load_solvers(suite_path)

        logger.info("scanning paths under %s", tasks_root)

        paths = list(borg.util.files_under(tasks_root, suite.domain.extensions))

        if not paths:
            raise ValueError("no paths found under specified root")

        if only_solver is None:
            solver_names = suite.solvers.keys()
        else:
            solver_names = [only_solver]

        for path in paths:
            run_data = None

            if only_missing and os.path.exists(path + suffix):
                run_data = numpy.recfromcsv(path + suffix, usemask = True)

            for solver_name in solver_names:
                if only_missing and run_data is not None:
                    count = runs - numpy.sum(run_data.solver == solver_name)
                else:
                    count = runs

                logger.info("scheduling %i run(s) of %s on %s", count, solver_name, os.path.basename(path))

                for _ in xrange(count):
                    yield (run_solver_on, [suite_path, solver_name, path, budget])

    for (task, row) in condor.do(yield_runs(), workers):
        # unpack run outcome
        (solver_name, budget, cost, succeeded, answer) = row

        if answer is None:
            answer_text = None
        else:
            answer_text = base64.b64encode(zlib.compress(pickle.dumps(answer)))

        # write it to disk
        (_, _, cnf_path, _) = task.args
        csv_path = cnf_path + suffix
        existed = os.path.exists(csv_path)

        with open(csv_path, "a") as csv_file:
            writer = csv.writer(csv_file)

            if not existed:
                writer.writerow(["solver", "budget", "cost", "succeeded", "answer"])

            writer.writerow([solver_name, budget, cost, succeeded, answer_text])

if __name__ == "__main__":
    borg.script(main)

<|endoftext|>


<commit_before>'''
synbiochem (c) University of Manchester 2015

synbiochem is licensed under the MIT License.

To view a copy of this license, visit <http://opensource.org/licenses/MIT/>.

@author:  neilswainston
'''
from matplotlib.colors import LinearSegmentedColormap
import numpy
import pylab
import scipy.spatial
import sys
import tempfile
import urllib

from Bio.PDB.PDBParser import PDBParser

# KD Hydrophobicity, EIIP, Helix, Sheet, Turn
AMINO_ACID_PROPERTIES = {
    'A': [1.8, -0.0667, 32.9, -23.6, -41.6],
    'R': [-4.5, 0.2674, 0, -6.2, -5.1],
    'N': [-3.5, -0.2589, -24.8, -41.6, 44.5],
    'D': [-3.5, 0.4408, 5.8, -41.6, 37.8],
    'C': [2.5, 0.1933, -5.1, 6.8, 17.4],
    'Q': [-3.5, 0.1545, 11.3, 0, -2],
    'E': [-3.5, -0.2463, 36.5, -67.3, -30.1],
    'G': [-0.4, -0.2509, -46.2, -13.9, 44.5],
    'H': [-3.2, -0.1414, 11.3, -18.6, -5.1],
    'I': [4.5, -0.2794, -1, 45.1, -75.5],
    'L': [3.8, -0.2794, 26.2, 15.7, -52.8],
    'K': [-3.9, -0.0679, 19.1, -31.5, 1],
    'M': [1.9, 0.1899, 27.8, 1, -51.1],
    'F': [2.8, 0.26, 10.4, 20.7, -51.1],
    'P': [-1.6, -0.1665, -59.8, -47.8, 41.9],
    'S': [-0.8, 0.1933, -32.9, -6.2, 35.8],
    'T': [-0.7, 0.2572, -24.8, 28.5, -4.1],
    'W': [-0.9, 0.0331, 3, 21.5, -4.1],
    'Y': [-1.3, 0.0148, -31.5, 27, 13.1],
    'V': [4.2, -0.2469, -3, 49.5, -69.3]
}

def calc_proximity(pdb_id):
    '''Calculates residue proximities from PDB file.'''
    with tempfile.TemporaryFile() as pdb_file:
        opener = urllib.URLopener()
        opener.retrieve('http://www.rcsb.org/pdb/files/' + pdb_id + '.pdb',
                        pdb_file.name)
        parser = PDBParser()

        chains = [c for c in parser.get_structure(pdb_id,
                                                  pdb_file.name).get_chains()]

        coords = [residue.child_dict['CA'].get_coord()
                  for residue in chains[0] if 'CA' in residue.child_dict]

        return scipy.spatial.distance.cdist(coords, coords, 'euclidean')

def plot(values, plot_filename, plot_format, title, max_value=None):
    '''Plots 3d matrix values.'''
    fig = pylab.figure()
    sub_plot = fig.add_subplot(111)

    cmap = LinearSegmentedColormap.from_list(name='name', colors=['g', 'w'],
                                             N=10)

    cax = sub_plot.imshow(values, interpolation='nearest', cmap=cmap)
    cax.set_clim(0.0, max_value)
    sub_plot.set_title(title)

    # Add colorbar, make sure to specify tick locations to match desired tick
    # labels
    min_val = numpy.min(values)
    max_val = numpy.max(values)
    cbar = fig.colorbar(cax, ticks=[min_val, max_val])
    cbar.set_ticks([min_val, max_val])

    pylab.savefig(plot_filename, format=plot_format)

def export_html(values):
    print ','.join(['[' + ','.join([str(v) for v in value.tolist()]) + ']'
                    for value in values])

def main(argv):
    '''main method.'''
    pdb_id = argv[1]
    proximities = calc_proximity(pdb_id)

    # export_html(proximities)

    plot_format = 'png'
    plot_filename = pdb_id + '.' + plot_format
    plot(proximities, plot_filename, plot_format,
         pdb_id + ' proximity plot')

if __name__ == '__main__':
    main(sys.argv)
<commit_msg>Added input/output generator for deep learning.<commit_after>'''
synbiochem (c) University of Manchester 2015

synbiochem is licensed under the MIT License.

To view a copy of this license, visit <http://opensource.org/licenses/MIT/>.

@author:  neilswainston
'''
import itertools
from matplotlib.colors import LinearSegmentedColormap
import numpy
import pylab
import scipy.spatial
import sys
import tempfile
import urllib

from Bio.PDB.PDBParser import PDBParser

# KD Hydrophobicity, EIIP, Helix, Sheet, Turn
AMINO_ACID_PROPERTIES = {
    'A': [1.8, -0.0667, 32.9, -23.6, -41.6],
    'R': [-4.5, 0.2674, 0, -6.2, -5.1],
    'N': [-3.5, -0.2589, -24.8, -41.6, 44.5],
    'D': [-3.5, 0.4408, 5.8, -41.6, 37.8],
    'C': [2.5, 0.1933, -5.1, 6.8, 17.4],
    'Q': [-3.5, 0.1545, 11.3, 0, -2],
    'E': [-3.5, -0.2463, 36.5, -67.3, -30.1],
    'G': [-0.4, -0.2509, -46.2, -13.9, 44.5],
    'H': [-3.2, -0.1414, 11.3, -18.6, -5.1],
    'I': [4.5, -0.2794, -1, 45.1, -75.5],
    'L': [3.8, -0.2794, 26.2, 15.7, -52.8],
    'K': [-3.9, -0.0679, 19.1, -31.5, 1],
    'M': [1.9, 0.1899, 27.8, 1, -51.1],
    'F': [2.8, 0.26, 10.4, 20.7, -51.1],
    'P': [-1.6, -0.1665, -59.8, -47.8, 41.9],
    'S': [-0.8, 0.1933, -32.9, -6.2, 35.8],
    'T': [-0.7, 0.2572, -24.8, 28.5, -4.1],
    'W': [-0.9, 0.0331, 3, 21.5, -4.1],
    'Y': [-1.3, 0.0148, -31.5, 27, 13.1],
    'V': [4.2, -0.2469, -3, 49.5, -69.3]
}

def calc_proximity(pdb_id):
    '''Calculates residue proximities from PDB file.'''
    with tempfile.TemporaryFile() as pdb_file:
        opener = urllib.URLopener()
        opener.retrieve('http://www.rcsb.org/pdb/files/' + pdb_id + '.pdb',
                        pdb_file.name)
        parser = PDBParser()

        chains = [c for c in parser.get_structure(pdb_id,
                                                  pdb_file.name).get_chains()]

        coords = [residue.child_dict['CA'].get_coord()
                  for residue in chains[0] if 'CA' in residue.child_dict]

        return scipy.spatial.distance.cdist(coords, coords, 'euclidean')

def plot(values, plot_filename, plot_format, title, max_value=None):
    '''Plots 3d matrix values.'''
    fig = pylab.figure()
    sub_plot = fig.add_subplot(111)

    cmap = LinearSegmentedColormap.from_list(name='name', colors=['g', 'w'],
                                             N=10)

    cax = sub_plot.imshow(values, interpolation='nearest', cmap=cmap)
    cax.set_clim(0.0, max_value)
    sub_plot.set_title(title)

    # Add colorbar, make sure to specify tick locations to match desired tick
    # labels
    min_val = numpy.min(values)
    max_val = numpy.max(values)
    cbar = fig.colorbar(cax, ticks=[min_val, max_val])
    cbar.set_ticks([min_val, max_val])

    pylab.savefig(plot_filename, format=plot_format)

def export_html(values):
    print ','.join(['[' + ','.join([str(v) for v in value.tolist()]) + ']'
                    for value in values])

def main(argv):
    '''main method.'''
    pdb_id = argv[1]
    proximities = calc_proximity(pdb_id)

    input = [AMINO_ACID_PROPERTIES[amino_acid] for amino_acid in amino_acids]
    output = itertools.chain.from_iterable(proximities)

    # export_html(proximities)

    plot_format = 'png'
    plot_filename = pdb_id + '.' + plot_format
    plot(proximities, plot_filename, plot_format,
         pdb_id + ' proximity plot')

if __name__ == '__main__':
    main(sys.argv)
<|endoftext|>


<commit_before>import os
import shutil
import stat
import time

import shlex
from subprocess import Popen, check_output, PIPE, CalledProcessError
from threading import Timer

from .models import Site
from ..users.models import User, Group
from ...settings import LE_WEBROOT

from django.utils.crypto import get_random_string
from django.template.loader import render_to_string

from raven.contrib.django.raven_compat.models import client

def create_site_users(site):
    try:
        user = User.objects.get(username="site_{}".format(site.name))
    except User.DoesNotExist:
        user = User.objects.create(id=get_next_id(), service=True, username="site_{}".format(site.name), email="site_{}@tjhsst.edu".format(site.name))
    try:
        group = Group.objects.get(name="site_{}".format(site.name))
    except Group.DoesNotExist:
        group = Group.objects.create(id=user.id, service=True, name="site_{}".format(site.name))
        group.users.add(user)
        group.save()
    site.user = user
    site.group = group
    site.save()
    return (user, group)

def add_access_token(func):
    """Decorator to add an access token if it is not set.

    Used to ensure that the user can authenticate with the Node.js server for cases like the web terminal.
    """
    def func_wrapper(request, **kwargs):
        if not request.user.access_token:
            request.user.access_token = get_random_string(24)
            request.user.save()
        return func(request, **kwargs)
    return func_wrapper

def get_next_id():
    if User.objects.filter(service=True).count() == 0:
        return 10000
    return User.objects.filter(service=True).order_by('-id')[0].id + 1

def make_site_dirs(site):
    for i in ["", "public", "private"]:
        path = os.path.join(site.path, i)
        if not os.path.exists(path):
            os.makedirs(path)
        os.chown(path, site.user.id, site.group.id)
        os.chmod(path, stat.S_IRUSR | stat.S_IWUSR | stat.S_IXUSR | stat.S_IRGRP | stat.S_IWGRP | stat.S_IXGRP | stat.S_ISGID)
    Popen(["/usr/bin/setfacl", "-m", "u:www-data:rx", site.path])
    Popen(["/usr/bin/setfacl", "-m", "u:www-data:rx", os.path.join(site.path, "public")])

def create_config_files(site):
    if not site.custom_nginx:
        with open("/etc/nginx/director.d/{}.conf".format(site.name), "w+") as f:
            f.write(render_to_string("config/nginx.conf", {"site": site}))
    if site.category == "php":
        with open("/etc/php/7.0/fpm/pool.d/{}.conf".format(site.name), "w+") as f:
            f.write(render_to_string("config/phpfpm.conf", {"site": site}))
    elif site.category == "dynamic" and hasattr(site, "process"):
        create_process_config(site.process)

def delete_php_config(site):
    filename = "/etc/php/7.0/fpm/pool.d/{}.conf".format(site.name)
    if os.path.exists(filename):
        os.remove(filename)

def write_new_index_file(site):
    """Creates a default index file for new sites."""
    with open(os.path.join(site.path, "public", "index.html"), "w+") as f:
        f.write(render_to_string("config/index.html", {"site": site}))

def delete_site_files(site):
    """Deletes all site content and configuration files."""
    files = ["/etc/nginx/director.d/{}.conf", "/etc/php/7.0/fpm/pool.d/{}.conf", "/etc/supervisor/director.d/{}.conf"]
    files = [x.format(site.name) for x in files]
    for f in files:
        if os.path.isfile(f):
            try:
                os.remove(f)
            except OSError:
                client.captureException()
    try:
        shutil.rmtree(site.path)
    except:
        client.captureException()

def create_process_config(process):
    with open("/etc/supervisor/director.d/{}.conf".format(process.site.name), "w+") as f:
        f.write(render_to_string("config/supervisor.conf", {"process": process}))

def delete_process_config(process):
    filename = "/etc/supervisor/director.d/{}.conf".format(process.site.name)
    if os.path.isfile(filename):
        os.remove(filename)

def restart_supervisor(site):
    try:
        site.process
        Popen(["supervisorctl", "restart", site.name])
    except Site.process.RelatedObjectDoesNotExist:
        pass

def get_supervisor_status(site):
    if not hasattr(site, "process"):
        return "No Process"

    try:
        return check_output(["supervisorctl", "status", site.name]).decode()
    except CalledProcessError:
        client.captureException()
        return "Status Retrieval Failure"

def get_supervisor_statuses(sites):
    try:
        out = check_output(["supervisorctl", "status"] + sites).decode()
    except CalledProcessError:
        client.captureException()
        return {}
    statuses = {}
    for line in out.split("\n"):
        line = line.strip()
        if not line:
            continue
        key, value = line.split(" ", 1)
        statuses[key] = True if "RUNNING" in value else False
    return statuses

def reload_services():
    reload_nginx_config()
    reload_php_fpm()
    update_supervisor()

def update_supervisor():
    if Popen(["supervisorctl", "update"]).wait() != 0:
        client.captureMessage("Failed to update supervisor.")

def reload_php_fpm():
    if Popen(["systemctl", "reload", "php7.0-fpm"]).wait() != 0:
        if Popen(["systemctl", "restart", "php7.0-fpm"]).wait() != 0:
            client.captureMessage("Failed to reload the PHP FPM service.")

def reload_nginx_config():
    if Popen(["/usr/sbin/nginx", "-s", "reload"]).wait() != 0:
        client.captureMessage("Failed to reload the nginx service.")

def check_nginx_config():
    return Popen(["/usr/sbin/nginx", "-t"]).wait() == 0

def flush_permissions():
    """Resets any cached users or groups."""
    if os.path.isfile("/proc/net/rpc/auth.unix.gid/flush"):
        with open("/proc/net/rpc/auth.unix.gid/flush", "w") as f:
            f.write(str(int(time.time())))
    Popen(["/usr/sbin/nscd", "-i", "group"])
    Popen(["/usr/sbin/nscd", "-i", "passwd"])

def run_as_site(site, cmd, cwd=None, env=None, timeout=15):
    proc = Popen(shlex.split(cmd) if isinstance(cmd, str) else cmd, preexec_fn=demote(
        site.user.id, site.group.id), cwd=cwd or site.path, env=env, stdout=PIPE, stderr=PIPE)
    timer = Timer(timeout, lambda p: p.terminate(), [proc])
    try:
        timer.start()
        out, err = proc.communicate()
    finally:
        timer.cancel()
    return (proc.returncode, out.decode("utf-8", "replace"), err.decode("utf-8", "replace"))

def demote(uid, gid):
    def result():
        os.setgroups([gid])
        os.setgid(gid)
        os.setuid(uid)
    return result

def generate_ssh_key(site, overwrite=True):
    sshpath = os.path.join(site.private_path, ".ssh")
    keypath = os.path.join(sshpath, "id_rsa")
    if not os.path.exists(sshpath):
        os.makedirs(sshpath)
    os.chown(sshpath, site.user.id, site.group.id)
    os.chmod(sshpath, stat.S_IRUSR | stat.S_IWUSR | stat.S_IXUSR)
    if os.path.isfile(keypath):
        if not overwrite:
            return False
        os.remove(keypath)
    if os.path.isfile(keypath + ".pub"):
        os.remove(keypath + ".pub")

    output = run_as_site(site, ["/usr/bin/ssh-keygen", "-t", "rsa", "-b", "4096", "-N", "", "-f", keypath])

    if not output[0] == 0:
        raise IOError("Could not generate RSA keys ({}) - {} - {}".format(output[0], output[1], output[2]))

    os.chown(keypath, site.user.id, site.group.id)
    os.chown(keypath + ".pub", site.user.id, site.group.id)
    os.chmod(keypath, stat.S_IRUSR | stat.S_IWUSR)
    os.chmod(keypath + ".pub", stat.S_IRUSR | stat.S_IWUSR)
    return True

def do_git_pull(site):
    fix_permissions(site)
    output = run_as_site(site, "git pull", cwd=site.git_path, env={
        "GIT_SSH_COMMAND": "ssh -o LogLevel=ERROR -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -i {}".format(os.path.join(site.private_path, ".ssh/id_rsa")),
        "HOME": site.private_path
    })
    if site.category == "dynamic":
        restart_supervisor(site)
    return output

def get_latest_commit(site):
    try:
        output = run_as_site(site, ["git", "log", "-n", "1"], cwd=site.git_path)
    except:
        client.captureException()
        return "Error"
    if not output[0] == 0:
        return "Error - {}".format(output[2].replace("\n", " ").replace("\r", ""))
    return output[1]

def fix_permissions(site):
    """Makes sure that all files are owned by the site user and the site group has access."""
    for root, dirs, files in os.walk(site.path):
        dirs[:] = [d for d in dirs if not d == ".ssh"]
        for f in files + dirs:
            path = os.path.join(root, f)
            try:
                st = os.lstat(path)
            except:
                client.captureException()
                continue
            os.lchown(path, site.user.id, site.group.id)
            if stat.S_ISLNK(st.st_mode):
                pass
            elif stat.S_ISDIR(st.st_mode):
                os.chmod(path, st.st_mode | stat.S_IRGRP | stat.S_IWGRP | stat.S_IXGRP)
            else:
                os.chmod(path, st.st_mode | stat.S_IRGRP | stat.S_IWGRP)

def list_executable_files(path, level):
    path = path.rstrip(os.path.sep)
    num_sep = path.count(os.path.sep)
    out = []

    for root, dirs, files in os.walk(path):
        # ignore hidden files and folders, ignore common npm install path
        files = [f for f in files if not f[0] == "."]
        dirs[:] = [d for d in dirs if not d[0] == "." and not d == "node_modules"]

        # ignore files in virtual environments
        if "pip-selfcheck.json" in files:
            del dirs[:]
            continue

        for f in files:
            p = os.path.abspath(os.path.join(root, f))
            if os.access(p, os.X_OK):
                out.append(p)

        # limit folder recursion
        cur_num_sep = root.count(os.path.sep)
        if num_sep + level <= cur_num_sep:
            del dirs[:]

    return out

def clean_site_type(instance):
    """Destroy any old site configuration caused by changing site types."""
    if instance.category != "php":
        delete_php_config(instance)

    if instance.category != "dynamic" and hasattr(instance, "process"):
        delete_process_config(instance.process)
        update_supervisor()
        instance.process.delete()

    if instance.category != "vm" and hasattr(instance, "virtual_machine"):
        vm = instance.virtual_machine
        vm.site = None
        vm.save()

def generate_ssl_certificate(domain):
    """Generate SSL certs for a domain and update the nginx config"""
    success = Popen([
        'sudo',
        '/usr/bin/certbot',
        'certonly',
        '--webroot',
        '-w',
        LE_WEBROOT,
        '-d',
        domain.domain,
        '-n']).wait() == 0
    if success:
        create_config_files(domain.site)
        reload_services(domain.site)
<commit_msg>settings lookup fix<commit_after>import os
import shutil
import stat
import time

import shlex
from subprocess import Popen, check_output, PIPE, CalledProcessError
from threading import Timer

from .models import Site
from ..users.models import User, Group

from django.conf import settings
from django.utils.crypto import get_random_string
from django.template.loader import render_to_string

from raven.contrib.django.raven_compat.models import client

def create_site_users(site):
    try:
        user = User.objects.get(username="site_{}".format(site.name))
    except User.DoesNotExist:
        user = User.objects.create(id=get_next_id(), service=True, username="site_{}".format(site.name), email="site_{}@tjhsst.edu".format(site.name))
    try:
        group = Group.objects.get(name="site_{}".format(site.name))
    except Group.DoesNotExist:
        group = Group.objects.create(id=user.id, service=True, name="site_{}".format(site.name))
        group.users.add(user)
        group.save()
    site.user = user
    site.group = group
    site.save()
    return (user, group)

def add_access_token(func):
    """Decorator to add an access token if it is not set.

    Used to ensure that the user can authenticate with the Node.js server for cases like the web terminal.
    """
    def func_wrapper(request, **kwargs):
        if not request.user.access_token:
            request.user.access_token = get_random_string(24)
            request.user.save()
        return func(request, **kwargs)
    return func_wrapper

def get_next_id():
    if User.objects.filter(service=True).count() == 0:
        return 10000
    return User.objects.filter(service=True).order_by('-id')[0].id + 1

def make_site_dirs(site):
    for i in ["", "public", "private"]:
        path = os.path.join(site.path, i)
        if not os.path.exists(path):
            os.makedirs(path)
        os.chown(path, site.user.id, site.group.id)
        os.chmod(path, stat.S_IRUSR | stat.S_IWUSR | stat.S_IXUSR | stat.S_IRGRP | stat.S_IWGRP | stat.S_IXGRP | stat.S_ISGID)
    Popen(["/usr/bin/setfacl", "-m", "u:www-data:rx", site.path])
    Popen(["/usr/bin/setfacl", "-m", "u:www-data:rx", os.path.join(site.path, "public")])

def create_config_files(site):
    if not site.custom_nginx:
        with open("/etc/nginx/director.d/{}.conf".format(site.name), "w+") as f:
            f.write(render_to_string("config/nginx.conf", {"site": site}))
    if site.category == "php":
        with open("/etc/php/7.0/fpm/pool.d/{}.conf".format(site.name), "w+") as f:
            f.write(render_to_string("config/phpfpm.conf", {"site": site}))
    elif site.category == "dynamic" and hasattr(site, "process"):
        create_process_config(site.process)

def delete_php_config(site):
    filename = "/etc/php/7.0/fpm/pool.d/{}.conf".format(site.name)
    if os.path.exists(filename):
        os.remove(filename)

def write_new_index_file(site):
    """Creates a default index file for new sites."""
    with open(os.path.join(site.path, "public", "index.html"), "w+") as f:
        f.write(render_to_string("config/index.html", {"site": site}))

def delete_site_files(site):
    """Deletes all site content and configuration files."""
    files = ["/etc/nginx/director.d/{}.conf", "/etc/php/7.0/fpm/pool.d/{}.conf", "/etc/supervisor/director.d/{}.conf"]
    files = [x.format(site.name) for x in files]
    for f in files:
        if os.path.isfile(f):
            try:
                os.remove(f)
            except OSError:
                client.captureException()
    try:
        shutil.rmtree(site.path)
    except:
        client.captureException()

def create_process_config(process):
    with open("/etc/supervisor/director.d/{}.conf".format(process.site.name), "w+") as f:
        f.write(render_to_string("config/supervisor.conf", {"process": process}))

def delete_process_config(process):
    filename = "/etc/supervisor/director.d/{}.conf".format(process.site.name)
    if os.path.isfile(filename):
        os.remove(filename)

def restart_supervisor(site):
    try:
        site.process
        Popen(["supervisorctl", "restart", site.name])
    except Site.process.RelatedObjectDoesNotExist:
        pass

def get_supervisor_status(site):
    if not hasattr(site, "process"):
        return "No Process"

    try:
        return check_output(["supervisorctl", "status", site.name]).decode()
    except CalledProcessError:
        client.captureException()
        return "Status Retrieval Failure"

def get_supervisor_statuses(sites):
    try:
        out = check_output(["supervisorctl", "status"] + sites).decode()
    except CalledProcessError:
        client.captureException()
        return {}
    statuses = {}
    for line in out.split("\n"):
        line = line.strip()
        if not line:
            continue
        key, value = line.split(" ", 1)
        statuses[key] = True if "RUNNING" in value else False
    return statuses

def reload_services():
    reload_nginx_config()
    reload_php_fpm()
    update_supervisor()

def update_supervisor():
    if Popen(["supervisorctl", "update"]).wait() != 0:
        client.captureMessage("Failed to update supervisor.")

def reload_php_fpm():
    if Popen(["systemctl", "reload", "php7.0-fpm"]).wait() != 0:
        if Popen(["systemctl", "restart", "php7.0-fpm"]).wait() != 0:
            client.captureMessage("Failed to reload the PHP FPM service.")

def reload_nginx_config():
    if Popen(["/usr/sbin/nginx", "-s", "reload"]).wait() != 0:
        client.captureMessage("Failed to reload the nginx service.")

def check_nginx_config():
    return Popen(["/usr/sbin/nginx", "-t"]).wait() == 0

def flush_permissions():
    """Resets any cached users or groups."""
    if os.path.isfile("/proc/net/rpc/auth.unix.gid/flush"):
        with open("/proc/net/rpc/auth.unix.gid/flush", "w") as f:
            f.write(str(int(time.time())))
    Popen(["/usr/sbin/nscd", "-i", "group"])
    Popen(["/usr/sbin/nscd", "-i", "passwd"])

def run_as_site(site, cmd, cwd=None, env=None, timeout=15):
    proc = Popen(shlex.split(cmd) if isinstance(cmd, str) else cmd, preexec_fn=demote(
        site.user.id, site.group.id), cwd=cwd or site.path, env=env, stdout=PIPE, stderr=PIPE)
    timer = Timer(timeout, lambda p: p.terminate(), [proc])
    try:
        timer.start()
        out, err = proc.communicate()
    finally:
        timer.cancel()
    return (proc.returncode, out.decode("utf-8", "replace"), err.decode("utf-8", "replace"))

def demote(uid, gid):
    def result():
        os.setgroups([gid])
        os.setgid(gid)
        os.setuid(uid)
    return result

def generate_ssh_key(site, overwrite=True):
    sshpath = os.path.join(site.private_path, ".ssh")
    keypath = os.path.join(sshpath, "id_rsa")
    if not os.path.exists(sshpath):
        os.makedirs(sshpath)
    os.chown(sshpath, site.user.id, site.group.id)
    os.chmod(sshpath, stat.S_IRUSR | stat.S_IWUSR | stat.S_IXUSR)
    if os.path.isfile(keypath):
        if not overwrite:
            return False
        os.remove(keypath)
    if os.path.isfile(keypath + ".pub"):
        os.remove(keypath + ".pub")

    output = run_as_site(site, ["/usr/bin/ssh-keygen", "-t", "rsa", "-b", "4096", "-N", "", "-f", keypath])

    if not output[0] == 0:
        raise IOError("Could not generate RSA keys ({}) - {} - {}".format(output[0], output[1], output[2]))

    os.chown(keypath, site.user.id, site.group.id)
    os.chown(keypath + ".pub", site.user.id, site.group.id)
    os.chmod(keypath, stat.S_IRUSR | stat.S_IWUSR)
    os.chmod(keypath + ".pub", stat.S_IRUSR | stat.S_IWUSR)
    return True

def do_git_pull(site):
    fix_permissions(site)
    output = run_as_site(site, "git pull", cwd=site.git_path, env={
        "GIT_SSH_COMMAND": "ssh -o LogLevel=ERROR -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -i {}".format(os.path.join(site.private_path, ".ssh/id_rsa")),
        "HOME": site.private_path
    })
    if site.category == "dynamic":
        restart_supervisor(site)
    return output

def get_latest_commit(site):
    try:
        output = run_as_site(site, ["git", "log", "-n", "1"], cwd=site.git_path)
    except:
        client.captureException()
        return "Error"
    if not output[0] == 0:
        return "Error - {}".format(output[2].replace("\n", " ").replace("\r", ""))
    return output[1]

def fix_permissions(site):
    """Makes sure that all files are owned by the site user and the site group has access."""
    for root, dirs, files in os.walk(site.path):
        dirs[:] = [d for d in dirs if not d == ".ssh"]
        for f in files + dirs:
            path = os.path.join(root, f)
            try:
                st = os.lstat(path)
            except:
                client.captureException()
                continue
            os.lchown(path, site.user.id, site.group.id)
            if stat.S_ISLNK(st.st_mode):
                pass
            elif stat.S_ISDIR(st.st_mode):
                os.chmod(path, st.st_mode | stat.S_IRGRP | stat.S_IWGRP | stat.S_IXGRP)
            else:
                os.chmod(path, st.st_mode | stat.S_IRGRP | stat.S_IWGRP)

def list_executable_files(path, level):
    path = path.rstrip(os.path.sep)
    num_sep = path.count(os.path.sep)
    out = []

    for root, dirs, files in os.walk(path):
        # ignore hidden files and folders, ignore common npm install path
        files = [f for f in files if not f[0] == "."]
        dirs[:] = [d for d in dirs if not d[0] == "." and not d == "node_modules"]

        # ignore files in virtual environments
        if "pip-selfcheck.json" in files:
            del dirs[:]
            continue

        for f in files:
            p = os.path.abspath(os.path.join(root, f))
            if os.access(p, os.X_OK):
                out.append(p)

        # limit folder recursion
        cur_num_sep = root.count(os.path.sep)
        if num_sep + level <= cur_num_sep:
            del dirs[:]

    return out

def clean_site_type(instance):
    """Destroy any old site configuration caused by changing site types."""
    if instance.category != "php":
        delete_php_config(instance)

    if instance.category != "dynamic" and hasattr(instance, "process"):
        delete_process_config(instance.process)
        update_supervisor()
        instance.process.delete()

    if instance.category != "vm" and hasattr(instance, "virtual_machine"):
        vm = instance.virtual_machine
        vm.site = None
        vm.save()

def generate_ssl_certificate(domain):
    """Generate SSL certs for a domain and update the nginx config."""
    success = Popen([
        "/usr/bin/certbot",
        "certonly",
        "--webroot",
        "-w",
        settings.LE_WEBROOT,
        "-d",
        domain.domain,
        "-n"]).wait() == 0

    if success:
        create_config_files(domain.site)
        reload_services(domain.site)
<|endoftext|>


<commit_before>from __future__ import absolute_import, division, print_function

import logging

from dynd import nd
from pandas import DataFrame
from .expr.table import *
from .api import *
from .data.csv import *
from .data.json import *
from .data.hdf5 import *
from .compute.python import *
from .data.pandas import *
from .data.meta import *
from .compute.pandas import *
from .compute.core import *
from .compute.core import compute

logging.basicConfig()
logger = logging.getLogger(__name__)
logger.setLevel(logging.WARNING)

inf = float('inf')
nan = float('nan')

__version__ = '0.6.0-dev'

# If IPython is already loaded, register the Blaze catalog magic
# from . import catalog
# import sys
# if 'IPython' in sys.modules:
#     catalog.register_ipy_magic()
# del sys

def print_versions():
    """Print all the versions of software that Blaze relies on."""
    import sys, platform
    import numpy as np
    import dynd
    import datashape
    import blz
    print("-=" * 38)
    print("Blaze version: %s" % __version__)
    print("Datashape version: %s" % datashape.__version__)
    print("NumPy version: %s" % np.__version__)
    print("DyND version: %s / LibDyND %s" %
                    (dynd.__version__, dynd.__libdynd_version__))
    print("Python version: %s" % sys.version)
    (sysname, nodename, release, version, machine, processor) = \
        platform.uname()
    print("Platform: %s-%s-%s (%s)" % (sysname, release, machine, version))
    if sysname == "Linux":
        print("Linux dist: %s" % " ".join(platform.linux_distribution()[:-1]))
    if not processor:
        processor = "not recognized"
    print("Processor: %s" % processor)
    print("Byte-ordering: %s" % sys.byteorder)
    print("Detected cores: %s" % blz.detect_number_of_cores())
    print("-=" * 38)

def test(verbosity=1, xunitfile=None, exit=False):
    """
    Runs the full Blaze test suite, outputting
    the results of the tests to sys.stdout.

    This uses nose tests to discover which tests to
    run, and runs tests in any 'tests' subdirectory
    within the Blaze module.

    Parameters
    ----------
    verbosity : int, optional
        Value 0 prints very little, 1 prints a little bit,
        and 2 prints the test names while testing.
    xunitfile : string, optional
        If provided, writes the test results to an xunit
        style xml file. This is useful for running the tests
        in a CI server such as Jenkins.
    exit : bool, optional
        If True, the function will call sys.exit with an
        error code after the tests are finished.
    """
    import nose
    import os
    import sys
    argv = ['nosetests', '--verbosity=%d' % verbosity]
    # Output an xunit file if requested
    if xunitfile:
        argv.extend(['--with-xunit', '--xunit-file=%s' % xunitfile])
    # Set the logging level to warn
    argv.extend(['--logging-level=WARN'])
    # Add all 'tests' subdirectories to the options
    rootdir = os.path.dirname(__file__)
    for root, dirs, files in os.walk(rootdir):
        if 'tests' in dirs:
            testsdir = os.path.join(root, 'tests')
            argv.append(testsdir)
            print('Test dir: %s' % testsdir[len(rootdir)+1:])
    # print versions (handy when reporting problems)
    print_versions()
    sys.stdout.flush()
    # Ask nose to do its thing
    return nose.main(argv=argv, exit=exit)
<commit_msg>Rework toplevel test function to use pytest<commit_after>from __future__ import absolute_import, division, print_function

import logging

from dynd import nd
from pandas import DataFrame
from .expr.table import *
from .api import *
from .data.csv import *
from .data.json import *
from .data.hdf5 import *
from .compute.python import *
from .data.pandas import *
from .data.meta import *
from .compute.pandas import *
from .compute.core import *
from .compute.core import compute

logging.basicConfig()
logger = logging.getLogger(__name__)
logger.setLevel(logging.WARNING)

inf = float('inf')
nan = float('nan')

__version__ = '0.6.0-dev'

# If IPython is already loaded, register the Blaze catalog magic
# from . import catalog
# import sys
# if 'IPython' in sys.modules:
#     catalog.register_ipy_magic()
# del sys

def print_versions():
    """Print all the versions of software that Blaze relies on."""
    import sys, platform
    import numpy as np
    import dynd
    import datashape
    import blz
    print("-=" * 38)
    print("Blaze version: %s" % __version__)
    print("Datashape version: %s" % datashape.__version__)
    print("NumPy version: %s" % np.__version__)
    print("DyND version: %s / LibDyND %s" %
                    (dynd.__version__, dynd.__libdynd_version__))
    print("Python version: %s" % sys.version)
    (sysname, nodename, release, version, machine, processor) = \
        platform.uname()
    print("Platform: %s-%s-%s (%s)" % (sysname, release, machine, version))
    if sysname == "Linux":
        print("Linux dist: %s" % " ".join(platform.linux_distribution()[:-1]))
    if not processor:
        processor = "not recognized"
    print("Processor: %s" % processor)
    print("Byte-ordering: %s" % sys.byteorder)
    print("Detected cores: %s" % blz.detect_number_of_cores())
    print("-=" * 38)

def test(verbose=False, junitfile=None, exit=False):
    """
    Runs the full Blaze test suite, outputting
    the results of the tests to sys.stdout.

    This uses py.test to discover which tests to
    run, and runs tests in any 'tests' subdirectory
    within the Blaze module.

    Parameters
    ----------
    verbose : int, optional
        Value 0 prints very little, 1 prints a little bit,
        and 2 prints the test names while testing.
    junitfile : string, optional
        If provided, writes the test results to an junit xml
        style xml file. This is useful for running the tests
        in a CI server such as Jenkins.
    exit : bool, optional
        If True, the function will call sys.exit with an
        error code after the tests are finished.
    """
    import os
    import sys
    import pytest

    args = []

    if verbose:
        args.append('--verbose')

    # Output an xunit file if requested
    if junitfile is not None:
        args.append('--junit-xml=%s' % junitfile)

    # Add all 'tests' subdirectories to the options
    rootdir = os.path.dirname(__file__)
    for root, dirs, files in os.walk(rootdir):
        if 'tests' in dirs:
            testsdir = os.path.join(root, 'tests')
            args.append(testsdir)
            print('Test dir: %s' % testsdir[len(rootdir) + 1:])
    # print versions (handy when reporting problems)
    print_versions()
    sys.stdout.flush()

    # Ask pytest to do its thing
    error_code = pytest.main(args=args)
    if exit:
        return sys.exit(error_code)
    return error_code
<|endoftext|>