<commit_before>import os
from datetime import datetime
from collections import OrderedDict
import matplotlib.pyplot as plt
import numpy as np

#-------------------------------------------------------------------------------

class SpecfitParser:
    """Parse and modify parameter files for the SpecFit task"""

    def __init__(self, filename):
        self.filename = filename
        self.n_components = 0
        self.components = OrderedDict()

        self.loadfile()

    def __str__(self):
        message = "Specfit database file {}\n".format(self.filename)
        message += '\n'.join(['-' + str(item) for item in self.components])
        return message

    def __iter__(self):
        for item in self.components.itervalues():
            yield item

    def loadfile(self):
        data = open(self.filename, 'r').readlines()

        if not len(data):
            raise ValueError('No lines in input datafile')

        for i, line in enumerate(data):
            line = line.strip()
            if line.startswith('components'):
                self.n_components = int(line.split()[1])
                self.read_components(data, i + self.n_components + 1)
                return

        raise ValueError('No components found')

    def plot(self, wlim, ax=None):

        if not ax:
            fig = plt.figure()
            ax = fig.add_subplot(1, 1, 1)
        else:
            fig = plt.gcf()

        xdata = np.arange(wlim[0], wlim[1], .1)
        ydata = np.zeros(xdata.shape)

        for comp in self.components.itervalues():
            print comp.name
            if 'powerlaw' in comp.name:
                index = comp.parameters['index'].value
                flux = comp.parameters['flux'].value
                ydata += flux * (xdata/1000.0)**(-1 * index)
            if 'gaussian' in comp.name:
                flux = comp.parameters['flux'].value
                centroid = comp.parameters['centroid'].value
                fwhm = comp.parameters['fwhm'].value
                fwhm = velocity_to_wavelength(fwhm, centroid)
                #skew = comp.parameters['skew'].value

                ydata += gaussian(xdata, flux, fwhm, centroid)

        ax.plot(xdata, ydata)

        ax.set_ylabel('Flux')
        ax.set_xlabel('Wavelength')

        fig.savefig('plot.pdf')

    def read_components(self, data, start):
        components_read = 0

        while components_read < self.n_components:
            comp_name, n_pars = data[start].strip().split()[0:2]
            n_pars = int(n_pars)

            if comp_name in self.components:
                raise KeyError('Component defined twice!')

            self.components[comp_name] = SpecfitComponent(data[start:start + n_pars + 1])

            components_read += 1
            start += n_pars + 1

        if components_read < self.n_components:
            raise ValueError("Not enough parameters supplied")

    def write(self, outname, clobber=False, mode='w'):
        if os.path.exists(outname) and not clobber:
            raise IOError("Will not overrite {} while clobber={}".format(outname,
                                                                         clobber))

        out_path, out_filename = os.path.split(outname)
        if not out_filename.startswith('sf'):
            print "adding 'sf' to db name"
            outname = os.path.join(out_path, 'sf'+out_filename)

        now = datetime.now()

        with open(outname, mode) as out_file:
            #-- Not sure if these are needed, keeping for now
            out_file.write('# {}\n'.format(now.ctime()))
            out_file.write('begin {}\n'.format(out_filename.lstrip('sf')))
            out_file.write('      task    specfit\n')

            #-- These are needed
            out_file.write('components    {}\n'.format(self.n_components))

            for item in self.components.itervalues():
                out_file.write('         {}\n'.format(item.type))

            for item in self.components.itervalues():
                out_file.write('{:>19}    {} # {}\n'.format(item.name,
                                                           item.n_pars,
                                                           item.label))
                for par in item.parameters.itervalues():
                    out_file.write('{:>33}{:>14}{:>14}{:>14}{:>10}{:>10}\n'.format(par.value,
                                                                              par.lower_lim,
                                                                              par.upper_lim,
                                                                              par.stepsize,
                                                                              par.tolerance,
                                                                              par.linkage))

#-------------------------------------------------------------------------------

class SpecfitComponent:
    """Parse and modify an individual compenent of a SpecFit paramter file"""

    _par_names = {'linear': ['flux', 'slope'],
                  'powerlaw': ['flux', 'index'],
                  'bpl': ['flux', 'break', 'index_above', 'index_below'],
                  'blackbody': ['flux', 'temperature'],
                  'gaussian': ['flux', 'centroid', 'fwhm', 'skew'],
                  'logarith': ['flux', 'centroid', 'fwhm', 'slew'],
                  'labsorp': ['eq_width', 'centroid', 'fwhm'],
                  'tauabs': ['depth', 'centroid', 'fwhm'],
                  'eabsorp': ['depth', 'wavelength'],
                  'recomb': ['flux', 'wavelength', 'temperature', 'fwhm'],
                  'extcor': ['e(v-b)'],
                  'usercont': ['norm', 'shift', 'redshift', 'key'],
                  'userline': ['norm', 'shift', 'redshift', 'key'],
                  'userabs': ['norm', 'shift', 'redshift', 'key'],
                  'lorentz': ['flux', 'centroid', 'fwhm', 'alpha'],
                  'dampabs': ['density', 'centroid', 'lifetime'],
                  'logabs': ['depth', 'centroid', 'fwhm'],
                  'ffree': ['norm', 'temperature'],
                  'extdrude': ['e(b-v)', 'w0', 'gamma', 'c1', 'c2', 'c3', 'c4'],
                  'disk': ['flux', 'beta', 'temperature'],
                  'ccmext': ['e(b-v)', 'rv']}

    def __init__(self, lines):
        self.name, self.n_pars = lines[0].strip().split()[0:2]
        self.n_pars = int(self.n_pars)
        self.label = ' '.join(lines[0].strip().split()[3:]) or ''

        #-- Strip out numbers to get type of model fit
        #-- ALL numbers will be stripped, but they should all be at the end
        self.type = ''.join([item for item in self.name if not item.isdigit()])
        self.number = int(''.join([item for item in self.name if item.isdigit()]))

        if self.n_pars != len(lines[1:]):
            message = "Incorrect number of parameters specified for"
            message += "{}, got {} instead of {}".format(self.type,
                                                         len(lines[1:]),
                                                         self.n_pars)
            raise ValueError(message)

        self.parameters = OrderedDict()
        for key, line in zip(self._par_names[self.type], lines[1:]):
            self.parameters[key] = SpecfitParameter(line)

    def __str__(self):
        output = "component: {} #{}\n".format(self.type, self.number)
        output += '\n'.join(['--' + str(item) for item in self.parameters])
        return output

    def __iter__(self):
        for item in self.parameters:
            yield item

#-------------------------------------------------------------------------------

class SpecfitParameter:
    """Parse and modify a parameter of a compenent"""

    def __init__(self, line):
        values = map(float, line.strip().split())

        if len(values) != 6:
            raise ValueError('Parameter can only have 6 values!')

        self.value = values[0]
        self.lower_lim = values[1]
        self.upper_lim = values[2]
        self.stepsize = values[3]
        self.tolerance = values[4]
        self.linkage = values[5]

    def __str__(self):
        return "parameter: {} {} {} {} {} {}".format(self.value,
                                                     self.lower_lim,
                                                     self.upper_lim,
                                                     self.stepsize,
                                                     self.tolerance,
                                                     self.linkage)

    def free(self):
        self.linkage = 0

    def fix(self):
        self.linkage = -1

#-------------------------------------------------------------------------------

def gaussian(x, area, sigma, center):
    sigma = sigma / (2 * np.sqrt(2 * np.log(2)))

    print area, sigma, center

    y = (area / (sigma * np.sqrt(2 * np.pi))) * np.exp((-(x-center)**2) / (2 * sigma**2))
    return y

#-------------------------------------------------------------------------------

def velocity_to_wavelength(velocity, zeropoint):
    LIGHTSPEED = 2.9979E5
    return zeropoint * velocity/LIGHTSPEED

#-------------------------------------------------------------------------------
<commit_msg>hopefully some error catches<commit_after>import os
from datetime import datetime
from collections import OrderedDict
import matplotlib.pyplot as plt
import numpy as np

#-------------------------------------------------------------------------------

class SpecfitParser:
    """Parse and modify parameter files for the SpecFit task"""

    def __init__(self, filename):
        self.filename = filename
        self.n_components = 0
        self.components = OrderedDict()

        self.loadfile()

    def __str__(self):
        message = "Specfit database file {}\n".format(self.filename)
        message += '\n'.join(['-' + str(item) for item in self.components])
        return message

    def __iter__(self):
        for item in self.components.itervalues():
            yield item

    def loadfile(self):
        data = open(self.filename, 'r').readlines()

        if not len(data):
            raise ValueError('No lines in input datafile')

        for i, line in enumerate(data):
            line = line.strip()
            if line.startswith('components'):
                self.n_components = int(line.split()[1])
                self.read_components(data, i + self.n_components + 1)
                return

        raise ValueError('No components found')

    def plot(self, wlim, ax=None):

        if not ax:
            fig = plt.figure()
            ax = fig.add_subplot(1, 1, 1)
        else:
            fig = plt.gcf()

        xdata = np.arange(wlim[0], wlim[1], .1)
        ydata = np.zeros(xdata.shape)

        for comp in self.components.itervalues():
            print comp.name
            if 'powerlaw' in comp.name:
                index = comp.parameters['index'].value
                flux = comp.parameters['flux'].value
                ydata += flux * (xdata/1000.0)**(-1 * index)
            if 'gaussian' in comp.name:
                flux = comp.parameters['flux'].value
                centroid = comp.parameters['centroid'].value
                fwhm = comp.parameters['fwhm'].value
                fwhm = velocity_to_wavelength(fwhm, centroid)
                #skew = comp.parameters['skew'].value

                ydata += gaussian(xdata, flux, fwhm, centroid)

        ax.plot(xdata, ydata)

        ax.set_ylabel('Flux')
        ax.set_xlabel('Wavelength')

        fig.savefig('plot.pdf')

    def read_components(self, data, start):
        components_read = 0

        while components_read < self.n_components:
            comp_name, n_pars = data[start].strip().split()[0:2]
            n_pars = int(n_pars)

            if comp_name in self.components:
                raise KeyError('Component defined twice!')

            self.components[comp_name] = SpecfitComponent(data[start:start + n_pars + 1])

            components_read += 1
            start += n_pars + 1

        if components_read < self.n_components:
            raise ValueError("Not enough parameters supplied")

    def write(self, outname, clobber=True, mode='w'):
        if os.path.exists(outname) and not clobber:
            raise IOError("Will not overrite {} while clobber={}".format(outname,
                                                                         clobber))

        out_path, out_filename = os.path.split(outname)
        if not out_filename.startswith('sf'):
            print "adding 'sf' to db name"
            outname = os.path.join(out_path, 'sf'+out_filename)

        now = datetime.now()

        with open(outname, mode) as out_file:
            #-- Not sure if these are needed, keeping for now
            out_file.write('# {}\n'.format(now.ctime()))
            out_file.write('begin {}\n'.format(out_filename.lstrip('sf')))
            out_file.write('      task    specfit\n')

            #-- These are needed
            out_file.write('components    {}\n'.format(self.n_components))

            for item in self.components.itervalues():
                out_file.write('         {}\n'.format(item.type))

            for item in self.components.itervalues():
                out_file.write('{:>19}    {} # {}\n'.format(item.name,
                                                           item.n_pars,
                                                           item.label))
                for par in item.parameters.itervalues():
                    out_file.write('{:>33}{:>14}{:>14}{:>14}{:>10}{:>10}\n'.format(par.value,
                                                                              par.lower_lim,
                                                                              par.upper_lim,
                                                                              par.stepsize,
                                                                              par.tolerance,
                                                                              par.linkage))

#-------------------------------------------------------------------------------

class SpecfitComponent:
    """Parse and modify an individual compenent of a SpecFit paramter file"""

    _par_names = {'linear': ['flux', 'slope'],
                  'powerlaw': ['flux', 'index'],
                  'bpl': ['flux', 'break', 'index_above', 'index_below'],
                  'blackbody': ['flux', 'temperature'],
                  'gaussian': ['flux', 'centroid', 'fwhm', 'skew'],
                  'logarith': ['flux', 'centroid', 'fwhm', 'slew'],
                  'labsorp': ['eq_width', 'centroid', 'fwhm'],
                  'tauabs': ['depth', 'centroid', 'fwhm'],
                  'eabsorp': ['depth', 'wavelength'],
                  'recomb': ['flux', 'wavelength', 'temperature', 'fwhm'],
                  'extcor': ['e(v-b)'],
                  'usercont': ['norm', 'shift', 'redshift', 'key'],
                  'userline': ['norm', 'shift', 'redshift', 'key'],
                  'userabs': ['norm', 'shift', 'redshift', 'key'],
                  'lorentz': ['flux', 'centroid', 'fwhm', 'alpha'],
                  'dampabs': ['density', 'centroid', 'lifetime'],
                  'logabs': ['depth', 'centroid', 'fwhm'],
                  'ffree': ['norm', 'temperature'],
                  'extdrude': ['e(b-v)', 'w0', 'gamma', 'c1', 'c2', 'c3', 'c4'],
                  'disk': ['flux', 'beta', 'temperature'],
                  'ccmext': ['e(b-v)', 'rv']}

    def __init__(self, lines):
        self.name, self.n_pars = lines[0].strip().split()[0:2]
        self.n_pars = int(self.n_pars)
        self.label = ' '.join(lines[0].strip().split()[3:]) or ''

        #-- Strip out numbers to get type of model fit
        #-- ALL numbers will be stripped, but they should all be at the end
        self.type = ''.join([item for item in self.name if not item.isdigit()])
        self.number = int(''.join([item for item in self.name if item.isdigit()]))

        if self.n_pars != len(lines[1:]):
            message = "Incorrect number of parameters specified for"
            message += "{}, got {} instead of {}".format(self.type,
                                                         len(lines[1:]),
                                                         self.n_pars)
            raise ValueError(message)

        self.parameters = OrderedDict()
        for key, line in zip(self._par_names[self.type], lines[1:]):
            self.parameters[key] = SpecfitParameter(line)

    def __str__(self):
        output = "component: {} #{}\n".format(self.type, self.number)
        output += '\n'.join(['--' + str(item) for item in self.parameters])
        return output

    def __iter__(self):
        for item in self.parameters:
            yield item

#-------------------------------------------------------------------------------

class SpecfitParameter:
    """Parse and modify a parameter of a compenent"""

    def __init__(self, line):
        values = map(float, line.strip().split())

        if len(values) != 6:
            raise ValueError('Parameter can only have 6 values!')

        self.value = values[0]
        self.lower_lim = values[1]
        self.upper_lim = values[2]
        self.stepsize = values[3]
        self.tolerance = values[4]
        self.linkage = values[5]

    def __str__(self):
        return "parameter: {} {} {} {} {} {}".format(self.value,
                                                     self.lower_lim,
                                                     self.upper_lim,
                                                     self.stepsize,
                                                     self.tolerance,
                                                     self.linkage)

    def free(self):
        if self.linkage <= 0:
            self.linkage = 0

    def fix(self):
        if self.linkage <= 0:
            self.linkage = -1

#-------------------------------------------------------------------------------

def gaussian(x, area, sigma, center):
    sigma = sigma / (2 * np.sqrt(2 * np.log(2)))

    print area, sigma, center

    y = (area / (sigma * np.sqrt(2 * np.pi))) * np.exp((-(x-center)**2) / (2 * sigma**2))
    return y

#-------------------------------------------------------------------------------

def velocity_to_wavelength(velocity, zeropoint):
    LIGHTSPEED = 2.9979E5
    return zeropoint * velocity/LIGHTSPEED

#-------------------------------------------------------------------------------
<|endoftext|>


<commit_before>#!/usr/bin/python3 -tt
# -*- coding: utf-8 -*-

import re
import json
from serial import Serial
from time import sleep
from http.server import HTTPServer, BaseHTTPRequestHandler
from urllib.parse import urlparse, parse_qs
import stream as streams

class Car(object):

    def __init__(self, *args, **kwargs):
        self.serial = Serial(*args, **kwargs)

    def forward(self):
        self.serial.write(b'f')

    def backward(self):
        self.serial.write(b'b')

    def left(self):
        self.serial.write(b'l')

    def right(self):
        self.serial.write(b'r')

car = Car('/dev/ttyACM0', 115200)

sleep(2)

def run(server_class=HTTPServer, handler_class=BaseHTTPRequestHandler, port=8000):
    server_address = ('', port)
    httpd = server_class(server_address, handler_class)
    httpd.serve_forever()

class StreamController():
    """ Controls the thread for raspivid and netcat """
    stream = None

    def control(self, t):
        if t:
            print(self.stream)
            if self.stream != None:
                if self.stream.isAlive():
                    return {"status": "stream is already running"}
                else:
                    self.stream = streams.Stream()
            else:
                self.stream = streams.Stream()
            self.stream.start()
            return {"status": "stream started"}
        else:
            self.stream.stop()
            self.stream.join()
            return {"status": "stream ended"}

stream_controller = StreamController()

class RequestHandler(BaseHTTPRequestHandler):

    print("CREATING A NEW REQUEST INSTANCE")

    log_file = open('marsem.log', 'w')
    stream = None

    urls = {
        'control': r'^/$',
        'stream_controller': r'^/stream/$'
    }

    def do_GET(self):
        self.send_response(200)
        self.send_header('Content-Type', 'application/json')
        self.end_headers()

        result = urlparse(self.path)
        path = result.path
        query = {}

        for key, value in parse_qs(result.query).items():
            if len(value) == 1:
                value = value[0]
                try:
                    value = json.loads(value)
                except ValueError:
                    pass
            query[key] = value

        for method, regex in self.urls.items():
            if re.match(regex, path):
                data = getattr(self, method)(query)
                self.wfile.write(json.dumps(data).encode())

    def control(self, query):
        action = query.get('action')
        if action:
            action = getattr(car, action, None)
            if action:
                action()

        return {}

    def stream_controller(self, query):
        param = query.get('stream')
        if param.lower() == 'true':
            return stream_controller.control(True)
        else:
            return stream_controller.control(False)

    def log_message(self, format, *args):
        self.log_file.write("%s - - [%s] %s\n" %
                            (self.client_address[0],
                             self.log_date_time_string(),
                             format%args))

if __name__ == '__main__':
    run(handler_class=RequestHandler)
<commit_msg>Removed debugging print<commit_after>#!/usr/bin/python3 -tt
# -*- coding: utf-8 -*-

import re
import json
from serial import Serial
from time import sleep
from http.server import HTTPServer, BaseHTTPRequestHandler
from urllib.parse import urlparse, parse_qs
import stream as streams

class Car(object):

    def __init__(self, *args, **kwargs):
        self.serial = Serial(*args, **kwargs)

    def forward(self):
        self.serial.write(b'f')

    def backward(self):
        self.serial.write(b'b')

    def left(self):
        self.serial.write(b'l')

    def right(self):
        self.serial.write(b'r')

car = Car('/dev/ttyACM0', 115200)

sleep(2)

def run(server_class=HTTPServer, handler_class=BaseHTTPRequestHandler, port=8000):
    server_address = ('', port)
    httpd = server_class(server_address, handler_class)
    httpd.serve_forever()

class StreamController():
    """ Controls the thread for raspivid and netcat """
    stream = None

    def control(self, t):
        if t:
            print(self.stream)
            if self.stream != None:
                if self.stream.isAlive():
                    return {"status": "stream is already running"}
                else:
                    self.stream = streams.Stream()
            else:
                self.stream = streams.Stream()
            self.stream.start()
            return {"status": "stream started"}
        else:
            self.stream.stop()
            self.stream.join()
            return {"status": "stream ended"}

stream_controller = StreamController()

class RequestHandler(BaseHTTPRequestHandler):

    log_file = open('marsem.log', 'w')
    stream = None

    urls = {
        'control': r'^/$',
        'stream_controller': r'^/stream/$'
    }

    def do_GET(self):
        self.send_response(200)
        self.send_header('Content-Type', 'application/json')
        self.end_headers()

        result = urlparse(self.path)
        path = result.path
        query = {}

        for key, value in parse_qs(result.query).items():
            if len(value) == 1:
                value = value[0]
                try:
                    value = json.loads(value)
                except ValueError:
                    pass
            query[key] = value

        for method, regex in self.urls.items():
            if re.match(regex, path):
                data = getattr(self, method)(query)
                self.wfile.write(json.dumps(data).encode())

    def control(self, query):
        action = query.get('action')
        if action:
            action = getattr(car, action, None)
            if action:
                action()

        return {}

    def stream_controller(self, query):
        param = query.get('stream')
        if param.lower() == 'true':
            return stream_controller.control(True)
        else:
            return stream_controller.control(False)

    def log_message(self, format, *args):
        self.log_file.write("%s - - [%s] %s\n" %
                            (self.client_address[0],
                             self.log_date_time_string(),
                             format%args))

if __name__ == '__main__':
    run(handler_class=RequestHandler)
<|endoftext|>


<commit_before>#!/usr/bin/env python
import sys
import libsedml

def create_plot_with_styles(file_name):
    doc = libsedml.SedDocument(1,4)

    create_styles(doc)

    create_plot_1(doc)

    create_plot_2(doc)

    libsedml.writeSedML(doc, file_name)
    pass

def create_styles(doc):
    # create styles
    style = doc.createStyle()
    style.setId('line_black')
    line = style.createLine()
    line.setStyle('solid')
    line.setColor('#000000FF')
    style = doc.createStyle()
    style.setId('area_blue')
    fill = style.createFill()
    fill.setColor('#0000FF7F')
    style = doc.createStyle()
    style.setId('area_red')
    fill = style.createFill()
    fill.setColor('#0000FF7F')

def create_plot_1(doc):
    # create plots
    plot = doc.createPlot2D()
    plot.setId('plot_errorbar')
    plot.setLegend(True)
    # shaded area
    sa = plot.createShadedArea()
    sa.setId('cy_err_area')
    sa.setName('error_area')
    sa.setOrder(1)
    sa.setXDataReference('x')
    sa.setYDataReferenceFrom('yerr_down')
    sa.setYDataReferenceTo('yerr_up')
    sa.setStyle('area_blue')
    # curve
    curve = plot.createCurve()
    curve.setId('cy_err')
    curve.setName('errorbar')
    curve.setOrder(2)
    curve.setXDataReference('x')
    curve.setYDataReference('y')
    curve.setXErrorUpper('xerr')
    curve.setYErrorUpper('yerr')
    curve.setStyle('line_black')

def create_plot_2(doc):
    # create plots
    plot = doc.createPlot2D()
    plot.setId('plot_area')
    plot.setLegend(True)
    # shaded area
    sa = plot.createShadedArea()
    sa.setId('cy_sin_area')
    sa.setName('area')
    sa.setOrder(1)
    sa.setXDataReference('x')
    sa.setYDataReferenceFrom('y_zero')
    sa.setYDataReferenceTo('y_sin')
    sa.setStyle('area_red')

if __name__ == "__main__":
    print('using libSEDML: %s' % libsedml.getLibSEDMLDottedVersion())
    file_name = 'plot_with_styles.xml'
    num_args = len(sys.argv)
    if num_args > 1:
        file_name = sys.argv[1]
    create_plot_with_styles(file_name)<commit_msg>- fix example<commit_after>#!/usr/bin/env python
import sys
import libsedml

def create_plot_with_styles(file_name):
    doc = libsedml.SedDocument(1,4)

    create_styles(doc)

    create_plot_1(doc)

    create_plot_2(doc)

    libsedml.writeSedML(doc, file_name)
    pass

def create_styles(doc):
    # create styles
    style = doc.createStyle()
    style.setId('line_black')
    line = style.createLineStyle()
    line.setStyle('solid')
    line.setColor('#000000FF')
    style = doc.createStyle()
    style.setId('area_blue')
    fill = style.createFillStyle()
    fill.setColor('#0000FF7F')
    style = doc.createStyle()
    style.setId('area_red')
    fill = style.createFillStyle()
    fill.setColor('#0000FF7F')

def create_plot_1(doc):
    # create plots
    plot = doc.createPlot2D()
    plot.setId('plot_errorbar')
    plot.setLegend(True)
    # shaded area
    sa = plot.createShadedArea()
    sa.setId('cy_err_area')
    sa.setName('error_area')
    sa.setOrder(1)
    sa.setXDataReference('x')
    sa.setYDataReferenceFrom('yerr_down')
    sa.setYDataReferenceTo('yerr_up')
    sa.setStyle('area_blue')
    # curve
    curve = plot.createCurve()
    curve.setId('cy_err')
    curve.setName('errorbar')
    curve.setOrder(2)
    curve.setXDataReference('x')
    curve.setYDataReference('y')
    curve.setXErrorUpper('xerr')
    curve.setYErrorUpper('yerr')
    curve.setStyle('line_black')

def create_plot_2(doc):
    # create plots
    plot = doc.createPlot2D()
    plot.setId('plot_area')
    plot.setLegend(True)
    # shaded area
    sa = plot.createShadedArea()
    sa.setId('cy_sin_area')
    sa.setName('area')
    sa.setOrder(1)
    sa.setXDataReference('x')
    sa.setYDataReferenceFrom('y_zero')
    sa.setYDataReferenceTo('y_sin')
    sa.setStyle('area_red')

if __name__ == "__main__":
    print('using libSEDML: %s' % libsedml.getLibSEDMLDottedVersion())
    file_name = 'plot_with_styles.xml'
    num_args = len(sys.argv)
    if num_args > 1:
        file_name = sys.argv[1]
    create_plot_with_styles(file_name)<|endoftext|>


<commit_before>"""Python module for generating fake spectra from an N-body catalogue.

Note in Arepo we have GFM_Metals and GFM_Metallicity.

GFM_Metallicity is the total mass in species not H or He
per unit gas mass (and is used for cooling).

GFM_Metals is a 9-component array of species:
H, He, C, N, O, Ne, Mg, Si, Fe

Because these are not all the species, GFM_Metals will not sum to 1
and sum(GFM_Metals[2:])  < GFM_Metallicity

However, it should be true that
1- sum(GFM_Metals[:]) + GFM_Metals[2:] = GFM_Metallicity

Also note that there is some instability at very low metallicities - the code will often return +-1e-20.
"""

import numpy as np
import hsml
import math
import h5py
import convert_cloudy
import line_data
from _spectra_priv import _SPH_Interpolate

def SPH_Interpolate(data, los_table, nbins, box, hub, atime):
    """Interpolate particles to lines of sight, calculating density, temperature and velocity
    of various species along the line of sight.

    This is a wrapper which calls the C function.
    Arguments:
    	data - HDF5 dataset from snapshot. Use f["PartType0"]
	    los_table - table of los positions. should have member arrays x, y, z and axis.
	    nbins - number of bins in each spectrum
	    box - box size
        hub - hubble constant (eg 0.71)
        atime - 1/(1+z)

    Returns:
        rho_HI
        vel_HI
        temp_HI
        all as arrays along the line of sight
    """
    pos = np.array(data["Coordinates"],dtype=np.float32)
    vel = np.array(data["Velocities"],dtype=np.float32)
    mass = np.array(data["Masses"],dtype=np.float32)
    u = np.array(data["InternalEnergy"],dtype=np.float32)
    nh0 = np.array(data["NeutralHydrogenAbundance"],dtype=np.float32)
    ne = np.array(data["ElectronAbundance"],dtype=np.float32)
    try:
        metals = np.array(data["GFM_Metals"],dtype=np.float32)[:,2:]
        #Deal with floating point roundoff - metals will sometimes be negative
        metals[np.where(np.abs(metals) < 1e-10)] = 0
    except IOError:
        metals = np.array()
    hh = np.array(hsml.get_smooth_length(data),dtype=np.float32)
    xx=np.array(los_table.xx, dtype=np.float32)
    yy=np.array(los_table.yy, dtype=np.float32)
    zz=np.array(los_table.zz, dtype=np.float32)
    axis=np.array(los_table.axis, dtype=np.int32)
    return  _SPH_Interpolate(nbins, box, hub, atime, pos, vel, mass, u, nh0, ne, metals, hh, axis, xx, yy, zz)

#Speed of light
C = 2.99e8
#Boltzmann constant
BOLTZMANN = 1.3806504e-23
KPC = 3.08568025e19
MPC = KPC * 1000
SIGMA_T = 6.652458558e-29
PROTONMASS = 1.66053886e-27 # 1 a.m.u

def compute_absorption(xbins, rho, vel, temp, line, Hz, h100, box100, atime, mass):
    """Computes the absorption spectrum (tau (u) ) from a binned set of interpolated
    densities, velocities and temperatures.
    xbins are the positions of each bin along the sightline. A good default is
    xbins = np.range(0,NBINS)*Box/NBINS

    Optical depth is given by:
    tau (u) = sigma_X c / H(z) int_infty^infty n_x(x) V( u - x - v_pec, b(x) ) dx
    where V is the Voigt profile, b(x)^2 = 2k_B T /m_x c^2 is the velocity dispersion.
    and v_pec is the peculiar velocity.
    sigma_X is the cross-section for this transition.
    """
    nbins = np.size(xbins)
    tau = np.zeros(nbins)
    #  Conversion factors from internal units
    rscale = (KPC*atime)/h100  # convert length to m
    #  Calculate the length scales to be used in the box
    vmax = box100 * Hz * rscale/ MPC # box size (kms^-1)
    dzgrid   = box100 * rscale / 1.*nbins # bin size m
    dvbin = dzgrid * Hz / MPC # velocity bin size (kms^-1)

    #Absorption cross-sections m^2
    sigma_X  = np.sqrt(3.0*math.pi*SIGMA_T/8.0) * line.lambda_X  * line.fosc_X
    # Prefactor for optical depth
    A_H1 = sigma_X*C*dzgrid/np.sqrt(math.pi)
    #Compute the spectra optical depth
    for i in np.arange(0, nbins):
        uu = dvbin*1.e3*np.arange(0,nbins)
        uu += vel*1.e3
        # Note this is indexed with i, above with j!
        # This is the difference in velocities between two clouds
        # on the same sightline
        vdiff  = np.abs(dvbin*i*1.0e3 - uu)  # ms^-1
        ind = np.where(vdiff > vmax *1.e3 /2.)
        vdiff[ind] = vmax*1.e3 - vdiff[ind]
        #Impact parameter
        bb = np.sqrt(2.0*BOLTZMANN*temp/(mass*PROTONMASS))
        T0 = (vdiff/bb)**2
        T1 = np.exp(-T0)
        # Voigt profile: Tepper-Garcia, 2006, MNRAS, 369, 2025
        aa_H1 = line.gamma_X*line.lambda_X/(4.0*math.pi*bb)
        T2 = 1.5/T0
        ind = np.where(T0 > 1.e-6)
        profile = np.array(T1)
        profile[ind] = T1[ind] - aa_H1[ind]/np.sqrt(math.pi)/T0[ind]*(T1[ind]**2*(4.0*T0[ind]**2 + 7.0*T0[ind] + 4.0 + T2[ind]) - T2[ind] -1.0)
        tau[i] += np.sum(A_H1  * rho  * profile /(mass*PROTONMASS*bb))

    return tau

class MetalLines:
    """Generate metal line spectra from simulation snapshot"""
    def __init__(self,los_table, snapshot, cloudy_dir, nbins = 1024):
        #Get los table from group list
        f = h5py.File(snapshot)
        self.box = f["Header"].attrs["BoxSize"]
        self.hubble = f["Header"].attrs["Hubble"]
        self.atime = f["Header"].attrs["Time"]
        self.redshift = f["Header"].attrs["Redshift"]
        Omega0 = f["Header"].attrs["Omega0"]
        OmegaLambda = f["Header"].attrs["OmegaLambda"]
        self.Hz = 100.0*self.hubble * np.sqrt(Omega0/self.atime**3 + OmegaLambda)
        self.nbins = nbins
        self.xbins = np.arange(0,self.nbins)*self.box/self.nbins
        self.species = ['C','N','O','Ne','Mg','Si','Fe']
        #Line data
        self.lines = line_data.LineData()
        #generate metal and hydrogen spectral densities
        #Indexing is: rho_metals [ NMETALS, NSPECTRA, NBIN ]
        (self.rho_HI, self.temp_HI, self.vel_HI, self.rho_metal, self.temp_metal, self.vel_metal) = SPH_Interpolate(f["PartType0"], los_table, nbins, self.box, self.hubble, self.atime)
        #Compute tau for HI
        self.tau_HI = compute_absorption(self.xbins, self.rho_HI, self.vel_HI, self.temp_HI, self.lines.get_line('H',1),self.Hz,self.hubble, self.box, self.atime,self.lines.get_mass('H'))

        #Generate cloudy tables
        self.cloudy = convert_cloudy.CloudyTable(cloudy_dir)

    def get_lines(self, species, ion):
        """Get the optical depth for a particular species out of self.species:
           (C, N, O, Ne, Mg, Si, Fe)
           and some ion number
           NOTE: May wish to special-case SiIII at some point
        """
        spec_ind = self.species.index(species)
        metal_density = self.rho_metal[:, spec_ind]
        #Use the total metallicity from summing metal species, not from the GFM_Metallicity
        #variable as the difference is small (~ 4%)
        tot_met = np.sum(metal_density,axis=0)
        ion = self.cloudy.interpolate(self.redshift, tot_met, metal_density, spec_ind, ion)
        #Generate density of this ion: cloudy densities are in log_10
        ion_density = 10**ion * metal_density
        #Compute tau for this metal ion
        tau_metal = compute_absorption(self.xbins, ion_density, self.vel_metal[:,spec_ind], self.temp_metal[:,spec_ind], self.lines.get_line(species,1),self.Hz,self.hubble, self.box, self.atime,self.lines.get_mass(species))
        return tau_metal

<commit_msg>Correctly order output arrays (oops)<commit_after>"""Python module for generating fake spectra from an N-body catalogue.

Note in Arepo we have GFM_Metals and GFM_Metallicity.

GFM_Metallicity is the total mass in species not H or He
per unit gas mass (and is used for cooling).

GFM_Metals is a 9-component array of species:
H, He, C, N, O, Ne, Mg, Si, Fe

Because these are not all the species, GFM_Metals will not sum to 1
and sum(GFM_Metals[2:])  < GFM_Metallicity

However, it should be true that
1- sum(GFM_Metals[:]) + GFM_Metals[2:] = GFM_Metallicity

Also note that there is some instability at very low metallicities - the code will often return +-1e-20.
"""

import numpy as np
import hsml
import math
import h5py
import convert_cloudy
import line_data
from _spectra_priv import _SPH_Interpolate

def SPH_Interpolate(data, los_table, nbins, box, hub, atime):
    """Interpolate particles to lines of sight, calculating density, temperature and velocity
    of various species along the line of sight.

    This is a wrapper which calls the C function.
    Arguments:
    	data - HDF5 dataset from snapshot. Use f["PartType0"]
	    los_table - table of los positions. should have member arrays x, y, z and axis.
	    nbins - number of bins in each spectrum
	    box - box size
        hub - hubble constant (eg 0.71)
        atime - 1/(1+z)

    Returns:
        rho_HI
        vel_HI
        temp_HI
        all as arrays along the line of sight
    """
    pos = np.array(data["Coordinates"],dtype=np.float32)
    vel = np.array(data["Velocities"],dtype=np.float32)
    mass = np.array(data["Masses"],dtype=np.float32)
    u = np.array(data["InternalEnergy"],dtype=np.float32)
    nh0 = np.array(data["NeutralHydrogenAbundance"],dtype=np.float32)
    ne = np.array(data["ElectronAbundance"],dtype=np.float32)
    try:
        metals = np.array(data["GFM_Metals"],dtype=np.float32)[:,2:]
        #Deal with floating point roundoff - metals will sometimes be negative
        metals[np.where(np.abs(metals) < 1e-10)] = 0
    except IOError:
        metals = np.array()
    hh = np.array(hsml.get_smooth_length(data),dtype=np.float32)
    xx=np.array(los_table.xx, dtype=np.float32)
    yy=np.array(los_table.yy, dtype=np.float32)
    zz=np.array(los_table.zz, dtype=np.float32)
    axis=np.array(los_table.axis, dtype=np.int32)
    return  _SPH_Interpolate(nbins, box, hub, atime, pos, vel, mass, u, nh0, ne, metals, hh, axis, xx, yy, zz)

#Speed of light
C = 2.99e8
#Boltzmann constant
BOLTZMANN = 1.3806504e-23
KPC = 3.08568025e19
MPC = KPC * 1000
SIGMA_T = 6.652458558e-29
PROTONMASS = 1.66053886e-27 # 1 a.m.u

def compute_absorption(xbins, rho, vel, temp, line, Hz, h100, box100, atime, mass):
    """Computes the absorption spectrum (tau (u) ) from a binned set of interpolated
    densities, velocities and temperatures.
    xbins are the positions of each bin along the sightline. A good default is
    xbins = np.range(0,NBINS)*Box/NBINS

    Optical depth is given by:
    tau (u) = sigma_X c / H(z) int_infty^infty n_x(x) V( u - x - v_pec, b(x) ) dx
    where V is the Voigt profile, b(x)^2 = 2k_B T /m_x c^2 is the velocity dispersion.
    and v_pec is the peculiar velocity.
    sigma_X is the cross-section for this transition.
    """
    nbins = np.size(xbins)
    tau = np.zeros(nbins)
    #  Conversion factors from internal units
    rscale = (KPC*atime)/h100  # convert length to m
    #  Calculate the length scales to be used in the box
    vmax = box100 * Hz * rscale/ MPC # box size (kms^-1)
    dzgrid   = box100 * rscale / 1.*nbins # bin size m
    dvbin = dzgrid * Hz / MPC # velocity bin size (kms^-1)

    #Absorption cross-sections m^2
    sigma_X  = np.sqrt(3.0*math.pi*SIGMA_T/8.0) * line.lambda_X  * line.fosc_X
    # Prefactor for optical depth
    A_H1 = sigma_X*C*dzgrid/np.sqrt(math.pi)
    #Compute the spectra optical depth
    for i in np.arange(0, nbins):
        uu = dvbin*1.e3*np.arange(0,nbins)
        uu += vel*1.e3
        # Note this is indexed with i, above with j!
        # This is the difference in velocities between two clouds
        # on the same sightline
        vdiff  = np.abs(dvbin*i*1.0e3 - uu)  # ms^-1
        ind = np.where(vdiff > vmax *1.e3 /2.)
        vdiff[ind] = vmax*1.e3 - vdiff[ind]
        #Impact parameter
        bb = np.sqrt(2.0*BOLTZMANN*temp/(mass*PROTONMASS))
        T0 = (vdiff/bb)**2
        T1 = np.exp(-T0)
        # Voigt profile: Tepper-Garcia, 2006, MNRAS, 369, 2025
        aa_H1 = line.gamma_X*line.lambda_X/(4.0*math.pi*bb)
        T2 = 1.5/T0
        ind = np.where(T0 > 1.e-6)
        profile = np.array(T1)
        profile[ind] = T1[ind] - aa_H1[ind]/np.sqrt(math.pi)/T0[ind]*(T1[ind]**2*(4.0*T0[ind]**2 + 7.0*T0[ind] + 4.0 + T2[ind]) - T2[ind] -1.0)
        tau[i] += np.sum(A_H1  * rho  * profile /(mass*PROTONMASS*bb))

    return tau

class MetalLines:
    """Generate metal line spectra from simulation snapshot"""
    def __init__(self,los_table, snapshot, cloudy_dir="/home/spb/codes/ArepoCoolingTables/tmp_spb/", nbins = 1024):
        #Get los table from group list
        f = h5py.File(snapshot)
        self.box = f["Header"].attrs["BoxSize"]
        self.hubble = f["Header"].attrs["HubbleParam"]
        self.atime = f["Header"].attrs["Time"]
        self.redshift = f["Header"].attrs["Redshift"]
        Omega0 = f["Header"].attrs["Omega0"]
        OmegaLambda = f["Header"].attrs["OmegaLambda"]
        self.Hz = 100.0*self.hubble * np.sqrt(Omega0/self.atime**3 + OmegaLambda)
        self.nbins = nbins
        self.xbins = np.arange(0,self.nbins)*self.box/self.nbins
        self.species = ['C','N','O','Ne','Mg','Si','Fe']
        self.NumLos = np.size(los_table.axis)
        #Line data
        self.lines = line_data.LineData()
        #generate metal and hydrogen spectral densities
        #Indexing is: rho_metals [ NMETALS, NSPECTRA, NBIN ]
        (self.rho_HI, self.vel_HI, self.temp_HI, self.rho_metal, self.vel_metal, self.temp_metal) = SPH_Interpolate(f["PartType0"], los_table, nbins, self.box, self.hubble, self.atime)
        #Compute tau for HI
        for n in np.arange(0,self.NumLos):
            self.tau_HI = compute_absorption(self.xbins, self.rho_HI[n], self.vel_HI[n], self.temp_HI[n], self.lines.get_line('H',1),self.Hz,self.hubble, self.box, self.atime,self.lines.get_mass('H'))

        #Generate cloudy tables
        self.cloudy = convert_cloudy.CloudyTable(cloudy_dir)

    def get_lines(self, species, ion):
        """Get the optical depth for a particular species out of self.species:
           (C, N, O, Ne, Mg, Si, Fe)
           and some ion number
           NOTE: May wish to special-case SiIII at some point
        """
        spec_ind = self.species.index(species)
        metal_density = self.rho_metal[:, :, spec_ind]
        #Use the total metallicity from summing metal species, not from the GFM_Metallicity
        #variable as the difference is small (~ 4%)
        tot_met = np.sum(metal_density,axis=0)
        ion = self.cloudy.interpolate(self.redshift, tot_met, metal_density, spec_ind, ion)
        #Generate density of this ion: cloudy densities are in log_10
        ion_density = 10**ion * metal_density
        #Compute tau for this metal ion
        for n in np.arange(0,self.NumLos):
            tau_metal = compute_absorption(self.xbins, ion_density[n], self.vel_metal[n,:,spec_ind], self.temp_metal[n,:,spec_ind], self.lines.get_line(species,1),self.Hz,self.hubble, self.box, self.atime,self.lines.get_mass(species))
        return tau_metal

<|endoftext|>


<commit_before># coding: utf-8

from django.db import models

from django.utils.translation import ugettext_lazy as _

class Region(models.Model):
    '''
    Region
    ======
    Common regional zones. All messages can be grouped by this territorial
    cluster.
    * TODO: use django-mptt
    * TODO: make nested regions
    * TODO: link message to nested regions
    '''
    class Meta():
        ordering = ['order']

    # Region number
    name = models.CharField(max_length=200, verbose_name=_("region name"))
    # Region slug
    slug = models.SlugField(_("slug"))
    # Region center coordinates
    latitude = models.FloatField(_("latitude"))
    longitude = models.FloatField(_("longitude"))
    # Region map default zoom
    zoom = models.SmallIntegerField(_("map zoom"))
    order = models.IntegerField(_("order"))

    def __unicode__(self):
        return self.name

class Location(models.Model):
    '''
    Geolocation
    ===========
    This data represent POI(Point Of Interest).
    Object contain small piece of data:
    * Geocoordinates - latitude and longitude
    * Description - textual name of POI or it's human-readable address
    * Optional link to georegion. If this link exists
    '''
    # Geocoordinates
    latitude = models.FloatField()
    longitude = models.FloatField(db_column='longitude')
    # Optional link for region
    region = models.ForeignKey(
        Region,
        verbose_name=_("region"),
        null=True, blank=True)
    # Short description or address
    description = models.CharField(max_length=200)

    def __unicode__(self):
        return u'%f %f' % (self.latitude, self.longitude)
<commit_msg>Added region to admin site<commit_after># coding: utf-8
from django.contrib import admin
from django.db import models

from django.utils.translation import ugettext_lazy as _

class Region(models.Model):
    '''
    Region
    ======
    Common regional zones. All messages can be grouped by this territorial
    cluster.
    * TODO: use django-mptt
    * TODO: make nested regions
    * TODO: link message to nested regions
    '''
    class Meta():
        ordering = ['order']

    # Region number
    name = models.CharField(max_length=200, verbose_name=_("region name"))
    # Region slug
    slug = models.SlugField(_("slug"))
    # Region center coordinates
    latitude = models.FloatField(_("latitude"))
    longitude = models.FloatField(_("longitude"))
    # Region map default zoom
    zoom = models.SmallIntegerField(_("map zoom"))
    order = models.IntegerField(_("order"))

    def __unicode__(self):
        return self.name

admin.site.register(Region)

class Location(models.Model):
    '''
    Geolocation
    ===========
    This data represent POI(Point Of Interest).
    Object contain small piece of data:
    * Geocoordinates - latitude and longitude
    * Description - textual name of POI or it's human-readable address
    * Optional link to georegion. If this link exists
    '''
    # Geocoordinates
    latitude = models.FloatField()
    longitude = models.FloatField(db_column='longitude')
    # Optional link for region
    region = models.ForeignKey(
        Region,
        verbose_name=_("region"),
        null=True, blank=True)
    # Short description or address
    description = models.CharField(max_length=200)

    def __unicode__(self):
        return u'%f %f' % (self.latitude, self.longitude)
<|endoftext|>


<commit_before>from django.conf.urls import include, url

urlpatterns = [
    url(r'^fafl', include('fafl.urls')),
    url(r'^', include('aristotle_mdr.urls')),
    url(r'^browse/', include('aristotle_mdr.contrib.browse.urls')),
    url(r'^help/', include('aristotle_mdr.contrib.help.urls', app_name="aristotle_help", namespace="aristotle_help")),
]
<commit_msg>add self pub<commit_after>from django.conf.urls import include, url

urlpatterns = [
    url(r'^fafl', include('fafl.urls')),
    url(r'^', include('aristotle_mdr.urls')),
    url(r'^browse/', include('aristotle_mdr.contrib.browse.urls')),
    url(r'^help/', include('aristotle_mdr.contrib.help.urls', app_name="aristotle_help", namespace="aristotle_help")),
    url(r'^publish/', include('aristotle_mdr.contrib.self_publish.urls', app_name="aristotle_self_publish", namespace="aristotle_self_publish")),
]
<|endoftext|>


<commit_before>"""
Created August 14 2014
James Houghton <james.p.houghton@gmail.com>

Changed May 03 2017
Alexey Prey Mulyukin <alexprey@yandex.ru> from sdCloud.io developement team
    Changes:

    [May 03 2017] Alexey Prey Mulyukin: Integrate support to
        logical operators like 'AND', 'OR' and 'NOT'.
        Fix support the whitespaces in expressions between
        operators and operands.
        Add support to modulo operator - 'MOD'.
        Fix support for case insensitive in function names.

This module converts a string of SMILE syntax into Python

"""
import parsimonious
from parsimonious.nodes import NodeVisitor
import pkg_resources
import re
from ...py_backend import builder

# Here we define which python function each XMILE keyword corresponds to
functions = {
    # ===
    # 3.5.1 Mathematical Functions
    # http://docs.oasis-open.org/xmile/xmile/v1.0/csprd01/xmile-v1.0-csprd01.html#_Toc398039980
    # ===

    "abs": "abs",
    "arccos": "np.arccos",
    "arcsin": "np.arcsin",
    "arctan": "np.arctan",
    "cos": "np.cos",
    "sin": "np.sin",
    "tan": "np.tan",
    "exp": "np.exp",
    "inf": "np.inf",
    "int": "int",
    "ln": "np.log",
    "log10": "np.log10",
    "max": "max",
    "min": "min",
    "pi": "np.pi",
    "sqrt": "np.sqrt",

    # ===
    # 3.5.2 Statistical Functions
    # http://docs.oasis-open.org/xmile/xmile/v1.0/csprd01/xmile-v1.0-csprd01.html#_Toc398039981
    # ===

    "exprnd": "np.random.exponential",
    "lognormal": "np.random.lognormal",
    "normal": "np.random.normal",
    "poisson": "np.random.poisson",
    "random": "np.random.rand",

    # ===
    # 3.5.4 Test Input Functions
    # http://docs.oasis-open.org/xmile/xmile/v1.0/csprd01/xmile-v1.0-csprd01.html#_Toc398039983
    # ===

    "pulse": "functions.pulse_magnitude",
    "step": "functions.step",
    "ramp": "functions.ramp",

    # ===
    # 3.5.6 Miscellaneous Functions
    # http://docs.oasis-open.org/xmile/xmile/v1.0/csprd01/xmile-v1.0-csprd01.html#_Toc398039985
    # ===
    "if_then_else": "functions.if_then_else",
    # "init" !TODO!
    # "previous" !TODO!
    # "self" !TODO!
}

prefix_operators = {
    "not": " not ",
    "-": "-",
    "+": " ",
}

infix_operators = {
    "and": " and ",
    "or": " or ",
    "=": "==",
    "<=": "<=",
    "<": "<",
    ">=": ">=",
    ">": ">",
    "<>": "!=",
    "^": "**",
    "+": "+",
    "-": "-",
    "*": "*",
    "/": "/",
    "mod": "%",
}

# ====
# 3.5.3 Delay Functions
# http://docs.oasis-open.org/xmile/xmile/v1.0/csprd01/xmile-v1.0-csprd01.html#_Toc398039982
# ====

builders = {
    # !TODO! Should correct handle the last args, because it's optionally
    # "delay" !TODO! How to add the infinity delay?
    "delay1": lambda element, subscript_dict, args: builder.add_n_delay(args[0], args[1], args[2], "1", element['subs'], subscript_dict),
    "delay3": lambda element, subscript_dict, args: builder.add_n_delay(args[0], args[1], args[2], "3", element['subs'], subscript_dict),
    "delayn": lambda element, subscript_dict, args: builder.add_n_delay(args[0], args[1], args[3], args[2], element['subs'], subscript_dict),

    # !TODO! Should correct handle the last args, because it's optionally
    "smth1": lambda element, subscript_dict, args: builder.add_n_smooth(args[0], args[1], args[2], "1", element['subs'], subscript_dict),
    "smth3": lambda element, subscript_dict, args: builder.add_n_smooth(args[0], args[1], args[2], "3", element['subs'], subscript_dict),
    "smthn": lambda element, subscript_dict, args: builder.add_n_smooth(args[0], args[1], args[3], args[2], element['subs'], subscript_dict),

    # "forcst" !TODO!
    # "trend" !TODO!
}

def format_word_list(word_list):
    return '|'.join(
        [re.escape(k) for k in reversed(sorted(word_list, key=len))])

class SMILEParser(NodeVisitor):
    def __init__(self, model_namespace=None, subscript_dict=None):
        if model_namespace is None:
            model_namespace = {}

        if subscript_dict is None:
            subscript_dict = {}

        self.model_namespace = model_namespace
        self.subscript_dict = subscript_dict
        self.extended_model_namespace = {
            key.replace(' ', '_'): value for key, value in self.model_namespace.items()}
        self.extended_model_namespace.update(self.model_namespace)

        # ===
        # 3.5.5 Time Functions
        # http://docs.oasis-open.org/xmile/xmile/v1.0/csprd01/xmile-v1.0-csprd01.html#_Toc398039984
        # ===
        self.extended_model_namespace.update({'dt': 'time_step'})
        self.extended_model_namespace.update({'starttime': 'initial_time'})
        self.extended_model_namespace.update({'endtime': 'final_time'})

        grammar = pkg_resources.resource_string("pysd", "py_backend/xmile/smile.grammar")
        grammar = grammar.decode('ascii').format(
            funcs=format_word_list(functions.keys()),
            in_ops=format_word_list(infix_operators.keys()),
            pre_ops=format_word_list(prefix_operators.keys()),
            identifiers=format_word_list(self.extended_model_namespace.keys()),
            build_keywords=format_word_list(builders.keys())
        )

        self.grammar = parsimonious.Grammar(grammar)

    def parse(self, text, element, context='eqn'):
        """
           context : <string> 'eqn', 'defn'
                If context is set to equation, lone identifiers will be parsed as calls to elements
                If context is set to definition, lone identifiers will be cleaned and returned.
        """

        # Remove the inline comments from `text` before parsing the grammar
        # http://docs.oasis-open.org/xmile/xmile/v1.0/csprd01/xmile-v1.0-csprd01.html#_Toc398039973
        text = re.sub(r"\{[^}]*\}", "", text)

        self.ast = self.grammar.parse(text)
        self.context = context
        self.element = element
        self.new_structure = []

        py_expr = self.visit(self.ast)

        return ({
            'py_expr': py_expr
        }, self.new_structure)

    def visit_conditional_statement(self, n, vc):
        _IF, _1, condition_expr, _2, _THEN, _3, then_expr, _4, _ELSE, _5, else_expr = vc
        return "functions.if_then_else(" + condition_expr + ", " + then_expr + ", " + else_expr + ")"

    def visit_identifier(self, n, vc):
        return self.extended_model_namespace[n.text] + '()'

    def visit_func(self, n, vc):
        return functions[n.text.lower()]

    def visit_build_call(self, n, vc):
        builder_name = vc[0].lower();
        arguments = [e.strip() for e in vc[4].split(",")]
        name, structure = builders[builder_name](self.element, self.subscript_dict, arguments)
        self.new_structure += structure
        return name;

    def visit_pre_oper(self, n, vc):
        return prefix_operators[n.text.lower()]

    def visit_in_oper(self, n, vc):
        return infix_operators[n.text.lower()]

    def generic_visit(self, n, vc):
        """
        Replace childbearing nodes with a list of their children;
        for leaves, return the node text;
        for empty nodes, return an empty string.

        Handles:
        - call
        - parens
        -
        """
        return ''.join(filter(None, vc)) or n.text or ''
<commit_msg>Make the initial value for delay* and smth* functions are optional<commit_after>"""
Created August 14 2014
James Houghton <james.p.houghton@gmail.com>

Changed May 03 2017
Alexey Prey Mulyukin <alexprey@yandex.ru> from sdCloud.io developement team
    Changes:

    [May 03 2017] Alexey Prey Mulyukin: Integrate support to
        logical operators like 'AND', 'OR' and 'NOT'.
        Fix support the whitespaces in expressions between
        operators and operands.
        Add support to modulo operator - 'MOD'.
        Fix support for case insensitive in function names.

This module converts a string of SMILE syntax into Python

"""
import parsimonious
from parsimonious.nodes import NodeVisitor
import pkg_resources
import re
from ...py_backend import builder

# Here we define which python function each XMILE keyword corresponds to
functions = {
    # ===
    # 3.5.1 Mathematical Functions
    # http://docs.oasis-open.org/xmile/xmile/v1.0/csprd01/xmile-v1.0-csprd01.html#_Toc398039980
    # ===

    "abs": "abs",
    "arccos": "np.arccos",
    "arcsin": "np.arcsin",
    "arctan": "np.arctan",
    "cos": "np.cos",
    "sin": "np.sin",
    "tan": "np.tan",
    "exp": "np.exp",
    "inf": "np.inf",
    "int": "int",
    "ln": "np.log",
    "log10": "np.log10",
    "max": "max",
    "min": "min",
    "pi": "np.pi",
    "sqrt": "np.sqrt",

    # ===
    # 3.5.2 Statistical Functions
    # http://docs.oasis-open.org/xmile/xmile/v1.0/csprd01/xmile-v1.0-csprd01.html#_Toc398039981
    # ===

    "exprnd": "np.random.exponential",
    "lognormal": "np.random.lognormal",
    "normal": "np.random.normal",
    "poisson": "np.random.poisson",
    "random": "np.random.rand",

    # ===
    # 3.5.4 Test Input Functions
    # http://docs.oasis-open.org/xmile/xmile/v1.0/csprd01/xmile-v1.0-csprd01.html#_Toc398039983
    # ===

    "pulse": "functions.pulse_magnitude",
    "step": "functions.step",
    "ramp": "functions.ramp",

    # ===
    # 3.5.6 Miscellaneous Functions
    # http://docs.oasis-open.org/xmile/xmile/v1.0/csprd01/xmile-v1.0-csprd01.html#_Toc398039985
    # ===
    "if_then_else": "functions.if_then_else",
    # "init" !TODO!
    # "previous" !TODO!
    # "self" !TODO!
}

prefix_operators = {
    "not": " not ",
    "-": "-",
    "+": " ",
}

infix_operators = {
    "and": " and ",
    "or": " or ",
    "=": "==",
    "<=": "<=",
    "<": "<",
    ">=": ">=",
    ">": ">",
    "<>": "!=",
    "^": "**",
    "+": "+",
    "-": "-",
    "*": "*",
    "/": "/",
    "mod": "%",
}

# ====
# 3.5.3 Delay Functions
# http://docs.oasis-open.org/xmile/xmile/v1.0/csprd01/xmile-v1.0-csprd01.html#_Toc398039982
# ====

builders = {
    # !TODO! Should correct handle the last args, should take the initial value of element instead the 0 if this parameter is missing
    # "delay" !TODO! How to add the infinity delay?
    "delay1": lambda element, subscript_dict, args: builder.add_n_delay(args[0], args[1], args[2] if len(args) > 2 else 0, "1", element['subs'], subscript_dict),
    "delay3": lambda element, subscript_dict, args: builder.add_n_delay(args[0], args[1], args[2] if len(args) > 2 else 0, "3", element['subs'], subscript_dict),
    "delayn": lambda element, subscript_dict, args: builder.add_n_delay(args[0], args[1], args[2] if len(args) > 3 else 0, args[2], element['subs'], subscript_dict),

    # !TODO! Should correct handle the last args, because it's optionally
    "smth1": lambda element, subscript_dict, args: builder.add_n_smooth(args[0], args[1], args[2] if len(args) > 2 else 0, "1", element['subs'], subscript_dict),
    "smth3": lambda element, subscript_dict, args: builder.add_n_smooth(args[0], args[1], args[2] if len(args) > 2 else 0, "3", element['subs'], subscript_dict),
    "smthn": lambda element, subscript_dict, args: builder.add_n_smooth(args[0], args[1], args[2] if len(args) > 3 else 0, args[2], element['subs'], subscript_dict),

    # "forcst" !TODO!
    # "trend" !TODO!
}

def format_word_list(word_list):
    return '|'.join(
        [re.escape(k) for k in reversed(sorted(word_list, key=len))])

class SMILEParser(NodeVisitor):
    def __init__(self, model_namespace=None, subscript_dict=None):
        if model_namespace is None:
            model_namespace = {}

        if subscript_dict is None:
            subscript_dict = {}

        self.model_namespace = model_namespace
        self.subscript_dict = subscript_dict
        self.extended_model_namespace = {
            key.replace(' ', '_'): value for key, value in self.model_namespace.items()}
        self.extended_model_namespace.update(self.model_namespace)

        # ===
        # 3.5.5 Time Functions
        # http://docs.oasis-open.org/xmile/xmile/v1.0/csprd01/xmile-v1.0-csprd01.html#_Toc398039984
        # ===
        self.extended_model_namespace.update({'dt': 'time_step'})
        self.extended_model_namespace.update({'starttime': 'initial_time'})
        self.extended_model_namespace.update({'endtime': 'final_time'})

        grammar = pkg_resources.resource_string("pysd", "py_backend/xmile/smile.grammar")
        grammar = grammar.decode('ascii').format(
            funcs=format_word_list(functions.keys()),
            in_ops=format_word_list(infix_operators.keys()),
            pre_ops=format_word_list(prefix_operators.keys()),
            identifiers=format_word_list(self.extended_model_namespace.keys()),
            build_keywords=format_word_list(builders.keys())
        )

        self.grammar = parsimonious.Grammar(grammar)

    def parse(self, text, element, context='eqn'):
        """
           context : <string> 'eqn', 'defn'
                If context is set to equation, lone identifiers will be parsed as calls to elements
                If context is set to definition, lone identifiers will be cleaned and returned.
        """

        # Remove the inline comments from `text` before parsing the grammar
        # http://docs.oasis-open.org/xmile/xmile/v1.0/csprd01/xmile-v1.0-csprd01.html#_Toc398039973
        text = re.sub(r"\{[^}]*\}", "", text)

        self.ast = self.grammar.parse(text)
        self.context = context
        self.element = element
        self.new_structure = []

        py_expr = self.visit(self.ast)

        return ({
            'py_expr': py_expr
        }, self.new_structure)

    def visit_conditional_statement(self, n, vc):
        _IF, _1, condition_expr, _2, _THEN, _3, then_expr, _4, _ELSE, _5, else_expr = vc
        return "functions.if_then_else(" + condition_expr + ", " + then_expr + ", " + else_expr + ")"

    def visit_identifier(self, n, vc):
        return self.extended_model_namespace[n.text] + '()'

    def visit_func(self, n, vc):
        return functions[n.text.lower()]

    def visit_build_call(self, n, vc):
        builder_name = vc[0].lower();
        arguments = [e.strip() for e in vc[4].split(",")]
        name, structure = builders[builder_name](self.element, self.subscript_dict, arguments)
        self.new_structure += structure
        return name;

    def visit_pre_oper(self, n, vc):
        return prefix_operators[n.text.lower()]

    def visit_in_oper(self, n, vc):
        return infix_operators[n.text.lower()]

    def generic_visit(self, n, vc):
        """
        Replace childbearing nodes with a list of their children;
        for leaves, return the node text;
        for empty nodes, return an empty string.

        Handles:
        - call
        - parens
        -
        """
        return ''.join(filter(None, vc)) or n.text or ''
<|endoftext|>


<commit_before>#!/usr/bin/env python

"""
Python script to obtain GitHub Release download count and other statistics.
"""

import os
import sys
import json
import time

try:
    import urllib2
except ImportError:
    import urllib.request as urllib2

try:
    input = raw_input
except NameError:
    pass

__version__ = "1.1.1"
__author__ = "Alexander Gorishnyak"
__email__ = "kefir500@gmail.com"
__license__ = "MIT"

class ConnectionError(Exception):
    """
    Raised on connection error.
    """

    def __init__(self, reason=None):
        """
        :param reason: Connection error reason.
        """
        self.reason = reason if reason else "Unknown reason."

    def __str__(self):
        return str(self.reason)

class GithubError(Exception):
    """
    Generic exception raised on GitHub API HTTP error.
    """

    def __init__(self, code, message=None):
        """
        :param code: HTTP error code.
        :param message: Exception message text.
        """
        self.code = code
        self.message = message

    def __str__(self):
        return "GitHub API HTTP {0}{1}".format(self.code, (": " + self.message) if self.message else "")

class GithubRepoError(GithubError):
    """
    Raised when accessing nonexistent GitHub username, repository or release tag.
    """

    def __init__(self, message=None):
        """
        :param message: Exception message text.
        """
        self.code = 404
        self.message = message if message else "Invalid GitHub username, repository or release tag."

class GithubLimitError(GithubError):
    """
    Raised when GitHub API request limit is exceeded.
    """

    def __init__(self, message=None):
        """
        :param message: Exception message text.
        """
        self.code = 403
        self.message = message if message else "Request limit exceeded."

class GithubTokenError(GithubError):
    """
    Raised when trying to pass invalid GitHub OAuth token.
    """

    def __init__(self, message=None):
        """
        :param message: Exception message text.
        """
        self.code = 401
        self.message = message if message else "Unauthorized. Check your \"GITHUB_TOKEN\" environment variable."

class _Text:
    """
    Definitions for colored output (no ANSI colors on Windows).
    """
    if os.name != "nt":
        HEADER = "\033[1;36m"
        SUCCESS = "\033[1;32m"
        ERROR = "\033[1;31m"
        BOLD = "\033[1m"
        ITALIC = "\033[3m"
        UNDERLINE = "\033[4m"
        END = "\033[0m"
    else:
        HEADER = SUCCESS = ERROR = BOLD = ITALIC = UNDERLINE = END = ""

def error(message):
    """
    Halt script due to critical error.
    :param message: Error text.
    """
    sys.exit(_Text.ERROR + "Error: " + message + _Text.END)

def get_env_token():
    """
    Get GitHub OAuth token from "GITHUB_TOKEN" environment variable.
    :return: GitHub OAuth token.
    """
    token = None if "GITHUB_TOKEN" not in os.environ else os.environ["GITHUB_TOKEN"]
    return token

def print_help():
    """
    Display command line help.
    """
    print("GitHub Download Stats")
    print("Python script to obtain GitHub Release download count and other statistics.\n")
    print("Usage:\n"
          "  python ghstats.py [{0}user{1}] [{0}repo{1}] [{0}tag{1}] [{0}options{1}]\n"
          "  python ghstats.py [{0}user{1}/{0}repo{1}] [{0}tag{1}] [{0}options{1}]\n"
          .format(_Text.ITALIC, _Text.END))
    print("Arguments:\n"
          "  {0}user{1}  Repository owner. If not present, user will be prompted for input.\n"
          "  {0}repo{1}  Repository title. If not present, user will be prompted for input.\n"
          "  {0}tag{1}   Release tag name. If not present, prints the total number of downloads.\n"
          .format(_Text.BOLD, _Text.END))
    print("Options:\n"
          "  {0}-d{1}, {0}--detail{1}  Print detailed statistics for release(s).\n"
          "  {0}-q{1}, {0}--quiet{1}   Print only resulting numbers and errors.\n"
          "                Overrides -d option.\n"
          "  {0}-l{1}, {0}--latest{1}  Get stats for the latest release.\n"
          "                Tag argument will be ignored.\n"
          "  {0}-h{1}, {0}--help{1}    Show this help.\n"
          .format(_Text.BOLD, _Text.END))
    print("Environment Variables:\n"
          "  {0}GITHUB_TOKEN{2}   GitHub OAuth token.\n"
          "                 Use to increase API request limit.\n"
          "                 {1}https://github.com/settings/tokens{2}"
          .format(_Text.BOLD, _Text.UNDERLINE, _Text.END))
    sys.exit(0)

def print_greeting():
    """
    Display greeting message.
    """
    print("GitHub Download Stats\n"
          "Author: Alexander Gorishnyak <kefir500@gmail.com>\n")

def download_stats(user=None, repo=None, tag=None, latest=False, token=None, quiet=False):
    """
    Get download statistics from GitHub API.
    :param user: GitHub repository owner username. If empty, user will be prompted for input.
    :param repo: GitHub repository name. If empty, user will be prompted for input.
    :param tag: Release tag name. If empty, get stats for all releases.
    :param latest: If True, ignore "tag" parameter and get stats for the latest release.
    :param token: GitHub OAuth token. If empty, API request limit will be reduced.
    :param quiet: If True, print nothing.
    :return: Statistics on downloads.
    :raises GithubRepoError: When accessing nonexistent GitHub username, repository or release tag.
    :raises GithubLimitError: When GitHub API request limit is exceeded.
    :raises GithubTokenError: When trying to pass invalid GitHub API OAuth token.
    :raises ConnectionError: On connection error.
    """
    if not user:
        user = input("GitHub Username: ")
    if not repo:
        repo = input("GitHub Repository: ")
    if not quiet:
        print("Downloading {0}/{1} stats...".format(user, repo))
    url = "https://api.github.com/repos/{0}/{1}/releases".format(user, repo)
    url += ("" if not tag else "/tags/" + tag) if not latest else "/latest"
    headers = {} if not token else {"Authorization": "token " + token}
    request = urllib2.Request(url, headers=headers)
    start = time.time()
    try:
        response = urllib2.urlopen(request).read().decode("utf-8")
    except urllib2.HTTPError as e:
        if e.code == 404:
            raise GithubRepoError()    # Invalid GitHub username, repository or release tag.
        elif e.code == 403:
            raise GithubLimitError()   # GitHub API request limit exceeded.
        elif e.code == 401:
            raise GithubTokenError()   # Invalid GitHub OAuth token.
        else:
            raise GithubError(e.code)  # Generic GitHub API exception.
    except urllib2.URLError as e:
        raise ConnectionError(e.reason)
    stats = json.loads(response)
    if not quiet:
        end = time.time()
        print("Downloaded in {0:.3f}s".format(end - start))
    return stats

def get_release_downloads(release, quiet=False):
    """
    Get number of downloads for a single release.
    :param release: Release statistics from GitHub API.
    :param quiet: If True, print nothing.
    :return: Number of downloads for a release.
    """
    downloads_total = 0
    if not quiet:
        encoding = sys.stdout.encoding or "ascii"
        title = release["name"].encode(encoding, 'ignore').decode(encoding)
        published = (time.strftime("%c", time.strptime(release["published_at"], "%Y-%m-%dT%H:%M:%SZ"))
                     if release["published_at"] else "Unpublished")
        print("")
        print("          --- " + _Text.HEADER + title + _Text.END + " ---\n")
        print("         Tag: " + release["tag_name"])
        print("      Author: " + release["author"]["login"])
        print("         URL: " + _Text.UNDERLINE + release["html_url"] + _Text.END)
        print("Published at: " + published)
        print("")
    if "assets" in release:
        indent = 13 + len(_Text.BOLD + _Text.END)
        for package in release["assets"]:
            downloads_release = package["download_count"]
            downloads_total += downloads_release
            if not quiet:
                print("{1:>{0}} {2}".format(
                      indent, _Text.BOLD + str(downloads_release) + _Text.END,
                      (package["label"] if package["label"] else package["name"])))
        if not quiet:
            print("{1:>{0}} Total".format(indent, _Text.BOLD + str(downloads_total) + _Text.END))
    return downloads_total

def get_stats_downloads(stats, quiet=False):
    """
    Get number of downloads from statistics.
    :param stats: Statistics from GitHub API.
    :param quiet: If True, print nothing.
    :return: Number of downloads.
    """
    total = 0
    if isinstance(stats, dict):
        total = get_release_downloads(stats, quiet)
    else:
        for release in stats:
            total += get_release_downloads(release, quiet)
    return total

def print_total(total, quiet=False, tag=None):
    """
    Print total number of downloads.
    :param total: Total number of downloads.
    :param quiet: If True, print only number of downloads without an additional text.
    :param tag: Release tag name (optional).
    :return: Total number of downloads passed via "total" parameter.
    """
    if not quiet:
        print("\n{0}Total Downloads{1}: {2}"
              .format(_Text.BOLD,
                      "" if not tag else " (" + tag + ")",
                      _Text.SUCCESS + str(total) + _Text.END))
    elif __name__ == "__main__":  # pragma: no cover
        print(str(total))
    return total

def main(user=None, repo=None, tag=None, latest=False, detail=False, token=None, quiet=False):
    """
    Get number of downloads for GitHub release(s).
    :param user: GitHub repository owner username. If empty, user will be prompted for input.
    :param repo: GitHub repository name. If empty, user will be prompted for input.
    :param tag: Release tag name. If empty, get stats for all releases.
    :param latest: If True, ignore "tag" parameter and get stats for the latest release.
    :param detail: Detailed output containing release information.
    :param token: GitHub OAuth token. If empty, API request limit will be reduced.
    :param quiet: If True, print nothing.
    :return: Number of downloads.
    """
    if not quiet:
        print_greeting()
    try:
        stats = download_stats(user, repo, tag, latest, token, quiet)
    except GithubError as e:
        error(e.message)
    except ConnectionError as e:
        error(str(e))
    else:
        total = get_stats_downloads(stats, quiet or not detail)
        print_total(total, quiet, tag or (stats["tag_name"] if latest else None))
        return total

def main_cli(args=None):
    """
    Parse command line arguments and pass to the main function.
    :param args: Command line arguments (without script name).
    :return: Number of downloads.
    """
    user = None              # GitHub username
    repo = None              # GitHub repository
    tag = None               # GitHub release tag
    latest = False           # Latest release
    detail = False           # Detailed output
    quiet = False            # Quiet output
    token = get_env_token()  # GitHub token
    if args is None:
        args = sys.argv[1:]
    for arg in args:
        if arg == "-q" or arg == "--quiet":
            quiet = True
        elif arg == "-d" or arg == "--detail":
            detail = True
        elif arg == "-l" or arg == "--latest":
            latest = True
        elif arg == "-h" or arg == "--help" or arg == "-?":
            print_help()
        else:
            if not user:
                if "/" not in arg:
                    user = arg
                else:
                    userrepo = arg.split("/")
                    user = userrepo[0]
                    repo = userrepo[1]
            elif not repo:
                repo = arg
            elif quiet and detail and latest:
                break
            elif not tag:
                tag = arg
    return main(user, repo, tag, latest, detail, token, quiet)

if __name__ == "__main__":  # pragma: no cover
    try:
        main_cli(sys.argv[1:])
    except KeyboardInterrupt:
        pass
<commit_msg>Fix quiet output for module import<commit_after>#!/usr/bin/env python

"""
Python script to obtain GitHub Release download count and other statistics.
"""

import os
import sys
import json
import time

try:
    import urllib2
except ImportError:
    import urllib.request as urllib2

try:
    input = raw_input
except NameError:
    pass

__version__ = "1.1.1"
__author__ = "Alexander Gorishnyak"
__email__ = "kefir500@gmail.com"
__license__ = "MIT"

class ConnectionError(Exception):
    """
    Raised on connection error.
    """

    def __init__(self, reason=None):
        """
        :param reason: Connection error reason.
        """
        self.reason = reason if reason else "Unknown reason."

    def __str__(self):
        return str(self.reason)

class GithubError(Exception):
    """
    Generic exception raised on GitHub API HTTP error.
    """

    def __init__(self, code, message=None):
        """
        :param code: HTTP error code.
        :param message: Exception message text.
        """
        self.code = code
        self.message = message

    def __str__(self):
        return "GitHub API HTTP {0}{1}".format(self.code, (": " + self.message) if self.message else "")

class GithubRepoError(GithubError):
    """
    Raised when accessing nonexistent GitHub username, repository or release tag.
    """

    def __init__(self, message=None):
        """
        :param message: Exception message text.
        """
        self.code = 404
        self.message = message if message else "Invalid GitHub username, repository or release tag."

class GithubLimitError(GithubError):
    """
    Raised when GitHub API request limit is exceeded.
    """

    def __init__(self, message=None):
        """
        :param message: Exception message text.
        """
        self.code = 403
        self.message = message if message else "Request limit exceeded."

class GithubTokenError(GithubError):
    """
    Raised when trying to pass invalid GitHub OAuth token.
    """

    def __init__(self, message=None):
        """
        :param message: Exception message text.
        """
        self.code = 401
        self.message = message if message else "Unauthorized. Check your \"GITHUB_TOKEN\" environment variable."

class _Text:
    """
    Definitions for colored output (no ANSI colors on Windows).
    """
    if os.name != "nt":
        HEADER = "\033[1;36m"
        SUCCESS = "\033[1;32m"
        ERROR = "\033[1;31m"
        BOLD = "\033[1m"
        ITALIC = "\033[3m"
        UNDERLINE = "\033[4m"
        END = "\033[0m"
    else:
        HEADER = SUCCESS = ERROR = BOLD = ITALIC = UNDERLINE = END = ""

def error(message):
    """
    Halt script due to critical error.
    :param message: Error text.
    """
    sys.exit(_Text.ERROR + "Error: " + message + _Text.END)

def get_env_token():
    """
    Get GitHub OAuth token from "GITHUB_TOKEN" environment variable.
    :return: GitHub OAuth token.
    """
    token = None if "GITHUB_TOKEN" not in os.environ else os.environ["GITHUB_TOKEN"]
    return token

def print_help():
    """
    Display command line help.
    """
    print("GitHub Download Stats")
    print("Python script to obtain GitHub Release download count and other statistics.\n")
    print("Usage:\n"
          "  python ghstats.py [{0}user{1}] [{0}repo{1}] [{0}tag{1}] [{0}options{1}]\n"
          "  python ghstats.py [{0}user{1}/{0}repo{1}] [{0}tag{1}] [{0}options{1}]\n"
          .format(_Text.ITALIC, _Text.END))
    print("Arguments:\n"
          "  {0}user{1}  Repository owner. If not present, user will be prompted for input.\n"
          "  {0}repo{1}  Repository title. If not present, user will be prompted for input.\n"
          "  {0}tag{1}   Release tag name. If not present, prints the total number of downloads.\n"
          .format(_Text.BOLD, _Text.END))
    print("Options:\n"
          "  {0}-d{1}, {0}--detail{1}  Print detailed statistics for release(s).\n"
          "  {0}-q{1}, {0}--quiet{1}   Print only resulting numbers and errors.\n"
          "                Overrides -d option.\n"
          "  {0}-l{1}, {0}--latest{1}  Get stats for the latest release.\n"
          "                Tag argument will be ignored.\n"
          "  {0}-h{1}, {0}--help{1}    Show this help.\n"
          .format(_Text.BOLD, _Text.END))
    print("Environment Variables:\n"
          "  {0}GITHUB_TOKEN{2}   GitHub OAuth token.\n"
          "                 Use to increase API request limit.\n"
          "                 {1}https://github.com/settings/tokens{2}"
          .format(_Text.BOLD, _Text.UNDERLINE, _Text.END))
    sys.exit(0)

def print_greeting():
    """
    Display greeting message.
    """
    print("GitHub Download Stats\n"
          "Author: Alexander Gorishnyak <kefir500@gmail.com>\n")

def download_stats(user=None, repo=None, tag=None, latest=False, token=None, quiet=False):
    """
    Get download statistics from GitHub API.
    :param user: GitHub repository owner username. If empty, user will be prompted for input.
    :param repo: GitHub repository name. If empty, user will be prompted for input.
    :param tag: Release tag name. If empty, get stats for all releases.
    :param latest: If True, ignore "tag" parameter and get stats for the latest release.
    :param token: GitHub OAuth token. If empty, API request limit will be reduced.
    :param quiet: If True, print nothing.
    :return: Statistics on downloads.
    :raises GithubRepoError: When accessing nonexistent GitHub username, repository or release tag.
    :raises GithubLimitError: When GitHub API request limit is exceeded.
    :raises GithubTokenError: When trying to pass invalid GitHub API OAuth token.
    :raises ConnectionError: On connection error.
    """
    if not user:
        user = input("GitHub Username: ")
    if not repo:
        repo = input("GitHub Repository: ")
    if not quiet:
        print("Downloading {0}/{1} stats...".format(user, repo))
    url = "https://api.github.com/repos/{0}/{1}/releases".format(user, repo)
    url += ("" if not tag else "/tags/" + tag) if not latest else "/latest"
    headers = {} if not token else {"Authorization": "token " + token}
    request = urllib2.Request(url, headers=headers)
    start = time.time()
    try:
        response = urllib2.urlopen(request).read().decode("utf-8")
    except urllib2.HTTPError as e:
        if e.code == 404:
            raise GithubRepoError()    # Invalid GitHub username, repository or release tag.
        elif e.code == 403:
            raise GithubLimitError()   # GitHub API request limit exceeded.
        elif e.code == 401:
            raise GithubTokenError()   # Invalid GitHub OAuth token.
        else:
            raise GithubError(e.code)  # Generic GitHub API exception.
    except urllib2.URLError as e:
        raise ConnectionError(e.reason)
    stats = json.loads(response)
    if not quiet:
        end = time.time()
        print("Downloaded in {0:.3f}s".format(end - start))
    return stats

def get_release_downloads(release, quiet=False):
    """
    Get number of downloads for a single release.
    :param release: Release statistics from GitHub API.
    :param quiet: If True, print nothing.
    :return: Number of downloads for a release.
    """
    downloads_total = 0
    if not quiet:
        encoding = sys.stdout.encoding or "ascii"
        title = release["name"].encode(encoding, 'ignore').decode(encoding)
        published = (time.strftime("%c", time.strptime(release["published_at"], "%Y-%m-%dT%H:%M:%SZ"))
                     if release["published_at"] else "Unpublished")
        print("")
        print("          --- " + _Text.HEADER + title + _Text.END + " ---\n")
        print("         Tag: " + release["tag_name"])
        print("      Author: " + release["author"]["login"])
        print("         URL: " + _Text.UNDERLINE + release["html_url"] + _Text.END)
        print("Published at: " + published)
        print("")
    if "assets" in release:
        indent = 13 + len(_Text.BOLD + _Text.END)
        for package in release["assets"]:
            downloads_release = package["download_count"]
            downloads_total += downloads_release
            if not quiet:
                print("{1:>{0}} {2}".format(
                      indent, _Text.BOLD + str(downloads_release) + _Text.END,
                      (package["label"] if package["label"] else package["name"])))
        if not quiet:
            print("{1:>{0}} Total".format(indent, _Text.BOLD + str(downloads_total) + _Text.END))
    return downloads_total

def get_stats_downloads(stats, quiet=False):
    """
    Get number of downloads from statistics.
    :param stats: Statistics from GitHub API.
    :param quiet: If True, print nothing.
    :return: Number of downloads.
    """
    total = 0
    if isinstance(stats, dict):
        total = get_release_downloads(stats, quiet)
    else:
        for release in stats:
            total += get_release_downloads(release, quiet)
    return total

def print_total(total, quiet=False, tag=None):
    """
    Print total number of downloads.
    :param total: Total number of downloads.
    :param quiet: If True, print only number of downloads without an additional text.
    :param tag: Release tag name (optional).
    :return: Total number of downloads passed via "total" parameter.
    """
    if not quiet:
        print("\n{0}Total Downloads{1}: {2}"
              .format(_Text.BOLD,
                      "" if not tag else " (" + tag + ")",
                      _Text.SUCCESS + str(total) + _Text.END))
    elif __name__ == "__main__" or __name__ == "ghstats.ghstats":  # pragma: no cover
        print(str(total))
    return total

def main(user=None, repo=None, tag=None, latest=False, detail=False, token=None, quiet=False):
    """
    Get number of downloads for GitHub release(s).
    :param user: GitHub repository owner username. If empty, user will be prompted for input.
    :param repo: GitHub repository name. If empty, user will be prompted for input.
    :param tag: Release tag name. If empty, get stats for all releases.
    :param latest: If True, ignore "tag" parameter and get stats for the latest release.
    :param detail: Detailed output containing release information.
    :param token: GitHub OAuth token. If empty, API request limit will be reduced.
    :param quiet: If True, print nothing.
    :return: Number of downloads.
    """
    if not quiet:
        print_greeting()
    try:
        stats = download_stats(user, repo, tag, latest, token, quiet)
    except GithubError as e:
        error(e.message)
    except ConnectionError as e:
        error(str(e))
    else:
        total = get_stats_downloads(stats, quiet or not detail)
        print_total(total, quiet, tag or (stats["tag_name"] if latest else None))
        return total

def main_cli(args=None):
    """
    Parse command line arguments and pass to the main function.
    :param args: Command line arguments (without script name).
    :return: Number of downloads.
    """
    user = None              # GitHub username
    repo = None              # GitHub repository
    tag = None               # GitHub release tag
    latest = False           # Latest release
    detail = False           # Detailed output
    quiet = False            # Quiet output
    token = get_env_token()  # GitHub token
    if args is None:
        args = sys.argv[1:]
    for arg in args:
        if arg == "-q" or arg == "--quiet":
            quiet = True
        elif arg == "-d" or arg == "--detail":
            detail = True
        elif arg == "-l" or arg == "--latest":
            latest = True
        elif arg == "-h" or arg == "--help" or arg == "-?":
            print_help()
        else:
            if not user:
                if "/" not in arg:
                    user = arg
                else:
                    userrepo = arg.split("/")
                    user = userrepo[0]
                    repo = userrepo[1]
            elif not repo:
                repo = arg
            elif quiet and detail and latest:
                break
            elif not tag:
                tag = arg
    return main(user, repo, tag, latest, detail, token, quiet)

if __name__ == "__main__":  # pragma: no cover
    try:
        main_cli(sys.argv[1:])
    except KeyboardInterrupt:
        pass
<|endoftext|>


<commit_before>"""Evaluation and skill scores for ensemble forecasts."""

import numpy as np

def rankhist_init(num_ens_members, X_min):
    """Initialize a rank histogram object.

    Parameters
    ----------
    num_ens_members : int
      Number ensemble members in the forecasts to accumulate into the rank
      histogram.
    X_min : float
      Threshold for minimum intensity. Forecast-observation pairs, where all
      ensemble members and verifying observations are below X_min, are not
      counted in the rank histogram.

    Returns
    -------
    out : dict
      The rank histogram object.
    """
    rankhist = {}

    rankhist["num_ens_members"] = num_ens_members
    rankhist["n"] = np.zeros(num_ens_members+1, dtype=int)
    rankhist["X_min"] = X_min

    return rankhist

def rankhist_accum(rankhist, X_f, X_o):
    """Accumulate forecast-observation pairs to the given rank histogram.

    Parameters
    ----------
    X_f : array-like
      Array of shape (n,m) containing the values from n ensemble forecasts with
      m members.
    X_o : array_like
      Array of length n containing the observed values corresponding to the
      forecast.
    """
    if X_f.shape[1] != rankhist["num_ens_members"]:
        raise ValueError("the number of ensemble members in X_f does not match the number of members in the rank histogram (%d!=%d)" % (X_f.shape[1], rankhist["num_ens_members"]))

    X_min = rankhist["X_min"]

    mask = np.logical_and(np.isfinite(X_o), np.all(np.isfinite(X_f), axis=1))
    X_f = X_f[mask, :].copy()
    X_o = X_o[mask]

    mask_nz = np.logical_or(X_o >= X_min, np.all(X_f >= X_min, axis=1))

    X_f.sort(axis=1)
    bin_idx = [np.digitize(v, f) for v,f in zip(X_o[mask_nz], X_f[mask_nz, :])]

    # handle ties, where the verifying observation lies between ensemble members
    # having the same value
    # ignore the cases where the verifying observations and all ensemble
    # members are below the threshold X_min
    for i in np.where(~mask_nz)[0]:
        if np.any(X_f[i, :] >= X_min):
            i_eq = np.where(X_f[i, :] < X_min)[0]
            if len(i_eq) > 1 and X_o[i] < X_min:
                bin_idx.append(np.random.randint(low=np.min(i_eq)+1,
                                                 high=np.max(i_eq)+1))

    for bi in bin_idx:
        rankhist["n"][bi] += 1

def rankhist_compute(rankhist, normalize=True):
    """Return the rank histogram counts and optionally normalize the histogram.

    Parameters
    ----------
    rankhist : dict
      A rank histogram object created with rankhist_init.

    Returns
    -------
    out : array_like
      The counts for the n+1 bins in the rank histogram, where n is the number
      of ensemble members.
    """
    if normalize:
        return 1.0*rankhist["n"] / sum(rankhist["n"])
    else:
        return rankhist["n"]
<commit_msg>Fix multiple issues with rank histograms, should now be correct<commit_after>"""Evaluation and skill scores for ensemble forecasts."""

import numpy as np

def rankhist_init(num_ens_members, X_min):
    """Initialize a rank histogram object.

    Parameters
    ----------
    num_ens_members : int
      Number ensemble members in the forecasts to accumulate into the rank
      histogram.
    X_min : float
      Threshold for minimum intensity. Forecast-observation pairs, where all
      ensemble members and verifying observations are below X_min, are not
      counted in the rank histogram.

    Returns
    -------
    out : dict
      The rank histogram object.
    """
    rankhist = {}

    rankhist["num_ens_members"] = num_ens_members
    rankhist["n"] = np.zeros(num_ens_members+1, dtype=int)
    rankhist["X_min"] = X_min

    return rankhist

def rankhist_accum(rankhist, X_f, X_o):
    """Accumulate forecast-observation pairs to the given rank histogram.

    Parameters
    ----------
    X_f : array-like
      Array of shape (n,m) containing the values from n ensemble forecasts with
      m members.
    X_o : array_like
      Array of length n containing the observed values corresponding to the
      forecast.
    """
    if X_f.shape[1] != rankhist["num_ens_members"]:
        raise ValueError("the number of ensemble members in X_f does not match the number of members in the rank histogram (%d!=%d)" % (X_f.shape[1], rankhist["num_ens_members"]))

    X_min = rankhist["X_min"]

    mask = np.logical_and(np.isfinite(X_o), np.all(np.isfinite(X_f), axis=1))
    X_f = X_f[mask, :].copy()
    X_o = X_o[mask]

    mask_nz = np.logical_or(X_o >= X_min, np.all(X_f >= X_min, axis=1))

    X_f.sort(axis=1)
    bin_idx = [np.digitize([v], f)[0] for v,f in zip(X_o[mask_nz], X_f[mask_nz, :])]

    # handle ties, where the verifying observation lies between ensemble
    # members having the same value
    # ignore the cases where the verifying observations and all ensemble
    # members are below the threshold X_min
    for i in np.where(~mask_nz)[0]:
        if np.any(X_f[i, :] >= X_min):
            i_eq = np.where(X_f[i, :] < X_min)[0]
            if len(i_eq) > 1 and X_o[i] < X_min:
                bin_idx.append(np.random.randint(low=np.min(i_eq),
                                                 high=np.max(i_eq)+1))

    for bi in bin_idx:
        rankhist["n"][bi] += 1

def rankhist_compute(rankhist, normalize=True):
    """Return the rank histogram counts and optionally normalize the histogram.

    Parameters
    ----------
    rankhist : dict
      A rank histogram object created with rankhist_init.

    Returns
    -------
    out : array_like
      The counts for the n+1 bins in the rank histogram, where n is the number
      of ensemble members.
    """
    if normalize:
        return 1.0*rankhist["n"] / sum(rankhist["n"])
    else:
        return rankhist["n"]
<|endoftext|>


<commit_before>#!/usr/bin/env python

__author__ = 'nwilson'

import threading
import os
import datetime

LOG_DATE_FORMAT = "%H:%M:%S"

class LogPipe(threading.Thread):
    def __init__(self, tag, logfile):
        """
        Setup the object with a logger
        and start the thread
        """
        threading.Thread.__init__(self)
        self.daemon = False
        self.tag = " " + tag
        self.fdRead, self.fdWrite = os.pipe()
        self.pipeReader = os.fdopen(self.fdRead)
        self.fdLog = logfile
        self.start()

    def fileno(self):
        """
        Return the write file descriptor of the pipe
        """
        return self.fdWrite

    def run(self):
        """
        Run the thread, logging everything.
        """
        for line in iter(self.pipeReader.readline, ''):
            # TODO
            self.fdLog.write("[%s] %s %s\n" % (datetime.now().strftime(LOG_DATE_FORMAT), self.tag, line.strip()))

        self.pipeReader.close()

    def close(self):
        """
        Close the write end of the pipe.
        """
        # TODO
        os.close(self.fdWrite)
        self.join()
        self.pipeReader.close()

    def __enter__(self):
        return self

    def __exit__(self, exc_type, exc_val, exc_tb):
        self.close()<commit_msg>WIP logging<commit_after>#!/usr/bin/env python

__author__ = 'nwilson'

import threading
import os
import datetime

LOG_DATE_FORMAT = "%H:%M:%S"
LOGGER_ENV = "GIDGET_LOG_FILE"

class Logger:
    _loggerThisProcess = None

    def __init__(self, logFilePath):
        self.fdLog = open(logFilePath, "a")

    def _log(self, tag, msg):
        self.fdLog.write("[%s] %s %s\n" % (datetime.now().strftime(LOG_DATE_FORMAT), tag, msg.strip()))

    def close(self):
        self.fdLog.close()

    @staticmethod
    def _getOrCreate():
        if Logger._loggerThisProcess is not None:
            return Logger._loggerThisProcess

        assert os.getenv(LOGGER_ENV) is not None

        Logger._loggerThisProcess = Logger(os.getenv(LOGGER_ENV))
        return Logger._loggerThisProcess

    @staticmethod
    def log(tag, msg):
        Logger._getOrCreate()

    def __enter__(self):
        pass

    def __exit__(self, exc_type, exc_val, exc_tb):
        if not self.logFile.closed():
            self.logFile.close()

class LogPipe(threading.Thread):
    def __init__(self, tag, logger):
        """
        Setup the object with a logger
        and start the thread
        """
        threading.Thread.__init__(self)
        self.daemon = False
        self.tag = " " + tag
        self.fdRead, self.fdWrite = os.pipe()
        self.pipeReader = os.fdopen(self.fdRead)
        self.logger = logger
        self.start()

    def fileno(self):
        """
        Return the write file descriptor of the pipe
        """
        return self.fdWrite

    def run(self):
        """
        Run the thread, logging everything.
        """
        for line in iter(self.pipeReader.readline, ''):
            self.logger.log(self.tag, line)

        self.pipeReader.close()

    def close(self):
        """
        Close the write end of the pipe.
        """
        # TODO
        os.close(self.fdWrite)
        self.join()
        self.pipeReader.close()

    def __enter__(self):
        return self

    def __exit__(self, exc_type, exc_val, exc_tb):
        self.close()
<|endoftext|>