<commit_before># -*- coding: utf-8 -*-
##############################################################################
#
#    Copyright (C) 2011 Domsense srl (<http://www.domsense.com>)
#    Copyright (C) 2011-2013 Agile Business Group sagl
#    (<http://www.agilebg.com>)
#
#    This program is free software: you can redistribute it and/or modify
#    it under the terms of the GNU Affero General Public License as published
#    by the Free Software Foundation, either version 3 of the License, or
#    (at your option) any later version.
#
#    This program is distributed in the hope that it will be useful,
#    but WITHOUT ANY WARRANTY; without even the implied warranty of
#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#    GNU Affero General Public License for more details.
#
#    You should have received a copy of the GNU Affero General Public License
#    along with this program.  If not, see <http://www.gnu.org/licenses/>.
#
##############################################################################
{
    'name': "HR - Attendance Analysis",
    'version': '0.1',
    'category': 'Generic Modules/Human Resources',
    'description': """
Dynamic reports based on employee's attendances and contract's calendar.
Among other things, it lets you see the amount of working hours outside and inside the contract's working schedule (overtime).
It also provides a daily based report, showing the detailed and total hours compared to calendar hours.
Several analysis settings can be configured, like:
 - Tolerance for sign-in and sign-out
 - Attendances and overtimes roundings
 - Diffrent types of overtime, according to the overtime amount

This module requires python 2.7 or later
----------------------------------------

""",
    'author': 'Agile Business Group',
    'website': 'http://www.agilebg.com',
    'license': 'AGPL-3',
    "depends" : ['hr_attendance', 'hr_contract', 'hr_holidays', 'report_webkit'],
    "init_xml" : [],
    "update_xml" : [
        'company_view.xml',
        'hr_attendance_view.xml',
        'reports.xml',
        'wizard/print_calendar_report.xml',
        'resource_view.xml',
        'security/ir.model.access.csv',
        ],
    "demo_xml" : [],
    "active": False,
    "installable": True
}
<commit_msg>[FIX] __openerp__<commit_after># -*- coding: utf-8 -*-
##############################################################################
#
#    Copyright (C) 2011 Domsense srl (<http://www.domsense.com>)
#    Copyright (C) 2011-2013 Agile Business Group sagl
#    (<http://www.agilebg.com>)
#
#    This program is free software: you can redistribute it and/or modify
#    it under the terms of the GNU Affero General Public License as published
#    by the Free Software Foundation, either version 3 of the License, or
#    (at your option) any later version.
#
#    This program is distributed in the hope that it will be useful,
#    but WITHOUT ANY WARRANTY; without even the implied warranty of
#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#    GNU Affero General Public License for more details.
#
#    You should have received a copy of the GNU Affero General Public License
#    along with this program.  If not, see <http://www.gnu.org/licenses/>.
#
##############################################################################
{
    'name': "HR - Attendance Analysis",
    'version': '0.1',
    'category': 'Generic Modules/Human Resources',
    'description': """
Dynamic reports based on employee's attendances and contract's calendar.
Among other things, it lets you see the amount of working hours outside and inside the contract's working schedule (overtime).
It also provides a daily based report, showing the detailed and total hours compared to calendar hours.
Several analysis settings can be configured, like:
 - Tolerance for sign-in and sign-out
 - Attendances and overtimes roundings
 - Diffrent types of overtime, according to the overtime amount

This module requires python 2.7 or later
----------------------------------------

""",
    'author': 'Agile Business Group',
    'website': 'http://www.agilebg.com',
    'license': 'AGPL-3',
    "depends" : ['hr_attendance', 'hr_contract', 'hr_holidays', 'report_webkit'],
    "data" : [
        'company_view.xml',
        'hr_attendance_view.xml',
        'reports.xml',
        'wizard/print_calendar_report.xml',
        'resource_view.xml',
        'security/ir.model.access.csv',
        ],
    "demo" : [],
    "active": False,
    "installable": True
}
<|endoftext|>


<commit_before># Copyright 2016 Tecnativa <vicent.cubells@tecnativa.com>
# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl).

from odoo import api, fields, models, _

class ProjectTask(models.Model):
    _inherit = 'project.task'

    code = fields.Char(
        string='Task Number',
        required=True,
        default='/',
        readonly=True,
    )

    _sql_constraints = [
        (
            'project_task_unique_code',
            'UNIQUE (code)',
            _('The code must be unique!')
        ),
    ]

    @api.model_create_multi
    def create(self, vals_list):
        for vals in vals_list:
            if vals.get('code', '/') == '/':
                vals['code'] = self.env['ir.sequence'].next_by_code(
                    'project.task'
                )
        return super().create(vals_list)

    def copy(self, default=None):
        if default is None:
            default = {}
        default['code'] = self.env['ir.sequence'].next_by_code('project.task')
        return super().copy(default)

    def name_get(self):
        result = super().name_get()
        new_result = []

        for task in result:
            rec = self.browse(task[0])
            name = '[%s] %s' % (rec.code, task[1])
            new_result.append((rec.id, name))
        return new_result
<commit_msg>[FIX][12.0] project_task_code missing api.multi<commit_after># Copyright 2016 Tecnativa <vicent.cubells@tecnativa.com>
# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl).

from odoo import api, fields, models, _

class ProjectTask(models.Model):
    _inherit = 'project.task'

    code = fields.Char(
        string='Task Number',
        required=True,
        default='/',
        readonly=True,
    )

    _sql_constraints = [
        (
            'project_task_unique_code',
            'UNIQUE (code)',
            _('The code must be unique!')
        ),
    ]

    @api.model_create_multi
    def create(self, vals_list):
        for vals in vals_list:
            if vals.get('code', '/') == '/':
                vals['code'] = self.env['ir.sequence'].next_by_code(
                    'project.task'
                )
        return super().create(vals_list)

    @api.multi
    def copy(self, default=None):
        self.ensure_one()
        if default is None:
            default = {}
        default['code'] = self.env['ir.sequence'].next_by_code('project.task')
        return super().copy(default)

    def name_get(self):
        result = super().name_get()
        new_result = []

        for task in result:
            rec = self.browse(task[0])
            name = '[%s] %s' % (rec.code, task[1])
            new_result.append((rec.id, name))
        return new_result
<|endoftext|>


<commit_before>"""Install the toolbox into ArcGIS Desktop 10.4"""
import sys
import shutil
import os.path

def main():
    """
    Perform the following steps:

    - add a link to the python.exe that ran setup.py to user's home directory in the file cea_python.pth
    - copy the file "CityEnergyAnalyst.py" to the "My Toolboxes" folder of ArcGIS Desktop and rename the
      extension to ".pyt"
    - sets up .pth files to access arcpy from the cea python interpreter.
    """
    # write out path to python.exe to the file cea_python.pth
    with open(os.path.expanduser('~/cea_python.pth'), 'w') as f:
        f.write(sys.executable)

    toolbox_dst = find_toolbox_destination()
    toolbox_folder = os.path.dirname(toolbox_dst)
    if not os.path.exists(toolbox_folder):
        os.makedirs(toolbox_folder)
    shutil.copy(find_toolbox_src(), toolbox_dst)

    with open(os.path.expanduser('~/cea_arcpy.pth'), 'w') as f:
        f.writelines('\n'.join(get_arcgis_paths()))

def find_toolbox_src():
    """
    Find the source path of the toolbox file (CityEnergyAnalyst.py) - hint: it is relative
    to the current file!
    """
    return os.path.abspath(os.path.join(os.path.dirname(__file__), 'CityEnergyAnalyst.py'))

def find_toolbox_destination():
    """
    Find the destination path for the toolbox file (City Energy Analyst.pyt) - hint: the
    folder is similar to "%APPDATA%\ESRI\Desktop10.4\ArcToolbox\My Toolboxes"
    """
    destination = os.path.join(
        os.path.expandvars(r"$APPDATA\ESRI\Desktop%s\ArcToolbox\My Toolboxes" % get_arcgis_version()),
        'City Energy Analyst.pyt')
    return destination

def get_arcgis_paths():
    """
    Use the windows registry to figure out the paths to the following folders:

    - bin
    - arcpy
    - scripts

    as subfolders of the installation directory.
    """
    import _winreg
    registry = _winreg.ConnectRegistry(None, _winreg.HKEY_LOCAL_MACHINE)
    arcgis_version = get_arcgis_version()
    try:
        key = _winreg.OpenKey(registry, r"SOFTWARE\wow6432Node\ESRI\Desktop%s" % arcgis_version)
    except WindowsError:
        key = _winreg.OpenKey(registry, r"SOFTWARE\ESRI\Desktop%s" % arcgis_version)
    install_dir, _ = _winreg.QueryValueEx(key, 'InstallDir')
    paths = [os.path.join(install_dir, 'bin64'),
            os.path.join(install_dir, 'arcpy'),
            os.path.join(install_dir, 'scripts')]
    return paths

def get_arcgis_version():
    """Check the registry for ArcGIS and return the version. Checks the following two locations:

    - HKLM\software\wow6432Node\esri\Arcgis\RealVersion
    - HKLM\SOFTWARE\ESRI\ArcGIS\RealVersion

    returns the version string as ``"major.minor"``, so ``"10.4"`` or ``"10.5"``
    """
    import _winreg
    registry = _winreg.ConnectRegistry(None, _winreg.HKEY_LOCAL_MACHINE)
    try:
        key = _winreg.OpenKey(registry, r"software\wow6432Node\esri\Arcgis")
    except WindowsError:
        key = _winreg.OpenKey(registry, r"SOFTWARE\ESRI\ArcGIS")
    value, _ = _winreg.QueryValueEx(key, 'RealVersion')
    return '.'.join(value.split('.')[:2])

if __name__ == '__main__':
    main()
<commit_msg>cea install-toolbox<commit_after>"""Install the toolbox into ArcGIS Desktop 10.4 and 10.5"""
import sys
import shutil
import os.path

def main(_):
    """
    Perform the following steps:

    - add a link to the python.exe that ran setup.py to user's home directory in the file cea_python.pth
    - copy the file "CityEnergyAnalyst.py" to the "My Toolboxes" folder of ArcGIS Desktop and rename the
      extension to ".pyt"
    - sets up .pth files to access arcpy from the cea python interpreter.
    """
    # write out path to python.exe to the file cea_python.pth
    with open(os.path.expanduser('~/cea_python.pth'), 'w') as f:
        f.write(sys.executable)

    toolbox_dst = find_toolbox_destination()
    toolbox_folder = os.path.dirname(toolbox_dst)
    if not os.path.exists(toolbox_folder):
        os.makedirs(toolbox_folder)
    shutil.copy(find_toolbox_src(), toolbox_dst)

    with open(os.path.expanduser('~/cea_arcpy.pth'), 'w') as f:
        f.writelines('\n'.join(get_arcgis_paths()))
    print('toolbox installed.')

def find_toolbox_src():
    """
    Find the source path of the toolbox file (CityEnergyAnalyst.py) - hint: it is relative
    to the current file!
    """
    return os.path.abspath(os.path.join(os.path.dirname(__file__), 'CityEnergyAnalyst.py'))

def find_toolbox_destination():
    """
    Find the destination path for the toolbox file (City Energy Analyst.pyt) - hint: the
    folder is similar to "%APPDATA%\ESRI\Desktop10.4\ArcToolbox\My Toolboxes"
    """
    destination = os.path.join(
        os.path.expandvars(r"$APPDATA\ESRI\Desktop%s\ArcToolbox\My Toolboxes" % get_arcgis_version()),
        'City Energy Analyst.pyt')
    return destination

def get_arcgis_paths():
    """
    Use the windows registry to figure out the paths to the following folders:

    - bin
    - arcpy
    - scripts

    as subfolders of the installation directory.
    """
    import _winreg
    registry = _winreg.ConnectRegistry(None, _winreg.HKEY_LOCAL_MACHINE)
    arcgis_version = get_arcgis_version()
    try:
        key = _winreg.OpenKey(registry, r"SOFTWARE\wow6432Node\ESRI\Desktop%s" % arcgis_version)
    except WindowsError:
        key = _winreg.OpenKey(registry, r"SOFTWARE\ESRI\Desktop%s" % arcgis_version)
    install_dir, _ = _winreg.QueryValueEx(key, 'InstallDir')
    paths = [os.path.join(install_dir, 'bin64'),
            os.path.join(install_dir, 'arcpy'),
            os.path.join(install_dir, 'scripts')]
    return paths

def get_arcgis_version():
    """Check the registry for ArcGIS and return the version. Checks the following two locations:

    - HKLM\software\wow6432Node\esri\Arcgis\RealVersion
    - HKLM\SOFTWARE\ESRI\ArcGIS\RealVersion

    returns the version string as ``"major.minor"``, so ``"10.4"`` or ``"10.5"``
    """
    import _winreg
    registry = _winreg.ConnectRegistry(None, _winreg.HKEY_LOCAL_MACHINE)
    try:
        key = _winreg.OpenKey(registry, r"software\wow6432Node\esri\Arcgis")
    except WindowsError:
        key = _winreg.OpenKey(registry, r"SOFTWARE\ESRI\ArcGIS")
    value, _ = _winreg.QueryValueEx(key, 'RealVersion')
    return '.'.join(value.split('.')[:2])

if __name__ == '__main__':
    main()
<|endoftext|>


<commit_before>from djtracker import models

from django.contrib.auth.models import User
from django.contrib.comments.models import Comment
from django.core.mail import send_mail
from django.db.models.signals import post_save
from django.contrib.sites.models import Site
from django.conf import settings

def create_profile(sender, instance, created, **kwargs):
    if created:
        profile = models.UserProfile()
        profile.user = instance
        profile.save()

def update_watchers(sender, instance, created, **kwargs):
    comment = instance
    site = Site.objects.get(id=settings.SITE_ID)
    if comment.content_type.name == "issue":
        users = comment.content_object.watched_by.all()
        email_addresses = []
        for x in users:
            email_addresses.append(x.user.email)
        email_title = "%s has been updated by %s" % (comment.content_object.name,
            comment.name)
        email_message = """
            Hello,
            This message is to inform you that the issue you are watching has
            been updated. The update is as follows:

            -------------------------------
            %s
            -------------------------------

            You can view this issue as http://%s%s

            Thanks,
            DjTracker Administration
            """ % (comment.comment, site,
                comment.content_object.get_absolute_url())
        send_mail(email_title, email_message, settings.ISSUE_ADDRESS,
            email_addresses, fail_silently=False)

post_save.connect(create_profile, sender=User)
post_save.connect(update_watchers, sender=Comment)
<commit_msg>users that have an issue assigned to them will also be emailed when the issue is updated<commit_after>from djtracker import models

from django.contrib.auth.models import User
from django.contrib.comments.models import Comment
from django.core.mail import send_mail
from django.db.models.signals import post_save
from django.contrib.sites.models import Site
from django.conf import settings

def create_profile(sender, instance, created, **kwargs):
    if created:
        profile = models.UserProfile()
        profile.user = instance
        profile.save()

def update_watchers(sender, instance, created, **kwargs):
    comment = instance
    site = Site.objects.get(id=settings.SITE_ID)
    if comment.content_type.name == "issue":
        users = comment.content_object.watched_by.all()
        email_addresses = []
        for x in users:
            email_addresses.append(x.user.email)
        if comment.content_object.assigned_to.user.email not in email_addresses:
            email_address.append(comment.content_object.assigned_to.user.email)
        email_title = "%s has been updated by %s" % (comment.content_object.name,
            comment.name)
        email_message = """
            Hello,
            This message is to inform you that the issue you are watching has
            been updated. The update is as follows:

            -------------------------------
            %s
            -------------------------------

            You can view this issue as http://%s%s

            Thanks,
            DjTracker Administration
            """ % (comment.comment, site,
                comment.content_object.get_absolute_url())
        send_mail(email_title, email_message, settings.ISSUE_ADDRESS,
            email_addresses, fail_silently=False)

post_save.connect(create_profile, sender=User)
post_save.connect(update_watchers, sender=Comment)
<|endoftext|>


<commit_before>from flask import Blueprint, render_template, current_app, request, abort, make_response, redirect, url_for

import cea.inputlocator
from cea.config import MultiChoiceParameter
import cea.plots
import cea.plots.categories

import importlib
import plotly.offline
import json
import yaml
import re

blueprint = Blueprint(
    'plots_blueprint',
    __name__,
    url_prefix='/plots',
    template_folder='templates',
    static_folder='static',
)

categories = {c.name: {'label': c.label, 'plots': [{'id': p.id(), 'name': p.name} for p in c.plots]}
              for c in cea.plots.categories.list_categories()}

@blueprint.route('/index')
def index():
    # TODO: Make this the entry point for showing dashboard

    return redirect(url_for('plots_blueprint.route_dashboard', dashboard_index=0))

@blueprint.route('/dashboard/<int:dashboard_index>')
def route_dashboard(dashboard_index):
    """
    Route the i-th dashboard from the dashboard configuratino file.
    In case of an out-of-bounds error, show the 0-th dashboard (that is guaranteed to exist)
    """
    cea_config = current_app.cea_config
    plot_cache = current_app.plot_cache
    dashboards = cea.plots.read_dashboards(cea_config, plot_cache)
    dashboard = dashboards[dashboard_index]
    layout = dashboard.layout
    html = 'default.html'
    if layout == 'map':
        html = 'map.html'
    else:
        print('No such layout: %s' % layout)
    return render_template('layout/'+html, dashboard_index=dashboard_index, dashboards=dashboards,
                           dashboard=dashboard, categories=categories)

@blueprint.route('/dashboard/manage')
def route_manage_dashboards():
    cea_config = current_app.cea_config
    plot_cache = current_app.plot_cache
    dashboards = cea.plots.read_dashboards(cea_config, plot_cache)
    return render_template('manage.html', dashboards=dashboards)

@blueprint.route('/dashboard/new')
def route_new_dashboard_view():
    return render_template('modal/new_dashboard.html')

@blueprint.route('/dashboard/new/save', methods=['POST'])
def route_new_dashboard():
    """
    Append a dashboard to the list of dashboards and open it for editing.
    """
    cea_config = current_app.cea_config
    plot_cache = current_app.plot_cache
    dashboard_index = cea.plots.new_dashboard(cea_config, plot_cache, request.form.get('name'), request.form.get('description'), request.form.get('layout'))
    return redirect(url_for('plots_blueprint.route_dashboard', dashboard_index=dashboard_index))

@blueprint.route('/dashboard/<int:dashboard_index>/<func>')
def route_manage_dashboards_function(dashboard_index, func):
    cea_config = current_app.cea_config
    plot_cache = current_app.plot_cache
    dashboards = cea.plots.read_dashboards(cea_config, plot_cache)
    dashboard_name = dashboards[dashboard_index].name
    if func == 'delete':
        return render_template('modal/delete_dashboard.html', dashboard_index=dashboard_index, dashboard_name=dashboard_name)
    if func == 'rename':
        dashboard_description = dashboards[dashboard_index].description
        return render_template('modal/rename_dashboard.html', dashboard_index=dashboard_index, dashboard_name=dashboard_name,
                               dashboard_description=dashboard_description)

@blueprint.route('/dashboard/delete/<int:dashboard_index>', methods=['POST'])
def route_delete_dashboard(dashboard_index):
    cea_config = current_app.cea_config
    cea.plots.delete_dashboard(cea_config, dashboard_index)
    return redirect(url_for('plots_blueprint.route_manage_dashboards'))

@blueprint.route('/dashboard/rename/<int:dashboard_index>', methods=['POST'])
def route_rename_dashboard(dashboard_index):
    dashboards = cea.plots.read_dashboards(current_app.cea_config, current_app.plot_cache)
    dashboard = dashboards[dashboard_index]
    dashboard.name = request.form.get('new-name', dashboard.name)
    dashboard.description = request.form.get('new-description', dashboard.description)
    cea.plots.write_dashboards(current_app.cea_config, dashboards)
    return redirect(url_for('plots_blueprint.route_manage_dashboards'))

@blueprint.route('/dashboard/add-plot/<int:dashboard_index>', methods=['POST'])
def route_add_plot_to_dashboard(dashboard_index):
    dashboards = cea.plots.read_dashboards(current_app.cea_config, current_app.plot_cache)
    dashboard = dashboards[dashboard_index]
    category = request.form.get('category', next(iter(categories)))
    plot_id = request.form.get('plot-id', next(iter(categories[category]['plots']))['id'])
    dashboard.add_plot(category, plot_id)
    cea.plots.write_dashboards(current_app.cea_config, dashboards)
    return redirect(url_for('plots_blueprint.route_dashboard', dashboard_index=dashboard_index))

@blueprint.route('/dashboard/replace-plot/<int:dashboard_index>/<int:plot_index>', methods=['POST'])
def route_replace_plot(dashboard_index, plot_index):
    dashboards = cea.plots.read_dashboards(current_app.cea_config, current_app.plot_cache)
    dashboard = dashboards[dashboard_index]
    category = request.form.get('category', next(iter(categories)))
    plot_id = request.form.get('plot-id', next(iter(categories[category]['plots']))['id'])
    dashboard.replace_plot(category, plot_id, plot_index)
    cea.plots.write_dashboards(current_app.cea_config, dashboards)
    return redirect(url_for('plots_blueprint.route_dashboard', dashboard_index=dashboard_index))

@blueprint.route('/dashboard/remove-plot/<int:dashboard_index>/<int:plot_index>')
def route_remove_plot_from_dashboard(dashboard_index, plot_index):
    """Remove a plot from a dashboard by index."""
    dashboards = cea.plots.read_dashboards(current_app.cea_config, current_app.plot_cache)
    dashboard = dashboards[dashboard_index]
    dashboard.remove_plot(plot_index)
    if len(dashboard.plots) == 0:
        cea.plots.delete_dashboard(current_app.cea_config, dashboard_index)
        return redirect(url_for('plots_blueprint.route_dashboard', dashboard_index=0))
    cea.plots.write_dashboards(current_app.cea_config, dashboards)
    return redirect(url_for('plots_blueprint.route_dashboard', dashboard_index=dashboard_index))

@blueprint.route('/dashboard/move_plot_up/<int:dashboard_index>/<int:plot_index>')
def route_move_plot_up(dashboard_index, plot_index):
    """Move a plot up in the dashboard"""
    if plot_index > 0:
        dashboards = cea.plots.read_dashboards(current_app.cea_config, current_app.plot_cache)
        dashboard = dashboards[dashboard_index]
        if plot_index < len(dashboard.plots):
            swap(dashboard.plots, plot_index - 1, plot_index)
            cea.plots.write_dashboards(current_app.cea_config, dashboards)
    return redirect(url_for('plots_blueprint.route_dashboard', dashboard_index=dashboard_index))

def swap(lst, i, j):
    """Swap positions of elements in a list as given by their indexes i and j"""
    lst[i], lst[j] = lst[j], lst[i]

@blueprint.route('/dashboard/move_plot_down/<int:dashboard_index>/<int:plot_index>')
def route_move_plot_down(dashboard_index, plot_index):
    """Move a plot down in the dashboard"""
    if plot_index >= 0:
        dashboards = cea.plots.read_dashboards(current_app.cea_config, current_app.plot_cache)
        dashboard = dashboards[dashboard_index]
        if plot_index < (len(dashboard.plots) - 1):
            swap(dashboard.plots, plot_index, plot_index + 1)
            cea.plots.write_dashboards(current_app.cea_config, dashboards)
    return redirect(url_for('plots_blueprint.route_dashboard', dashboard_index=dashboard_index))

@blueprint.route('/dashboard/plot-parameters/<int:dashboard_index>/<int:plot_index>', methods=['GET'])
def route_get_plot_parameters(dashboard_index, plot_index):
    dashboards = cea.plots.read_dashboards(current_app.cea_config, current_app.plot_cache)
    dashboard = dashboards[dashboard_index]
    plot = dashboard.plots[plot_index]
    parameters = []
    for pname, fqname in plot.expected_parameters.items():
        parameter = current_app.cea_config.get_parameter(fqname)
        if pname in plot.parameters:
            parameter.set(plot.parameters[pname])
        parameters.append(parameter)
    return render_template('parameters.html', parameters=parameters, weather_dict={})

@blueprint.route('/dashboard/plot-parameters/<int:dashboard_index>/<int:plot_index>', methods=['POST'])
def route_post_plot_parameters(dashboard_index, plot_index):
    dashboards = cea.plots.read_dashboards(current_app.cea_config, current_app.plot_cache)
    dashboard = dashboards[dashboard_index]
    plot = dashboard.plots[plot_index]
    print('route_post_plot_parameters: expected_parameters: {}'.format(plot.expected_parameters.items()))
    for pname, fqname in plot.expected_parameters.items():
        parameter = current_app.cea_config.get_parameter(fqname)
        print('route_post_plot_parameters: fqname={fqname}, pname={pname}'.format(fqname=fqname, pname=pname))
        if isinstance(parameter, MultiChoiceParameter):
            plot.parameters[pname] = parameter.decode(','.join(request.form.getlist(pname)))
        else:
            plot.parameters[pname] = parameter.decode(request.form[pname])
    cea.plots.write_dashboards(current_app.cea_config, dashboards)
    return redirect(url_for('plots_blueprint.route_dashboard', dashboard_index=dashboard_index))

@blueprint.route('/category/<category>')
def route_category(category):
    """FIXME: this will be removed soon..."""
    if not cea.plots.categories.is_valid_category(category):
        return abort(404)

    cea_config = current_app.cea_config
    locator = cea.inputlocator.InputLocator(scenario=cea_config.scenario)
    buildings = cea_config.plots.buildings

    category = cea.plots.categories.load_category(category)
    plots = [plot_class(cea_config, locator, parameters={'buildings': buildings}) for plot_class in category.plots]
    return render_template('category.html', category=category, plots=plots)

@blueprint.route('/div/<int:dashboard_index>/<int:plot_index>')
def route_div(dashboard_index, plot_index):
    """Return the plot as a div to be used in an AJAX call"""
    try:
        plot = load_plot(dashboard_index, plot_index)
    except Exception as ex:
        return abort(500, ex)
    if not plot.missing_input_files():
        plot_div = plot.plot_div()
        # BUGFIX for (#2102 - Can't add the same plot twice in a dashboard)
        # update id of div to include dashboard_index and plot_index
        if plot_div.startswith("<div id="):
            div_id = re.match('<div id="([0-9a-f-]+)"', plot_div).group(1)
            plot_div = plot_div.replace(div_id, "{div_id}-{dashboard_index}-{plot_index}".format(
                div_id=div_id, dashboard_index=dashboard_index, plot_index=plot_index))
        return make_response(plot_div, 200)
    else:
        return render_template('missing_input_files.html',
                               missing_input_files=[lm(*args) for lm, args in plot.missing_input_files()],
                               script_suggestions=script_suggestions(lm.__name__ for lm, _ in plot.missing_input_files()))

@blueprint.route('/table/<int:dashboard_index>/<int:plot_index>')
def route_table(dashboard_index, plot_index):
    """Return the table for the plot as a div to be used in an AJAX call"""
    try:
        plot = load_plot(dashboard_index, plot_index)
    except Exception as ex:
        return abort(500, ex)
    if not plot.missing_input_files():
        return make_response(plot.table_div(), 200)

def script_suggestions(locator_names):
    """Return a list of CeaScript objects that produce the output for each locator name"""
    import cea.scripts
    schemas = cea.scripts.schemas()
    script_names = []
    for name in locator_names:
        script_names.extend(schemas[name]['created_by'])
    return [cea.scripts.by_name(n) for n in sorted(set(script_names))]

def load_plot(dashboard_index, plot_index):
    """Load a plot from the dashboard_yml"""
    cea_config = current_app.cea_config
    dashboards = cea.plots.read_dashboards(cea_config, current_app.plot_cache)
    dashboard_index = dashboards[dashboard_index]
    plot = dashboard_index.plots[plot_index]
    return plot

@blueprint.route('/plot/<int:dashboard_index>/<int:plot_index>')
def route_plot(dashboard_index, plot_index):
    try:
        plot = load_plot(dashboard_index, plot_index)
    except Exception as ex:
        return abort(500, ex)

    return render_template('plot.html', dashboard_index=dashboard_index, plot_index=plot_index, plot=plot)<commit_msg>Allow dashboards with 0 plots<commit_after>from flask import Blueprint, render_template, current_app, request, abort, make_response, redirect, url_for

import cea.inputlocator
from cea.config import MultiChoiceParameter
import cea.plots
import cea.plots.categories

import importlib
import plotly.offline
import json
import yaml
import re

blueprint = Blueprint(
    'plots_blueprint',
    __name__,
    url_prefix='/plots',
    template_folder='templates',
    static_folder='static',
)

categories = {c.name: {'label': c.label, 'plots': [{'id': p.id(), 'name': p.name} for p in c.plots]}
              for c in cea.plots.categories.list_categories()}

@blueprint.route('/index')
def index():
    # TODO: Make this the entry point for showing dashboard

    return redirect(url_for('plots_blueprint.route_dashboard', dashboard_index=0))

@blueprint.route('/dashboard/<int:dashboard_index>')
def route_dashboard(dashboard_index):
    """
    Route the i-th dashboard from the dashboard configuratino file.
    In case of an out-of-bounds error, show the 0-th dashboard (that is guaranteed to exist)
    """
    cea_config = current_app.cea_config
    plot_cache = current_app.plot_cache
    dashboards = cea.plots.read_dashboards(cea_config, plot_cache)
    dashboard = dashboards[dashboard_index]
    layout = dashboard.layout
    html = 'default.html'
    if layout == 'map':
        html = 'map.html'
    else:
        print('No such layout: %s' % layout)
    return render_template('layout/'+html, dashboard_index=dashboard_index, dashboards=dashboards,
                           dashboard=dashboard, categories=categories)

@blueprint.route('/dashboard/manage')
def route_manage_dashboards():
    cea_config = current_app.cea_config
    plot_cache = current_app.plot_cache
    dashboards = cea.plots.read_dashboards(cea_config, plot_cache)
    return render_template('manage.html', dashboards=dashboards)

@blueprint.route('/dashboard/new')
def route_new_dashboard_view():
    return render_template('modal/new_dashboard.html')

@blueprint.route('/dashboard/new/save', methods=['POST'])
def route_new_dashboard():
    """
    Append a dashboard to the list of dashboards and open it for editing.
    """
    cea_config = current_app.cea_config
    plot_cache = current_app.plot_cache
    dashboard_index = cea.plots.new_dashboard(cea_config, plot_cache, request.form.get('name'), request.form.get('description'), request.form.get('layout'))
    return redirect(url_for('plots_blueprint.route_dashboard', dashboard_index=dashboard_index))

@blueprint.route('/dashboard/<int:dashboard_index>/<func>')
def route_manage_dashboards_function(dashboard_index, func):
    cea_config = current_app.cea_config
    plot_cache = current_app.plot_cache
    dashboards = cea.plots.read_dashboards(cea_config, plot_cache)
    dashboard_name = dashboards[dashboard_index].name
    if func == 'delete':
        return render_template('modal/delete_dashboard.html', dashboard_index=dashboard_index, dashboard_name=dashboard_name)
    if func == 'rename':
        dashboard_description = dashboards[dashboard_index].description
        return render_template('modal/rename_dashboard.html', dashboard_index=dashboard_index, dashboard_name=dashboard_name,
                               dashboard_description=dashboard_description)

@blueprint.route('/dashboard/delete/<int:dashboard_index>', methods=['POST'])
def route_delete_dashboard(dashboard_index):
    cea_config = current_app.cea_config
    cea.plots.delete_dashboard(cea_config, dashboard_index)
    return redirect(url_for('plots_blueprint.route_manage_dashboards'))

@blueprint.route('/dashboard/rename/<int:dashboard_index>', methods=['POST'])
def route_rename_dashboard(dashboard_index):
    dashboards = cea.plots.read_dashboards(current_app.cea_config, current_app.plot_cache)
    dashboard = dashboards[dashboard_index]
    dashboard.name = request.form.get('new-name', dashboard.name)
    dashboard.description = request.form.get('new-description', dashboard.description)
    cea.plots.write_dashboards(current_app.cea_config, dashboards)
    return redirect(url_for('plots_blueprint.route_manage_dashboards'))

@blueprint.route('/dashboard/add-plot/<int:dashboard_index>', methods=['POST'])
def route_add_plot_to_dashboard(dashboard_index):
    dashboards = cea.plots.read_dashboards(current_app.cea_config, current_app.plot_cache)
    dashboard = dashboards[dashboard_index]
    category = request.form.get('category', next(iter(categories)))
    plot_id = request.form.get('plot-id', next(iter(categories[category]['plots']))['id'])
    dashboard.add_plot(category, plot_id)
    cea.plots.write_dashboards(current_app.cea_config, dashboards)
    return redirect(url_for('plots_blueprint.route_dashboard', dashboard_index=dashboard_index))

@blueprint.route('/dashboard/replace-plot/<int:dashboard_index>/<int:plot_index>', methods=['POST'])
def route_replace_plot(dashboard_index, plot_index):
    dashboards = cea.plots.read_dashboards(current_app.cea_config, current_app.plot_cache)
    dashboard = dashboards[dashboard_index]
    category = request.form.get('category', next(iter(categories)))
    plot_id = request.form.get('plot-id', next(iter(categories[category]['plots']))['id'])
    dashboard.replace_plot(category, plot_id, plot_index)
    cea.plots.write_dashboards(current_app.cea_config, dashboards)
    return redirect(url_for('plots_blueprint.route_dashboard', dashboard_index=dashboard_index))

@blueprint.route('/dashboard/remove-plot/<int:dashboard_index>/<int:plot_index>')
def route_remove_plot_from_dashboard(dashboard_index, plot_index):
    """Remove a plot from a dashboard by index."""
    dashboards = cea.plots.read_dashboards(current_app.cea_config, current_app.plot_cache)
    dashboard = dashboards[dashboard_index]
    dashboard.remove_plot(plot_index)
    # if len(dashboard.plots) == 0:
    #     cea.plots.delete_dashboard(current_app.cea_config, dashboard_index)
    #     return redirect(url_for('plots_blueprint.route_dashboard', dashboard_index=0))
    cea.plots.write_dashboards(current_app.cea_config, dashboards)
    return redirect(url_for('plots_blueprint.route_dashboard', dashboard_index=dashboard_index))

@blueprint.route('/dashboard/move_plot_up/<int:dashboard_index>/<int:plot_index>')
def route_move_plot_up(dashboard_index, plot_index):
    """Move a plot up in the dashboard"""
    if plot_index > 0:
        dashboards = cea.plots.read_dashboards(current_app.cea_config, current_app.plot_cache)
        dashboard = dashboards[dashboard_index]
        if plot_index < len(dashboard.plots):
            swap(dashboard.plots, plot_index - 1, plot_index)
            cea.plots.write_dashboards(current_app.cea_config, dashboards)
    return redirect(url_for('plots_blueprint.route_dashboard', dashboard_index=dashboard_index))

def swap(lst, i, j):
    """Swap positions of elements in a list as given by their indexes i and j"""
    lst[i], lst[j] = lst[j], lst[i]

@blueprint.route('/dashboard/move_plot_down/<int:dashboard_index>/<int:plot_index>')
def route_move_plot_down(dashboard_index, plot_index):
    """Move a plot down in the dashboard"""
    if plot_index >= 0:
        dashboards = cea.plots.read_dashboards(current_app.cea_config, current_app.plot_cache)
        dashboard = dashboards[dashboard_index]
        if plot_index < (len(dashboard.plots) - 1):
            swap(dashboard.plots, plot_index, plot_index + 1)
            cea.plots.write_dashboards(current_app.cea_config, dashboards)
    return redirect(url_for('plots_blueprint.route_dashboard', dashboard_index=dashboard_index))

@blueprint.route('/dashboard/plot-parameters/<int:dashboard_index>/<int:plot_index>', methods=['GET'])
def route_get_plot_parameters(dashboard_index, plot_index):
    dashboards = cea.plots.read_dashboards(current_app.cea_config, current_app.plot_cache)
    dashboard = dashboards[dashboard_index]
    plot = dashboard.plots[plot_index]
    parameters = []
    for pname, fqname in plot.expected_parameters.items():
        parameter = current_app.cea_config.get_parameter(fqname)
        if pname in plot.parameters:
            parameter.set(plot.parameters[pname])
        parameters.append(parameter)
    return render_template('parameters.html', parameters=parameters, weather_dict={})

@blueprint.route('/dashboard/plot-parameters/<int:dashboard_index>/<int:plot_index>', methods=['POST'])
def route_post_plot_parameters(dashboard_index, plot_index):
    dashboards = cea.plots.read_dashboards(current_app.cea_config, current_app.plot_cache)
    dashboard = dashboards[dashboard_index]
    plot = dashboard.plots[plot_index]
    print('route_post_plot_parameters: expected_parameters: {}'.format(plot.expected_parameters.items()))
    for pname, fqname in plot.expected_parameters.items():
        parameter = current_app.cea_config.get_parameter(fqname)
        print('route_post_plot_parameters: fqname={fqname}, pname={pname}'.format(fqname=fqname, pname=pname))
        if isinstance(parameter, MultiChoiceParameter):
            plot.parameters[pname] = parameter.decode(','.join(request.form.getlist(pname)))
        else:
            plot.parameters[pname] = parameter.decode(request.form[pname])
    cea.plots.write_dashboards(current_app.cea_config, dashboards)
    return redirect(url_for('plots_blueprint.route_dashboard', dashboard_index=dashboard_index))

@blueprint.route('/category/<category>')
def route_category(category):
    """FIXME: this will be removed soon..."""
    if not cea.plots.categories.is_valid_category(category):
        return abort(404)

    cea_config = current_app.cea_config
    locator = cea.inputlocator.InputLocator(scenario=cea_config.scenario)
    buildings = cea_config.plots.buildings

    category = cea.plots.categories.load_category(category)
    plots = [plot_class(cea_config, locator, parameters={'buildings': buildings}) for plot_class in category.plots]
    return render_template('category.html', category=category, plots=plots)

@blueprint.route('/div/<int:dashboard_index>/<int:plot_index>')
def route_div(dashboard_index, plot_index):
    """Return the plot as a div to be used in an AJAX call"""
    try:
        plot = load_plot(dashboard_index, plot_index)
    except Exception as ex:
        return abort(500, ex)
    if not plot.missing_input_files():
        plot_div = plot.plot_div()
        # BUGFIX for (#2102 - Can't add the same plot twice in a dashboard)
        # update id of div to include dashboard_index and plot_index
        if plot_div.startswith("<div id="):
            div_id = re.match('<div id="([0-9a-f-]+)"', plot_div).group(1)
            plot_div = plot_div.replace(div_id, "{div_id}-{dashboard_index}-{plot_index}".format(
                div_id=div_id, dashboard_index=dashboard_index, plot_index=plot_index))
        return make_response(plot_div, 200)
    else:
        return render_template('missing_input_files.html',
                               missing_input_files=[lm(*args) for lm, args in plot.missing_input_files()],
                               script_suggestions=script_suggestions(lm.__name__ for lm, _ in plot.missing_input_files()))

@blueprint.route('/table/<int:dashboard_index>/<int:plot_index>')
def route_table(dashboard_index, plot_index):
    """Return the table for the plot as a div to be used in an AJAX call"""
    try:
        plot = load_plot(dashboard_index, plot_index)
    except Exception as ex:
        return abort(500, ex)
    if not plot.missing_input_files():
        return make_response(plot.table_div(), 200)

def script_suggestions(locator_names):
    """Return a list of CeaScript objects that produce the output for each locator name"""
    import cea.scripts
    schemas = cea.scripts.schemas()
    script_names = []
    for name in locator_names:
        script_names.extend(schemas[name]['created_by'])
    return [cea.scripts.by_name(n) for n in sorted(set(script_names))]

def load_plot(dashboard_index, plot_index):
    """Load a plot from the dashboard_yml"""
    cea_config = current_app.cea_config
    dashboards = cea.plots.read_dashboards(cea_config, current_app.plot_cache)
    dashboard_index = dashboards[dashboard_index]
    plot = dashboard_index.plots[plot_index]
    return plot

@blueprint.route('/plot/<int:dashboard_index>/<int:plot_index>')
def route_plot(dashboard_index, plot_index):
    try:
        plot = load_plot(dashboard_index, plot_index)
    except Exception as ex:
        return abort(500, ex)

    return render_template('plot.html', dashboard_index=dashboard_index, plot_index=plot_index, plot=plot)<|endoftext|>


<commit_before>#!/usr/bin/env python

'''
Create the database structure for the monitoring program and the scripts
that operate on the data.  Populate it with some random rows for development
purposes.

This file essentially contains the current (as of writing) configuration
of sensors (and groups) on campus.
'''

import sys
sys.path.append('/var/local/energy/lib')
import psycopg2
from psycopg2.extensions import ISOLATION_LEVEL_AUTOCOMMIT
from random import random, randint
from energyconfig2 import *

NUM_INSERTS = 10**3 # per sensor
INSERT_INTERVAL = 10 # seconds

def main():
    '''
    Build the database structure and populate with random rows.
    '''
    conn = psycopg2.connect(PSQL_CONNSTR)
    conn.set_isolation_level(ISOLATION_LEVEL_AUTOCOMMIT)
    cur = conn.cursor()

    cur.execute('''CREATE TABLE sensor_groups
                     (id serial primary key,
                     name varchar(32) not null unique,
                     color char(6));''')
    cur.execute('''CREATE TABLE sensors
                     (id serial primary key,
                     name varchar(32) not null,
                     ip inet,
                     sensor_group_id integer REFERENCES sensor_groups (id),
                     three_phase boolean,
                     factor double precision,
                     UNIQUE(name, sensor_group_id));''')

    # (The colors are for graphing purposes.)
    cur.execute('''INSERT INTO sensor_groups (name, color)
                   VALUES ('Atwood', '00bd39');''')
    cur.execute('''INSERT INTO sensor_groups (name, color)
                   VALUES ('Case', '0776a0');''')
    cur.execute('''INSERT INTO sensor_groups (name, color)
                   VALUES ('East', '6600cc');''')
    cur.execute('''INSERT INTO sensor_groups (name, color)
                   VALUES ('Linde', 'd100a0');''')
    cur.execute('''INSERT INTO sensor_groups (name, color)
                   VALUES ('North', 'ff0000');''')
    cur.execute('''INSERT INTO sensor_groups (name, color)
                   VALUES ('Sontag', 'ffa000');''')
    cur.execute('''INSERT INTO sensor_groups (name, color)
                   VALUES ('South', 'a63f00');''')
    cur.execute('''INSERT INTO sensor_groups (name, color)
                   VALUES ('West', '000000');''')

    # Determine IDs of sensor groups for later use
    cur.execute('''SELECT id, name FROM sensor_groups;''')
    sensor_groups = {}
    for r in cur:
        sensor_groups[r[1]] = r[0]

    # Insert sensor information---the important part.
    cur.execute('''INSERT INTO sensors (name, ip, sensor_group_id,
                     three_phase, factor)
                   VALUES ('', '172.31.10.11', %s,
                     'true', 8);''',
                (sensor_groups['Atwood'],))
    cur.execute('''INSERT INTO sensors (name, ip, sensor_group_id,
                     three_phase, factor)
                   VALUES ('', '172.31.10.31', %s,
                     'true', 24);''',
                (sensor_groups['Case'],))
    cur.execute('''INSERT INTO sensors (name, ip, sensor_group_id,
                     three_phase, factor)
                   VALUES ('1', '172.31.10.71', %s,
                     'true', 4.2 / 10.0);''',
                (sensor_groups['East'],))
    cur.execute('''INSERT INTO sensors (name, ip, sensor_group_id,
                     three_phase, factor)
                   VALUES ('2', '172.31.10.72', %s,
                     'false', 8);''',
                (sensor_groups['East'],))
    cur.execute('''INSERT INTO sensors (name, ip, sensor_group_id,
                     three_phase, factor)
                   VALUES ('', '172.31.10.21', %s,
                     'true', 16);''',
                (sensor_groups['Linde'],))
    cur.execute('''INSERT INTO sensors (name, ip, sensor_group_id,
                     three_phase, factor)
                   VALUES ('1', '172.31.10.61', %s,
                     'true', 4.2 / 10.0);''',
                (sensor_groups['North'],))
    cur.execute('''INSERT INTO sensors (name, ip, sensor_group_id,
                     three_phase, factor)
                   VALUES ('2', '172.31.10.62', %s,
                     'false', 8);''',
                (sensor_groups['North'],))
    cur.execute('''INSERT INTO sensors (name, ip, sensor_group_id,
                     three_phase, factor)
                   VALUES ('HVAC', '172.31.10.63', %s,
                     'true', 4);''',
                (sensor_groups['North'],))
    cur.execute('''INSERT INTO sensors (name, ip, sensor_group_id,
                     three_phase, factor)
                   VALUES ('', '172.31.10.41', %s,
                     'true', 32);''',
                (sensor_groups['Sontag'],))
    cur.execute('''INSERT INTO sensors (name, ip, sensor_group_id,
                     three_phase, factor)
                   VALUES ('', '172.31.10.51', %s,
                     'true', 12);''',
                (sensor_groups['South'],))
    cur.execute('''INSERT INTO sensors (name, ip, sensor_group_id,
                     three_phase, factor)
                   VALUES ('1', '172.31.10.81', %s,
                     'true', 4.2 / 10.0);''',
                (sensor_groups['West'],))
    cur.execute('''INSERT INTO sensors (name, ip, sensor_group_id,
                     three_phase, factor)
                   VALUES ('2', '172.31.10.82', %s,
                     'false', 8);''',
                (sensor_groups['West'],))
    cur.execute('''INSERT INTO sensors (name, ip, sensor_group_id,
                     three_phase, factor)
                   VALUES ('HVAC', '172.31.10.73', %s,
                     'true', 6);''',
                (sensor_groups['West'],))

    # For a note about three phase sensors, see the monitoring program
    cur.execute('''SELECT id, name, three_phase FROM sensors;''')
    sensors = {}
    phase_factor = {}
    for r in cur:
        sensors[r[0]] = r[1]
        phase_factor[r[0]] = (r[2] and 1 or 0)

    # This is the central table where data is stored directly from the sensors
    cur.execute('''CREATE TABLE sensor_readings
                     (rdngtime timestamp not null,
                     sensor_id integer REFERENCES sensors (id),
                     rindex smallint,
                     awatthr integer,
                     bwatthr integer,
                     cwatthr integer,
                     avarhr integer,
                     bvarhr integer,
                     cvarhr integer,
                     avahr integer,
                     bvahr integer,
                     cvahr integer,
                     airms integer,
                     birms integer,
                     cirms integer,
                     avrms integer,
                     bvrms integer,
                     cvrms integer,
                     freq integer,
                     tempc integer);''')
    cur.execute('''CREATE INDEX sensor_readings_index
                   ON sensor_readings (rdngtime);''')
    for sensor_id in sensors.keys():
        # Populate the database so we can have something to develop with
        for i in range(NUM_INSERTS):
            v = [int(10000 + 2000 * random()**2) for j in range(18)]
            v[3] *= phase_factor[sensor_id]
            cur.execute('''INSERT INTO sensor_readings (rdngtime, sensor_id,
                             rindex, awatthr, bwatthr, cwatthr, avarhr, bvarhr,
                             cvarhr, avahr, bvahr, cvahr, airms, birms, cirms,
                             avrms, bvrms, cvrms, freq, tempc)
                           VALUES (now() - interval '%d seconds', %%s, %%s,
                             %%s, %%s, %%s, %%s, %%s, %%s, %%s, %%s, %%s, %%s,
                             %%s, %%s, %%s, %%s, %%s, %%s, %%s);'''
                        % (INSERT_INTERVAL*i + randint(-1, 1)),
                        [sensor_id] + v)

    # This table will be for averages (per week, and per month) of power
    # readings.  Postgres doesn't seem to like to compute these for each
    # web request, whereas the 10 seconds in between messages from a
    # sensor offer plenty of thinking time.
    cur.execute('''CREATE TABLE power_averages
                     (rdngtime_first timestamp not null,
                     rdngtime_last timestamp not null,
                     sensor_id integer REFERENCES sensors (id),
                     avg_type integer not null,
                     num_pts integer,
                     watts double precision);''')

    # Take the random data from sensor_readings and insert the appropriate
    # averages into power_averages.  (These figures would normally be
    # computed and inserted immediately after an insert to sensor_readings.)
    for sensor_id in sensors.keys():
        cur.execute('''INSERT INTO power_averages (rdngtime_first,
                         rdngtime_last, sensor_id, avg_type, num_pts, watts)
                       SELECT MIN(rdngtime), MAX(rdngtime), %s, %s, COUNT(*),
                         AVG(awatthr + bwatthr + cwatthr)
                         FROM sensor_readings
                         WHERE sensor_id = %s
                         GROUP BY date_trunc('month', rdngtime);''',
                    (sensor_id, AVG_TYPE_MONTH, sensor_id))
        cur.execute('''INSERT INTO power_averages (rdngtime_first,
                         rdngtime_last, sensor_id, avg_type, num_pts, watts)
                       SELECT MIN(rdngtime), MAX(rdngtime), %s, %s, COUNT(*),
                         AVG(awatthr + bwatthr + cwatthr)
                         FROM sensor_readings
                         WHERE sensor_id = %s
                         GROUP BY date_trunc('week', rdngtime);''',
                    (sensor_id, AVG_TYPE_WEEK, sensor_id))

    cur.execute('''CREATE INDEX power_averages_first_index
                   ON power_averages (rdngtime_first);''')
    cur.execute('''CREATE INDEX power_averages_last_index
                   ON power_averages (rdngtime_last);''')

    cur.execute('''CREATE INDEX sensor_readings_rdngtime_per_index
                   ON sensor_readings ((date_trunc('minute'::text, rdngtime)
                     + (FLOOR((date_part('second'::text, rdngtime)
                     / 10::double precision)) * '00:00:10'::interval)));''')

    cur.execute('''ANALYZE;''')

if __name__ == '__main__':
    main()
<commit_msg>Fixed bug in dev (no energyconfig2 for a while now!)<commit_after>#!/usr/bin/env python

'''
Create the database structure for the monitoring program and the scripts
that operate on the data.  Populate it with some random rows for development
purposes.

This file essentially contains the current (as of writing) configuration
of sensors (and groups) on campus.
'''

import sys
sys.path.append('/var/local/energy/lib')
import psycopg2
from psycopg2.extensions import ISOLATION_LEVEL_AUTOCOMMIT
from random import random, randint
from energyconfig import *

NUM_INSERTS = 10**3 # per sensor
INSERT_INTERVAL = 10 # seconds

def main():
    '''
    Build the database structure and populate with random rows.
    '''
    conn = psycopg2.connect(PSQL_CONNSTR)
    conn.set_isolation_level(ISOLATION_LEVEL_AUTOCOMMIT)
    cur = conn.cursor()

    cur.execute('''CREATE TABLE sensor_groups
                     (id serial primary key,
                     name varchar(32) not null unique,
                     color char(6));''')
    cur.execute('''CREATE TABLE sensors
                     (id serial primary key,
                     name varchar(32) not null,
                     ip inet,
                     sensor_group_id integer REFERENCES sensor_groups (id),
                     three_phase boolean,
                     factor double precision,
                     UNIQUE(name, sensor_group_id));''')

    # (The colors are for graphing purposes.)
    cur.execute('''INSERT INTO sensor_groups (name, color)
                   VALUES ('Atwood', '00bd39');''')
    cur.execute('''INSERT INTO sensor_groups (name, color)
                   VALUES ('Case', '0776a0');''')
    cur.execute('''INSERT INTO sensor_groups (name, color)
                   VALUES ('East', '6600cc');''')
    cur.execute('''INSERT INTO sensor_groups (name, color)
                   VALUES ('Linde', 'd100a0');''')
    cur.execute('''INSERT INTO sensor_groups (name, color)
                   VALUES ('North', 'ff0000');''')
    cur.execute('''INSERT INTO sensor_groups (name, color)
                   VALUES ('Sontag', 'ffa000');''')
    cur.execute('''INSERT INTO sensor_groups (name, color)
                   VALUES ('South', 'a63f00');''')
    cur.execute('''INSERT INTO sensor_groups (name, color)
                   VALUES ('West', '000000');''')

    # Determine IDs of sensor groups for later use
    cur.execute('''SELECT id, name FROM sensor_groups;''')
    sensor_groups = {}
    for r in cur:
        sensor_groups[r[1]] = r[0]

    # Insert sensor information---the important part.
    cur.execute('''INSERT INTO sensors (name, ip, sensor_group_id,
                     three_phase, factor)
                   VALUES ('', '172.31.10.11', %s,
                     'true', 8);''',
                (sensor_groups['Atwood'],))
    cur.execute('''INSERT INTO sensors (name, ip, sensor_group_id,
                     three_phase, factor)
                   VALUES ('', '172.31.10.31', %s,
                     'true', 24);''',
                (sensor_groups['Case'],))
    cur.execute('''INSERT INTO sensors (name, ip, sensor_group_id,
                     three_phase, factor)
                   VALUES ('1', '172.31.10.71', %s,
                     'true', 4.2 / 10.0);''',
                (sensor_groups['East'],))
    cur.execute('''INSERT INTO sensors (name, ip, sensor_group_id,
                     three_phase, factor)
                   VALUES ('2', '172.31.10.72', %s,
                     'false', 8);''',
                (sensor_groups['East'],))
    cur.execute('''INSERT INTO sensors (name, ip, sensor_group_id,
                     three_phase, factor)
                   VALUES ('', '172.31.10.21', %s,
                     'true', 16);''',
                (sensor_groups['Linde'],))
    cur.execute('''INSERT INTO sensors (name, ip, sensor_group_id,
                     three_phase, factor)
                   VALUES ('1', '172.31.10.61', %s,
                     'true', 4.2 / 10.0);''',
                (sensor_groups['North'],))
    cur.execute('''INSERT INTO sensors (name, ip, sensor_group_id,
                     three_phase, factor)
                   VALUES ('2', '172.31.10.62', %s,
                     'false', 8);''',
                (sensor_groups['North'],))
    cur.execute('''INSERT INTO sensors (name, ip, sensor_group_id,
                     three_phase, factor)
                   VALUES ('HVAC', '172.31.10.63', %s,
                     'true', 4);''',
                (sensor_groups['North'],))
    cur.execute('''INSERT INTO sensors (name, ip, sensor_group_id,
                     three_phase, factor)
                   VALUES ('', '172.31.10.41', %s,
                     'true', 32);''',
                (sensor_groups['Sontag'],))
    cur.execute('''INSERT INTO sensors (name, ip, sensor_group_id,
                     three_phase, factor)
                   VALUES ('', '172.31.10.51', %s,
                     'true', 12);''',
                (sensor_groups['South'],))
    cur.execute('''INSERT INTO sensors (name, ip, sensor_group_id,
                     three_phase, factor)
                   VALUES ('1', '172.31.10.81', %s,
                     'true', 4.2 / 10.0);''',
                (sensor_groups['West'],))
    cur.execute('''INSERT INTO sensors (name, ip, sensor_group_id,
                     three_phase, factor)
                   VALUES ('2', '172.31.10.82', %s,
                     'false', 8);''',
                (sensor_groups['West'],))
    cur.execute('''INSERT INTO sensors (name, ip, sensor_group_id,
                     three_phase, factor)
                   VALUES ('HVAC', '172.31.10.73', %s,
                     'true', 6);''',
                (sensor_groups['West'],))

    # For a note about three phase sensors, see the monitoring program
    cur.execute('''SELECT id, name, three_phase FROM sensors;''')
    sensors = {}
    phase_factor = {}
    for r in cur:
        sensors[r[0]] = r[1]
        phase_factor[r[0]] = (r[2] and 1 or 0)

    # This is the central table where data is stored directly from the sensors
    cur.execute('''CREATE TABLE sensor_readings
                     (rdngtime timestamp not null,
                     sensor_id integer REFERENCES sensors (id),
                     rindex smallint,
                     awatthr integer,
                     bwatthr integer,
                     cwatthr integer,
                     avarhr integer,
                     bvarhr integer,
                     cvarhr integer,
                     avahr integer,
                     bvahr integer,
                     cvahr integer,
                     airms integer,
                     birms integer,
                     cirms integer,
                     avrms integer,
                     bvrms integer,
                     cvrms integer,
                     freq integer,
                     tempc integer);''')
    cur.execute('''CREATE INDEX sensor_readings_index
                   ON sensor_readings (rdngtime);''')
    for sensor_id in sensors.keys():
        # Populate the database so we can have something to develop with
        for i in range(NUM_INSERTS):
            v = [int(10000 + 2000 * random()**2) for j in range(18)]
            v[3] *= phase_factor[sensor_id]
            cur.execute('''INSERT INTO sensor_readings (rdngtime, sensor_id,
                             rindex, awatthr, bwatthr, cwatthr, avarhr, bvarhr,
                             cvarhr, avahr, bvahr, cvahr, airms, birms, cirms,
                             avrms, bvrms, cvrms, freq, tempc)
                           VALUES (now() - interval '%d seconds', %%s, %%s,
                             %%s, %%s, %%s, %%s, %%s, %%s, %%s, %%s, %%s, %%s,
                             %%s, %%s, %%s, %%s, %%s, %%s, %%s);'''
                        % (INSERT_INTERVAL*i + randint(-1, 1)),
                        [sensor_id] + v)

    # This table will be for averages (per week, and per month) of power
    # readings.  Postgres doesn't seem to like to compute these for each
    # web request, whereas the 10 seconds in between messages from a
    # sensor offer plenty of thinking time.
    cur.execute('''CREATE TABLE power_averages
                     (rdngtime_first timestamp not null,
                     rdngtime_last timestamp not null,
                     sensor_id integer REFERENCES sensors (id),
                     avg_type integer not null,
                     num_pts integer,
                     watts double precision);''')

    # Take the random data from sensor_readings and insert the appropriate
    # averages into power_averages.  (These figures would normally be
    # computed and inserted immediately after an insert to sensor_readings.)
    for sensor_id in sensors.keys():
        cur.execute('''INSERT INTO power_averages (rdngtime_first,
                         rdngtime_last, sensor_id, avg_type, num_pts, watts)
                       SELECT MIN(rdngtime), MAX(rdngtime), %s, %s, COUNT(*),
                         AVG(awatthr + bwatthr + cwatthr)
                         FROM sensor_readings
                         WHERE sensor_id = %s
                         GROUP BY date_trunc('month', rdngtime);''',
                    (sensor_id, AVG_TYPE_MONTH, sensor_id))
        cur.execute('''INSERT INTO power_averages (rdngtime_first,
                         rdngtime_last, sensor_id, avg_type, num_pts, watts)
                       SELECT MIN(rdngtime), MAX(rdngtime), %s, %s, COUNT(*),
                         AVG(awatthr + bwatthr + cwatthr)
                         FROM sensor_readings
                         WHERE sensor_id = %s
                         GROUP BY date_trunc('week', rdngtime);''',
                    (sensor_id, AVG_TYPE_WEEK, sensor_id))

    cur.execute('''CREATE INDEX power_averages_first_index
                   ON power_averages (rdngtime_first);''')
    cur.execute('''CREATE INDEX power_averages_last_index
                   ON power_averages (rdngtime_last);''')

    cur.execute('''CREATE INDEX sensor_readings_rdngtime_per_index
                   ON sensor_readings ((date_trunc('minute'::text, rdngtime)
                     + (FLOOR((date_part('second'::text, rdngtime)
                     / 10::double precision)) * '00:00:10'::interval)));''')

    cur.execute('''ANALYZE;''')

if __name__ == '__main__':
    main()
<|endoftext|>


<commit_before>#from django.apps import apps
from django.test import TransactionTestCase
from django.db.migrations.executor import MigrationExecutor
from django.db import connection

from django.utils import timezone

from django.contrib.auth.models import User
from .factories import UserFactory, KeyboardFactory
from .serializers import KeyboardListItemSerializer
from .models import Keyboard

from django.db import models

class TestMigrations(TransactionTestCase):

    @property
    def app(self):
        return apps.get_containing_app_config(type(self).__module__).name

    migrate_from = None
    migrate_to = None

    def setUp(self):
        assert self.migrate_from and self.migrate_to, \
            "TestCase '{}' must define migrate_from and migrate_to properties".format(type(self).__name__)
        #self.migrate_from = [(self.app, self.migrate_from)]
        #self.migrate_to = [(self.app, self.migrate_to)]
        connection.prepare_database()
        executor = MigrationExecutor(connection)
        old_apps = executor.loader.project_state(self.migrate_from).apps

        # Reverse to the original migration
        executor.migrate(self.migrate_from)

        self.setUpBeforeMigration(old_apps)

        # Run the migration to test
        executor.loader.build_graph()
        executor.migrate(self.migrate_to)

        self.apps = executor.loader.project_state(self.migrate_to).apps

    def setUpBeforeMigration(self, apps):
        pass

class DefaultsTestCase(TestMigrations):

    migrate_from = [('cc_kapi', u'0001_initial')]
    migrate_to = [('cc_kapi', u'0002_v2api')]

    def setUpBeforeMigration(self, japps):
        # create users and keyboards under the old schema
        User = japps.get_model('auth', 'User')

        class KeyboardFactory2(KeyboardFactory):
            class Meta:
                model = japps.get_model('cc_kapi', 'Keyboard')

        user = User(username="test")
        user.save()
        self.user_id = user.id

        keyboard1 = KeyboardFactory2(user=user)
        keyboard1.is_primary = True
        keyboard1.save()
        self.keyboard1_id = keyboard1.id

        keyboard2 = KeyboardFactory2(user=user)
        keyboard2.save()
        self.keyboard2_id = keyboard2.id

        user1 = User(username="test2")
        user1.save()
        self.user1_id = user1.id

        keyboard3 = KeyboardFactory2(user=user1)
        keyboard3.save()
        self.keyboard3_id = keyboard3.id

    def test_defaults_migrated(self):
        # check that there are two users
        users = User.objects.all()
        self.assertEqual(len(users), 2)

        user = User.objects.get(id=self.user_id)
        user1 = User.objects.get(id=self.user1_id)

        # check that the three keyboards in the database exist
        kb1 = Keyboard.objects.get(id=self.keyboard1_id)
        kb2 = Keyboard.objects.get(id=self.keyboard2_id)
        kb3 = Keyboard.objects.get(id=self.keyboard3_id)

        # check that there are two keyboards for user and that the ids match
        keyboards = user.keyboard_set.all()
        self.assertEqual(len(keyboards), 2)
        kids = [k.id for k in keyboards]
        kids.sort()
        self.assertEqual(kids, [self.keyboard1_id, self.keyboard2_id])

        # check that there is one keyboard for user1 and that the id matches
        keyboards1 = user1.keyboard_set.all()
        self.assertEqual(len(keyboards1), 1)
        self.assertEqual(keyboards1[0].id, self.keyboard3_id)

        # check that there are five keyboards in total
        keyboards_all = Keyboard.objects.all()
        self.assertEqual(len(keyboards_all), 5)

        # check that kb is user's default desktop keyboard and there is no mobile default
        self.assertEqual(user.defaultkeyboards.desktop.id, kb1.id)
        self.assertEqual(user.defaultkeyboards.mobile, None)

        # check that user1 has no default desktop or mobile keyboard
        self.assertFalse(hasattr(user1, 'defaultkeyboards'))

<commit_msg>Commit the transaction before attempting to rollback the migration.<commit_after>#from django.apps import apps
from django.test import TransactionTestCase
from django.db.migrations.executor import MigrationExecutor
from django.db import connection, transaction

from django.utils import timezone

from django.contrib.auth.models import User
from .factories import UserFactory, KeyboardFactory
from .serializers import KeyboardListItemSerializer
from .models import Keyboard

from django.db import models

class TestMigrations(TransactionTestCase):

    @property
    def app(self):
        return apps.get_containing_app_config(type(self).__module__).name

    migrate_from = None
    migrate_to = None

    def setUp(self):
        assert self.migrate_from and self.migrate_to, \
            "TestCase '{}' must define migrate_from and migrate_to properties".format(type(self).__name__)
        #self.migrate_from = [(self.app, self.migrate_from)]
        #self.migrate_to = [(self.app, self.migrate_to)]
        connection.prepare_database()

        transaction.commit()

        executor = MigrationExecutor(connection)
        old_apps = executor.loader.project_state(self.migrate_from).apps

        # Reverse to the original migration
        executor.migrate(self.migrate_from)

        self.setUpBeforeMigration(old_apps)

        # Run the migration to test
        executor.loader.build_graph()
        executor.migrate(self.migrate_to)

        self.apps = executor.loader.project_state(self.migrate_to).apps

    def setUpBeforeMigration(self, apps):
        pass

class DefaultsTestCase(TestMigrations):

    migrate_from = [('cc_kapi', u'0001_initial')]
    migrate_to = [('cc_kapi', u'0002_v2api')]

    def setUpBeforeMigration(self, japps):
        # create users and keyboards under the old schema
        User = japps.get_model('auth', 'User')

        class KeyboardFactory2(KeyboardFactory):
            class Meta:
                model = japps.get_model('cc_kapi', 'Keyboard')

        user = User(username="test")
        user.save()
        self.user_id = user.id

        keyboard1 = KeyboardFactory2(user=user)
        keyboard1.is_primary = True
        keyboard1.save()
        self.keyboard1_id = keyboard1.id

        keyboard2 = KeyboardFactory2(user=user)
        keyboard2.save()
        self.keyboard2_id = keyboard2.id

        user1 = User(username="test2")
        user1.save()
        self.user1_id = user1.id

        keyboard3 = KeyboardFactory2(user=user1)
        keyboard3.save()
        self.keyboard3_id = keyboard3.id

    def test_defaults_migrated(self):
        # check that there are two users
        users = User.objects.all()
        self.assertEqual(len(users), 2)

        user = User.objects.get(id=self.user_id)
        user1 = User.objects.get(id=self.user1_id)

        # check that the three keyboards in the database exist
        kb1 = Keyboard.objects.get(id=self.keyboard1_id)
        kb2 = Keyboard.objects.get(id=self.keyboard2_id)
        kb3 = Keyboard.objects.get(id=self.keyboard3_id)

        # check that there are two keyboards for user and that the ids match
        keyboards = user.keyboard_set.all()
        self.assertEqual(len(keyboards), 2)
        kids = [k.id for k in keyboards]
        kids.sort()
        self.assertEqual(kids, [self.keyboard1_id, self.keyboard2_id])

        # check that there is one keyboard for user1 and that the id matches
        keyboards1 = user1.keyboard_set.all()
        self.assertEqual(len(keyboards1), 1)
        self.assertEqual(keyboards1[0].id, self.keyboard3_id)

        # check that there are five keyboards in total
        keyboards_all = Keyboard.objects.all()
        self.assertEqual(len(keyboards_all), 5)

        # check that kb is user's default desktop keyboard and there is no mobile default
        self.assertEqual(user.defaultkeyboards.desktop.id, kb1.id)
        self.assertEqual(user.defaultkeyboards.mobile, None)

        # check that user1 has no default desktop or mobile keyboard
        self.assertFalse(hasattr(user1, 'defaultkeyboards'))

<|endoftext|>


<commit_before>#!/usr/bin/env python
#-------------------------------------------------------------------------
# It extracts candidate sites applying some simple rules.
# This script works with Christian's file format.
#-------------------------------------------------------------------------

import codecs
import sys
import re

def cleanOfSpaces(myString):
    """Clean a string of trailing spaces"""

    myString = re.sub("^( )+", "", myString)
    myString = re.sub("( )+$", "", myString)
    return myString

def normalize(testURL):
    """Normalize the URL."""

    testURL = re.sub("^http(s)?://", "", testURL)
    testURL = re.sub("/$", "", testURL)
    components = testURL.split('/')
    return components

def isHTML(fileS):
    """If the last file ends with HTML"""

    if fileS.endswith("htm") or fileS.endswith("html"):
        return 1
    return 0

def hasLangParameter(lang, fileS):
    """It has the lang parameter in the form: lang=en etc.  """
    if re.search("=" + lang, fileS):
        return 1

def rule1(url, lang):
    """The rule says that the destination file should be same after replacing the language code: mi0064_en.htm====>mi0064_it.htm
    or it should have the structure of type =en"""

    components = normalize(url)
    fileS = components[-1]
    if isHTML(fileS):
        regex = "([^A-Za-z])" + lang + "\\.htm(l?)$"
        m = re.search(regex, fileS)
        if m:
            return 1
    elif hasLangParameter(lang, fileS):
        return 1
    return 0

def rule2(url, lang):
    """The rule says that the path to the file should differ by a language code e.g www.cvc.com/en/ => www.cvc.com/it/  """

    components = normalize(url)
    components.pop()
    urlPart = "/".join(components)
    regex = "([^A-Za-z]){1}" + lang + "([^A-Za-z]){1}"
    m = re.search(regex, urlPart)
    if m:
        return 1
    return 0

def rulesOn(url, lang):
    """If the URL has a certain form I say I have a promising URL."""

    mapRes = {}

    res1 = rule1(url, lang)
    res2 = rule2(url, lang)

    mapRes["rule1"] = res1
    mapRes["rule2"] = res2

    return mapRes

def getURL(line):
    """It gets the record from the line provided"""

    rest, json = line.split("\t")
    components = re.split("\s+", rest)
    for component in components:
        if re.search("^http(s)?://", component):
            return component

def getCandidates(fi, fo, sLang, dLang):
    """Check if a url could be a candidate or not."""

    nCandidates = 0
    for line in fi:
        line = line.rstrip()

        # tld url crawl<TAB>json_data
        url = getURL(line)

        mapS = rulesOn(url, sLang)
        mapD = rulesOn(url, dLang)

        #---------If-Elif to impede that the same link is put under both source
        if mapS["rule1"] or mapS["rule2"]:
            nCandidates += 1
            fo.write(
                "S#:" + url + "\t" + str(mapS["rule1"]) + "\t" + str(mapS["rule2"]) + "\n")
        elif mapD["rule1"] or mapD["rule2"]:
            nCandidates += 1
            fo.write(
                "D#:" + url + "\t" + str(mapD["rule1"]) + "\t" + str(mapD["rule2"]) + "\n")

    return nCandidates

def getFileOutput(dOutput, sLang, dLang):
    return dOutput + "/candidates-" + sLang + "-" + dLang + ".txt"

def getLogFile(dOutput, sLang, dLang):
    return dOutput + "/run-" + sLang + "-" + dLang + ".log"

def main():

    sLang = sys.argv[1]
    dLang = sys.argv[2]

    getCandidates(sys.stdin, sys.stdout, sLang, dLang)

if __name__ == '__main__':
    main()
<commit_msg>100% speedup<commit_after>#!/usr/bin/env python
#-------------------------------------------------------------------------
# It extracts candidate sites applying some simple rules.
# This script works with Christian's file format.
#-------------------------------------------------------------------------

import codecs
import sys
import re

def cleanOfSpaces(myString):
    """Clean a string of trailing spaces"""

    myString = re.sub("^( )+", "", myString)
    myString = re.sub("( )+$", "", myString)
    return myString

def normalize(testURL):
    """Normalize the URL."""
    testURL = testURL.split('//', 1)[1]

    # testURL = re.sub("^http(s)?://", "", testURL)
    if testURL.endswith('/'):
        testURL = testURL[:-1]
    # testURL = re.sub("/$", "", testURL)
    components = testURL.split('/')
    return components

def isHTML(fileS):
    """If the last file ends with HTML"""

    if fileS.endswith("htm") or fileS.endswith("html"):
        return 1
    return 0

def hasLangParameter(lang, fileS):
    """It has the lang parameter in the form: lang=en etc.  """
    re_lang = re.compile("=%s" % lang)
    if re_lang.search(fileS):
        return 1

def rule1(url, url_components, lang):
    """The rule says that the destination file should be same after replacing the language code: mi0064_en.htm====>mi0064_it.htm
    or it should have the structure of type =en"""

    fileS = url_components[-1]
    if isHTML(fileS):
        re_rule1 = re.compile("([^A-Za-z])" + lang + "\\.htm(l?)$")
        m = re_rule1.search(fileS)
        if m:
            return 1
    elif hasLangParameter(lang, fileS):
        return 1
    return 0

def rule2(url, url_components, lang):
    """The rule says that the path to the file should differ by a language code e.g www.cvc.com/en/ => www.cvc.com/it/  """

    urlPart = "/".join(url_components[:-1])
    # regex = "([^A-Za-z]){1}" + lang + "([^A-Za-z]){1}"
    re_rule2 = re.compile("([^A-Za-z])" + lang + "([^A-Za-z])")
    m = re_rule2.search(urlPart)
    if m:
        return 1
    return 0

def rulesOn(url, url_components, lang):
    """If the URL has a certain form I say I have a promising URL."""

    mapRes = {}

    res1 = rule1(url, url_components, lang)
    res2 = rule2(url, url_components, lang)

    mapRes["rule1"] = res1
    mapRes["rule2"] = res2

    return mapRes

def getURL(line):
    """It gets the record from the line provided"""

    rest, json = line.split("\t")
    components = re.split("\s+", rest)
    for component in components:
        if re.search("^http(s)?://", component):
            return component

def getCandidates(fi, fo, sLang, dLang):
    """Check if a url could be a candidate or not."""

    nCandidates = 0
    for line in fi:
        line = line.rstrip()

        # tld url crawl<TAB>json_data
        url = getURL(line)

        url_components = normalize(url)
        mapS = rulesOn(url, url_components, sLang)
        mapD = rulesOn(url, url_components, dLang)

        #---------If-Elif to impede that the same link is put under both source
        if mapS["rule1"] or mapS["rule2"]:
            nCandidates += 1
            fo.write(
                "S#:" + url + "\t" + str(mapS["rule1"]) + "\t" + str(mapS["rule2"]) + "\n")
        elif mapD["rule1"] or mapD["rule2"]:
            nCandidates += 1
            fo.write(
                "D#:" + url + "\t" + str(mapD["rule1"]) + "\t" + str(mapD["rule2"]) + "\n")

    return nCandidates

def getFileOutput(dOutput, sLang, dLang):
    return dOutput + "/candidates-" + sLang + "-" + dLang + ".txt"

def getLogFile(dOutput, sLang, dLang):
    return dOutput + "/run-" + sLang + "-" + dLang + ".log"

def main():

    sLang = sys.argv[1]
    dLang = sys.argv[2]

    getCandidates(sys.stdin, sys.stdout, sLang, dLang)

if __name__ == '__main__':
    main()
<|endoftext|>


<commit_before># Copyright 2015 by Raytheon BBN Technologies Corp.  All Rights Reserved.

import ast

# For testing only
if __name__ == '__main__':
    import os
    import sys

    # Find the directory that this executable lives in;
    # munge the path to look within the parent module
    #
    DIRNAME = os.path.normpath(
            os.path.abspath(os.path.dirname(sys.argv[0]) or '.'))
    sys.path.append(os.path.normpath(os.path.join(DIRNAME, '..')))

from pyqgl2.ast_util import NodeTransformerWithFname
from pyqgl2.ast_util import NodeVisitorWithFname

# This is just a smattering of possible waveforms.
# (I'm not sure whether these are even correct, or
# meaningful, but they're good enough for test cases)
#
UNI_WAVEFORMS = set(['MEAS', 'Y90', 'Y180', 'X90', 'X180', 'Z90', 'Z180'])

# Like UNI_WAVEFORMS, BI_OPS is fictitious
#
BI_OPS = set(['SWAP'])

class CheckType(NodeTransformerWithFname):

    def __init__(self, fname):
        super(CheckType, self).__init__(fname)

        # for each qbit, track where it is created
        #
        # the key is the qbit number, and the val is the name
        # and where it's created
        #
        self.qbit_origins = dict()

        # a list of scope tuples: (name, qbit?, context)
        #
        self.scope = list()

        self.class_defs = dict()

        self.func_level = 0

    def _push_scope(self, qbit_scope):
        self.scope.append(qbit_scope)

    def _pop_scope(self):
        self.scope = self.scope[:-1]

    def _curr_scope(self):
        return self.scope[-1]

    def _extend_scope(self, name):
        self.scope[-1].append(name)

    @staticmethod
    def _qbit_decl(node):

        if type(node) != ast.FunctionDef:
            return None

        if not node.decorator_list:
            return None

        decls = None
        for dec in node.decorator_list:
            if (type(dec) == ast.Call) and (dec.func.id == 'qtypes'):
                decls = dec.args
                break

        if not decls:
            return None

        params = list()

        for decl in decls:
            if type(decl) == ast.Str:
                params.append(decl.s)
            elif type(decl) == ast.Tuple:
                params.append(decl.elts[0].s)

        return params

    def assign_simple(self, node):

        target = node.targets[0]
        value = node.value

        if type(target) != ast.Name:
            return node

        if target.id in self._curr_scope():
            msg = 'reassignment of qbit \'%s\' forbidden' % target.id
            print msg
            self.error_msg(node,
                    ('reassignment of qbit \'%s\' forbidden' % target.id))

        if (type(value) == ast.Call) and (value.func.id == 'Qbit'):
            self._extend_scope(target.id)
        elif (type(value) == ast.Name) and value.id in self._curr_scope():
            self.warning_msg(node, 'alias of qbit \'%s\' as \'%s\'' %
                    (value.id, target.id))
            self._extend_scope(target.id)
        else:
            return node

        print 'new scope %s' % str(self._curr_scope())

        return node

    def visit_Assign(self, node):

        # We only do singleton assignments, not tuples,
        # and not expressions

        if type(node.targets[0]) == ast.Name:
            self.assign_simple(node)

        self.generic_visit(node)
        return node

    def visit_FunctionDef(self, node):
        decls = self._qbit_decl(node)
        if decls is not None:
            # diagnostic only
            print 'DECL: %s qbits %s' % (node.name, str(decls))
            self._push_scope(decls)
            self.func_level += 1
            self.generic_visit(node)
            self.func_level -= 1
            self._pop_scope()

        return node

    def visit_Call(self, node):

        # We can only check functions referenced by name, not arbitrary
        # expressions that return a function
        #
        if type(node.func) != ast.Name:
            self.warning_msg(node, 'function not referenced by name')
            return node

        func_name = node.func.id

        def check_arg(arg, argpos):
            if type(arg) != ast.Name:
                self.error_msg(node, '%s param to %s must be a symbol' %
                        (argpos, func_name))
                return False

            if arg.id not in self._curr_scope():
                self.error_msg(node, '%s param to %s must be a qbit' %
                        (argpos, func_name))
                return False

            return True

        if func_name in UNI_WAVEFORMS:
            if len(node.args) < 1:
                self.error_msg(node,
                        '%s requires a qbit parameter' % func_name)
                return node

            first_arg = node.args[0]
            check_arg(first_arg, 'first')

        elif func_name in BI_OPS:
            if len(node.args) < 2:
                self.error_msg(node,
                        '%s requires two qbit parameters' % func_name)
                return node

            arg1 = node.args[0]
            arg2 = node.args[1]

            check_arg(arg1, 'first')
            check_arg(arg2, 'second')

        return node

class CompileQGLFunctions(ast.NodeTransformer):

    LEVEL = 0

    def __init__(self, *args, **kwargs):
        super(CompileQGLFunctions, self).__init__(*args, **kwargs)

        self.concur_finder = FindConcurBlocks()

    def visit_FunctionDef(self, node):
        qglmode = False
        found_other = False

        print '>>> %s' % ast.dump(node)

        if node.decorator_list:
            for dec in node.decorator_list:
                if (type(dec) == ast.Name) and (dec.id == 'qglmode'):
                    qglmode = True
                elif (type(dec) == ast.Call) and (dec.func.id == 'qtypes'):
                    print 'yahoo'
                else:
                    found_other = True

            if qglmode and found_other:
                self.error_msg(node, 'qtypes must be the sole decorator')

        if not qglmode:
            return node

        if self.LEVEL > 0:
            self.error_msg(node, 'QGL mode functions cannot be nested')

        self.LEVEL += 1
        # check for nested qglmode functions
        self.generic_visit(node)
        self.LEVEL -= 1

        # First, find and check all the concur blocks

        body = node.body
        for ind in xrange(len(body)):
            stmnt = body[ind]
            body[ind] = self.concur_finder.visit(stmnt)

class FindWaveforms(ast.NodeTransformer):

    def __init__(self, *args, **kwargs):
        super(FindWaveforms, self).__init__(*args, **kwargs)

        self.seq = list()

    def visit_Call(self, node):

        # This is just a sketch

        if node.func.id == 'MEAS':
            self.seq.append('MEAS ' + ast.dump(node))
        elif node.func.id == 'X90':
            self.seq.append('X90 ' + ast.dump(node))
        elif node.func.id == 'Y90':
            self.seq.append('Y90 ' + ast.dump(node))

        return node

class FindConcurBlocks(ast.NodeTransformer):

    LEVEL = 0

    def __init__(self, *args, **kwargs):
        super(FindConcurBlocks, self).__init__(*args, **kwargs)

        self.concur_stmnts = set()
        self.qbit_sets = dict()

    def visit_With(self, node):
        if ((type(node.context_expr) != ast.Name) or
                (node.context_expr.id != 'concur')):
            return node

        if self.LEVEL > 0:
            self.error_msg(node, 'nested concur blocks are not supported')

        self.LEVEL += 1

        body = node.body
        for ind in xrange(len(body)):
            stmnt = body[ind]
            find_ref = FindQbitReferences()
            find_ref.generic_visit(stmnt)
            self.qbit_sets[ind] = find_ref.qbit_refs

            self.visit(stmnt)

        self.LEVEL -= 1

        # check_conflicts will halt the program if it detects an error
        #
        qbits_referenced = self.check_conflicts(node.lineno)
        print 'qbits in concur block (line: %d): %s' % (
                node.lineno, str(qbits_referenced))

        for ind in xrange(len(body)):
            stmnt = body[ind]
            find_waveforms = FindWaveforms()
            find_waveforms.generic_visit(stmnt)

            for waveform in find_waveforms.seq:
                print 'concur %d: WAVEFORM: %s' % (stmnt.lineno, waveform)

    def check_conflicts(self, lineno):

        all_seen = set()

        for refs in self.qbit_sets.values():
            if not refs.isdisjoint(all_seen):
                conflict = refs.intersection(all_seen)
                self.error_msg(node,
                        '%s appear in multiple concurrent statements' %
                        str(', '.join(list(conflict))))

            all_seen.update(refs)

        return all_seen

class FindQbitReferences(ast.NodeTransformer):
    """
    Find all the references to qbits in a node

    Assumes that all qbits are referenced by variables with
    names that start with 'qbit', rather than arbitrary expressions

    For example, if you do something like

        arr[ind] = qbit1
        foo = arr[ind]

    Then "qbit1" will be detected as a reference to a qbit,
    but "arr[ind]" or "foo" will not, even though all three
    expressions evaluate to a reference to the same qbit.
    """

    def __init__(self, *args, **kwargs):
        super(FindQbitReferences, self).__init__(*args, **kwargs)

        self.qbit_refs = set()

    def visit_Name(self, node):
        if node.id.startswith('qbit'):
            self.qbit_refs.add(node.id)

        return node

if __name__ == '__main__':
    import sys

    def preprocess(fname):

        text = open(fname, 'r').read()

        ptree = ast.parse(text, mode='exec')
        nptree = CheckType(fname).visit(ptree)

    preprocess(sys.argv[1])
<commit_msg>start of waveform checks<commit_after># Copyright 2015 by Raytheon BBN Technologies Corp.  All Rights Reserved.

import ast

# For testing only
if __name__ == '__main__':
    import os
    import sys

    # Find the directory that this executable lives in;
    # munge the path to look within the parent module
    #
    DIRNAME = os.path.normpath(
            os.path.abspath(os.path.dirname(sys.argv[0]) or '.'))
    sys.path.append(os.path.normpath(os.path.join(DIRNAME, '..')))
    print os.path.normpath(os.path.join(DIRNAME, '..'))

from pyqgl2.ast_util import NodeError
from pyqgl2.ast_util import NodeTransformerWithFname
from pyqgl2.ast_util import NodeVisitorWithFname

# This is just a smattering of possible waveforms.
# (I'm not sure whether these are even correct, or
# meaningful, but they're good enough for test cases)
#
UNI_WAVEFORMS = set(
        ['MEAS', 'Y90', 'Y180', 'X90', 'X180', 'Z90', 'Z180', 'UTheta'])

# Like UNI_WAVEFORMS, BI_OPS is fictitious
#
BI_OPS = set(['SWAP'])

class CheckType(NodeTransformerWithFname):

    def __init__(self, fname):
        super(CheckType, self).__init__(fname)

        # for each qbit, track where it is created
        #
        # the key is the qbit number, and the val is the name
        # and where it's created
        #
        self.qbit_origins = dict()

        # a list of scope tuples: (name, qbit?, context)
        #
        # We begin with the global scope, initially empty
        #
        self.scope = list(list())

        self.class_defs = dict()

        self.func_level = 0

        self.waveforms = dict()

    def _push_scope(self, qbit_scope):
        self.scope.append(qbit_scope)

    def _pop_scope(self):
        self.scope = self.scope[:-1]

    def _curr_scope(self):
        return self.scope[-1]

    def _extend_scope(self, name):
        self.scope[-1].append(name)

    @staticmethod
    def _qbit_decl(node):

        if type(node) != ast.FunctionDef:
            return None

        if not node.decorator_list:
            return None

        decls = None
        for dec in node.decorator_list:
            if (type(dec) == ast.Call) and (dec.func.id == 'qtypes'):
                decls = dec.args
                break

        if not decls:
            return None

        params = list()

        for decl in decls:
            if type(decl) == ast.Str:
                params.append(decl.s)
            elif type(decl) == ast.Tuple:
                params.append(decl.elts[0].s)

        return params

    def gen_waveform(self, name, args, kwargs):
        arg_text = ', '.join([ast.dump(arg) for arg in args])
        kwarg_text = ', '.join(sorted([ast.dump(arg) for arg in kwargs]))

        errs = 0
        for arg_index in xrange(1, len(args)):
            if type(args[arg_index]) != ast.Num:
                self.error_msg(arg, 'arg %d must be a number' % arg_index)
                errs += 1

        if errs:
            return

        signature = '%s-%s' % (name, arg_text)
        if kwarg_text:
            signature += '-%s' % kwarg_text

        # print 'WAVEFORM %s %s %s' % (name, arg_text, kwarg_text)
        # print signature

        if signature in self.waveforms:
            print 'NOTE: already generated waveform %s' % signature
        else:
            print 'generating waveform %s' % signature
            self.waveforms[signature] = 1 # BOGUS

    def assign_simple(self, node):

        target = node.targets[0]
        value = node.value

        if type(target) != ast.Name:
            return node

        if target.id in self._curr_scope():
            msg = 'reassignment of qbit \'%s\' forbidden' % target.id
            print msg
            self.error_msg(node,
                    ('reassignment of qbit \'%s\' forbidden' % target.id))

        if (type(value) == ast.Call) and (value.func.id == 'Qbit'):
            self._extend_scope(target.id)
        elif (type(value) == ast.Name) and value.id in self._curr_scope():
            self.warning_msg(node, 'alias of qbit \'%s\' as \'%s\'' %
                    (value.id, target.id))
            self._extend_scope(target.id)
        else:
            return node

        print 'new scope %s' % str(self._curr_scope())

        return node

    def visit_Assign(self, node):

        # We only do singleton assignments, not tuples,
        # and not expressions

        if type(node.targets[0]) == ast.Name:
            self.assign_simple(node)

        self.generic_visit(node)
        return node

    def visit_FunctionDef(self, node):
        decls = self._qbit_decl(node)
        if decls is not None:
            # diagnostic only
            self.diag_msg(
                    node, '%s declares qbits %s' % (node.name, str(decls)))
            self._push_scope(decls)
            self.func_level += 1
            self.generic_visit(node)
            self.func_level -= 1
            self._pop_scope()

        return node

    def visit_Call(self, node):

        # We can only check functions referenced by name, not arbitrary
        # expressions that return a function
        #
        if type(node.func) != ast.Name:
            self.warning_msg(node, 'function not referenced by name')
            return node

        func_name = node.func.id

        def check_arg(arg, argpos):
            if type(arg) != ast.Name:
                self.error_msg(node, '%s param to %s must be a symbol' %
                        (argpos, func_name))
                return False

            if arg.id not in self._curr_scope():
                self.error_msg(node, '%s param to %s must be a qbit' %
                        (argpos, func_name))
                return False

            return True

        if func_name in UNI_WAVEFORMS:
            if len(node.args) < 1:
                self.error_msg(node,
                        '%s requires a qbit parameter' % func_name)
                return node

            first_arg = node.args[0]
            check_arg(first_arg, 'first')

            self.gen_waveform(func_name, node.args, node.keywords)

        elif func_name in BI_OPS:
            if len(node.args) < 2:
                self.error_msg(node,
                        '%s requires two qbit parameters' % func_name)
                return node

            arg1 = node.args[0]
            arg2 = node.args[1]

            check_arg(arg1, 'first')
            check_arg(arg2, 'second')

        return node

class CompileQGLFunctions(ast.NodeTransformer):

    LEVEL = 0

    def __init__(self, *args, **kwargs):
        super(CompileQGLFunctions, self).__init__(*args, **kwargs)

        self.concur_finder = FindConcurBlocks()

    def visit_FunctionDef(self, node):
        qglmode = False
        found_other = False

        print '>>> %s' % ast.dump(node)

        if node.decorator_list:
            for dec in node.decorator_list:
                if (type(dec) == ast.Name) and (dec.id == 'qglmode'):
                    qglmode = True
                elif (type(dec) == ast.Call) and (dec.func.id == 'qtypes'):
                    print 'yahoo'
                else:
                    found_other = True

            if qglmode and found_other:
                self.error_msg(node, 'qtypes must be the sole decorator')

        if not qglmode:
            return node

        if self.LEVEL > 0:
            self.error_msg(node, 'QGL mode functions cannot be nested')

        self.LEVEL += 1
        # check for nested qglmode functions
        self.generic_visit(node)
        self.LEVEL -= 1

        # First, find and check all the concur blocks

        body = node.body
        for ind in xrange(len(body)):
            stmnt = body[ind]
            body[ind] = self.concur_finder.visit(stmnt)

class FindWaveforms(ast.NodeTransformer):

    def __init__(self, *args, **kwargs):
        super(FindWaveforms, self).__init__(*args, **kwargs)

        self.seq = list()

    def visit_Call(self, node):

        # This is just a sketch

        if node.func.id == 'MEAS':
            self.seq.append('MEAS ' + ast.dump(node))
        elif node.func.id == 'X90':
            self.seq.append('X90 ' + ast.dump(node))
        elif node.func.id == 'Y90':
            self.seq.append('Y90 ' + ast.dump(node))

        return node

class FindConcurBlocks(ast.NodeTransformer):

    LEVEL = 0

    def __init__(self, *args, **kwargs):
        super(FindConcurBlocks, self).__init__(*args, **kwargs)

        self.concur_stmnts = set()
        self.qbit_sets = dict()

    def visit_With(self, node):
        if ((type(node.context_expr) != ast.Name) or
                (node.context_expr.id != 'concur')):
            return node

        if self.LEVEL > 0:
            self.error_msg(node, 'nested concur blocks are not supported')

        self.LEVEL += 1

        body = node.body
        for ind in xrange(len(body)):
            stmnt = body[ind]
            find_ref = FindQbitReferences()
            find_ref.generic_visit(stmnt)
            self.qbit_sets[ind] = find_ref.qbit_refs

            self.visit(stmnt)

        self.LEVEL -= 1

        # check_conflicts will halt the program if it detects an error
        #
        qbits_referenced = self.check_conflicts(node.lineno)
        print 'qbits in concur block (line: %d): %s' % (
                node.lineno, str(qbits_referenced))

        for ind in xrange(len(body)):
            stmnt = body[ind]
            find_waveforms = FindWaveforms()
            find_waveforms.generic_visit(stmnt)

            for waveform in find_waveforms.seq:
                print 'concur %d: WAVEFORM: %s' % (stmnt.lineno, waveform)

    def check_conflicts(self, lineno):

        all_seen = set()

        for refs in self.qbit_sets.values():
            if not refs.isdisjoint(all_seen):
                conflict = refs.intersection(all_seen)
                self.error_msg(node,
                        '%s appear in multiple concurrent statements' %
                        str(', '.join(list(conflict))))

            all_seen.update(refs)

        return all_seen

class FindQbitReferences(ast.NodeTransformer):
    """
    Find all the references to qbits in a node

    Assumes that all qbits are referenced by variables with
    names that start with 'qbit', rather than arbitrary expressions

    For example, if you do something like

        arr[ind] = qbit1
        foo = arr[ind]

    Then "qbit1" will be detected as a reference to a qbit,
    but "arr[ind]" or "foo" will not, even though all three
    expressions evaluate to a reference to the same qbit.
    """

    def __init__(self, *args, **kwargs):
        super(FindQbitReferences, self).__init__(*args, **kwargs)

        self.qbit_refs = set()

    def visit_Name(self, node):
        if node.id.startswith('qbit'):
            self.qbit_refs.add(node.id)

        return node

if __name__ == '__main__':
    import sys

    def preprocess(fname):

        text = open(fname, 'r').read()

        ptree = ast.parse(text, mode='exec')
        type_check = CheckType(fname)
        nptree = type_check.visit(ptree)
        if type_check.max_err_level >= NodeError.NODE_ERROR_ERROR:
            print 'bailing out'
            sys.exit(1)

    preprocess(sys.argv[1])
<|endoftext|>