<commit_before>#!/usr/bin/env python

import os
import sys
import argparse
import subprocess

# ==============================================================================
def _run_cmd(cmd, path=None):

    if path:
        env = os.environ.copy()
        env['PYTHONPATH'] = path
    else:
        env = None

    print(cmd)

    p = subprocess.Popen(cmd, env=env, shell=False)
    result = p.wait()

    if result:
        sys.exit(result)

# ==============================================================================
def _fetch_repo(cur_dir, repo_name, repo_dir=None):
    if repo_dir:
        return os.path.abspath(repo_dir)

    repo_dir = os.path.join(cur_dir, repo_name)

    _run_cmd(["git", "clone", "-b", "master", "--depth", "1",
              "https://github.com/aqualid/%s.git" % repo_name, repo_dir])

    return repo_dir

# ==============================================================================

def _run_example(core_dir, tools_dir, example_dir):
    tools_dir = os.path.join(tools_dir, 'tools')

    cmd = [sys.executable, "-c", "import aql;import sys;sys.exit(aql.main())", "-C", example_dir, "-I", tools_dir]
    _run_cmd(cmd, core_dir)
    _run_cmd(cmd + ['-R'], core_dir)

# ==============================================================================

def _get_examples(examples_dir):
    for item in os.listdir(examples_dir):
        example_dir = os.path.join(examples_dir, item)
        makefile = os.path.join(example_dir, 'make.aql')
        if os.path.isfile(makefile):
            yield example_dir

# ==============================================================================

def run(core_dir, tools_dir, examples_dir):
    for example_dir in _get_examples(examples_dir):
        _run_example(core_dir, tools_dir, example_dir)

# ==============================================================================
def _parse_args():
    args_parser = argparse.ArgumentParser()

    args_parser.add_argument('--core-dir', action='store',
                             dest='core_dir', metavar='PATH',
                             help="Aqualid core directory. "
                                  "By default it will be fetched from GitHub.")

    args_parser.add_argument('--tools-dir', action='store',
                             dest='tools_dir', metavar='PATH',
                             help="Aqualid tools directory. "
                                  "By default it will be fetched from GitHub.")

    return args_parser.parse_args()

# ==============================================================================

def main():
    args = _parse_args()

    examples_dir = os.path.abspath(os.path.dirname(__file__))

    core_dir = _fetch_repo(examples_dir, 'aqualid', args.core_dir)
    tools_dir = _fetch_repo(examples_dir, 'tools', args.tools_dir)

    run(core_dir, tools_dir, examples_dir)

# ==============================================================================
if __name__ == '__main__':
    main()
<commit_msg>Cosmetic fix<commit_after>#!/usr/bin/env python

import os
import sys
import argparse
import subprocess

# ==============================================================================
def _run_cmd(cmd, path=None):

    if path:
        env = os.environ.copy()
        env['PYTHONPATH'] = path
    else:
        env = None

    print(cmd)

    p = subprocess.Popen(cmd, env=env, shell=False)
    result = p.wait()

    if result:
        sys.exit(result)

# ==============================================================================
def _fetch_repo(cur_dir, repo_name, repo_dir=None):
    if repo_dir:
        return os.path.abspath(repo_dir)

    repo_dir = os.path.join(cur_dir, repo_name)

    _run_cmd(["git", "clone", "-b", "master", "--depth", "1",
              "https://github.com/aqualid/%s.git" % repo_name, repo_dir])

    return repo_dir

# ==============================================================================

def _run_example(core_dir, tools_dir, example_dir):
    tools_dir = os.path.join(tools_dir, 'tools')

    cmd = [sys.executable, "-c", "import aql;import sys;sys.exit(aql.main())",
           "-C", example_dir, "-I", tools_dir]
    _run_cmd(cmd, core_dir)
    _run_cmd(cmd + ['-R'], core_dir)

# ==============================================================================

def _get_examples(examples_dir):
    for item in os.listdir(examples_dir):
        example_dir = os.path.join(examples_dir, item)
        makefile = os.path.join(example_dir, 'make.aql')
        if os.path.isfile(makefile):
            yield example_dir

# ==============================================================================

def run(core_dir, tools_dir, examples_dir):
    for example_dir in _get_examples(examples_dir):
        _run_example(core_dir, tools_dir, example_dir)

# ==============================================================================
def _parse_args():
    args_parser = argparse.ArgumentParser()

    args_parser.add_argument('--core-dir', action='store',
                             dest='core_dir', metavar='PATH',
                             help="Aqualid core directory. "
                                  "By default it will be fetched from GitHub.")

    args_parser.add_argument('--tools-dir', action='store',
                             dest='tools_dir', metavar='PATH',
                             help="Aqualid tools directory. "
                                  "By default it will be fetched from GitHub.")

    return args_parser.parse_args()

# ==============================================================================

def main():
    args = _parse_args()

    examples_dir = os.path.abspath(os.path.dirname(__file__))

    core_dir = _fetch_repo(examples_dir, 'aqualid', args.core_dir)
    tools_dir = _fetch_repo(examples_dir, 'tools', args.tools_dir)

    run(core_dir, tools_dir, examples_dir)

# ==============================================================================
if __name__ == '__main__':
    main()
<|endoftext|>


<commit_before>#!/usr/bin/env python
# ***** BEGIN LICENSE BLOCK *****
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this file,
# You can obtain one at http://mozilla.org/MPL/2.0/.
# ***** END LICENSE BLOCK *****
import os
import sys
import copy

# load modules from parent dir
sys.path.insert(1, os.path.dirname(sys.path[0]))

from mozharness.base.script import PreScriptAction
from mozharness.base.vcs.vcsbase import MercurialScript
from mozharness.mozilla.blob_upload import BlobUploadMixin, blobupload_config_options
from mozharness.mozilla.testing.testbase import TestingMixin, testing_config_options

from mozharness.mozilla.structuredlog import StructuredOutputParser
from mozharness.base.log import INFO

class WebPlatformTest(TestingMixin, MercurialScript, BlobUploadMixin):
    config_options = [
        [['--test-type'], {
            "action": "extend",
            "dest": "test_type",
            "help": "Specify the test types to run."}
         ],
        [["--total-chunks"], {
            "action": "store",
            "dest": "total_chunks",
            "help": "Number of total chunks"}
         ],
        [["--this-chunk"], {
            "action": "store",
            "dest": "this_chunk",
            "help": "Number of this chunk"}]
    ] + copy.deepcopy(testing_config_options) + \
        copy.deepcopy(blobupload_config_options)

    def __init__(self, require_config_file=True):
        super(WebPlatformTest, self).__init__(
            config_options=self.config_options,
            all_actions=[
                'clobber',
                'read-buildbot-config',
                'download-and-extract',
                'create-virtualenv',
                'pull',
                'install',
                'run-tests',
            ],
            require_config_file=require_config_file,
            config={'require_test_zip': True})

        # Surely this should be in the superclass
        c = self.config
        self.installer_url = c.get('installer_url')
        self.test_url = c.get('test_url')
        self.installer_path = c.get('installer_path')
        self.binary_path = c.get('binary_path')
        self.abs_app_dir = None

    def query_abs_app_dir(self):
        """We can't set this in advance, because OSX install directories
        change depending on branding and opt/debug.
        """
        if self.abs_app_dir:
            return self.abs_app_dir
        if not self.binary_path:
            self.fatal("Can't determine abs_app_dir (binary_path not set!)")
        self.abs_app_dir = os.path.dirname(self.binary_path)
        return self.abs_app_dir

    def query_abs_dirs(self):
        if self.abs_dirs:
            return self.abs_dirs
        abs_dirs = super(WebPlatformTest, self).query_abs_dirs()

        dirs = {}
        dirs['abs_app_install_dir'] = os.path.join(abs_dirs['abs_work_dir'], 'application')
        dirs['abs_test_install_dir'] = os.path.join(abs_dirs['abs_work_dir'], 'tests')
        dirs["abs_wpttest_dir"] = os.path.join(dirs['abs_test_install_dir'], "web-platform")
        dirs['abs_blob_upload_dir'] = os.path.join(abs_dirs['abs_work_dir'], 'blobber_upload_dir')

        abs_dirs.update(dirs)
        self.abs_dirs = abs_dirs

        return self.abs_dirs

    @PreScriptAction('create-virtualenv')
    def _pre_create_virtualenv(self, action):
        dirs = self.query_abs_dirs()

        requirements = os.path.join(dirs['abs_test_install_dir'],
                                    'config',
                                    'marionette_requirements.txt')

        self.register_virtualenv_module(requirements=[requirements],
                                        two_pass=True)

        self.register_virtualenv_module(name='fxos_appgen')

    def _query_cmd(self):
        if not self.binary_path:
            self.fatal("Binary path could not be determined")
            #And exit

        c = self.config
        dirs = self.query_abs_dirs()
        abs_app_dir = self.query_abs_app_dir()
        run_file_name = "runtests.py"

        base_cmd = [self.query_python_path('python'), '-u']
        base_cmd.append(os.path.join(dirs["abs_wpttest_dir"], run_file_name))

        # Make sure that the logging directory exists
        if self.mkdir_p(dirs["abs_blob_upload_dir"]) == -1:
            self.fatal("Could not create blobber upload directory")
            # Exit

        base_cmd += ["--log-raw=-",
                     "--log-raw=%s" % os.path.join(dirs["abs_blob_upload_dir"],
                                                   "wpt_raw.log"),
                     "--binary=%s" % self.binary_path,
                     "--symbols-path=%s" % self.query_symbols_url(),
                     "--stackwalk-binary=%s" % self.query_minidump_stackwalk()]

        for test_type in c.get("test_type", []):
            base_cmd.append("--test-type=%s" % test_type)

        for opt in ["total_chunks", "this_chunk"]:
            val = c.get(opt)
            if val:
                base_cmd.append("--%s=%s" % (opt.replace("_", "-"), val))

        options = list(c.get("options", [])) + list(self.tree_config["options"])

        str_format_values = {
            'binary_path': self.binary_path,
            'test_path': dirs["abs_wpttest_dir"],
            'test_install_path': dirs["abs_test_install_dir"],
            'abs_app_dir': abs_app_dir,
            'abs_work_dir': dirs["abs_work_dir"]
            }

        opt_cmd = [item % str_format_values for item in options]

        return base_cmd + opt_cmd

    def download_and_extract(self):
        super(WebPlatformTest, self).download_and_extract(
            target_unzip_dirs=["bin/*",
                               "config/*",
                               "mozbase/*",
                               "marionette/*",
                               "web-platform/*"])

    def run_tests(self):
        dirs = self.query_abs_dirs()
        cmd = self._query_cmd()

        parser = StructuredOutputParser(config=self.config,
                                        log_obj=self.log_obj)

        env = {'MINIDUMP_SAVE_PATH': dirs['abs_blob_upload_dir']}
        env = self.query_env(partial_env=env, log_level=INFO)

        return_code = self.run_command(cmd,
                                       cwd=dirs['abs_work_dir'],
                                       output_timeout=1000,
                                       output_parser=parser,
                                       env=env)

        tbpl_status, log_level = parser.evaluate_parser(return_code)

        self.buildbot_status(tbpl_status, level=log_level)

# main {{{1
if __name__ == '__main__':
    web_platform_tests = WebPlatformTest()
    web_platform_tests.run_and_exit()
<commit_msg>Bug 1143634 - Remove dependency on fxos-appgen from web platform tests. r=jgriffin<commit_after>#!/usr/bin/env python
# ***** BEGIN LICENSE BLOCK *****
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this file,
# You can obtain one at http://mozilla.org/MPL/2.0/.
# ***** END LICENSE BLOCK *****
import os
import sys
import copy

# load modules from parent dir
sys.path.insert(1, os.path.dirname(sys.path[0]))

from mozharness.base.script import PreScriptAction
from mozharness.base.vcs.vcsbase import MercurialScript
from mozharness.mozilla.blob_upload import BlobUploadMixin, blobupload_config_options
from mozharness.mozilla.testing.testbase import TestingMixin, testing_config_options

from mozharness.mozilla.structuredlog import StructuredOutputParser
from mozharness.base.log import INFO

class WebPlatformTest(TestingMixin, MercurialScript, BlobUploadMixin):
    config_options = [
        [['--test-type'], {
            "action": "extend",
            "dest": "test_type",
            "help": "Specify the test types to run."}
         ],
        [["--total-chunks"], {
            "action": "store",
            "dest": "total_chunks",
            "help": "Number of total chunks"}
         ],
        [["--this-chunk"], {
            "action": "store",
            "dest": "this_chunk",
            "help": "Number of this chunk"}]
    ] + copy.deepcopy(testing_config_options) + \
        copy.deepcopy(blobupload_config_options)

    def __init__(self, require_config_file=True):
        super(WebPlatformTest, self).__init__(
            config_options=self.config_options,
            all_actions=[
                'clobber',
                'read-buildbot-config',
                'download-and-extract',
                'create-virtualenv',
                'pull',
                'install',
                'run-tests',
            ],
            require_config_file=require_config_file,
            config={'require_test_zip': True})

        # Surely this should be in the superclass
        c = self.config
        self.installer_url = c.get('installer_url')
        self.test_url = c.get('test_url')
        self.installer_path = c.get('installer_path')
        self.binary_path = c.get('binary_path')
        self.abs_app_dir = None

    def query_abs_app_dir(self):
        """We can't set this in advance, because OSX install directories
        change depending on branding and opt/debug.
        """
        if self.abs_app_dir:
            return self.abs_app_dir
        if not self.binary_path:
            self.fatal("Can't determine abs_app_dir (binary_path not set!)")
        self.abs_app_dir = os.path.dirname(self.binary_path)
        return self.abs_app_dir

    def query_abs_dirs(self):
        if self.abs_dirs:
            return self.abs_dirs
        abs_dirs = super(WebPlatformTest, self).query_abs_dirs()

        dirs = {}
        dirs['abs_app_install_dir'] = os.path.join(abs_dirs['abs_work_dir'], 'application')
        dirs['abs_test_install_dir'] = os.path.join(abs_dirs['abs_work_dir'], 'tests')
        dirs["abs_wpttest_dir"] = os.path.join(dirs['abs_test_install_dir'], "web-platform")
        dirs['abs_blob_upload_dir'] = os.path.join(abs_dirs['abs_work_dir'], 'blobber_upload_dir')

        abs_dirs.update(dirs)
        self.abs_dirs = abs_dirs

        return self.abs_dirs

    @PreScriptAction('create-virtualenv')
    def _pre_create_virtualenv(self, action):
        dirs = self.query_abs_dirs()

        requirements = os.path.join(dirs['abs_test_install_dir'],
                                    'config',
                                    'marionette_requirements.txt')

        self.register_virtualenv_module(requirements=[requirements],
                                        two_pass=True)

    def _query_cmd(self):
        if not self.binary_path:
            self.fatal("Binary path could not be determined")
            #And exit

        c = self.config
        dirs = self.query_abs_dirs()
        abs_app_dir = self.query_abs_app_dir()
        run_file_name = "runtests.py"

        base_cmd = [self.query_python_path('python'), '-u']
        base_cmd.append(os.path.join(dirs["abs_wpttest_dir"], run_file_name))

        # Make sure that the logging directory exists
        if self.mkdir_p(dirs["abs_blob_upload_dir"]) == -1:
            self.fatal("Could not create blobber upload directory")
            # Exit

        base_cmd += ["--log-raw=-",
                     "--log-raw=%s" % os.path.join(dirs["abs_blob_upload_dir"],
                                                   "wpt_raw.log"),
                     "--binary=%s" % self.binary_path,
                     "--symbols-path=%s" % self.query_symbols_url(),
                     "--stackwalk-binary=%s" % self.query_minidump_stackwalk()]

        for test_type in c.get("test_type", []):
            base_cmd.append("--test-type=%s" % test_type)

        for opt in ["total_chunks", "this_chunk"]:
            val = c.get(opt)
            if val:
                base_cmd.append("--%s=%s" % (opt.replace("_", "-"), val))

        options = list(c.get("options", [])) + list(self.tree_config["options"])

        str_format_values = {
            'binary_path': self.binary_path,
            'test_path': dirs["abs_wpttest_dir"],
            'test_install_path': dirs["abs_test_install_dir"],
            'abs_app_dir': abs_app_dir,
            'abs_work_dir': dirs["abs_work_dir"]
            }

        opt_cmd = [item % str_format_values for item in options]

        return base_cmd + opt_cmd

    def download_and_extract(self):
        super(WebPlatformTest, self).download_and_extract(
            target_unzip_dirs=["bin/*",
                               "config/*",
                               "mozbase/*",
                               "marionette/*",
                               "web-platform/*"])

    def run_tests(self):
        dirs = self.query_abs_dirs()
        cmd = self._query_cmd()

        parser = StructuredOutputParser(config=self.config,
                                        log_obj=self.log_obj)

        env = {'MINIDUMP_SAVE_PATH': dirs['abs_blob_upload_dir']}
        env = self.query_env(partial_env=env, log_level=INFO)

        return_code = self.run_command(cmd,
                                       cwd=dirs['abs_work_dir'],
                                       output_timeout=1000,
                                       output_parser=parser,
                                       env=env)

        tbpl_status, log_level = parser.evaluate_parser(return_code)

        self.buildbot_status(tbpl_status, level=log_level)

# main {{{1
if __name__ == '__main__':
    web_platform_tests = WebPlatformTest()
    web_platform_tests.run_and_exit()
<|endoftext|>


<commit_before>from pyramid.view import view_config

from pyramid.httpexceptions import HTTPException, HTTPFound, HTTPNotFound
from deform import Form, Button
from deform import ValidationFailure

from phoenix.views.settings import SettingsView
from phoenix.grid import MyGrid

import logging
logger = logging.getLogger(__name__)

class CatalogService(SettingsView):
    def __init__(self, request):
        super(CatalogService, self).__init__(
            request, name='catalog_settings', title='Catalog Service')
        self.csw = self.request.csw
        self.description = "%s (%s)" % (self.csw.identification.title, self.csw.url)

    def generate_service_form(self, formid="deform"):
        from phoenix.schema import CatalogAddServiceSchema
        schema = CatalogAddServiceSchema()
        return Form(
            schema,
            buttons=(Button(name='add_service', title='Add Service'),),
            formid=formid)

    def process_service_form(self, form):
        try:
            controls = self.request.POST.items()
            appstruct = form.validate(controls)
            url = appstruct.get('url')
            self.request.csw.harvest(
                source=url,
                resourcetype=appstruct.get('resource_type'))
            self.session.flash('Added WPS %s' % (url), queue="success")
        except ValidationFailure, e:
            logger.exception('validation of catalog form failed')
            return dict(form = e.render())
        except Exception, e:
            logger.exception('could not harvest wps.')
            self.session.flash('Could not add WPS %s. %s' % (url, e), queue="error")
        return HTTPFound(location=self.request.route_url('catalog_settings'))

    def generate_dataset_form(self, formid="deform"):
        from phoenix.schema import PublishSchema
        schema = PublishSchema().bind(email=self.user_email())
        return Form(
            schema,
            buttons=(Button(name='add_dataset', title='Add Dataset'),),
            formid=formid)

    def process_dataset_form(self, form):
        try:
            controls = self.request.POST.items()
            appstruct = form.validate(controls)

            from mako.template import Template
            import os
            # TODO: fix location and usage of publish templates
            import phoenix
            templ_dc = Template(filename=os.path.join(os.path.dirname(phoenix.__file__), "templates", "dc.xml"))
            record = templ_dc.render(**appstruct)
            logger.debug('record=%s', record)
            self.csw.transaction(ttype="insert", typename='csw:Record', record=str(record))
            self.session.flash('Added Dataset %s' % (appstruct.get('title')), queue="success")
        except ValidationFailure, e:
            logger.exception('validation of catalog form failed')
            return dict(form = e.render())
        except Exception, e:
            logger.exception('could not harvest wps.')
            self.session.flash('Could not add Dataset %s. %s' % (appstruct.get('source'), e), queue="error")
        return HTTPFound(location=self.request.route_url('catalog_settings'))

    @view_config(route_name='remove_all_records')
    def remove_all(self):
        try:
            self.csw.getrecords(maxrecords=0)
            count = self.csw.results.get('matches'),
            # TODO: self destruction is out of order ... see exception
            self.csw.transaction(ttype='delete', typename='csw:Record')
            self.session.flash("%d Records deleted." % count, queue='info')
        except Exception,e:
            logger.exception('could not remove datasets.')
            self.session.flash('Ooops ... self destruction out of order. %s' % e, queue="error")
        return HTTPFound(location=self.request.route_url('catalog_settings'))

    @view_config(route_name='remove_record')
    def remove(self):
        try:
            recordid = self.request.matchdict.get('recordid')
            self.csw.transaction(ttype='delete', typename='csw:Record', identifier=recordid )
            self.session.flash('Removed record %s.' % recordid, queue="info")
        except Exception,e:
            logger.exception("Could not remove record")
            self.session.flash('Could not remove record. %s' % e, queue="error")
        return HTTPFound(location=self.request.route_url('catalog_settings'))

    def get_csw_items(self):
        results = []
        try:
            self.csw.getrecords(esn="full")
            logger.debug('csw results %s', self.csw.results)
            for rec in self.csw.records:
                myrec = self.csw.records[rec]
                results.append(dict(
                    source = myrec.source,
                    identifier = myrec.identifier,
                    title = myrec.title,
                    abstract = myrec.abstract,
                    subjects = myrec.subjects,
                    format = myrec.format,
                    creator = myrec.creator,
                    modified = myrec.modified,
                    bbox = myrec.bbox,
                    references = myrec.references,
                    ))
        except:
            logger.exception('could not get items for csw.')
        return results

    @view_config(route_name="catalog_settings", renderer='phoenix:templates/settings/catalog.pt')
    def view(self):
        service_form = self.generate_service_form()
        dataset_form = self.generate_dataset_form()
        if 'add_service' in self.request.POST:
            return self.process_service_form(service_form)
        elif 'add_dataset' in self.request.POST:
            return self.process_dataset_form(dataset_form)
        items = self.get_csw_items()

        grid = CSWGrid(
                self.request,
                items,
                ['title', 'creator', 'modified', 'format', ''],
            )
        self.csw.getrecords(maxrecords=0)
        return dict(
            datasets_found=self.csw.results.get('matches'),
            grid=grid,
            items=items,
            service_form=service_form.render(),
            dataset_form=dataset_form.render())

class CSWGrid(MyGrid):
    def __init__(self, request, *args, **kwargs):
        super(CSWGrid, self).__init__(request, *args, **kwargs)
        self.column_formats[''] = self.action_td
        self.column_formats['title'] = self.title_td
        self.column_formats['format'] = self.format_td
        self.column_formats['modified'] = self.modified_td

    def title_td(self, col_num, i, item):
        return self.render_title_td(item['title'], item['abstract'], item.get('subjects'))

    def format_td(self, col_num, i, item):
        return self.render_format_td(item['format'], item['source'])

    def modified_td(self, col_num, i, item):
        return self.render_timestamp_td(timestamp=item.get('modified'))

    def action_td(self, col_num, i, item):
        buttongroup = []
        buttongroup.append(
            ("remove", item.get('identifier'), "glyphicon glyphicon-trash", "",
            self.request.route_url('remove_record', recordid=item.get('identifier')),
            False) )
        return self.render_action_td(buttongroup)

<commit_msg>fixed breadcrumbs in catalog<commit_after>from pyramid.view import view_config

from pyramid.httpexceptions import HTTPException, HTTPFound, HTTPNotFound
from deform import Form, Button
from deform import ValidationFailure

from phoenix.views.settings import SettingsView
from phoenix.grid import MyGrid

import logging
logger = logging.getLogger(__name__)

class Catalog(SettingsView):
    def __init__(self, request):
        super(Catalog, self).__init__(
            request, name='catalog_settings', title='Catalog Service')
        self.csw = self.request.csw
        self.description = "%s (%s)" % (self.csw.identification.title, self.csw.url)

    def breadcrumbs(self):
        breadcrumbs = super(Catalog, self).breadcrumbs()
        breadcrumbs.append(dict(route_path=self.request.route_path(self.name), title=self.title))
        return breadcrumbs

    def generate_service_form(self, formid="deform"):
        from phoenix.schema import CatalogAddServiceSchema
        schema = CatalogAddServiceSchema()
        return Form(
            schema,
            buttons=(Button(name='add_service', title='Add Service'),),
            formid=formid)

    def process_service_form(self, form):
        try:
            controls = self.request.POST.items()
            appstruct = form.validate(controls)
            url = appstruct.get('url')
            self.request.csw.harvest(
                source=url,
                resourcetype=appstruct.get('resource_type'))
            self.session.flash('Added WPS %s' % (url), queue="success")
        except ValidationFailure, e:
            logger.exception('validation of catalog form failed')
            return dict(form = e.render())
        except Exception, e:
            logger.exception('could not harvest wps.')
            self.session.flash('Could not add WPS %s. %s' % (url, e), queue="error")
        return HTTPFound(location=self.request.route_url('catalog_settings'))

    def generate_dataset_form(self, formid="deform"):
        from phoenix.schema import PublishSchema
        schema = PublishSchema().bind(email=self.user_email())
        return Form(
            schema,
            buttons=(Button(name='add_dataset', title='Add Dataset'),),
            formid=formid)

    def process_dataset_form(self, form):
        try:
            controls = self.request.POST.items()
            appstruct = form.validate(controls)

            from mako.template import Template
            import os
            # TODO: fix location and usage of publish templates
            import phoenix
            templ_dc = Template(filename=os.path.join(os.path.dirname(phoenix.__file__), "templates", "dc.xml"))
            record = templ_dc.render(**appstruct)
            logger.debug('record=%s', record)
            self.csw.transaction(ttype="insert", typename='csw:Record', record=str(record))
            self.session.flash('Added Dataset %s' % (appstruct.get('title')), queue="success")
        except ValidationFailure, e:
            logger.exception('validation of catalog form failed')
            return dict(form = e.render())
        except Exception, e:
            logger.exception('could not harvest wps.')
            self.session.flash('Could not add Dataset %s. %s' % (appstruct.get('source'), e), queue="error")
        return HTTPFound(location=self.request.route_url('catalog_settings'))

    @view_config(route_name='remove_all_records')
    def remove_all(self):
        try:
            self.csw.getrecords(maxrecords=0)
            count = self.csw.results.get('matches'),
            # TODO: self destruction is out of order ... see exception
            self.csw.transaction(ttype='delete', typename='csw:Record')
            self.session.flash("%d Records deleted." % count, queue='info')
        except Exception,e:
            logger.exception('could not remove datasets.')
            self.session.flash('Ooops ... self destruction out of order. %s' % e, queue="error")
        return HTTPFound(location=self.request.route_url('catalog_settings'))

    @view_config(route_name='remove_record')
    def remove(self):
        try:
            recordid = self.request.matchdict.get('recordid')
            self.csw.transaction(ttype='delete', typename='csw:Record', identifier=recordid )
            self.session.flash('Removed record %s.' % recordid, queue="info")
        except Exception,e:
            logger.exception("Could not remove record")
            self.session.flash('Could not remove record. %s' % e, queue="error")
        return HTTPFound(location=self.request.route_url('catalog_settings'))

    def get_csw_items(self):
        results = []
        try:
            self.csw.getrecords(esn="full")
            logger.debug('csw results %s', self.csw.results)
            for rec in self.csw.records:
                myrec = self.csw.records[rec]
                results.append(dict(
                    source = myrec.source,
                    identifier = myrec.identifier,
                    title = myrec.title,
                    abstract = myrec.abstract,
                    subjects = myrec.subjects,
                    format = myrec.format,
                    creator = myrec.creator,
                    modified = myrec.modified,
                    bbox = myrec.bbox,
                    references = myrec.references,
                    ))
        except:
            logger.exception('could not get items for csw.')
        return results

    @view_config(route_name="catalog_settings", renderer='phoenix:templates/settings/catalog.pt')
    def view(self):
        service_form = self.generate_service_form()
        dataset_form = self.generate_dataset_form()
        if 'add_service' in self.request.POST:
            return self.process_service_form(service_form)
        elif 'add_dataset' in self.request.POST:
            return self.process_dataset_form(dataset_form)
        items = self.get_csw_items()

        grid = CSWGrid(
                self.request,
                items,
                ['title', 'creator', 'modified', 'format', ''],
            )
        self.csw.getrecords(maxrecords=0)
        return dict(
            datasets_found=self.csw.results.get('matches'),
            grid=grid,
            items=items,
            service_form=service_form.render(),
            dataset_form=dataset_form.render())

class CSWGrid(MyGrid):
    def __init__(self, request, *args, **kwargs):
        super(CSWGrid, self).__init__(request, *args, **kwargs)
        self.column_formats[''] = self.action_td
        self.column_formats['title'] = self.title_td
        self.column_formats['format'] = self.format_td
        self.column_formats['modified'] = self.modified_td

    def title_td(self, col_num, i, item):
        return self.render_title_td(item['title'], item['abstract'], item.get('subjects'))

    def format_td(self, col_num, i, item):
        return self.render_format_td(item['format'], item['source'])

    def modified_td(self, col_num, i, item):
        return self.render_timestamp_td(timestamp=item.get('modified'))

    def action_td(self, col_num, i, item):
        buttongroup = []
        buttongroup.append(
            ("remove", item.get('identifier'), "glyphicon glyphicon-trash", "",
            self.request.route_url('remove_record', recordid=item.get('identifier')),
            False) )
        return self.render_action_td(buttongroup)

<|endoftext|>


<commit_before>#!/home/pi/.virtualenvs/sunlight_rfid_doorman/bin/python

import time
import re
import sys
import itertools

import serial
import envoy

from settings import *
from acl import *

QUIET = '--quiet' in map(lambda x: x.strip(), sys.argv)

def allow(ser, user_id):
	ser.write('G')
	envoy.run('ssh gatekeeper /usr/bin/python /root/cycle.py')
	if not QUIET:
		print '[PASS] %s | %s' % (datetime.datetime.now().isoformat(), user_id)
	log((time.time(), user_id, 'PASS'))

def deny(ser, rfid_serial):
	ser.write('R')
	if not QUIET:
		print '[DENY] %s | %s' % (datetime.datetime.now().isoformat(), rfid_serial)
	log((time.time(), rfid_serial, 'DENY'))

def main():

	denied_rfid_serials = []

	refresh_access_control_list()

	re_identifiers = re.compile(r'\[(.{10})\]')

	if not QUIET:
		print 'Starting...'

	# open serial connection
	ser = serial.Serial(SERIAL_PORT, SERIAL_RATE)

	# wait for Arduino to restart due to serial connection
	time.sleep(2)

	if not QUIET:
		print 'Ready.'
		print ('-' * 60)

	while True:
		line = ser.readline()
		acl = get_access_control_list() # refresh from redis

		# pull the RFID serial number from the serial input
		match = re_identifiers.search(line)
		if match is not None:
			rfid_serial = match.group(1)
			if rfid_serial in acl.keys():
				allow(ser, acl[rfid_serial])
				denied_rfid_serials = []
			else:
				# is this the first time this serial has failed?
				# if so, check for an updated ACL & rerun check
				if rfid_serial not in denied_rfid_serials:
					ser.write('Y') # signal we're thinking...

					# pull a fresh ACL
					try:
						refresh_access_control_list()
					except:
						pass
					acl = get_access_control_list()

					# recheck
					if rfid_serial in acl.keys():
						allow(ser, acl[rfid_serial])
						denied_rfid_serials = []
					else:
						deny(ser, rfid_serial)
						denied_rfid_serials.append(rfid_serial)

				else:
					deny(ser, rfid_serial)
					denied_rfid_serials.append(rfid_serial)

if __name__ == '__main__':
	main()
<commit_msg>start ssh process before LED for better user experience<commit_after>#!/home/pi/.virtualenvs/sunlight_rfid_doorman/bin/python

import time
import re
import sys
import itertools

import serial
import envoy

from settings import *
from acl import *

QUIET = '--quiet' in map(lambda x: x.strip(), sys.argv)

def allow(ser, user_id):
	envoy.run('ssh gatekeeper /usr/bin/python /root/cycle.py')
	ser.write('G')
	if not QUIET:
		print '[PASS] %s | %s' % (datetime.datetime.now().isoformat(), user_id)
	log((time.time(), user_id, 'PASS'))

def deny(ser, rfid_serial):
	ser.write('R')
	if not QUIET:
		print '[DENY] %s | %s' % (datetime.datetime.now().isoformat(), rfid_serial)
	log((time.time(), rfid_serial, 'DENY'))

def main():

	denied_rfid_serials = []

	refresh_access_control_list()

	re_identifiers = re.compile(r'\[(.{10})\]')

	if not QUIET:
		print 'Starting...'

	# open serial connection
	ser = serial.Serial(SERIAL_PORT, SERIAL_RATE)

	# wait for Arduino to restart due to serial connection
	time.sleep(2)

	if not QUIET:
		print 'Ready.'
		print ('-' * 60)

	while True:
		line = ser.readline()
		acl = get_access_control_list() # refresh from redis

		# pull the RFID serial number from the serial input
		match = re_identifiers.search(line)
		if match is not None:
			rfid_serial = match.group(1)
			if rfid_serial in acl.keys():
				allow(ser, acl[rfid_serial])
				denied_rfid_serials = []
			else:
				# is this the first time this serial has failed?
				# if so, check for an updated ACL & rerun check
				if rfid_serial not in denied_rfid_serials:
					ser.write('Y') # signal we're thinking...

					# pull a fresh ACL
					try:
						refresh_access_control_list()
					except:
						pass
					acl = get_access_control_list()

					# recheck
					if rfid_serial in acl.keys():
						allow(ser, acl[rfid_serial])
						denied_rfid_serials = []
					else:
						deny(ser, rfid_serial)
						denied_rfid_serials.append(rfid_serial)

				else:
					deny(ser, rfid_serial)
					denied_rfid_serials.append(rfid_serial)

if __name__ == '__main__':
	main()
<|endoftext|>


<commit_before>"""
Timsort implementation.  Mostly adapted from CPython's listobject.c.

For more information, see listsort.txt in CPython's source tree.
"""

from __future__ import print_function, absolute_import, division

def LT(a, b):
    """
    Trivial comparison function between two keys.  This is factored out to
    make it clear where comparison occurs.
    """
    return a < b

def binarysort(keys, values, lo, hi, start):
    """
    binarysort is the best method for sorting small arrays: it does
    few compares, but can do data movement quadratic in the number of
    elements.
    [lo, hi) is a contiguous slice of a list, and is sorted via
    binary insertion.  This sort is stable.
    On entry, must have lo <= start <= hi, and that [lo, start) is already
    sorted (pass start == lo if you don't know!).
    """
    assert lo <= start and start <= hi, "Bad input for binarysort()"
    if lo == start:
        start += 1
    while start < hi:
        pivot = keys[start]
        # Bisect to find where to insert `pivot`
        l = lo
        r = start
        # Invariants:
        # pivot >= all in [lo, l).
        # pivot  < all in [r, start).
        # The second is vacuously true at the start.
        while l < r:
            p = l + ((r - l) >> 1)
            if LT(pivot, keys[p]):
                r = p
            else:
                l = p+1

        # The invariants still hold, so pivot >= all in [lo, l) and
        # pivot < all in [l, start), so pivot belongs at l.  Note
        # that if there are elements equal to pivot, l points to the
        # first slot after them -- that's why this sort is stable.
        # Slide over to make room (aka memmove()).
        for p in range(start, l, -1):
            keys[p] = keys[p - 1]
        keys[l] = pivot
        if values:
            pivot_val = values[start]
            for p in range(start, l, -1):
                values[p] = values[p - 1]
            values[l] = pivot_val

        start += 1

def count_run(keys, lo, hi):
    """
    Return the length of the run beginning at lo, in the slice [lo, hi).
    lo < hi is required on entry.  "A run" is the longest ascending sequence, with

        lo[0] <= lo[1] <= lo[2] <= ...

    or the longest descending sequence, with

        lo[0] > lo[1] > lo[2] > ...

    A tuple (length, descending) is returned, where boolean *descending*
    is set to 0 in the former case, or to 1 in the latter.
    For its intended use in a stable mergesort, the strictness of the defn of
    "descending" is needed so that the caller can safely reverse a descending
    sequence without violating stability (strict > ensures there are no equal
    elements to get out of order).
    """
    assert lo < hi, "Bad input for count_run()"
    if lo + 1 == hi:
        # Trivial 1-long run
        return 1, False
    if LT(keys[lo + 1], keys[lo]):
        # Descending run
        for k in range(lo + 2, hi):
            if not LT(keys[k], keys[k - 1]):
                return k - lo, True
        return hi - lo, True
    else:
        # Ascending run
        for k in range(lo + 2, hi):
            if LT(keys[k], keys[k - 1]):
                return k - lo, False
        return hi - lo, False

def gallop_left(key, a, start, stop, hint):
    """
    Locate the proper position of key in a sorted vector; if the vector contains
    an element equal to key, return the position immediately to the left of
    the leftmost equal element.  [gallop_right() does the same except returns
    the position to the right of the rightmost equal element (if any).]

    "a" is a sorted vector with stop elements, starting at a[start].
    stop must be > start.

    "hint" is an index at which to begin the search, start <= hint < stop.
    The closer hint is to the final result, the faster this runs.

    The return value is the int k in start..stop such that

        a[k-1] < key <= a[k]

    pretending that a[start-1] is minus infinity and a[stop] is plus infinity.
    IOW, key belongs at index k; or, IOW, the first k elements of a should
    precede key, and the last stop-start-k should follow key.

    See listsort.txt for info on the method.
    """
    assert stop > start, "gallop_left(): stop <= start"
    assert hint >= start and hint < stop, "gallop_left(): hint not in [start, stop)"
    n = stop - start

    # First, gallop from the hint to find a "good" subinterval for bisecting
    lastofs = 0
    ofs = 1
    if LT(a[hint], key):
        # a[hint] < key => gallop right, until
        #                  a[hint + lastofs] < key <= a[hint + ofs]
        maxofs = stop - hint
        while ofs < maxofs:
            if LT(a[hint + ofs], key):
                lastofs = ofs
                ofs = (ofs << 1) + 1
                if ofs <= 0:
                    # Int overflow
                    ofs = maxofs
            else:
                # key <= a[hint + ofs]
                break
        if ofs > maxofs:
            ofs = maxofs
        # Translate back to offsets relative to a[0]
        lastofs += hint
        ofs += hint
    else:
        # key <= a[hint] => gallop left, until
        #                   a[hint - ofs] < key <= a[hint - lastofs]
        maxofs = hint - start + 1
        while ofs < maxofs:
            if LT(a[hint - ofs], key):
                break
            else:
                # key <= a[hint - ofs]
                lastofs = ofs
                ofs = (ofs << 1) + 1
                if ofs <= 0:
                    # Int overflow
                    ofs = maxofs
        if ofs > maxofs:
            ofs = maxofs
        # Translate back to positive offsets relative to a[0]
        lastofs, ofs = hint - ofs, hint - lastofs

    assert start - 1 <= lastofs and lastofs < ofs and ofs <= stop
    # Now a[lastofs] < key <= a[ofs], so key belongs somewhere to the
    # right of lastofs but no farther right than ofs.  Do a binary
    # search, with invariant a[lastofs-1] < key <= a[ofs].
    lastofs += 1
    while lastofs < ofs:
        m = lastofs + ((ofs - lastofs) >> 1)
        if LT(a[m], key):
            # a[m] < key
            lastofs = m + 1
        else:
            # key <= a[m]
            ofs = m
    # Now lastofs == ofs, so a[ofs - 1] < key <= a[ofs]
    return ofs

def gallop_right(key, a, start, stop, hint):
    """
    Exactly like gallop_left(), except that if key already exists in a[start:stop],
    finds the position immediately to the right of the rightmost equal value.

    The return value is the int k in start..stop such that

        a[k-1] <= key < a[k]

    The code duplication is massive, but this is enough different given that
    we're sticking to "<" comparisons that it's much harder to follow if
    written as one routine with yet another "left or right?" flag.
    """
    assert stop > start, "gallop_right(): stop <= start"
    assert hint >= start and hint < stop, "gallop_right(): hint not in [start, stop)"
    n = stop - start

    # First, gallop from the hint to find a "good" subinterval for bisecting
    lastofs = 0
    ofs = 1
    if LT(key, a[hint]):
        # key < a[hint] => gallop left, until
        #                  a[hint - ofs] <= key < a[hint - lastofs]
        maxofs = hint - start + 1
        while ofs < maxofs:
            if LT(key, a[hint - ofs]):
                lastofs = ofs
                ofs = (ofs << 1) + 1
                if ofs <= 0:
                    # Int overflow
                    ofs = maxofs
            else:
                # a[hint - ofs] <= key
                break
        if ofs > maxofs:
            ofs = maxofs
        # Translate back to positive offsets relative to a[0]
        lastofs, ofs = hint - ofs, hint - lastofs
    else:
        # a[hint] <= key -- gallop right, until
        # a[hint + lastofs] <= key < a[hint + ofs]
        maxofs = stop - hint
        while ofs < maxofs:
            if LT(key, a[hint + ofs]):
                break
            else:
                # a[hint + ofs] <= key
                lastofs = ofs
                ofs = (ofs << 1) + 1
                if ofs <= 0:
                    # Int overflow
                    ofs = maxofs
        if ofs > maxofs:
            ofs = maxofs
        # Translate back to offsets relative to a[0]
        lastofs += hint
        ofs += hint

    assert start - 1 <= lastofs and lastofs < ofs and ofs <= stop
    # Now a[lastofs] <= key < a[ofs], so key belongs somewhere to the
    # right of lastofs but no farther right than ofs.  Do a binary
    # search, with invariant a[lastofs-1] <= key < a[ofs].
    lastofs += 1
    while lastofs < ofs:
        m = lastofs + ((ofs - lastofs) >> 1)
        if LT(key, a[m]):
            # key < a[m]
            ofs = m
        else:
            # a[m] <= key
            lastofs = m + 1
    # Now lastofs == ofs, so a[ofs - 1] <= key < a[ofs]
    return ofs

def merge_compute_minrun(n):
    """
    Compute a good value for the minimum run length; natural runs shorter
    than this are boosted artificially via binary insertion.

    If n < 64, return n (it's too small to bother with fancy stuff).
    Else if n is an exact power of 2, return 32.
    Else return an int k, 32 <= k <= 64, such that n/k is close to, but
    strictly less than, an exact power of 2.

    See listsort.txt for more info.
    """
    r = 0
    assert n >= 0
    while n >= 64:
        r |= n & 1
        n >>= 1
    return n + r
<commit_msg>Add comments<commit_after>"""
Timsort implementation.  Mostly adapted from CPython's listobject.c.

For more information, see listsort.txt in CPython's source tree.
"""

from __future__ import print_function, absolute_import, division

def LT(a, b):
    """
    Trivial comparison function between two keys.  This is factored out to
    make it clear where comparison occurs.
    """
    return a < b

def binarysort(keys, values, lo, hi, start):
    """
    binarysort is the best method for sorting small arrays: it does
    few compares, but can do data movement quadratic in the number of
    elements.
    [lo, hi) is a contiguous slice of a list, and is sorted via
    binary insertion.  This sort is stable.
    On entry, must have lo <= start <= hi, and that [lo, start) is already
    sorted (pass start == lo if you don't know!).
    """
    assert lo <= start and start <= hi, "Bad input for binarysort()"
    if lo == start:
        start += 1
    while start < hi:
        pivot = keys[start]
        # Bisect to find where to insert `pivot`
        # NOTE: bisection only wins over linear search if the comparison
        # function is much more expensive than simply moving data.
        l = lo
        r = start
        # Invariants:
        # pivot >= all in [lo, l).
        # pivot  < all in [r, start).
        # The second is vacuously true at the start.
        while l < r:
            p = l + ((r - l) >> 1)
            if LT(pivot, keys[p]):
                r = p
            else:
                l = p+1

        # The invariants still hold, so pivot >= all in [lo, l) and
        # pivot < all in [l, start), so pivot belongs at l.  Note
        # that if there are elements equal to pivot, l points to the
        # first slot after them -- that's why this sort is stable.
        # Slide over to make room (aka memmove()).
        for p in range(start, l, -1):
            keys[p] = keys[p - 1]
        keys[l] = pivot
        if values:
            pivot_val = values[start]
            for p in range(start, l, -1):
                values[p] = values[p - 1]
            values[l] = pivot_val

        start += 1

def count_run(keys, lo, hi):
    """
    Return the length of the run beginning at lo, in the slice [lo, hi).
    lo < hi is required on entry.  "A run" is the longest ascending sequence, with

        lo[0] <= lo[1] <= lo[2] <= ...

    or the longest descending sequence, with

        lo[0] > lo[1] > lo[2] > ...

    A tuple (length, descending) is returned, where boolean *descending*
    is set to 0 in the former case, or to 1 in the latter.
    For its intended use in a stable mergesort, the strictness of the defn of
    "descending" is needed so that the caller can safely reverse a descending
    sequence without violating stability (strict > ensures there are no equal
    elements to get out of order).
    """
    assert lo < hi, "Bad input for count_run()"
    if lo + 1 == hi:
        # Trivial 1-long run
        return 1, False
    if LT(keys[lo + 1], keys[lo]):
        # Descending run
        for k in range(lo + 2, hi):
            if not LT(keys[k], keys[k - 1]):
                return k - lo, True
        return hi - lo, True
    else:
        # Ascending run
        for k in range(lo + 2, hi):
            if LT(keys[k], keys[k - 1]):
                return k - lo, False
        return hi - lo, False

def gallop_left(key, a, start, stop, hint):
    """
    Locate the proper position of key in a sorted vector; if the vector contains
    an element equal to key, return the position immediately to the left of
    the leftmost equal element.  [gallop_right() does the same except returns
    the position to the right of the rightmost equal element (if any).]

    "a" is a sorted vector with stop elements, starting at a[start].
    stop must be > start.

    "hint" is an index at which to begin the search, start <= hint < stop.
    The closer hint is to the final result, the faster this runs.

    The return value is the int k in start..stop such that

        a[k-1] < key <= a[k]

    pretending that a[start-1] is minus infinity and a[stop] is plus infinity.
    IOW, key belongs at index k; or, IOW, the first k elements of a should
    precede key, and the last stop-start-k should follow key.

    See listsort.txt for info on the method.
    """
    assert stop > start, "gallop_left(): stop <= start"
    assert hint >= start and hint < stop, "gallop_left(): hint not in [start, stop)"
    n = stop - start

    # First, gallop from the hint to find a "good" subinterval for bisecting
    lastofs = 0
    ofs = 1
    if LT(a[hint], key):
        # a[hint] < key => gallop right, until
        #                  a[hint + lastofs] < key <= a[hint + ofs]
        maxofs = stop - hint
        while ofs < maxofs:
            if LT(a[hint + ofs], key):
                lastofs = ofs
                ofs = (ofs << 1) + 1
                if ofs <= 0:
                    # Int overflow
                    ofs = maxofs
            else:
                # key <= a[hint + ofs]
                break
        if ofs > maxofs:
            ofs = maxofs
        # Translate back to offsets relative to a[0]
        lastofs += hint
        ofs += hint
    else:
        # key <= a[hint] => gallop left, until
        #                   a[hint - ofs] < key <= a[hint - lastofs]
        maxofs = hint - start + 1
        while ofs < maxofs:
            if LT(a[hint - ofs], key):
                break
            else:
                # key <= a[hint - ofs]
                lastofs = ofs
                ofs = (ofs << 1) + 1
                if ofs <= 0:
                    # Int overflow
                    ofs = maxofs
        if ofs > maxofs:
            ofs = maxofs
        # Translate back to positive offsets relative to a[0]
        lastofs, ofs = hint - ofs, hint - lastofs

    assert start - 1 <= lastofs and lastofs < ofs and ofs <= stop
    # Now a[lastofs] < key <= a[ofs], so key belongs somewhere to the
    # right of lastofs but no farther right than ofs.  Do a binary
    # search, with invariant a[lastofs-1] < key <= a[ofs].
    lastofs += 1
    while lastofs < ofs:
        m = lastofs + ((ofs - lastofs) >> 1)
        if LT(a[m], key):
            # a[m] < key
            lastofs = m + 1
        else:
            # key <= a[m]
            ofs = m
    # Now lastofs == ofs, so a[ofs - 1] < key <= a[ofs]
    return ofs

def gallop_right(key, a, start, stop, hint):
    """
    Exactly like gallop_left(), except that if key already exists in a[start:stop],
    finds the position immediately to the right of the rightmost equal value.

    The return value is the int k in start..stop such that

        a[k-1] <= key < a[k]

    The code duplication is massive, but this is enough different given that
    we're sticking to "<" comparisons that it's much harder to follow if
    written as one routine with yet another "left or right?" flag.
    """
    assert stop > start, "gallop_right(): stop <= start"
    assert hint >= start and hint < stop, "gallop_right(): hint not in [start, stop)"
    n = stop - start

    # First, gallop from the hint to find a "good" subinterval for bisecting
    lastofs = 0
    ofs = 1
    if LT(key, a[hint]):
        # key < a[hint] => gallop left, until
        #                  a[hint - ofs] <= key < a[hint - lastofs]
        maxofs = hint - start + 1
        while ofs < maxofs:
            if LT(key, a[hint - ofs]):
                lastofs = ofs
                ofs = (ofs << 1) + 1
                if ofs <= 0:
                    # Int overflow
                    ofs = maxofs
            else:
                # a[hint - ofs] <= key
                break
        if ofs > maxofs:
            ofs = maxofs
        # Translate back to positive offsets relative to a[0]
        lastofs, ofs = hint - ofs, hint - lastofs
    else:
        # a[hint] <= key -- gallop right, until
        # a[hint + lastofs] <= key < a[hint + ofs]
        maxofs = stop - hint
        while ofs < maxofs:
            if LT(key, a[hint + ofs]):
                break
            else:
                # a[hint + ofs] <= key
                lastofs = ofs
                ofs = (ofs << 1) + 1
                if ofs <= 0:
                    # Int overflow
                    ofs = maxofs
        if ofs > maxofs:
            ofs = maxofs
        # Translate back to offsets relative to a[0]
        lastofs += hint
        ofs += hint

    assert start - 1 <= lastofs and lastofs < ofs and ofs <= stop
    # Now a[lastofs] <= key < a[ofs], so key belongs somewhere to the
    # right of lastofs but no farther right than ofs.  Do a binary
    # search, with invariant a[lastofs-1] <= key < a[ofs].
    lastofs += 1
    while lastofs < ofs:
        m = lastofs + ((ofs - lastofs) >> 1)
        if LT(key, a[m]):
            # key < a[m]
            ofs = m
        else:
            # a[m] <= key
            lastofs = m + 1
    # Now lastofs == ofs, so a[ofs - 1] <= key < a[ofs]
    return ofs

def merge_compute_minrun(n):
    """
    Compute a good value for the minimum run length; natural runs shorter
    than this are boosted artificially via binary insertion.

    If n < 64, return n (it's too small to bother with fancy stuff).
    Else if n is an exact power of 2, return 32.
    Else return an int k, 32 <= k <= 64, such that n/k is close to, but
    strictly less than, an exact power of 2.

    See listsort.txt for more info.
    """
    r = 0
    assert n >= 0
    while n >= 64:
        r |= n & 1
        n >>= 1
    return n + r

# A mergestate is a (min_gallop, temp, pending) tuple

<|endoftext|>


<commit_before>import os

from paste.deploy import loadapp
from waitress import serve

if __name__ == "__main__":
    port = int(os.environ.get("PORT", 5000))
    app = loadapp('config:production.ini', relative_to='.')

    serve(app, host='0.0.0.0', port=port)
<commit_msg>Force the use of https unless the settings says otherwise<commit_after>import os

from paste.deploy import loadapp
from waitress import serve

if __name__ == "__main__":
    port = int(os.environ.get("PORT", 5000))
    scheme = os.environ.get("SCHEME", "https")
    app = loadapp('config:production.ini', relative_to='.')

    serve(app, host='0.0.0.0', port=port, scheme=scheme)
<|endoftext|>


<commit_before>import os
import json
import time
from datetime import datetime

from hitcpy import HITC

import web

from influxclient import saveResult, listSensors, getSensorData

DATE_FORMAT = "%Y-%m-%d %H:%M:%S.%f"
# 2015-12-08 23:12:47.105

HITC_URL = os.environ["HITC"]

urls = (
  "/", "Index",
  "/_models", "Models",
  "/_data/sensors", "SensorsData",
  "/_data/sensor/(.+)/(.+)", "SensorData"
)
app = web.application(urls, globals())
render = web.template.render("templates/")
hitcClient = HITC(HITC_URL)

# Utility functions

def createModelFromDataPoint(modelId, point):
  with open("anomaly_params.json") as inputParams:
    modelSpec = json.loads(inputParams.read())
  modelSpec["guid"] = modelId
  createdModel = hitcClient.create_model(modelSpec)
  print "Created {0}".format(createdModel)

def runOneDataPoint(modelId, point):
  timeString = point["time"]
  timestamp = int(time.mktime(datetime.strptime(timeString, DATE_FORMAT).timetuple()))
  dataRow = {
    "c0": timestamp,
    "c1": point["value"]
  }
  # There is only one value in the result list, so pop() it off.
  return hitcClient.get_model(modelId).run(dataRow).pop()

def getSensorIds(sensors):
  sensorIds = []
  for sensor in sensors:
    name = sensor["name"]
    for tag in sensor["tags"]:
      sensorIds.append(name + "/" + tag["component"])
  return sorted(list(set(sensorIds)))

# HTTP Handlers

class Models:

  def GET(self):
    modelIds = [m.guid for m in hitcClient.get_all_models()]
    return json.dumps(modelIds)

class Index:

  def GET(self):
    modelIds = [m.guid for m in listModels()]
    return json.dumps(modelIds)

class Index:

  def GET(self):
    raise web.seeother('/static/index.html')

  def POST(self):
    data = json.loads(web.data())
    modelIds = [m.guid for m in hitcClient.get_all_models()]
    modelId = data["component"] + '_' +  data["stream"]
    if modelId not in modelIds:
      createModelFromDataPoint(modelId, data)
    htmResult = runOneDataPoint(modelId, data)
    saveResult(htmResult, data)
    return json.dumps({"result": "success"})

class SensorData:

  def GET(self, measurement, component):
    limit = None
    since = None
    query = web.input()
    if "limit" in query:
      limit = query["limit"]
    if "since" in query:
      since = query["since"]
    sensor = getSensorData(measurement, component, limit=limit, since=since)
    return json.dumps(sensor)

class SensorsData:

  def GET(self):
    return json.dumps(getSensorIds(listSensors()))

# Start here

if __name__ == "__main__":
  app.run()
<commit_msg>Fixes #26.<commit_after>import os
import json
import time
from datetime import datetime

from hitcpy import HITC

import web

from influxclient import saveResult, listSensors, getSensorData

DATE_FORMAT = "%Y-%m-%d %H:%M:%S.%f"
# 2015-12-08 23:12:47.105

HITC_URL = os.environ["HITC"]

urls = (
  "/", "Index",
  "/_models", "Models",
  "/_data/sensors", "SensorsData",
  "/_data/sensor/(.+)/(.+)", "SensorData"
)
app = web.application(urls, globals())
render = web.template.render("templates/")
hitcClient = HITC(HITC_URL)

# Utility functions

def createModelFromDataPoint(modelId, point):
  with open("anomaly_params.json") as inputParams:
    modelSpec = json.loads(inputParams.read())
  modelSpec["guid"] = modelId
  createdModel = hitcClient.create_model(modelSpec)
  print "Created {0}".format(createdModel)

def runOneDataPoint(modelId, point):
  timeString = point["time"]
  timestamp = int(time.mktime(datetime.strptime(timeString, DATE_FORMAT).timetuple()))
  dataRow = {
    "c0": timestamp,
    "c1": point["value"]
  }
  # There is only one value in the result list, so pop() it off.
  return hitcClient.get_model(modelId).run(dataRow).pop()

def getSensorIds(sensors):
  sensorIds = []
  for sensor in sensors:
    name = sensor["name"]
    for tag in sensor["tags"]:
      sensorIds.append(name + "/" + tag["component"])
  return sorted(list(set(sensorIds)))

# HTTP Handlers

class Models:

  def GET(self):
    modelIds = [m.guid for m in hitcClient.get_all_models()]
    return json.dumps(modelIds)

class Index:

  def GET(self):
    raise web.seeother('/static/index.html')

  def POST(self):
    data = json.loads(web.data())
    modelIds = [m.guid for m in hitcClient.get_all_models()]
    modelId = data["component"] + '_' +  data["stream"]
    if modelId not in modelIds:
      createModelFromDataPoint(modelId, data)
    htmResult = runOneDataPoint(modelId, data)
    saveResult(htmResult, data)
    return json.dumps({"result": "success"})

class SensorData:

  def GET(self, measurement, component):
    limit = None
    since = None
    query = web.input()
    if "limit" in query:
      limit = query["limit"]
    if "since" in query:
      since = query["since"]
    sensor = getSensorData(measurement, component, limit=limit, since=since)
    return json.dumps(sensor)

class SensorsData:

  def GET(self):
    return json.dumps(getSensorIds(listSensors()))

# Start here

if __name__ == "__main__":
  app.run()
<|endoftext|>


<commit_before>"""
Support for the Withings API.

For more details about this platform, please refer to the documentation at
"""
from __future__ import annotations

import asyncio

from aiohttp.web import Request, Response
import voluptuous as vol
from withings_api.common import NotifyAppli

from homeassistant.components import webhook
from homeassistant.components.application_credentials import (
    ClientCredential,
    async_import_client_credential,
)
from homeassistant.components.webhook import (
    async_unregister as async_unregister_webhook,
)
from homeassistant.config_entries import ConfigEntry
from homeassistant.const import (
    CONF_CLIENT_ID,
    CONF_CLIENT_SECRET,
    CONF_WEBHOOK_ID,
    Platform,
)
from homeassistant.core import HomeAssistant, callback
from homeassistant.helpers import config_validation as cv
from homeassistant.helpers.event import async_call_later
from homeassistant.helpers.typing import ConfigType

from . import const
from .common import (
    _LOGGER,
    async_get_data_manager,
    async_remove_data_manager,
    get_data_manager_by_webhook_id,
    json_message_response,
)

DOMAIN = const.DOMAIN
PLATFORMS = [Platform.BINARY_SENSOR, Platform.SENSOR]

CONFIG_SCHEMA = vol.Schema(
    {
        DOMAIN: vol.All(
            cv.deprecated(const.CONF_PROFILES),
            cv.deprecated(CONF_CLIENT_ID),
            cv.deprecated(CONF_CLIENT_SECRET),
            vol.Schema(
                {
                    vol.Optional(CONF_CLIENT_ID): vol.All(cv.string, vol.Length(min=1)),
                    vol.Optional(CONF_CLIENT_SECRET): vol.All(
                        cv.string, vol.Length(min=1)
                    ),
                    vol.Optional(const.CONF_USE_WEBHOOK, default=False): cv.boolean,
                    vol.Optional(const.CONF_PROFILES): vol.All(
                        cv.ensure_list,
                        vol.Unique(),
                        vol.Length(min=1),
                        [vol.All(cv.string, vol.Length(min=1))],
                    ),
                }
            ),
        )
    },
    extra=vol.ALLOW_EXTRA,
)

async def async_setup(hass: HomeAssistant, config: ConfigType) -> bool:
    """Set up the Withings component."""
    conf = config.get(DOMAIN, {})
    if not (conf := config.get(DOMAIN, {})):
        return True

    # Make the config available to the oauth2 config flow.
    hass.data[DOMAIN] = {const.CONFIG: conf}

    # Setup the oauth2 config flow.
    if CONF_CLIENT_ID in conf:
        await async_import_client_credential(
            hass,
            DOMAIN,
            ClientCredential(
                conf[CONF_CLIENT_ID],
                conf[CONF_CLIENT_SECRET],
            ),
        )
        _LOGGER.warning(
            "Configuration of Withings integration OAuth2 credentials in YAML "
            "is deprecated and will be removed in a future release; Your "
            "existing OAuth Application Credentials have been imported into "
            "the UI automatically and can be safely removed from your "
            "configuration.yaml file"
        )

    return True

async def async_setup_entry(hass: HomeAssistant, entry: ConfigEntry) -> bool:
    """Set up Withings from a config entry."""
    config_updates = {}

    # Add a unique id if it's an older config entry.
    if entry.unique_id != entry.data["token"]["userid"] or not isinstance(
        entry.unique_id, str
    ):
        config_updates["unique_id"] = str(entry.data["token"]["userid"])

    # Add the webhook configuration.
    if CONF_WEBHOOK_ID not in entry.data:
        webhook_id = webhook.async_generate_id()
        config_updates["data"] = {
            **entry.data,
            **{
                const.CONF_USE_WEBHOOK: hass.data[DOMAIN][const.CONFIG][
                    const.CONF_USE_WEBHOOK
                ],
                CONF_WEBHOOK_ID: webhook_id,
            },
        }

    if config_updates:
        hass.config_entries.async_update_entry(entry, **config_updates)

    data_manager = await async_get_data_manager(hass, entry)

    _LOGGER.debug("Confirming %s is authenticated to withings", data_manager.profile)
    await data_manager.poll_data_update_coordinator.async_config_entry_first_refresh()

    webhook.async_register(
        hass,
        const.DOMAIN,
        "Withings notify",
        data_manager.webhook_config.id,
        async_webhook_handler,
    )

    # Perform first webhook subscription check.
    if data_manager.webhook_config.enabled:
        data_manager.async_start_polling_webhook_subscriptions()

        @callback
        def async_call_later_callback(now) -> None:
            hass.async_create_task(
                data_manager.subscription_update_coordinator.async_refresh()
            )

        # Start subscription check in the background, outside this component's setup.
        async_call_later(hass, 1, async_call_later_callback)

    hass.config_entries.async_setup_platforms(entry, PLATFORMS)

    return True

async def async_unload_entry(hass: HomeAssistant, entry: ConfigEntry) -> bool:
    """Unload Withings config entry."""
    data_manager = await async_get_data_manager(hass, entry)
    data_manager.async_stop_polling_webhook_subscriptions()

    async_unregister_webhook(hass, data_manager.webhook_config.id)

    await asyncio.gather(
        data_manager.async_unsubscribe_webhook(),
        hass.config_entries.async_unload_platforms(entry, PLATFORMS),
    )

    async_remove_data_manager(hass, entry)

    return True

async def async_webhook_handler(
    hass: HomeAssistant, webhook_id: str, request: Request
) -> Response | None:
    """Handle webhooks calls."""
    # Handle http head calls to the path.
    # When creating a notify subscription, Withings will check that the endpoint is running by sending a HEAD request.
    if request.method.upper() == "HEAD":
        return Response()

    if request.method.upper() != "POST":
        return json_message_response("Invalid method", message_code=2)

    # Handle http post calls to the path.
    if not request.body_exists:
        return json_message_response("No request body", message_code=12)

    params = await request.post()

    if "appli" not in params:
        return json_message_response("Parameter appli not provided", message_code=20)

    try:
        appli = NotifyAppli(int(params.getone("appli")))
    except ValueError:
        return json_message_response("Invalid appli provided", message_code=21)

    data_manager = get_data_manager_by_webhook_id(hass, webhook_id)
    if not data_manager:
        _LOGGER.error(
            "Webhook id %s not handled by data manager. This is a bug and should be reported",
            webhook_id,
        )
        return json_message_response("User not found", message_code=1)

    # Run this in the background and return immediately.
    hass.async_create_task(data_manager.async_webhook_data_updated(appli))

    return json_message_response("Success", message_code=0)
<commit_msg>Guard withings accessing hass.data without it being set (#73454)<commit_after>"""
Support for the Withings API.

For more details about this platform, please refer to the documentation at
"""
from __future__ import annotations

import asyncio

from aiohttp.web import Request, Response
import voluptuous as vol
from withings_api.common import NotifyAppli

from homeassistant.components import webhook
from homeassistant.components.application_credentials import (
    ClientCredential,
    async_import_client_credential,
)
from homeassistant.components.webhook import (
    async_unregister as async_unregister_webhook,
)
from homeassistant.config_entries import ConfigEntry
from homeassistant.const import (
    CONF_CLIENT_ID,
    CONF_CLIENT_SECRET,
    CONF_WEBHOOK_ID,
    Platform,
)
from homeassistant.core import HomeAssistant, callback
from homeassistant.helpers import config_validation as cv
from homeassistant.helpers.event import async_call_later
from homeassistant.helpers.typing import ConfigType

from . import const
from .common import (
    _LOGGER,
    async_get_data_manager,
    async_remove_data_manager,
    get_data_manager_by_webhook_id,
    json_message_response,
)

DOMAIN = const.DOMAIN
PLATFORMS = [Platform.BINARY_SENSOR, Platform.SENSOR]

CONFIG_SCHEMA = vol.Schema(
    {
        DOMAIN: vol.All(
            cv.deprecated(const.CONF_PROFILES),
            cv.deprecated(CONF_CLIENT_ID),
            cv.deprecated(CONF_CLIENT_SECRET),
            vol.Schema(
                {
                    vol.Optional(CONF_CLIENT_ID): vol.All(cv.string, vol.Length(min=1)),
                    vol.Optional(CONF_CLIENT_SECRET): vol.All(
                        cv.string, vol.Length(min=1)
                    ),
                    vol.Optional(const.CONF_USE_WEBHOOK, default=False): cv.boolean,
                    vol.Optional(const.CONF_PROFILES): vol.All(
                        cv.ensure_list,
                        vol.Unique(),
                        vol.Length(min=1),
                        [vol.All(cv.string, vol.Length(min=1))],
                    ),
                }
            ),
        )
    },
    extra=vol.ALLOW_EXTRA,
)

async def async_setup(hass: HomeAssistant, config: ConfigType) -> bool:
    """Set up the Withings component."""
    if not (conf := config.get(DOMAIN)):
        # Apply the defaults.
        conf = CONFIG_SCHEMA({DOMAIN: {}})[DOMAIN]
        hass.data[DOMAIN] = {const.CONFIG: conf}
        return True

    hass.data[DOMAIN] = {const.CONFIG: conf}

    # Setup the oauth2 config flow.
    if CONF_CLIENT_ID in conf:
        await async_import_client_credential(
            hass,
            DOMAIN,
            ClientCredential(
                conf[CONF_CLIENT_ID],
                conf[CONF_CLIENT_SECRET],
            ),
        )
        _LOGGER.warning(
            "Configuration of Withings integration OAuth2 credentials in YAML "
            "is deprecated and will be removed in a future release; Your "
            "existing OAuth Application Credentials have been imported into "
            "the UI automatically and can be safely removed from your "
            "configuration.yaml file"
        )

    return True

async def async_setup_entry(hass: HomeAssistant, entry: ConfigEntry) -> bool:
    """Set up Withings from a config entry."""
    config_updates = {}

    # Add a unique id if it's an older config entry.
    if entry.unique_id != entry.data["token"]["userid"] or not isinstance(
        entry.unique_id, str
    ):
        config_updates["unique_id"] = str(entry.data["token"]["userid"])

    # Add the webhook configuration.
    if CONF_WEBHOOK_ID not in entry.data:
        webhook_id = webhook.async_generate_id()
        config_updates["data"] = {
            **entry.data,
            **{
                const.CONF_USE_WEBHOOK: hass.data[DOMAIN][const.CONFIG][
                    const.CONF_USE_WEBHOOK
                ],
                CONF_WEBHOOK_ID: webhook_id,
            },
        }

    if config_updates:
        hass.config_entries.async_update_entry(entry, **config_updates)

    data_manager = await async_get_data_manager(hass, entry)

    _LOGGER.debug("Confirming %s is authenticated to withings", data_manager.profile)
    await data_manager.poll_data_update_coordinator.async_config_entry_first_refresh()

    webhook.async_register(
        hass,
        const.DOMAIN,
        "Withings notify",
        data_manager.webhook_config.id,
        async_webhook_handler,
    )

    # Perform first webhook subscription check.
    if data_manager.webhook_config.enabled:
        data_manager.async_start_polling_webhook_subscriptions()

        @callback
        def async_call_later_callback(now) -> None:
            hass.async_create_task(
                data_manager.subscription_update_coordinator.async_refresh()
            )

        # Start subscription check in the background, outside this component's setup.
        async_call_later(hass, 1, async_call_later_callback)

    hass.config_entries.async_setup_platforms(entry, PLATFORMS)

    return True

async def async_unload_entry(hass: HomeAssistant, entry: ConfigEntry) -> bool:
    """Unload Withings config entry."""
    data_manager = await async_get_data_manager(hass, entry)
    data_manager.async_stop_polling_webhook_subscriptions()

    async_unregister_webhook(hass, data_manager.webhook_config.id)

    await asyncio.gather(
        data_manager.async_unsubscribe_webhook(),
        hass.config_entries.async_unload_platforms(entry, PLATFORMS),
    )

    async_remove_data_manager(hass, entry)

    return True

async def async_webhook_handler(
    hass: HomeAssistant, webhook_id: str, request: Request
) -> Response | None:
    """Handle webhooks calls."""
    # Handle http head calls to the path.
    # When creating a notify subscription, Withings will check that the endpoint is running by sending a HEAD request.
    if request.method.upper() == "HEAD":
        return Response()

    if request.method.upper() != "POST":
        return json_message_response("Invalid method", message_code=2)

    # Handle http post calls to the path.
    if not request.body_exists:
        return json_message_response("No request body", message_code=12)

    params = await request.post()

    if "appli" not in params:
        return json_message_response("Parameter appli not provided", message_code=20)

    try:
        appli = NotifyAppli(int(params.getone("appli")))
    except ValueError:
        return json_message_response("Invalid appli provided", message_code=21)

    data_manager = get_data_manager_by_webhook_id(hass, webhook_id)
    if not data_manager:
        _LOGGER.error(
            "Webhook id %s not handled by data manager. This is a bug and should be reported",
            webhook_id,
        )
        return json_message_response("User not found", message_code=1)

    # Run this in the background and return immediately.
    hass.async_create_task(data_manager.async_webhook_data_updated(appli))

    return json_message_response("Success", message_code=0)
<|endoftext|>