<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-

import os, sys
import subprocess, threading
import logging as loggers
import time
loggers.basicConfig(level=loggers.INFO)
logging = loggers.getLogger(__name__)

class ParallelExecutor(object):
    def __init__(self, cmd):
        self.cmd = cmd
        self.process = None
        self.stdout_output = ""
        self.stderr_output = ""

    def run(self, timeout):
        def target():
            logging.info(self.cmd)
            self.process = subprocess.Popen(self.cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
            self.stdout_output, self.stderr_output = self.process.communicate()
            logging.info("Thread finished")

        thread = threading.Thread(target=target)
        thread.daemon = True
        thread.start()

        thread.join(timeout)
        if thread.is_alive():
            logging.info("Time out, terminating process")
            self.process.terminate()
            time.sleep(1)
            try:
                self.process.kill()
            except OSError:
                pass
            # thread.join()
        try:
            thread._stop()
        except:
            pass
        return self.process.returncode

ERROR_KEYWORD = "Traceback (most recent call last)"

def test_experiment(path, timeout=60):
    path = os.path.join("experiments", path)
    executor = ParallelExecutor("python %s" % path)
    executor.run(timeout=timeout)
    if executor.stderr_output == None:
        logging.info("stderr is none")
        return
    logging.info("stdout:")
    logging.info(executor.stdout_output)
    if ERROR_KEYWORD in executor.stderr_output:
        logging.info("stderr:")
        logging.info(executor.stderr_output)
    else:
        logging.info("No error was found")
    assert ERROR_KEYWORD not in executor.stderr_output

if __name__ == '__main__':
    # MNIST
    test_experiment("mnist/mlp_dropout.py", timeout=30)
    test_experiment("mnist/deep_convolution.py", timeout=30)
    # Auto-encoders
    test_experiment("auto_encoders/rnn_auto_encoder.py", timeout=120)
    # LMs
    test_experiment("lm/baseline_rnnlm.py", timeout=180)
    # Highway networks
    test_experiment("highway_networks/mnist_highway.py", timeout=120)
    # CG Training
    test_experiment("scipy_training/mnist_cg.py", timeout=180)
    # Tutorials
    test_experiment("tutorials/tutorial1.py", timeout=120)
    test_experiment("tutorials/tutorial2.py", timeout=120)
    test_experiment("tutorials/tutorial3.py", timeout=120)
    sys.exit(0)
<commit_msg>Output details in the test<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-

import os, sys
import subprocess, threading
import logging as loggers
import time
loggers.basicConfig(level=loggers.INFO)
logging = loggers.getLogger(__name__)

class ParallelExecutor(object):
    def __init__(self, cmd):
        self.cmd = cmd
        self.process = None
        self.stdout_output = ""
        self.stderr_output = ""

    def run(self, timeout):
        def target():
            logging.info(self.cmd)
            self.process = subprocess.Popen(self.cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
            self.stdout_output, self.stderr_output = self.process.communicate()
            logging.info("Thread finished")

        thread = threading.Thread(target=target)
        thread.daemon = True
        thread.start()

        thread.join(timeout)
        if thread.is_alive():
            logging.info("Time out, terminating process")
            self.process.terminate()
            time.sleep(1)
            try:
                self.process.kill()
            except OSError:
                pass
            # thread.join()
        try:
            thread._stop()
        except:
            pass
        return self.process.returncode

ERROR_KEYWORD = "Traceback (most recent call last)"

def test_experiment(path, timeout=60):
    path = os.path.join("experiments", path)
    executor = ParallelExecutor("python %s" % path)
    executor.run(timeout=timeout)
    if executor.stderr_output == None:
        logging.info("stderr is none")
        return
    logging.info(executor.stdout_output)
    logging.info(executor.stderr_output)
    if ERROR_KEYWORD in executor.stderr_output:
        logging.info("------ Error was found ------")
        logging.info(executor.stderr_output)
        logging.info("-----------------------------")
    else:
        logging.info("------ No error was found ------")
    assert ERROR_KEYWORD not in executor.stderr_output

if __name__ == '__main__':
    # MNIST
    test_experiment("mnist/mlp_dropout.py", timeout=30)
    test_experiment("mnist/deep_convolution.py", timeout=30)
    # Auto-encoders
    test_experiment("auto_encoders/rnn_auto_encoder.py", timeout=120)
    # LMs
    test_experiment("lm/baseline_rnnlm.py", timeout=180)
    # Highway networks
    test_experiment("highway_networks/mnist_highway.py", timeout=120)
    # CG Training
    test_experiment("scipy_training/mnist_cg.py", timeout=180)
    # Tutorials
    test_experiment("tutorials/tutorial1.py", timeout=120)
    test_experiment("tutorials/tutorial2.py", timeout=120)
    test_experiment("tutorials/tutorial3.py", timeout=120)
    sys.exit(0)
<|endoftext|>


<commit_before>import selenium.webdriver as webdriver
import lxml
import lxml.html
from bs4 import BeautifulSoup
import requests
from trawler.browsers import exceptions
from selenium.webdriver.common.desired_capabilities import DesiredCapabilities
from trawler.browsers.utils import start_browser
import logging
import random, time
from trawler.settings import AVAILABLE_METHODS, DEFAULT_MAX_PAGES

class BrowserBase(object):
    """
    Base class for making new browser classes like BingBrowser, GoogleBrowser, DuckDuckGoBrowser etc

    USAGE:
        _BASE_URL : https://www.bing.com # this should be changed for each Child class
        _PHANTOMJS_PATH : #phantomjs binary path
        _DEFAULT_METHOD : #default method used to scrape ? selenium-chrome or selenium-htmlunit

    """

    def __init__(self, kw=None, max_page=None, method='selenium-chrome', driver=None):
        """
        Make some quick calculations to proceed with the run
        """

        self._AVAILABLE_SCRAPE_METHODS = AVAILABLE_METHODS

        self._BASE_URL = None
        self._SEARCH_QS = None
        self._SEARCH_TERM = None
        self._SEARCH_URL = self._BASE_URL
        if self._SEARCH_QS: self._SEARCH_URL = self._SEARCH_URL + self._SEARCH_QS
        if self._SEARCH_TERM: self._SEARCH_URL = self._SEARCH_URL + self._SEARCH_TERM
        self._PAUSE_RUN_RANDOMLY = lambda: random.randint(1, 4)

        self._HTML_DATA = None
        self._SOUPED_HTML_DATA = None

        self._RESULTS_MAIN = []
        self._RESULTS_KEYWORDS = []

        self._SEARCH_MAIN_CSS_SELECTOR = None
        self._SEARCH_KEYWORDS_CSS_SELECTOR = None
        self._SEARCH_NEXT_CSS_SELECTOR = None

        self._NEXT_PAGE_URL = None

        self._ITER = 0
        self._ITER_MAX = DEFAULT_MAX_PAGES

        self._SEARCH_TERM = kw

        if max_page:
            self._ITER_MAX = max_page

        self._DEFAULT_SCRAPE_METHOD = method
        if self._DEFAULT_SCRAPE_METHOD in ['selenium-htmlunit', 'selenium-chrome', ]:
            if driver is None:
                self._init_browser_instance()
            else:
                self._DRIVER = driver
        else:
            self._DRIVER = None

        if self._DEFAULT_SCRAPE_METHOD not in self._AVAILABLE_SCRAPE_METHODS:
            raise exceptions.BrowerScrapeMethodNotImplemented('Not implemented')

    def _test_config(self):
        """
        this will check the inputs and executables being in place
        :return:
        """
        logging.debug('testing config')

    def _soup_data(self):
        # return lxml.html.fromstring(self._HTML_DATA)
        return BeautifulSoup(self._HTML_DATA, "lxml")

    def _init_browser_instance(self):
        self._DRIVER = start_browser(self._DEFAULT_SCRAPE_METHOD)

    def evaluate_url(self):
        """
        decide which url to scrape now
        :return:
        """
        if self._NEXT_PAGE_URL:
            return self._NEXT_PAGE_URL
        else:
            return self._SEARCH_URL

    def get_html_with_selenium(self):
        """
        scrapes the html content using requests module

        https://stackoverflow.com/a/18102579/3448851
        :return:
        """
        url = self.evaluate_url()
        self._DRIVER.get(url)
        return self._DRIVER.page_source

    def get_html_with_requests(self):
        """
        scrapes the html content using requests module
        :return:
        """
        url = self.evaluate_url()
        try:
            req = requests.get(url, timeout=10)
            if req.status_code == 200:
                return req.text
            else:
                return None
        except:
            return None

    def get_html(self, method=None):
        if method is None:  method = self.get_current_method()
        if method in ['selenium-htmlunit', 'selenium-chrome', ]: return self.get_html_with_selenium()
        if method == 'requests': return self.get_html_with_requests()

    def dry_run(self):
        """
        This will run a dry run with plain python requests, and check if requests is good enough,
        and if there is some issue, the driver will be switched to Selenium
        :return:
        """
        pass

    def get_current_method(self):
        """
        Returns the current Browser driver being used by this class (requests or selenium)
        :return:
        """
        return self._DEFAULT_SCRAPE_METHOD

    def shift_method(self):
        """
        swaps the current method to other method. If python requests is current method, it will shift to next one, which is
        selenium
        :return:
        """
        index = self._AVAILABLE_SCRAPE_METHODS.index(self._DEFAULT_SCRAPE_METHOD)
        return self._AVAILABLE_SCRAPE_METHODS[index - 1]

    def search(self):
        for i in range(self._ITER_MAX):
            print(self._NEXT_PAGE_URL, self._ITER, self._ITER_MAX, "======")
            if i == 0 or self._NEXT_PAGE_URL:
                self._ITER += 1
                time.sleep(self._PAUSE_RUN_RANDOMLY())
                self.search_single_page(i)

    def search_single_page(self, iter_num=0):
        """
         1. Perform a dry run
         2. shift _DEFAULT_SCRAPE_METHOD if needed
         3. get results
         """

        self.dry_run()
        self._test_config()
        self._HTML_DATA = self.get_html()
        self._SOUPED_HTML_DATA = self._soup_data()
        self._RESULTS_MAIN += self.get_search_results()
        self._RESULTS_KEYWORDS += self.get_related_keywords()
        self._NEXT_PAGE_URL = self._get_next_page()

    @property
    def data(self):
        # make the data unique
        self._RESULTS_MAIN = [dict(y) for y in set(tuple(x.items()) for x in self._RESULTS_MAIN)]
        self._RESULTS_KEYWORDS = [dict(y) for y in set(tuple(x.items()) for x in self._RESULTS_KEYWORDS)]
        return {
            'results': self._RESULTS_MAIN,
            'results_count': len(self._RESULTS_MAIN),
            'related_keywords': self._RESULTS_KEYWORDS,
            'related_keywords_count': len(self._RESULTS_KEYWORDS),
            'next_url': self._NEXT_PAGE_URL
        }

    def _scrape_css_selector(self, selector=None):
        results = self._SOUPED_HTML_DATA.select(selector)
        data = []
        for result in results:
            link = result.get('href').strip() if result.get('href') else None
            datum = {
                'url': link.encode('utf-8') if link.startswith('http') else self._BASE_URL + link,
                'text': result.getText().strip().encode('utf-8') if result.getText() else None
            }
            data.append(datum)
        return data

    def _get_next_page(self):
        """
        :return:
        """
        if self._SEARCH_NEXT_CSS_SELECTOR:
            el = self._SOUPED_HTML_DATA.select(self._SEARCH_NEXT_CSS_SELECTOR)
            if len(el) >= 1:
                el = el[0]
                href = el.get('href').strip()
                if "http://" in href or "https://" in href:
                    return href
                return self._BASE_URL + href
        else:
            return None

    def get_search_results(self):
        return self._scrape_css_selector(self._SEARCH_MAIN_CSS_SELECTOR)

    def get_related_keywords(self):
        if self._SEARCH_KEYWORDS_CSS_SELECTOR:
            return self._scrape_css_selector(self._SEARCH_KEYWORDS_CSS_SELECTOR)
        else:
            return []

    def close(self):
        if self._DRIVER:
            self._DRIVER.close()
<commit_msg>encoding data removed<commit_after>import selenium.webdriver as webdriver
import lxml
import lxml.html
from bs4 import BeautifulSoup
import requests
from trawler.browsers import exceptions
from selenium.webdriver.common.desired_capabilities import DesiredCapabilities
from trawler.browsers.utils import start_browser
import logging
import random, time
from trawler.settings import AVAILABLE_METHODS, DEFAULT_MAX_PAGES

class BrowserBase(object):
    """
    Base class for making new browser classes like BingBrowser, GoogleBrowser, DuckDuckGoBrowser etc

    USAGE:
        _BASE_URL : https://www.bing.com # this should be changed for each Child class
        _PHANTOMJS_PATH : #phantomjs binary path
        _DEFAULT_METHOD : #default method used to scrape ? selenium-chrome or selenium-htmlunit

    """

    def __init__(self, kw=None, max_page=None, method='selenium-chrome', driver=None):
        """
        Make some quick calculations to proceed with the run
        """

        self._AVAILABLE_SCRAPE_METHODS = AVAILABLE_METHODS

        self._BASE_URL = None
        self._SEARCH_QS = None
        self._SEARCH_TERM = None
        self._SEARCH_URL = self._BASE_URL
        if self._SEARCH_QS: self._SEARCH_URL = self._SEARCH_URL + self._SEARCH_QS
        if self._SEARCH_TERM: self._SEARCH_URL = self._SEARCH_URL + self._SEARCH_TERM
        self._PAUSE_RUN_RANDOMLY = lambda: random.randint(1, 4)

        self._HTML_DATA = None
        self._SOUPED_HTML_DATA = None

        self._RESULTS_MAIN = []
        self._RESULTS_KEYWORDS = []

        self._SEARCH_MAIN_CSS_SELECTOR = None
        self._SEARCH_KEYWORDS_CSS_SELECTOR = None
        self._SEARCH_NEXT_CSS_SELECTOR = None

        self._NEXT_PAGE_URL = None

        self._ITER = 0
        self._ITER_MAX = DEFAULT_MAX_PAGES

        self._SEARCH_TERM = kw

        if max_page:
            self._ITER_MAX = max_page

        self._DEFAULT_SCRAPE_METHOD = method
        if self._DEFAULT_SCRAPE_METHOD in ['selenium-htmlunit', 'selenium-chrome', ]:
            if driver is None:
                self._init_browser_instance()
            else:
                self._DRIVER = driver
        else:
            self._DRIVER = None

        if self._DEFAULT_SCRAPE_METHOD not in self._AVAILABLE_SCRAPE_METHODS:
            raise exceptions.BrowerScrapeMethodNotImplemented('Not implemented')

    def _test_config(self):
        """
        this will check the inputs and executables being in place
        :return:
        """
        logging.debug('testing config')

    def _soup_data(self):
        # return lxml.html.fromstring(self._HTML_DATA)
        return BeautifulSoup(self._HTML_DATA, "lxml")

    def _init_browser_instance(self):
        self._DRIVER = start_browser(self._DEFAULT_SCRAPE_METHOD)

    def evaluate_url(self):
        """
        decide which url to scrape now
        :return:
        """
        if self._NEXT_PAGE_URL:
            return self._NEXT_PAGE_URL
        else:
            return self._SEARCH_URL

    def get_html_with_selenium(self):
        """
        scrapes the html content using requests module

        https://stackoverflow.com/a/18102579/3448851
        :return:
        """
        url = self.evaluate_url()
        self._DRIVER.get(url)
        return self._DRIVER.page_source

    def get_html_with_requests(self):
        """
        scrapes the html content using requests module
        :return:
        """
        url = self.evaluate_url()
        try:
            req = requests.get(url, timeout=10)
            if req.status_code == 200:
                return req.text
            else:
                return None
        except:
            return None

    def get_html(self, method=None):
        if method is None:  method = self.get_current_method()
        if method in ['selenium-htmlunit', 'selenium-chrome', ]: return self.get_html_with_selenium()
        if method == 'requests': return self.get_html_with_requests()

    def dry_run(self):
        """
        This will run a dry run with plain python requests, and check if requests is good enough,
        and if there is some issue, the driver will be switched to Selenium
        :return:
        """
        pass

    def get_current_method(self):
        """
        Returns the current Browser driver being used by this class (requests or selenium)
        :return:
        """
        return self._DEFAULT_SCRAPE_METHOD

    def shift_method(self):
        """
        swaps the current method to other method. If python requests is current method, it will shift to next one, which is
        selenium
        :return:
        """
        index = self._AVAILABLE_SCRAPE_METHODS.index(self._DEFAULT_SCRAPE_METHOD)
        return self._AVAILABLE_SCRAPE_METHODS[index - 1]

    def search(self):
        for i in range(self._ITER_MAX):
            print(self._NEXT_PAGE_URL, self._ITER, self._ITER_MAX, "======")
            if i == 0 or self._NEXT_PAGE_URL:
                self._ITER += 1
                time.sleep(self._PAUSE_RUN_RANDOMLY())
                self.search_single_page(i)

    def search_single_page(self, iter_num=0):
        """
         1. Perform a dry run
         2. shift _DEFAULT_SCRAPE_METHOD if needed
         3. get results
         """

        self.dry_run()
        self._test_config()
        self._HTML_DATA = self.get_html()
        self._SOUPED_HTML_DATA = self._soup_data()
        self._RESULTS_MAIN += self.get_search_results()
        self._RESULTS_KEYWORDS += self.get_related_keywords()
        self._NEXT_PAGE_URL = self._get_next_page()

    @property
    def data(self):
        # make the data unique
        self._RESULTS_MAIN = [dict(y) for y in set(tuple(x.items()) for x in self._RESULTS_MAIN)]
        self._RESULTS_KEYWORDS = [dict(y) for y in set(tuple(x.items()) for x in self._RESULTS_KEYWORDS)]
        return {
            'results': self._RESULTS_MAIN,
            'results_count': len(self._RESULTS_MAIN),
            'related_keywords': self._RESULTS_KEYWORDS,
            'related_keywords_count': len(self._RESULTS_KEYWORDS),
            'next_url': self._NEXT_PAGE_URL
        }

    def _scrape_css_selector(self, selector=None):
        results = self._SOUPED_HTML_DATA.select(selector)
        data = []
        for result in results:
            link = result.get('href').strip() if result.get('href') else None
            datum = {
                'url': link if link.startswith('http') else self._BASE_URL + link,
                'text': result.getText().strip() if result.getText() else None
            }
            data.append(datum)
        return data

    def _get_next_page(self):
        """
        :return:
        """
        if self._SEARCH_NEXT_CSS_SELECTOR:
            el = self._SOUPED_HTML_DATA.select(self._SEARCH_NEXT_CSS_SELECTOR)
            if len(el) >= 1:
                el = el[0]
                href = el.get('href').strip()
                if "http://" in href or "https://" in href:
                    return href
                return self._BASE_URL + href
        else:
            return None

    def get_search_results(self):
        return self._scrape_css_selector(self._SEARCH_MAIN_CSS_SELECTOR)

    def get_related_keywords(self):
        if self._SEARCH_KEYWORDS_CSS_SELECTOR:
            return self._scrape_css_selector(self._SEARCH_KEYWORDS_CSS_SELECTOR)
        else:
            return []

    def close(self):
        if self._DRIVER:
            self._DRIVER.close()
<|endoftext|>


<commit_before>"""
Utilities for extracting data from Opal applications
"""
from collections import OrderedDict
import csv
import datetime
import functools
import logging
import os
import tempfile
import zipfile

from django.template import Context, loader
from six import text_type

from collections import defaultdict
from opal.models import Episode
from opal.core.subrecords import (
    episode_subrecords, subrecords
)

class CsvColumn(object):
    """ A custom column class that will render a custom value

        * name is similar to api_name on a field
          if it matches and existig field api name in the extract
          fields this will override it
        * value is that takes in whatever arguments
          are passed to get_row
        * display name is what is used in the header
    """
    def __init__(self, name, value=None, display_name=None):
        self.name = name
        self.value = value

        if value:
            self.value = value
        else:
            self.value = lambda renderer, obj: getattr(obj, self.name)

        if display_name:
            self.display_name = display_name
        else:
            self.display_name = self.name.title()

class CsvRenderer(object):
    """
        An Abstract base class of the other csv renderers
    """

    # overrides of model fields for the csv columns
    non_field_csv_columns = []

    def __init__(self, model, queryset, user, fields=None):
        self.queryset = queryset
        self.model = model
        self.user = user
        if fields:
            self.fields = fields
        else:
            self.fields = self.get_field_names_to_render()

    def get_non_field_csv_column_names(self):
        return [csv_column.name for csv_column in self.non_field_csv_columns]

    def get_non_field_csv_columns(self, field_name):
        return next(
            i for i in self.non_field_csv_columns if i.name == field_name
        )

    def get_field_names_to_render(self):
        field_names = self.model._get_fieldnames_to_extract()
        field_names.remove("consistency_token")
        result = self.get_non_field_csv_column_names()
        non_field_csv_columns_set = set(result)
        for field_name in field_names:
            if field_name not in non_field_csv_columns_set:
                result.append(field_name)

        return result

    def get_field_title(self, field_name):
        return self.model._get_field_title(field_name)

    def get_headers(self):
        result = []
        for field in self.fields:
            if field in self.get_non_field_csv_column_names():
                result.append(
                    self.get_non_field_csv_columns(field).display_name
                )
            else:
                result.append(self.get_field_title(field))
        return result

    def get_field_value(self, field_name, data):
        col_value = data[field_name]
        if isinstance(col_value, list):
            return "; ".join(text_type(i) for i in col_value)
        else:
            return text_type(col_value)

    def get_row(self, instance, *args, **kwargs):
        as_dict = instance.to_dict(user=self.user)

        result = []

        for field in self.fields:
            if field in self.get_non_field_csv_column_names():
                some_fn = self.get_non_field_csv_columns(field).value
                result.append(
                    some_fn(self, instance, *args, **kwargs)
                )
            else:
                result.append(self.get_field_value(field, as_dict))

        return result

    def get_rows(self):
        for instance in self.queryset:
            yield self.get_row(instance)

    def count(self):
        return self.queryset.count()

    def write_to_file(self, file_name):
        logging.info("writing for {}".format(self.model))

        with open(file_name, "w") as csv_file:
            writer = csv.writer(csv_file)
            writer.writerow(self.get_headers())
            for row in self.get_rows():
                writer.writerow(row)

        logging.info("finished writing for {}".format(self.model))

class EpisodeCsvRenderer(CsvRenderer):
    non_field_csv_columns = (
        CsvColumn(
            "tagging",
            value=lambda renderer, instance: text_type(";".join(
                instance.get_tag_names(renderer.user, historic=True)
            ))
        ),
        CsvColumn("start"),
        CsvColumn("end"),
        CsvColumn("created"),
        CsvColumn("updated"),
        CsvColumn("created_by_id", display_name="Created By"),
        CsvColumn("updated_by_id", display_name="Updated By"),
        CsvColumn("patient_id", display_name="Patient"),
    )

class PatientSubrecordCsvRenderer(CsvRenderer):
    non_field_csv_columns = (
        CsvColumn(
            "episode_id",
            display_name="Episode",
            value=lambda renderer, instance, episode_id: text_type(episode_id)
        ),
    )

    def __init__(self, model, episode_queryset, user, fields=None):
        self.patient_to_episode = defaultdict(list)

        for episode in episode_queryset:
            self.patient_to_episode[episode.patient_id].append(episode.id)

        queryset = model.objects.filter(
            patient__in=list(self.patient_to_episode.keys()))

        super(PatientSubrecordCsvRenderer, self).__init__(
            model, queryset, user, fields
        )

    def get_field_names_to_render(self):
        field_names = super(
            PatientSubrecordCsvRenderer, self
        ).get_field_names_to_render()
        field_names.remove("id")
        return field_names

    def get_rows(self):
        for sub in self.queryset:
            for episode_id in self.patient_to_episode[sub.patient_id]:
                yield self.get_row(sub, episode_id)

def field_to_dict(subrecord, field_name):
    return dict(
        display_name=subrecord._get_field_title(field_name),
        description=subrecord.get_field_description(field_name),
        type_display_name=subrecord.get_human_readable_type(field_name),
    )

def get_data_dictionary():
    schema = {}
    for subrecord in subrecords():
        field_names = subrecord._get_fieldnames_to_extract()
        record_schema = [field_to_dict(subrecord, i) for i in field_names]
        schema[subrecord.get_display_name()] = record_schema
    field_names = Episode._get_fieldnames_to_extract()
    field_names.remove("start")
    field_names.remove("end")
    schema["Episode"] = [field_to_dict(Episode, i) for i in field_names]
    schema["Episode"].append(dict(
        display_name="Start",
        type_display_name="Date & Time"
    ))
    schema["Episode"].append(dict(
        display_name="End",
        type_display_name="Date & Time"
    ))
    return OrderedDict(sorted(schema.items(), key=lambda t: t[0]))

def write_data_dictionary(file_name):
    dictionary = get_data_dictionary()
    t = loader.get_template("extract_data_schema.html")
    ctx = Context(dict(schema=dictionary))
    rendered = t.render(ctx)
    with open(file_name, "w") as f:
        f.write(rendered)

class EpisodeSubrecordCsvRenderer(CsvRenderer):
    non_field_csv_columns = (
        CsvColumn(
            "patient_id",
            display_name="Patient",
            value=lambda self, instance: text_type(instance.episode.patient_id)
        ),
    )

    def get_field_names_to_render(self):
        field_names = super(
            EpisodeSubrecordCsvRenderer, self
        ).get_field_names_to_render()
        field_names.remove("id")
        return field_names

    def __init__(self, model, episode_queryset, user, fields=None):
        queryset = model.objects.filter(episode__in=episode_queryset)

        super(EpisodeSubrecordCsvRenderer, self).__init__(
            model, queryset, user, fields
        )

def zip_archive(episodes, description, user):
    """
    Given an iterable of EPISODES, the DESCRIPTION of this set of episodes,
    and the USER for which we are extracting, create a zip archive suitable
    for download with all of these episodes as CSVs.
    """
    target_dir = tempfile.mkdtemp()
    target = os.path.join(target_dir, 'extract.zip')

    with zipfile.ZipFile(target, mode='w') as z:
        zipfolder = '{0}.{1}'.format(user.username, datetime.date.today())
        os.mkdir(os.path.join(target_dir, zipfolder))
        make_file_path = functools.partial(os.path.join, target_dir, zipfolder)
        zip_relative_file_path = functools.partial(os.path.join, zipfolder)

        file_name = "data_dictionary.html"
        full_file_name = make_file_path(file_name)
        write_data_dictionary(full_file_name)
        z.write(full_file_name, zip_relative_file_path(file_name))

        file_name = "episodes.csv"
        full_file_name = make_file_path(file_name)
        renderer = EpisodeCsvRenderer(Episode, episodes, user)
        renderer.write_to_file(full_file_name)
        z.write(full_file_name, zip_relative_file_path(file_name))

        for subrecord in subrecords():
            if getattr(subrecord, '_exclude_from_extract', False):
                continue
            file_name = '{0}.csv'.format(subrecord.get_api_name())
            full_file_name = make_file_path(file_name)
            if subrecord in episode_subrecords():
                renderer = EpisodeSubrecordCsvRenderer(
                    subrecord, episodes, user
                )
            else:
                renderer = PatientSubrecordCsvRenderer(
                    subrecord, episodes, user
                )
            if renderer.count():
                renderer.write_to_file(full_file_name)
                z.write(full_file_name, zip_relative_file_path(file_name))

    return target

def async_extract(user, criteria):
    """
    Given the user and the criteria, let's run an async extract.
    """
    from opal.core.search import tasks
    return tasks.extract.delay(user, criteria).id
<commit_msg>seperate out the zip write from the file writes<commit_after>"""
Utilities for extracting data from Opal applications
"""
from collections import OrderedDict
import csv
import datetime
import functools
import logging
import os
import tempfile
import zipfile

from django.template import Context, loader
from six import text_type

from collections import defaultdict
from opal.models import Episode
from opal.core.subrecords import (
    episode_subrecords, subrecords
)

class CsvColumn(object):
    """ A custom column class that will render a custom value

        * name is similar to api_name on a field
          if it matches and existig field api name in the extract
          fields this will override it
        * value is that takes in whatever arguments
          are passed to get_row
        * display name is what is used in the header
    """
    def __init__(self, name, value=None, display_name=None):
        self.name = name
        self.value = value

        if value:
            self.value = value
        else:
            self.value = lambda renderer, obj: getattr(obj, self.name)

        if display_name:
            self.display_name = display_name
        else:
            self.display_name = self.name.title()

class CsvRenderer(object):
    """
        An Abstract base class of the other csv renderers
    """

    # overrides of model fields for the csv columns
    non_field_csv_columns = []

    def __init__(self, model, queryset, user, fields=None):
        self.queryset = queryset
        self.model = model
        self.user = user
        if fields:
            self.fields = fields
        else:
            self.fields = self.get_field_names_to_render()

    def get_non_field_csv_column_names(self):
        return [csv_column.name for csv_column in self.non_field_csv_columns]

    def get_non_field_csv_columns(self, field_name):
        return next(
            i for i in self.non_field_csv_columns if i.name == field_name
        )

    def get_field_names_to_render(self):
        field_names = self.model._get_fieldnames_to_extract()
        field_names.remove("consistency_token")
        result = self.get_non_field_csv_column_names()
        non_field_csv_columns_set = set(result)
        for field_name in field_names:
            if field_name not in non_field_csv_columns_set:
                result.append(field_name)

        return result

    def get_field_title(self, field_name):
        return self.model._get_field_title(field_name)

    def get_headers(self):
        result = []
        for field in self.fields:
            if field in self.get_non_field_csv_column_names():
                result.append(
                    self.get_non_field_csv_columns(field).display_name
                )
            else:
                result.append(self.get_field_title(field))
        return result

    def get_field_value(self, field_name, data):
        col_value = data[field_name]
        if isinstance(col_value, list):
            return "; ".join(text_type(i) for i in col_value)
        else:
            return text_type(col_value)

    def get_row(self, instance, *args, **kwargs):
        as_dict = instance.to_dict(user=self.user)

        result = []

        for field in self.fields:
            if field in self.get_non_field_csv_column_names():
                some_fn = self.get_non_field_csv_columns(field).value
                result.append(
                    some_fn(self, instance, *args, **kwargs)
                )
            else:
                result.append(self.get_field_value(field, as_dict))

        return result

    def get_rows(self):
        for instance in self.queryset:
            yield self.get_row(instance)

    def count(self):
        return self.queryset.count()

    def write_to_file(self, file_name):
        logging.info("writing for {}".format(self.model))

        with open(file_name, "w") as csv_file:
            writer = csv.writer(csv_file)
            writer.writerow(self.get_headers())
            for row in self.get_rows():
                writer.writerow(row)

        logging.info("finished writing for {}".format(self.model))

class EpisodeCsvRenderer(CsvRenderer):
    non_field_csv_columns = (
        CsvColumn(
            "tagging",
            value=lambda renderer, instance: text_type(";".join(
                instance.get_tag_names(renderer.user, historic=True)
            ))
        ),
        CsvColumn("start"),
        CsvColumn("end"),
        CsvColumn("created"),
        CsvColumn("updated"),
        CsvColumn("created_by_id", display_name="Created By"),
        CsvColumn("updated_by_id", display_name="Updated By"),
        CsvColumn("patient_id", display_name="Patient"),
    )

class PatientSubrecordCsvRenderer(CsvRenderer):
    non_field_csv_columns = (
        CsvColumn(
            "episode_id",
            display_name="Episode",
            value=lambda renderer, instance, episode_id: text_type(episode_id)
        ),
    )

    def __init__(self, model, episode_queryset, user, fields=None):
        self.patient_to_episode = defaultdict(list)

        for episode in episode_queryset:
            self.patient_to_episode[episode.patient_id].append(episode.id)

        queryset = model.objects.filter(
            patient__in=list(self.patient_to_episode.keys()))

        super(PatientSubrecordCsvRenderer, self).__init__(
            model, queryset, user, fields
        )

    def get_field_names_to_render(self):
        field_names = super(
            PatientSubrecordCsvRenderer, self
        ).get_field_names_to_render()
        field_names.remove("id")
        return field_names

    def get_rows(self):
        for sub in self.queryset:
            for episode_id in self.patient_to_episode[sub.patient_id]:
                yield self.get_row(sub, episode_id)

def field_to_dict(subrecord, field_name):
    return dict(
        display_name=subrecord._get_field_title(field_name),
        description=subrecord.get_field_description(field_name),
        type_display_name=subrecord.get_human_readable_type(field_name),
    )

def get_data_dictionary():
    schema = {}
    for subrecord in subrecords():
        field_names = subrecord._get_fieldnames_to_extract()
        record_schema = [field_to_dict(subrecord, i) for i in field_names]
        schema[subrecord.get_display_name()] = record_schema
    field_names = Episode._get_fieldnames_to_extract()
    field_names.remove("start")
    field_names.remove("end")
    schema["Episode"] = [field_to_dict(Episode, i) for i in field_names]
    schema["Episode"].append(dict(
        display_name="Start",
        type_display_name="Date & Time"
    ))
    schema["Episode"].append(dict(
        display_name="End",
        type_display_name="Date & Time"
    ))
    return OrderedDict(sorted(schema.items(), key=lambda t: t[0]))

def write_data_dictionary(file_name):
    dictionary = get_data_dictionary()
    t = loader.get_template("extract_data_schema.html")
    ctx = Context(dict(schema=dictionary))
    rendered = t.render(ctx)
    with open(file_name, "w") as f:
        f.write(rendered)

class EpisodeSubrecordCsvRenderer(CsvRenderer):
    non_field_csv_columns = (
        CsvColumn(
            "patient_id",
            display_name="Patient",
            value=lambda self, instance: text_type(instance.episode.patient_id)
        ),
    )

    def get_field_names_to_render(self):
        field_names = super(
            EpisodeSubrecordCsvRenderer, self
        ).get_field_names_to_render()
        field_names.remove("id")
        return field_names

    def __init__(self, model, episode_queryset, user, fields=None):
        queryset = model.objects.filter(episode__in=episode_queryset)

        super(EpisodeSubrecordCsvRenderer, self).__init__(
            model, queryset, user, fields
        )

def generate_files(root_dir, episodes, user):
    """ Generate the files and return a tuple of absolute_file_name, file_name
    """
    file_names = []

    file_name = "data_dictionary.html"
    full_file_name = os.path.join(root_dir, file_name)
    write_data_dictionary(full_file_name)
    file_names.append((full_file_name, file_name,))

    file_name = "episodes.csv"
    full_file_name = os.path.join(root_dir, file_name)
    renderer = EpisodeCsvRenderer(Episode, episodes, user)
    renderer.write_to_file(full_file_name)
    file_names.append((full_file_name, file_name,))

    for subrecord in subrecords():
        if getattr(subrecord, '_exclude_from_extract', False):
            continue
        file_name = '{0}.csv'.format(subrecord.get_api_name())
        full_file_name = os.path.join(root_dir, file_name)
        if subrecord in episode_subrecords():
            renderer = EpisodeSubrecordCsvRenderer(
                subrecord, episodes, user
            )
        else:
            renderer = PatientSubrecordCsvRenderer(
                subrecord, episodes, user
            )
        if renderer.count():
            renderer.write_to_file(full_file_name)
            file_names.append((full_file_name, file_name,))

    return file_names

def zip_archive(episodes, description, user):
    """
    Given an iterable of EPISODES, the DESCRIPTION of this set of episodes,
    and the USER for which we are extracting, create a zip archive suitable
    for download with all of these episodes as CSVs.
    """
    target_dir = tempfile.mkdtemp()
    target = os.path.join(target_dir, 'extract.zip')
    zipfolder = '{0}.{1}'.format(user.username, datetime.date.today())
    root_dir = os.path.join(target_dir, zipfolder)
    os.mkdir(root_dir)
    zip_relative_file_path = functools.partial(os.path.join, zipfolder)
    generate_files(os.path.join(target_dir, zipfolder), episodes, user)

    with zipfile.ZipFile(target, mode='w') as z:
        zipfolder = '{0}.{1}'.format(user.username, datetime.date.today())
        file_names = generate_files(root_dir, episodes, user)
        for full_file_name, file_name in file_names:
            z.write(
                full_file_name,
                os.path.join(full_file_name, zip_relative_file_path(file_name))
            )

    return target

def async_extract(user, criteria):
    """
    Given the user and the criteria, let's run an async extract.
    """
    from opal.core.search import tasks
    return tasks.extract.delay(user, criteria).id
<|endoftext|>


<commit_before>#!/usr/bin/env python
import redis, sys, os, json, jinja2, pycurl
from jinja2 import Template
from StringIO import StringIO

r_server = redis.StrictRedis('127.0.0.1', db=2)
check = r_server.get("need_CSR")
if check == None:
    print "need run registration job"

if len(sys.argv) > 1:
    need_haproxy = sys.argv[1]
else:
    need_haproxy = 'no'

#### get hostname
try:
    buffer = StringIO()
    c = pycurl.Curl()
    c.setopt(c.URL, 'http://169.254.169.253/latest/meta-data/public-hostname')
    c.setopt(pycurl.CONNECTTIMEOUT, 5)
    c.setopt(c.WRITEDATA, buffer)
    c.perform()
    c.close()
    hostname = buffer.getvalue()
except:
    hostname = "localhost"

i_key = "owner-info"
json_data = r_server.get(i_key)
if json_data != None:
    data = json.loads(json_data)
    email = data['Email']
    hostname = data['Hostname']
frontend_conf = ""
backend_conf = ""
acl = ""
redirect = ""

if check == "1":
    #### get certificate
    if os.environ["ENV"] == 'AWS':
        os.system("mkdir -p /opt/certs/letsencrypt")
        os.system("cd /opt/certs && openssl req -inform pem -outform der -in server.csr -out ./letsencrypt/server.der")
        request = (
        "cd /opt/certs/letsencrypt && letsencrypt certonly --csr server.der --standalone --non-interactive --agree-tos --email %s --standalone-supported-challenges http-01" % email)
        os.system(request)
        os.system(" cd /opt/certs/letsencrypt && cat 0001_chain.pem ../server.key > ../server.bundle.pem")
        os.system("rm -rf /opt/certs/letsencrypt")
        r_server.set("need_CSR", "0")
        r_server.bgsave
else:
    print "Don't need new certificate"

### reconfigure haproxy
if (check == "1") or (need_haproxy == "yes"):

    app_key = "apps"
    data_apps = r_server.get(app_key)
    os.system("rm -rf /opt/haproxy/haproxy.cfg")
    config_template = open('/opt/madcore/bin/templates/haproxy.cfg').read()
    if os.environ["ENV"] == 'AWS':
        crt_path = "/opt/certs/server.bundle.pem"
    else:
        crt_path = "/etc/pki/tls/certs/server.bundle.pem"

    if data_apps:
        apps = json.loads(data_apps)
        for app in apps:
            if app["name"] == 'kubedash':
                ai = "acl is_%s hdr_end(host) -i %s.%s \n    " % (app["name"], app["name"], data["Hostname"])
                acl = acl + ai
                ri = "redirect  code 301 location https://kubeapi.%s/api/v1/proxy/namespaces/kube-system/services/kubernetes-dashboard/ if is_%s \n    " % (
                data["Hostname"], app["name"])
                redirect = redirect + ri
            elif app["name"] == 'grafana':
                ai = "acl is_%s hdr_end(host) -i %s.%s \n    " % (app["name"], app["name"], data["Hostname"])
                acl = acl + ai
                ri = "redirect  code 301 location https://kubeapi.%s/api/v1/proxy/namespaces/kube-system/services/monitoring-grafana/ if is_%s \n    " % (
                data["Hostname"], app["name"])
                redirect = redirect + ri
            elif app["name"] == 'spark':
                ai = "acl is_%s hdr_end(host) -i %s.%s \n    " % (app["name"], app["name"], data["Hostname"])
                acl = acl + ai
                ri = "redirect  code 301 location https://kubeapi.%s:8080/api/v1/proxy/namespaces/spark-cluster/services/spark-master:8080/ if is_%s \n    " % (
                data["Hostname"], app["name"])
                redirect = redirect + ri
            elif app["name"] == 'zeppelin':
                ai = "acl is_%s hdr_end(host) -i %s.%s \n    " % (app["name"], app["name"], data["Hostname"])
                acl = acl + ai
                ri = "redirect  code 301 location https://kubeapi.%s:8080/api/v1/proxy/namespaces/spark-cluster/services/zeppelin/ if is_%s \n    " % (
                data["Hostname"], app["name"])
                redirect = redirect + ri
            else:
                i = "use_backend %s if { hdr_end(host) -i %s }\n    " % (
                app["name"], app["name"] + "." + data['Hostname'])
                frontend_conf = frontend_conf + i
                i = ("backend %s\n    balance roundrobin\n    server %s 127.0.0.1:%s check\n  " % (
                app["name"], app["name"], app["port"]))
                backend_conf = backend_conf + i
        template = Template(config_template)
        config = (
        template.render(hostname=hostname, crt_path=crt_path, subdomain=frontend_conf, backend=backend_conf, acl=acl,
                        redirect=redirect))
    else:
        template = Template(config_template)
        config = (template.render(hostname=hostname, crt_path=crt_path))

    open("/opt/haproxy/haproxy.cfg", "w").write(config)
    os.system("haproxy -f /opt/haproxy/haproxy.cfg -p /var/run/haproxy.pid -sf $(cat /var/run/haproxy.pid)")

else:
    print "Don't need reconfigure HAproxy"
<commit_msg>added check letsencrypt cert exist<commit_after>#!/usr/bin/env python
import redis, sys, os, json, jinja2, pycurl
from jinja2 import Template
from StringIO import StringIO

r_server = redis.StrictRedis('127.0.0.1', db=2)
check = r_server.get("need_CSR")
if check == None:
    print "need run registration job"

if len(sys.argv) > 1:
    need_haproxy = sys.argv[1]
else:
    need_haproxy = 'no'

#### get hostname
try:
    buffer = StringIO()
    c = pycurl.Curl()
    c.setopt(c.URL, 'http://169.254.169.253/latest/meta-data/public-hostname')
    c.setopt(pycurl.CONNECTTIMEOUT, 5)
    c.setopt(c.WRITEDATA, buffer)
    c.perform()
    c.close()
    hostname = buffer.getvalue()
except:
    hostname = "localhost"

i_key = "owner-info"
json_data = r_server.get(i_key)
if json_data != None:
    data = json.loads(json_data)
    email = data['Email']
    hostname = data['Hostname']
frontend_conf = ""
backend_conf = ""
acl = ""
redirect = ""

if check == "1":
    #### get certificate
    if os.environ["ENV"] == 'AWS':
        os.system("mkdir -p /opt/certs/letsencrypt")
        os.system("cd /opt/certs && openssl req -inform pem -outform der -in server.csr -out ./letsencrypt/server.der")
        request = (
        "cd /opt/certs/letsencrypt && letsencrypt certonly --csr server.der --standalone --non-interactive --agree-tos --email %s --standalone-supported-challenges http-01" % email)
        os.system(request)
        if !os.path.exists("/opt/certs/letsencrypt/0001_chain.pem"):
            sys.exet(2)
        os.system(" cd /opt/certs/letsencrypt && cat 0001_chain.pem ../server.key > ../server.bundle.pem")
        os.system("rm -rf /opt/certs/letsencrypt")
        r_server.set("need_CSR", "0")
        r_server.bgsave
else:
    print "Don't need new certificate"

if !os.path.exists("/opt/certs/letsencrypt/0001_chain.pem"):
    sys.exet(2)

### reconfigure haproxy
if (check == "1") or (need_haproxy == "yes"):

    app_key = "apps"
    data_apps = r_server.get(app_key)
    os.system("rm -rf /opt/haproxy/haproxy.cfg")
    config_template = open('/opt/madcore/bin/templates/haproxy.cfg').read()
    if os.environ["ENV"] == 'AWS':
        crt_path = "/opt/certs/server.bundle.pem"
    else:
        crt_path = "/etc/pki/tls/certs/server.bundle.pem"

    if data_apps:
        apps = json.loads(data_apps)
        for app in apps:
            if app["name"] == 'kubedash':
                ai = "acl is_%s hdr_end(host) -i %s.%s \n    " % (app["name"], app["name"], data["Hostname"])
                acl = acl + ai
                ri = "redirect  code 301 location https://kubeapi.%s/api/v1/proxy/namespaces/kube-system/services/kubernetes-dashboard/ if is_%s \n    " % (
                data["Hostname"], app["name"])
                redirect = redirect + ri
            elif app["name"] == 'grafana':
                ai = "acl is_%s hdr_end(host) -i %s.%s \n    " % (app["name"], app["name"], data["Hostname"])
                acl = acl + ai
                ri = "redirect  code 301 location https://kubeapi.%s/api/v1/proxy/namespaces/kube-system/services/monitoring-grafana/ if is_%s \n    " % (
                data["Hostname"], app["name"])
                redirect = redirect + ri
            elif app["name"] == 'spark':
                ai = "acl is_%s hdr_end(host) -i %s.%s \n    " % (app["name"], app["name"], data["Hostname"])
                acl = acl + ai
                ri = "redirect  code 301 location https://kubeapi.%s:8080/api/v1/proxy/namespaces/spark-cluster/services/spark-master:8080/ if is_%s \n    " % (
                data["Hostname"], app["name"])
                redirect = redirect + ri
            elif app["name"] == 'zeppelin':
                ai = "acl is_%s hdr_end(host) -i %s.%s \n    " % (app["name"], app["name"], data["Hostname"])
                acl = acl + ai
                ri = "redirect  code 301 location https://kubeapi.%s:8080/api/v1/proxy/namespaces/spark-cluster/services/zeppelin/ if is_%s \n    " % (
                data["Hostname"], app["name"])
                redirect = redirect + ri
            else:
                i = "use_backend %s if { hdr_end(host) -i %s }\n    " % (
                app["name"], app["name"] + "." + data['Hostname'])
                frontend_conf = frontend_conf + i
                i = ("backend %s\n    balance roundrobin\n    server %s 127.0.0.1:%s check\n  " % (
                app["name"], app["name"], app["port"]))
                backend_conf = backend_conf + i
        template = Template(config_template)
        config = (
        template.render(hostname=hostname, crt_path=crt_path, subdomain=frontend_conf, backend=backend_conf, acl=acl,
                        redirect=redirect))
    else:
        template = Template(config_template)
        config = (template.render(hostname=hostname, crt_path=crt_path))

    open("/opt/haproxy/haproxy.cfg", "w").write(config)
    os.system("haproxy -f /opt/haproxy/haproxy.cfg -p /var/run/haproxy.pid -sf $(cat /var/run/haproxy.pid)")

else:
    print "Don't need reconfigure HAproxy"
<|endoftext|>


<commit_before>"""
Templatetags for panels
"""
from django import template
from django.template.loader import get_template

from opal.utils import camelcase_to_underscore

register = template.Library()

@register.inclusion_tag('_helpers/record_panel.html')
def record_panel(model, editable=1, title=None, name=None, detail_template=None, angular_filter=None, noentries=None, only_display_if_exists=False):
    """
    Register a panel for our record.
    Editable is an angular expression
    to be evaluated
    """

    if name is None:
        name = camelcase_to_underscore(model.__class__.__name__)

    if detail_template is None:
        detail_template = model.__class__.get_detail_template()

    if title is None:
        title = getattr(model, '_title', name.replace('_', ' ').title())

    return {
        'name': name,
        'singleton': getattr(model, '_is_singleton', False),
        'title': title,
        'detail_template': detail_template,
        'icon': getattr(model, '_icon', None),
        'editable': editable,
        'angular_filter': angular_filter,
        'noentries': noentries
        'only_display_if_exists': only_display_if_exists,
    }

@register.inclusion_tag('_helpers/record_timeline.html')
def record_timeline(model, whenfield):
    try:
        name = camelcase_to_underscore(model.__class__.__name__)

        return {
            'name': name,
            'editable': True,
            'title': getattr(model, '_title', name.replace('_', ' ').title()),
            'detail_template': model.__class__.get_detail_template(),
            'icon': getattr(model, '_icon', None),
            'whenfield': whenfield,
        }
    except Exception as err:
        print err
        raise
<commit_msg>Typo in panels dict literal<commit_after>"""
Templatetags for panels
"""
from django import template
from django.template.loader import get_template

from opal.utils import camelcase_to_underscore

register = template.Library()

@register.inclusion_tag('_helpers/record_panel.html')
def record_panel(model, editable=1, title=None, name=None, detail_template=None, angular_filter=None, noentries=None, only_display_if_exists=False):
    """
    Register a panel for our record.
    Editable is an angular expression
    to be evaluated
    """

    if name is None:
        name = camelcase_to_underscore(model.__class__.__name__)

    if detail_template is None:
        detail_template = model.__class__.get_detail_template()

    if title is None:
        title = getattr(model, '_title', name.replace('_', ' ').title())

    return {
        'name': name,
        'singleton': getattr(model, '_is_singleton', False),
        'title': title,
        'detail_template': detail_template,
        'icon': getattr(model, '_icon', None),
        'editable': editable,
        'angular_filter': angular_filter,
        'noentries': noentries,
        'only_display_if_exists': only_display_if_exists
    }

@register.inclusion_tag('_helpers/record_timeline.html')
def record_timeline(model, whenfield):
    try:
        name = camelcase_to_underscore(model.__class__.__name__)

        return {
            'name': name,
            'editable': True,
            'title': getattr(model, '_title', name.replace('_', ' ').title()),
            'detail_template': model.__class__.get_detail_template(),
            'icon': getattr(model, '_icon', None),
            'whenfield': whenfield,
        }
    except Exception as err:
        print err
        raise
<|endoftext|>


<commit_before># -*- encoding: utf-8 -*-
from __future__ import (unicode_literals, absolute_import,
                        division, print_function)
import json
from schematics.models import Model

class OParl(object):

    def __init__(self, string):
        self.string = string
        self.data = json.loads(string)

    def validate(self):
        raise Exception('Ponies!')
<commit_msg>Removed import<commit_after># -*- encoding: utf-8 -*-
from __future__ import (unicode_literals, absolute_import,
                        division, print_function)
import json

class OParl(object):

    def __init__(self, string):
        self.string = string
        self.data = json.loads(string)

    def validate(self):
        raise Exception('Ponies!')
<|endoftext|>


<commit_before>import toolz
import numpy as np
import lasagne

from .. import utils
from .. import core
from .. import nodes

# ############################ utils for wrapping ############################

def wrap_lasagne_node(network, in_vw, param_kwargs, constructor, kwargs):
    """
    param_kwargs:
    dict from param name to map of keyword arguments for constructing
    (eg. inits, tags, etc.)
    """
    l_in = lasagne.layers.InputLayer(
        input_var=in_vw.variable,
        shape=in_vw.shape)
    l_out = constructor(l_in, **kwargs)
    output = lasagne.layers.get_output(l_out)
    output_shape = lasagne.layers.get_output_shape(l_out)
    params = lasagne.layers.get_all_params(l_out)
    to_replace = {}
    for param in params:
        name = param.name
        assert name in param_kwargs
        assert "tags" in param_kwargs[name]
        assert "inits" in param_kwargs[name]
        vw = network.create_variable(
            name=name,
            is_shared=True,
            shape=param.get_value().shape,
            **param_kwargs[name]
        )
        to_replace[param] = vw.variable
    new_output, = utils.deep_clone([output], to_replace)
    network.create_variable(
        name="default",
        variable=new_output,
        shape=output_shape,
        tags={"output"},
    )

class LasagneUpdatesNode(nodes.StandardUpdatesNode):

    """
    node that wraps lasagne update functions
    """

    hyperparameter_names = ("cost_reference",
                            "reference",
                            "sgd_learning_rate",
                            "learning_rate")

    def _new_update_deltas(self, network, parameters, grads):
        parameter_variables = [p.variable for p in parameters]
        updates = self._lasagne_updates(network, parameter_variables, grads)
        return core.UpdateDeltas.from_updates(updates)

    def _lasagne_updates(self, network, parameter_variables, grads):
        raise NotImplementedError

# ################################## nodes ##################################

@core.register_node("lasagne_dense")
class DenseNode(core.NodeImpl):

    """
    node wrapping lasagne's DenseLayer
    """

    hyperparameter_names = ("inits",
                            "dense_num_units",
                            "num_units")

    def compute_output(self, network, in_vw):
        inits = list(toolz.concat(network.find_hyperparameters(
            ["inits"],
            [])))
        num_units = network.find_hyperparameter(["dense_num_units",
                                                 "num_units"])
        wrap_lasagne_node(
            network=network,
            in_vw=in_vw,
            param_kwargs=dict(
                W=dict(tags={"parameter", "weight"},
                       inits=inits),
                b=dict(tags={"parameter", "bias"},
                       inits=inits)
            ),
            constructor=lasagne.layers.DenseLayer,
            kwargs=dict(
                num_units=num_units,
                nonlinearity=lasagne.nonlinearities.identity,
            )
        )

def ReLUNode(name):
    return nodes.ApplyNode(name,
                           fn=lasagne.nonlinearities.rectify,
                           shape_fn=utils.identity)

# ################################# updates #################################

@core.register_node("lasagne_sgd")
class SGDNode(LasagneUpdatesNode):

    """
    node that provides updates via SGD
    """

    hyperparameter_names = ("sgd_learning_rate",
                            "learning_rate")

    def _lasagne_updates(self, network, parameter_variables, grads):
        learning_rate = network.find_hyperparameter(["sgd_learning_rate",
                                                     "learning_rate"])
        return lasagne.updates.sgd(grads,
                                   parameter_variables,
                                   learning_rate=learning_rate)

@core.register_node("lasagne_nesterov_momentum")
class NesterovMomentumNode(LasagneUpdatesNode):

    """
    node that provides updates via SGD
    """

    hyperparameter_names = ("learning_rate",
                            "momentum")

    def _lasagne_updates(self, network, parameter_variables, grads):
        learning_rate = network.find_hyperparameter(["learning_rate"])
        momentum = network.find_hyperparameter(["momentum"], 0.9)
        return lasagne.updates.nesterov_momentum(grads,
                                                 parameter_variables,
                                                 learning_rate=learning_rate,
                                                 momentum=momentum)

# ############################### convolutions ###############################

@core.register_node("lasagne_conv2d")
class Conv2DNode(core.NodeImpl):

    """
    node wrapping lasagne's Conv2DLayer
    """

    hyperparameter_names = ("inits",
                            "num_filters",
                            "filter_size",
                            "conv_stride",
                            "stride",
                            "conv_pad",
                            "pad",
                            "untie_biases")

    def compute_output(self, network, in_vw):
        inits = list(toolz.concat(network.find_hyperparameters(
            ["inits"],
            [])))
        wrap_lasagne_node(
            network=network,
            in_vw=in_vw,
            param_kwargs=dict(
                W=dict(tags={"parameter", "weight"},
                       inits=inits),
                b=dict(tags={"parameter", "bias"},
                       inits=inits)
            ),
            constructor=lasagne.layers.Conv2DLayer,
            kwargs=dict(
                num_filters=network.find_hyperparameter(["num_filters"]),
                filter_size=network.find_hyperparameter(["filter_size"]),
                stride=network.find_hyperparameter(["conv_stride",
                                                    "stride"],
                                                   (1, 1)),
                pad=network.find_hyperparameter(["conv_pad",
                                                 "pad"],
                                                (0, 0)),
                untie_biases=network.find_hyperparameter(["untie_biases"],
                                                         False),
                nonlinearity=lasagne.nonlinearities.identity,
            )
        )

@core.register_node("lasagne_conv2d_dnn")
class Conv2DDNNNode(core.NodeImpl):

    """
    node wrapping lasagne's Conv2DDNNLayer
    """

    hyperparameter_names = ("inits",
                            "num_filters",
                            "filter_size",
                            "conv_stride",
                            "stride",
                            "conv_pad",
                            "pad",
                            "untie_biases",
                            "flip_filters",)

    def compute_output(self, network, in_vw):
        import lasagne.layers.dnn
        inits = list(toolz.concat(network.find_hyperparameters(
            ["inits"],
            [])))
        wrap_lasagne_node(
            network=network,
            in_vw=in_vw,
            param_kwargs=dict(
                W=dict(tags={"parameter", "weight"},
                       inits=inits),
                b=dict(tags={"parameter", "bias"},
                       inits=inits)
            ),
            constructor=lasagne.layers.dnn.Conv2DDNNLayer,
            kwargs=dict(
                num_filters=network.find_hyperparameter(["num_filters"]),
                filter_size=network.find_hyperparameter(["filter_size"]),
                stride=network.find_hyperparameter(["conv_stride",
                                                    "stride"],
                                                   (1, 1)),
                pad=network.find_hyperparameter(["conv_pad",
                                                 "pad"],
                                                (0, 0)),
                untie_biases=network.find_hyperparameter(["untie_biases"],
                                                         False),
                flip_filters=network.find_hyperparameter(["flip_filters"],
                                                         False),
                nonlinearity=lasagne.nonlinearities.identity,
            )
        )

# ############################### downsampling ###############################

@core.register_node("lasagne_maxpool2d")
class MaxPool2DNode(core.NodeImpl):

    """
    node wrapping lasagne's MaxPool2DLayer
    """

    hyperparameter_names = ("pool_size",
                            "pool_stride",
                            "stride",
                            "pad",
                            "ignore_border")

    def compute_output(self, network, in_vw):
        pool_size = network.find_hyperparameter(["pool_size"])
        stride = network.find_hyperparameter(["pool_stride",
                                              "stride"],
                                             None)
        ignore_border = network.find_hyperparameter(["ignore_border"],
                                                    False)
        if ((stride is not None)
                and (stride != pool_size)
                and (not ignore_border)):
            # as of 20150813
            # for more information, see:
            # https://groups.google.com/forum/#!topic/lasagne-users/t_rMTLAtpZo
            msg = ("Setting stride not equal to pool size and not ignoring"
                   " border results in using a slower (cpu-based)"
                   " implementation")
            # making this an assertion instead of a warning to make sure it
            # is done
            assert False, msg
        wrap_lasagne_node(
            network=network,
            in_vw=in_vw,
            param_kwargs={},
            constructor=lasagne.layers.MaxPool2DLayer,
            kwargs=dict(
                pool_size=pool_size,
                stride=stride,
                pad=network.find_hyperparameter(["pad"], (0, 0)),
                ignore_border=ignore_border,
            )
        )

@core.register_node("lasagne_maxpool2d_dnn")
class MaxPool2DDNNNode(core.NodeImpl):

    """
    node wrapping lasagne's MaxPool2DDNNLayer
    """

    hyperparameter_names = ("pool_size",
                            "pool_stride",
                            "stride",
                            "pad")

    def compute_output(self, network, in_vw):
        import lasagne.layers.dnn
        wrap_lasagne_node(
            network=network,
            in_vw=in_vw,
            param_kwargs={},
            constructor=lasagne.layers.dnn.MaxPool2DDNNLayer,
            kwargs=dict(
                pool_size=network.find_hyperparameter(["pool_size"]),
                stride=network.find_hyperparameter(["pool_stride",
                                                    "stride"],
                                                   None),
                pad=network.find_hyperparameter(["pad"], (0, 0)),
            )
        )

@core.register_node("lasagne_meanpool2d_dnn")
class MeanPool2DDNNNode(core.NodeImpl):

    """
    node wrapping lasagne's Pool2DDNNLayer with mode = average_exc_pad
    """

    hyperparameter_names = ("pool_size",
                            "pool_stride",
                            "stride",
                            "pad")

    def compute_output(self, network, in_vw):
        import lasagne.layers.dnn
        wrap_lasagne_node(
            network=network,
            in_vw=in_vw,
            param_kwargs={},
            constructor=lasagne.layers.dnn.Pool2DDNNLayer,
            kwargs=dict(
                # TODO look into which is better
                # can also be mode="average_inc_pad",
                mode="average_exc_pad",
                pool_size=network.find_hyperparameter(["pool_size"]),
                stride=network.find_hyperparameter(["pool_stride",
                                                    "stride"],
                                                   None),
                pad=network.find_hyperparameter(["pad"], (0, 0)),
            )
        )

# ############################## normalization ##############################

@core.register_node("lasagne_lrn2d")
class LocalResponseNormalization2DNode(core.NodeImpl):

    """
    node wrapping lasagne's Pool2DDNNLayer with mode = average_exc_pad
    """

    hyperparameter_names = ("lrn_alpha",
                            "lrn_k",
                            "lrn_beta",
                            "lrn_n")

    def compute_output(self, network, in_vw):
        import lasagne.layers.dnn
        wrap_lasagne_node(
            network=network,
            in_vw=in_vw,
            param_kwargs={},
            constructor=lasagne.layers.LocalResponseNormalization2DLayer,
            kwargs=dict(
                alpha=network.find_hyperparameter(["lrn_alpha"], 1e-4),
                k=network.find_hyperparameter(["lrn_k"], 2),
                beta=network.find_hyperparameter(["lrn_beta"], 0.75),
                n=network.find_hyperparameter(["lrn_n"], 5)))
<commit_msg>match lasagne's new default of max pool ignore_border=True<commit_after>import toolz
import numpy as np
import lasagne

from .. import utils
from .. import core
from .. import nodes

# ############################ utils for wrapping ############################

def wrap_lasagne_node(network, in_vw, param_kwargs, constructor, kwargs):
    """
    param_kwargs:
    dict from param name to map of keyword arguments for constructing
    (eg. inits, tags, etc.)
    """
    l_in = lasagne.layers.InputLayer(
        input_var=in_vw.variable,
        shape=in_vw.shape)
    l_out = constructor(l_in, **kwargs)
    output = lasagne.layers.get_output(l_out)
    output_shape = lasagne.layers.get_output_shape(l_out)
    params = lasagne.layers.get_all_params(l_out)
    to_replace = {}
    for param in params:
        name = param.name
        assert name in param_kwargs
        assert "tags" in param_kwargs[name]
        assert "inits" in param_kwargs[name]
        vw = network.create_variable(
            name=name,
            is_shared=True,
            shape=param.get_value().shape,
            **param_kwargs[name]
        )
        to_replace[param] = vw.variable
    new_output, = utils.deep_clone([output], to_replace)
    network.create_variable(
        name="default",
        variable=new_output,
        shape=output_shape,
        tags={"output"},
    )

class LasagneUpdatesNode(nodes.StandardUpdatesNode):

    """
    node that wraps lasagne update functions
    """

    hyperparameter_names = ("cost_reference",
                            "reference",
                            "sgd_learning_rate",
                            "learning_rate")

    def _new_update_deltas(self, network, parameters, grads):
        parameter_variables = [p.variable for p in parameters]
        updates = self._lasagne_updates(network, parameter_variables, grads)
        return core.UpdateDeltas.from_updates(updates)

    def _lasagne_updates(self, network, parameter_variables, grads):
        raise NotImplementedError

# ################################## nodes ##################################

@core.register_node("lasagne_dense")
class DenseNode(core.NodeImpl):

    """
    node wrapping lasagne's DenseLayer
    """

    hyperparameter_names = ("inits",
                            "dense_num_units",
                            "num_units")

    def compute_output(self, network, in_vw):
        inits = list(toolz.concat(network.find_hyperparameters(
            ["inits"],
            [])))
        num_units = network.find_hyperparameter(["dense_num_units",
                                                 "num_units"])
        wrap_lasagne_node(
            network=network,
            in_vw=in_vw,
            param_kwargs=dict(
                W=dict(tags={"parameter", "weight"},
                       inits=inits),
                b=dict(tags={"parameter", "bias"},
                       inits=inits)
            ),
            constructor=lasagne.layers.DenseLayer,
            kwargs=dict(
                num_units=num_units,
                nonlinearity=lasagne.nonlinearities.identity,
            )
        )

def ReLUNode(name):
    return nodes.ApplyNode(name,
                           fn=lasagne.nonlinearities.rectify,
                           shape_fn=utils.identity)

# ################################# updates #################################

@core.register_node("lasagne_sgd")
class SGDNode(LasagneUpdatesNode):

    """
    node that provides updates via SGD
    """

    hyperparameter_names = ("sgd_learning_rate",
                            "learning_rate")

    def _lasagne_updates(self, network, parameter_variables, grads):
        learning_rate = network.find_hyperparameter(["sgd_learning_rate",
                                                     "learning_rate"])
        return lasagne.updates.sgd(grads,
                                   parameter_variables,
                                   learning_rate=learning_rate)

@core.register_node("lasagne_nesterov_momentum")
class NesterovMomentumNode(LasagneUpdatesNode):

    """
    node that provides updates via SGD
    """

    hyperparameter_names = ("learning_rate",
                            "momentum")

    def _lasagne_updates(self, network, parameter_variables, grads):
        learning_rate = network.find_hyperparameter(["learning_rate"])
        momentum = network.find_hyperparameter(["momentum"], 0.9)
        return lasagne.updates.nesterov_momentum(grads,
                                                 parameter_variables,
                                                 learning_rate=learning_rate,
                                                 momentum=momentum)

# ############################### convolutions ###############################

@core.register_node("lasagne_conv2d")
class Conv2DNode(core.NodeImpl):

    """
    node wrapping lasagne's Conv2DLayer
    """

    hyperparameter_names = ("inits",
                            "num_filters",
                            "filter_size",
                            "conv_stride",
                            "stride",
                            "conv_pad",
                            "pad",
                            "untie_biases")

    def compute_output(self, network, in_vw):
        inits = list(toolz.concat(network.find_hyperparameters(
            ["inits"],
            [])))
        wrap_lasagne_node(
            network=network,
            in_vw=in_vw,
            param_kwargs=dict(
                W=dict(tags={"parameter", "weight"},
                       inits=inits),
                b=dict(tags={"parameter", "bias"},
                       inits=inits)
            ),
            constructor=lasagne.layers.Conv2DLayer,
            kwargs=dict(
                num_filters=network.find_hyperparameter(["num_filters"]),
                filter_size=network.find_hyperparameter(["filter_size"]),
                stride=network.find_hyperparameter(["conv_stride",
                                                    "stride"],
                                                   (1, 1)),
                pad=network.find_hyperparameter(["conv_pad",
                                                 "pad"],
                                                (0, 0)),
                untie_biases=network.find_hyperparameter(["untie_biases"],
                                                         False),
                nonlinearity=lasagne.nonlinearities.identity,
            )
        )

@core.register_node("lasagne_conv2d_dnn")
class Conv2DDNNNode(core.NodeImpl):

    """
    node wrapping lasagne's Conv2DDNNLayer
    """

    hyperparameter_names = ("inits",
                            "num_filters",
                            "filter_size",
                            "conv_stride",
                            "stride",
                            "conv_pad",
                            "pad",
                            "untie_biases",
                            "flip_filters",)

    def compute_output(self, network, in_vw):
        import lasagne.layers.dnn
        inits = list(toolz.concat(network.find_hyperparameters(
            ["inits"],
            [])))
        wrap_lasagne_node(
            network=network,
            in_vw=in_vw,
            param_kwargs=dict(
                W=dict(tags={"parameter", "weight"},
                       inits=inits),
                b=dict(tags={"parameter", "bias"},
                       inits=inits)
            ),
            constructor=lasagne.layers.dnn.Conv2DDNNLayer,
            kwargs=dict(
                num_filters=network.find_hyperparameter(["num_filters"]),
                filter_size=network.find_hyperparameter(["filter_size"]),
                stride=network.find_hyperparameter(["conv_stride",
                                                    "stride"],
                                                   (1, 1)),
                pad=network.find_hyperparameter(["conv_pad",
                                                 "pad"],
                                                (0, 0)),
                untie_biases=network.find_hyperparameter(["untie_biases"],
                                                         False),
                flip_filters=network.find_hyperparameter(["flip_filters"],
                                                         False),
                nonlinearity=lasagne.nonlinearities.identity,
            )
        )

# ############################### downsampling ###############################

@core.register_node("lasagne_maxpool2d")
class MaxPool2DNode(core.NodeImpl):

    """
    node wrapping lasagne's MaxPool2DLayer
    """

    hyperparameter_names = ("pool_size",
                            "pool_stride",
                            "stride",
                            "pad",
                            "ignore_border")

    def compute_output(self, network, in_vw):
        pool_size = network.find_hyperparameter(["pool_size"])
        stride = network.find_hyperparameter(["pool_stride",
                                              "stride"],
                                             None)
        ignore_border = network.find_hyperparameter(["ignore_border"],
                                                    True)
        if ((stride is not None)
                and (stride != pool_size)
                and (not ignore_border)):
            # as of 20150813
            # for more information, see:
            # https://groups.google.com/forum/#!topic/lasagne-users/t_rMTLAtpZo
            msg = ("Setting stride not equal to pool size and not ignoring"
                   " border results in using a slower (cpu-based)"
                   " implementation")
            # making this an assertion instead of a warning to make sure it
            # is done
            assert False, msg
        wrap_lasagne_node(
            network=network,
            in_vw=in_vw,
            param_kwargs={},
            constructor=lasagne.layers.MaxPool2DLayer,
            kwargs=dict(
                pool_size=pool_size,
                stride=stride,
                pad=network.find_hyperparameter(["pad"], (0, 0)),
                ignore_border=ignore_border,
            )
        )

@core.register_node("lasagne_maxpool2d_dnn")
class MaxPool2DDNNNode(core.NodeImpl):

    """
    node wrapping lasagne's MaxPool2DDNNLayer
    """

    hyperparameter_names = ("pool_size",
                            "pool_stride",
                            "stride",
                            "pad")

    def compute_output(self, network, in_vw):
        import lasagne.layers.dnn
        wrap_lasagne_node(
            network=network,
            in_vw=in_vw,
            param_kwargs={},
            constructor=lasagne.layers.dnn.MaxPool2DDNNLayer,
            kwargs=dict(
                pool_size=network.find_hyperparameter(["pool_size"]),
                stride=network.find_hyperparameter(["pool_stride",
                                                    "stride"],
                                                   None),
                pad=network.find_hyperparameter(["pad"], (0, 0)),
            )
        )

@core.register_node("lasagne_meanpool2d_dnn")
class MeanPool2DDNNNode(core.NodeImpl):

    """
    node wrapping lasagne's Pool2DDNNLayer with mode = average_exc_pad
    """

    hyperparameter_names = ("pool_size",
                            "pool_stride",
                            "stride",
                            "pad")

    def compute_output(self, network, in_vw):
        import lasagne.layers.dnn
        wrap_lasagne_node(
            network=network,
            in_vw=in_vw,
            param_kwargs={},
            constructor=lasagne.layers.dnn.Pool2DDNNLayer,
            kwargs=dict(
                # TODO look into which is better
                # can also be mode="average_inc_pad",
                mode="average_exc_pad",
                pool_size=network.find_hyperparameter(["pool_size"]),
                stride=network.find_hyperparameter(["pool_stride",
                                                    "stride"],
                                                   None),
                pad=network.find_hyperparameter(["pad"], (0, 0)),
            )
        )

# ############################## normalization ##############################

@core.register_node("lasagne_lrn2d")
class LocalResponseNormalization2DNode(core.NodeImpl):

    """
    node wrapping lasagne's Pool2DDNNLayer with mode = average_exc_pad
    """

    hyperparameter_names = ("lrn_alpha",
                            "lrn_k",
                            "lrn_beta",
                            "lrn_n")

    def compute_output(self, network, in_vw):
        import lasagne.layers.dnn
        wrap_lasagne_node(
            network=network,
            in_vw=in_vw,
            param_kwargs={},
            constructor=lasagne.layers.LocalResponseNormalization2DLayer,
            kwargs=dict(
                alpha=network.find_hyperparameter(["lrn_alpha"], 1e-4),
                k=network.find_hyperparameter(["lrn_k"], 2),
                beta=network.find_hyperparameter(["lrn_beta"], 0.75),
                n=network.find_hyperparameter(["lrn_n"], 5)))
<|endoftext|>


<commit_before># -*- encoding: utf-8 -*-
from __future__ import (unicode_literals, absolute_import,
                        division, print_function)
import json
from collections import namedtuple
from jsonschema.validators import Draft4Validator
from functools import wraps
import jsonschema.exceptions
from .schema import OPARL, TYPES
from .utils import build_object_type, import_from_string
from .statistics import with_stats

class ValidationNotice(namedtuple('ValidationNotice',
                       ['message', 'section'])):

    def __new__(cls, message, section=None):
        return super(ValidationNotice, cls).__new__(cls, message, section)

class ValidationError(namedtuple('ValidationError',
                      ['message', 'section'])):

    def __new__(cls, message, section=None):
        return super(ValidationError, cls).__new__(cls, message, section)

def prune(*args):
    def decorator(func):
        @wraps(func)
        def wrapper(*args_, **kwargs_):
            return (item for item in func(*args_, **kwargs_)
                    if item not in args)
        return wrapper
    return decorator

def types(*args):
    def decorator(func):
        func.types = args
        return func
    return decorator

class OParlResponse(object):

    def __init__(self, response):
        # TODO: doc me
        self.response = response
        self.validators = [name for name in dir(self)
                           if name.startswith('_validate_')]

    @types('AgendaItem', 'Document', 'Membership', 'Person',
           'Body', 'Location', 'Organization', 'System',
           'Consultation', 'Meeting', 'Paper')  # Or all by default?
    def _validate_success(self):
        return self.response.status_code in range(200, 400)  # O(1) in Py 3

    @prune(None)
    def validate(self):
        # TODO: doc me
        if self.response:
            for name in self.validators:
                yield getattr(self, name)()

class OParlJson(object):

    def __init__(self, string):
        # TODO: doc me
        self.string = string

    @classmethod
    def _get_validator(cls, object_type):
        object_type = build_object_type(object_type)
        if object_type not in OPARL:
            return None
        return Draft4Validator(OPARL[object_type])

    @staticmethod
    def _validate_type(data):
        """
        Check if the document contains a type and we have a schema for it.

        :param data: dictionary with the parsed document for validation
        :returns: internal document type, could be used to access the
           approriate schema using the OPARL dictionary
        """
        type_check = {
            'type': 'object',
            'properties': {
                'type': {
                    'type': 'string',
                    'enum': TYPES.keys()
                }
            },
            'required': ['type']
        }
        Draft4Validator(type_check).validate(data)
        return TYPES[data['type']]

    @classmethod
    def _validate_schema(cls, obj_type, data):
        validator = cls._get_validator(obj_type)
        return validator.iter_errors(data)

    @classmethod
    def _validate_custom(cls, schema, data):
        if 'oparl:validate' in schema:
            for test in schema['oparl:validate']:
                func = import_from_string(test['method'])
                if not func(data):
                    yield ValidationError(section=test['section'],
                                          message=test['message'])

    @prune(None)
    @with_stats
    def validate(self, stats=None):
        # TODO: doc me
        try:
            data = json.loads(self.string)
            stats.count_document()
        except ValueError as excp:
            yield ValidationError('JSON error: %s' % excp)
            return
        try:
            obj_type = self._validate_type(data)
            stats.count_type(obj_type)
            # simple pass all errors to the caller
            for error in self._validate_schema(obj_type, data):
                yield error
            for error in self._validate_custom(OPARL[obj_type], data):
                yield error
        except (jsonschema.exceptions.ValidationError,
                jsonschema.exceptions.SchemaError) as excp:
            # _validate_type may raise an exception, we translate it
            # here into en apropriate ValidationError and pass it to
            # the caller (if the type does not validate, it does not
            # make sense to continue validation
            if len(excp.path) > 0:
                yield ValidationError('"{}": {}'.format(''.join(excp.path),
                                                        excp.message))
            else:
                yield ValidationError(excp.message)
<commit_msg>add decorator for error msg<commit_after># -*- encoding: utf-8 -*-
from __future__ import (unicode_literals, absolute_import,
                        division, print_function)
import json
from collections import namedtuple
from jsonschema.validators import Draft4Validator
from functools import wraps
import jsonschema.exceptions
from .schema import OPARL, TYPES
from .utils import build_object_type, import_from_string
from .statistics import with_stats

class ValidationNotice(namedtuple('ValidationNotice',
                       ['message', 'section'])):

    def __new__(cls, message, section=None):
        return super(ValidationNotice, cls).__new__(cls, message, section)

class ValidationError(namedtuple('ValidationError',
                      ['message', 'section'])):

    def __new__(cls, message, section=None):
        return super(ValidationError, cls).__new__(cls, message, section)

def prune(*args):
    def decorator(func):
        @wraps(func)
        def wrapper(*args_, **kwargs_):
            return (item for item in func(*args_, **kwargs_)
                    if item not in args)
        return wrapper
    return decorator

def types(*args):
    def decorator(func):
        func.types = args
        return func
    return decorator

def errormsg(msg):
    def decorator(func):
        @wraps(func)
        def wrapper(*args, **kwargs):
            if func(*args, **kwargs) is False:
                raise Exception(msg)
        return wrapper
    return decorator

class OParlResponse(object):

    def __init__(self, response):
        # TODO: doc me
        self.response = response
        self.validators = [name for name in dir(self)
                           if name.startswith('_validate_')]

    @types('AgendaItem', 'Document', 'Membership', 'Person',
           'Body', 'Location', 'Organization', 'System',
           'Consultation', 'Meeting', 'Paper')  # Or all by default?

    @errormsg("Invalid Status Code")
    def _validate_success(self):
        return self.response.status_code in range(200, 400)  # O(1) in Py 3

    @prune(None)
    def validate(self):
        # TODO: doc me
        if self.response:
            for name in self.validators:
                try:
                    getattr(self, name)()
                except Exception as error:
                    yield error

class OParlJson(object):

    def __init__(self, string):
        # TODO: doc me
        self.string = string

    @classmethod
    def _get_validator(cls, object_type):
        object_type = build_object_type(object_type)
        if object_type not in OPARL:
            return None
        return Draft4Validator(OPARL[object_type])

    @staticmethod
    def _validate_type(data):
        """
        Check if the document contains a type and we have a schema for it.

        :param data: dictionary with the parsed document for validation
        :returns: internal document type, could be used to access the
           approriate schema using the OPARL dictionary
        """
        type_check = {
            'type': 'object',
            'properties': {
                'type': {
                    'type': 'string',
                    'enum': TYPES.keys()
                }
            },
            'required': ['type']
        }
        Draft4Validator(type_check).validate(data)
        return TYPES[data['type']]

    @classmethod
    def _validate_schema(cls, obj_type, data):
        validator = cls._get_validator(obj_type)
        return validator.iter_errors(data)

    @classmethod
    def _validate_custom(cls, schema, data):
        if 'oparl:validate' in schema:
            for test in schema['oparl:validate']:
                func = import_from_string(test['method'])
                if not func(data):
                    yield ValidationError(section=test['section'],
                                          message=test['message'])

    @prune(None)
    @with_stats
    def validate(self, stats=None):
        # TODO: doc me
        try:
            data = json.loads(self.string)
            stats.count_document()
        except ValueError as excp:
            yield ValidationError('JSON error: %s' % excp)
            return
        try:
            obj_type = self._validate_type(data)
            stats.count_type(obj_type)
            # simple pass all errors to the caller
            for error in self._validate_schema(obj_type, data):
                yield error
            for error in self._validate_custom(OPARL[obj_type], data):
                yield error
        except (jsonschema.exceptions.ValidationError,
                jsonschema.exceptions.SchemaError) as excp:
            # _validate_type may raise an exception, we translate it
            # here into en apropriate ValidationError and pass it to
            # the caller (if the type does not validate, it does not
            # make sense to continue validation
            if len(excp.path) > 0:
                yield ValidationError('"{}": {}'.format(''.join(excp.path),
                                                        excp.message))
            else:
                yield ValidationError(excp.message)
<|endoftext|>


<commit_before>import ldap

from django.conf import settings
from django.utils.encoding import smart_str

def get_attrs_of_sciper(sciper):
    con = ldap.initialize(settings.LDAP)
    con.simple_bind()

    base_dn = 'c=ch'
    filter = '(uniqueIdentifier=' + str(sciper) + ')'
    attrs = ['sn', 'givenName', 'mail']

    name, firstname, email = '', '', ''
    for someone in con.search_s(base_dn, ldap.SCOPE_SUBTREE, filter, attrs):
        name = getattr(someone[1], 'sn', [''])[0].split(',')[0]
        firstname = getattr(someone[1], 'givenName', [''])[0].split(',')[0]
        email = getattr(someone[1], 'mail', [''])[0]

    if not name:
        print "No user in LDAP with sciper {}".format(sciper)

    return (name, firstname, email)

def search_sciper(s):

    con = ldap.initialize(settings.LDAP)
    con.simple_bind()

    base_dn = 'c=ch'
    filter = '(|(cn=*%s*)(uniqueIdentifier=%s))' % (smart_str(s), smart_str(s))
    attrs = ['uniqueIdentifier', 'sn', 'givenName', 'mail']

    results = {}

    for someone in con.search_s(base_dn, ldap.SCOPE_SUBTREE, filter, attrs):
        try:
            sciper = someone[1]['uniqueIdentifier'][0]
            name = someone[1]['sn'][0].split(',')[0]
            firstname = someone[1]['givenName'][0].split(',')[0]
            email = someone[1]['mail'][0]

            results[sciper] = (sciper, firstname, name, email)
        except:
            pass

        if len(results) >= 7:
            return results

    return results
<commit_msg>Hotfix: wrong ldap get (its a dict, not an object)<commit_after>import ldap

from django.conf import settings
from django.utils.encoding import smart_str

def get_attrs_of_sciper(sciper):
    con = ldap.initialize(settings.LDAP)
    con.simple_bind()

    base_dn = 'c=ch'
    filter = '(uniqueIdentifier=' + str(sciper) + ')'
    attrs = ['sn', 'givenName', 'mail']

    name, firstname, email = '', '', ''
    for someone in con.search_s(base_dn, ldap.SCOPE_SUBTREE, filter, attrs):
        name = someone[1].get('sn', [''])[0].split(',')[0]
        firstname = someone[1].get('givenName', [''])[0].split(',')[0]
        email = someone[1].get('mail', [''])[0]

    if not name:
        print "No user in LDAP with sciper {}".format(sciper)

    return (name, firstname, email)

def search_sciper(s):

    con = ldap.initialize(settings.LDAP)
    con.simple_bind()

    base_dn = 'c=ch'
    filter = '(|(cn=*%s*)(uniqueIdentifier=%s))' % (smart_str(s), smart_str(s))
    attrs = ['uniqueIdentifier', 'sn', 'givenName', 'mail']

    results = {}

    for someone in con.search_s(base_dn, ldap.SCOPE_SUBTREE, filter, attrs):
        try:
            sciper = someone[1]['uniqueIdentifier'][0]
            name = someone[1]['sn'][0].split(',')[0]
            firstname = someone[1]['givenName'][0].split(',')[0]
            email = someone[1]['mail'][0]

            results[sciper] = (sciper, firstname, name, email)
        except:
            pass

        if len(results) >= 7:
            return results

    return results
<|endoftext|>


<commit_before>
import os

import aql.local_host
import aql.utils
import aql.setup

_prependEnvPath = aql.utils.prependEnvPath

if aql.local_host.os == 'cygwin':
    _drive_d = '/cygdrive/d'
else:
    _drive_d = 'd:'

#//===========================================================================//

def     setup_gcc( options, os_env, env ):

    if options.cc_name != 'gcc':
        return 0

    #//-------------------------------------------------------//

    if options.target_os == 'windows':

        cc_path = _drive_d

        if options.cc_ver == '4.3':
            cc_path += '/bin/development/compilers/gcc/mingw_4.3'
            options.gcc_prefix = 'mingw32-'
            options.gcc_suffix = ''

        elif options.cc_ver == '4.2':
            cc_path += '/bin/development/compilers/gcc/mingw_4.2.1_dw2'
            options.gcc_prefix = 'mingw32-'
            options.gcc_suffix = '-dw2'

        elif options.cc_ver == '4.1':
            cc_path += '/bin/development/compilers/gcc/mingw'

        else:
            return 0

        _prependEnvPath( os_env, 'PATH', cc_path + '/bin' )

        return 1

    #//-------------------------------------------------------//

    if (options.target_os == 'cygwin') and (aql.local_host.os == 'cygwin'):
        os_env.update( os.environ )
        return 1

#//===========================================================================//

aql.setup.AddToolSetup( 'aql_tool_gcc', setup_gcc )
<commit_msg>GCC setup for none windows platform uses existing environment<commit_after>
import os

import aql.local_host
import aql.utils
import aql.setup

_prependEnvPath = aql.utils.prependEnvPath

if aql.local_host.os == 'cygwin':
    _drive_d = '/cygdrive/d'
else:
    _drive_d = 'd:'

#//===========================================================================//

def     setup_gcc( options, os_env, env ):

    if options.cc_name != 'gcc':
        return 0

    #//-------------------------------------------------------//

    if options.target_os == 'windows':

        cc_path = _drive_d

        if options.cc_ver == '4.3':
            cc_path += '/bin/development/compilers/gcc/mingw_4.3'
            options.gcc_prefix = 'mingw32-'
            options.gcc_suffix = ''

        elif options.cc_ver == '4.2':
            cc_path += '/bin/development/compilers/gcc/mingw_4.2.1_dw2'
            options.gcc_prefix = 'mingw32-'
            options.gcc_suffix = '-dw2'

        elif options.cc_ver == '4.1':
            cc_path += '/bin/development/compilers/gcc/mingw'

        else:
            return 0

        _prependEnvPath( os_env, 'PATH', cc_path + '/bin' )

        return 1

    #//-------------------------------------------------------//

    os_env.update( os.environ )
    return 1

#//===========================================================================//

aql.setup.AddToolSetup( 'aql_tool_gcc', setup_gcc )
<|endoftext|>