<commit_before># -*- coding: utf-8 -*-
from __future__ import unicode_literals

import re
from builtins import map, filter, str
from operator import itemgetter

import mistune
import requests
from bs4 import BeautifulSoup
from future.utils import iteritems
from html2text import html2text
from six.moves.urllib.parse import urljoin

from anpy.utils import extract_datetime

AN_BASE_URL = 'http://www.assemblee-nationale.fr'

class InvalidResponseException(Exception):
    pass

class Dossier(object):
    def __init__(self, url=None, senat_url=None, title=None,
                 legislature=None, procedure=None, steps=None):
        self.url = url
        self.senat_url = senat_url
        self.title = title
        self.legislature = legislature
        self.procedure = procedure
        self.steps = steps or []

    @staticmethod
    def download_and_build(url):
        resp = requests.get(url)
        if resp.status_code >= 400:
            raise InvalidResponseException('%s: %d' % (url, resp.status_code))
        return DossierParser(url, resp.content).parse()

    def to_dict(self):
        return {
            'url': self.url,
            'senat_url': self.senat_url,
            'title': self.title,
            'legislature': self.legislature,
            'procedure': self.procedure,
            'steps': self.steps
        }

class ProcedureParlementaire(object):
    PJL = 'PJL'
    PPL = 'PPL'

class LegislativeStepType(object):
    AN_PREMIERE_LECTURE = 'AN_PREMIERE_LECTURE'
    SENAT_PREMIERE_LECTURE = 'SENAT_PREMIERE_LECTURE'
    AN_DEUXIEME_LECTURE = 'AN_DEUXIEME_LECTURE'
    SENAT_DEUXIEME_LECTURE = 'SENAT_DEUXIEME_LECTURE'
    CMP = 'CMP'
    CONSEIL_CONSTIT = 'CONSEIL_CONSTITUT'
    AN_NOUVELLE_LECTURE = 'AN_NOUVELLE_LECTURE'
    SENAT_NOUVELLE_LECTURE = 'SENAT_NOUVELLE_LECTURE'
    AN_LECTURE_DEFINITIVE = 'AN_LECTURE_DEFINITIVE'

class DecisionStatus(object):
    ADOPTE = 'ADOPTE'
    REJETE = 'REJETE'
    MODIFIE = 'MODIFIE'

class LegislativeActType(object):
    ETUDE_IMPACT = 'ETUDE_IMPACT'
    AVIS_CONSEIL_ETAT = 'AVIS_CONSEIL_DETAT'
    PROCEDURE_ACCELEREE = 'PROCEDURE_ACCELEREE'
    REUNION_COMMISSION = 'REUNION_COMMISSION'
    DISCUSSION_COMMISSION = 'DISCUSSION_COMMISSION'
    DEPOT_RAPPORT = 'DEPOT_RAPPORT'
    SAISIE_COM_AVIS = 'SAISIE_COM_AVIS'
    NOMINATION_RAPPORTEURS = 'NOMINATION_RAPPORTEURS'
    DISCUSSION_SEANCE_PUBLIQUE = 'DISCUSSION_SEANCE_PUBLIQUE'
    DECISION = 'DECISION'
    DEPOT_INITIATIVE = 'DEPOT_INITIATIVE'
    SAISIE_COM_FOND = 'SAISIE_COM_FOND'
    PROMULGATION = 'PROMULGATION'
    SAISINE_CONSEIL_CONSTIT = 'SAISINE_CONSEIL_CONSTIT'

class DossierParser(object):
    def __init__(self, url, html):
        self.url = url
        self.html = html
        self.soup = BeautifulSoup(clean_html(html), 'html5lib')

    def parse(self):
        return Dossier(
            url=self.url,
            title=self.parse_title(),
            legislature=self.parse_legislature(),
            procedure=self.parse_procedure(),
            steps=self.parse_steps(),
            senat_url=self.parse_senat_url())

    def parse_title(self):
        return self.soup.strong.text if self.soup.strong else None

    def parse_procedure(self):
        re_procedure = re.compile('(projet de loi|proposition de loi)', re.I)

        def match_procedure(a):
            return re_procedure.match(a.text)

        def get_procedure(a):
            proc = match_procedure(a).group(1).lower()
            if proc == 'projet de loi':
                return ProcedureParlementaire.PJL
            elif proc == 'proposition de loi':
                return ProcedureParlementaire.PPL

        return next(
            map(get_procedure,
                filter(match_procedure, self.soup.find_all('a'))), None)

    def parse_senat_url(self):
        re_url = re.compile('http://www.senat.fr/dossier-legislatif')
        re_url_alt = re.compile('http://www.senat.fr/dossierleg/')

        return next(
            filter(lambda url: re_url.match(url) or re_url_alt.match(url),
                   map(lambda a: a['href'], self.soup.find_all('a'))), None)

    def parse_legislature(self):
        re_legislature = re.compile('/(\d{2})/')

        def match_legislature(a):
            return re_legislature.match(a['href'])

        def get_matched_group(a):
            return match_legislature(a).group(1) \
                if match_legislature(a) else None

        return next(map(get_matched_group,
                        filter(match_legislature, self.soup.find_all('a'))))

    def parse_steps(self):
        tree = self.build_step_tree()
        return tree.extract_data()

    def build_step_tree(self):
        root_node = DossierNode()
        current_node = root_node

        for element in filter(filter_dossier_element,
                              self.soup.find_all(['p', 'h2'])):
            new_node_class = BaseNode.match_node_class(element)

            if new_node_class:
                parent = current_node.get_relevant_parent(new_node_class)
                current_node = new_node_class(parent=parent)
                parent.add_child(current_node)

            current_node.add_element(element)

        return root_node

def clean_html(html):
    soup = BeautifulSoup(html, 'html5lib')
    if soup.body.header:
        soup.body.header.extract()
    md_text = html2text(str(soup.body), bodywidth=0)
    return mistune.markdown(md_text).replace('
\n', '
\n
')

def filter_dossier_element(element):
    return element.text.strip() and \
           not element.text.startswith('_') and \
           not element.text.startswith('Accueil')

class BaseNode(object):
    def __init__(self, parent=None):
        self.parent = parent
        self.elements = []
        self.children = []

    @classmethod
    def match(cls, html):
        raise NotImplementedError

    @classmethod
    def match_node_class(cls, element):
        for class_ in [LegislativeStepNode,
                       DiscussionSeancePubliqueNode,
                       DepotLoiNode,
                       ProcedureAccelereeNode,
                       AvisConseilEtatNode,
                       EtudeImpactNode,
                       DecisionNode]:
            if class_.match(element):
                return class_

    def get_relevant_parent(self, node_class):
        return self

    def add_child(self, node):
        self.children.append(node)

    def add_element(self, element):
        self.elements.append(element)

    def extract_data(self):
        raise NotImplementedError

    def __repr__(self):
        return '<%s [%s]>' % (
            self.__class__.__name__,
            self.elements[0].__repr__() if self.elements else '')

class DossierNode(BaseNode):
    def extract_data(self):
        return [step.extract_data() for step in self.children]

class LegislativeStepNode(BaseNode):
    steps_re = {
        LegislativeStepType.AN_PREMIERE_LECTURE:
            re.compile('^assemblée nationale - 1ère lecture',
                       re.I | re.UNICODE),
        LegislativeStepType.SENAT_PREMIERE_LECTURE:
            re.compile('^sénat - 1ère lecture',
                       re.I | re.UNICODE),
        LegislativeStepType.AN_DEUXIEME_LECTURE:
            re.compile('^assemblée nationale - 2e lecture',
                       re.I | re.UNICODE),
        LegislativeStepType.SENAT_DEUXIEME_LECTURE:
            re.compile('^sénat - 2e lecture',
                       re.I | re.UNICODE),
        LegislativeStepType.AN_NOUVELLE_LECTURE:
            re.compile('^assemblée nationale - nouvelle lecture',
                       re.I | re.UNICODE),
        LegislativeStepType.SENAT_NOUVELLE_LECTURE:
            re.compile('^sénat - nouvelle lecture',
                       re.I | re.UNICODE),
        LegislativeStepType.AN_LECTURE_DEFINITIVE:
            re.compile('^assemblée nationale - lecture définitive',
                       re.I | re.UNICODE),
        LegislativeStepType.CONSEIL_CONSTIT:
            re.compile('^conseil constitutionnel',
                       re.I | re.UNICODE),
        LegislativeStepType.CMP:
            re.compile('^commission mixte paritaire \((accord|désaccord)?\)$',
                       re.I | re.UNICODE)
    }

    @classmethod
    def match(cls, html):
        return any(map(lambda r: r.match(html.text), cls.steps_re.values()))

    def get_relevant_parent(self, node_class):
        return self.parent if node_class == LegislativeStepNode else self

    def extract_data(self):
        data = {
            'type': self.extract_type(),
            'acts': [act for child in self.children
                     for act in child.extract_data()]
        }

        if data['type'] == LegislativeStepType.CMP and self.elements:
            if 'Accord' in self.elements[0].text:
                data['status'] = 'ACCORD'
            if 'Désaccord' in self.elements[0].text:
                data['status'] = 'DESACCORD'

        return data

    def extract_type(self):
        if not self.elements:
            return

        return next(
            map(itemgetter(0),
                filter(lambda item: item[1].match(self.elements[0].text),
                       iteritems(self.steps_re))))

class LegislativeActNode(BaseNode):
    rule = None

    @classmethod
    def match(cls, html):
        return cls.rule.match(html.text) if cls.rule else False

    def get_relevant_parent(self, node_class):
        return self.parent if issubclass(node_class, LegislativeActNode)\
            else self.parent.parent

    def extract_data(self):
        raise NotImplementedError

class ProcedureAccelereeNode(LegislativeActNode):
    rule = re.compile('^le gouvernement a engagé la procédure accélérée',
                      re.I | re.UNICODE)

    def extract_data(self):
        if not self.elements:
            return

        matched_dates = re.findall(' le (\d+\s?\w* \\w+ \d{4})',
                                   self.elements[0].text,
                                   re.I | re.UNICODE)

        return [{
            'type': LegislativeActType.PROCEDURE_ACCELEREE,
            'date': extract_datetime(matched_dates[0])
            if matched_dates else None
        }]

class EtudeImpactNode(LegislativeActNode):
    rule = re.compile('^etude d\'impact', re.I | re.UNICODE)

    def extract_data(self):
        return [{
            'type': LegislativeActType.ETUDE_IMPACT,
            'url': urljoin(AN_BASE_URL, self.elements[0].a['href'])
        }]

class DepotLoiNode(LegislativeActNode):
    rule = re.compile('^(projet de loi|proposition de loi).+déposée? le',
                      re.I | re.UNICODE)

    @classmethod
    def match(cls, html):
        return cls.rule.match(html.text) and html.a if cls.rule else False

    def extract_data(self):
        if not self.elements:
            return

        return [{
            'type': LegislativeActType.DEPOT_INITIATIVE,
            'url': self.extract_url(),
            'date': self.extract_date()
        }]

    def extract_date(self):
        matched_dates = re.findall(' déposée? le (\d+ \w+ \d{4})',
                                   self.elements[0].text, re.UNICODE)

        return extract_datetime(matched_dates[0]) if matched_dates else None

    def extract_url(self):
        return urljoin(AN_BASE_URL, self.elements[0].a['href']) \
            if self.elements else None

    def extract_legislature(self):
        link = self.elements[0].a
        matched = re.match('/(\d{2})/', link['href'])

        if matched:
            return matched.group(1)

    def extract_procedure(self):
        if self.elements and 'Projet de loi' in self.elements[0].text:
            return ProcedureParlementaire.PJL
        elif self.elements and 'Proposition de loi' in self.elements[0].text:
            return ProcedureParlementaire.PPL

class DecisionNode(LegislativeActNode):
    rule = re.compile('^(projet de loi|proposition de loi)(?!.*déposé)',
                      re.I | re.UNICODE)

    def extract_data(self):
        status = None
        if 'adopté' in self.elements[0].text:
            status = DecisionStatus.ADOPTE
        elif 'modifié' in self.elements[0].text:
            status = DecisionStatus.MODIFIE
        elif 'rejeté' in self.elements[0].text:
            status = DecisionStatus.REJETE

        matched_dates = re.findall(' le (\d+\s?\w* \\w+ \d{4})',
                                   self.elements[0].text,
                                   re.I | re.UNICODE)

        return [{
            'type': LegislativeActType.DECISION,
            'status': status,
            'date': extract_datetime(matched_dates[0])
            if matched_dates else None,
            'url': urljoin(AN_BASE_URL, self.elements[0].a['href'])
            if self.elements[0].a else None
        }]

class DiscussionSeancePubliqueNode(LegislativeActNode):
    rule = re.compile('^discussion en séance publique', re.I | re.UNICODE)

    def extract_data(self):
        return self.extract_seances_from_first_element() \
            if self.seances_in_first_element() \
            else self.extract_seances_from_all_elements()

    def seances_in_first_element(self):
        return 'au cours des séances' in self.elements[0].text

    def extract_seances_from_first_element(self):
        return [{'url': urljoin(AN_BASE_URL, a['href']),
                 'type': LegislativeActType.DISCUSSION_SEANCE_PUBLIQUE}
                for a in self.elements[0].find_all('a')]

    def extract_seances_from_all_elements(self):
        def extract_seance(element):
            return {
                'url': urljoin(AN_BASE_URL, element.a['href'])
                if element.a else None,
                'date': extract_datetime(element.text.split('séance du ')[1]),
                'type': LegislativeActType.DISCUSSION_SEANCE_PUBLIQUE
            }

        def filter_seance(element):
            return 'séance du ' in element.text and \
                   'Scrutin' not in element.text

        return list(map(extract_seance, filter(filter_seance, self.elements)))

class AvisConseilEtatNode(LegislativeActNode):
    rule = re.compile('^avis du conseil d\'État', re.I | re.UNICODE)

    def extract_data(self):
        return [{
            'type': LegislativeActType.AVIS_CONSEIL_ETAT,
            'url': urljoin(AN_BASE_URL, self.elements[0].a['href'])
        }]
<commit_msg>be more liberal for legislative step detection<commit_after># -*- coding: utf-8 -*-
from __future__ import unicode_literals

import re
from builtins import map, filter, str
from operator import itemgetter

import mistune
import requests
from bs4 import BeautifulSoup
from future.utils import iteritems
from html2text import html2text
from six.moves.urllib.parse import urljoin

from anpy.utils import extract_datetime

AN_BASE_URL = 'http://www.assemblee-nationale.fr'

class InvalidResponseException(Exception):
    pass

class Dossier(object):
    def __init__(self, url=None, senat_url=None, title=None,
                 legislature=None, procedure=None, steps=None):
        self.url = url
        self.senat_url = senat_url
        self.title = title
        self.legislature = legislature
        self.procedure = procedure
        self.steps = steps or []

    @staticmethod
    def download_and_build(url):
        resp = requests.get(url)
        if resp.status_code >= 400:
            raise InvalidResponseException('%s: %d' % (url, resp.status_code))
        return DossierParser(url, resp.content).parse()

    def to_dict(self):
        return {
            'url': self.url,
            'senat_url': self.senat_url,
            'title': self.title,
            'legislature': self.legislature,
            'procedure': self.procedure,
            'steps': self.steps
        }

class ProcedureParlementaire(object):
    PJL = 'PJL'
    PPL = 'PPL'

class LegislativeStepType(object):
    AN_PREMIERE_LECTURE = 'AN_PREMIERE_LECTURE'
    SENAT_PREMIERE_LECTURE = 'SENAT_PREMIERE_LECTURE'
    AN_DEUXIEME_LECTURE = 'AN_DEUXIEME_LECTURE'
    SENAT_DEUXIEME_LECTURE = 'SENAT_DEUXIEME_LECTURE'
    CMP = 'CMP'
    CONSEIL_CONSTIT = 'CONSEIL_CONSTITUT'
    AN_NOUVELLE_LECTURE = 'AN_NOUVELLE_LECTURE'
    SENAT_NOUVELLE_LECTURE = 'SENAT_NOUVELLE_LECTURE'
    AN_LECTURE_DEFINITIVE = 'AN_LECTURE_DEFINITIVE'

class DecisionStatus(object):
    ADOPTE = 'ADOPTE'
    REJETE = 'REJETE'
    MODIFIE = 'MODIFIE'

class LegislativeActType(object):
    ETUDE_IMPACT = 'ETUDE_IMPACT'
    AVIS_CONSEIL_ETAT = 'AVIS_CONSEIL_DETAT'
    PROCEDURE_ACCELEREE = 'PROCEDURE_ACCELEREE'
    REUNION_COMMISSION = 'REUNION_COMMISSION'
    DISCUSSION_COMMISSION = 'DISCUSSION_COMMISSION'
    DEPOT_RAPPORT = 'DEPOT_RAPPORT'
    SAISIE_COM_AVIS = 'SAISIE_COM_AVIS'
    NOMINATION_RAPPORTEURS = 'NOMINATION_RAPPORTEURS'
    DISCUSSION_SEANCE_PUBLIQUE = 'DISCUSSION_SEANCE_PUBLIQUE'
    DECISION = 'DECISION'
    DEPOT_INITIATIVE = 'DEPOT_INITIATIVE'
    SAISIE_COM_FOND = 'SAISIE_COM_FOND'
    PROMULGATION = 'PROMULGATION'
    SAISINE_CONSEIL_CONSTIT = 'SAISINE_CONSEIL_CONSTIT'

class DossierParser(object):
    def __init__(self, url, html):
        self.url = url
        self.html = html
        self.soup = BeautifulSoup(clean_html(html), 'html5lib')

    def parse(self):
        return Dossier(
            url=self.url,
            title=self.parse_title(),
            legislature=self.parse_legislature(),
            procedure=self.parse_procedure(),
            steps=self.parse_steps(),
            senat_url=self.parse_senat_url())

    def parse_title(self):
        return self.soup.strong.text if self.soup.strong else None

    def parse_procedure(self):
        re_procedure = re.compile('(projet de loi|proposition de loi)', re.I)

        def match_procedure(a):
            return re_procedure.match(a.text)

        def get_procedure(a):
            proc = match_procedure(a).group(1).lower()
            if proc == 'projet de loi':
                return ProcedureParlementaire.PJL
            elif proc == 'proposition de loi':
                return ProcedureParlementaire.PPL

        return next(
            map(get_procedure,
                filter(match_procedure, self.soup.find_all('a'))), None)

    def parse_senat_url(self):
        re_url = re.compile('http://www.senat.fr/dossier-legislatif')
        re_url_alt = re.compile('http://www.senat.fr/dossierleg/')

        return next(
            filter(lambda url: re_url.match(url) or re_url_alt.match(url),
                   map(lambda a: a['href'], self.soup.find_all('a'))), None)

    def parse_legislature(self):
        re_legislature = re.compile('/(\d{2})/')

        def match_legislature(a):
            return re_legislature.match(a['href'])

        def get_matched_group(a):
            return match_legislature(a).group(1) \
                if match_legislature(a) else None

        return next(map(get_matched_group,
                        filter(match_legislature, self.soup.find_all('a'))))

    def parse_steps(self):
        tree = self.build_step_tree()
        return tree.extract_data()

    def build_step_tree(self):
        root_node = DossierNode()
        current_node = root_node

        for element in filter(filter_dossier_element,
                              self.soup.find_all(['p', 'h2'])):
            new_node_class = BaseNode.match_node_class(element)

            if new_node_class:
                parent = current_node.get_relevant_parent(new_node_class)
                current_node = new_node_class(parent=parent)
                parent.add_child(current_node)

            current_node.add_element(element)

        return root_node

def clean_html(html):
    soup = BeautifulSoup(html, 'html5lib')
    if soup.body.header:
        soup.body.header.extract()
    md_text = html2text(str(soup.body), bodywidth=0)
    return mistune.markdown(md_text).replace('
\n', '
\n
')

def filter_dossier_element(element):
    return element.text.strip() and \
           not element.text.startswith('_') and \
           not element.text.startswith('Accueil')

class BaseNode(object):
    def __init__(self, parent=None):
        self.parent = parent
        self.elements = []
        self.children = []

    @classmethod
    def match(cls, html):
        raise NotImplementedError

    @classmethod
    def match_node_class(cls, element):
        for class_ in [LegislativeStepNode,
                       DiscussionSeancePubliqueNode,
                       DepotLoiNode,
                       ProcedureAccelereeNode,
                       AvisConseilEtatNode,
                       EtudeImpactNode,
                       DecisionNode]:
            if class_.match(element):
                return class_

    def get_relevant_parent(self, node_class):
        return self

    def add_child(self, node):
        self.children.append(node)

    def add_element(self, element):
        self.elements.append(element)

    def extract_data(self):
        raise NotImplementedError

    def __repr__(self):
        return '<%s [%s]>' % (
            self.__class__.__name__,
            self.elements[0].__repr__() if self.elements else '')

class DossierNode(BaseNode):
    def extract_data(self):
        return [step.extract_data() for step in self.children]

class LegislativeStepNode(BaseNode):
    steps_re = {
        LegislativeStepType.AN_PREMIERE_LECTURE:
            re.compile('^assemblée nationale - 1 ?ère lecture',
                       re.I | re.UNICODE),
        LegislativeStepType.SENAT_PREMIERE_LECTURE:
            re.compile('^sénat - 1 ?ère lecture',
                       re.I | re.UNICODE),
        LegislativeStepType.AN_DEUXIEME_LECTURE:
            re.compile('^assemblée nationale - 2e lecture',
                       re.I | re.UNICODE),
        LegislativeStepType.SENAT_DEUXIEME_LECTURE:
            re.compile('^sénat - 2e lecture',
                       re.I | re.UNICODE),
        LegislativeStepType.AN_NOUVELLE_LECTURE:
            re.compile('^assemblée nationale - nouvelle lecture',
                       re.I | re.UNICODE),
        LegislativeStepType.SENAT_NOUVELLE_LECTURE:
            re.compile('^sénat - nouvelle lecture',
                       re.I | re.UNICODE),
        LegislativeStepType.AN_LECTURE_DEFINITIVE:
            re.compile('^assemblée nationale - lecture définitive',
                       re.I | re.UNICODE),
        LegislativeStepType.CONSEIL_CONSTIT:
            re.compile('^conseil constitutionnel',
                       re.I | re.UNICODE),
        LegislativeStepType.CMP:
            re.compile('^commission mixte paritaire \((accord|désaccord)?\)$',
                       re.I | re.UNICODE)
    }

    @classmethod
    def match(cls, html):
        return any(map(lambda r: r.match(html.text), cls.steps_re.values()))

    def get_relevant_parent(self, node_class):
        return self.parent if node_class == LegislativeStepNode else self

    def extract_data(self):
        data = {
            'type': self.extract_type(),
            'acts': [act for child in self.children
                     for act in child.extract_data()]
        }

        if data['type'] == LegislativeStepType.CMP and self.elements:
            if 'Accord' in self.elements[0].text:
                data['status'] = 'ACCORD'
            if 'Désaccord' in self.elements[0].text:
                data['status'] = 'DESACCORD'

        return data

    def extract_type(self):
        if not self.elements:
            return

        return next(
            map(itemgetter(0),
                filter(lambda item: item[1].match(self.elements[0].text),
                       iteritems(self.steps_re))))

class LegislativeActNode(BaseNode):
    rule = None

    @classmethod
    def match(cls, html):
        return cls.rule.match(html.text) if cls.rule else False

    def get_relevant_parent(self, node_class):
        return self.parent if issubclass(node_class, LegislativeActNode)\
            else self.parent.parent

    def extract_data(self):
        raise NotImplementedError

class ProcedureAccelereeNode(LegislativeActNode):
    rule = re.compile('^le gouvernement a engagé la procédure accélérée',
                      re.I | re.UNICODE)

    def extract_data(self):
        if not self.elements:
            return

        matched_dates = re.findall(' le (\d+\s?\w* \\w+ \d{4})',
                                   self.elements[0].text,
                                   re.I | re.UNICODE)

        return [{
            'type': LegislativeActType.PROCEDURE_ACCELEREE,
            'date': extract_datetime(matched_dates[0])
            if matched_dates else None
        }]

class EtudeImpactNode(LegislativeActNode):
    rule = re.compile('^etude d\'impact', re.I | re.UNICODE)

    def extract_data(self):
        return [{
            'type': LegislativeActType.ETUDE_IMPACT,
            'url': urljoin(AN_BASE_URL, self.elements[0].a['href'])
        }]

class DepotLoiNode(LegislativeActNode):
    rule = re.compile('^(projet de loi|proposition de loi).+déposée? le',
                      re.I | re.UNICODE)

    @classmethod
    def match(cls, html):
        return cls.rule.match(html.text) and html.a if cls.rule else False

    def extract_data(self):
        if not self.elements:
            return

        return [{
            'type': LegislativeActType.DEPOT_INITIATIVE,
            'url': self.extract_url(),
            'date': self.extract_date()
        }]

    def extract_date(self):
        matched_dates = re.findall(' déposée? le (\d+ \w+ \d{4})',
                                   self.elements[0].text, re.UNICODE)

        return extract_datetime(matched_dates[0]) if matched_dates else None

    def extract_url(self):
        return urljoin(AN_BASE_URL, self.elements[0].a['href']) \
            if self.elements else None

    def extract_legislature(self):
        link = self.elements[0].a
        matched = re.match('/(\d{2})/', link['href'])

        if matched:
            return matched.group(1)

    def extract_procedure(self):
        if self.elements and 'Projet de loi' in self.elements[0].text:
            return ProcedureParlementaire.PJL
        elif self.elements and 'Proposition de loi' in self.elements[0].text:
            return ProcedureParlementaire.PPL

class DecisionNode(LegislativeActNode):
    rule = re.compile('^(projet de loi|proposition de loi)(?!.*déposé)',
                      re.I | re.UNICODE)

    def extract_data(self):
        status = None
        if 'adopté' in self.elements[0].text:
            status = DecisionStatus.ADOPTE
        elif 'modifié' in self.elements[0].text:
            status = DecisionStatus.MODIFIE
        elif 'rejeté' in self.elements[0].text:
            status = DecisionStatus.REJETE

        matched_dates = re.findall(' le (\d+\s?\w* \\w+ \d{4})',
                                   self.elements[0].text,
                                   re.I | re.UNICODE)

        return [{
            'type': LegislativeActType.DECISION,
            'status': status,
            'date': extract_datetime(matched_dates[0])
            if matched_dates else None,
            'url': urljoin(AN_BASE_URL, self.elements[0].a['href'])
            if self.elements[0].a else None
        }]

class DiscussionSeancePubliqueNode(LegislativeActNode):
    rule = re.compile('^discussion en séance publique', re.I | re.UNICODE)

    def extract_data(self):
        return self.extract_seances_from_first_element() \
            if self.seances_in_first_element() \
            else self.extract_seances_from_all_elements()

    def seances_in_first_element(self):
        return 'au cours des séances' in self.elements[0].text

    def extract_seances_from_first_element(self):
        return [{'url': urljoin(AN_BASE_URL, a['href']),
                 'type': LegislativeActType.DISCUSSION_SEANCE_PUBLIQUE}
                for a in self.elements[0].find_all('a')]

    def extract_seances_from_all_elements(self):
        def extract_seance(element):
            return {
                'url': urljoin(AN_BASE_URL, element.a['href'])
                if element.a else None,
                'date': extract_datetime(element.text.split('séance du ')[1]),
                'type': LegislativeActType.DISCUSSION_SEANCE_PUBLIQUE
            }

        def filter_seance(element):
            return 'séance du ' in element.text and \
                   'Scrutin' not in element.text

        return list(map(extract_seance, filter(filter_seance, self.elements)))

class AvisConseilEtatNode(LegislativeActNode):
    rule = re.compile('^avis du conseil d\'État', re.I | re.UNICODE)

    def extract_data(self):
        return [{
            'type': LegislativeActType.AVIS_CONSEIL_ETAT,
            'url': urljoin(AN_BASE_URL, self.elements[0].a['href'])
        }]
<|endoftext|>


<commit_before># Write the benchmarking functions here.
# See "Writing benchmarks" in the asv docs for more information.
import os.path as osp
import numpy as np
from tempfile import TemporaryDirectory
import h5py

class TimeSuite:
    """
    An example benchmark that times the performance of various kinds
    of iterating over dictionaries in Python.
    """
    def setup(self):
        self._td = TemporaryDirectory()
        path = osp.join(self._td.name, 'test.h5')
        with h5py.File(path, 'w') as f:
            f['a'] = np.arange(100000)

        self.f = h5py.File(path, 'r')

    def teardown(self):
        self.f.close()
        self._td.cleanup()

    def time_many_small_reads(self):
        ds = self.f['a']
        for i in range(10000):
            arr = ds[i * 10:(i + 1) * 10]
<commit_msg>Add benchmark for writing with index of last dimension<commit_after># Write the benchmarking functions here.
# See "Writing benchmarks" in the asv docs for more information.
import os.path as osp
import numpy as np
from tempfile import TemporaryDirectory
import h5py

class TimeSuite:
    """
    An example benchmark that times the performance of various kinds
    of iterating over dictionaries in Python.
    """
    def setup(self):
        self._td = TemporaryDirectory()
        path = osp.join(self._td.name, 'test.h5')
        with h5py.File(path, 'w') as f:
            f['a'] = np.arange(100000)

        self.f = h5py.File(path, 'r')

    def teardown(self):
        self.f.close()
        self._td.cleanup()

    def time_many_small_reads(self):
        ds = self.f['a']
        for i in range(10000):
            arr = ds[i * 10:(i + 1) * 10]

class WritingTimeSuite:
    """Based on example in GitHub issue 492:
    https://github.com/h5py/h5py/issues/492
    """
    def setup(self):
        self._td = TemporaryDirectory()
        path = osp.join(self._td.name, 'test.h5')
        self.f = h5py.File(path, 'w')
        self.shape = shape = (128, 1024, 512)
        self.f.create_dataset(
            'a', shape=shape, dtype=np.float32, chunks=(1, shape[1], 64)
        )

    def teardown(self):
        self.f.close()
        self._td.cleanup()

    def time_write_index_last_axis(self):
        ds = self.f['a']
        data = np.zeros(self.shape[:2])
        for i in range(self.shape[2]):
            ds[..., i] = data

    def time_write_slice_last_axis(self):
        ds = self.f['a']
        data = np.zeros(self.shape[:2])
        for i in range(self.shape[2]):
            ds[..., i:i+1] = data[..., np.newaxis]
<|endoftext|>


<commit_before>from openid import cryptutil
import copy

class ServerAssocs(object):
    def __init__(self):
        self.assocs = {}

    def set(self, assoc):
        self.assocs[assoc.handle] = assoc

    def get(self, handle):
        return self.assocs.get(handle)

    def remove(self, handle):
        try:
            del self.assocs[handle]
        except KeyError:
            return False
        else:
            return True

    def best(self):
        """Returns association with the oldest issued date.

        or None if there are no associations.
        """
        best = None
        for assoc in self.assocs.values():
            if best is None or best.issued < assoc.issued:
                best = assoc
        return best

class MemoryStore(object):
    """In-process memory store.

    Use for single long-running processes.  No persistence supplied.
    """
    def __init__(self):
        self.server_assocs = {}
        self.nonces = {}

    def _getServerAssocs(self, server_url):
        try:
            return self.server_assocs[server_url]
        except KeyError:
            assocs = self.server_assocs[server_url] = ServerAssocs()
            return assocs

    def storeAssociation(self, server_url, assoc):
        assocs = self._getServerAssocs(server_url)
        assocs.set(copy.deepcopy(assoc))

    def getAssociation(self, server_url, handle=None):
        assocs = self._getServerAssocs(server_url)
        if handle is None:
            return assocs.best()
        else:
            return assocs.get(handle)

    def removeAssociation(self, server_url, handle):
        assocs = self._getServerAssocs(server_url)
        return assocs.remove(handle)

    def useNonce(self, server_url, timestamp, salt):
        nonce = (str(server_url), int(timestamp), str(salt))
        if nonce in self.nonces:
            return False
        else:
            self.nonces[nonce] = None
            return True

    def getExpired(self):
        expired = []
        for server_url, assocs in self.server_assocs.iteritems():
            best = assocs.best()
            if best is None or best.getExpiresIn() == 0:
                print best
                expired.append(server_url)

        return expired

    def __eq__(self, other):
        return ((self.server_assocs == other.server_assocs) and
                (self.nonces == other.nonces))

    def __ne__(self, other):
        return not (self == other)
<commit_msg>[project @ store.memstore: add module docstring]<commit_after>"""A simple store using only in-process memory."""
from openid import cryptutil
import copy

class ServerAssocs(object):
    def __init__(self):
        self.assocs = {}

    def set(self, assoc):
        self.assocs[assoc.handle] = assoc

    def get(self, handle):
        return self.assocs.get(handle)

    def remove(self, handle):
        try:
            del self.assocs[handle]
        except KeyError:
            return False
        else:
            return True

    def best(self):
        """Returns association with the oldest issued date.

        or None if there are no associations.
        """
        best = None
        for assoc in self.assocs.values():
            if best is None or best.issued < assoc.issued:
                best = assoc
        return best

class MemoryStore(object):
    """In-process memory store.

    Use for single long-running processes.  No persistence supplied.
    """
    def __init__(self):
        self.server_assocs = {}
        self.nonces = {}

    def _getServerAssocs(self, server_url):
        try:
            return self.server_assocs[server_url]
        except KeyError:
            assocs = self.server_assocs[server_url] = ServerAssocs()
            return assocs

    def storeAssociation(self, server_url, assoc):
        assocs = self._getServerAssocs(server_url)
        assocs.set(copy.deepcopy(assoc))

    def getAssociation(self, server_url, handle=None):
        assocs = self._getServerAssocs(server_url)
        if handle is None:
            return assocs.best()
        else:
            return assocs.get(handle)

    def removeAssociation(self, server_url, handle):
        assocs = self._getServerAssocs(server_url)
        return assocs.remove(handle)

    def useNonce(self, server_url, timestamp, salt):
        nonce = (str(server_url), int(timestamp), str(salt))
        if nonce in self.nonces:
            return False
        else:
            self.nonces[nonce] = None
            return True

    def getExpired(self):
        expired = []
        for server_url, assocs in self.server_assocs.iteritems():
            best = assocs.best()
            if best is None or best.getExpiresIn() == 0:
                print best
                expired.append(server_url)

        return expired

    def __eq__(self, other):
        return ((self.server_assocs == other.server_assocs) and
                (self.nonces == other.nonces))

    def __ne__(self, other):
        return not (self == other)
<|endoftext|>


<commit_before>"""
OpenNSA NSI Service -> Backend adaptor (router).

Author: Henrik Thostrup Jensen <htj@nordu.net>
Copyright: NORDUnet (2011)
"""

import random

from zope.interface import implements

from twisted.python import log
from twisted.internet import defer

from opennsa.interface import NSIServiceInterface
from opennsa import nsa, error, topology, jsonrpc, proxy, connection

class NSIAggregator:

    implements(NSIServiceInterface)

    def __init__(self, network, backend, topology_file):
        self.network = network
        self.backend = backend

        self.topology = topology.Topology()
        self.topology.parseTopology(open(topology_file))

        # get own nsa from topology
        self.nsa = self.topology.getNetwork(self.network).nsa
        self.proxy = proxy.NSIProxy(jsonrpc.JSONRPCNSIClient(), self.nsa, self.topology)

        self.connections = {} # persistence, ha!

    # utility functionality

    def getConnection(self, requester_nsa, connection_id):

        conn = self.connections.get(requester_nsa.address, {}).get(connection_id, None)
        if conn is None:
            raise error.NoSuchConnectionError('No connection with id %s for NSA with address %s' % (connection_id, requester_nsa.address))
        else:
            return conn

    # command functionality

    def reserve(self, requester_nsa, provider_nsa, connection_id, global_reservation_id, description, service_parameters, session_security_attributes):

#        log.msg("Reserve request: %s, %s, %s" % (connection_id, global_reservation_id, description))

        nsa_identity = requester_nsa.address

        if connection_id in self.connections.get(nsa_identity, {}):
            raise error.ReserveError('Reservation with connection id %s already exists' % connection_id)

        source_stp = service_parameters.source_stp
        dest_stp   = service_parameters.dest_stp

        conn = connection.Connection(connection_id, source_stp, dest_stp, global_reservation_id)

        def reservationMade(results):

            local_conn = results[0][1]

            conn.switchState(connection.RESERVED)
            self.connections.setdefault(nsa_identity, {})[connection_id] = conn
            log.msg('Reservation created. Connection id: %s (%s). Global id %s' % (connection_id, local_conn.internal_reservation_id, global_reservation_id), system='opennsa.NSIAggregator')
            return conn.connection_id

        def localReservationMade(local_conn):
            # should we do anything here..?
            return local_conn

        # figure out nature of request

        link_info = ( connection_id, source_stp.network, source_stp.endpoint, dest_stp.network, dest_stp.endpoint, self.network)

        if source_stp.network == self.network and dest_stp.network == self.network:
            log.msg('Reserve %s: Simple link creation: %s:%s -> %s:%s (%s)' % link_info, system='opennsa.NSIAggregator')
            # make an internal link, no sub requests

            local_conn = connection.LocalConnection(source_stp.endpoint, dest_stp.endpoint, backend=self.backend)

            conn.local_connection = local_conn
            conn.switchState(connection.RESERVING)

            d = local_conn.reserve(service_parameters)
            d.addCallback(localReservationMade)

            dl = defer.DeferredList( [ d ] )
            dl.addCallback(reservationMade)
            return dl

        elif source_stp.network == self.network:
            # make link and chain on - common chaining
            log.msg('Reserve %s: Common chain creation: %s:%s -> %s:%s (%s)' % link_info, system='opennsa.NSIAggregator')

            links = self.topology.findLinks(source_stp, dest_stp)
            # check for no links
            links.sort(key=lambda e : len(e.endpoint_pairs))
            selected_link = links[0] # shortest link
            log.msg('Attempting to create link %s' % selected_link, system='opennsa.NSIAggregator')

            assert selected_link.source_stp.network == self.network

            chain_network = selected_link.endpoint_pairs[0].stp2.network

            conn.switchState(connection.RESERVING)

            def issueChainReservation(local_conn):

                def chainedReservationMade(sub_conn):
                    # do we need this for something?
                    return sub_conn

                sub_conn_id = 'int-ccid' + ''.join( [ str(int(random.random() * 10)) for _ in range(4) ] )

                new_source_stp      = selected_link.endpoint_pairs[0].stp2
                new_service_params  = nsa.ServiceParameters('', '', new_source_stp, dest_stp)

                sub_conn = connection.SubConnection(sub_conn_id, chain_network, new_source_stp, dest_stp, proxy=self.proxy)
                conn.sub_connections.append(sub_conn)

                d = sub_conn.reserve(new_service_params, global_reservation_id, description)
                d.addCallback(chainedReservationMade)
                d.addCallback(lambda sub_conn : [ (True, local_conn), (True, conn) ] )
                return d

            local_conn = connection.LocalConnection(selected_link.source_stp.endpoint, selected_link.endpoint_pairs[0].stp1.endpoint, backend=self.backend)

            conn.local_connection = local_conn

            d = local_conn.reserve(service_parameters) # should probably make a new service params...
            d.addCallback(localReservationMade)
            d.addCallback(issueChainReservation)
            d.addCallback(reservationMade)
            return d

        elif dest_stp.network == self.network:
            # make link and chain on - backwards chaining
            log.msg('Backwards chain creation: %s:%s -> %s:%s (%s)' % link_info, system='opennsa.NSIAggregator')
            raise NotImplementedError('Backwards chain reservation')

        else:
            log.msg('Tree creation:  %s:%s -> %s:%s (%s)' % link_info, system='opennsa.NSIAggregator')
            raise NotImplementedError('Tree reservation')

    def cancelReservation(self, requester_nsa, provider_nsa, connection_id, session_security_attributes):

        def connectionCancelled(results):
            conn.switchState(connection.CANCELLED)
            successes = [ r[0] for r in results ]
            if all(successes):
                if len(successes) > 1:
                    log.msg('Connection %s and all sub connections(%i) cancelled' % (conn.connection_id, len(results)-1), system='opennsa.NSIAggregator')
                return conn.connection_id
            if any(successes):
                print "Partial cancelation, gahh"
            else:
                log.msg('Failed to cancel connection %s and all sub connections(%i)' % (conn.connection_id, len(results)-1), system='opennsa.NSIAggregator')

        conn = self.getConnection(requester_nsa, connection_id)

        conn.switchState(connection.CANCELLING)

        defs = []
        for sc in conn.connections():
            d = sc.cancelReservation()
            defs.append(d)

        dl = defer.DeferredList(defs)
        dl.addCallback(connectionCancelled)
        return dl

    def provision(self, requester_nsa, provider_nsa, connection_id, session_security_attributes):

        def provisionComplete(results):
            conn.switchState(connection.PROVISIONED)
            if len(results) > 1:
                log.msg('Connection %s and all sub connections(%i) provisioned' % (connection_id, len(results)-1), system='opennsa.NSIAggregator')
            return connection_id

        conn = self.getConnection(requester_nsa, connection_id)

        conn.switchState(connection.PROVISIONING)

        defs = []
        for sc in conn.connections():
            d = sc.provision()
            defs.append(d)

        dl = defer.DeferredList(defs)
        dl.addCallback(provisionComplete)
        return dl

    def releaseProvision(self, requester_nsa, provider_nsa, connection_id, session_security_attributes):

        def connectionReleased(results):
            conn.switchState(connection.RESERVED)
            if len(results) > 1:
                log.msg('Connection %s and all sub connections(%i) released' % (connection_id, len(results)-1), system='opennsa.NSIAggregator')
            return conn.connection_id

        conn = self.getConnection(requester_nsa, connection_id)
        conn.switchState(connection.RELEASING)

        defs = []
        for sc in conn.connections():
            d = sc.releaseProvision()
            defs.append(d)

        dl = defer.DeferredList(defs)
        dl.addCallback(connectionReleased)
        return dl

    def query(self, requester_nsa, provider_nsa, query_filter, session_security_attributes):

        log.msg('', system='opennsa.NSIAggregator')

<commit_msg>clean up reservation code a bit (approaching readable now)<commit_after>"""
OpenNSA NSI Service -> Backend adaptor (router).

Author: Henrik Thostrup Jensen <htj@nordu.net>
Copyright: NORDUnet (2011)
"""

import random

from zope.interface import implements

from twisted.python import log
from twisted.internet import defer

from opennsa.interface import NSIServiceInterface
from opennsa import nsa, error, topology, jsonrpc, proxy, connection

class NSIAggregator:

    implements(NSIServiceInterface)

    def __init__(self, network, backend, topology_file):
        self.network = network
        self.backend = backend

        self.topology = topology.Topology()
        self.topology.parseTopology(open(topology_file))

        # get own nsa from topology
        self.nsa = self.topology.getNetwork(self.network).nsa
        self.proxy = proxy.NSIProxy(jsonrpc.JSONRPCNSIClient(), self.nsa, self.topology)

        self.connections = {} # persistence, ha!

    # utility functionality

    def getConnection(self, requester_nsa, connection_id):

        conn = self.connections.get(requester_nsa.address, {}).get(connection_id, None)
        if conn is None:
            raise error.NoSuchConnectionError('No connection with id %s for NSA with address %s' % (connection_id, requester_nsa.address))
        else:
            return conn

    # command functionality

    def reserve(self, requester_nsa, provider_nsa, connection_id, global_reservation_id, description, service_parameters, session_security_attributes):

        nsa_identity = requester_nsa.address

        if connection_id in self.connections.get(nsa_identity, {}):
            raise error.ReserveError('Reservation with connection id %s already exists' % connection_id)

        source_stp = service_parameters.source_stp
        dest_stp   = service_parameters.dest_stp

        conn = connection.Connection(connection_id, source_stp, dest_stp, global_reservation_id)

        def reservationMade(results):
            local_conn = results[0][1]
            conn.switchState(connection.RESERVED)
            self.connections.setdefault(nsa_identity, {})[connection_id] = conn
            log.msg('Reservation created. Connection id: %s (%s). Global id %s' % (connection_id, local_conn.internal_reservation_id, global_reservation_id), system='opennsa.NSIAggregator')
            return conn.connection_id

        # figure out nature of request

        link_info = ( connection_id, source_stp.network, source_stp.endpoint, dest_stp.network, dest_stp.endpoint, self.network)

        if source_stp.network == self.network and dest_stp.network == self.network:
            log.msg('Reserve %s: Simple link creation: %s:%s -> %s:%s (%s)' % link_info, system='opennsa.NSIAggregator')
            # make an internal link, no sub requests

            local_conn = connection.LocalConnection(source_stp.endpoint, dest_stp.endpoint, backend=self.backend)

            conn.local_connection = local_conn
            conn.switchState(connection.RESERVING)

            d = local_conn.reserve(service_parameters)
            dl = defer.DeferredList( [ d ] )
            dl.addCallback(reservationMade)
            return dl

        elif source_stp.network == self.network:
            # make link and chain on - common chaining
            log.msg('Reserve %s: Common chain creation: %s:%s -> %s:%s (%s)' % link_info, system='opennsa.NSIAggregator')

            links = self.topology.findLinks(source_stp, dest_stp)
            # check for no links
            links.sort(key=lambda e : len(e.endpoint_pairs))
            selected_link = links[0] # shortest link
            log.msg('Attempting to create link %s' % selected_link, system='opennsa.NSIAggregator')

            assert selected_link.source_stp.network == self.network

            chain_network = selected_link.endpoint_pairs[0].stp2.network

            def issueChainReservation(local_conn):

                sub_conn_id = 'int-ccid' + ''.join( [ str(int(random.random() * 10)) for _ in range(4) ] )
                new_source_stp      = selected_link.endpoint_pairs[0].stp2
                new_service_params  = nsa.ServiceParameters('', '', new_source_stp, dest_stp)

                sub_conn = connection.SubConnection(sub_conn_id, chain_network, new_source_stp, dest_stp, proxy=self.proxy)
                conn.sub_connections.append(sub_conn)

                d = sub_conn.reserve(new_service_params, global_reservation_id, description)
                d.addCallback(lambda sub_conn : [ (True, local_conn), (True, conn) ] )
                return d

            local_conn = connection.LocalConnection(selected_link.source_stp.endpoint, selected_link.endpoint_pairs[0].stp1.endpoint, backend=self.backend)

            conn.local_connection = local_conn
            conn.switchState(connection.RESERVING)

            d = local_conn.reserve(service_parameters) # should probably make a new service params...
            d.addCallback(issueChainReservation)
            d.addCallback(reservationMade)
            return d

        elif dest_stp.network == self.network:
            # make link and chain on - backwards chaining
            log.msg('Backwards chain creation: %s:%s -> %s:%s (%s)' % link_info, system='opennsa.NSIAggregator')
            raise NotImplementedError('Backwards chain reservation')

        else:
            log.msg('Tree creation:  %s:%s -> %s:%s (%s)' % link_info, system='opennsa.NSIAggregator')
            raise NotImplementedError('Tree reservation')

    def cancelReservation(self, requester_nsa, provider_nsa, connection_id, session_security_attributes):

        def connectionCancelled(results):
            conn.switchState(connection.CANCELLED)
            successes = [ r[0] for r in results ]
            if all(successes):
                if len(successes) > 1:
                    log.msg('Connection %s and all sub connections(%i) cancelled' % (conn.connection_id, len(results)-1), system='opennsa.NSIAggregator')
                return conn.connection_id
            if any(successes):
                print "Partial cancelation, gahh"
            else:
                log.msg('Failed to cancel connection %s and all sub connections(%i)' % (conn.connection_id, len(results)-1), system='opennsa.NSIAggregator')

        conn = self.getConnection(requester_nsa, connection_id)

        conn.switchState(connection.CANCELLING)

        defs = []
        for sc in conn.connections():
            d = sc.cancelReservation()
            defs.append(d)

        dl = defer.DeferredList(defs)
        dl.addCallback(connectionCancelled)
        return dl

    def provision(self, requester_nsa, provider_nsa, connection_id, session_security_attributes):

        def provisionComplete(results):
            conn.switchState(connection.PROVISIONED)
            if len(results) > 1:
                log.msg('Connection %s and all sub connections(%i) provisioned' % (connection_id, len(results)-1), system='opennsa.NSIAggregator')
            return connection_id

        conn = self.getConnection(requester_nsa, connection_id)

        conn.switchState(connection.PROVISIONING)

        defs = []
        for sc in conn.connections():
            d = sc.provision()
            defs.append(d)

        dl = defer.DeferredList(defs)
        dl.addCallback(provisionComplete)
        return dl

    def releaseProvision(self, requester_nsa, provider_nsa, connection_id, session_security_attributes):

        def connectionReleased(results):
            conn.switchState(connection.RESERVED)
            if len(results) > 1:
                log.msg('Connection %s and all sub connections(%i) released' % (connection_id, len(results)-1), system='opennsa.NSIAggregator')
            return conn.connection_id

        conn = self.getConnection(requester_nsa, connection_id)
        conn.switchState(connection.RELEASING)

        defs = []
        for sc in conn.connections():
            d = sc.releaseProvision()
            defs.append(d)

        dl = defer.DeferredList(defs)
        dl.addCallback(connectionReleased)
        return dl

    def query(self, requester_nsa, provider_nsa, query_filter, session_security_attributes):

        log.msg('', system='opennsa.NSIAggregator')

<|endoftext|>


<commit_before>from openpassword.exceptions import InvalidPasswordException

class Keychain:

    def __init__(self, encryption_key):
        self.encryption_key = encryption_key
        self._locked = True

    def unlock(self, password):
        try:
            self.encryption_key.decrypt(password)
            self._locked = False
        except InvalidPasswordException as e:
            self._locked = True
            raise(e)

    def lock(self):
        self._locked = True

    def is_locked(self):
        return self._locked
<commit_msg>Remove unecessary parentesis from raise call<commit_after>from openpassword.exceptions import InvalidPasswordException

class Keychain:

    def __init__(self, encryption_key):
        self.encryption_key = encryption_key
        self._locked = True

    def unlock(self, password):
        try:
            self.encryption_key.decrypt(password)
            self._locked = False
        except InvalidPasswordException as e:
            self._locked = True
            raise e

    def lock(self):
        self._locked = True

    def is_locked(self):
        return self._locked
<|endoftext|>


<commit_before># vim: tabstop=4 shiftwidth=4 softtabstop=4

#    Copyright (C) 2012 Yahoo! Inc. All Rights Reserved.
#    Copyright (C) 2012 New Dream Network, LLC (DreamHost) All Rights Reserved.
#
#    Licensed under the Apache License, Version 2.0 (the "License"); you may
#    not use this file except in compliance with the License. You may obtain
#    a copy of the License at
#
#         http://www.apache.org/licenses/LICENSE-2.0
#
#    Unless required by applicable law or agreed to in writing, software
#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
#    License for the specific language governing permissions and limitations
#    under the License.

import collections
import copy
import glob
import jsonpatch
import os
import platform
import re
import shlex

import six

from anvil import exceptions as excp
from anvil import importer
from anvil import log as logging
from anvil import pprint
from anvil import shell as sh
from anvil import utils

LOG = logging.getLogger(__name__)

Component = collections.namedtuple(  # pylint: disable=C0103
    "Component", 'entry_point,options,siblings')

class Distro(object):
    def __init__(self,
                 name, platform_pattern,
                 install_helper, dependency_handler,
                 commands, components, **kwargs):
        self.name = name
        self._platform_pattern_text = platform_pattern
        self._platform_pattern = re.compile(platform_pattern, re.IGNORECASE)
        self._install_helper = install_helper
        self._dependency_handler = dependency_handler
        self._commands = commands
        self._components = components
        self.inject_platform_overrides(kwargs)

    def inject_platform_overrides(self, potential_data, source='??'):
        d = _linux_distribution()

        def merge(merge_into):
            if 'platform_overrides' in potential_data:
                overrides = potential_data['platform_overrides']
                if d in overrides:
                    LOG.info("Merging in 'platform_overrides' that matched distro %s from %s", d, source)
                    return utils.recursive_merge(merge_into, overrides[d])
                else:
                    LOG.debug("Distro %s not in 'platform_overrides' (valid are %s) from %s",
                              d, list(six.iterkeys(overrides)), source)
            return merge_into

        self._dependency_handler = merge(self._dependency_handler)

    def pformat(self, item_max_len=None):
        data = {
            'name': self.name,
            'dependency_handler': self._dependency_handler,
            'commands': self._commands,
            'pattern': "/%s/i" % self._platform_pattern_text,
        }
        return pprint.pformat(data, item_max_len=item_max_len)

    def _fetch_value(self, root, keys, quiet):
        end_key = keys[-1]
        for k in keys[0:-1]:
            if quiet:
                root = root.get(k)
                if root is None:
                    return None
            else:
                root = root[k]
        end_value = None
        if not quiet:
            end_value = root[end_key]
        else:
            end_value = root.get(end_key)
        return end_value

    def get_dependency_config(self, key, *more_keys, **kwargs):
        root = dict(self._dependency_handler)
        # NOTE(harlowja): Don't allow access to the dependency handler class
        # name. Access should be via the property instead.
        root.pop('name', None)
        keys = [key] + list(more_keys)
        return self._fetch_value(root, keys, kwargs.get('quiet', False))

    def get_command_config(self, key, *more_keys, **kwargs):
        root = dict(self._commands)
        keys = [key] + list(more_keys)
        return self._fetch_value(root, keys, kwargs.get('quiet', False))

    def get_command(self, key, *more_keys, **kwargs):
        """Retrieves a string for running a command from the setup
        and splits it to return a list.
        """
        val = self.get_command_config(key, *more_keys, **kwargs)
        if not val:
            return []
        else:
            return shlex.split(val)

    def known_component(self, name):
        return name in self._components

    def supports_platform(self, platform_name):
        """Does this distro support the named platform?

        :param platform_name: Return value from platform.platform().
        """
        return bool(self._platform_pattern.search(platform_name))

    @property
    def install_helper_class(self):
        """Return an install helper that will work for this distro."""
        return importer.import_entry_point(self._install_helper)

    @property
    def dependency_handler_class(self):
        """Return a dependency handler that will work for this distro."""
        return importer.import_entry_point(self._dependency_handler["name"])

    def extract_component(self, name, action, default_entry_point_creator=None):
        """Return the class + component info to use for doing the action w/the component."""
        try:
            # Use a copy instead of the original since we will be
            # modifying this dictionary which may not be wanted for future
            # usages of this dictionary (so keep the original clean)...
            component_info = copy.deepcopy(self._components[name])
        except KeyError:
            component_info = {}
        action_classes = component_info.pop('action_classes', {})
        if default_entry_point_creator is not None:
            default_action_classes = default_entry_point_creator(name,
                                                                 copy.deepcopy(component_info))
            if default_action_classes:
                for (an_action, entry_point) in six.iteritems(default_action_classes):
                    if an_action not in action_classes:
                        action_classes[an_action] = entry_point
        try:
            entry_point = action_classes.pop(action)
        except KeyError:
            raise RuntimeError('No entrypoint configured/generated for'
                               ' %r %r for distribution %r' % (action, name, self.name))
        else:
            return Component(entry_point, component_info, action_classes)

def _match_distros(distros):
    plt = platform.platform()
    matches = []
    for d in distros:
        if d.supports_platform(plt):
            matches.append(d)
    if not matches:
        raise excp.ConfigException('No distro matched for platform %r' % plt)
    else:
        return matches

def _linux_distribution():
    linux_plt = platform.linux_distribution()[0:2]
    linux_plt = "-".join(linux_plt)
    linux_plt = linux_plt.replace(" ", "-")
    linux_plt = linux_plt.lower().strip()
    return linux_plt

def load(path, distros_patch=None):
    """Load configuration for all distros found in path.

    :param path: path containing distro configuration in yaml format
    :param distros_patch: distros file patch, jsonpath format (rfc6902)
    """
    distro_possibles = []
    patch = jsonpatch.JsonPatch(distros_patch) if distros_patch else None
    input_files = glob.glob(sh.joinpths(path, '*.yaml'))
    if not input_files:
        raise excp.ConfigException('Did not find any distro definition files in %r' % path)
    for fn in input_files:
        LOG.debug("Attempting to load distro definition from %r", fn)
        try:
            cls_kvs = utils.load_yaml(fn)
            # Apply any user specified patches to distros file
            if patch:
                patch.apply(cls_kvs, in_place=True)
        except Exception as err:
            LOG.warning('Could not load distro definition from %r: %s', fn, err)
        else:
            if 'name' not in cls_kvs:
                name, _ext = os.path.splitext(sh.basename(fn))
                cls_kvs['name'] = name
            distro_possibles.append(Distro(**cls_kvs))
    return _match_distros(distro_possibles)
<commit_msg>Allow overrides keys to be regex patterns that match the platform<commit_after># vim: tabstop=4 shiftwidth=4 softtabstop=4

#    Copyright (C) 2012 Yahoo! Inc. All Rights Reserved.
#    Copyright (C) 2012 New Dream Network, LLC (DreamHost) All Rights Reserved.
#
#    Licensed under the Apache License, Version 2.0 (the "License"); you may
#    not use this file except in compliance with the License. You may obtain
#    a copy of the License at
#
#         http://www.apache.org/licenses/LICENSE-2.0
#
#    Unless required by applicable law or agreed to in writing, software
#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
#    License for the specific language governing permissions and limitations
#    under the License.

import collections
import copy
import glob
import jsonpatch
import os
import platform
import re
import shlex

import six

from anvil import exceptions as excp
from anvil import importer
from anvil import log as logging
from anvil import pprint
from anvil import shell as sh
from anvil import utils

LOG = logging.getLogger(__name__)

Component = collections.namedtuple(  # pylint: disable=C0103
    "Component", 'entry_point,options,siblings')

class Distro(object):
    def __init__(self,
                 name, platform_pattern,
                 install_helper, dependency_handler,
                 commands, components, **kwargs):
        self.name = name
        self._platform_pattern_text = platform_pattern
        self._platform_pattern = re.compile(platform_pattern, re.IGNORECASE)
        self._install_helper = install_helper
        self._dependency_handler = dependency_handler
        self._commands = commands
        self._components = components
        self.inject_platform_overrides(kwargs)

    def inject_platform_overrides(self, potential_data, source='??'):
        if 'platform_overrides' not in potential_data:
            return
        overrides = potential_data['platform_overrides']
        plts = _get_platform_names()
        patterns = [(k, re.compile(k, re.IGNORECASE), override)
                    for k, override in six.iteritems(overrides)]
        for k, pat, override in patterns:
            if any(pat.search(plt) for plt in plts):
                LOG.info("Merging in 'platform_overrides' that matched"
                         " platform %s from %s (sub-key %s)", plts, source, k)
                self._dependency_handler = utils.recursive_merge(
                    self._dependency_handler, override)

    def pformat(self, item_max_len=None):
        data = {
            'name': self.name,
            'dependency_handler': self._dependency_handler,
            'commands': self._commands,
            'pattern': "/%s/i" % self._platform_pattern_text,
        }
        return pprint.pformat(data, item_max_len=item_max_len)

    def _fetch_value(self, root, keys, quiet):
        end_key = keys[-1]
        for k in keys[0:-1]:
            if quiet:
                root = root.get(k)
                if root is None:
                    return None
            else:
                root = root[k]
        end_value = None
        if not quiet:
            end_value = root[end_key]
        else:
            end_value = root.get(end_key)
        return end_value

    def get_dependency_config(self, key, *more_keys, **kwargs):
        root = dict(self._dependency_handler)
        # NOTE(harlowja): Don't allow access to the dependency handler class
        # name. Access should be via the property instead.
        root.pop('name', None)
        keys = [key] + list(more_keys)
        return self._fetch_value(root, keys, kwargs.get('quiet', False))

    def get_command_config(self, key, *more_keys, **kwargs):
        root = dict(self._commands)
        keys = [key] + list(more_keys)
        return self._fetch_value(root, keys, kwargs.get('quiet', False))

    def get_command(self, key, *more_keys, **kwargs):
        """Retrieves a string for running a command from the setup
        and splits it to return a list.
        """
        val = self.get_command_config(key, *more_keys, **kwargs)
        if not val:
            return []
        else:
            return shlex.split(val)

    def known_component(self, name):
        return name in self._components

    def supports_platform(self, platform_name):
        """Does this distro support the named platform?

        :param platform_name: Return value from platform.platform().
        """
        return bool(self._platform_pattern.search(platform_name))

    @property
    def install_helper_class(self):
        """Return an install helper that will work for this distro."""
        return importer.import_entry_point(self._install_helper)

    @property
    def dependency_handler_class(self):
        """Return a dependency handler that will work for this distro."""
        return importer.import_entry_point(self._dependency_handler["name"])

    def extract_component(self, name, action, default_entry_point_creator=None):
        """Return the class + component info to use for doing the action w/the component."""
        try:
            # Use a copy instead of the original since we will be
            # modifying this dictionary which may not be wanted for future
            # usages of this dictionary (so keep the original clean)...
            component_info = copy.deepcopy(self._components[name])
        except KeyError:
            component_info = {}
        action_classes = component_info.pop('action_classes', {})
        if default_entry_point_creator is not None:
            default_action_classes = default_entry_point_creator(name,
                                                                 copy.deepcopy(component_info))
            if default_action_classes:
                for (an_action, entry_point) in six.iteritems(default_action_classes):
                    if an_action not in action_classes:
                        action_classes[an_action] = entry_point
        try:
            entry_point = action_classes.pop(action)
        except KeyError:
            raise RuntimeError('No entrypoint configured/generated for'
                               ' %r %r for distribution %r' % (action, name, self.name))
        else:
            return Component(entry_point, component_info, action_classes)

def _get_platform_names():
    plts = [
        platform.platform(),
    ]
    linux_plt = platform.linux_distribution()[0:2]
    linux_plt = "-".join(linux_plt)
    linux_plt = linux_plt.replace(" ", "-")
    plts.append(linux_plt)
    return [plt.lower() for plt in plts]

def _match_distros(distros):
    plts = _get_platform_names()
    matches = []
    for d in distros:
        if any(d.supports_platform(plt) for plt in plts):
            matches.append(d)
    if not matches:
        raise excp.ConfigException('No distro matched for platform %s' % plts)
    else:
        return matches

def load(path, distros_patch=None):
    """Load configuration for all distros found in path.

    :param path: path containing distro configuration in yaml format
    :param distros_patch: distros file patch, jsonpath format (rfc6902)
    """
    distro_possibles = []
    patch = jsonpatch.JsonPatch(distros_patch) if distros_patch else None
    input_files = glob.glob(sh.joinpths(path, '*.yaml'))
    if not input_files:
        raise excp.ConfigException('Did not find any distro definition files in %r' % path)
    for fn in input_files:
        LOG.debug("Attempting to load distro definition from %r", fn)
        try:
            cls_kvs = utils.load_yaml(fn)
            # Apply any user specified patches to distros file
            if patch:
                patch.apply(cls_kvs, in_place=True)
        except Exception as err:
            LOG.warning('Could not load distro definition from %r: %s', fn, err)
        else:
            if 'name' not in cls_kvs:
                name, _ext = os.path.splitext(sh.basename(fn))
                cls_kvs['name'] = name
            distro_possibles.append(Distro(**cls_kvs))
    return _match_distros(distro_possibles)
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
"""
===============================================================================
Cubic: Generate lattice-like networks
===============================================================================

"""
import numpy as np
import scipy as sp
from openpnm.network import GenericNetwork
from openpnm import topotools
from openpnm.utils import logging
logger = logging.getLogger(__name__)
from openpnm.utils.misc import tic, toc

class Cubic(GenericNetwork):
    r"""
    Simple cubic lattice with connectivity from 6 to 26

    Though simple, the Cubic network offers many advantages such as easy
    visualization and accurate determination of domain area and length in
    transport calculations.

    Parameters
    ----------
    shape : array_like
        The [Nx, Ny, Nz] size of the network in terms of the number of pores in
        each direction

    spacing : array_like, optional
        The spacing between pore centers in each direction. If not given, then
        [1, 1, 1] is assumed.

    connectivity : int, optional
        The number of connections to neighboring pores.  Connections are made
        symmetrically to any combination of face, edge, or corners neighbors.
        The default is 6 to create a simple cubic structure, but options are:

        - 6: Faces only
        - 8: Corners only
        - 12: Edges only
        - 14: Faces and Corners
        - 18: Faces and Edges
        - 20: Edges and Corners
        - 26: Faces, Edges and Corners

        For a more random distribution of connectivity, use a high
        ``connectivity`` (i.e. 26) and then delete a fraction of the throats
        using ``openpnm.topotools.reduce_coordination``.

    name : string
        An optional name for the object to help identify it.  If not given,
        one will be generated.

    project : OpenPNM Project object, optional
        Each OpenPNM object must be part of a *Project*.  If none is supplied
        then one will be created and this Network will be automatically
        assigned to it.  To create a *Project* use ``openpnm.Project()``.

    Attributes
    ----------
    spacing : int or array
        The distance between pore centers.  This value becomes meaningless
        if the topology is manipulated at all (i.e. by adding boundary pores)
        since there is not unique or consistent value.  In such cases an
        exception is thrown.

    shape : array
        The shape of the network.  Like ``spacing`` this values is meaningless
        if the topology is manipulated, so an Exception is thrown.

    Examples
    --------
    >>> import openpnm as op
    >>> pn = op.network.Cubic(shape=[5, 5, 5], spacing=[1, 1, 1])
    >>> pn.Np
    125

    And it can be plotted for quick visualization using:

    >>> fig = op.topotools.plot_connections(network=pn)
    >>> fig = op.topotools.plot_coordinates(network=pn, c='r', s=75, fig=fig)

    .. image:: /../docs/static/images/cubic_network.png
        :align: center

    For larger networks and more control over presentation use `Paraview
    <http://www.paraview.org>`_.
    """
    def __init__(self, shape, spacing=[1, 1, 1], connectivity=6, name=None,
                 project=None):

        arr = np.atleast_3d(np.empty(shape))

        # Store original network shape
        self._shape = sp.shape(arr)
        # Store network spacing
        spacing = sp.around(spacing, decimals=15)
        if spacing.size == 2:
            spacing = sp.concatenate((spacing, [1]))
        self._spacing = sp.ones(3)*sp.array(spacing, ndmin=1)

        points = np.vstack(np.meshgrid(np.arange(shape[0]),
                                       np.arange(shape[1]),
                                       np.arange(shape[2]))).reshape(3,-1).T
        points = points.astype(float) + 0.5

        I = np.arange(arr.size).reshape(arr.shape)

        face_joints = [(I[:, :, :-1], I[:, :, 1:]),
                       (I[:, :-1], I[:, 1:]),
                       (I[:-1], I[1:])]

        corner_joints = [(I[:-1, :-1, :-1], I[1:, 1:, 1:]),
                         (I[:-1, :-1, 1:], I[1:, 1:, :-1]),
                         (I[:-1, 1:, :-1], I[1:, :-1, 1:]),
                         (I[1:, :-1, :-1], I[:-1, 1:, 1:])]

        edge_joints = [(I[:, :-1, :-1], I[:, 1:, 1:]),
                       (I[:, :-1, 1:], I[:, 1:, :-1]),
                       (I[:-1, :, :-1], I[1:, :, 1:]),
                       (I[1:, :, :-1], I[:-1, :, 1:]),
                       (I[1:, 1:, :], I[:-1, :-1, :]),
                       (I[1:, :-1, :], I[:-1, 1:, :])]

        if connectivity == 6:
            joints = face_joints
        elif connectivity == 8:
            joints = corner_joints
        elif connectivity == 12:
            joints = edge_joints
        elif connectivity == 14:
            joints = face_joints + corner_joints
        elif connectivity == 18:
            joints = face_joints + edge_joints
        elif connectivity == 20:
            joints = edge_joints + corner_joints
        elif connectivity == 26:
            joints = face_joints + corner_joints + edge_joints
        else:
            raise Exception('Invalid connectivity receieved. Must be 6, 8, '
                            '12, 14, 18, 20 or 26')

        I = np.arange(arr.size).reshape(arr.shape)
        tails, heads = [], []
        # tic()
        for T, H in joints:
            tails.extend(T.flat)
            heads.extend(H.flat)
        # toc()
        pairs = np.vstack([tails, heads]).T

        super().__init__(Np=points.shape[0], Nt=pairs.shape[0], name=name,
                         project=project)

        self['pore.coords'] = points
        self['throat.conns'] = pairs
        self['pore.internal'] = True
        self['throat.internal'] = True
        self._label_surface_pores()
        topotools.label_faces(network=self)
        Ps = self['pore.surface']
        self['throat.surface'] = sp.all(Ps[self['throat.conns']], axis=1)
        # Scale network to requested spacing
        self['pore.coords'] *= spacing

    def _label_surface_pores(self):
        r"""
        """
        hits = sp.zeros_like(self.Ps, dtype=bool)
        dims = topotools.dimensionality(self)
        mn = sp.amin(self['pore.coords'], axis=0)
        mx = sp.amax(self['pore.coords'], axis=0)
        for ax in [0, 1, 2]:
            if dims[ax]:
                hits += self['pore.coords'][:, ax] <= mn[ax]
                hits += self['pore.coords'][:, ax] >= mx[ax]
        self['pore.surface'] = hits

    def add_boundary_pores(self, labels=['top', 'bottom', 'front', 'back',
                                         'left', 'right'], spacing=None):
        r"""
        Add pores to the faces of the network for use as boundary pores.

        Pores are offset from the faces by 1/2 a lattice spacing such that
        they lie directly on the boundaries.

        Parameters
        ----------
        labels : string or list of strings
            The labels indicating the pores defining each face where boundary
            pores are to be added (e.g. 'left' or ['left', 'right'])

        spacing : scalar or array_like
            The spacing of the network (e.g. [1, 1, 1]).  This should be given
            since it can be quite difficult to infer from the network, for
            instance if boundary pores have already added to other faces.

        """
        if type(labels) == str:
            labels = [labels]
        x, y, z = self['pore.coords'].T
        if spacing is None:
            spacing = self._get_spacing()
        else:
            spacing = sp.array(spacing)
            if spacing.size == 1:
                spacing = sp.ones(3)*spacing
        Lcx, Lcy, Lcz = spacing

        offset = {}
        offset['front'] = offset['left'] = offset['bottom'] = [0, 0, 0]
        offset['back'] = [Lcx*self._shape[0], 0, 0]
        offset['right'] = [0, Lcy*self._shape[1], 0]
        offset['top'] = [0, 0, Lcz*self._shape[2]]

        scale = {}
        scale['front'] = scale['back'] = [0, 1, 1]
        scale['left'] = scale['right'] = [1, 0, 1]
        scale['bottom'] = scale['top'] = [1, 1, 0]

        for label in labels:
            try:
                Ps = self.pores(label)
                topotools.clone_pores(network=self, pores=Ps,
                                      labels=label+'_boundary')
                # Translate cloned pores
                ind = self.pores(label+'_boundary')
                coords = self['pore.coords'][ind]
                coords = coords*scale[label] + offset[label]
                self['pore.coords'][ind] = coords
            except KeyError:
                logger.warning('No pores labelled ' + label + ' were found, ' +
                               'skipping boundary addition')

    def _get_spacing(self):
        # Find Network spacing
        P12 = self['throat.conns']
        C12 = self['pore.coords'][P12]
        mag = np.sqrt(np.sum(np.diff(C12, axis=1)**2, axis=2))
        vec = sp.around(sp.squeeze(np.diff(C12, axis=1))/mag,
                        decimals=10)
        spacing = [0, 0, 0]
        dims = topotools.dimensionality(self)
        # Ensure vectors point in n-dims unique directions
        c = {tuple(row): 1 for row in vec}
        if len(c.keys()) > sum(dims):
            raise Exception('Spacing is undefined when throats point in ' +
                            'more directions than network has dimensions')
        mag = sp.around(mag.squeeze(), decimals=10)
        for ax in [0, 1, 2]:
            if dims[ax]:
                inds = sp.where(vec[:, ax] == vec[:, ax].max())[0]
                temp = sp.unique(mag[inds])
                if np.size(temp) > 1:
                    raise Exception('A unique value of spacing could not be found')
                else:
                    spacing[ax] = temp[0]
        return sp.array(spacing)

    spacing = property(fget=_get_spacing)

    def _get_shape(self):
        L = np.ptp(self['pore.coords'], axis=0)
        mask = L.astype(bool)
        S = self.spacing
        shape = sp.array([1, 1, 1], int)
        shape[mask] = L[mask]/S[mask] + 1
        return shape

    shape = property(fget=_get_shape)

    def to_array(self, values):
        r"""
        Converts the values to a rectangular array with the same shape as the
        network

        Parameters
        ----------
        values : array_like
            An Np-long array of values to convert to

        Notes
        -----
        This method can break on networks that have had boundaries added.  It
        will usually work IF the given values came only from 'internal'
        pores.

        """
        if sp.shape(values)[0] > self.num_pores('internal'):
            raise Exception('The array shape does not match the network')
        Ps = sp.array(self['pore.index'][self.pores('internal')], dtype=int)
        arr = sp.ones(self._shape)*sp.nan
        ind = sp.unravel_index(Ps, self._shape)
        arr[ind[0], ind[1], ind[2]] = values
        return arr

    def from_array(self, array, propname):
        r"""
        Apply data to the network based on a rectangular array filled with
        values.  Each array location corresponds to a pore in the network.

        Parameters
        ----------
        array : array_like
            The rectangular array containing the values to be added to the
            network. This array must be the same shape as the original network.

        propname : string
            The name of the pore property being added.

        """
        array = sp.atleast_3d(array)
        if sp.shape(array) != self._shape:
            raise Exception('The array shape does not match the network')
        temp = array.flatten()
        Ps = sp.array(self['pore.index'][self.pores('internal')], dtype=int)
        propname = 'pore.' + propname.split('.')[-1]
        self[propname] = sp.nan
        self[propname][self.pores('internal')] = temp[Ps]
<commit_msg>Remove profiling tools (tic, toc) from Cubic<commit_after># -*- coding: utf-8 -*-
"""
===============================================================================
Cubic: Generate lattice-like networks
===============================================================================

"""
import numpy as np
import scipy as sp
from openpnm.network import GenericNetwork
from openpnm import topotools
from openpnm.utils import logging
logger = logging.getLogger(__name__)

class Cubic(GenericNetwork):
    r"""
    Simple cubic lattice with connectivity from 6 to 26

    Though simple, the Cubic network offers many advantages such as easy
    visualization and accurate determination of domain area and length in
    transport calculations.

    Parameters
    ----------
    shape : array_like
        The [Nx, Ny, Nz] size of the network in terms of the number of pores in
        each direction

    spacing : array_like, optional
        The spacing between pore centers in each direction. If not given, then
        [1, 1, 1] is assumed.

    connectivity : int, optional
        The number of connections to neighboring pores.  Connections are made
        symmetrically to any combination of face, edge, or corners neighbors.
        The default is 6 to create a simple cubic structure, but options are:

        - 6: Faces only
        - 8: Corners only
        - 12: Edges only
        - 14: Faces and Corners
        - 18: Faces and Edges
        - 20: Edges and Corners
        - 26: Faces, Edges and Corners

        For a more random distribution of connectivity, use a high
        ``connectivity`` (i.e. 26) and then delete a fraction of the throats
        using ``openpnm.topotools.reduce_coordination``.

    name : string
        An optional name for the object to help identify it.  If not given,
        one will be generated.

    project : OpenPNM Project object, optional
        Each OpenPNM object must be part of a *Project*.  If none is supplied
        then one will be created and this Network will be automatically
        assigned to it.  To create a *Project* use ``openpnm.Project()``.

    Attributes
    ----------
    spacing : int or array
        The distance between pore centers.  This value becomes meaningless
        if the topology is manipulated at all (i.e. by adding boundary pores)
        since there is not unique or consistent value.  In such cases an
        exception is thrown.

    shape : array
        The shape of the network.  Like ``spacing`` this values is meaningless
        if the topology is manipulated, so an Exception is thrown.

    Examples
    --------
    >>> import openpnm as op
    >>> pn = op.network.Cubic(shape=[5, 5, 5], spacing=[1, 1, 1])
    >>> pn.Np
    125

    And it can be plotted for quick visualization using:

    >>> fig = op.topotools.plot_connections(network=pn)
    >>> fig = op.topotools.plot_coordinates(network=pn, c='r', s=75, fig=fig)

    .. image:: /../docs/static/images/cubic_network.png
        :align: center

    For larger networks and more control over presentation use `Paraview
    <http://www.paraview.org>`_.
    """
    def __init__(self, shape, spacing=[1, 1, 1], connectivity=6, name=None,
                 project=None):

        arr = np.atleast_3d(np.empty(shape))

        # Store original network shape
        self._shape = sp.shape(arr)
        # Store network spacing
        spacing = sp.around(spacing, decimals=15)
        if spacing.size == 2:
            spacing = sp.concatenate((spacing, [1]))
        self._spacing = sp.ones(3)*sp.array(spacing, ndmin=1)

        points = np.vstack(np.meshgrid(np.arange(shape[0]),
                                       np.arange(shape[1]),
                                       np.arange(shape[2]))).reshape(3,-1).T
        points = points.astype(float) + 0.5

        I = np.arange(arr.size).reshape(arr.shape)

        face_joints = [(I[:, :, :-1], I[:, :, 1:]),
                       (I[:, :-1], I[:, 1:]),
                       (I[:-1], I[1:])]

        corner_joints = [(I[:-1, :-1, :-1], I[1:, 1:, 1:]),
                         (I[:-1, :-1, 1:], I[1:, 1:, :-1]),
                         (I[:-1, 1:, :-1], I[1:, :-1, 1:]),
                         (I[1:, :-1, :-1], I[:-1, 1:, 1:])]

        edge_joints = [(I[:, :-1, :-1], I[:, 1:, 1:]),
                       (I[:, :-1, 1:], I[:, 1:, :-1]),
                       (I[:-1, :, :-1], I[1:, :, 1:]),
                       (I[1:, :, :-1], I[:-1, :, 1:]),
                       (I[1:, 1:, :], I[:-1, :-1, :]),
                       (I[1:, :-1, :], I[:-1, 1:, :])]

        if connectivity == 6:
            joints = face_joints
        elif connectivity == 8:
            joints = corner_joints
        elif connectivity == 12:
            joints = edge_joints
        elif connectivity == 14:
            joints = face_joints + corner_joints
        elif connectivity == 18:
            joints = face_joints + edge_joints
        elif connectivity == 20:
            joints = edge_joints + corner_joints
        elif connectivity == 26:
            joints = face_joints + corner_joints + edge_joints
        else:
            raise Exception('Invalid connectivity receieved. Must be 6, 8, '
                            '12, 14, 18, 20 or 26')

        I = np.arange(arr.size).reshape(arr.shape)
        tails, heads = [], []
        for T, H in joints:
            tails.extend(T.flat)
            heads.extend(H.flat)
        pairs = np.vstack([tails, heads]).T

        super().__init__(Np=points.shape[0], Nt=pairs.shape[0], name=name,
                         project=project)

        self['pore.coords'] = points
        self['throat.conns'] = pairs
        self['pore.internal'] = True
        self['throat.internal'] = True
        self._label_surface_pores()
        topotools.label_faces(network=self)
        Ps = self['pore.surface']
        self['throat.surface'] = sp.all(Ps[self['throat.conns']], axis=1)
        # Scale network to requested spacing
        self['pore.coords'] *= spacing

    def _label_surface_pores(self):
        r"""
        """
        hits = sp.zeros_like(self.Ps, dtype=bool)
        dims = topotools.dimensionality(self)
        mn = sp.amin(self['pore.coords'], axis=0)
        mx = sp.amax(self['pore.coords'], axis=0)
        for ax in [0, 1, 2]:
            if dims[ax]:
                hits += self['pore.coords'][:, ax] <= mn[ax]
                hits += self['pore.coords'][:, ax] >= mx[ax]
        self['pore.surface'] = hits

    def add_boundary_pores(self, labels=['top', 'bottom', 'front', 'back',
                                         'left', 'right'], spacing=None):
        r"""
        Add pores to the faces of the network for use as boundary pores.

        Pores are offset from the faces by 1/2 a lattice spacing such that
        they lie directly on the boundaries.

        Parameters
        ----------
        labels : string or list of strings
            The labels indicating the pores defining each face where boundary
            pores are to be added (e.g. 'left' or ['left', 'right'])

        spacing : scalar or array_like
            The spacing of the network (e.g. [1, 1, 1]).  This should be given
            since it can be quite difficult to infer from the network, for
            instance if boundary pores have already added to other faces.

        """
        if type(labels) == str:
            labels = [labels]
        x, y, z = self['pore.coords'].T
        if spacing is None:
            spacing = self._get_spacing()
        else:
            spacing = sp.array(spacing)
            if spacing.size == 1:
                spacing = sp.ones(3)*spacing
        Lcx, Lcy, Lcz = spacing

        offset = {}
        offset['front'] = offset['left'] = offset['bottom'] = [0, 0, 0]
        offset['back'] = [Lcx*self._shape[0], 0, 0]
        offset['right'] = [0, Lcy*self._shape[1], 0]
        offset['top'] = [0, 0, Lcz*self._shape[2]]

        scale = {}
        scale['front'] = scale['back'] = [0, 1, 1]
        scale['left'] = scale['right'] = [1, 0, 1]
        scale['bottom'] = scale['top'] = [1, 1, 0]

        for label in labels:
            try:
                Ps = self.pores(label)
                topotools.clone_pores(network=self, pores=Ps,
                                      labels=label+'_boundary')
                # Translate cloned pores
                ind = self.pores(label+'_boundary')
                coords = self['pore.coords'][ind]
                coords = coords*scale[label] + offset[label]
                self['pore.coords'][ind] = coords
            except KeyError:
                logger.warning('No pores labelled ' + label + ' were found, ' +
                               'skipping boundary addition')

    def _get_spacing(self):
        # Find Network spacing
        P12 = self['throat.conns']
        C12 = self['pore.coords'][P12]
        mag = np.sqrt(np.sum(np.diff(C12, axis=1)**2, axis=2))
        vec = sp.around(sp.squeeze(np.diff(C12, axis=1))/mag,
                        decimals=10)
        spacing = [0, 0, 0]
        dims = topotools.dimensionality(self)
        # Ensure vectors point in n-dims unique directions
        c = {tuple(row): 1 for row in vec}
        if len(c.keys()) > sum(dims):
            raise Exception('Spacing is undefined when throats point in ' +
                            'more directions than network has dimensions')
        mag = sp.around(mag.squeeze(), decimals=10)
        for ax in [0, 1, 2]:
            if dims[ax]:
                inds = sp.where(vec[:, ax] == vec[:, ax].max())[0]
                temp = sp.unique(mag[inds])
                if np.size(temp) > 1:
                    raise Exception('A unique value of spacing could not be found')
                else:
                    spacing[ax] = temp[0]
        return sp.array(spacing)

    spacing = property(fget=_get_spacing)

    def _get_shape(self):
        L = np.ptp(self['pore.coords'], axis=0)
        mask = L.astype(bool)
        S = self.spacing
        shape = sp.array([1, 1, 1], int)
        shape[mask] = L[mask]/S[mask] + 1
        return shape

    shape = property(fget=_get_shape)

    def to_array(self, values):
        r"""
        Converts the values to a rectangular array with the same shape as the
        network

        Parameters
        ----------
        values : array_like
            An Np-long array of values to convert to

        Notes
        -----
        This method can break on networks that have had boundaries added.  It
        will usually work IF the given values came only from 'internal'
        pores.

        """
        if sp.shape(values)[0] > self.num_pores('internal'):
            raise Exception('The array shape does not match the network')
        Ps = sp.array(self['pore.index'][self.pores('internal')], dtype=int)
        arr = sp.ones(self._shape)*sp.nan
        ind = sp.unravel_index(Ps, self._shape)
        arr[ind[0], ind[1], ind[2]] = values
        return arr

    def from_array(self, array, propname):
        r"""
        Apply data to the network based on a rectangular array filled with
        values.  Each array location corresponds to a pore in the network.

        Parameters
        ----------
        array : array_like
            The rectangular array containing the values to be added to the
            network. This array must be the same shape as the original network.

        propname : string
            The name of the pore property being added.

        """
        array = sp.atleast_3d(array)
        if sp.shape(array) != self._shape:
            raise Exception('The array shape does not match the network')
        temp = array.flatten()
        Ps = sp.array(self['pore.index'][self.pores('internal')], dtype=int)
        propname = 'pore.' + propname.split('.')[-1]
        self[propname] = sp.nan
        self[propname][self.pores('internal')] = temp[Ps]
<|endoftext|>