<commit_before>#! /usr/bin/env python3
# -*- coding: utf-8 -*-
"""
.. module:: pysdmx
    :platform: Unix, Windows
    :synopsis: Python interface for SDMX

.. :moduleauthor :: Widukind team <widukind-dev@cepremap.org>
"""

import requests
import pandas
import lxml.etree
import datetime
import numpy
from io import BytesIO
import re
import zipfile
import time
from collections import OrderedDict

def date_parser(date, frequency):
    """Generate proper index for pandas

    :param date: A date
    :type date: str
    :param frequency: A frequency as specified in SDMX, A for Annual, Q for Quarterly, M for Monthly and D for Daily
    :type frequency: str
    :return: datetime.datetime()

    >>> date_parser('1987-02-02','D')
    datetime.datetime(1987, 2, 2, 0, 0)
    """

    if frequency == 'A':
        return datetime.datetime.strptime(date, '%Y')
    if frequency == 'Q':
        date = date.split('-Q')
        date = str(int(date[1])*3) + date[0]
        return datetime.datetime.strptime(date, '%m%Y')
    if frequency == 'M':
        return datetime.datetime.strptime(date, '%Y-%m')
    if frequency == 'D':
        return datetime.datetime.strptime(date, '%Y-%m-%d')

class SDMX_REST(object):
    """Data provider. This is the main class that allows practical access to all the data.

    :ivar sdmx_url: The URL of the SDMX endpoint, the webservice employed to access the data.
    :type sdmx_url: str
    :ivar agencyID: An identifier of the statistical provider.
    :type agencyID: str
    """
    def __init__(self, sdmx_url, version, agencyID):
        self.sdmx_url = sdmx_url
        self.agencyID = agencyID
        self._dataflows = None
        self.version = version

    @staticmethod
    def query_rest(url):
        """Retrieve SDMX messages.

        :param url: The URL of the message.
        :type url: str
        :return: An lxml.etree.ElementTree() of the SDMX message
        """
        parser = lxml.etree.XMLParser(
            ns_clean=True, recover=True, encoding='utf-8')
        request = requests.get(url, timeout= 20)
        if request.status_code == requests.codes.ok:
            response =  lxml.etree.fromstring(
                request.text.encode('utf-8'), parser=parser)
        elif request.status_code == 430:
            #Sometimes, eurostat creates a zipfile when the query is too large. We have to wait for the file to be generated.
            messages = response.xpath('.//footer:Message/common:Text',
                                      namespaces=response.nsmap)
            regex_ = re.compile("Due to the large query the response will be written "
                                "to a file which will be located under URL: (.*)")
            matches = [regex_.match(element.text) for element in messages]
            if bool(matches):
                response_ = None
                i = 30
                while i<101:
                    time.sleep(i)
                    i = i+10
                    url = matches[0].groups()[0]
                    request = requests.get(url)
                    if request.headers['content-type'] == "application/octet-stream":
                        buffer = BytesIO(request.content)
                        file = zipfile.ZipFile(buffer)
                        filename = file.namelist()[0]
                        response_ =  lxml.etree.fromstring(
                            file.read(filename), parser=parser)
                        break
                        if response_ is None:
                            raise Exception("Eurostat didn't provide the file you are looking for.")
            else:
                raise ValueError("Error getting client({})".format(request.status_code))
        else:
            raise ValueError("Error getting client({})".format(request.status_code))
        return response

    @property
    def dataflows(self):
        """Index of available dataflows

        :type: dict"""
        if not self._dataflows:
            if self.version == '2_1':
                url = '/'.join([self.sdmx_url, 'dataflow', self.agencyID, 'all', 'latest'])
                tree = self.query_rest(url)
                dataflow_path = ".//str:Dataflow"
                name_path = ".//com:Name"
                if not self._dataflows:
                    self._dataflows = {}
                    for dataflow in tree.iterfind(dataflow_path,
                                                       namespaces=tree.nsmap):
                        id = dataflow.get('id')
                        agencyID = dataflow.get('agencyID')
                        version = dataflow.get('version')
                        titles = {}
                        for title in dataflow.iterfind(name_path,
                                                       namespaces=tree.nsmap):
                            language = title.values()
                            language = language[0]
                            titles[language] = title.text
                        self._dataflows[id] = (agencyID, version, titles)
            if self.version == '2_0':
                url = '/'.join([self.sdmx_url, 'Dataflow'])
                tree = self.query_rest(url)
                dataflow_path = ".//structure:Dataflow"
                name_path = ".//structure:Name"
                keyid_path = ".//structure:KeyFamilyID"
                if not self._dataflows:
                    self._dataflows = {}
                    for dataflow in tree.iterfind(dataflow_path,
                                                       namespaces=tree.nsmap):
                        for id in dataflow.iterfind(keyid_path,
                                                       namespaces=tree.nsmap):
                            id = id.text
                        agencyID = dataflow.get('agencyID')
                        version = dataflow.get('version')
                        titles = {}
                        for title in dataflow.iterfind(name_path,
                                                       namespaces=tree.nsmap):
                            titles['en'] = title.text
                        self._dataflows[id] = (agencyID, version, titles)
        return self._dataflows

    def codes(self, flowRef):
        """Data definitions

        Returns a dictionnary describing the available dimensions for a specific flowRef.

        :param flowRef: Identifier of the dataflow
        :type flowRef: str
        :return: dict"""
        if self.version == '2_1':
            url = '/'.join([self.sdmx_url, 'datastructure', self.agencyID, 'DSD_' + flowRef])
            tree = self.query_rest(url)
            codelists_path = ".//str:Codelists"
            codelist_path = ".//str:Codelist"
            name_path = ".//com:Name"
            code_path = ".//str:Code"
            self._codes = {}
            codelists = tree.xpath(codelists_path,
                                          namespaces=tree.nsmap)
            for codelists_ in codelists:
                for codelist in codelists_.iterfind(codelist_path,
                                                    namespaces=tree.nsmap):
                    name = codelist.xpath(name_path, namespaces=tree.nsmap)
                    name = name[0]
                    name = name.text
                    # a dot "." can't be part of a JSON field name
                    name = re.sub(r"\.","",name)
                    code = OrderedDict({})
                    for code_ in codelist.iterfind(code_path,
                                                   namespaces=tree.nsmap):
                        code_key = code_.get('id')
                        code_name = code_.xpath(name_path,
                                                namespaces=tree.nsmap)
                        code_name = code_name[0]
                        code[code_key] = code_name.text
                    self._codes[name] = code
        if self.version == '2_0':
            codelists_path = ".//message:CodeLists"
            codelist_path = ".//structure:CodeList"
            code_path = ".//structure:Code"
            description_path = ".//structure:Description"
            url = '/'.join([self.sdmx_url, 'KeyFamily', flowRef])
            tree = self.query_rest(url)
            self._codes = {}
            codelists = tree.xpath(codelists_path,
                                          namespaces=tree.nsmap)
            for codelists_ in codelists:
                for codelist in codelists_.iterfind(codelist_path,
                                                    namespaces=tree.nsmap):
                    name = codelist.get('id')
                    name = name[3:]
                    # a dot "." can't be part of a JSON field name
                    name = re.sub(r"\.","",name)
                    code = {}
                    for code_ in codelist.iterfind(code_path,
                                                   namespaces=tree.nsmap):
                        code_key = code_.get('value')
                        code_name = code_.xpath(description_path,
                                                namespaces=tree.nsmap)
                        code_name = code_name[0]
                        code[code_key] = code_name.text
                    self._codes[name] = code
        return self._codes

    def dataframe(self, flowRef, key, startperiod=None, endperiod=None):
        """Get data

        :param flowRef: an identifier of the data
        :type flowRef: str
        :param key: a filter using codes (for example, .... for no filter ...BE for all the series related to Belgium) if using v2_1. In 2_0, you should be providing a dict following that syntax {dimension:value}
        :type key: str or dict
        :param startperiod: the starting date of the time series that will be downloaded (optional, default: None)
        :type startperiod: datetime.datetime()
        :param endperiod: the ending date of the time series that will be downloaded (optional, default: None)
        :type endperiod: datetime.datetime()
        :return: DataFrame"""
        raw_data, metadata = self.data(flowRef, key, startperiod=None, endperiod=None)
        column_names = ['__'.join(d.values()) for d in metadata]
        return pandas.DataFrame(dict(zip(column_names, raw_data)))

    def data(self, flowRef, key, startperiod=None, endperiod=None):
        """Get data

        :param flowRef: an identifier of the data
        :type flowRef: str
        :param key: a filter using codes (for example, .... for no filter ...BE for all the series related to Belgium) if using v2_1. In 2_0, you should be providing a dict following that syntax {dimension:value}
        :type key: str or dict
        :param startperiod: the starting date of the time series that will be downloaded (optional, default: None)
        :type startperiod: datetime.datetime()
        :param endperiod: the ending date of the time series that will be downloaded (optional, default: None)
        :type endperiod: datetime.datetime()
        :return: list of timeseries, list of metadata"""
        if self.version == '2_1':
            resource = 'data'
            if startperiod is not None and endperiod is not None:
                query = '/'.join([resource, flowRef, key
                        + '?startperiod=' + startperiod
                        + '&endPeriod=' + endperiod])
            else:
                query = '/'.join([resource, flowRef, key])
            url = '/'.join([self.sdmx_url,query])
            tree = self.query_rest(url)
            GENERIC = '{'+tree.nsmap['generic']+'}'
            self._time_series = []
            self._metadata = []
            for series in tree.iterfind(".//generic:Series",
                                             namespaces=tree.nsmap):
                time_series_ = []
                for elem in series.iterchildren():
                    if elem.tag == GENERIC + 'SeriesKey':
                        codes = {}
                        for value in elem.iter(GENERIC + "Value"):
                            codes[value.get('id')] = value.get('value')
                    elif elem.tag == GENERIC + 'Obs':
                        for elem1 in elem.iterchildren():
                            observation_status = 'A'
                            if elem1.tag == GENERIC + 'ObsDimension':
                                dimension = elem1.get('value')
                                dimension = date_parser(dimension, codes['FREQ'])
                            elif elem1.tag == GENERIC + 'ObsValue':
                                value = elem1.get('value')
                            elif elem1.tag == GENERIC + 'Attibutes':
                                for elem2 in elem1.iter(".//generic:Value[@id='OBS_STATUS']",
                                    namespaces=tree.nsmap):
                                    observation_status = elem2.get('value')
                        time_series_.append((dimension, value, observation_status))
                time_series_.sort()
                dates = numpy.array(
                    [observation[0] for observation in time_series_])
                values = numpy.array(
                    [observation[1] for observation in time_series_])
                time_series_ = pandas.Series(values, index=dates, dtype='float64')
                self._time_series.append(time_series_)
                self._metadata.append(codes)
            return self._time_series,self._metadata

        if self.version == '2_0':
            resource = 'GenericData'
            key__ = ''
            for key_, value_ in key.items():
                key__ += '&' + key_ + '=' + value_
            key = key__

            if startperiod is not None and endperiod is not None:
                query = (resource + '?dataflow=' + flowRef + key
                        + 'startperiod=' + startperiod
                        + '&endPeriod=' + endperiod)
            else:
                query = resource + '?dataflow=' + flowRef + key
            url = '/'.join([self.sdmx_url,query])
            tree = self.query_rest(url)
            _time_series = []
            _metadata = []
            for series in tree.iterfind(".//generic:Series",
                                             namespaces=tree.nsmap):
                codes = {}
                for codes_ in series.iterfind(".//generic:SeriesKey",
                                           namespaces=tree.nsmap):
                    for key in codes_.iterfind(".//generic:Value",
                                               namespaces=tree.nsmap):
                        codes[key.get('concept')] = key.get('value')
                time_series_ = []
                for observation in series.iterfind(".//generic:Obs",
                                                   namespaces=tree.nsmap):
                    dimensions = observation.xpath(".//generic:Time",
                                                   namespaces=tree.nsmap)
                    dimension = date_parser(dimensions[0].text, codes['FREQ'])
                    values = observation.xpath(".//generic:ObsValue",
                                               namespaces=tree.nsmap)
                    value = values[0].get('value')
                    observation_status = 'A'
                    for attribute in \
                        observation.iterfind(".//generic:Attributes",
                                             namespaces=tree.nsmap):
                        for observation_status_ in \
                            attribute.xpath(
                                ".//generic:Value[@concept='OBS_STATUS']",
                                namespaces=tree.nsmap):
                            if observation_status_ is not None:
                                observation_status \
                                    = observation_status_.get('value')
                    time_series_.append((dimension, value, observation_status))
                time_series_.sort()
                dates = numpy.array(
                    [observation[0] for observation in time_series_])
                values = numpy.array(
                    [observation[1] for observation in time_series_])
                time_series_ = pandas.Series(values, index=dates, dtype='float64')
                _time_series.append(time_series_)
                _metadata.append(codes)
            return _time_series, _metadata

eurostat = SDMX_REST('http://www.ec.europa.eu/eurostat/SDMX/diss-web/rest',
                     '2_1','ESTAT')
eurostat_test = SDMX_REST('http://localhost:8800/eurostat',
                     '2_1','ESTAT')
ecb = SDMX_REST('http://sdw-ws.ecb.europa.eu',
                     '2_0','ECB')
ilo = SDMX_REST('http://www.ilo.org/ilostat/sdmx/ws/rest/',
                     '2_1','ILO')
fao = SDMX_REST('http://data.fao.org/sdmx',
                     '2_1','FAO')
<commit_msg>Refactor data() to return list of 1d DataFrames with namedtuples as columns.<commit_after>#! /usr/bin/env python3
# -*- coding: utf-8 -*-
"""
.. module:: pysdmx
    :platform: Unix, Windows
    :synopsis: Python interface for SDMX

.. :moduleauthor :: Widukind team <widukind-dev@cepremap.org>
"""

import requests
import pandas
import lxml.etree
import datetime
import numpy
from io import BytesIO
import re
import zipfile
import time
from collections import OrderedDict, namedtuple

def date_parser(date, frequency):
    """Generate proper index for pandas

    :param date: A date
    :type date: str
    :param frequency: A frequency as specified in SDMX, A for Annual, Q for Quarterly, M for Monthly and D for Daily
    :type frequency: str
    :return: datetime.datetime()

    >>> date_parser('1987-02-02','D')
    datetime.datetime(1987, 2, 2, 0, 0)
    """

    if frequency == 'A':
        return datetime.datetime.strptime(date, '%Y')
    if frequency == 'Q':
        date = date.split('-Q')
        date = str(int(date[1])*3) + date[0]
        return datetime.datetime.strptime(date, '%m%Y')
    if frequency == 'M':
        return datetime.datetime.strptime(date, '%Y-%m')
    if frequency == 'D':
        return datetime.datetime.strptime(date, '%Y-%m-%d')

class SDMX_REST(object):
    """Data provider. This is the main class that allows practical access to all the data.

    :ivar sdmx_url: The URL of the SDMX endpoint, the webservice employed to access the data.
    :type sdmx_url: str
    :ivar agencyID: An identifier of the statistical provider.
    :type agencyID: str
    """
    def __init__(self, sdmx_url, version, agencyID):
        self.sdmx_url = sdmx_url
        self.agencyID = agencyID
        self._dataflows = None
        self.version = version

    @staticmethod
    def query_rest(url):
        """Retrieve SDMX messages.

        :param url: The URL of the message.
        :type url: str
        :return: An lxml.etree.ElementTree() of the SDMX message
        """
        parser = lxml.etree.XMLParser(
            ns_clean=True, recover=True, encoding='utf-8')
        request = requests.get(url, timeout= 20)
        if request.status_code == requests.codes.ok:
            response =  lxml.etree.fromstring(
                request.text.encode('utf-8'), parser=parser)
        elif request.status_code == 430:
            #Sometimes, eurostat creates a zipfile when the query is too large. We have to wait for the file to be generated.
            messages = response.xpath('.//footer:Message/common:Text',
                                      namespaces=response.nsmap)
            regex_ = re.compile("Due to the large query the response will be written "
                                "to a file which will be located under URL: (.*)")
            matches = [regex_.match(element.text) for element in messages]
            if bool(matches):
                response_ = None
                i = 30
                while i<101:
                    time.sleep(i)
                    i = i+10
                    url = matches[0].groups()[0]
                    request = requests.get(url)
                    if request.headers['content-type'] == "application/octet-stream":
                        buffer = BytesIO(request.content)
                        file = zipfile.ZipFile(buffer)
                        filename = file.namelist()[0]
                        response_ =  lxml.etree.fromstring(
                            file.read(filename), parser=parser)
                        break
                        if response_ is None:
                            raise Exception("Eurostat didn't provide the file you are looking for.")
            else:
                raise ValueError("Error getting client({})".format(request.status_code))
        else:
            raise ValueError("Error getting client({})".format(request.status_code))
        return response

    @property
    def dataflows(self):
        """Index of available dataflows

        :type: dict"""
        if not self._dataflows:
            if self.version == '2_1':
                url = '/'.join([self.sdmx_url, 'dataflow', self.agencyID, 'all', 'latest'])
                tree = self.query_rest(url)
                dataflow_path = ".//str:Dataflow"
                name_path = ".//com:Name"
                if not self._dataflows:
                    self._dataflows = {}
                    for dataflow in tree.iterfind(dataflow_path,
                                                       namespaces=tree.nsmap):
                        id = dataflow.get('id')
                        agencyID = dataflow.get('agencyID')
                        version = dataflow.get('version')
                        titles = {}
                        for title in dataflow.iterfind(name_path,
                                                       namespaces=tree.nsmap):
                            language = title.values()
                            language = language[0]
                            titles[language] = title.text
                        self._dataflows[id] = (agencyID, version, titles)
            if self.version == '2_0':
                url = '/'.join([self.sdmx_url, 'Dataflow'])
                tree = self.query_rest(url)
                dataflow_path = ".//structure:Dataflow"
                name_path = ".//structure:Name"
                keyid_path = ".//structure:KeyFamilyID"
                if not self._dataflows:
                    self._dataflows = {}
                    for dataflow in tree.iterfind(dataflow_path,
                                                       namespaces=tree.nsmap):
                        for id in dataflow.iterfind(keyid_path,
                                                       namespaces=tree.nsmap):
                            id = id.text
                        agencyID = dataflow.get('agencyID')
                        version = dataflow.get('version')
                        titles = {}
                        for title in dataflow.iterfind(name_path,
                                                       namespaces=tree.nsmap):
                            titles['en'] = title.text
                        self._dataflows[id] = (agencyID, version, titles)
        return self._dataflows

    def codes(self, flowRef):
        """Data definitions

        Returns a dictionnary describing the available dimensions for a specific flowRef.

        :param flowRef: Identifier of the dataflow
        :type flowRef: str
        :return: dict"""
        if self.version == '2_1':
            url = '/'.join([self.sdmx_url, 'datastructure', self.agencyID, 'DSD_' + flowRef])
            tree = self.query_rest(url)
            codelists_path = ".//str:Codelists"
            codelist_path = ".//str:Codelist"
            name_path = ".//com:Name"
            code_path = ".//str:Code"
            self._codes = {}
            codelists = tree.xpath(codelists_path,
                                          namespaces=tree.nsmap)
            for codelists_ in codelists:
                for codelist in codelists_.iterfind(codelist_path,
                                                    namespaces=tree.nsmap):
                    name = codelist.xpath(name_path, namespaces=tree.nsmap)
                    name = name[0]
                    name = name.text
                    # a dot "." can't be part of a JSON field name
                    name = re.sub(r"\.","",name)
                    code = OrderedDict({})
                    for code_ in codelist.iterfind(code_path,
                                                   namespaces=tree.nsmap):
                        code_key = code_.get('id')
                        code_name = code_.xpath(name_path,
                                                namespaces=tree.nsmap)
                        code_name = code_name[0]
                        code[code_key] = code_name.text
                    self._codes[name] = code
        if self.version == '2_0':
            codelists_path = ".//message:CodeLists"
            codelist_path = ".//structure:CodeList"
            code_path = ".//structure:Code"
            description_path = ".//structure:Description"
            url = '/'.join([self.sdmx_url, 'KeyFamily', flowRef])
            tree = self.query_rest(url)
            self._codes = {}
            codelists = tree.xpath(codelists_path,
                                          namespaces=tree.nsmap)
            for codelists_ in codelists:
                for codelist in codelists_.iterfind(codelist_path,
                                                    namespaces=tree.nsmap):
                    name = codelist.get('id')
                    name = name[3:]
                    # a dot "." can't be part of a JSON field name
                    name = re.sub(r"\.","",name)
                    code = {}
                    for code_ in codelist.iterfind(code_path,
                                                   namespaces=tree.nsmap):
                        code_key = code_.get('value')
                        code_name = code_.xpath(description_path,
                                                namespaces=tree.nsmap)
                        code_name = code_name[0]
                        code[code_key] = code_name.text
                    self._codes[name] = code
        return self._codes

    def dataframe(self, flowRef, key, startperiod=None, endperiod=None):
        """Get data

        :param flowRef: an identifier of the data
        :type flowRef: str
        :param key: a filter using codes (for example, .... for no filter ...BE for all the series related to Belgium) if using v2_1. In 2_0, you should be providing a dict following that syntax {dimension:value}
        :type key: str or dict
        :param startperiod: the starting date of the time series that will be downloaded (optional, default: None)
        :type startperiod: datetime.datetime()
        :param endperiod: the ending date of the time series that will be downloaded (optional, default: None)
        :type endperiod: datetime.datetime()
        :return: DataFrame"""
        raw_data, metadata = self.data(flowRef, key, startperiod=None, endperiod=None)
        column_names = ['__'.join(d.values()) for d in metadata]
        return pandas.DataFrame(dict(zip(column_names, raw_data)))

    def data(self, flowRef, key, startperiod=None, endperiod=None):
        """Get data

        :param flowRef: an identifier of the data
        :type flowRef: str
        :param key: a filter using codes (for example, .... for no filter ...BE for all the series related to Belgium) if using v2_1. In 2_0, you should be providing a dict following that syntax {dimension:value}
        :type key: str or dict
        :param startperiod: the starting date of the time series that will be downloaded (optional, default: None)
        :type startperiod: datetime.datetime()
        :param endperiod: the ending date of the time series that will be downloaded (optional, default: None)
        :type endperiod: datetime.datetime()
        :return: list of 1-dimensional DataFrames."""

        # Container for namedtuple classes for metadata of each column
        tuple_classes = []
        result = []

        if self.version == '2_1':
            resource = 'data'
            if startperiod is not None and endperiod is not None:
                query = '/'.join([resource, flowRef, key
                        + '?startperiod=' + startperiod
                        + '&endPeriod=' + endperiod])
            else:
                query = '/'.join([resource, flowRef, key])
            url = '/'.join([self.sdmx_url,query])
            tree = self.query_rest(url)
            GENERIC = '{'+tree.nsmap['generic']+'}'

            for series in tree.iterfind(".//generic:Series",
                                             namespaces=tree.nsmap):
                time_series_ = []
                for elem in series.iterchildren():
                    if elem.tag == GENERIC + 'SeriesKey':
                        codes = {}
                        for value in elem.iter(GENERIC + "Value"):
                            codes[value.get('id')] = value.get('value')
                    elif elem.tag == GENERIC + 'Obs':
                        for elem1 in elem.iterchildren():
                            observation_status = 'A'
                            if elem1.tag == GENERIC + 'ObsDimension':
                                dimension = elem1.get('value')
                                dimension = date_parser(dimension, codes['FREQ'])
                            elif elem1.tag == GENERIC + 'ObsValue':
                                value = elem1.get('value')
                            elif elem1.tag == GENERIC + 'Attibutes':
                                for elem2 in elem1.iter(".//generic:Value[@id='OBS_STATUS']",
                                    namespaces=tree.nsmap):
                                    observation_status = elem2.get('value')
                        time_series_.append((dimension, value, observation_status))
                time_series_ = self.make_dataframe      (time_series_, codes,
                tuple_classes)
                result.append(time_series_)

        if self.version == '2_0':
            resource = 'GenericData'
            key__ = ''
            for key_, value_ in key.items():
                key__ += '&' + key_ + '=' + value_
            key = key__

            if startperiod is not None and endperiod is not None:
                query = (resource + '?dataflow=' + flowRef + key
                        + 'startperiod=' + startperiod
                        + '&endPeriod=' + endperiod)
            else:
                query = resource + '?dataflow=' + flowRef + key
            url = '/'.join([self.sdmx_url,query])
            tree = self.query_rest(url)
            _time_series = []
            _metadata = []
            for series in tree.iterfind(".//generic:Series",
                                             namespaces=tree.nsmap):
                codes = {}
                for codes_ in series.iterfind(".//generic:SeriesKey",
                                           namespaces=tree.nsmap):
                    for key in codes_.iterfind(".//generic:Value",
                                               namespaces=tree.nsmap):
                        codes[key.get('concept')] = key.get('value')
                time_series_ = []
                for observation in series.iterfind(".//generic:Obs",
                                                   namespaces=tree.nsmap):
                    dimensions = observation.xpath(".//generic:Time",
                                                   namespaces=tree.nsmap)
                    dimension = date_parser(dimensions[0].text, codes['FREQ'])
                    values = observation.xpath(".//generic:ObsValue",
                                               namespaces=tree.nsmap)
                    value = values[0].get('value')
                    observation_status = 'A'
                    for attribute in \
                        observation.iterfind(".//generic:Attributes",
                                             namespaces=tree.nsmap):
                        for observation_status_ in \
                            attribute.xpath(
                                ".//generic:Value[@concept='OBS_STATUS']",
                                namespaces=tree.nsmap):
                            if observation_status_ is not None:
                                observation_status \
                                    = observation_status_.get('value')
                    time_series_.append((dimension, value, observation_status))
                time_series_ = self.make_dataframe(time_series_, codes,
                    tuple_classes)
                result.append(time_series_)
        return result

    def make_dataframe(self, raw_series, codes, tuple_classes):
        raw_series.sort()
        for t in tuple_classes:
                try:
                    code_tuple = t(**codes)
                    break
                except TypeError:
                    if t is tuple_classes[-1]:
                        tuple_classes.append(namedtuple(
                        'CodeTuple' + str(len(tuple_classes)), codes.keys()))
                        code_tuple = tuple_classes[-1](**codes)
        else:
            tuple_classes.append(namedtuple(
                'CodeTuple' + str(len(tuple_classes)), codes.keys()))
            code_tuple = tuple_classes[0](**codes)
        dates = numpy.array(
            [observation[0] for observation in raw_series])
        values = numpy.array(
            [observation[1] for observation in raw_series])
        return pandas.DataFrame(values, columns = [code_tuple], index=dates, dtype = numpy.float64)

eurostat = SDMX_REST('http://www.ec.europa.eu/eurostat/SDMX/diss-web/rest',
                     '2_1','ESTAT')
eurostat_test = SDMX_REST('http://localhost:8800/eurostat',
                     '2_1','ESTAT')
ecb = SDMX_REST('http://sdw-ws.ecb.europa.eu',
                     '2_0','ECB')
ilo = SDMX_REST('http://www.ilo.org/ilostat/sdmx/ws/rest/',
                     '2_1','ILO')
fao = SDMX_REST('http://data.fao.org/sdmx',
                     '2_1','FAO')
<|endoftext|>


<commit_before>from ipScanner import IpScanner
from arpPoison import ArpPoison
import sys

def main():
    print 'by default the scanner will scan 192.168.0'
    args = sys.argv
    interactive = False
    ip = ''
    if '-i' in args:
        interactive = True;
        args.remove('-i')
    scanner = None

    routerIP = args[1]

    try:
        ip = args[2]
        scanner = IpScanner(ip)
    except:
        scanner = IpScanner()
    targetlist = scanner.scan()

    attacklist = []
    #who to target?
    if interactive:
        while True:
            if targetlist == []:
                print 'target list empty'
                break

            print 'get the os for the specified IP addresse, write next to go to next step.'
            print targetlist
            ip = raw_input()
            if ip == 'next':
                break
            else:
                attackOS = scanner.getOsForIp(ip)
                print attackOS

            print 'do you want to add this to the attack list? (y/n)'
            answer = raw_input()
            if answer == 'y' or answer =='yes':
                attacklist.append(ip)
                targetlist.remove(ip)

    #peut faire la meme chose pour le hostname mais sa ne me retourne jamais rien de bon
    else:
        attacklist = targetlist
        #iteraction avec usager pour les target
    print attacklist

    stopList = []
    for i in attacklist:
        t = threading.Thread(target=worker,args=(i,routerIP,stopList))
        t.start()

    while True:
        answer = raw_input()
        if answer == 'stop':
            break

def worker(ipVictim, ipAttack, stopList):
    l = threading.Lock()
    l.lock()
    stopList.append(ArpPoison(ipAttack, ipVictim))
    l.release()

    #launch atttackkk!
if __name__ == "__main__":
    main()<commit_msg>wireshark screenshot<commit_after>from ipScanner import IpScanner
from arpPoison import ArpPoison
import sys
import threading

def main():
    print 'by default the scanner will scan 192.168.0'
    args = sys.argv
    interactive = False
    ip = ''
    if '-i' in args:
        interactive = True;
        args.remove('-i')
    scanner = None

    routerIP = args[1]

    try:
        ip = args[2]
        scanner = IpScanner(ip)
    except:
        scanner = IpScanner()
    targetlist = scanner.scan()

    attacklist = []
    #who to target?
    if interactive:
        while True:
            if targetlist == []:
                print 'target list empty'
                break

            print 'get the os for the specified IP addresse, write next to go to next step.'
            print targetlist
            ip = raw_input()
            if ip == 'next':
                break
            else:
                attackOS = scanner.getOsForIp(ip)
                print attackOS

            print 'do you want to add this to the attack list? (y/n)'
            answer = raw_input()
            if answer == 'y' or answer =='yes':
                attacklist.append(ip)
                targetlist.remove(ip)

    #peut faire la meme chose pour le hostname mais sa ne me retourne jamais rien de bon
    else:
        attacklist = targetlist
        #iteraction avec usager pour les target
    print attacklist

    stopList = []
    for i in attacklist:
        t = threading.Thread(target=worker,args=(i,routerIP,stopList))
        t.start()

    while True:
        answer = raw_input()
        if answer == 'stop':
            break

def worker(ipVictim, ipAttack, stopList):
    l = threading.Lock()
    #l.lock(True)
    stopList.append(ArpPoison(ipAttack, ipVictim))
    #l.release()

    #launch atttackkk!
if __name__ == "__main__":
    main()<|endoftext|>


<commit_before>import os

types_map = {
    # http://www.iana.org/assignments/media-types
    '.a': 'application/octet-stream',
    '.ai': 'application/postscript',
    '.aif': 'audio/x-aiff',
    '.aifc': 'audio/x-aiff',
    '.aiff': 'audio/x-aiff',
    '.au': 'audio/basic',
    '.avi': 'video/x-msvideo',
    '.bat': 'text/plain',
    '.bcpio': 'application/x-bcpio',
    '.bin': 'application/octet-stream',
    '.bmp': 'image/bmp',
    '.c': 'text/plain',
    '.cdf': 'application/x-netcdf',
    '.cpio': 'application/x-cpio',
    '.csh': 'application/x-csh',
    '.css': 'text/css',
    '.csv': 'text/csv',
    '.dll': 'application/octet-stream',
    '.doc': 'application/msword',
    '.dot': 'application/msword',
    '.dvi': 'application/x-dvi',
    '.eml': 'message/rfc822',
    '.eps': 'application/postscript',
    '.epub': 'application/epub+zip',
    '.etx': 'text/x-setext',
    '.exe': 'application/octet-stream',
    '.gif': 'image/gif',
    '.gtar': 'application/x-gtar',
    '.h': 'text/plain',
    '.hdf': 'application/x-hdf',
    '.htm': 'text/html',
    '.html': 'text/html',
    '.ico': 'image/vnd.microsoft.icon',
    '.ief': 'image/ief',
    '.iges': 'model/iges',
    '.jpe': 'image/jpeg',
    '.jpeg': 'image/jpeg',
    '.jpg': 'image/jpeg',
    '.js': 'application/javascript',
    '.json': 'application/json',
    '.ksh': 'text/plain',
    '.latex': 'application/x-latex',
    '.m1v': 'video/mpeg',
    '.m3u': 'application/vnd.apple.mpegurl',
    '.m3u8': 'application/vnd.apple.mpegurl',
    '.man': 'application/x-troff-man',
    '.markdown': 'text/markdown',
    '.md': 'text/markdown',
    '.me': 'application/x-troff-me',
    '.mht': 'message/rfc822',
    '.mhtml': 'message/rfc822',
    '.mif': 'application/x-mif',
    '.mov': 'video/quicktime',
    '.movie': 'video/x-sgi-movie',
    '.mp2': 'audio/mpeg',
    '.mp3': 'audio/mpeg',
    '.mp4': 'video/mp4',
    '.mpa': 'video/mpeg',
    '.mpe': 'video/mpeg',
    '.mpeg': 'video/mpeg',
    '.mpg': 'video/mpeg',
    '.ms': 'application/x-troff-ms',
    '.nc': 'application/x-netcdf',
    '.nws': 'message/rfc822',
    '.o': 'application/octet-stream',
    '.obj': 'application/octet-stream',
    '.oda': 'application/oda',
    '.p12': 'application/x-pkcs12',
    '.p7c': 'application/pkcs7-mime',
    '.pbm': 'image/x-portable-bitmap',
    '.pdf': 'application/pdf',
    '.pfx': 'application/x-pkcs12',
    '.pgm': 'image/x-portable-graymap',
    '.pl': 'text/plain',
    '.png': 'image/png',
    '.pnm': 'image/x-portable-anymap',
    '.pot': 'application/vnd.ms-powerpoint',
    '.ppa': 'application/vnd.ms-powerpoint',
    '.ppm': 'image/x-portable-pixmap',
    '.pps': 'application/vnd.ms-powerpoint',
    '.ppt': 'application/vnd.ms-powerpoint',
    '.ps': 'application/postscript',
    '.pwz': 'application/vnd.ms-powerpoint',
    '.py': 'text/x-python',
    '.pyc': 'application/x-python-code',
    '.pyo': 'application/x-python-code',
    '.qt': 'video/quicktime',
    '.ra': 'audio/x-pn-realaudio',
    '.ram': 'application/x-pn-realaudio',
    '.ras': 'image/x-cmu-raster',
    '.rdf': 'application/xml',
    '.rgb': 'image/x-rgb',
    '.roff': 'application/x-troff',
    '.rtx': 'text/richtext',
    '.sgm': 'text/x-sgml',
    '.sgml': 'text/x-sgml',
    '.sh': 'application/x-sh',
    '.shar': 'application/x-shar',
    '.snd': 'audio/basic',
    '.so': 'application/octet-stream',
    '.src': 'application/x-wais-source',
    '.stl': 'model/stl',
    '.sv4cpio': 'application/x-sv4cpio',
    '.sv4crc': 'application/x-sv4crc',
    '.svg': 'image/svg+xml',
    '.swf': 'application/x-shockwave-flash',
    '.t': 'application/x-troff',
    '.tar': 'application/x-tar',
    '.tcl': 'application/x-tcl',
    '.tex': 'application/x-tex',
    '.texi': 'application/x-texinfo',
    '.texinfo': 'application/x-texinfo',
    '.tif': 'image/tiff',
    '.tiff': 'image/tiff',
    '.tr': 'application/x-troff',
    '.tsv': 'text/tab-separated-values',
    '.txt': 'text/plain',
    '.ustar': 'application/x-ustar',
    '.vcf': 'text/x-vcard',
    '.wav': 'audio/x-wav',
    '.webm': 'video/webm',
    '.wiz': 'application/msword',
    '.wsdl': 'application/xml',
    '.xbm': 'image/x-xbitmap',
    '.xlb': 'application/vnd.ms-excel',
    '.xls': 'application/vnd.ms-excel',
    '.xml': 'text/xml',
    '.xpdl': 'application/xml',
    '.xpm': 'image/x-xpixmap',
    '.xsl': 'application/xml',
    '.xwd': 'image/x-xwindowdump',
    '.zip': 'application/zip',

    # These are non-standard types, commonly found in the wild.
    '.mid': 'audio/midi',
    '.midi': 'audio/midi',
    '.pct': 'image/pict',
    '.pic': 'image/pict',
    '.pict': 'image/pict',
    '.rtf': 'application/rtf',
    '.xul': 'text/xul',
    '.m4v': 'video/m4v',

    # microsoft is special and has its own 'standard'
    # https://docs.microsoft.com/en-us/windows/desktop/wmp/file-name-extensions
    '.wmv': 'video/x-ms-wmv'
}

def guess_media_type(path):
    _, ext = os.path.splitext(path)
    return types_map.get(
        ext.strip().lower(),
        'application/octet-stream' if not (ext and ext[1:].strip()) else f'application/x-ext-{ext[1:].strip().lower()}'
    )
<commit_msg>stream type mapping<commit_after>import os

types_map = {
    # http://www.iana.org/assignments/media-types
    # Type mapping for automated metadata extraction (video, audio, image, document, binary, model)
    '.a': ('application/octet-stream', 'binary'),
    '.ai': ('application/postscript', 'image'),
    '.aif': ('audio/x-aiff', 'audio'),
    '.aifc': ('audio/x-aiff', 'audio'),
    '.aiff': ('audio/x-aiff', 'audio'),
    '.au': ('audio/basic', 'audio'),
    '.avi': ('video/x-msvideo', 'video'),
    '.bat': ('text/plain', 'document'),
    '.bcpio': ('application/x-bcpio', 'binary'),
    '.bin': ('application/octet-stream', 'binary'),
    '.bmp': ('image/bmp', 'image'),
    '.c': ('text/plain', 'document'),
    '.cdf': ('application/x-netcdf', 'binary'),
    '.cpio': ('application/x-cpio', 'binary'),
    '.csh': ('application/x-csh', 'binary'),
    '.css': ('text/css', 'document'),
    '.csv': ('text/csv', 'document'),
    '.dll': ('application/octet-stream', 'binary'),
    '.doc': ('application/msword', 'document'),
    '.dot': ('application/msword', 'document'),
    '.dvi': ('application/x-dvi', 'binary'),
    '.eml': ('message/rfc822', 'document'),
    '.eps': ('application/postscript', 'document'),
    '.epub': ('application/epub+zip', 'document'),
    '.etx': ('text/x-setext', 'document'),
    '.exe': ('application/octet-stream', 'binary'),
    '.gif': ('image/gif', 'image'),
    '.gtar': ('application/x-gtar', 'binary'),
    '.h': ('text/plain', 'document'),
    '.hdf': ('application/x-hdf', 'binary'),
    '.htm': ('text/html', 'document'),
    '.html': ('text/html', 'document'),
    '.ico': ('image/vnd.microsoft.icon', 'image'),
    '.ief': ('image/ief', 'image'),
    '.iges': ('model/iges', 'model'),
    '.jpe': ('image/jpeg', 'image'),
    '.jpeg': ('image/jpeg', 'image'),
    '.jpg': ('image/jpeg', 'image'),
    '.js': ('application/javascript', 'document'),
    '.json': ('application/json', 'document'),
    '.ksh': ('text/plain', 'document'),
    '.latex': ('application/x-latex', 'binary'),
    '.m1v': ('video/mpeg', 'video'),
    '.m3u': ('application/vnd.apple.mpegurl', 'audio'),
    '.m3u8': ('application/vnd.apple.mpegurl', 'audio'),
    '.man': ('application/x-troff-man', 'document'),
    '.markdown': ('text/markdown', 'document'),
    '.md': ('text/markdown', 'document'),
    '.me': ('application/x-troff-me', 'binary'),
    '.mht': ('message/rfc822', 'document'),
    '.mhtml': ('message/rfc822', 'document'),
    '.mif': ('application/x-mif', 'binary'),
    '.mov': ('video/quicktime', 'video'),
    '.movie': ('video/x-sgi-movie', 'video'),
    '.mp2': ('audio/mpeg', 'audio'),
    '.mp3': ('audio/mpeg', 'audio'),
    '.mp4': ('video/mp4', 'video'),
    '.mpa': ('video/mpeg', 'video'),
    '.mpe': ('video/mpeg', 'video'),
    '.mpeg': ('video/mpeg', 'video'),
    '.mpg': ('video/mpeg', 'video'),
    '.ms': ('application/x-troff-ms', 'binary'),
    '.nc': ('application/x-netcdf', 'binary'),
    '.nws': ('message/rfc822', 'document'),
    '.o': ('application/octet-stream', 'binary'),
    '.obj': ('application/octet-stream', 'model'),
    '.oda': ('application/oda', 'binary'),
    '.p12': ('application/x-pkcs12', 'binary'),
    '.p7c': ('application/pkcs7-mime', 'binary'),
    '.pbm': ('image/x-portable-bitmap', 'image'),
    '.pdf': ('application/pdf', 'document'),
    '.pfx': ('application/x-pkcs12', 'binary'),
    '.pgm': ('image/x-portable-graymap', 'image'),
    '.pl': ('text/plain', 'document'),
    '.png': ('image/png', 'image'),
    '.pnm': ('image/x-portable-anymap', 'image'),
    '.pot': ('application/vnd.ms-powerpoint', 'document'),
    '.ppa': ('application/vnd.ms-powerpoint', 'document'),
    '.ppm': ('image/x-portable-pixmap', 'image'),
    '.pps': ('application/vnd.ms-powerpoint', 'document'),
    '.ppt': ('application/vnd.ms-powerpoint', 'document'),
    '.ps': ('application/postscript', 'document'),
    '.pwz': ('application/vnd.ms-powerpoint', 'document'),
    '.py': ('text/x-python', 'document'),
    '.pyc': ('application/x-python-code', 'binary'),
    '.pyo': ('application/x-python-code', 'binary'),
    '.qt': ('video/quicktime', 'video'),
    '.ra': ('audio/x-pn-realaudio', 'audio'),
    '.ram': ('application/x-pn-realaudio', 'audio'),
    '.ras': ('image/x-cmu-raster', 'image'),
    '.rdf': ('application/xml', 'binary'),
    '.rgb': ('image/x-rgb', 'image'),
    '.roff': ('application/x-troff', 'binary'),
    '.rtx': ('text/richtext', 'document'),
    '.sgm': ('text/x-sgml', 'document'),
    '.sgml': ('text/x-sgml', 'document'),
    '.sh': ('application/x-sh', 'document'),
    '.shar': ('application/x-shar', 'binary'),
    '.snd': ('audio/basic', 'audio'),
    '.so': ('application/octet-stream', 'binary'),
    '.src': ('application/x-wais-source', 'binary'),
    '.stl': ('model/stl', 'model'),
    '.sv4cpio': ('application/x-sv4cpio', 'binary'),
    '.sv4crc': ('application/x-sv4crc', 'binary'),
    '.svg': ('image/svg+xml', 'image'),
    '.swf': ('application/x-shockwave-flash', 'binary'),
    '.t': ('application/x-troff', 'binary'),
    '.tar': ('application/x-tar', 'binary'),
    '.tcl': ('application/x-tcl', 'binary'),
    '.tex': ('application/x-tex', 'binary'),
    '.texi': ('application/x-texinfo', 'binary'),
    '.texinfo': ('application/x-texinfo', 'binary'),
    '.tif': ('image/tiff', 'image'),
    '.tiff': ('image/tiff', 'image'),
    '.tr': ('application/x-troff', 'binary'),
    '.tsv': ('text/tab-separated-values', 'document'),
    '.txt': ('text/plain', 'document'),
    '.ustar': ('application/x-ustar', 'binary'),
    '.vcf': ('text/x-vcard', 'document'),
    '.wav': ('audio/x-wav', 'audio'),
    '.webm': ('video/webm', 'video'),
    '.wiz': ('application/msword', 'document'),
    '.wsdl': ('application/xml', 'document'),
    '.xbm': ('image/x-xbitmap', 'image'),
    '.xlb': ('application/vnd.ms-excel', 'document'),
    '.xls': ('application/vnd.ms-excel', 'document'),
    '.xml': ('text/xml', 'document'),
    '.xpdl': ('application/xml', 'document'),
    '.xpm': ('image/x-xpixmap', 'image'),
    '.xsl': ('application/xml', 'document'),
    '.xwd': ('image/x-xwindowdump', 'image'),
    '.zip': ('application/zip', 'binary'),

    # These are non-standard types, commonly found in the wild.
    '.cbr': ('application/vnd.comicbook+zip', 'document'),
    '.cbz': ('application/vnd.comicbook+zip', 'document'),
    '.lbry': ('application/x-ext-lbry', 'document'),
    '.mid': ('audio/midi', 'audio'),
    '.midi': ('audio/midi', 'audio'),
    '.mobi': ('application/x-mobipocket-ebook', 'document'),
    '.pct': ('image/pict', 'image'),
    '.pic': ('image/pict', 'image'),
    '.pict': ('image/pict', 'image'),
    '.prc': ('application/x-mobipocket-ebook', 'document'),
    '.rtf': ('application/rtf', 'document'),
    '.xul': ('text/xul', 'document'),
    '.m4v': ('video/m4v', 'video'),

    # microsoft is special and has its own 'standard'
    # https://docs.microsoft.com/en-us/windows/desktop/wmp/file-name-extensions
    '.wmv': ('video/x-ms-wmv', 'video')
}

def guess_media_type(path):
    _, ext = os.path.splitext(path)
    extension = ext.strip().lower()
    if extension:
        if extension in types_map:
            return types_map[extension][0]
        return f'application/x-ext-{extension[1:]}'
    return 'application/octet-stream'
<|endoftext|>


<commit_before>import asyncio
import typing
import logging
from lbrynet.utils import resolve_host
from lbrynet.stream.assembler import StreamAssembler
from lbrynet.stream.descriptor import StreamDescriptor
from lbrynet.blob_exchange.downloader import BlobDownloader
from lbrynet.dht.peer import KademliaPeer
if typing.TYPE_CHECKING:
    from lbrynet.conf import Config
    from lbrynet.dht.node import Node
    from lbrynet.blob.blob_manager import BlobFileManager
    from lbrynet.blob.blob_file import BlobFile

log = logging.getLogger(__name__)

def drain_into(a: list, b: list):
    while a:
        b.append(a.pop())

class StreamDownloader(StreamAssembler):
    def __init__(self, loop: asyncio.BaseEventLoop, config: 'Config', blob_manager: 'BlobFileManager', sd_hash: str,
                 output_dir: typing.Optional[str] = None, output_file_name: typing.Optional[str] = None):
        super().__init__(loop, blob_manager, sd_hash, output_file_name)
        self.config = config
        self.output_dir = output_dir or self.config.download_dir
        self.output_file_name = output_file_name
        self.blob_downloader: typing.Optional[BlobDownloader] = None
        self.search_queue = asyncio.Queue(loop=loop)
        self.peer_queue = asyncio.Queue(loop=loop)
        self.accumulate_task: typing.Optional[asyncio.Task] = None
        self.descriptor: typing.Optional[StreamDescriptor]
        self.node: typing.Optional['Node'] = None
        self.assemble_task: typing.Optional[asyncio.Task] = None
        self.fixed_peers_handle: typing.Optional[asyncio.Handle] = None

    async def setup(self):  # start the peer accumulator and initialize the downloader
        if self.blob_downloader:
            raise Exception("downloader is already set up")
        if self.node:
            _, self.accumulate_task = self.node.accumulate_peers(self.search_queue, self.peer_queue)
        self.blob_downloader = BlobDownloader(self.loop, self.config, self.blob_manager, self.peer_queue)
        self.search_queue.put_nowait(self.sd_hash)

    async def after_got_descriptor(self):
        self.search_queue.put_nowait(self.descriptor.blobs[0].blob_hash)
        log.info("added head blob to search")

    async def after_finished(self):
        log.info("downloaded stream %s -> %s", self.sd_hash, self.output_path)
        await self.blob_manager.storage.change_file_status(self.descriptor.stream_hash, 'finished')
        self.blob_downloader.close()

    def stop(self):
        if self.accumulate_task:
            self.accumulate_task.cancel()
            self.accumulate_task = None
        if self.assemble_task:
            self.assemble_task.cancel()
            self.assemble_task = None
        if self.fixed_peers_handle:
            self.fixed_peers_handle.cancel()
            self.fixed_peers_handle = None
        self.blob_downloader = None
        if self.stream_handle:
            if not self.stream_handle.closed:
                self.stream_handle.close()
            self.stream_handle = None

    async def get_blob(self, blob_hash: str, length: typing.Optional[int] = None) -> 'BlobFile':
        return await self.blob_downloader.download_blob(blob_hash, length)

    def add_fixed_peers(self):
        async def _add_fixed_peers():
            self.peer_queue.put_nowait([
                KademliaPeer(self.loop, address=(await resolve_host(url, port + 1, proto='tcp')), tcp_port=port + 1)
                for url, port in self.config.reflector_servers
            ])
        if self.config.reflector_servers:
            self.fixed_peers_handle = self.loop.call_later(
                self.config.fixed_peer_delay if (
                        'dht' not in self.config.components_to_skip
                        and self.node
                        and len(self.node.protocol.routing_table.get_peers())
                ) else 0.0,
                lambda: self.loop.create_task(_add_fixed_peers())
            )

    def download(self, node: typing.Optional['Node'] = None):
        self.node = node
        self.assemble_task = self.loop.create_task(self.assemble_decrypted_stream(self.config.download_dir))
        self.add_fixed_peers()
<commit_msg>resovle reflector IP before adding it<commit_after>import asyncio
import typing
import logging
from lbrynet.utils import resolve_host
from lbrynet.stream.assembler import StreamAssembler
from lbrynet.stream.descriptor import StreamDescriptor
from lbrynet.blob_exchange.downloader import BlobDownloader
from lbrynet.dht.peer import KademliaPeer
if typing.TYPE_CHECKING:
    from lbrynet.conf import Config
    from lbrynet.dht.node import Node
    from lbrynet.blob.blob_manager import BlobFileManager
    from lbrynet.blob.blob_file import BlobFile

log = logging.getLogger(__name__)

def drain_into(a: list, b: list):
    while a:
        b.append(a.pop())

class StreamDownloader(StreamAssembler):
    def __init__(self, loop: asyncio.BaseEventLoop, config: 'Config', blob_manager: 'BlobFileManager', sd_hash: str,
                 output_dir: typing.Optional[str] = None, output_file_name: typing.Optional[str] = None):
        super().__init__(loop, blob_manager, sd_hash, output_file_name)
        self.config = config
        self.output_dir = output_dir or self.config.download_dir
        self.output_file_name = output_file_name
        self.blob_downloader: typing.Optional[BlobDownloader] = None
        self.search_queue = asyncio.Queue(loop=loop)
        self.peer_queue = asyncio.Queue(loop=loop)
        self.accumulate_task: typing.Optional[asyncio.Task] = None
        self.descriptor: typing.Optional[StreamDescriptor]
        self.node: typing.Optional['Node'] = None
        self.assemble_task: typing.Optional[asyncio.Task] = None
        self.fixed_peers_handle: typing.Optional[asyncio.Handle] = None

    async def setup(self):  # start the peer accumulator and initialize the downloader
        if self.blob_downloader:
            raise Exception("downloader is already set up")
        if self.node:
            _, self.accumulate_task = self.node.accumulate_peers(self.search_queue, self.peer_queue)
        self.blob_downloader = BlobDownloader(self.loop, self.config, self.blob_manager, self.peer_queue)
        self.search_queue.put_nowait(self.sd_hash)

    async def after_got_descriptor(self):
        self.search_queue.put_nowait(self.descriptor.blobs[0].blob_hash)
        log.info("added head blob to search")

    async def after_finished(self):
        log.info("downloaded stream %s -> %s", self.sd_hash, self.output_path)
        await self.blob_manager.storage.change_file_status(self.descriptor.stream_hash, 'finished')
        self.blob_downloader.close()

    def stop(self):
        if self.accumulate_task:
            self.accumulate_task.cancel()
            self.accumulate_task = None
        if self.assemble_task:
            self.assemble_task.cancel()
            self.assemble_task = None
        if self.fixed_peers_handle:
            self.fixed_peers_handle.cancel()
            self.fixed_peers_handle = None
        self.blob_downloader = None
        if self.stream_handle:
            if not self.stream_handle.closed:
                self.stream_handle.close()
            self.stream_handle = None

    async def get_blob(self, blob_hash: str, length: typing.Optional[int] = None) -> 'BlobFile':
        return await self.blob_downloader.download_blob(blob_hash, length)

    def add_fixed_peers(self):
        async def _add_fixed_peers():
            addresses = [
                (await resolve_host(url, port + 1, proto='tcp'), port)
                for url, port in self.config.reflector_servers
            ]
            delay = self.config.fixed_peer_delay if (
                    'dht' not in self.config.components_to_skip
                    and self.node and len(self.node.protocol.routing_table.get_peers())
            ) else 0.0
            self.loop.call_later(delay, lambda:
                self.peer_queue.put_nowait([
                    KademliaPeer(self.loop, address=address, tcp_port=port + 1)
                    for address, port in addresses
            ]))
        if self.config.reflector_servers:
            self.loop.create_task(_add_fixed_peers())

    def download(self, node: typing.Optional['Node'] = None):
        self.node = node
        self.assemble_task = self.loop.create_task(self.assemble_decrypted_stream(self.config.download_dir))
        self.add_fixed_peers()
<|endoftext|>


<commit_before>"""
Django settings for testproject project.

Generated by 'django-admin startproject' using Django 1.10.6.

For more information on this file, see
https://docs.djangoproject.com/en/1.10/topics/settings/

For the full list of settings and their values, see
https://docs.djangoproject.com/en/1.10/ref/settings/
"""

import os

# Build paths inside the project like this: os.path.join(BASE_DIR, ...)
BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))

# Quick-start development settings - unsuitable for production
# See https://docs.djangoproject.com/en/1.10/howto/deployment/checklist/

# SECURITY WARNING: keep the secret key used in production secret!
SECRET_KEY = '*lx!g2o%m)b613n$709334=+ulwi^&6e8=o6h3upwn4&3c$o^p'

# SECURITY WARNING: don't run with debug turned on in production!
DEBUG = True

ALLOWED_HOSTS = []

# Application definition

INSTALLED_APPS = [
    'django.contrib.admin',
    'django.contrib.auth',
    'django.contrib.contenttypes',
    'django.contrib.sessions',
    'django.contrib.messages',
    'django.contrib.staticfiles',

    'lbattachment',

    'lbworkflow',
    'lbworkflow.tests.leave',
]

MIDDLEWARE = [
    'django.middleware.security.SecurityMiddleware',
    'django.contrib.sessions.middleware.SessionMiddleware',
    'django.middleware.common.CommonMiddleware',
    'django.middleware.csrf.CsrfViewMiddleware',
    'django.contrib.auth.middleware.AuthenticationMiddleware',
    'django.contrib.messages.middleware.MessageMiddleware',
    'django.middleware.clickjacking.XFrameOptionsMiddleware',
]

ROOT_URLCONF = 'lbworkflow.tests.urls'

TEMPLATES = [
    {
        'BACKEND': 'django.template.backends.django.DjangoTemplates',
        'DIRS': [],
        'APP_DIRS': True,
        'OPTIONS': {
            'context_processors': [
                'django.template.context_processors.debug',
                'django.template.context_processors.request',
                'django.contrib.auth.context_processors.auth',
                'django.contrib.messages.context_processors.messages',
            ],
        },
    },
]

WSGI_APPLICATION = 'testproject.wsgi.application'

# Database
# https://docs.djangoproject.com/en/1.10/ref/settings/#databases

DATABASES = {
    'default': {
        'ENGINE': 'django.db.backends.sqlite3',
    }
}

# Password validation
# https://docs.djangoproject.com/en/1.10/ref/settings/#auth-password-validators

AUTH_PASSWORD_VALIDATORS = [
    {
        'NAME': 'django.contrib.auth.password_validation.UserAttributeSimilarityValidator',
    },
    {
        'NAME': 'django.contrib.auth.password_validation.MinimumLengthValidator',
    },
    {
        'NAME': 'django.contrib.auth.password_validation.CommonPasswordValidator',
    },
    {
        'NAME': 'django.contrib.auth.password_validation.NumericPasswordValidator',
    },
]

# Internationalization
# https://docs.djangoproject.com/en/1.10/topics/i18n/

LANGUAGE_CODE = 'en-us'

TIME_ZONE = 'UTC'

USE_I18N = True

USE_L10N = True

USE_TZ = True

# Static files (CSS, JavaScript, Images)
# https://docs.djangoproject.com/en/1.10/howto/static-files/

STATIC_URL = '/static/'

LBWF_APPS = {
    'leave': 'lbworkflow.tests.leave',
    'leave2': 'lbworkflow.tests.leave2',
}
<commit_msg>remove leave2 from LBWF_APPS<commit_after>"""
Django settings for testproject project.

Generated by 'django-admin startproject' using Django 1.10.6.

For more information on this file, see
https://docs.djangoproject.com/en/1.10/topics/settings/

For the full list of settings and their values, see
https://docs.djangoproject.com/en/1.10/ref/settings/
"""

import os

# Build paths inside the project like this: os.path.join(BASE_DIR, ...)
BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))

# Quick-start development settings - unsuitable for production
# See https://docs.djangoproject.com/en/1.10/howto/deployment/checklist/

# SECURITY WARNING: keep the secret key used in production secret!
SECRET_KEY = '*lx!g2o%m)b613n$709334=+ulwi^&6e8=o6h3upwn4&3c$o^p'

# SECURITY WARNING: don't run with debug turned on in production!
DEBUG = True

ALLOWED_HOSTS = []

# Application definition

INSTALLED_APPS = [
    'django.contrib.admin',
    'django.contrib.auth',
    'django.contrib.contenttypes',
    'django.contrib.sessions',
    'django.contrib.messages',
    'django.contrib.staticfiles',

    'lbattachment',

    'lbworkflow',
    'lbworkflow.tests.leave',
]

MIDDLEWARE = [
    'django.middleware.security.SecurityMiddleware',
    'django.contrib.sessions.middleware.SessionMiddleware',
    'django.middleware.common.CommonMiddleware',
    'django.middleware.csrf.CsrfViewMiddleware',
    'django.contrib.auth.middleware.AuthenticationMiddleware',
    'django.contrib.messages.middleware.MessageMiddleware',
    'django.middleware.clickjacking.XFrameOptionsMiddleware',
]

ROOT_URLCONF = 'lbworkflow.tests.urls'

TEMPLATES = [
    {
        'BACKEND': 'django.template.backends.django.DjangoTemplates',
        'DIRS': [],
        'APP_DIRS': True,
        'OPTIONS': {
            'context_processors': [
                'django.template.context_processors.debug',
                'django.template.context_processors.request',
                'django.contrib.auth.context_processors.auth',
                'django.contrib.messages.context_processors.messages',
            ],
        },
    },
]

WSGI_APPLICATION = 'testproject.wsgi.application'

# Database
# https://docs.djangoproject.com/en/1.10/ref/settings/#databases

DATABASES = {
    'default': {
        'ENGINE': 'django.db.backends.sqlite3',
    }
}

# Password validation
# https://docs.djangoproject.com/en/1.10/ref/settings/#auth-password-validators

AUTH_PASSWORD_VALIDATORS = [
    {
        'NAME': 'django.contrib.auth.password_validation.UserAttributeSimilarityValidator',
    },
    {
        'NAME': 'django.contrib.auth.password_validation.MinimumLengthValidator',
    },
    {
        'NAME': 'django.contrib.auth.password_validation.CommonPasswordValidator',
    },
    {
        'NAME': 'django.contrib.auth.password_validation.NumericPasswordValidator',
    },
]

# Internationalization
# https://docs.djangoproject.com/en/1.10/topics/i18n/

LANGUAGE_CODE = 'en-us'

TIME_ZONE = 'UTC'

USE_I18N = True

USE_L10N = True

USE_TZ = True

# Static files (CSS, JavaScript, Images)
# https://docs.djangoproject.com/en/1.10/howto/static-files/

STATIC_URL = '/static/'

LBWF_APPS = {
    'leave': 'lbworkflow.tests.leave',
}
<|endoftext|>


<commit_before>class LRUCache:

    # @param capacity, an integer
    def __init__(self, capacity):
        self.store = {} # {key: [value, times_called_by_get]}
        self.capacity = capacity

    # @return an integer
    def get(self, key):
        if key in self.store:
            self.store[key][1] += 1
            return self.store[key][0]
        else:
            return -1

    # @param key, an integer
    # @param value, an integer
    # @return nothing
    def set(self, key, value):
        if len(self.store) >= self.capacity:
            key_to_discard = min(self.store.items(), key=lambda x: x[1][1])[0] # return the first occurrence if multiple entries exist
            self.store.pop(key_to_discard)
        self.store[key] = [value, 0]
        return None
<commit_msg>need better data structure for keeping track of get usage<commit_after>class LRUCache:

    # @param capacity, an integer
    def __init__(self, capacity):
        self.store = {} # {key: [value, times_called_by_get]}
        self.capacity = capacity

    # @return an integer
    def get(self, key):
        if key in self.store:
            self.store[key][1] += 1
            return self.store[key][0]
        else:
            return -1

    # @param key, an integer
    # @param value, an integer
    # @return nothing
    def set(self, key, value):
        if len(self.store) >= self.capacity:
            # the following operation is too costly, when capacity is reached
            key_to_discard = min(self.store.items(), key=lambda x: x[1][1])[0] # return the first occurrence if multiple entries exist
            self.store.pop(key_to_discard)
        self.store[key] = [value, 0]
        return None
<|endoftext|>


<commit_before>"""
.. module: lemur.certificates.models
    :platform: Unix
    :copyright: (c) 2015 by Netflix Inc., see AUTHORS for more
    :license: Apache, see LICENSE for more details.
.. moduleauthor:: Kevin Glisson <kglisson@netflix.com>
"""
import os
import datetime
from cryptography import x509
from cryptography.hazmat.backends import default_backend

from flask import current_app

from sqlalchemy.orm import relationship
from sqlalchemy import Integer, ForeignKey, String, DateTime, PassiveDefault, func, Column, Text, Boolean

from sqlalchemy_utils import EncryptedType

from lemur.database import db

from lemur.domains.models import Domain
from lemur.users import service as user_service

from lemur.constants import SAN_NAMING_TEMPLATE, DEFAULT_NAMING_TEMPLATE, NONSTANDARD_NAMING_TEMPLATE
from lemur.models import certificate_associations, certificate_account_associations

def cert_get_cn(cert):
    """
    Attempts to get a sane common name from a given certificate.

    :param cert:
    :return: Common name or None
    """
    try:
        return cert.subject.get_attributes_for_oid(
            x509.OID_COMMON_NAME
        )[0].value.strip()
    except Exception as e:
        current_app.logger.error("Unable to get CN! {0}".format(e))

def cert_get_domains(cert):
    """
    Attempts to get an domains listed in a certificate.
    If 'subjectAltName' extension is not available we simply
    return the common name.

    :param cert:
    :return: List of domains
    """
    domains = []
    try:
        ext = cert.extensions.get_extension_for_oid(x509.OID_SUBJECT_ALTERNATIVE_NAME)
        entries = ext.get_values_for(x509.DNSName)
        for entry in entries:
            domains.append(entry.split(":")[1].strip(", "))
    except Exception as e:
        current_app.logger.warning("Failed to get SubjectAltName: {0}".format(e))
        domains.append(cert_get_cn(cert))
    return domains

def cert_get_serial(cert):
    """
    Fetch the serial number from the certificate.

    :param cert:
    :return: serial number
    """
    return cert.serial

def cert_is_san(cert):
    """
    Determines if a given certificate is a SAN certificate.
    SAN certificates are simply certificates that cover multiple domains.

    :param cert:
    :return: Bool
    """
    domains = cert_get_domains(cert)
    if len(domains) > 1:
        return True
    return False

def cert_is_wildcard(cert):
    """
    Determines if certificate is a wildcard certificate.

    :param cert:
    :return: Bool
    """
    domains = cert_get_domains(cert)
    if len(domains) == 1 and domains[0][0:1] == "*":
        return True
    return False

def cert_get_bitstrength(cert):
    """
    Calculates a certificates public key bit length.

    :param cert:
    :return: Integer
    """
    return cert.public_key().key_size * 8

def cert_get_issuer(cert):
    """
    Gets a sane issuer from a given certificate.

    :param cert:
    :return: Issuer
    """
    try:
        return cert.subject.get_attributes_for_oid(x509.OID_ORGANIZATION_NAME)[0].value
    except Exception as e:
        current_app.logger.error("Unable to get issuer! {0}".format(e))

def cert_is_internal(cert):
    """
    Uses an internal resource in order to determine if
    a given certificate was issued by an 'internal' certificate
    authority.

    :param cert:
    :return: Bool
    """
    if cert_get_issuer(cert) in current_app.config.get('INTERNAL_CA', []):
        return True
    return False

def cert_get_not_before(cert):
    """
    Gets the naive datetime of the certificates 'not_before' field.
    This field denotes the first date in time which the given certificate
    is valid.

    :param cert:
    :return: Datetime
    """
    return cert.not_valid_before

def cert_get_not_after(cert):
    """
    Gets the naive datetime of the certificates 'not_after' field.
    This field denotes the last date in time which the given certificate
    is valid.

    :param cert:
    :return: Datetime
    """
    return cert.not_valid_after

def get_name_from_arn(arn):
    """
    Extract the certificate name from an arn.

    :param arn: IAM SSL arn
    :return: name of the certificate as uploaded to AWS
    """
    return arn.split("/", 1)[1]

def get_account_number(arn):
    """
    Extract the account number from an arn.

    :param arn: IAM SSL arn
    :return: account number associated with ARN
    """
    return arn.split(":")[4]

class Certificate(db.Model):
    __tablename__ = 'certificates'
    id = Column(Integer, primary_key=True)
    owner = Column(String(128))
    body = Column(Text())
    private_key = Column(EncryptedType(String, os.environ.get('LEMUR_ENCRYPTION_KEY')))
    challenge = Column(EncryptedType(String, os.environ.get('LEMUR_ENCRYPTION_KEY')))
    csr_config = Column(Text())
    status = Column(String(128))
    deleted = Column(Boolean, index=True)
    name = Column(String(128))
    chain = Column(Text())
    bits = Column(Integer())
    issuer = Column(String(128))
    serial = Column(String(128))
    cn = Column(String(128))
    description = Column(String(1024))
    active = Column(Boolean, default=True)
    san = Column(String(1024))
    not_before = Column(DateTime)
    not_after = Column(DateTime)
    date_created = Column(DateTime, PassiveDefault(func.now()), nullable=False)
    user_id = Column(Integer, ForeignKey('users.id'))
    authority_id = Column(Integer, ForeignKey('authorities.id'))
    accounts = relationship("Account", secondary=certificate_account_associations, backref='certificate')
    domains = relationship("Domain", secondary=certificate_associations, backref="certificate")
    elb_listeners = relationship("Listener", lazy='dynamic', backref='certificate')

    def __init__(self, body, private_key=None, challenge=None, chain=None, csr_config=None):
        self.body = body
        # We encrypt the private_key on creation
        self.private_key = private_key
        self.chain = chain
        self.csr_config = csr_config
        self.challenge = challenge
        cert = x509.load_pem_x509_certificate(str(self.body), default_backend())
        self.bits = cert_get_bitstrength(cert)
        self.issuer = cert_get_issuer(cert)
        self.serial = cert_get_serial(cert)
        self.cn = cert_get_cn(cert)
        self.san = cert_is_san(cert)
        self.not_before = cert_get_not_before(cert)
        self.not_after = cert_get_not_after(cert)
        self.name = self.create_name

        for domain in cert_get_domains(cert):
            self.domains.append(Domain(name=domain))

    @property
    def create_name(self):
        """
        Create a name for our certificate. A naming standard
        is based on a series of templates. The name includes
        useful information such as Common Name, Validation dates,
        and Issuer.

        :rtype : str
        :return:
        """
        # aws doesn't allow special chars
        if self.cn:
            subject = self.cn.replace('*', "WILDCARD")

            if self.san:
                t = SAN_NAMING_TEMPLATE
            else:
                t = DEFAULT_NAMING_TEMPLATE

            temp = t.format(
                subject=subject,
                issuer=self.issuer,
                not_before=self.not_before.strftime('%Y%m%d'),
                not_after=self.not_after.strftime('%Y%m%d')
            )

        else:
            t = NONSTANDARD_NAMING_TEMPLATE

            temp = t.format(
                issuer=self.issuer,
                not_before=self.not_before.strftime('%Y%m%d'),
                not_after=self.not_after.strftime('%Y%m%d')
            )

        return temp

    @property
    def is_expired(self):
        if self.not_after < datetime.datetime.now():
            return True

    @property
    def is_unused(self):
        if self.elb_listeners.count() == 0:
            return True

    @property
    def is_revoked(self):
        # we might not yet know the condition of the cert
        if self.status:
            if 'revoked' in self.status:
                return True

    def get_arn(self, account_number):
        """
        Generate a valid AWS IAM arn

        :rtype : str
        :param account_number:
        :return:
        """
        return "arn:aws:iam::{}:server-certificate/{}".format(account_number, self.name)

    def as_dict(self):
        return {c.name: getattr(self, c.name) for c in self.__table__.columns}

    def serialize(self):
        blob = self.as_dict()
        # TODO this should be done with relationships
        user = user_service.get(self.user_id)
        if user:
            blob['creator'] = user.username

        return blob

<commit_msg>Fixes an issue where the issuer has special chars in the name. AWS dislikes special chars in certificate names so we strip them out here. In general we want to have the name tracked by Lemur be the same as what is uploaded to various destinations.<commit_after>"""
.. module: lemur.certificates.models
    :platform: Unix
    :copyright: (c) 2015 by Netflix Inc., see AUTHORS for more
    :license: Apache, see LICENSE for more details.
.. moduleauthor:: Kevin Glisson <kglisson@netflix.com>
"""
import os
import datetime
from cryptography import x509
from cryptography.hazmat.backends import default_backend

from flask import current_app

from sqlalchemy.orm import relationship
from sqlalchemy import Integer, ForeignKey, String, DateTime, PassiveDefault, func, Column, Text, Boolean

from sqlalchemy_utils import EncryptedType

from lemur.database import db

from lemur.domains.models import Domain
from lemur.users import service as user_service

from lemur.constants import SAN_NAMING_TEMPLATE, DEFAULT_NAMING_TEMPLATE, NONSTANDARD_NAMING_TEMPLATE
from lemur.models import certificate_associations, certificate_account_associations

def cert_get_cn(cert):
    """
    Attempts to get a sane common name from a given certificate.

    :param cert:
    :return: Common name or None
    """
    try:
        return cert.subject.get_attributes_for_oid(
            x509.OID_COMMON_NAME
        )[0].value.strip()
    except Exception as e:
        current_app.logger.error("Unable to get CN! {0}".format(e))

def cert_get_domains(cert):
    """
    Attempts to get an domains listed in a certificate.
    If 'subjectAltName' extension is not available we simply
    return the common name.

    :param cert:
    :return: List of domains
    """
    domains = []
    try:
        ext = cert.extensions.get_extension_for_oid(x509.OID_SUBJECT_ALTERNATIVE_NAME)
        entries = ext.get_values_for(x509.DNSName)
        for entry in entries:
            domains.append(entry.split(":")[1].strip(", "))
    except Exception as e:
        current_app.logger.warning("Failed to get SubjectAltName: {0}".format(e))
        domains.append(cert_get_cn(cert))
    return domains

def cert_get_serial(cert):
    """
    Fetch the serial number from the certificate.

    :param cert:
    :return: serial number
    """
    return cert.serial

def cert_is_san(cert):
    """
    Determines if a given certificate is a SAN certificate.
    SAN certificates are simply certificates that cover multiple domains.

    :param cert:
    :return: Bool
    """
    domains = cert_get_domains(cert)
    if len(domains) > 1:
        return True
    return False

def cert_is_wildcard(cert):
    """
    Determines if certificate is a wildcard certificate.

    :param cert:
    :return: Bool
    """
    domains = cert_get_domains(cert)
    if len(domains) == 1 and domains[0][0:1] == "*":
        return True
    return False

def cert_get_bitstrength(cert):
    """
    Calculates a certificates public key bit length.

    :param cert:
    :return: Integer
    """
    return cert.public_key().key_size * 8

def cert_get_issuer(cert):
    """
    Gets a sane issuer from a given certificate.

    :param cert:
    :return: Issuer
    """
    delchars = ''.join(c for c in map(chr, range(256)) if not c.isalnum())
    try:
        issuer = str(cert.subject.get_attributes_for_oid(x509.OID_ORGANIZATION_NAME)[0].value)
        return issuer.translate(None, delchars)
    except Exception as e:
        current_app.logger.error("Unable to get issuer! {0}".format(e))

def cert_is_internal(cert):
    """
    Uses an internal resource in order to determine if
    a given certificate was issued by an 'internal' certificate
    authority.

    :param cert:
    :return: Bool
    """
    if cert_get_issuer(cert) in current_app.config.get('INTERNAL_CA', []):
        return True
    return False

def cert_get_not_before(cert):
    """
    Gets the naive datetime of the certificates 'not_before' field.
    This field denotes the first date in time which the given certificate
    is valid.

    :param cert:
    :return: Datetime
    """
    return cert.not_valid_before

def cert_get_not_after(cert):
    """
    Gets the naive datetime of the certificates 'not_after' field.
    This field denotes the last date in time which the given certificate
    is valid.

    :param cert:
    :return: Datetime
    """
    return cert.not_valid_after

def get_name_from_arn(arn):
    """
    Extract the certificate name from an arn.

    :param arn: IAM SSL arn
    :return: name of the certificate as uploaded to AWS
    """
    return arn.split("/", 1)[1]

def get_account_number(arn):
    """
    Extract the account number from an arn.

    :param arn: IAM SSL arn
    :return: account number associated with ARN
    """
    return arn.split(":")[4]

class Certificate(db.Model):
    __tablename__ = 'certificates'
    id = Column(Integer, primary_key=True)
    owner = Column(String(128))
    body = Column(Text())
    private_key = Column(EncryptedType(String, os.environ.get('LEMUR_ENCRYPTION_KEY')))
    challenge = Column(EncryptedType(String, os.environ.get('LEMUR_ENCRYPTION_KEY')))
    csr_config = Column(Text())
    status = Column(String(128))
    deleted = Column(Boolean, index=True)
    name = Column(String(128))
    chain = Column(Text())
    bits = Column(Integer())
    issuer = Column(String(128))
    serial = Column(String(128))
    cn = Column(String(128))
    description = Column(String(1024))
    active = Column(Boolean, default=True)
    san = Column(String(1024))
    not_before = Column(DateTime)
    not_after = Column(DateTime)
    date_created = Column(DateTime, PassiveDefault(func.now()), nullable=False)
    user_id = Column(Integer, ForeignKey('users.id'))
    authority_id = Column(Integer, ForeignKey('authorities.id'))
    accounts = relationship("Account", secondary=certificate_account_associations, backref='certificate')
    domains = relationship("Domain", secondary=certificate_associations, backref="certificate")
    elb_listeners = relationship("Listener", lazy='dynamic', backref='certificate')

    def __init__(self, body, private_key=None, challenge=None, chain=None, csr_config=None):
        self.body = body
        # We encrypt the private_key on creation
        self.private_key = private_key
        self.chain = chain
        self.csr_config = csr_config
        self.challenge = challenge
        cert = x509.load_pem_x509_certificate(str(self.body), default_backend())
        self.bits = cert_get_bitstrength(cert)
        self.issuer = cert_get_issuer(cert)
        self.serial = cert_get_serial(cert)
        self.cn = cert_get_cn(cert)
        self.san = cert_is_san(cert)
        self.not_before = cert_get_not_before(cert)
        self.not_after = cert_get_not_after(cert)
        self.name = self.create_name

        for domain in cert_get_domains(cert):
            self.domains.append(Domain(name=domain))

    @property
    def create_name(self):
        """
        Create a name for our certificate. A naming standard
        is based on a series of templates. The name includes
        useful information such as Common Name, Validation dates,
        and Issuer.

        :rtype : str
        :return:
        """
        # aws doesn't allow special chars
        if self.cn:
            subject = self.cn.replace('*', "WILDCARD")

            if self.san:
                t = SAN_NAMING_TEMPLATE
            else:
                t = DEFAULT_NAMING_TEMPLATE

            temp = t.format(
                subject=subject,
                issuer=self.issuer,
                not_before=self.not_before.strftime('%Y%m%d'),
                not_after=self.not_after.strftime('%Y%m%d')
            )

        else:
            t = NONSTANDARD_NAMING_TEMPLATE

            temp = t.format(
                issuer=self.issuer,
                not_before=self.not_before.strftime('%Y%m%d'),
                not_after=self.not_after.strftime('%Y%m%d')
            )

        return temp

    @property
    def is_expired(self):
        if self.not_after < datetime.datetime.now():
            return True

    @property
    def is_unused(self):
        if self.elb_listeners.count() == 0:
            return True

    @property
    def is_revoked(self):
        # we might not yet know the condition of the cert
        if self.status:
            if 'revoked' in self.status:
                return True

    def get_arn(self, account_number):
        """
        Generate a valid AWS IAM arn

        :rtype : str
        :param account_number:
        :return:
        """
        return "arn:aws:iam::{}:server-certificate/{}".format(account_number, self.name)

    def as_dict(self):
        return {c.name: getattr(self, c.name) for c in self.__table__.columns}

    def serialize(self):
        blob = self.as_dict()
        # TODO this should be done with relationships
        user = user_service.get(self.user_id)
        if user:
            blob['creator'] = user.username

        return blob

<|endoftext|>


<commit_before>
from django import forms
from django.contrib import admin
from django.contrib.admin.options import InlineModelAdmin, ModelAdmin
from django.contrib.contenttypes.admin import GenericTabularInline
from django.contrib.contenttypes.models import ContentType
from django.utils.translation import ugettext_lazy as _
from feincms.admin import item_editor, tree_editor
from feincms.admin.item_editor import FeinCMSInline, ItemEditorForm
from feincms.module.page.modeladmins import PageAdmin as FeinPageAdmin

from .models import *

class PageDimensionAdmin(InlineModelAdmin):

    model = PageDimension

class PageAdmin(FeinPageAdmin):

    fieldsets = [
        (None, {
            'fields': [
                ('title', 'slug'),
                ('active', 'in_navigation'),
            ],
        }),
        (_('Other options'), {
            'classes': ['collapse'],
            'fields': [
                'template_key', 'parent', 'override_url', 'redirect_to', 'theme'],
        }),
        # <-- insertion point, extensions appear here, see insertion_index
        # above
        item_editor.FEINCMS_CONTENT_FIELDSET,
    ]

admin.site.register(Page, PageAdmin)

class PageDimensionAdmin(ModelAdmin):

    pass

admin.site.register(PageDimension, PageDimensionAdmin)

class WidgetDimensionAdmin(ModelAdmin):

    pass

admin.site.register(WidgetDimension, WidgetDimensionAdmin)

class WidgetThemeForm(forms.ModelForm):

    widget_class = forms.ChoiceField(
        choices=[],
        required=False,
    )

    class Meta:

        model = WidgetTheme

    def __init__(self, *args, **kwargs):
        super(WidgetThemeForm, self).__init__(*args, **kwargs)

        choices = [(t.model_class().__name__, t)
                   for t in ContentType.objects.filter(app_label__in=['web'])]
        self.fields['widget_class'].choices = choices

class WidgetThemeAdmin(ModelAdmin):

    form = WidgetThemeForm

admin.site.register(WidgetTheme, WidgetThemeAdmin)

class PageThemeAdmin(ModelAdmin):

    pass

admin.site.register(PageTheme, PageThemeAdmin)
<commit_msg>admin tune<commit_after>
from django import forms
from django.contrib import admin
from django.contrib.admin.options import InlineModelAdmin, ModelAdmin
from django.contrib.contenttypes.admin import GenericTabularInline
from django.contrib.contenttypes.models import ContentType
from django.utils.translation import ugettext_lazy as _
from feincms.admin import item_editor, tree_editor
from feincms.admin.item_editor import FeinCMSInline, ItemEditorForm
from feincms.module.page.modeladmins import PageAdmin as FeinPageAdmin

from .models import *

class PageDimensionAdmin(InlineModelAdmin):

    model = PageDimension

class PageAdmin(FeinPageAdmin):

    fieldsets = [
        (None, {
            'fields': [
                ('title', 'slug'),
                ('active', 'in_navigation'),
            ],
        }),
        (_('Other options'), {
            'classes': ['collapse'],
            'fields': [
                'template_key', 'parent', 'override_url', 'redirect_to', 'theme'],
        }),
        # <-- insertion point, extensions appear here, see insertion_index
        # above
        item_editor.FEINCMS_CONTENT_FIELDSET,
    ]

admin.site.register(Page, PageAdmin)

class PageDimensionAdmin(ModelAdmin):

    pass

admin.site.register(PageDimension, PageDimensionAdmin)

class WidgetDimensionAdmin(ModelAdmin):

    pass

admin.site.register(WidgetDimension, WidgetDimensionAdmin)

class WidgetThemeForm(forms.ModelForm):

    widget_class = forms.ChoiceField(
        choices=[],
        required=False,
    )

    class Meta:

        model = WidgetTheme

    def __init__(self, *args, **kwargs):
        super(WidgetThemeForm, self).__init__(*args, **kwargs)

        choices = [(t.model_class().__name__, t)
                   for t in ContentType.objects.filter(app_label__in=['web'])]
        self.fields['widget_class'].choices = choices

class WidgetThemeAdmin(ModelAdmin):

    form = WidgetThemeForm

    list_display = ('__str__', 'widget_class')

    list_filter = ('widget_class',)

admin.site.register(WidgetTheme, WidgetThemeAdmin)

class PageThemeAdmin(ModelAdmin):

    pass

admin.site.register(PageTheme, PageThemeAdmin)
<|endoftext|>