<commit_before># Copyright 2018 KMEE INFORMATICA LTDA
#   Gabriel Cardoso de Faria <gabriel.cardoso@kmee.com.br>
# Copyright (C) 2020 Renato Lima - Akretion <renato.lima@akretion.com.br>
# License AGPL-3 or later (http://www.gnu.org/licenses/agpl)

import os
import base64
import re
import tempfile
import zipfile
import io
import logging
from datetime import datetime
import calendar

from odoo import _, api, fields, models
from odoo.exceptions import RedirectWarning

from ..constants.fiscal import (
    MODELO_FISCAL_EMISSAO_PRODUTO,
    MODELO_FISCAL_EMISSAO_SERVICO,
    MODELO_FISCAL_NFE,
    MODELO_FISCAL_NFCE,
    MODELO_FISCAL_CFE,
    MODELO_FISCAL_CUPOM_FISCAL_ECF,
    MODELO_FISCAL_NFSE,
    MODELO_FISCAL_RL,
    SITUACAO_EDOC_EM_DIGITACAO,
    SITUACAO_EDOC_A_ENVIAR,
    SITUACAO_EDOC_ENVIADA,
    SITUACAO_EDOC_REJEITADA,
    SITUACAO_EDOC_AUTORIZADA,
    SITUACAO_EDOC_CANCELADA,
    SITUACAO_EDOC_DENEGADA,
    SITUACAO_EDOC_INUTILIZADA,
)

from odoo.addons.l10n_br_nfe.constants.nfe import (
    NFE_ENVIRONMENTS
)

_logger = logging.getLogger(__name__)

try:
    from erpbrasil.base import misc
except ImportError:
    _logger.error("Biblioteca erpbrasil.base não instalada")

PATH_MODELO = {
    MODELO_FISCAL_NFE: 'nfe',
    MODELO_FISCAL_NFCE: 'nfce',
    MODELO_FISCAL_CFE: 'cfe',
    MODELO_FISCAL_CUPOM_FISCAL_ECF: 'cfe_ecf',
    MODELO_FISCAL_NFSE: 'nfse',
    MODELO_FISCAL_RL: 'rl',
}

PATH_AMBIENTE = dict(NFE_ENVIRONMENTS)

SITUACAO_EDOC = [
    SITUACAO_EDOC_EM_DIGITACAO,
    SITUACAO_EDOC_A_ENVIAR,
    SITUACAO_EDOC_ENVIADA,
    SITUACAO_EDOC_REJEITADA,
    SITUACAO_EDOC_AUTORIZADA,
    SITUACAO_EDOC_CANCELADA,
    SITUACAO_EDOC_DENEGADA,
    SITUACAO_EDOC_INUTILIZADA,
]

XMLS_IMPORTANTES = [
    'file_xml_autorizacao_id',
    'file_xml_autorizacao_cancelamento_id',
    'file_xml_autorizacao_inutilizacao_id',
    'file_pdf_id',
]

class FiscalClosing(models.Model):
    _name = 'l10n_br_fiscal.closing'
    _inherit = ['mail.thread', 'mail.activity.mixin']
    _description = 'Fiscal Closing Period'

    @api.depends('month', 'year', 'export_type')
    def _compute_name(self):
        for record in self:
            if record.export_type == 'period':
                record.name = "{}/{}".format(record.month, record.year)
                record.file_name = "{}-{}".format(record.month, record.year) + '.zip'
            else:
                now = fields.Datetime.now().strftime('%d/%m/%Y')
                record.name = now
                record.file_name = now + '.zip'

    state = fields.Selection(
        selection=[
            ('draft', 'Draft'),
            ('open', 'Open'),
            ('closed', 'Closed')],
        string='state',
        default='draft',
        readonly=True)

    name = fields.Char(
        string='Name',
        compute='_compute_name',
        store=True,
        size=7,
        index=True)

    file_name = fields.Char(
        string='File Name',
        compute='_compute_name',
        store=True,
        size=11,
        index=True)

    year = fields.Char(
        string='Year',
        size=4,
        index=True)

    month = fields.Char(
        string='Month',
        size=2,
        index=True)

    zip_file = fields.Binary(
        string='Zip Files',
        readonly=True)

    export_type = fields.Selection(
        selection=[
            ('period', 'Por período'),
            ('all', 'Tudo')],
        string='Export',
        default='period',
        required=True)

    group_folder = fields.Boolean(
        string='Group documents')

    raiz = fields.Char(
        string='Folder structure path',
        default=False)

    document_nfe_ids = fields.One2many(
        comodel_name="l10n_br_fiscal.document",
        string=" NFe Documents",
        inverse_name="close_id",
        domain=[('document_type', '=', MODELO_FISCAL_NFE)])

    document_nfce_ids = fields.One2many(
        comodel_name="l10n_br_fiscal.document",
        string="NFCe Documents",
        inverse_name="close_id",
        domain=[('document_type', '=', MODELO_FISCAL_NFCE)])

    document_cfe_ids = fields.One2many(
        comodel_name="l10n_br_fiscal.document",
        string="CFe Documents",
        inverse_name="close_id",
        domain=[('document_type', '=', MODELO_FISCAL_CFE)])

    document_cfeecf_ids = fields.One2many(
        comodel_name="l10n_br_fiscal.document",
        string="CFe ECF Documents",
        inverse_name="close_id",
        domain=[('document_type', '=', MODELO_FISCAL_CUPOM_FISCAL_ECF)])

    document_nfse_ids = fields.One2many(
        comodel_name="l10n_br_fiscal.document",
        string="NFSe Documents",
        inverse_name="close_id",
        domain=[('document_type', '=', MODELO_FISCAL_NFSE)])

    document_rl_ids = fields.One2many(
        comodel_name="l10n_br_fiscal.document",
        string="RL Documents",
        inverse_name="close_id",
        domain=[('document_type', '=', MODELO_FISCAL_RL)])

    attachment_ids = fields.Many2many(
        comodel_name='ir.attachment',
        string='Other accountant files')

    company_id = fields.Many2one(
        comodel_name='res.company',
        string='Company')

    file_icms = fields.Binary(
        string='ICMS')

    file_icms_st = fields.Binary(
        string='ICMS ST')

    file_ipi = fields.Binary(
        string='IPI')

    file_iss = fields.Binary(
        string='ISS')

    file_pis = fields.Binary(
        string='PIS')

    file_cofins = fields.Binary(
        string='COFINS')

    file_csll = fields.Binary(
        string='CSLL')

    file_irpj = fields.Binary(
        string='IRPJ')

    file_simples = fields.Binary(
        string='Simples')

    file_honorarios = fields.Binary(
        string='Accountant Fee')

    block = fields.Boolean(
        string='Block period',
        help="Avoid new fiscal moves")

    notes = fields.Text(
        string="Accountant notes")

    _sql_constraints = [(
        'fiscal_closing_unique',
        'unique (company_id, month, year, export_type)',
        "The closing must be unique for the company in a period of time.")]

    def monta_caminho(self, document):
        document_path = '/'.join([
            # TODO: Colocar ambiente
            # PATH_AMBIENTE[documento.nfe_environment],
            misc.punctuation_rm(document.company_cnpj_cpf),
            PATH_MODELO[document.document_type_id.code],
            document.date.strftime("%m-%Y"),
            document.document_serie_id.code.zfill(3) +
            ('-' + misc.punctuation_rm(str(document.number)).zfill(9)
             if self.group_folder else '')
        ])
        if self.raiz:
            if not os.path.exists(self.raiz + '/' + document_path):
                try:
                    os.makedirs(self.raiz + '/' + document_path)
                except OSError:
                    raise RedirectWarning(
                        _('Error!'),
                        _('Check write permissions and folder path'))
        return document_path

    def _date_range(self):
        date_range = calendar.monthrange(int(self.year), int(self.month))
        date_min = '-'.join((self.year, self.month, str(date_range[0])))
        date_min = datetime.strptime(date_min, '%Y-%m-%d')

        date_max = '-'.join((self.year, self.month, str(date_range[1])))
        date_max = datetime.strptime(date_max, '%Y-%m-%d')

        return date_min, date_max

    def _save_temp_file(self, document_path, anexo, temp_dir):
        document_path = document_path + '/' + anexo.datas_fname
        filename = os.path.join(temp_dir.name, document_path)

        if not os.path.exists(os.path.dirname(filename)):
            try:
                os.makedirs(os.path.dirname(filename))
            except OSError:
                raise RedirectWarning(
                    _('Error!'),
                    _('Check write permissions and folder path'))

            with open(filename, 'wb') as file:
                file.write(base64.b64decode(anexo.datas))

    def _prepara_arquivos(self, temp_dir, periodic_export=False):
        domain = [('document_type', 'in', MODELO_FISCAL_EMISSAO_PRODUTO +
                   MODELO_FISCAL_EMISSAO_SERVICO),
                  ('state_edoc', 'in', SITUACAO_EDOC)]

        domain += [
            ('close_id', '=', False)
        ] if periodic_export else []

        domain += [
            ('company_id', '=', self.company_id.ids)
        ] if self.company_id else []

        if self.export_type == 'period':
            date_min, date_max = self._date_range()

            domain += [
                ('date', '>=', date_min),
                ('date', '<=', date_max),
            ]

        documents = self.env['l10n_br_fiscal.document'].search(domain)
        # documents += self.env['l10n_br_fiscal.document_correction'].search(domain)
        # documents += self.env['l10n_br_fiscal.document_cancel'].search(domain)

        for document in documents:
            anexos = [getattr(document, campo) for campo in XMLS_IMPORTANTES
                      if hasattr(document, campo)
                      and getattr(document, campo).id is not False]

            document.close_id = self.id

            if not anexos:
                continue
            document_path = self.monta_caminho(document)
            try:
                for anexo in anexos:
                    self._save_temp_file(document_path, anexo, temp_dir)

            except Exception:
                _logger.error(_('Replication failed: document attachments '
                                '[id =% s] is not present in the database.'
                                % document.id))
        return temp_dir.name

    def zipdir(self, path, ziph):
        for root, dirs, files in os.walk(path):
            for file in files:
                zip_path = re.sub(".*/odoo/.*?/.", "/", root)
                ziph.writestr(os.path.join(zip_path, file), file)

    def action_export(self):
        temp_dir = tempfile.TemporaryDirectory(dir=os.getcwd())

        files_dir = self._prepara_arquivos(temp_dir)
        order_file = io.BytesIO()
        order_zip = zipfile.ZipFile(
            order_file, mode="w", compression=zipfile.ZIP_DEFLATED
        )

        self.zipdir(files_dir, order_zip)
        temp_dir.cleanup()

        order_zip.close()
        self.write({
            'zip_file': base64.b64encode(order_file.getvalue()),
            'state': 'open'
        })

    def action_close(self):
        """ Sobrescrever este método para, notificar seguidores,
        processar documentos anexos, criar contas a pagar"""
        self.state = 'closed'
<commit_msg>[FIX] Fiscal closing date_range<commit_after># Copyright 2018 KMEE INFORMATICA LTDA
#   Gabriel Cardoso de Faria <gabriel.cardoso@kmee.com.br>
# Copyright (C) 2020 Renato Lima - Akretion <renato.lima@akretion.com.br>
# License AGPL-3 or later (http://www.gnu.org/licenses/agpl)

import os
import base64
import re
import tempfile
import zipfile
import io
import logging
from datetime import datetime
import calendar

from odoo import _, api, fields, models
from odoo.exceptions import RedirectWarning

from ..constants.fiscal import (
    MODELO_FISCAL_EMISSAO_PRODUTO,
    MODELO_FISCAL_EMISSAO_SERVICO,
    MODELO_FISCAL_NFE,
    MODELO_FISCAL_NFCE,
    MODELO_FISCAL_CFE,
    MODELO_FISCAL_CUPOM_FISCAL_ECF,
    MODELO_FISCAL_NFSE,
    MODELO_FISCAL_RL,
    SITUACAO_EDOC_EM_DIGITACAO,
    SITUACAO_EDOC_A_ENVIAR,
    SITUACAO_EDOC_ENVIADA,
    SITUACAO_EDOC_REJEITADA,
    SITUACAO_EDOC_AUTORIZADA,
    SITUACAO_EDOC_CANCELADA,
    SITUACAO_EDOC_DENEGADA,
    SITUACAO_EDOC_INUTILIZADA,
)

from odoo.addons.l10n_br_nfe.constants.nfe import (
    NFE_ENVIRONMENTS
)

_logger = logging.getLogger(__name__)

try:
    from erpbrasil.base import misc
except ImportError:
    _logger.error("Biblioteca erpbrasil.base não instalada")

PATH_MODELO = {
    MODELO_FISCAL_NFE: 'nfe',
    MODELO_FISCAL_NFCE: 'nfce',
    MODELO_FISCAL_CFE: 'cfe',
    MODELO_FISCAL_CUPOM_FISCAL_ECF: 'cfe_ecf',
    MODELO_FISCAL_NFSE: 'nfse',
    MODELO_FISCAL_RL: 'rl',
}

PATH_AMBIENTE = dict(NFE_ENVIRONMENTS)

SITUACAO_EDOC = [
    SITUACAO_EDOC_EM_DIGITACAO,
    SITUACAO_EDOC_A_ENVIAR,
    SITUACAO_EDOC_ENVIADA,
    SITUACAO_EDOC_REJEITADA,
    SITUACAO_EDOC_AUTORIZADA,
    SITUACAO_EDOC_CANCELADA,
    SITUACAO_EDOC_DENEGADA,
    SITUACAO_EDOC_INUTILIZADA,
]

XMLS_IMPORTANTES = [
    'file_xml_autorizacao_id',
    'file_xml_autorizacao_cancelamento_id',
    'file_xml_autorizacao_inutilizacao_id',
    'file_pdf_id',
]

class FiscalClosing(models.Model):
    _name = 'l10n_br_fiscal.closing'
    _inherit = ['mail.thread', 'mail.activity.mixin']
    _description = 'Fiscal Closing Period'

    @api.depends('month', 'year', 'export_type')
    def _compute_name(self):
        for record in self:
            if record.export_type == 'period':
                record.name = "{}/{}".format(record.month, record.year)
                record.file_name = "{}-{}".format(record.month, record.year) + '.zip'
            else:
                now = fields.Datetime.now().strftime('%d/%m/%Y')
                record.name = now
                record.file_name = now + '.zip'

    state = fields.Selection(
        selection=[
            ('draft', 'Draft'),
            ('open', 'Open'),
            ('closed', 'Closed')],
        string='state',
        default='draft',
        readonly=True)

    name = fields.Char(
        string='Name',
        compute='_compute_name',
        store=True,
        size=7,
        index=True)

    file_name = fields.Char(
        string='File Name',
        compute='_compute_name',
        store=True,
        size=11,
        index=True)

    year = fields.Char(
        string='Year',
        size=4,
        index=True)

    month = fields.Char(
        string='Month',
        size=2,
        index=True)

    zip_file = fields.Binary(
        string='Zip Files',
        readonly=True)

    export_type = fields.Selection(
        selection=[
            ('period', 'Por período'),
            ('all', 'Tudo')],
        string='Export',
        default='period',
        required=True)

    group_folder = fields.Boolean(
        string='Group documents')

    raiz = fields.Char(
        string='Folder structure path',
        default=False)

    document_nfe_ids = fields.One2many(
        comodel_name="l10n_br_fiscal.document",
        string=" NFe Documents",
        inverse_name="close_id",
        domain=[('document_type', '=', MODELO_FISCAL_NFE)])

    document_nfce_ids = fields.One2many(
        comodel_name="l10n_br_fiscal.document",
        string="NFCe Documents",
        inverse_name="close_id",
        domain=[('document_type', '=', MODELO_FISCAL_NFCE)])

    document_cfe_ids = fields.One2many(
        comodel_name="l10n_br_fiscal.document",
        string="CFe Documents",
        inverse_name="close_id",
        domain=[('document_type', '=', MODELO_FISCAL_CFE)])

    document_cfeecf_ids = fields.One2many(
        comodel_name="l10n_br_fiscal.document",
        string="CFe ECF Documents",
        inverse_name="close_id",
        domain=[('document_type', '=', MODELO_FISCAL_CUPOM_FISCAL_ECF)])

    document_nfse_ids = fields.One2many(
        comodel_name="l10n_br_fiscal.document",
        string="NFSe Documents",
        inverse_name="close_id",
        domain=[('document_type', '=', MODELO_FISCAL_NFSE)])

    document_rl_ids = fields.One2many(
        comodel_name="l10n_br_fiscal.document",
        string="RL Documents",
        inverse_name="close_id",
        domain=[('document_type', '=', MODELO_FISCAL_RL)])

    attachment_ids = fields.Many2many(
        comodel_name='ir.attachment',
        string='Other accountant files')

    company_id = fields.Many2one(
        comodel_name='res.company',
        string='Company')

    file_icms = fields.Binary(
        string='ICMS')

    file_icms_st = fields.Binary(
        string='ICMS ST')

    file_ipi = fields.Binary(
        string='IPI')

    file_iss = fields.Binary(
        string='ISS')

    file_pis = fields.Binary(
        string='PIS')

    file_cofins = fields.Binary(
        string='COFINS')

    file_csll = fields.Binary(
        string='CSLL')

    file_irpj = fields.Binary(
        string='IRPJ')

    file_simples = fields.Binary(
        string='Simples')

    file_honorarios = fields.Binary(
        string='Accountant Fee')

    block = fields.Boolean(
        string='Block period',
        help="Avoid new fiscal moves")

    notes = fields.Text(
        string="Accountant notes")

    _sql_constraints = [(
        'fiscal_closing_unique',
        'unique (company_id, month, year, export_type)',
        "The closing must be unique for the company in a period of time.")]

    def monta_caminho(self, document):
        document_path = '/'.join([
            # TODO: Colocar ambiente
            # PATH_AMBIENTE[documento.nfe_environment],
            misc.punctuation_rm(document.company_cnpj_cpf),
            PATH_MODELO[document.document_type_id.code],
            document.date.strftime("%m-%Y"),
            document.document_serie_id.code.zfill(3) +
            ('-' + misc.punctuation_rm(str(document.number)).zfill(9)
             if self.group_folder else '')
        ])
        if self.raiz:
            if not os.path.exists(self.raiz + '/' + document_path):
                try:
                    os.makedirs(self.raiz + '/' + document_path)
                except OSError:
                    raise RedirectWarning(
                        _('Error!'),
                        _('Check write permissions and folder path'))
        return document_path

    def _date_range(self):
        date_range = calendar.monthrange(int(self.year), int(self.month))
        date_min = '-'.join((self.year, self.month, str(date_range[0])))
        date_min = datetime.strptime(date_min, '%Y-%m-%d').replace(day=1)

        date_max = '-'.join((self.year, self.month, str(date_range[1])))
        date_max = datetime.strptime(date_max, '%Y-%m-%d')
        date_max = datetime.combine(date_max, date_max.time().max)

        return date_min, date_max

    def _save_temp_file(self, document_path, anexo, temp_dir):
        document_path = document_path + '/' + anexo.datas_fname
        filename = os.path.join(temp_dir.name, document_path)

        if not os.path.exists(os.path.dirname(filename)):
            try:
                os.makedirs(os.path.dirname(filename))
            except OSError:
                raise RedirectWarning(
                    _('Error!'),
                    _('Check write permissions and folder path'))

            with open(filename, 'wb') as file:
                file.write(base64.b64decode(anexo.datas))

    def _prepara_arquivos(self, temp_dir, periodic_export=False):
        domain = [('document_type', 'in', MODELO_FISCAL_EMISSAO_PRODUTO +
                   MODELO_FISCAL_EMISSAO_SERVICO),
                  ('state_edoc', 'in', SITUACAO_EDOC)]

        domain += [
            ('close_id', '=', False)
        ] if periodic_export else []

        domain += [
            ('company_id', '=', self.company_id.ids)
        ] if self.company_id else []

        if self.export_type == 'period':
            date_min, date_max = self._date_range()

            domain += [
                ('date', '>=', date_min),
                ('date', '<=', date_max),
            ]

        documents = self.env['l10n_br_fiscal.document'].search(domain)
        # documents += self.env['l10n_br_fiscal.document_correction'].search(domain)
        # documents += self.env['l10n_br_fiscal.document_cancel'].search(domain)

        for document in documents:
            anexos = [getattr(document, campo) for campo in XMLS_IMPORTANTES
                      if hasattr(document, campo)
                      and getattr(document, campo).id is not False]

            document.close_id = self.id

            if not anexos:
                continue
            document_path = self.monta_caminho(document)
            try:
                for anexo in anexos:
                    self._save_temp_file(document_path, anexo, temp_dir)

            except Exception:
                _logger.error(_('Replication failed: document attachments '
                                '[id =% s] is not present in the database.'
                                % document.id))
        return temp_dir.name

    def zipdir(self, path, ziph):
        for root, dirs, files in os.walk(path):
            for file in files:
                zip_path = re.sub(".*/odoo/.*?/.", "/", root)
                ziph.writestr(os.path.join(zip_path, file), file)

    def action_export(self):
        temp_dir = tempfile.TemporaryDirectory(dir=os.getcwd())

        files_dir = self._prepara_arquivos(temp_dir)
        order_file = io.BytesIO()
        order_zip = zipfile.ZipFile(
            order_file, mode="w", compression=zipfile.ZIP_DEFLATED
        )

        self.zipdir(files_dir, order_zip)
        temp_dir.cleanup()

        order_zip.close()
        self.write({
            'zip_file': base64.b64encode(order_file.getvalue()),
            'state': 'open'
        })

    def action_close(self):
        """ Sobrescrever este método para, notificar seguidores,
        processar documentos anexos, criar contas a pagar"""
        self.state = 'closed'
<|endoftext|>


<commit_before># coding: utf-8
# Copyright 2013 The Font Bakery Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
# See AUTHORS.txt for the list of Authors and LICENSE.txt for the License.

from flask import Blueprint, request, flash, g, session, redirect, url_for

from .models import User
from ..extensions import db, github
from flask.ext.babel import gettext as _

gitauth = Blueprint('gitauth', __name__, url_prefix='/auth')

@gitauth.after_request
def after_request(response):
    return response

@gitauth.route('/me')
def me():
    # Visit this URL and log the user in as Mikhail's 'xen' username
    # This enables developers to work offline when they have no access to github authentication
    # FOR MULTI USER INSTALLATIONS THIS IS AN EXPLOIT :)
    user = User.get_or_init('xen')
    if user.id == 1:
        # XXX: this is temporary
        session['user_id'] = user.id
        g.user = user
        flash('You were signed in')

    return redirect(url_for('frontend.splash'))

@gitauth.route('/login')
def login():
    # import ipdb; ipdb.set_trace()
    if session.get('user_id', None) is None:
        redirect_uri = url_for('.authorized',
            next=request.args.get('next') or request.referrer or None,
            _external=True)
        params = {'redirect_uri': redirect_uri, 'scope': 'user:email,public_repo'}
        return redirect(github.get_authorize_url(**params))
    else:
        flash('Already logged in')
        return redirect(url_for('frontend.splash'))

@gitauth.route('/callback')
def authorized(next = None):
    next_url = request.args.get('next') or url_for('frontend.splash')

    if not 'code' in request.args:
        flash('You did not authorize the request')
        return redirect(next_url)

    redirect_uri = url_for('.authorized', _external=True)
    data = dict(code=request.args['code'], redirect_uri=redirect_uri)
    auth = github.get_auth_session(data=data)

    token = auth.access_token
    me = auth.get('user').json()

    user = User.get_or_init(me['login'])

    # if user.id is None:
    #     new record isn't saved yet
    #     flash(_('Welcome to Bakery.'))

    # update user data
    user.name = me['name']
    user.github_access_token = token
    user.avatar = me['gravatar_id']
    user.email = me['email']

    # save to database
    db.session.add(user)
    db.session.commit()

    session['user_id'] = user.id
    g.user = user
    flash('You were signed in')

    return redirect(next_url)

@gitauth.route('/logout')
def logout():
    session.pop('user_id', None)
    return redirect(url_for('frontend.splash'))

<commit_msg>Fixing issue #100<commit_after># coding: utf-8
# Copyright 2013 The Font Bakery Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
# See AUTHORS.txt for the list of Authors and LICENSE.txt for the License.

from flask import Blueprint, request, flash, g, session, redirect, url_for

from .models import User
from ..extensions import db, github
from flask.ext.babel import gettext as _

gitauth = Blueprint('gitauth', __name__, url_prefix='/auth')

@gitauth.after_request
def after_request(response):
    return response

@gitauth.route('/me')
def me():
    # Visit this URL and log the user in as Mikhail's 'xen' username
    # This enables developers to work offline when they have no access to github authentication
    # FOR MULTI USER INSTALLATIONS THIS IS AN EXPLOIT :)
    user = User.get_or_init('xen')
    if user.id == 1:
        # XXX: this is temporary
        session['user_id'] = user.id
        g.user = user
        flash('Welcome!')

    return redirect(url_for('frontend.splash'))

@gitauth.route('/login')
def login():
    # import ipdb; ipdb.set_trace()
    if session.get('user_id', None) is None:
        redirect_uri = url_for('.authorized',
            next=request.args.get('next') or request.referrer or None,
            _external=True)
        params = {'redirect_uri': redirect_uri, 'scope': 'user:email,public_repo'}
        return redirect(github.get_authorize_url(**params))
    else:
        flash('Already logged in')
        return redirect(url_for('frontend.splash'))

@gitauth.route('/callback')
def authorized(next = None):
    next_url = request.args.get('next') or url_for('frontend.splash')

    if not 'code' in request.args:
        flash('You did not authorize the request')
        return redirect(next_url)

    redirect_uri = url_for('.authorized', _external=True)
    data = dict(code=request.args['code'], redirect_uri=redirect_uri)
    auth = github.get_auth_session(data=data)

    token = auth.access_token
    me = auth.get('user').json()

    user = User.get_or_init(me['login'])

    # if user.id is None:
    #     new record isn't saved yet
    #     flash(_('Welcome to Bakery.'))

    # update user data
    user.name = me['name']
    user.github_access_token = token
    user.avatar = me['gravatar_id']
    user.email = me['email']

    # save to database
    db.session.add(user)
    db.session.commit()

    session['user_id'] = user.id
    g.user = user
    flash('Welcome!')

    return redirect(next_url)

@gitauth.route('/logout')
def logout():
    session.pop('user_id', None)
    return redirect(url_for('frontend.splash'))

<|endoftext|>


<commit_before>#!/usr/bin/python

"""
created_by:         John Fang
created_date:       3/2/2015
last_modified_by:   Micah Halter
last_modified date: 3/4/2015
"""

# imports
import constants
import json
import mysql.connector
from user import User
from answer import Answer
from mysql_connect_config import getConfig
# classes
class Comment(object):
    'Comment object to hold attributes and functions for an assessment'

    def __init__(self, id, created, created_by, answer, content, active):
        """
        self       - the comment in question
        id         - the id number of the comment 'self' in the database
        created    - the date when the comment 'self' was created
        created_by - the user that created the comment 'self'
        answer     - the answer of the comment 'self'
        content    - the content of the topic 'self'

        this function acts as the constructor to define a new topic object
        """
        self.id         = id
        self.created    = created
        self.created_by = created_by
        self.answer     = answer
        self.content    = content
        self.active     = active
    def add(self):

        if self.id is not None:
            return

        cnx = mysql.connector.connect(**getConfig())
        cursor = cnx.cursor()

        insert = ("INSERT INTO comment (created_by, answer_id, content, active) VALUES ('%s', %s, '%s', '%s', %s); SELECT LAST_LAST_INSERT_ID();" %(self.created, self.created_by.id, self.answer.id, self.content, self.active))
        cursor.execute(insert)

        for (id) in cursor:
            self.id=id

        cnx.commit()
        cursor.close()
        cnx.close()

    def update(self):

        cnx = mysql.connector.connect(**getConfig())
        cursor = cnx.cursor()

        if self.id is not None:
            update = ("UPDATE comment SET answer_id = %s,  content = '%s', active = %s WHERE id = %s;" % (self.asnwer.id, self.content, self.active, self.id))
            cursor.execute(update)

        cnx.commit()
        cursor.close()
        cnx.close()

    def activate(self, bool):
        cnx = mysql.connector.connect(**getConfig())
        cursor = cnx.cursor()

        if self.id is not None:
            self.active = int(bool)
            update = ("UPDATE comment SET active=%s WHERE id=%S;" % (int(bool), self.id))
            cursor.execute(update)

        cnx.commit()
        cursor.close()
        cnx.close()

    @classmethod
    def get(self, search="all", testActive=1):
        cnx = mysql.connector.connect(**getcConfig())
        cursor = cnx.cursor()

        returnList = []
        query = ""
        if search == "all":
            query = "SELECT * FROM comment"
        elif type(search) is int:
            query = ("SELECT * FROM comment WHERE id=%s" % (search))
        elif type(search) is User:
            query = ("SELECT * FROM comment WHERE created_by=%s" % (search.id))
        elif type(search) is Answer:
            query = ("SELECT * FROM comment WHERE answer_id=%s" % (search.id))

        query += (" WHERE active=%s" if search == "all" else " AND active =%s;") % (testActive)
        cursor.execute(query)
        for (id, created, created_by, answer_id, content, active) in cursor:
            user = User.get(created_by)[0]
            answer = Answer.get(answer_id)[0]
            returnList.append(Comment(id, created, user, answer, content, active))

        cnx.commit()
        cursor.close()
        cnx.close

        return returnList

    def noID(self, created, created_by, answer, content, active):
        """
        the parameters correspond with the parameters in the constructor above

        returns a new comment with the id parameter set as None and the other
        parameters as they are passed in

        this function acts as a second constructor where you have created a
        comment that has not yet been assigned an id from the database
        """
        return self(None, created, created_by, answer, content, active)

    def __eq__(self, other):
        """
        self  - the comment in question
        other - other comment that you are comparing 'self' to

        returns a boolean telling if the attributes of 'self'
        and 'other' are the same

        this function overrides the way python compares two comments
        and decides if they are equal instead of checking if they
        are the exact same object
        """
        return (
        self.id         == other.id         and
        self.created    == other.created    and
        self.created_by == other.created_by and
        self.answer     == other.answer     and
        self.content    == other.content    and
        self.active     == other.active)

    def __str__(self):
        """
        self - the comment in question

        returns a string that fully describes the comment 'self'

        this function allows you to convert the comment object into
        a human-readable string for viewing the information in it
        """
        string = ""
        string += "id: "         +      str(self.id)         + "\n"
        string += "created: "    +      str(self.created)    + "\n"
        string += "created by: " +      str(self.created_by) + "\n"
        string += "active: "     + str(bool(self.active))    + "\n"
        string += "answer: "     +      str(self.answer)     + "\n"
        string += "content: "    +      self.content         + "\n"
        return string
    def toJson(self):
        data = {
        "id"        : self.id,
        "created"   : self.created,
        "created_by": self.created_by,
        "active"    : self.active,
        "answer"    : self.answer,
        "content"   : self.content
        }
        return json.dumps(data)
<commit_msg>Fixes in comment.py<commit_after>#!/usr/bin/python

"""
created_by:         John Fang
created_date:       3/2/2015
last_modified_by:   LZ
last_modified date: 3/4/2015
"""

# imports
import constants
import json
import mysql.connector
from user import User
from answer import Answer
from mysql_connect_config import getConfig
# classes
class Comment(object):
    'Comment object to hold attributes and functions for an assessment'

    def __init__(self, id, created, created_by, answer, content, active):
        """
        self       - the comment in question
        id         - the id number of the comment 'self' in the database
        created    - the date when the comment 'self' was created
        created_by - the user that created the comment 'self'
        answer     - the answer of the comment 'self'
        content    - the content of the topic 'self'

        this function acts as the constructor to define a new topic object
        """
        self.id         = id
        self.created    = created
        self.created_by = created_by
        self.answer     = answer
        self.content    = content
        self.active     = active

    def add(self):

        if self.id is not None:
            return

        cnx = mysql.connector.connect(**getConfig())
        cursor = cnx.cursor()

        insert = ("INSERT INTO comment (created_by, answer_id, content, active) VALUES (%s, '%s', '%s', %s); SELECT LAST_LAST_INSERT_ID();" %(self.created_by.id, self.answer.id, self.content, self.active))
        cursor.execute(insert)

        for (id) in cursor:
            self.id=id

        cnx.commit()
        cursor.close()
        cnx.close()

    def update(self):

        cnx = mysql.connector.connect(**getConfig())
        cursor = cnx.cursor()

        if self.id is not None:
            update = ("UPDATE comment SET answer_id = %s,  content = '%s', active = %s WHERE id = %s;" % (self.answer.id, self.content, self.active, self.id))
            cursor.execute(update)

        cnx.commit()
        cursor.close()
        cnx.close()

    def activate(self, bool):
        cnx = mysql.connector.connect(**getConfig())
        cursor = cnx.cursor()

        if self.id is not None:
            self.active = int(bool)
            update = ("UPDATE comment SET active=%s WHERE id=%s;" % (int(bool), self.id))
            cursor.execute(update)

        cnx.commit()
        cursor.close()
        cnx.close()

    @classmethod
    def get(self, search="all", testActive=1):
        cnx = mysql.connector.connect(**getcConfig())
        cursor = cnx.cursor()

        returnList = []
        query = ""
        if search == "all":
            query = "SELECT * FROM comment"
        elif type(search) is int:
            query = ("SELECT * FROM comment WHERE id=%s" % (search))
        elif type(search) is User:
            query = ("SELECT * FROM comment WHERE created_by=%s" % (search.id))
        elif type(search) is Answer:
            query = ("SELECT * FROM comment WHERE answer_id=%s" % (search.id))

        query += (" WHERE active=%s;" if search == "all" else " AND active =%s;") % (testActive)
        cursor.execute(query)
        for (id, created, created_by, answer_id, content, active) in cursor:
            user = User.get(created_by)[0]
            answer = Answer.get(answer_id)[0]
            returnList.append(Comment(id, created, user, answer, content, active))

        cnx.commit()
        cursor.close()
        cnx.close

        return returnList

    def noID(self, created, created_by, answer, content, active):
        """
        the parameters correspond with the parameters in the constructor above

        returns a new comment with the id parameter set as None and the other
        parameters as they are passed in

        this function acts as a second constructor where you have created a
        comment that has not yet been assigned an id from the database
        """
        return self(None, created, created_by, answer, content, active)

    def __eq__(self, other):
        """
        self  - the comment in question
        other - other comment that you are comparing 'self' to

        returns a boolean telling if the attributes of 'self'
        and 'other' are the same

        this function overrides the way python compares two comments
        and decides if they are equal instead of checking if they
        are the exact same object
        """
        return (
        self.id         == other.id         and
        self.created    == other.created    and
        self.created_by == other.created_by and
        self.answer     == other.answer     and
        self.content    == other.content    and
        self.active     == other.active)

    def __str__(self):
        """
        self - the comment in question

        returns a string that fully describes the comment 'self'

        this function allows you to convert the comment object into
        a human-readable string for viewing the information in it
        """
        string = ""
        string += "id: "         +      str(self.id)         + "\n"
        string += "created: "    +      str(self.created)    + "\n"
        string += "created by: " +      str(self.created_by) + "\n"
        string += "active: "     + str(bool(self.active))    + "\n"
        string += "answer: "     +      str(self.answer)     + "\n"
        string += "content: "    +          self.content     + "\n"

        return string

    def toJson(self):
        data = {
        "id"        : self.id,
        "created"   : self.created,
        "created_by": self.created_by,
        "active"    : self.active,
        "answer"    : self.answer,
        "content"   : self.content
        }
        return json.dumps(data)
<|endoftext|>


<commit_before>
__version__ = "0.6.0"
__author__ = 'Michael Davis'
__license__ = 'MIT'
__copyright__ = 'Copyright 2015 Michael Davis'

from .exceptions import JOSEError
from .exceptions import JWSError
from .exceptions import ExpiredSignatureError
from .exceptions import JWTError
<commit_msg>Bump copyright year<commit_after>
__version__ = "0.6.0"
__author__ = 'Michael Davis'
__license__ = 'MIT'
__copyright__ = 'Copyright 2016 Michael Davis'

from .exceptions import JOSEError
from .exceptions import JWSError
from .exceptions import ExpiredSignatureError
from .exceptions import JWTError
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-

import urllib2

def api():
    if request.args[0] == 'zonky':
        url = 'https://api.zonky.cz/loans/marketplace?rating__eq=%s' % request.args[1]
        request = urllib2.Request(url)
        request.add_header('X-Page', '0')
        request.add_header('X-Size', '30')

    if request:
        response.headers = {'Access-Control-Allow-Origin': '*'}
        return urllib2.urlopen(request).read()
    else:
        raise HTTP(403)
<commit_msg>cors proxy<commit_after># -*- coding: utf-8 -*-

import urllib2

def api():
    if request.args[0] == 'zonky':
        url = 'https://api.zonky.cz/loans/marketplace?rating__eq=%s' % request.args[1]
        urllib_request = urllib2.Request(url)
        urllib_request.add_header('X-Page', '0')
        urllib_request.add_header('X-Size', '30')

    if urllib_request:
        response.headers = {'Access-Control-Allow-Origin': '*'}
        return urllib2.urlopen(urllib_request).read()
    else:
        raise HTTP(403)
<|endoftext|>


<commit_before>def once(command, log):

    net_id = command('neutron net-create')
    command('neutron subnet-create {net_id} 10.0.100.0/24'.format(net_id=net_id))
    command('neutron port-create {net_id}'.format(net_id=net_id))

def start(lab, log, args):
    import time
    import re

    how_many = args['how_many']
    unique_pattern_in_name = args.get('unique_pattern_in_name', 'sqe-test')

    server = lab.controllers()[0]

    def command(cmd):
        name_part = '{0}-{1}'.format(unique_pattern_in_name, i)
        if 'subnet-create' in cmd:  # subnet-create needs to be before net-create since net-create is a substring of subnet-create
            name_part = '--name ' + name_part + '-subnet'
        elif 'net-create' in cmd:
            name_part += '-net'
        elif 'port-create' in cmd:
            name_part = '--name ' + name_part + '-port'

        res = server.run(command='{cmd} {name_part} {lab_creds}'.format(cmd=cmd, name_part=name_part, lab_creds=lab.cloud))
        if res:
            return re.search('id\s+\|\s+([-\w]+)', ''.join(res.stdout)).group(1)

    start_time = time.time()
    for i in xrange(0, how_many):
        once(command=command, log=log)
        log.info('n_ports={0} status=created'.format(i+1))
    log.info('n_ports={0} status=created in time={1} secs'.format(how_many, time.time()-start_time))
<commit_msg>set quota with some margin (20) calculate v4 cidr even if number of networks is more the 256<commit_after>def once(command, number):

    net_id = command('neutron net-create')
    command('neutron subnet-create {net_id} {cidr}'.format(net_id=net_id, cidr=cidr4(number)))
    command('neutron port-create {net_id}'.format(net_id=net_id))

def cidr4(i):
    return '10.{0}.{1}.0/24'.format(i / 256, i % 256)

def start(lab, log, args):
    import time
    import re

    how_many = args['how_many']
    unique_pattern_in_name = args.get('unique_pattern_in_name', 'sqe-test')

    server = lab.controllers()[0]

    def command(cmd):
        name_part = '{0}-{1}'.format(unique_pattern_in_name, i)
        if 'subnet-create' in cmd:  # subnet-create needs to be before net-create since net-create is a substring of subnet-create
            name_part = '--name ' + name_part + '-subnet'
        elif 'net-create' in cmd:
            name_part += '-net'
        elif 'port-create' in cmd:
            name_part = '--name ' + name_part + '-port'

        res = server.run(command='{cmd} {name_part} {lab_creds}'.format(cmd=cmd, name_part=name_part, lab_creds=lab.cloud))
        if res:
            return re.search('id\s+\|\s+([-\w]+)', ''.join(res.stdout)).group(1)

    start_time = time.time()

    quota = 20 + how_many
    server.run(command='neutron quota-update --network {n} --subnet {n} --port {n} {lab_creds}'.format(n=quota, lab_creds=lab.cloud))
    for i in xrange(0, how_many):
        once(command=command, number=i)
        log.info('n_ports={0} status=created'.format(i+1))
    log.info('n_ports={0} status=created in time={1} secs'.format(how_many, time.time()-start_time))
<|endoftext|>


<commit_before>from openpathsampling.todict import ops_object
import openpathsampling as paths

__author__ = 'jan-hendrikprinz'

@ops_object
class ToyTopology(paths.Topology):
    '''
    Attributes
    ----------
    masses : numpy.ndarray (n_atoms, dtype=float)
        The masses associated with each atom
    '''
    def __init__(self, n_spatial, masses, pes, n_atoms = 1):
        super(ToyTopology, self).__init__(n_atoms = n_atoms, n_spatial=n_spatial)
        self.masses = masses
        self.pes = pes

    def subset(self, list_of_atoms):
        return self<commit_msg>rm toy_topology.py<commit_after><|endoftext|>


<commit_before>import numpy
import scipy.sparse

def sparse_matrix(shape, integer=False):
    """
    Create a new sparse matrix (either integer or floating point) in a form
    suitable for populating with data
    """
    dtype = numpy.int_ if integer else numpy.float_
    return scipy.sparse.lil_matrix(shape, dtype=dtype)

def finalise_matrix(matrix):
    """
    Return a copy of a sparse matrix in a form suitable for storage
    """
    matrix = matrix.tocsc()
    matrix.sort_indices()
    if is_integer(matrix):
        matrix = convert_to_smallest_int_type(matrix)
    matrix = convert_to_dense_if_smaller(matrix)
    return matrix

def zeros_like(matrix, order=None):
    """
    Return a zero-valued matrix of the same shape as `matrix` and with
    equivalent integer or floating point type

    Note this differs from `numpy.zeros_like` in that it always uses the
    largest integer type (int_) even if the source matrix uses a smaller type
    (e.g.  uint8). This is so that we have sufficient headroom to sum many
    matrices together.
    """
    dtype = numpy.int_ if is_integer(matrix) else numpy.float_
    return numpy.zeros(matrix.shape, dtype=dtype, order=order)

def is_integer(matrix):
    """
    Return whether or not the matrix has integer type
    """
    return numpy.issubdtype(matrix.dtype, numpy.integer)

def convert_to_dense_if_smaller(matrix):
    """
    Convert a sparse matrix to a dense one if that would result in less overall
    memory usage

    This isn't primarily about storage space as once compressed there isn't
    much difference between the two forms, but the smaller representations are
    faster to work with when summing.
    """
    if get_dense_memory_usage(matrix) < get_sparse_memory_usage(matrix):
        return matrix.toarray()
    else:
        return matrix

def get_sparse_memory_usage(matrix):
    """
    Return the number of bytes need to store a sparse matrix
    """
    return matrix.data.nbytes + matrix.indices.nbytes + matrix.indptr.nbytes

def get_dense_memory_usage(matrix):
    """
    Return the number of bytes need to store the equivalent dense
    representation of a sparse matrix
    """
    return matrix.dtype.itemsize * matrix.shape[0] * matrix.shape[1]

def convert_to_smallest_int_type(matrix):
    """
    Convert a matrix to use the smallest integer type capable of representing
    all the values currently stored in it
    """
    target_type = smallest_int_type_for_range(matrix.min(), matrix.max())
    if target_type != matrix.dtype:
        matrix = matrix.astype(target_type, copy=False)
    return matrix

def smallest_int_type_for_range(minimum, maximum):
    """
    Return smallest numpy integer type capable of representing all values in
    the supplied range
    """
    signed = minimum < 0
    abs_max = max(maximum, abs(minimum))
    if signed:
        if abs_max < 1 << 7:
            return numpy.int8
        elif abs_max < 1 << 15:
            return numpy.int16
        elif abs_max < 1 << 31:
            return numpy.int32
    else:
        if abs_max < 1 << 8:
            return numpy.uint8
        elif abs_max < 1 << 16:
            return numpy.uint16
        elif abs_max < 1 << 32:
            return numpy.uint32
    # Return default integer type (other than in the exceptional case that the
    # value is too big to store in a signed 64-bit int)
    if not signed and abs_max > 1 << 63:
        return numpy.uint64
    else:
        return numpy.int64
<commit_msg>Add get_submatrix function<commit_after>import numpy
import scipy.sparse
from scipy.sparse import csc_matrix
from scipy.sparse.compressed import _process_slice, get_csr_submatrix

def sparse_matrix(shape, integer=False):
    """
    Create a new sparse matrix (either integer or floating point) in a form
    suitable for populating with data
    """
    dtype = numpy.int_ if integer else numpy.float_
    return scipy.sparse.lil_matrix(shape, dtype=dtype)

def finalise_matrix(matrix):
    """
    Return a copy of a sparse matrix in a form suitable for storage
    """
    matrix = matrix.tocsc()
    matrix.sort_indices()
    if is_integer(matrix):
        matrix = convert_to_smallest_int_type(matrix)
    matrix = convert_to_dense_if_smaller(matrix)
    return matrix

def zeros_like(matrix, order=None):
    """
    Return a zero-valued matrix of the same shape as `matrix` and with
    equivalent integer or floating point type

    Note this differs from `numpy.zeros_like` in that it always uses the
    largest integer type (int_) even if the source matrix uses a smaller type
    (e.g.  uint8). This is so that we have sufficient headroom to sum many
    matrices together.
    """
    dtype = numpy.int_ if is_integer(matrix) else numpy.float_
    return numpy.zeros(matrix.shape, dtype=dtype, order=order)

def is_integer(matrix):
    """
    Return whether or not the matrix has integer type
    """
    return numpy.issubdtype(matrix.dtype, numpy.integer)

def convert_to_dense_if_smaller(matrix):
    """
    Convert a sparse matrix to a dense one if that would result in less overall
    memory usage

    This isn't primarily about storage space as once compressed there isn't
    much difference between the two forms, but the smaller representations are
    faster to work with when summing.
    """
    if get_dense_memory_usage(matrix) < get_sparse_memory_usage(matrix):
        return matrix.toarray()
    else:
        return matrix

def get_sparse_memory_usage(matrix):
    """
    Return the number of bytes need to store a sparse matrix
    """
    return matrix.data.nbytes + matrix.indices.nbytes + matrix.indptr.nbytes

def get_dense_memory_usage(matrix):
    """
    Return the number of bytes need to store the equivalent dense
    representation of a sparse matrix
    """
    return matrix.dtype.itemsize * matrix.shape[0] * matrix.shape[1]

def convert_to_smallest_int_type(matrix):
    """
    Convert a matrix to use the smallest integer type capable of representing
    all the values currently stored in it
    """
    target_type = smallest_int_type_for_range(matrix.min(), matrix.max())
    if target_type != matrix.dtype:
        matrix = matrix.astype(target_type, copy=False)
    return matrix

def smallest_int_type_for_range(minimum, maximum):
    """
    Return smallest numpy integer type capable of representing all values in
    the supplied range
    """
    signed = minimum < 0
    abs_max = max(maximum, abs(minimum))
    if signed:
        if abs_max < 1 << 7:
            return numpy.int8
        elif abs_max < 1 << 15:
            return numpy.int16
        elif abs_max < 1 << 31:
            return numpy.int32
    else:
        if abs_max < 1 << 8:
            return numpy.uint8
        elif abs_max < 1 << 16:
            return numpy.uint16
        elif abs_max < 1 << 32:
            return numpy.uint32
    # Return default integer type (other than in the exceptional case that the
    # value is too big to store in a signed 64-bit int)
    if not signed and abs_max > 1 << 63:
        return numpy.uint64
    else:
        return numpy.int64

def get_submatrix(matrix, rows=slice(None, None), cols=slice(None, None)):
    """
    Return a submatrix sliced by the supplied rows and columns, with a special
    fast path for Compressed Sparse Column matrices

    Workaroud for https://github.com/scipy/scipy/issues/11496
    """
    # Default slicing behaviour for types which don't need the fast path
    if not isinstance(matrix, csc_matrix):
        return matrix[rows, cols]
    # This is based on the code found in the following file, but skips the
    # redundant initialisation checks that would get run on the new matrix
    # instance:
    # scipy/sparse/compressed.py:_cs_matrix._get_submatrix
    N, M = matrix.shape
    i0, i1 = _process_slice(cols, M)
    j0, j1 = _process_slice(rows, N)
    if i0 == 0 and j0 == 0 and i1 == M and j1 == N:
        return matrix
    indptr, indices, data = get_csr_submatrix(
        M, N, matrix.indptr, matrix.indices, matrix.data, i0, i1, j0, j1
    )
    shape = (j1 - j0, i1 - i0)
    # Construct the new matrix instance by directly assigning its members,
    # rather than using `__init__` which runs additional checks that we don't
    # need
    new_matrix = csc_matrix.__new__(csc_matrix)
    new_matrix.data = data
    new_matrix.indices = indices
    new_matrix.indptr = indptr
    new_matrix._shape = shape
    return new_matrix
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
# vim: tabstop=4 shiftwidth=4 softtabstop=4
#
# Copyright (C) 2020, GEM Foundation
#
# OpenQuake is free software: you can redistribute it and/or modify it
# under the terms of the GNU Affero General Public License as published
# by the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# OpenQuake is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with OpenQuake.  If not, see <http://www.gnu.org/licenses/>.

import numpy
from scipy.stats import norm
from openquake.baselib.general import group_array
from openquake.hazardlib.imt import from_string
from openquake.hazardlib.probability_map import ProbabilityCurve
from openquake.commonlib.oqvalidation import check_same_levels

# this is necessary since norm.cdf for scale=0 returns NaNs
def norm_cdf(x, a, s):
    """
    Gaussian cumulative distribution function; if s=0, returns an
    Heaviside function instead. NB: for x=a, 0.5 is returned for all s.

    >>> norm_cdf(1.2, 1, .1)
    0.9772498680518208
    >>> norm_cdf(1.2, 1, 0)
    1.0
    >>> norm_cdf(.8, 1, .1)
    0.022750131948179216
    >>> norm_cdf(.8, 1, 0)
    0.0
    >>> norm_cdf(1, 1, .1)
    0.5
    >>> norm_cdf(1, 1, 0)
    0.5
    """
    if s == 0:
        return numpy.heaviside(x - a, .5)
    else:
        return norm.cdf(x, loc=a, scale=s)

def digitize(name, values, edges):
    """
    :param name: 'period' or 'level'
    :param values: periods or levels
    :param edges: available periods or levels
    :returns: the indices of the values in the bins

    If there are V values and E edges this functions returns an array
    with V elements in the range 0 .. E - 2; for instance:

    >>> digitize('period', [0, .1, .2], [0, .05, .1, .15, .20, .25])
    array([0, 2, 4])
    """
    if max(values) > max(edges):
        raise ValueError(
            f'The {name} {max(values)} is outside the edges {edges}')
    if min(values) < min(edges):
        raise ValueError(
            f'The {name} {min(values)} is outside the edges {edges}')
    return numpy.digitize(values, edges) - 1

class Amplifier(object):
    """
    :param imtls: intensity measure types and levels DictArray M x I
    :param ampl_funcs: an ArrayWrapper containing amplification functions
    :param amplevels: levels used for the amplified curves
    :attr periods: array of M periods
    :attr midlevels: array of I-1 levels
    :attr alpha: dict code, imt-> I-1 amplification coefficients
    :attr sigma: dict code, imt-> I-1 amplification sigmas
    """
    def __init__(self, imtls, ampl_funcs, amplevels=None):
        self.imtls = imtls
        self.periods, levels = check_same_levels(imtls)
        self.amplevels = levels if amplevels is None else amplevels
        self.midlevels = numpy.diff(levels) / 2 + levels[:-1]  # mid levels
        imls = ampl_funcs.imls
        imts = [from_string(imt) for imt in ampl_funcs.dtype.names[2:]
                if not imt.startswith('sigma_')]
        m_indices = digitize(
            'period', self.periods, [imt.period for imt in imts])
        if len(imls) == 1:  # one level means same values for all levels
            l_indices = [0] * len(levels)
        else:
            l_indices = digitize('level', self.midlevels, imls)
        L = len(l_indices)
        self.imtdict = {imt: str(imts[m]) for m, imt in zip(m_indices, imtls)}
        self.alpha = {}  # code, imt -> alphas
        self.sigma = {}  # code, imt -> sigmas
        self.ampcodes = []
        for code, arr in group_array(ampl_funcs, 'ampcode').items():
            self.ampcodes.append(code)
            for m in set(m_indices):
                im = str(imts[m])
                self.alpha[code, im] = alpha = numpy.zeros(L)
                self.sigma[code, im] = sigma = numpy.zeros(L)
                idx = 0
                for rec in arr[l_indices]:
                    alpha[idx] = rec[im]
                    try:
                        sigma[idx] = rec['sigma_' + im]
                    except ValueError:  # missing sigma
                        pass
                    idx += 1

    def amplify_one(self, ampl_code, imt, poes):
        """
        :param ampl_code: 2-letter code for the amplification function
        :param imt: an intensity measure type
        :param poes: the original PoEs
        :returns: the amplified PoEs
        """
        if ampl_code == b'' and len(self.ampcodes) == 1:
            # manage the case of a site collection with empty ampcode
            ampl_code = self.ampcodes[0]
        stored_imt = self.imtdict[imt]
        alphas = self.alpha[ampl_code, stored_imt]
        sigmas = self.sigma[ampl_code, stored_imt]
        ampl_poes = numpy.zeros_like(self.amplevels)
        for mid, prob, a, s in zip(
                self.midlevels, -numpy.diff(poes), alphas, sigmas):
            ampl_poes += (1. - norm_cdf(self.amplevels / mid, a, s)) * prob
        return ampl_poes

    def amplify(self, ampl_code, pcurves):
        """
        :param ampl_code: 2-letter code for the amplification function
        :param pcurves: a list of ProbabilityCurves containing PoEs
        :returns: amplified ProbabilityCurves
        """
        out = []
        for pcurve in pcurves:
            lst = []
            for imt in self.imtls:
                slc = self.imtls(imt)
                new = self.amplify_one(ampl_code, imt, pcurve.array[slc, 0])
                lst.append(new)
            out.append(ProbabilityCurve(numpy.concatenate(lst).reshape(-1, 1)))
        return out
<commit_msg>Small fix<commit_after># -*- coding: utf-8 -*-
# vim: tabstop=4 shiftwidth=4 softtabstop=4
#
# Copyright (C) 2020, GEM Foundation
#
# OpenQuake is free software: you can redistribute it and/or modify it
# under the terms of the GNU Affero General Public License as published
# by the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# OpenQuake is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with OpenQuake.  If not, see <http://www.gnu.org/licenses/>.

import numpy
from scipy.stats import norm
from openquake.baselib.general import group_array
from openquake.hazardlib.imt import from_string
from openquake.hazardlib.probability_map import ProbabilityCurve
from openquake.commonlib.oqvalidation import check_same_levels

# this is necessary since norm.cdf for scale=0 returns NaNs
def norm_cdf(x, a, s):
    """
    Gaussian cumulative distribution function; if s=0, returns an
    Heaviside function instead. NB: for x=a, 0.5 is returned for all s.

    >>> norm_cdf(1.2, 1, .1)
    0.9772498680518208
    >>> norm_cdf(1.2, 1, 0)
    1.0
    >>> norm_cdf(.8, 1, .1)
    0.022750131948179216
    >>> norm_cdf(.8, 1, 0)
    0.0
    >>> norm_cdf(1, 1, .1)
    0.5
    >>> norm_cdf(1, 1, 0)
    0.5
    """
    if s == 0:
        return numpy.heaviside(x - a, .5)
    else:
        return norm.cdf(x, loc=a, scale=s)

def digitize(name, values, edges):
    """
    :param name: 'period' or 'level'
    :param values: periods or levels
    :param edges: available periods or levels
    :returns: the indices of the values in the bins

    If there are V values and E edges this functions returns an array
    with V elements in the range 0 .. E - 2; for instance:

    >>> digitize('period', [0, .1, .2], [0, .05, .1, .15, .20, .25])
    array([0, 2, 4])
    """
    if max(values) > max(edges):
        raise ValueError(
            f'The {name} {max(values)} is outside the edges {edges}')
    if min(values) < min(edges):
        raise ValueError(
            f'The {name} {min(values)} is outside the edges {edges}')
    return numpy.digitize(values, edges) - 1

class Amplifier(object):
    """
    :param imtls: intensity measure types and levels DictArray M x I
    :param ampl_funcs: an ArrayWrapper containing amplification functions
    :param amplevels: levels used for the amplified curves
    :attr periods: array of M periods
    :attr midlevels: array of I-1 levels
    :attr alpha: dict code, imt-> I-1 amplification coefficients
    :attr sigma: dict code, imt-> I-1 amplification sigmas
    """
    def __init__(self, imtls, ampl_funcs, amplevels=None):
        self.imtls = imtls
        self.periods, levels = check_same_levels(imtls)
        self.amplevels = levels if amplevels is None else amplevels
        self.midlevels = numpy.diff(levels) / 2 + levels[:-1]  # mid levels
        imls = ampl_funcs.imls
        imts = [from_string(imt) for imt in ampl_funcs.dtype.names[2:]
                if not imt.startswith('sigma_')]
        m_indices = digitize(
            'period', self.periods, [imt.period for imt in imts])
        if len(imls) == 1:  # one level means same values for all levels
            l_indices = [0] * len(self.midlevels)
        else:
            l_indices = digitize('level', self.midlevels, imls)
        L = len(l_indices)
        self.imtdict = {imt: str(imts[m]) for m, imt in zip(m_indices, imtls)}
        self.alpha = {}  # code, imt -> alphas
        self.sigma = {}  # code, imt -> sigmas
        self.ampcodes = []
        for code, arr in group_array(ampl_funcs, 'ampcode').items():
            self.ampcodes.append(code)
            for m in set(m_indices):
                im = str(imts[m])
                self.alpha[code, im] = alpha = numpy.zeros(L)
                self.sigma[code, im] = sigma = numpy.zeros(L)
                idx = 0
                for rec in arr[l_indices]:
                    alpha[idx] = rec[im]
                    try:
                        sigma[idx] = rec['sigma_' + im]
                    except ValueError:  # missing sigma
                        pass
                    idx += 1

    def amplify_one(self, ampl_code, imt, poes):
        """
        :param ampl_code: 2-letter code for the amplification function
        :param imt: an intensity measure type
        :param poes: the original PoEs
        :returns: the amplified PoEs
        """
        if ampl_code == b'' and len(self.ampcodes) == 1:
            # manage the case of a site collection with empty ampcode
            ampl_code = self.ampcodes[0]
        stored_imt = self.imtdict[imt]
        alphas = self.alpha[ampl_code, stored_imt]
        sigmas = self.sigma[ampl_code, stored_imt]
        ampl_poes = numpy.zeros_like(self.amplevels)
        for mid, prob, a, s in zip(
                self.midlevels, -numpy.diff(poes), alphas, sigmas):
            ampl_poes += (1. - norm_cdf(self.amplevels / mid, a, s)) * prob
        return ampl_poes

    def amplify(self, ampl_code, pcurves):
        """
        :param ampl_code: 2-letter code for the amplification function
        :param pcurves: a list of ProbabilityCurves containing PoEs
        :returns: amplified ProbabilityCurves
        """
        out = []
        for pcurve in pcurves:
            lst = []
            for imt in self.imtls:
                slc = self.imtls(imt)
                new = self.amplify_one(ampl_code, imt, pcurve.array[slc, 0])
                lst.append(new)
            out.append(ProbabilityCurve(numpy.concatenate(lst).reshape(-1, 1)))
        return out
<|endoftext|>


<commit_before>from pprint import pprint  # noqa
from datetime import datetime

from opensanctions.util import EntityEmitter
from opensanctions.util import jointext

ORG_HEADER = ["No", "Last Name", "Name", "Middle Name", "Date of birth",
              "Place of birth", "Reason for inclusion",
              "Category of entity", "Date of inclusion"]

PER_HEADER = ["No", "Name", "Reason for inclusion",
              "Category of entity", "Date of inclusion"]

def has_integer_id(row):
    try:
        int(row[0])
        return True
    except Exception:
        return False

def parse_date(date):
    if date is not None:
        date = datetime.strptime(date, '%d.%m.%Y')
        return date.date()

def extract_rows(node):
    for table in node.findall('.//table'):
        for row in table.findall('./row'):
            cells = []
            for cell in row.findall('./cell'):
                cells.append(cell.text)
            if not len(cells):
                continue
            if not has_integer_id(cells[0]):
                continue
            if len(cells) == 5:
                yield {k: v for k, v in zip(PER_HEADER, cells)}
            if len(cells) == 9:
                yield {k: v for k, v in zip(ORG_HEADER, cells)}

def parse_individual(emitter, data):
    person = emitter.make('Person')
    name = jointext(data["Name"], data["Middle Name"], data["Last Name"])
    person.make_id(name, data["Reason for inclusion"])
    person.add('name', name)
    person.add('lastName', data["Last Name"])
    person.add('firstName', data["Name"])
    person.add('fatherName', data["Middle Name"])
    person.add('birthDate', parse_date(data["Date of birth"]))
    person.add('birthPlace', data["Place of birth"])

    sanction = emitter.make('Sanction')
    sanction.make_id('Sanction', person.id)
    sanction.add('entity', person)
    sanction.add('authority', 'Kyrgyz Financial Intelligence Unit')
    sanction.add('reason', data["Reason for inclusion"])
    sanction.add('program', data["Category of entity"])
    sanction.add('startDate', parse_date(data["Date of inclusion"]))
    emitter.emit(person)
    emitter.emit(sanction)

def parse_organisation(emitter, data):
    entity = emitter.make('LegalEntity')
    entity.make_id(data["Name"], data["Reason for inclusion"])
    sanction = emitter.make('Sanction')
    sanction.make_id('Sanction', entity.id)
    sanction.add('entity', entity)
    sanction.add('authority', 'Kyrgyz Financial Intelligence Unit')
    sanction.add('reason', data["Reason for inclusion"])
    sanction.add('program', data["Category of entity"])
    sanction.add('startDate', parse_date(data["Date of inclusion"]))
    names = data["Name"].split(',')
    entity.add('name', names[0])
    entity.add('alias', names[1:])
    emitter.emit(entity)
    emitter.emit(sanction)

def parse(context, data):
    emitter = EntityEmitter(context)
    with context.http.rehash(data) as res:
        for row in extract_rows(res.xml):
            if len(row) == 5:
                parse_organisation(emitter, row)
            if len(row) == 9:
                parse_individual(emitter, row)
    emitter.finalize()
<commit_msg>Fix kg_fiu_national crawler<commit_after>from pprint import pprint  # noqa
from datetime import datetime
import re

from opensanctions.util import EntityEmitter
from opensanctions.util import jointext

ORG_HEADER = ["No", "Last Name", "Name", "Middle Name", "Date of birth",
              "Place of birth", "Reason for inclusion",
              "Category of entity", "Date of inclusion"]

PER_HEADER = ["No", "Name", "Reason for inclusion",
              "Category of entity", "Date of inclusion"]

def has_integer_id(row):
    try:
        int(row[0])
        return True
    except Exception:
        return False

def parse_date(date):
    pattern = re.compile(r'[^0-9\.]')
    if date is not None:
        date = pattern.sub('', date)
        try:
            date = datetime.strptime(date, '%d.%m.%Y')
        except ValueError:
            date = datetime.strptime(date, '%Y')
        return date.date()

def extract_rows(node):
    for table in node.findall('.//table'):
        for row in table.findall('./row'):
            cells = []
            for cell in row.findall('./cell'):
                cells.append(cell.text)
            if not len(cells):
                continue
            if not has_integer_id(cells[0]):
                continue
            if len(cells) == 5:
                yield {k: v for k, v in zip(PER_HEADER, cells)}
            if len(cells) == 9:
                yield {k: v for k, v in zip(ORG_HEADER, cells)}

def parse_individual(emitter, data):
    person = emitter.make('Person')
    name = jointext(data["Name"], data["Middle Name"], data["Last Name"])
    person.make_id(name, data["Reason for inclusion"])
    person.add('name', name)
    person.add('lastName', data["Last Name"])
    person.add('firstName', data["Name"])
    person.add('fatherName', data["Middle Name"])
    # Some records have multiple dobs
    dob = data["Date of birth"]
    if dob is not None:
        dobs = dob.split()
        for date in dobs:
            person.add('birthDate', parse_date(date))
    person.add('birthPlace', data["Place of birth"])

    sanction = emitter.make('Sanction')
    sanction.make_id('Sanction', person.id)
    sanction.add('entity', person)
    sanction.add('authority', 'Kyrgyz Financial Intelligence Unit')
    sanction.add('reason', data["Reason for inclusion"])
    sanction.add('program', data["Category of entity"])
    sanction.add('startDate', parse_date(data["Date of inclusion"]))
    emitter.emit(person)
    emitter.emit(sanction)

def parse_organisation(emitter, data):
    entity = emitter.make('LegalEntity')
    entity.make_id(data["Name"], data["Reason for inclusion"])
    sanction = emitter.make('Sanction')
    sanction.make_id('Sanction', entity.id)
    sanction.add('entity', entity)
    sanction.add('authority', 'Kyrgyz Financial Intelligence Unit')
    sanction.add('reason', data["Reason for inclusion"])
    sanction.add('program', data["Category of entity"])
    sanction.add('startDate', parse_date(data["Date of inclusion"]))
    names = data["Name"].split(',')
    entity.add('name', names[0])
    entity.add('alias', names[1:])
    emitter.emit(entity)
    emitter.emit(sanction)

def parse(context, data):
    emitter = EntityEmitter(context)
    with context.http.rehash(data) as res:
        for row in extract_rows(res.xml):
            if len(row) == 5:
                parse_organisation(emitter, row)
            if len(row) == 9:
                parse_individual(emitter, row)
    emitter.finalize()
<|endoftext|>


<commit_before>from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
from __future__ import unicode_literals

import numpy as np
import networkx as nx
import functools
import math

from opensfm import types
from opensfm import pygeometry

import opensfm.synthetic_data.synthetic_metrics as sm
import opensfm.synthetic_data.synthetic_generator as sg

def get_camera(type, id, focal, k1, k2):
    camera = None
    if type == 'perspective':
        camera = pygeometry.Camera.create_perspective(focal, k1, k2)
    if type == 'fisheye':
        camera = pygeometry.Camera.create_fisheye(focal, k1, k2)
    camera.id = id

    camera.height = 1600
    camera.width = 2000
    return camera

def get_scene_generator(type, length):
    generator = None
    if type == 'circle':
        generator = functools.partial(sg.ellipse_generator, length, length)
    if type == 'ellipse':
        ellipse_ratio = 2
        generator = functools.partial(sg.ellipse_generator, length,
                                      length / ellipse_ratio)
    if type == 'line':
        generator = functools.partial(sg.line_generator, length)
    if type == 'curve':
        generator = functools.partial(sg.weird_curve, length)
    return generator

def normalized(x):
    return x / np.linalg.norm(x)

def camera_pose(position, lookat, up):
    '''
    Pose from position and look at direction

    >>> position = [1.0, 2.0, 3.0]
    >>> lookat = [0., 10.0, 2.0]
    >>> up = [0.0, 0.0, 1.0]
    >>> pose = camera_pose(position, lookat, up)
    >>> np.allclose(pose.get_origin(), position)
    True
    >>> d = normalized(pose.transform(lookat))
    >>> np.allclose(d, [0, 0, 1])
    True
    '''
    ez = normalized(np.array(lookat) - np.array(position))
    ex = normalized(np.cross(ez, up))
    ey = normalized(np.cross(ez, ex))
    pose = types.Pose()
    pose.set_rotation_matrix([ex, ey, ez])
    pose.set_origin(position)
    return pose

class SyntheticScene(object):
    def get_reconstruction(self, rotation_noise=0.0,
                           position_noise=0.0,
                           camera_noise=0.0):
        raise NotImplementedError()

    def get_scene_exifs(self, gps_noise):
        raise NotImplementedError()

    def get_tracks_data(self, maximum_depth, noise):
        raise NotImplementedError()

class SyntheticCubeScene(SyntheticScene):
    """ Scene consisting in of cameras looking at point in a cube. """
    def __init__(self, num_cameras, num_points, noise):
        self.cameras = {}
        for i in range(num_cameras):
            camera = camera = pygeometry.Camera.create_perspective(0.9, -0.1, 0.01)
            camera.id = 'camera%04d' % i
            camera.height = 600
            camera.width = 800
            self.cameras[camera.id] = camera

        self.shots = {}
        r = 2.0
        for i in range(num_cameras):
            phi = np.random.rand()*math.pi
            theta = np.random.rand()*2.0*math.pi
            x = r*np.sin(theta)*np.cos(phi)
            y = r*np.sin(theta)*np.sin(phi)
            z = r*np.cos(theta)
            position = [x, y, z]

            alpha = np.random.rand()
            lookat = [0.0, 0, 0]
            up = [alpha * 0.2, alpha * 0.2, 1.0]

            shot = types.Shot()
            shot.id = 'shot%04d' % i
            shot.camera = self.cameras['camera%04d' % i]
            shot.pose = camera_pose(position, lookat, up)
            self.shots[shot.id] = shot

        points = np.random.rand(num_points, 3)-[0.5, 0.5, 0.5]
        self.points = {}
        for i, p in enumerate(points):
            pt = types.Point()
            pt.id = 'point' + str(i)
            pt.coordinates = p
            pt.color = [100, 100, 20]
            self.points[pt.id] = pt

    def get_reconstruction(self, rotation_noise=0.0,
                           position_noise=0.0,
                           camera_noise=0.0):
        reconstruction = types.Reconstruction()
        reconstruction.shots = self.shots
        reconstruction.points = self.points
        reconstruction.cameras = self.cameras
        return reconstruction

    def get_tracks_data(self, maximum_depth, noise):
        return sg.generate_track_data(self.get_reconstruction(),
                                      maximum_depth, noise)

class SyntheticStreetScene(SyntheticScene):
    """ Scene consisting in a virtual street extruded along some
        parametric shape (line, ellipse), with camera placed along
        the shape.
    """
    def __init__(self, generator):
        self.generator = generator
        self.wall_points = None
        self.floor_points = None
        self.width = None
        self.shot_positions = []
        self.shot_rotations = []
        self.cameras = []

    def add_street(self, points_count, height, width):
        self.wall_points, self.floor_points = sg.generate_street(
            sg.samples_generator_random_count(
                int(points_count // 3)), self.generator,
            height, width)
        self.width = width
        return self

    def perturb_walls(self, walls_pertubation):
        sg.perturb_points(self.wall_points, walls_pertubation)
        return self

    def perturb_floor(self, floor_pertubation):
        sg.perturb_points(self.floor_points, floor_pertubation)
        return self

    def add_camera_sequence(self, camera, start, length, height, interval,
                            position_noise=None, rotation_noise=None,
                            gps_noise=None):
        default_noise_interval = 0.25*interval
        positions, rotations = sg.generate_cameras(
            sg.samples_generator_interval(start, length, interval,
                                          default_noise_interval),
            self.generator, height)
        sg.perturb_points(positions, position_noise)
        sg.perturb_rotations(rotations, rotation_noise)
        self.shot_rotations.append(rotations)
        self.shot_positions.append(positions)
        self.cameras.append(camera)
        return self

    def get_reconstruction(self, rotation_noise=0.0,
                           position_noise=0.0,
                           camera_noise=0.0):
        floor_color = [120, 90, 10]
        wall_color = [10, 90, 130]

        positions = self.shot_positions
        if position_noise != 0.0:
            for p in positions:
                sg.perturb_points(p, position_noise)
        rotations = self.shot_rotations
        if position_noise != 0.0:
            for r in rotations:
                sg.perturb_rotations(r, rotation_noise)
        cameras = self.cameras
        if camera_noise != 0.0:
            for c in cameras:
                c.focal *= (1+camera_noise)

        return sg.create_reconstruction(
            [self.floor_points, self.wall_points],
            [floor_color, wall_color],
            cameras, positions, rotations)

    def get_scene_exifs(self, gps_noise):
        return sg.generate_exifs(self.get_reconstruction(),
                                 gps_noise)

    def get_tracks_data(self, maximum_depth, noise):
        return sg.generate_track_data(self.get_reconstruction(),
                                      maximum_depth, noise)

import json
from opensfm import io
def compare(reference, reconstruction):
    completeness = sm.completeness_errors(reference, reconstruction)

    absolute_position = sm.position_errors(reference, reconstruction)
    absolute_rotation = sm.rotation_errors(reference, reconstruction)
    absolute_points = sm.points_errors(reference, reconstruction)
    absolute_gps = sm.gps_errors(reconstruction)

    aligned = sm.aligned_to_reference(reference, reconstruction)
    aligned_position = sm.position_errors(reference, aligned)
    aligned_rotation = sm.rotation_errors(reference, aligned)
    aligned_points = sm.points_errors(reference, aligned)
    aligned_gps = sm.gps_errors(aligned)

    json.dump(io.reconstructions_to_json([reference, reconstruction, aligned]), open('/tmp/mapillary_sfm_test/aligned.json', 'wt'))

    return {
        'ratio_cameras': completeness[0],
        'ratio_points': completeness[1],

        'absolute_position_rmse': sm.rmse(absolute_position),
        'absolute_rotation_rmse': sm.rmse(absolute_rotation),
        'absolute_points_rmse': sm.rmse(absolute_points),
        'absolute_gps_rmse': sm.rmse(absolute_gps),

        'aligned_position_rmse': sm.rmse(aligned_position),
        'aligned_rotation_rmse': sm.rmse(aligned_rotation),
        'aligned_points_rmse': sm.rmse(aligned_points),
        'aligned_gps_rmse': sm.rmse(aligned_gps),
    }
<commit_msg>fix: remove debug code<commit_after>from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
from __future__ import unicode_literals

import numpy as np
import networkx as nx
import functools
import math

from opensfm import types
from opensfm import pygeometry

import opensfm.synthetic_data.synthetic_metrics as sm
import opensfm.synthetic_data.synthetic_generator as sg

def get_camera(type, id, focal, k1, k2):
    camera = None
    if type == 'perspective':
        camera = pygeometry.Camera.create_perspective(focal, k1, k2)
    if type == 'fisheye':
        camera = pygeometry.Camera.create_fisheye(focal, k1, k2)
    camera.id = id

    camera.height = 1600
    camera.width = 2000
    return camera

def get_scene_generator(type, length):
    generator = None
    if type == 'circle':
        generator = functools.partial(sg.ellipse_generator, length, length)
    if type == 'ellipse':
        ellipse_ratio = 2
        generator = functools.partial(sg.ellipse_generator, length,
                                      length / ellipse_ratio)
    if type == 'line':
        generator = functools.partial(sg.line_generator, length)
    if type == 'curve':
        generator = functools.partial(sg.weird_curve, length)
    return generator

def normalized(x):
    return x / np.linalg.norm(x)

def camera_pose(position, lookat, up):
    '''
    Pose from position and look at direction

    >>> position = [1.0, 2.0, 3.0]
    >>> lookat = [0., 10.0, 2.0]
    >>> up = [0.0, 0.0, 1.0]
    >>> pose = camera_pose(position, lookat, up)
    >>> np.allclose(pose.get_origin(), position)
    True
    >>> d = normalized(pose.transform(lookat))
    >>> np.allclose(d, [0, 0, 1])
    True
    '''
    ez = normalized(np.array(lookat) - np.array(position))
    ex = normalized(np.cross(ez, up))
    ey = normalized(np.cross(ez, ex))
    pose = types.Pose()
    pose.set_rotation_matrix([ex, ey, ez])
    pose.set_origin(position)
    return pose

class SyntheticScene(object):
    def get_reconstruction(self, rotation_noise=0.0,
                           position_noise=0.0,
                           camera_noise=0.0):
        raise NotImplementedError()

    def get_scene_exifs(self, gps_noise):
        raise NotImplementedError()

    def get_tracks_data(self, maximum_depth, noise):
        raise NotImplementedError()

class SyntheticCubeScene(SyntheticScene):
    """ Scene consisting in of cameras looking at point in a cube. """
    def __init__(self, num_cameras, num_points, noise):
        self.cameras = {}
        for i in range(num_cameras):
            camera = camera = pygeometry.Camera.create_perspective(0.9, -0.1, 0.01)
            camera.id = 'camera%04d' % i
            camera.height = 600
            camera.width = 800
            self.cameras[camera.id] = camera

        self.shots = {}
        r = 2.0
        for i in range(num_cameras):
            phi = np.random.rand()*math.pi
            theta = np.random.rand()*2.0*math.pi
            x = r*np.sin(theta)*np.cos(phi)
            y = r*np.sin(theta)*np.sin(phi)
            z = r*np.cos(theta)
            position = [x, y, z]

            alpha = np.random.rand()
            lookat = [0.0, 0, 0]
            up = [alpha * 0.2, alpha * 0.2, 1.0]

            shot = types.Shot()
            shot.id = 'shot%04d' % i
            shot.camera = self.cameras['camera%04d' % i]
            shot.pose = camera_pose(position, lookat, up)
            self.shots[shot.id] = shot

        points = np.random.rand(num_points, 3)-[0.5, 0.5, 0.5]
        self.points = {}
        for i, p in enumerate(points):
            pt = types.Point()
            pt.id = 'point' + str(i)
            pt.coordinates = p
            pt.color = [100, 100, 20]
            self.points[pt.id] = pt

    def get_reconstruction(self, rotation_noise=0.0,
                           position_noise=0.0,
                           camera_noise=0.0):
        reconstruction = types.Reconstruction()
        reconstruction.shots = self.shots
        reconstruction.points = self.points
        reconstruction.cameras = self.cameras
        return reconstruction

    def get_tracks_data(self, maximum_depth, noise):
        return sg.generate_track_data(self.get_reconstruction(),
                                      maximum_depth, noise)

class SyntheticStreetScene(SyntheticScene):
    """ Scene consisting in a virtual street extruded along some
        parametric shape (line, ellipse), with camera placed along
        the shape.
    """
    def __init__(self, generator):
        self.generator = generator
        self.wall_points = None
        self.floor_points = None
        self.width = None
        self.shot_positions = []
        self.shot_rotations = []
        self.cameras = []

    def add_street(self, points_count, height, width):
        self.wall_points, self.floor_points = sg.generate_street(
            sg.samples_generator_random_count(
                int(points_count // 3)), self.generator,
            height, width)
        self.width = width
        return self

    def perturb_walls(self, walls_pertubation):
        sg.perturb_points(self.wall_points, walls_pertubation)
        return self

    def perturb_floor(self, floor_pertubation):
        sg.perturb_points(self.floor_points, floor_pertubation)
        return self

    def add_camera_sequence(self, camera, start, length, height, interval,
                            position_noise=None, rotation_noise=None,
                            gps_noise=None):
        default_noise_interval = 0.25*interval
        positions, rotations = sg.generate_cameras(
            sg.samples_generator_interval(start, length, interval,
                                          default_noise_interval),
            self.generator, height)
        sg.perturb_points(positions, position_noise)
        sg.perturb_rotations(rotations, rotation_noise)
        self.shot_rotations.append(rotations)
        self.shot_positions.append(positions)
        self.cameras.append(camera)
        return self

    def get_reconstruction(self, rotation_noise=0.0,
                           position_noise=0.0,
                           camera_noise=0.0):
        floor_color = [120, 90, 10]
        wall_color = [10, 90, 130]

        positions = self.shot_positions
        if position_noise != 0.0:
            for p in positions:
                sg.perturb_points(p, position_noise)
        rotations = self.shot_rotations
        if position_noise != 0.0:
            for r in rotations:
                sg.perturb_rotations(r, rotation_noise)
        cameras = self.cameras
        if camera_noise != 0.0:
            for c in cameras:
                c.focal *= (1+camera_noise)

        return sg.create_reconstruction(
            [self.floor_points, self.wall_points],
            [floor_color, wall_color],
            cameras, positions, rotations)

    def get_scene_exifs(self, gps_noise):
        return sg.generate_exifs(self.get_reconstruction(),
                                 gps_noise)

    def get_tracks_data(self, maximum_depth, noise):
        return sg.generate_track_data(self.get_reconstruction(),
                                      maximum_depth, noise)

def compare(reference, reconstruction):
    completeness = sm.completeness_errors(reference, reconstruction)

    absolute_position = sm.position_errors(reference, reconstruction)
    absolute_rotation = sm.rotation_errors(reference, reconstruction)
    absolute_points = sm.points_errors(reference, reconstruction)
    absolute_gps = sm.gps_errors(reconstruction)

    aligned = sm.aligned_to_reference(reference, reconstruction)
    aligned_position = sm.position_errors(reference, aligned)
    aligned_rotation = sm.rotation_errors(reference, aligned)
    aligned_points = sm.points_errors(reference, aligned)
    aligned_gps = sm.gps_errors(aligned)

    return {
        'ratio_cameras': completeness[0],
        'ratio_points': completeness[1],

        'absolute_position_rmse': sm.rmse(absolute_position),
        'absolute_rotation_rmse': sm.rmse(absolute_rotation),
        'absolute_points_rmse': sm.rmse(absolute_points),
        'absolute_gps_rmse': sm.rmse(absolute_gps),

        'aligned_position_rmse': sm.rmse(aligned_position),
        'aligned_rotation_rmse': sm.rmse(aligned_rotation),
        'aligned_points_rmse': sm.rmse(aligned_points),
        'aligned_gps_rmse': sm.rmse(aligned_gps),
    }
<|endoftext|>


<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-

from __future__ import absolute_import
from __future__ import division, print_function, unicode_literals
import fsed.ahocorasick

@cli.command()
@click.argument('batch_filename', type=click.Path(exists=True))
@click.argument('input_filename', default='-')
@click.argument('output_filename', default='-')
def make_input_acsr(batch_filename, input_filename, output_filename):
    # load the batch
    logger.info('make_input_acsr {} {} {}'.format(batch_filename, input_filename,
                                                  output_filename))
    candidates = []
    with utils.open_file(batch_filename) as batch_file:
        for line in batch_file:
            line = line.decode('utf-8').strip()
            candidates.append(line)
    logger.info('{} MWE candidates loaded'.format(len(candidates)))
    trie = ahocorasick.AhoCorasickTrie()
    for cand in candidates:
        trie[cand] = cand.replace(' ', '_')
    # search and replace on the
    with utils.open_file(input_filename) as input_file:
        with utils.open_file(output_filename, 'w') as output_file:
            logger.info('writing {} to {}'.format(input_filename,
                                                  output_filename))
            num_lines = 0
            for line in input_file:
                line = line.decode('utf-8').strip()
                line = trie.greedy_replace_w_sep(line)
                output_file.write((line + '\n').encode('utf-8'))
                num_lines += 1
            logger.info('{} lines written'.format(num_lines))
<commit_msg>fsed/fsed.py: fix package paths<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-

from __future__ import absolute_import
from __future__ import division, print_function, unicode_literals
import fsed.ahocorasick

@cli.command()
@click.argument('batch_filename', type=click.Path(exists=True))
@click.argument('input_filename', default='-')
@click.argument('output_filename', default='-')
def make_input_acsr(batch_filename, input_filename, output_filename):
    # load the batch
    logger.info('make_input_acsr {} {} {}'.format(batch_filename, input_filename,
                                                  output_filename))
    candidates = []
    with utils.open_file(batch_filename) as batch_file:
        for line in batch_file:
            line = line.decode('utf-8').strip()
            candidates.append(line)
    logger.info('{} MWE candidates loaded'.format(len(candidates)))
    trie = fsed.ahocorasick.AhoCorasickTrie()
    for cand in candidates:
        trie[cand] = cand.replace(' ', '_')
    # search and replace on the
    with utils.open_file(input_filename) as input_file:
        with utils.open_file(output_filename, 'w') as output_file:
            logger.info('writing {} to {}'.format(input_filename,
                                                  output_filename))
            num_lines = 0
            for line in input_file:
                line = line.decode('utf-8').strip()
                line = trie.greedy_replace_w_sep(line)
                output_file.write((line + '\n').encode('utf-8'))
                num_lines += 1
            logger.info('{} lines written'.format(num_lines))
<|endoftext|>


<commit_before>
import skimage.morphology as mo
import scipy.ndimage as nd
import warnings

try:
    import cv2
    CV2_FLAG = True
except ImportError:
    warnings.warn("Cannot import cv2. Computing with scipy.ndimage")
    CV2_FLAG = False

from utils import eight_conn

def smooth_edges(mask, filter_size, min_pixels):

    no_small = mo.remove_small_holes(mask, min_size=min_pixels,
                                     connectivity=2)

    open_close = \
        nd.binary_closing(nd.binary_opening(no_small, eight_conn), eight_conn)

    medianed = nd.median_filter(open_close, filter_size)

    return mo.remove_small_holes(medianed, min_size=min_pixels,
                                 connectivity=2)

def remove_spurs(mask, min_distance=9):
    '''
    Remove spurious mask features with reconstruction.
    '''

    # Distance transform of the mask
    dist_trans = nd.distance_transform_edt(mask)

    # We don't want to return local maxima within the minimum distance
    # Use reconstruction to remove.
    seed = dist_trans + min_distance
    reconst = mo.reconstruction(seed, dist_trans, method='erosion') - \
        min_distance

    if CV2_FLAG:
        return cv2.morphologyEx((reconst > 0).astype("uint8"),
                                cv2.MORPH_DILATE,
                                mo.disk(min_distance).astype("uint8")).astype(bool)
    else:
        return mo.dilation(reconst > 0, selem=mo.disk(min_distance))
<commit_msg>Find the fraction of a region that falls within the mask<commit_after>
import skimage.morphology as mo
import scipy.ndimage as nd
import warnings
from astropy.modeling.models import Ellipse2D
import numpy as np

try:
    import cv2
    CV2_FLAG = True
except ImportError:
    warnings.warn("Cannot import cv2. Computing with scipy.ndimage")
    CV2_FLAG = False

from utils import eight_conn, ceil_int

def smooth_edges(mask, filter_size, min_pixels):

    no_small = mo.remove_small_holes(mask, min_size=min_pixels,
                                     connectivity=2)

    open_close = \
        nd.binary_closing(nd.binary_opening(no_small, eight_conn), eight_conn)

    medianed = nd.median_filter(open_close, filter_size)

    return mo.remove_small_holes(medianed, min_size=min_pixels,
                                 connectivity=2)

def remove_spurs(mask, min_distance=9):
    '''
    Remove spurious mask features with reconstruction.
    '''

    # Distance transform of the mask
    dist_trans = nd.distance_transform_edt(mask)

    # We don't want to return local maxima within the minimum distance
    # Use reconstruction to remove.
    seed = dist_trans + min_distance
    reconst = mo.reconstruction(seed, dist_trans, method='erosion') - \
        min_distance

    if CV2_FLAG:
        return cv2.morphologyEx((reconst > 0).astype("uint8"),
                                cv2.MORPH_DILATE,
                                mo.disk(min_distance).astype("uint8")).astype(bool)
    else:
        return mo.dilation(reconst > 0, selem=mo.disk(min_distance))

def fraction_in_mask(blob, mask):
    '''
    Find the fraction of a blob within the mask. This is intended to be an
    added check for bad 2D fits.
    '''

    ellipse = Ellipse2D(True, blob[1], blob[0], blob[2], blob[3], blob[4])

    # Cut the mask to the bounding box
    yextents, xextents = ellipse.bounding_box

    yy, xx = np.mgrid[yextents[0]:yextents[1] + 1,
                      xextents[0]:xextents[1] + 1]

    ellip_mask = ellipse(xx, yy).astype(np.bool)

    local_mask = mask[yextents[0]:yextents[1] + 1,
                      xextents[0]:xextents[1] + 1]

    return (ellip_mask * local_mask).sum() / float(ellip_mask.sum())
<|endoftext|>