<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-
#
# Copyright (C) 2007-2008 Christopher Lenz
# All rights reserved.
#
# This software is licensed as described in the file COPYING, which
# you should have received as part of this distribution.

"""Implementation of a view server for functions written in Python."""

from codecs import BOM_UTF8
import logging
import os
import sys
import traceback
from types import FunctionType

from couchdb import json, util

__all__ = ['main', 'run']
__docformat__ = 'restructuredtext en'

log = logging.getLogger('couchdb.view')

def run(input=sys.stdin, output=sys.stdout):
    r"""CouchDB view function handler implementation for Python.

    :param input: the readable file-like object to read input from
    :param output: the writable file-like object to write output to
    """
    functions = []

    def _writejson(obj):
        obj = json.encode(obj)
        if isinstance(obj, util.utype):
            obj = obj.encode('utf-8')
        output.write(obj)
        output.write('\n')
        output.flush()

    def _log(message):
        if not isinstance(message, util.strbase):
            message = json.encode(message)
        _writejson({'log': message})

    def reset(config=None):
        del functions[:]
        return True

    def add_fun(string):
        string = BOM_UTF8 + string.encode('utf-8')
        globals_ = {}
        try:
            util.pyexec(string, {'log': _log}, globals_)
        except Exception as e:
            return {'error': {
                'id': 'map_compilation_error',
                'reason': e.args[0]
            }}
        err = {'error': {
            'id': 'map_compilation_error',
            'reason': 'string must eval to a function '
                      '(ex: "def(doc): return 1")'
        }}
        if len(globals_) != 1:
            return err
        function = list(globals_.values())[0]
        if type(function) is not FunctionType:
            return err
        functions.append(function)
        return True

    def map_doc(doc):
        results = []
        for function in functions:
            try:
                results.append([[key, value] for key, value in function(doc)])
            except Exception as e:
                log.error('runtime error in map function: %s', e,
                          exc_info=True)
                results.append([])
                _log(traceback.format_exc())
        return results

    def reduce(*cmd, **kwargs):
        code = BOM_UTF8 + cmd[0][0].encode('utf-8')
        args = cmd[1]
        globals_ = {}
        try:
            util.pyexec(code, {'log': _log}, globals_)
        except Exception as e:
            log.error('runtime error in reduce function: %s', e,
                      exc_info=True)
            return {'error': {
                'id': 'reduce_compilation_error',
                'reason': e.args[0]
            }}
        err = {'error': {
            'id': 'reduce_compilation_error',
            'reason': 'string must eval to a function '
                      '(ex: "def(keys, values): return 1")'
        }}
        if len(globals_) != 1:
            return err
        function = list(globals_.values())[0]
        if type(function) is not FunctionType:
            return err

        rereduce = kwargs.get('rereduce', False)
        results = []
        if rereduce:
            keys = None
            vals = args
        else:
            if args:
                keys, vals = zip(*args)
            else:
                keys, vals = [], []
        if util.funcode(function).co_argcount == 3:
            results = function(keys, vals, rereduce)
        else:
            results = function(keys, vals)
        return [True, [results]]

    def rereduce(*cmd):
        # Note: weird kwargs is for Python 2.5 compat
        return reduce(*cmd, **{'rereduce': True})

    handlers = {'reset': reset, 'add_fun': add_fun, 'map_doc': map_doc,
                'reduce': reduce, 'rereduce': rereduce}

    try:
        while True:
            line = input.readline()
            if not line:
                break
            try:
                cmd = json.decode(line)
                log.debug('Processing %r', cmd)
            except ValueError as e:
                log.error('Error: %s', e, exc_info=True)
                return 1
            else:
                retval = handlers[cmd[0]](*cmd[1:])
                log.debug('Returning  %r', retval)
                _writejson(retval)
    except KeyboardInterrupt:
        return 0
    except Exception as e:
        log.error('Error: %s', e, exc_info=True)
        return 1

_VERSION = """%(name)s - CouchDB Python %(version)s

Copyright (C) 2007 Christopher Lenz <cmlenz@gmx.de>.
"""

_HELP = """Usage: %(name)s [OPTION]

The %(name)s command runs the CouchDB Python view server.

The exit status is 0 for success or 1 for failure.

Options:

  --version             display version information and exit
  -h, --help            display a short help message and exit
  --json-module=<name>  set the JSON module to use ('simplejson', 'cjson',
                        or 'json' are supported)
  --log-file=<file>     name of the file to write log messages to, or '-' to
                        enable logging to the standard error stream
  --debug               enable debug logging; requires --log-file to be
                        specified

Report bugs via the web at <http://code.google.com/p/couchdb-python>.
"""

def main():
    """Command-line entry point for running the view server."""
    import getopt
    from couchdb import __version__ as VERSION

    try:
        option_list, argument_list = getopt.gnu_getopt(
            sys.argv[1:], 'h',
            ['version', 'help', 'json-module=', 'debug', 'log-file=']
        )

        message = None
        for option, value in option_list:
            if option in ('--version'):
                message = _VERSION % dict(name=os.path.basename(sys.argv[0]),
                                      version=VERSION)
            elif option in ('-h', '--help'):
                message = _HELP % dict(name=os.path.basename(sys.argv[0]))
            elif option in ('--json-module'):
                json.use(module=value)
            elif option in ('--debug'):
                log.setLevel(logging.DEBUG)
            elif option in ('--log-file'):
                if value == '-':
                    handler = logging.StreamHandler(sys.stderr)
                    handler.setFormatter(logging.Formatter(
                        ' -> [%(levelname)s] %(message)s'
                    ))
                else:
                    handler = logging.FileHandler(value)
                    handler.setFormatter(logging.Formatter(
                        '[%(asctime)s] [%(levelname)s] %(message)s'
                    ))
                log.addHandler(handler)
        if message:
            sys.stdout.write(message)
            sys.stdout.flush()
            sys.exit(0)

    except getopt.GetoptError as error:
        message = '%s\n\nTry `%s --help` for more information.\n' % (
            str(error), os.path.basename(sys.argv[0])
        )
        sys.stderr.write(message)
        sys.stderr.flush()
        sys.exit(1)

    sys.exit(run())

if __name__ == '__main__':
    main()
<commit_msg>Python 3: the view server writes bytes.<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-
#
# Copyright (C) 2007-2008 Christopher Lenz
# All rights reserved.
#
# This software is licensed as described in the file COPYING, which
# you should have received as part of this distribution.

"""Implementation of a view server for functions written in Python."""

from codecs import BOM_UTF8
import logging
import os
import sys
import traceback
from types import FunctionType

from couchdb import json, util

__all__ = ['main', 'run']
__docformat__ = 'restructuredtext en'

log = logging.getLogger('couchdb.view')

def run(input=sys.stdin, output=sys.stdout):
    r"""CouchDB view function handler implementation for Python.

    :param input: the readable file-like object to read input from
    :param output: the writable file-like object to write output to
    """
    functions = []

    def _writejson(obj):
        obj = json.encode(obj)
        if isinstance(obj, util.utype):
            obj = obj.encode('utf-8')
        output.write(obj)
        output.write(b'\n')
        output.flush()

    def _log(message):
        if not isinstance(message, util.strbase):
            message = json.encode(message)
        _writejson({'log': message})

    def reset(config=None):
        del functions[:]
        return True

    def add_fun(string):
        string = BOM_UTF8 + string.encode('utf-8')
        globals_ = {}
        try:
            util.pyexec(string, {'log': _log}, globals_)
        except Exception as e:
            return {'error': {
                'id': 'map_compilation_error',
                'reason': e.args[0]
            }}
        err = {'error': {
            'id': 'map_compilation_error',
            'reason': 'string must eval to a function '
                      '(ex: "def(doc): return 1")'
        }}
        if len(globals_) != 1:
            return err
        function = list(globals_.values())[0]
        if type(function) is not FunctionType:
            return err
        functions.append(function)
        return True

    def map_doc(doc):
        results = []
        for function in functions:
            try:
                results.append([[key, value] for key, value in function(doc)])
            except Exception as e:
                log.error('runtime error in map function: %s', e,
                          exc_info=True)
                results.append([])
                _log(traceback.format_exc())
        return results

    def reduce(*cmd, **kwargs):
        code = BOM_UTF8 + cmd[0][0].encode('utf-8')
        args = cmd[1]
        globals_ = {}
        try:
            util.pyexec(code, {'log': _log}, globals_)
        except Exception as e:
            log.error('runtime error in reduce function: %s', e,
                      exc_info=True)
            return {'error': {
                'id': 'reduce_compilation_error',
                'reason': e.args[0]
            }}
        err = {'error': {
            'id': 'reduce_compilation_error',
            'reason': 'string must eval to a function '
                      '(ex: "def(keys, values): return 1")'
        }}
        if len(globals_) != 1:
            return err
        function = list(globals_.values())[0]
        if type(function) is not FunctionType:
            return err

        rereduce = kwargs.get('rereduce', False)
        results = []
        if rereduce:
            keys = None
            vals = args
        else:
            if args:
                keys, vals = zip(*args)
            else:
                keys, vals = [], []
        if util.funcode(function).co_argcount == 3:
            results = function(keys, vals, rereduce)
        else:
            results = function(keys, vals)
        return [True, [results]]

    def rereduce(*cmd):
        # Note: weird kwargs is for Python 2.5 compat
        return reduce(*cmd, **{'rereduce': True})

    handlers = {'reset': reset, 'add_fun': add_fun, 'map_doc': map_doc,
                'reduce': reduce, 'rereduce': rereduce}

    try:
        while True:
            line = input.readline()
            if not line:
                break
            try:
                cmd = json.decode(line)
                log.debug('Processing %r', cmd)
            except ValueError as e:
                log.error('Error: %s', e, exc_info=True)
                return 1
            else:
                retval = handlers[cmd[0]](*cmd[1:])
                log.debug('Returning  %r', retval)
                _writejson(retval)
    except KeyboardInterrupt:
        return 0
    except Exception as e:
        log.error('Error: %s', e, exc_info=True)
        return 1

_VERSION = """%(name)s - CouchDB Python %(version)s

Copyright (C) 2007 Christopher Lenz <cmlenz@gmx.de>.
"""

_HELP = """Usage: %(name)s [OPTION]

The %(name)s command runs the CouchDB Python view server.

The exit status is 0 for success or 1 for failure.

Options:

  --version             display version information and exit
  -h, --help            display a short help message and exit
  --json-module=<name>  set the JSON module to use ('simplejson', 'cjson',
                        or 'json' are supported)
  --log-file=<file>     name of the file to write log messages to, or '-' to
                        enable logging to the standard error stream
  --debug               enable debug logging; requires --log-file to be
                        specified

Report bugs via the web at <http://code.google.com/p/couchdb-python>.
"""

def main():
    """Command-line entry point for running the view server."""
    import getopt
    from couchdb import __version__ as VERSION

    try:
        option_list, argument_list = getopt.gnu_getopt(
            sys.argv[1:], 'h',
            ['version', 'help', 'json-module=', 'debug', 'log-file=']
        )

        message = None
        for option, value in option_list:
            if option in ('--version'):
                message = _VERSION % dict(name=os.path.basename(sys.argv[0]),
                                      version=VERSION)
            elif option in ('-h', '--help'):
                message = _HELP % dict(name=os.path.basename(sys.argv[0]))
            elif option in ('--json-module'):
                json.use(module=value)
            elif option in ('--debug'):
                log.setLevel(logging.DEBUG)
            elif option in ('--log-file'):
                if value == '-':
                    handler = logging.StreamHandler(sys.stderr)
                    handler.setFormatter(logging.Formatter(
                        ' -> [%(levelname)s] %(message)s'
                    ))
                else:
                    handler = logging.FileHandler(value)
                    handler.setFormatter(logging.Formatter(
                        '[%(asctime)s] [%(levelname)s] %(message)s'
                    ))
                log.addHandler(handler)
        if message:
            sys.stdout.write(message)
            sys.stdout.flush()
            sys.exit(0)

    except getopt.GetoptError as error:
        message = '%s\n\nTry `%s --help` for more information.\n' % (
            str(error), os.path.basename(sys.argv[0])
        )
        sys.stderr.write(message)
        sys.stderr.flush()
        sys.exit(1)

    sys.exit(run())

if __name__ == '__main__':
    main()
<|endoftext|>


<commit_before>#! /usr/bin/env python
# -*- coding: utf-8 -*-

# OpenFisca -- A versatile microsimulation software
# By: OpenFisca Team <contact@openfisca.fr>
#
# Copyright (C) 2011, 2012, 2013, 2014, 2015 OpenFisca Team
# https://github.com/openfisca
#
# This file is part of OpenFisca.
#
# OpenFisca is free software; you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as
# published by the Free Software Foundation, either version 3 of the
# License, or (at your option) any later version.
#
# OpenFisca is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program.  If not, see <http://www.gnu.org/licenses/>.

import logging

from openfisca_survey_manager.surveys import SurveyCollection

log = logging.getLogger(__name__)

from openfisca_france_data.temporary import TemporaryStore

temporary_store = TemporaryStore.create(file_name = "indirect_taxation_tmp")

#**************************************************************************************************************************
#* Etape n° 0-1-2 : IMPUTATION DE LOYERS POUR LES MENAGES PROPRIETAIRES
#**************************************************************************************************************************
#
#	if ${yearrawdata} == 1995 {
#		* L'enquête BdF 1995 ne contient pas de loyer imputés, donc Super-Roy les calcule!
#
#		use "$rawdatadir\socioscm.dta", clear
#		tempfile imput00
#
#		* On sélectionne les variables sur les ménages nécessaires au calcul des loyers imputés.
#		keep MENA STALOG SURFHAB CONFORT1 CONFORT2 CONFORT3 CONFORT4 ANCONS SITLOG PONDERRD NBPHAB RG CC
#		rename MENA ident_men
#		keep if PONDERRD != .
#		sort ident_men
#		des ident_men
#		save "`imput00'", replace
#
#		* On récupère les loyers réels dans la base de consommation.
#		tempfile loyers
#		use "`depenses'", clear
#		keep if posteCOICOP=="0411"
#		keep ident_men posteCOICOP depense
#		sort ident_men
#		merge 1:1 ident_men using "`imput00'"
#		tab _m
#		drop _m
#		rename depense loyer_reel
#		rename PONDERRD PONDMEN
#
#		* une indicatrice pour savoir si le loyer est connu et l'occupant est locataire
#		gen observe = (loyer_reel != . & inlist(STALOG,"3","4"))
#		gen loyer_impute = loyer_reel
#		gen maison_appart = (SITLOG == "1")
#		gen catsurf = 1 if SURF < 16
#		replace catsurf = 1 if SURF > 15 & SURF < 31
#		replace catsurf = 3 if SURF > 30 & SURF < 41
#		replace catsurf = 4 if SURF > 40 & SURF < 61
#		replace catsurf = 5 if SURF > 60 & SURF < 81
#		replace catsurf = 6 if SURF > 80 & SURF < 101
#		replace catsurf = 7 if SURF > 100 & SURF < 151
#		replace catsurf = 8 if SURF > 150
#		replace maison  = 0 if CC == "5" & catsurf == 1 & maison == 1
#		replace maison  = 0 if CC == "5" & catsurf == 3 & maison == 1
#		replace maison  = 0 if CC == "5" & catsurf == 8 & maison == 1
#		replace maison  = 0 if CC == "4" & catsurf == 1 & maison == 1
# 		sort ident_men
#
#		save "`loyers'", replace
#
#		hotdeck loyer_imput using "$rawdatadir\hotdeck", store by(catsurf CC maison_appart) keep(ident_men loyer_imput observe)
#		replace loyer_imput = . if observe == 1
#		use "$rawdatadir\hotdeck1.dta", clear
#		keep ident_men loyer_imput
#		sort ident_men
#		save "$rawdatadir\hotdeck1.dta", replace
#
#		use "`loyers'", clear
#		merge ident_men using "$rawdatadir\hotdeck1.dta", update
#		tab _m observe
#		drop _m
#
#		replace loyer_impute = 0 if observe == 1
#		gen imputation = (observe == 0)
#		label var imputation "Un loyer a été imputé (oui = 1, non = 0)"
#		rename STALOG stalog
#		keep ident_men loyer_imp imputation observe stalog
#		sort ident_men
#		save "`loyers'", replace
#		use "`depenses'", clear
#		sort ident_men posteCOICOP
#		merge m:1 ident_men using "`loyers'"
#		noisily: replace depense = 0 if posteCOICOP == "0411" & inlist(stalog,"1","2","5") & depense > 0 & depense != .
#		noisily: replace depense = 0 if posteCOICOP == "0411" & inlist(stalog,"1","2","5") & depense == .
#		replace depense = loyer_imp if posteCOICOP == "0421"  & observe == 0
#		replace depense = 0 		if posteCOICOP == "0421"  & observe == 1 & depense == .
#		drop observe stalog loyer_impute
#		tab  _m
#		drop _m
#	}
#
#
#
#			* POUR BdF 2000 ET 2005, ON UTILISE LES LOYERS IMPUTES CALCULES PAR L'INSEE
#
#
#	if ${yearrawdata} == 2000 {
#		tempfile loyers_imputes
#		* Garder les loyers imputés (disponibles dans la table sur les ménages)
#		use "$rawdatadir\menage.dta", clear
#		keep IDENT REV81
#		rename IDENT ident_men
#		gen posteCOICOP = "0421"
#		rename REV81 depense
#		sort ident_men posteCOICOP
#		save "`loyers_imputes'", replace
#		use "`depenses'", clear
#		sort ident_men posteCOICOP
#		merge 1:1 ident_men posteCOICOP using "`loyers_imputes'", update
#		tab _m
#		tab _m if posteCOICOP == "0421"
#		drop _m
#	}
#
#	if ${yearrawdata} == 2005 {
#		* Garder les loyers imputés (disponibles dans la table sur les ménages)
#		tempfile loyers_imputes
#		use "$rawdatadir\menage.dta", clear
#		keep ident_men rev801_d
#		gen posteCOICOP = "0421"
#		rename rev801_d depense
#		sort ident_men poste
#		save "`loyers_imputes'", replace
#		use "`depenses'", clear
#		sort ident_men posteCOICOP
#		merge 1:1 ident_men posteCOICOP using "`loyers_imputes'", update keepusing(depense)
#		tab _m
#		tab _m if posteCOICOP == "0421"
#		drop _m
#	}

def build_imputation_loyers_proprietaires(year = None):
    """Build menage consumption by categorie fiscale dataframe """

    assert year is not None
    # Load data
    bdf_survey_collection = SurveyCollection.load(collection = 'budget_des_familles')
    survey = bdf_survey_collection.surveys['budget_des_familles_{}'.format(year)]

    if year == 2000:
        loyers_imputes = survey.get_values(table = "menage", variables = ['ident', 'rev81'])
        loyers_imputes.rename(
            columns = {
                'ident': 'ident_men',
                'rev81': '0421',
                },
            inplace = True,
            )

    if year == 2005:
        loyers_imputes = survey.get_values(table = "menage")
        kept_variables = ['ident_men', 'rev801_d']
        loyers_imputes = loyers_imputes[kept_variables]
        loyers_imputes.rename(columns = {'rev801_d': '0421'}, inplace = True)

    # Joindre à la table des dépenses par COICOP
    loyers_imputes.set_index('ident_men', inplace = True)
    temporary_store['loyers_imputes_{}'.format(year)] = loyers_imputes
    depenses = temporary_store['depenses_{}'.format(year)]
    depenses = depenses.merge(loyers_imputes, left_index = True, right_index = True)

    # Sauvegarde de la base depenses mise à jour

#**************************************************************************************************************************
#* Etape n° 0-1-3 : SAUVER LES BASES DE DEPENSES HOMOGENEISEES DANS LE BON DOSSIER
#**************************************************************************************************************************
#
#
#	sort posteCOICOP
#	save "`depenses'", replace
#
#	keep  ident_men pondmen posteCOICOP poste13 grosposte depense description supplementaire nondurable semidurable durable servicenondurable servicedurable loyer depensenonconso
#	order ident_men pondmen posteCOICOP poste13 grosposte depense description supplementaire nondurable semidurable durable servicenondurable servicedurable loyer depensenonconso
#	save "${datadir}\dépenses_BdF.dta", replace

    # Save in temporary store
    temporary_store['depenses_bdf_{}'.format(year)] = depenses

if __name__ == '__main__':
    import sys
    import time
    logging.basicConfig(level = logging.INFO, stream = sys.stdout)
    deb = time.clock()
    year = 2005
    build_imputation_loyers_proprietaires(year = year)
    log.info("step 0_1_2_build_imputation_loyers_proprietaires duration is {}".format(time.clock() - deb))
<commit_msg>le fichier temporaire des dépenses en 2000 ne semblait pas se créer donc j'ai écrit une ligne en plus pour lui dire explicitement de le faire<commit_after>#! /usr/bin/env python
# -*- coding: utf-8 -*-

# OpenFisca -- A versatile microsimulation software
# By: OpenFisca Team <contact@openfisca.fr>
#
# Copyright (C) 2011, 2012, 2013, 2014, 2015 OpenFisca Team
# https://github.com/openfisca
#
# This file is part of OpenFisca.
#
# OpenFisca is free software; you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as
# published by the Free Software Foundation, either version 3 of the
# License, or (at your option) any later version.
#
# OpenFisca is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program.  If not, see <http://www.gnu.org/licenses/>.

import logging

from openfisca_survey_manager.surveys import SurveyCollection

log = logging.getLogger(__name__)

from openfisca_france_data.temporary import TemporaryStore

temporary_store = TemporaryStore.create(file_name = "indirect_taxation_tmp")

#**************************************************************************************************************************
#* Etape n° 0-1-2 : IMPUTATION DE LOYERS POUR LES MENAGES PROPRIETAIRES
#**************************************************************************************************************************
#
#	if ${yearrawdata} == 1995 {
#		* L'enquête BdF 1995 ne contient pas de loyer imputés, donc Super-Roy les calcule!
#
#		use "$rawdatadir\socioscm.dta", clear
#		tempfile imput00
#
#		* On sélectionne les variables sur les ménages nécessaires au calcul des loyers imputés.
#		keep MENA STALOG SURFHAB CONFORT1 CONFORT2 CONFORT3 CONFORT4 ANCONS SITLOG PONDERRD NBPHAB RG CC
#		rename MENA ident_men
#		keep if PONDERRD != .
#		sort ident_men
#		des ident_men
#		save "`imput00'", replace
#
#		* On récupère les loyers réels dans la base de consommation.
#		tempfile loyers
#		use "`depenses'", clear
#		keep if posteCOICOP=="0411"
#		keep ident_men posteCOICOP depense
#		sort ident_men
#		merge 1:1 ident_men using "`imput00'"
#		tab _m
#		drop _m
#		rename depense loyer_reel
#		rename PONDERRD PONDMEN
#
#		* une indicatrice pour savoir si le loyer est connu et l'occupant est locataire
#		gen observe = (loyer_reel != . & inlist(STALOG,"3","4"))
#		gen loyer_impute = loyer_reel
#		gen maison_appart = (SITLOG == "1")
#		gen catsurf = 1 if SURF < 16
#		replace catsurf = 1 if SURF > 15 & SURF < 31
#		replace catsurf = 3 if SURF > 30 & SURF < 41
#		replace catsurf = 4 if SURF > 40 & SURF < 61
#		replace catsurf = 5 if SURF > 60 & SURF < 81
#		replace catsurf = 6 if SURF > 80 & SURF < 101
#		replace catsurf = 7 if SURF > 100 & SURF < 151
#		replace catsurf = 8 if SURF > 150
#		replace maison  = 0 if CC == "5" & catsurf == 1 & maison == 1
#		replace maison  = 0 if CC == "5" & catsurf == 3 & maison == 1
#		replace maison  = 0 if CC == "5" & catsurf == 8 & maison == 1
#		replace maison  = 0 if CC == "4" & catsurf == 1 & maison == 1
# 		sort ident_men
#
#		save "`loyers'", replace
#
#		hotdeck loyer_imput using "$rawdatadir\hotdeck", store by(catsurf CC maison_appart) keep(ident_men loyer_imput observe)
#		replace loyer_imput = . if observe == 1
#		use "$rawdatadir\hotdeck1.dta", clear
#		keep ident_men loyer_imput
#		sort ident_men
#		save "$rawdatadir\hotdeck1.dta", replace
#
#		use "`loyers'", clear
#		merge ident_men using "$rawdatadir\hotdeck1.dta", update
#		tab _m observe
#		drop _m
#
#		replace loyer_impute = 0 if observe == 1
#		gen imputation = (observe == 0)
#		label var imputation "Un loyer a été imputé (oui = 1, non = 0)"
#		rename STALOG stalog
#		keep ident_men loyer_imp imputation observe stalog
#		sort ident_men
#		save "`loyers'", replace
#		use "`depenses'", clear
#		sort ident_men posteCOICOP
#		merge m:1 ident_men using "`loyers'"
#		noisily: replace depense = 0 if posteCOICOP == "0411" & inlist(stalog,"1","2","5") & depense > 0 & depense != .
#		noisily: replace depense = 0 if posteCOICOP == "0411" & inlist(stalog,"1","2","5") & depense == .
#		replace depense = loyer_imp if posteCOICOP == "0421"  & observe == 0
#		replace depense = 0 		if posteCOICOP == "0421"  & observe == 1 & depense == .
#		drop observe stalog loyer_impute
#		tab  _m
#		drop _m
#	}
#
#
#
#			* POUR BdF 2000 ET 2005, ON UTILISE LES LOYERS IMPUTES CALCULES PAR L'INSEE
#
#
#	if ${yearrawdata} == 2000 {
#		tempfile loyers_imputes
#		* Garder les loyers imputés (disponibles dans la table sur les ménages)
#		use "$rawdatadir\menage.dta", clear
#		keep IDENT REV81
#		rename IDENT ident_men
#		gen posteCOICOP = "0421"
#		rename REV81 depense
#		sort ident_men posteCOICOP
#		save "`loyers_imputes'", replace
#		use "`depenses'", clear
#		sort ident_men posteCOICOP
#		merge 1:1 ident_men posteCOICOP using "`loyers_imputes'", update
#		tab _m
#		tab _m if posteCOICOP == "0421"
#		drop _m
#	}
#
#	if ${yearrawdata} == 2005 {
#		* Garder les loyers imputés (disponibles dans la table sur les ménages)
#		tempfile loyers_imputes
#		use "$rawdatadir\menage.dta", clear
#		keep ident_men rev801_d
#		gen posteCOICOP = "0421"
#		rename rev801_d depense
#		sort ident_men poste
#		save "`loyers_imputes'", replace
#		use "`depenses'", clear
#		sort ident_men posteCOICOP
#		merge 1:1 ident_men posteCOICOP using "`loyers_imputes'", update keepusing(depense)
#		tab _m
#		tab _m if posteCOICOP == "0421"
#		drop _m
#	}

def build_imputation_loyers_proprietaires(year = None):
    """Build menage consumption by categorie fiscale dataframe """

    assert year is not None
    # Load data
    bdf_survey_collection = SurveyCollection.load(collection = 'budget_des_familles')
    survey = bdf_survey_collection.surveys['budget_des_familles_{}'.format(year)]

    if year == 2000:
        loyers_imputes = survey.get_values(table = "menage", variables = ['ident', 'rev81'])
        loyers_imputes.rename(
            columns = {
                'ident': 'ident_men',
                'rev81': '0421',
                },
            inplace = True,
            )
        depenses = survey.get_values(table = 'depmen')

    if year == 2005:
        loyers_imputes = survey.get_values(table = "menage")
        kept_variables = ['ident_men', 'rev801_d']
        loyers_imputes = loyers_imputes[kept_variables]
        loyers_imputes.rename(columns = {'rev801_d': '0421'}, inplace = True)

    # Joindre à la table des dépenses par COICOP
    loyers_imputes.set_index('ident_men', inplace = True)
    temporary_store['loyers_imputes_{}'.format(year)] = loyers_imputes
    temporary_store['depenses_{}'.format(year)] = depenses
    depenses = depenses.merge(loyers_imputes, left_index = True, right_index = True)

    # Sauvegarde de la base depenses mise à jour

#**************************************************************************************************************************
#* Etape n° 0-1-3 : SAUVER LES BASES DE DEPENSES HOMOGENEISEES DANS LE BON DOSSIER
#**************************************************************************************************************************
#
#
#	sort posteCOICOP
#	save "`depenses'", replace
#
#	keep  ident_men pondmen posteCOICOP poste13 grosposte depense description supplementaire nondurable semidurable durable servicenondurable servicedurable loyer depensenonconso
#	order ident_men pondmen posteCOICOP poste13 grosposte depense description supplementaire nondurable semidurable durable servicenondurable servicedurable loyer depensenonconso
#	save "${datadir}\dépenses_BdF.dta", replace

    # Save in temporary store
    temporary_store['depenses_bdf_{}'.format(year)] = depenses

if __name__ == '__main__':
    import sys
    import time
    logging.basicConfig(level = logging.INFO, stream = sys.stdout)
    deb = time.clock()
    year = 2005
    build_imputation_loyers_proprietaires(year = year)
    log.info("step 0_1_2_build_imputation_loyers_proprietaires duration is {}".format(time.clock() - deb))
<|endoftext|>


<commit_before>
__version__ = '0.20.0'
NEWEST_CONAN_SUPPORTED = "1.9"
<commit_msg>0.20.2<commit_after>
__version__ = '0.20.2'
NEWEST_CONAN_SUPPORTED = "1.9"
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
"""
@file
@brief  quelques fonctions à propos de la première séance

"""

import datetime

def commentaire_accentues():
    """
    L'aide de cette fonction contient assuréments des accents.

    .. faqref::
        :tag: python
        :title: Python n'accepte pas les accents

        .. index:: accent, accents, utf8, encoding

        Le langage Python a été conçu en langage anglais. Dès qu'on on ajoute un caractère
        qui ne fait pas partie de l'alphabet anglais (ponctuation comprise), il déclenche une erreur :

        ::

            File "faq_cvxopt.py", line 3
            SyntaxError: Non-UTF-8 code starting with '\xe8' in file faq_cvxopt.py on line 4, but no encoding declared;
                see http://python.org/dev/peps/pep-0263/ for details

        Pour la résoudre, il faut dire à l'interpréteur que des caractères non anglais peuvent apparaître
        et écrire sur la première ligne du programme :

        ::

            # -*- coding: latin-1 -*-

        Ou pour tout caractère y compris chinois :

        ::

            # -*- coding: utf-8 -*-

        Si vous utilisez l'éditeur `SciTE <http://www.scintilla.org/SciTE.html>`_ sous Windows,
        après avoir ajouté cette ligne avec l'encoding `utf-8`,
        il est conseillé de fermer le fichier puis de le réouvrir.
        SciTE le traitera différemment.

        **L'encodage ``utf-8`` est la norme sur Internet.** C'est pourquoi il est préférable d'utiliser celui-ci pour
        partager son code via une page Web.
    """
    pass

def dix_entiers_carre():
    """
    fait la somme des dix premiers entiers au carré

    :returns: nombre réel

    .. faqref::
        :tag: python
        :title: Quelle est la différence entre return et print ?

        La fonction ``print`` sert à afficher un résultat sur la sortie standard.
        Elle peut être utilisée à tout moment
        mais elle n'a pas d'impact sur le déroulement programme. Le mot-clé ``return``
        n'est utilisé que dans une fonction. Lorsque le programme rencontre
        une instruction commençant par ``return``, il quitte la fonction
        et transmet le résultat à l'instruction qui a appelé la fonction.
        La fonction ``print`` ne modifie pas votre algorithme. La fonction ``return``
        spécifie le résultat de votre fonction : elle modifie l'algorithme.

    .. exref::
        :title: calcul de la somme des dix premiers entiers au carré
        :tag: Base

        Ce calcul simple peut s'écrire de diffèrentes manières.

        ::

            s = 0
            for i in range(1,11):
                s += i**2

        D'une façon abrégée :

        ::

            s = sum ( [ i**2 for i in range(1,11) ] )
    """
    s = 0
    for i in range(1, 11):
        s += i ** 2
    return s

def racine_carree(x):
    """
    retourne la racine carrée d'un nombre

    @param  x       nombre
    @return         racine carrée
    """
    return x ** 0.5

def repetition_a_eviter(serie):
    """
    Une répétition à éviter.

    .. exref::
        :title: Eviter d'effectuer le même appel deux fois
        :tag: Base -

        Dans cette fonction on calcule la variance d'une série d'observations.

        ::

            def moyenne(serie):
                return sum(serie) / len(serie)

            def variance_a_eviter(serie):
                s = 0
                for obs in serie :
                    s += (obs-moyenne(serie))**2
                return s / len(serie)

        La fonction ``variance_a_eviter`` appelle la fonction ``moyenne`` à chaque passage
        dans la boucle. Or, rien ne change d'un passage à l'autre. Il vaut mieux stocker
        le résultat dans une variable :

    ::

        def moyenne(serie):
            return sum(serie) / len(serie)

        def variance(serie):
            s = 0
            moy = moyenne(serie)
            for obs in serie :
                s += (obs-moy)**2
            return s / len(serie)

    Le coût de la variance passe alors d'un coût en :math:`O(n^2)` à :math:`O(n)`.
    Ce n'est pas le seul endroit où cette erreur survient. Dans le code suivant,
    on appelle deux fois la fonction ``major`` avec le même argument.
    C'est à éviter.

    ::

        meilleur = major(data)[0]  # retourne ("quelque chose", True)
        if major(data)[1]:
            return {"leaf":guess}

    @endexample
    """

    def moyenne(serie):
        return sum(serie) / len(serie)

    def variance_a_eviter(serie):
        s = 0
        for obs in serie:
            s += (obs - moyenne(serie)) ** 2
        return s / len(serie)

    def variance(serie):
        s = 0
        moy = moyenne(serie)
        for obs in serie:
            s += (obs - moy) ** 2
        return s / len(serie)

    return variance(serie)

def dictionnaire_modifie_dans_la_boucle():
    """
    Dictionnaires, listes modifiés dans la boucle qui les parcourt.

    .. exref::
        :title: Modifier un dictionnaire en le parcourant
        :tag: Base -

        Il faut éviter de modifier un container lorsqu'on le parcourt.
        Lorsqu'on supprime un élément d'un dictionnaire, la structure de celui-ci
        s'en trouve modifiée et affecte la boucle qui le parcourt. La boucle parcourt
        toujours l'ancienne structure du dictionnaire, celle qui existait au début
        au début de la boucle.

        ::

            d = { k: k for k in range(10) }
            for k, v in d.items():
                if k == 4 :
                    del d[k]

        En Python, cela produit l'erreur qui suit mais d'autres langages ne préviennent
        pas (C++) et cela aboutit à une erreur qui intervient plus tard dans le code
        (comme une valeur numérique inattendue).

    ::

        Traceback (most recent call last):
        File "session1.py", line 176, in <module>
            l = liste_modifie_dans_la_boucle()
        File "session1.py", line 169, in liste_modifie_dans_la_boucle
            for k,v in d.items():
        RuntimeError: dictionary changed size during iteration

    Il faut pour éviter cela stocker les éléments qu'on veut modifier pour les supprimer
    ensuite.

    ::

        d = { k:k for k in l }
        rem = [ ]
        for k,v in d.items():
            if k == 4 :
                rem.append(k)
        for r in rem :
            del d[r]

    Même si Python autorise cela pour les listes,
    il est conseillé de s'en abstenir ainsi que pour tout type d'objets qui en contient d'autres.
    C'est une habitude qui vous servira pour la plupart des autres langages.

    @endexample
    """
    li = [0, 1, 2, 3, 4, 5, 6]
    for i in li:
        if i == 2:
            li.remove(3)

    d = {k: k for k in li}
    rem = []
    for k, v in d.items():
        if k == 4:
            rem.append(k)
    for r in rem:
        del d[r]

    return li, d

def str2date(s, format="%d/%m/%Y"):
    """
    convertit une chaîne de caractères en datetime

    @param      s       chaîne de caractères
    @param      format  format de la conversion

    .. exref::
        :title: conversion d'une chaîne de caractère en datetime
        :tag: Base

        C'est le genre de fonction qu'on n'utilise pas souvent mais qu'on peine à retrouver
        lorsqu'on en a besoin.
        Il faut utiliser la fonction `strftime <https://docs.python.org/3/library/datetime.html#strftime-and-strptime-behavior>`_.

        ::

            import datetime
            dt = datetime.datetime.strptime ("16/01/2014", "%d/%m/%Y")
    """
    return datetime.datetime.strptime(s, format)
<commit_msg>documentation<commit_after># -*- coding: utf-8 -*-
"""
@file
@brief  quelques fonctions à propos de la première séance

"""

import datetime

def commentaire_accentues():
    """
    L'aide de cette fonction contient assuréments des accents.

    .. faqref::
        :tag: python
        :title: Python n'accepte pas les accents

        .. index:: accent, accents, utf8, encoding

        Le langage Python a été conçu en langage anglais. Dès qu'on on ajoute un caractère
        qui ne fait pas partie de l'alphabet anglais (ponctuation comprise), il déclenche une erreur :

        ::

            File "faq_cvxopt.py", line 3
            SyntaxError: Non-UTF-8 code starting with '\xe8' in file faq_cvxopt.py on line 4, but no encoding declared;
                see http://python.org/dev/peps/pep-0263/ for details

        Pour la résoudre, il faut dire à l'interpréteur que des caractères non anglais peuvent apparaître
        et écrire sur la première ligne du programme :

        ::

            # -*- coding: latin-1 -*-

        Ou pour tout caractère y compris chinois :

        ::

            # -*- coding: utf-8 -*-

        Si vous utilisez l'éditeur `SciTE <http://www.scintilla.org/SciTE.html>`_ sous Windows,
        après avoir ajouté cette ligne avec l'encoding `utf-8`,
        il est conseillé de fermer le fichier puis de le réouvrir.
        SciTE le traitera différemment.

        **L'encodage ``utf-8`` est la norme sur Internet.** C'est pourquoi il est préférable d'utiliser celui-ci pour
        partager son code via une page Web.
    """
    pass

def dix_entiers_carre():
    """
    fait la somme des dix premiers entiers au carré

    :returns: nombre réel

    .. faqref::
        :tag: python
        :title: Quelle est la différence entre return et print ?

        La fonction ``print`` sert à afficher un résultat sur la sortie standard.
        Elle peut être utilisée à tout moment
        mais elle n'a pas d'impact sur le déroulement programme. Le mot-clé ``return``
        n'est utilisé que dans une fonction. Lorsque le programme rencontre
        une instruction commençant par ``return``, il quitte la fonction
        et transmet le résultat à l'instruction qui a appelé la fonction.
        La fonction ``print`` ne modifie pas votre algorithme. La fonction ``return``
        spécifie le résultat de votre fonction : elle modifie l'algorithme.

    .. exref::
        :title: calcul de la somme des dix premiers entiers au carré
        :tag: Base

        Ce calcul simple peut s'écrire de diffèrentes manières.

        ::

            s = 0
            for i in range(1,11):
                s += i**2

        D'une façon abrégée :

        ::

            s = sum ( [ i**2 for i in range(1,11) ] )
    """
    s = 0
    for i in range(1, 11):
        s += i ** 2
    return s

def racine_carree(x):
    """
    retourne la racine carrée d'un nombre

    @param  x       nombre
    @return         racine carrée
    """
    return x ** 0.5

def repetition_a_eviter(serie):
    """
    Une répétition à éviter.

    .. exref::
        :title: Eviter d'effectuer le même appel deux fois
        :tag: Base -

        Dans cette fonction on calcule la variance d'une série d'observations.

        ::

            def moyenne(serie):
                return sum(serie) / len(serie)

            def variance_a_eviter(serie):
                s = 0
                for obs in serie :
                    s += (obs-moyenne(serie))**2
                return s / len(serie)

        La fonction ``variance_a_eviter`` appelle la fonction ``moyenne`` à chaque passage
        dans la boucle. Or, rien ne change d'un passage à l'autre. Il vaut mieux stocker
        le résultat dans une variable :

        ::

            def moyenne(serie):
                return sum(serie) / len(serie)

            def variance(serie):
                s = 0
                moy = moyenne(serie)
                for obs in serie :
                    s += (obs-moy)**2
                return s / len(serie)

        Le coût de la variance passe alors d'un coût en :math:`O(n^2)` à :math:`O(n)`.
        Ce n'est pas le seul endroit où cette erreur survient. Dans le code suivant,
        on appelle deux fois la fonction ``major`` avec le même argument.
        C'est à éviter.

        ::

            meilleur = major(data)[0]  # retourne ("quelque chose", True)
            if major(data)[1]:
                return {"leaf":guess}
    """

    def moyenne(serie):
        return sum(serie) / len(serie)

    def variance_a_eviter(serie):
        s = 0
        for obs in serie:
            s += (obs - moyenne(serie)) ** 2
        return s / len(serie)

    def variance(serie):
        s = 0
        moy = moyenne(serie)
        for obs in serie:
            s += (obs - moy) ** 2
        return s / len(serie)

    return variance(serie)

def dictionnaire_modifie_dans_la_boucle():
    """
    Dictionnaires, listes modifiés dans la boucle qui les parcourt.

    .. exref::
        :title: Modifier un dictionnaire en le parcourant
        :tag: Base -

        Il faut éviter de modifier un container lorsqu'on le parcourt.
        Lorsqu'on supprime un élément d'un dictionnaire, la structure de celui-ci
        s'en trouve modifiée et affecte la boucle qui le parcourt. La boucle parcourt
        toujours l'ancienne structure du dictionnaire, celle qui existait au début
        au début de la boucle.

        ::

            d = { k: k for k in range(10) }
            for k, v in d.items():
                if k == 4 :
                    del d[k]

        En Python, cela produit l'erreur qui suit mais d'autres langages ne préviennent
        pas (C++) et cela aboutit à une erreur qui intervient plus tard dans le code
        (comme une valeur numérique inattendue).

        ::

            Traceback (most recent call last):
            File "session1.py", line 176, in <module>
                l = liste_modifie_dans_la_boucle()
            File "session1.py", line 169, in liste_modifie_dans_la_boucle
                for k,v in d.items():
            RuntimeError: dictionary changed size during iteration

        Il faut pour éviter cela stocker les éléments qu'on veut modifier pour les supprimer
        ensuite.

        ::

            d = { k:k for k in l }
            rem = [ ]
            for k,v in d.items():
                if k == 4 :
                    rem.append(k)
            for r in rem :
                del d[r]

        Même si Python autorise cela pour les listes,
        il est conseillé de s'en abstenir ainsi que pour tout type d'objets qui en contient d'autres.
        C'est une habitude qui vous servira pour la plupart des autres langages.
    """
    li = [0, 1, 2, 3, 4, 5, 6]
    for i in li:
        if i == 2:
            li.remove(3)

    d = {k: k for k in li}
    rem = []
    for k, v in d.items():
        if k == 4:
            rem.append(k)
    for r in rem:
        del d[r]

    return li, d

def str2date(s, format="%d/%m/%Y"):
    """
    convertit une chaîne de caractères en datetime

    @param      s       chaîne de caractères
    @param      format  format de la conversion

    .. exref::
        :title: conversion d'une chaîne de caractère en datetime
        :tag: Base

        C'est le genre de fonction qu'on n'utilise pas souvent mais qu'on peine à retrouver
        lorsqu'on en a besoin.
        Il faut utiliser la fonction `strftime <https://docs.python.org/3/library/datetime.html#strftime-and-strptime-behavior>`_.

        ::

            import datetime
            dt = datetime.datetime.strptime ("16/01/2014", "%d/%m/%Y")
    """
    return datetime.datetime.strptime(s, format)
<|endoftext|>


<commit_before>import argparse
import requests
import os.path
import json
import time
from future.utils import iteritems

# Confirm program arguments
def confirmArguments():
	parser = argparse.ArgumentParser(description='Program to convert \
		Json formatted file from Swagger editor for Elastic Search indexing.')
	parser.add_argument('filename', nargs='?')
	args = parser.parse_args()

	if args.filename is not None:
	    input_file = args.filename
	    return input_file
	else:
	    print('Usage: python convertJson4ESindexing.py [filename or uri]')
	    exit()

# Get file contents
def getFileContents(filename):
	# Check if URL provided
	if(filename.startswith("http")):
		print('DEBUG: Get file from URL')
		r = requests.get(filename)
		if r.status_code == 200:
			print('DEBUG: Got the file')
			# file_contents = r.text
			json_file_contents = r.json()
			return json_file_contents
		else:
			print('ERROR: Unable to access URL. Failed with status code ', r.status_code)
			exit()
	else:
		# Check if local file exists with file name provided
		if(os.path.isfile(filename)):
			print('DEBUG: Local file exists!')
			# Read in file contents
			f = open(filename, 'r')
			file_contents = f.read()
			if(file_contents == '' or os.stat(filename).st_size == 0):
				print('ERROR: No content in ', filename)
				exit()
			else:
				print('DEBUG: File contents exist')
				json_file_contents = json.loads(file_contents)
				return json_file_contents
		else:
			print('ERROR: No file found')
			exit()

# Convert Swagger JSON file
def convert_file(the_file_contents):
	converted_data = {}
	operations_list = []

	# Get each first level path key
	for key in the_file_contents.keys():
		# Modify object in paths
		if(key == "paths"):
			for pathname_key in the_file_contents[key]:
				print("\n")
				print("** Pathname: ", pathname_key)
				path_obj = {}
				for method_key in the_file_contents[key][pathname_key]:
					print("\n")
					print("** Method name: ", method_key)

					# Convert path object
					path_obj["httpOperation"] = method_key
					path_obj["path"] = pathname_key

					# Convert response object
					for stuff in the_file_contents[key][pathname_key][method_key]:
						response_list = []
						if(stuff == "responses"):
							response_obj = {}
							for response in the_file_contents[key][pathname_key][method_key][stuff]:
								response_obj["httpCode"] = response
								for response_item in the_file_contents[key][pathname_key][method_key][stuff][response]:
									response_obj[response_item] = the_file_contents[key][pathname_key][method_key][stuff][response].get(response_item)
							response_list.append(response_obj)
							path_obj["responses"] = response_list
						else:
							path_obj[stuff] = the_file_contents[key][pathname_key][method_key].get(stuff)

				operations_list.append(path_obj)
			converted_data["operations"] = operations_list
		else:
			cd = converted_data[key] = the_file_contents.get(key)

	# Print converted data
	# print(json.dumps(converted_data, sort_keys=True, indent=4, separators=(',', ': ') ))

	# adding a meta field
	meta = {'timestamp': time.ctime()}
	converted_data['meta'] = meta

	return converted_data

# Convert ES indexed file format to Swagger format
def convert_to_swagger(the_file_contents):
	swagger_formatted_data = {}

	# Get each first level path key
	for key in the_file_contents.keys():
		# Modify object in operations
		if(key == "operations"):
			# Iterate through objects and re-format
			path_obj = {}
			for operations_dict in the_file_contents[key]:
				http_obj = {}
				operations_obj = {}
				# print("\n** New operations object **")
				path_key = operations_dict['path']  # E.g. /gene
				http_operation_key = operations_dict['httpOperation'] # E.g. post or get

				for key,value in iteritems(operations_dict):
					# print(key)
					if(key == "path" or key == "httpOperation" or
						key == "parameters"):
						continue
					elif(key == "responses"):
						# Process responses
						http_response_obj = {}
						for response in value:
							http_code = response['httpCode']
							response_obj = {}
							# print("** Response: ", http_code)
							# print("Response-Value: ", response)
							for k,v in iteritems(response):
								# print("K-V:", k,v)
								if(k != 'httpCode'):
									response_obj[k] = v
							# print("RespObj: " + json.dumps(response_obj, sort_keys=True, indent=4, separators=(',', ': ')))

						http_response_obj[http_code] = response_obj
						operations_obj[key] = http_response_obj
						# print("Http Code: " + json.dumps(operations_obj, sort_keys=True, indent=4, separators=(',', ': ')))
					else:
						operations_obj[key] = value

				# Add to object
				http_obj[http_operation_key] = operations_obj

				# Check if path_key in path_obj
				if(path_key) in path_obj:
					path_obj[path_key].update(http_obj)
					# print(json.dumps(path_obj, sort_keys=True, indent=4, separators=(',', ': ')))
				else:
					path_obj[path_key] = http_obj

			# Add path object to data file
			swagger_formatted_data["paths"] = path_obj
		else:
			swagger_formatted_data[key] = the_file_contents.get(key)
	print(json.dumps(swagger_formatted_data, sort_keys=True, indent=4, separators=(',', ': ')))

## Main Program ##
if __name__ == '__main__':
	filename = confirmArguments()

	the_file_contents = getFileContents(filename)

	converted_data = convert_file(the_file_contents)

	swagger_formatted_data = convert_to_swagger(the_file_contents)

<commit_msg>remove timestamp from output, remove debug print statements<commit_after>import argparse
import requests
import os.path
import json
import time
from future.utils import iteritems

# Confirm program arguments
def confirmArguments():
	parser = argparse.ArgumentParser(description='Program to convert \
		Json formatted file from Swagger editor for Elastic Search indexing.')
	parser.add_argument('filename', nargs='?')
	args = parser.parse_args()

	if args.filename is not None:
	    input_file = args.filename
	    return input_file
	else:
	    print('Usage: python convertJson4ESindexing.py [filename or uri]')
	    exit()

# Get file contents
def getFileContents(filename):
	# Check if URL provided
	if(filename.startswith("http")):
		print('DEBUG: Get file from URL')
		r = requests.get(filename)
		if r.status_code == 200:
			print('DEBUG: Got the file')
			# file_contents = r.text
			json_file_contents = r.json()
			return json_file_contents
		else:
			print('ERROR: Unable to access URL. Failed with status code ', r.status_code)
			exit()
	else:
		# Check if local file exists with file name provided
		if(os.path.isfile(filename)):
			print('DEBUG: Local file exists!')
			# Read in file contents
			f = open(filename, 'r')
			file_contents = f.read()
			if(file_contents == '' or os.stat(filename).st_size == 0):
				print('ERROR: No content in ', filename)
				exit()
			else:
				print('DEBUG: File contents exist')
				json_file_contents = json.loads(file_contents)
				return json_file_contents
		else:
			print('ERROR: No file found')
			exit()

# Convert Swagger JSON file
def convert_file(the_file_contents):
	es_formatted_data = {}
	operations_list = []

	# Get each first level path key
	for key in the_file_contents.keys():
		# Modify object in paths
		if(key == "paths"):
			for pathname_key in the_file_contents[key]:
				# print("\n")
				# print("** Pathname: ", pathname_key)
				path_obj = {}
				for method_key in the_file_contents[key][pathname_key]:
					# print("\n")
					# print("** Method name: ", method_key)

					# Convert path object
					path_obj["httpOperation"] = method_key
					path_obj["path"] = pathname_key

					# Convert response object
					for stuff in the_file_contents[key][pathname_key][method_key]:
						response_list = []
						if(stuff == "responses"):
							response_obj = {}
							for response in the_file_contents[key][pathname_key][method_key][stuff]:
								response_obj["httpCode"] = response
								for response_item in the_file_contents[key][pathname_key][method_key][stuff][response]:
									response_obj[response_item] = the_file_contents[key][pathname_key][method_key][stuff][response].get(response_item)
							response_list.append(response_obj)
							path_obj["responses"] = response_list
						else:
							path_obj[stuff] = the_file_contents[key][pathname_key][method_key].get(stuff)

				operations_list.append(path_obj)
			es_formatted_data["operations"] = operations_list
		else:
			cd = es_formatted_data[key] = the_file_contents.get(key)

	# adding a meta field
	meta = {'timestamp': time.ctime()}
	es_formatted_data['meta'] = meta

	return es_formatted_data

# Convert ES indexed file format to Swagger format
def convert_to_swagger(the_file_contents):
	swagger_formatted_data = {}

	# Get each first level path key
	for key in the_file_contents.keys():
		# Modify object in operations
		if(key == "operations"):
			# Iterate through objects and re-format
			path_obj = {}
			for operations_dict in the_file_contents[key]:
				http_obj = {}
				operations_obj = {}
				# print("\n** New operations object **")
				path_key = operations_dict['path']  # E.g. /gene
				http_operation_key = operations_dict['httpOperation'] # E.g. post or get

				for key,value in iteritems(operations_dict):
					if(key == "path" or key == "httpOperation"):
						continue
					elif(key == "responses"):
						# Process responses
						http_response_obj = {}
						for response in value:
							http_code = response['httpCode']
							response_obj = {}
							for k,v in iteritems(response):
								if(k != 'httpCode'):
									response_obj[k] = v

						http_response_obj[http_code] = response_obj
						operations_obj[key] = http_response_obj
					else:
						operations_obj[key] = value

				# Add to object
				http_obj[http_operation_key] = operations_obj

				# Check if path_key in path_obj
				if(path_key) in path_obj:
					path_obj[path_key].update(http_obj)
				else:
					path_obj[path_key] = http_obj

			# Add path object to data file
			swagger_formatted_data["paths"] = path_obj
		# Do not add timestamp into converted file
		elif(key == "meta"):
			continue
		else:
			swagger_formatted_data[key] = the_file_contents.get(key)

	return swagger_formatted_data

## Main Program ##
if __name__ == '__main__':
	filename = confirmArguments()

	the_file_contents = getFileContents(filename)

	es_formatted_data = convert_file(the_file_contents)

	swagger_formatted_data = convert_to_swagger(the_file_contents)

<|endoftext|>


<commit_before>import chainerx
from chainerx import _docs

def set_docs():
    _docs_creation()
    _docs_indexing()
    _docs_linalg()
    _docs_logic()
    _docs_manipulation()
    _docs_math()
    _docs_statistics()
    _docs_connection()
    _docs_normalization()
    _docs_pooling()

def _docs_creation():
    pass

def _docs_indexing():
    pass

def _docs_linalg():
    pass

def _docs_logic():
    _docs.set_doc(
        chainerx.logical_not,
        """logical_not(x)
Returns an array of NOT x element-wise.

Args:
    x (~chainerx.ndarray): Input array.

Returns:
    :class:`~chainerx.ndarray`: Output array of type bool.

Note:
    During backpropagation, this function does not propagate gradients.

.. seealso:: :func:`numpy.logical_not`
""")

    _docs.set_doc(
        chainerx.greater,
        """greater(x1, x2)
Returns an array of (x1 > x2) element-wise.

Args:
    x1 (~chainerx.ndarray): Input array.
    x2 (~chainerx.ndarray): Input array.

Returns:
    :class:`~chainerx.ndarray`: Output array of type bool.

Note:
    During backpropagation, this function does not propagate gradients.

.. seealso:: :func:`numpy.greater`
""")

    _docs.set_doc(
        chainerx.greater_equal,
        """greater_equal(x1, x2)
Returns an array of (x1 >= x2) element-wise.

Args:
    x1 (~chainerx.ndarray): Input array.
    x2 (~chainerx.ndarray): Input array.

Returns:
    :class:`~chainerx.ndarray`: Output array of type bool.

Note:
    During backpropagation, this function does not propagate gradients.

.. seealso:: :func:`numpy.greater_equal`
""")

    _docs.set_doc(
        chainerx.less,
        """less(x1, x2)
Returns an array of (x1 < x2) element-wise.

Args:
    x1 (~chainerx.ndarray): Input array.
    x2 (~chainerx.ndarray): Input array.

Returns:
    :class:`~chainerx.ndarray`: Output array of type bool.

Note:
    During backpropagation, this function does not propagate gradients.

.. seealso:: :func:`numpy.less`
""")

    _docs.set_doc(
        chainerx.less_equal,
        """less_equal(x1, x2)
Returns an array of (x1 <= x2) element-wise.

Args:
    x1 (~chainerx.ndarray): Input array.
    x2 (~chainerx.ndarray): Input array.

Returns:
    :class:`~chainerx.ndarray`: Output array of type bool.

Note:
    During backpropagation, this function does not propagate gradients.

.. seealso:: :func:`numpy.less_equal`
""")

    _docs.set_doc(
        chainerx.equal,
        """equal(x1, x2)
Returns an array of (x1 == x2) element-wise.

Args:
    x1 (~chainerx.ndarray): Input array.
    x2 (~chainerx.ndarray): Input array.

Returns:
    :class:`~chainerx.ndarray`: Output array of type bool.

Note:
    During backpropagation, this function does not propagate gradients.

.. seealso:: :func:`numpy.equal`
""")

    _docs.set_doc(
        chainerx.not_equal,
        """not_equal(x1, x2)
Returns an array of (x1 != x2) element-wise.

Args:
    x1 (~chainerx.ndarray): Input array.
    x2 (~chainerx.ndarray): Input array.

Returns:
    :class:`~chainerx.ndarray`: Output array of type bool.

Note:
    During backpropagation, this function does not propagate gradients.

.. seealso:: :func:`numpy.not_equal`
""")

def _docs_manipulation():
    _docs.set_doc(
        chainerx.reshape,
        """reshape(a, newshape)
Returns a reshaped array.

Args:
    a (~chainerx.ndarray): Array to be reshaped.
    newshape (int or tuple of ints): The new shape of the array to return.
        If it is an integer, then it is treated as a tuple of length one.
        It should be compatible with ``a.size``. One of the elements can be
        -1, which is automatically replaced with the appropriate value to
        make the shape compatible with ``a.size``.

Returns:
    :class:`~chainerx.ndarray`: A reshaped view of ``a`` if possible,
    otherwise a copy.

Note:
    During backpropagation, this function propagates the gradient of the
    output array to the input array ``a``.

.. seealso:: :func:`numpy.reshape`
""")

def _docs_math():
    pass

def _docs_statistics():
    pass

def _docs_connection():
    pass

def _docs_normalization():
    pass

def _docs_pooling():
    pass
<commit_msg>fix numpy links<commit_after>import chainerx
from chainerx import _docs

def set_docs():
    _docs_creation()
    _docs_indexing()
    _docs_linalg()
    _docs_logic()
    _docs_manipulation()
    _docs_math()
    _docs_statistics()
    _docs_connection()
    _docs_normalization()
    _docs_pooling()

def _docs_creation():
    pass

def _docs_indexing():
    pass

def _docs_linalg():
    pass

def _docs_logic():
    _docs.set_doc(
        chainerx.logical_not,
        """logical_not(x)
Returns an array of NOT x element-wise.

Args:
    x (~chainerx.ndarray): Input array.

Returns:
    :class:`~chainerx.ndarray`: Output array of type bool.

Note:
    During backpropagation, this function does not propagate gradients.

.. seealso:: :data:`numpy.logical_not`
""")

    _docs.set_doc(
        chainerx.greater,
        """greater(x1, x2)
Returns an array of (x1 > x2) element-wise.

Args:
    x1 (~chainerx.ndarray): Input array.
    x2 (~chainerx.ndarray): Input array.

Returns:
    :class:`~chainerx.ndarray`: Output array of type bool.

Note:
    During backpropagation, this function does not propagate gradients.

.. seealso:: :data:`numpy.greater`
""")

    _docs.set_doc(
        chainerx.greater_equal,
        """greater_equal(x1, x2)
Returns an array of (x1 >= x2) element-wise.

Args:
    x1 (~chainerx.ndarray): Input array.
    x2 (~chainerx.ndarray): Input array.

Returns:
    :class:`~chainerx.ndarray`: Output array of type bool.

Note:
    During backpropagation, this function does not propagate gradients.

.. seealso:: :data:`numpy.greater_equal`
""")

    _docs.set_doc(
        chainerx.less,
        """less(x1, x2)
Returns an array of (x1 < x2) element-wise.

Args:
    x1 (~chainerx.ndarray): Input array.
    x2 (~chainerx.ndarray): Input array.

Returns:
    :class:`~chainerx.ndarray`: Output array of type bool.

Note:
    During backpropagation, this function does not propagate gradients.

.. seealso:: :data:`numpy.less`
""")

    _docs.set_doc(
        chainerx.less_equal,
        """less_equal(x1, x2)
Returns an array of (x1 <= x2) element-wise.

Args:
    x1 (~chainerx.ndarray): Input array.
    x2 (~chainerx.ndarray): Input array.

Returns:
    :class:`~chainerx.ndarray`: Output array of type bool.

Note:
    During backpropagation, this function does not propagate gradients.

.. seealso:: :data:`numpy.less_equal`
""")

    _docs.set_doc(
        chainerx.equal,
        """equal(x1, x2)
Returns an array of (x1 == x2) element-wise.

Args:
    x1 (~chainerx.ndarray): Input array.
    x2 (~chainerx.ndarray): Input array.

Returns:
    :class:`~chainerx.ndarray`: Output array of type bool.

Note:
    During backpropagation, this function does not propagate gradients.

.. seealso:: :data:`numpy.equal`
""")

    _docs.set_doc(
        chainerx.not_equal,
        """not_equal(x1, x2)
Returns an array of (x1 != x2) element-wise.

Args:
    x1 (~chainerx.ndarray): Input array.
    x2 (~chainerx.ndarray): Input array.

Returns:
    :class:`~chainerx.ndarray`: Output array of type bool.

Note:
    During backpropagation, this function does not propagate gradients.

.. seealso:: :data:`numpy.not_equal`
""")

def _docs_manipulation():
    _docs.set_doc(
        chainerx.reshape,
        """reshape(a, newshape)
Returns a reshaped array.

Args:
    a (~chainerx.ndarray): Array to be reshaped.
    newshape (int or tuple of ints): The new shape of the array to return.
        If it is an integer, then it is treated as a tuple of length one.
        It should be compatible with ``a.size``. One of the elements can be
        -1, which is automatically replaced with the appropriate value to
        make the shape compatible with ``a.size``.

Returns:
    :class:`~chainerx.ndarray`: A reshaped view of ``a`` if possible,
    otherwise a copy.

Note:
    During backpropagation, this function propagates the gradient of the
    output array to the input array ``a``.

.. seealso:: :func:`numpy.reshape`
""")

def _docs_math():
    pass

def _docs_statistics():
    pass

def _docs_connection():
    pass

def _docs_normalization():
    pass

def _docs_pooling():
    pass
<|endoftext|>


<commit_before>#!/usr/bin/env python

"""
Check each candidate for hot pixels and nearest variable sources
"""

import argparse
import os
import sys

from chandra_suli import find_files
from chandra_suli import logging_system
from chandra_suli.run_command import CommandRunner

if __name__=="__main__":

    parser = argparse.ArgumentParser(description='Check each candidate transient for hot pixels and '
                                                 'closest variable source')

    parser.add_argument("-o","--obsid",help="Observation ID Numbers", type=int, required=True)

    # Get the logger
    logger = logging_system.get_logger(os.path.basename(sys.argv[0]))

    # Get the command runner
    runner = CommandRunner(logger)

    args = parser.parse_args()

    ccd_files = find_files.find_files('.','ccd*%s*fits'%args.obsid)
    ccd_files = sorted(ccd_files)

    ccd_bb_files = find_files.find_files('.', 'ccd*%s*res.txt' %args.obsid)
    ccd_bb_files = sorted(ccd_bb_files)

    evtfile = find_files.find_files('.','*%s*evt3.fits' %args.obsid)[0]

    for i in xrange(len(ccd_bb_files)):

        og_file = os.path.basename(ccd_bb_files[i])

        ccd_bb_file = ccd_bb_files[i]
        ccd_file = ccd_files[i]

        check_hp_file = "check_hp_%s" %og_file

        cmd_line = "check_hot_pixel_revised.py --evtfile %s --bbfile %s --outfile %s --debug no" \
                   %(ccd_file, ccd_bb_file, check_hp_file)

        runner.run(cmd_line)

        check_var_file = "check_var_%s" %og_file

        cmd_line = "check_variable_revised.py --bbfile %s --outfile %s --eventfile %s" \
                   %(check_hp_file,check_var_file, evtfile)

        runner.run(cmd_line)<commit_msg>Check that directory exists<commit_after>#!/usr/bin/env python

"""
Check each candidate for hot pixels and nearest variable sources
"""

import argparse
import os
import sys

from chandra_suli import find_files
from chandra_suli import logging_system
from chandra_suli.run_command import CommandRunner
from chandra_suli.work_within_directory import work_within_directory

if __name__=="__main__":

    parser = argparse.ArgumentParser(description='Check each candidate transient for hot pixels and '
                                                 'closest variable source')

    parser.add_argument("-o","--obsid",help="Observation ID Numbers", type=int, required=True)

    # Get the logger
    logger = logging_system.get_logger(os.path.basename(sys.argv[0]))

    # Get the command runner
    runner = CommandRunner(logger)

    args = parser.parse_args()

    if not os.path.exists(str(args.obsid)):

        raise IOError("Directory not found for obsid %s" %args.obsid)

    with work_within_directory(str(args.obsid)):

        ccd_files = find_files.find_files('.','ccd*%s*fits'%args.obsid)
        ccd_files = sorted(ccd_files)

        ccd_bb_files = find_files.find_files('.', 'ccd*%s*res.txt' %args.obsid)
        ccd_bb_files = sorted(ccd_bb_files)

        evtfile = find_files.find_files('.','*%s*evt3.fits' %args.obsid)[0]

        for i in xrange(len(ccd_bb_files)):

            og_file = os.path.basename(ccd_bb_files[i])

            ccd_bb_file = ccd_bb_files[i]
            ccd_file = ccd_files[i]

            check_hp_file = "check_hp_%s" %og_file

            cmd_line = "check_hot_pixel_revised.py --evtfile %s --bbfile %s --outfile %s --debug no" \
                       %(ccd_file, ccd_bb_file, check_hp_file)

            runner.run(cmd_line)

            check_var_file = "check_var_%s" %og_file

            cmd_line = "check_variable_revised.py --bbfile %s --outfile %s --eventfile %s" \
                       %(check_hp_file,check_var_file, evtfile)

            runner.run(cmd_line)<|endoftext|>


<commit_before>from __future__ import absolute_import, division

import logging
from operator import add

from lxml import etree

from flask import current_app

from changes.config import db, statsreporter
from changes.constants import Result
from changes.db.utils import try_create
from changes.models.failurereason import FailureReason
from changes.models.testresult import TestResult, TestResultManager
from changes.utils.agg import aggregate_result
from changes.utils.http import build_uri

from .base import ArtifactHandler, ArtifactParseError

class XunitHandler(ArtifactHandler):
    FILENAMES = ('xunit.xml', 'junit.xml', 'nosetests.xml', '*.xunit.xml', '*.junit.xml', '*.nosetests.xml')
    logger = logging.getLogger('xunit')

    def process(self, fp):
        test_list = self.get_tests(fp)

        manager = TestResultManager(self.step)
        manager.save(test_list)

        return test_list

    @statsreporter.timer('xunithandler_get_tests')
    def get_tests(self, fp):
        try:
            # libxml has a limit on the size of a text field by default, but we encode stdout/stderr.
            #
            # Its not good to have such huge text fields in the first place but we still want to
            # avoid hard failing here if we do.
            parser = etree.XMLParser(huge_tree=True)
            root = etree.fromstring(fp.read(), parser=parser)
        except Exception:
            uri = build_uri('/find_build/{0}/'.format(self.step.job.build_id.hex))
            self.logger.warning('Failed to parse XML; (step=%s, build=%s)', self.step.id.hex, uri, exc_info=True)
            self._record_malformed()
            return []

        # We've parsed the XML successful, but we still need to make sure it is well-formed for our reasons.
        try:
            if root.tag == 'unittest-results':
                return self.get_bitten_tests(root)
            return self.get_xunit_tests(root)
        except ArtifactParseError:
            # There may be valid test data to be extracted, but we discard them all to be safe.
            self._record_malformed()
            return []

    def _record_malformed(self):
        try_create(FailureReason, {
            'step_id': self.step.id,
            'job_id': self.step.job_id,
            'build_id': self.step.job.build_id,
            'project_id': self.step.project_id,
            'reason': 'malformed_artifact'
        })
        db.session.commit()

    def get_bitten_tests(self, root):
        step = self.step

        results = []

        message_limit = current_app.config.get('TEST_MESSAGE_MAX_LEN')
        # XXX(dcramer): bitten xml syntax, no clue what this
        for node in root.iter('test'):
            # classname, name, time
            attrs = dict(node.items())
            # AFAIK the spec says only one tag can be present
            # http://windyroad.com.au/dl/Open%20Source/JUnit.xsd
            if attrs['status'] == 'success':
                result = Result.passed
            elif attrs['status'] == 'skipped':
                result = Result.skipped
            elif attrs['status'] in ('error', 'failure'):
                result = Result.failed
            else:
                result = None

            try:
                message = list(node.iter('traceback'))[0].text
            except IndexError:
                message = ''

            # no matching status tags were found
            if result is None:
                result = Result.passed

            results.append(TestResult(
                step=step,
                name=attrs['name'],
                package=attrs.get('fixture') or None,
                duration=float(attrs['duration']) * 1000,
                result=result,
                message=_truncate_message(message, message_limit),
            ))

        return results

    def get_xunit_tests(self, root):
        step = self.step

        message_limit = current_app.config.get('TEST_MESSAGE_MAX_LEN')
        results = []
        for node in root.iter('testcase'):
            # classname, name, time
            attrs = dict(node.items())
            # AFAIK the spec says only one tag can be present
            # http://windyroad.com.au/dl/Open%20Source/JUnit.xsd
            try:
                r_node = list(node.iterchildren())[0]
            except IndexError:
                result = Result.passed
                message = ''
            else:
                # TODO(cramer): whitelist tags that are not statuses
                if r_node.tag == 'failure':
                    result = Result.failed
                elif r_node.tag == 'skipped':
                    result = Result.skipped
                elif r_node.tag == 'error':
                    result = Result.failed
                else:
                    result = None

                message = r_node.text

            # If there's a previous failure in addition to stdout or stderr,
            # prioritize showing the previous failure because that's what's
            # useful for debugging flakiness.
            message = attrs.get("last_failure_output") or message
            # no matching status tags were found
            if result is None:
                result = Result.passed

            if attrs.get('quarantined'):
                if result == Result.passed:
                    result = Result.quarantined_passed
                elif result == Result.failed:
                    result = Result.quarantined_failed
                elif result == Result.skipped:
                    result = Result.quarantined_skipped

            owner = attrs.get('owner', None)

            if attrs.get('time'):
                duration = float(attrs['time']) * 1000
            else:
                duration = None

            if not attrs.get('name'):
                raise ArtifactParseError('testcase is missing required "name" property.')

            results.append(TestResult(
                step=step,
                name=attrs['name'],
                package=attrs.get('classname') or None,
                duration=duration,
                result=result,
                # We truncate before deduplication; this gives us a weaker guarantee on maximum size,
                # but ensures that we have at least some message from each test.
                message=_truncate_message(message, message_limit),
                reruns=int(attrs.get('rerun')) if attrs.get('rerun') else None,
                artifacts=self._get_testartifacts(node),
                owner=owner,
            ))

        results = _deduplicate_testresults(results)
        return results

    def _get_testartifacts(self, node):
        test_artifacts_node = node.find('test-artifacts')
        if test_artifacts_node is None:
            return None

        results = []
        for artifact_node in node.iter('artifact'):
            attrs = dict(artifact_node.items())
            results.append(attrs)
        return results

def _deduplicate_testresults(results):
    """Combine TestResult objects until every package+name is unique.

    The traditions surrounding jUnit do not prohibit a single test from
    producing two or more <testcase> elements.  In fact, py.test itself
    will produce two such elements for a single test if the test both
    fails and then hits an error during tear-down.  To impedance-match
    this situation with the Changes constraint of one result per test,
    we combine <testcase> elements that belong to the same test.

    """
    result_dict = {}
    deduped = []

    for result in results:
        key = (result.package, result.name)
        existing_result = result_dict.get(key)

        if existing_result is not None:
            e, r = existing_result, result
            e.duration = _careful(add, e.duration, r.duration)
            e.result = aggregate_result((e.result, r.result))
            if e.message is None:
                e.message = ''
            if r.message is None:
                r.message = ''
            e.message += '\n\n' + r.message
            e.reruns = _careful(max, e.reruns, r.reruns)
            e.artifacts = _careful(add, e.artifacts, r.artifacts)
        else:
            result_dict[key] = result
            deduped.append(result)

    return deduped

def _careful(op, a, b):
    """Return `op(a, b)` if neither is `None`, else the non-`None` value."""
    if a is None:
        return b
    if b is None:
        return a
    return op(a, b)

_TRUNCATION_HEADER = " -- CONTENT TRUNCATED; Look for original XML file for the full data. --\n"

def _truncate_message(msg, limit):
    """Truncate a message if necessary, retaining as many ending lines as possible.
    If truncated, a header line explaining may be prepended to the beginning.

    Args:
        msg (str): The message to potentially truncate.
        limit (Optional[int]): Maximum number of bytes to retain of the message.
    """
    if msg is None or len(msg) <= limit:
        return msg
    nl = msg.find('\n', len(msg) - limit)
    if nl == -1:
        msg = ''  # No line ending? This message is going to be unreadable.
    else:
        msg = msg[nl + 1:]
    return _TRUNCATION_HEADER + msg
<commit_msg>Provide more context to Sentry when xunit parsing fails<commit_after>from __future__ import absolute_import, division

import logging
from operator import add

from lxml import etree

from flask import current_app

from changes.config import db, statsreporter
from changes.constants import Result
from changes.db.utils import try_create
from changes.models.failurereason import FailureReason
from changes.models.testresult import TestResult, TestResultManager
from changes.utils.agg import aggregate_result
from changes.utils.http import build_uri

from .base import ArtifactHandler, ArtifactParseError

class XunitHandler(ArtifactHandler):
    FILENAMES = ('xunit.xml', 'junit.xml', 'nosetests.xml', '*.xunit.xml', '*.junit.xml', '*.nosetests.xml')
    logger = logging.getLogger('xunit')

    def process(self, fp):
        test_list = self.get_tests(fp)

        manager = TestResultManager(self.step)
        manager.save(test_list)

        return test_list

    @statsreporter.timer('xunithandler_get_tests')
    def get_tests(self, fp):
        content_size = None
        try:
            # libxml has a limit on the size of a text field by default, but we encode stdout/stderr.
            #
            # Its not good to have such huge text fields in the first place but we still want to
            # avoid hard failing here if we do.
            parser = etree.XMLParser(huge_tree=True)
            content = fp.read()
            content_size = len(content)
            root = etree.fromstring(content, parser=parser)
        except Exception:
            uri = build_uri('/find_build/{0}/'.format(self.step.job.build_id.hex))
            self.logger.warning('Failed to parse XML; (step=%s, build=%s, size=%s)',
                                self.step.id.hex, uri, content_size, exc_info=True)
            self._record_malformed()
            return []

        # We've parsed the XML successful, but we still need to make sure it is well-formed for our reasons.
        try:
            if root.tag == 'unittest-results':
                return self.get_bitten_tests(root)
            return self.get_xunit_tests(root)
        except ArtifactParseError:
            # There may be valid test data to be extracted, but we discard them all to be safe.
            self._record_malformed()
            return []

    def _record_malformed(self):
        try_create(FailureReason, {
            'step_id': self.step.id,
            'job_id': self.step.job_id,
            'build_id': self.step.job.build_id,
            'project_id': self.step.project_id,
            'reason': 'malformed_artifact'
        })
        db.session.commit()

    def get_bitten_tests(self, root):
        step = self.step

        results = []

        message_limit = current_app.config.get('TEST_MESSAGE_MAX_LEN')
        # XXX(dcramer): bitten xml syntax, no clue what this
        for node in root.iter('test'):
            # classname, name, time
            attrs = dict(node.items())
            # AFAIK the spec says only one tag can be present
            # http://windyroad.com.au/dl/Open%20Source/JUnit.xsd
            if attrs['status'] == 'success':
                result = Result.passed
            elif attrs['status'] == 'skipped':
                result = Result.skipped
            elif attrs['status'] in ('error', 'failure'):
                result = Result.failed
            else:
                result = None

            try:
                message = list(node.iter('traceback'))[0].text
            except IndexError:
                message = ''

            # no matching status tags were found
            if result is None:
                result = Result.passed

            results.append(TestResult(
                step=step,
                name=attrs['name'],
                package=attrs.get('fixture') or None,
                duration=float(attrs['duration']) * 1000,
                result=result,
                message=_truncate_message(message, message_limit),
            ))

        return results

    def get_xunit_tests(self, root):
        step = self.step

        message_limit = current_app.config.get('TEST_MESSAGE_MAX_LEN')
        results = []
        for node in root.iter('testcase'):
            # classname, name, time
            attrs = dict(node.items())
            # AFAIK the spec says only one tag can be present
            # http://windyroad.com.au/dl/Open%20Source/JUnit.xsd
            try:
                r_node = list(node.iterchildren())[0]
            except IndexError:
                result = Result.passed
                message = ''
            else:
                # TODO(cramer): whitelist tags that are not statuses
                if r_node.tag == 'failure':
                    result = Result.failed
                elif r_node.tag == 'skipped':
                    result = Result.skipped
                elif r_node.tag == 'error':
                    result = Result.failed
                else:
                    result = None

                message = r_node.text

            # If there's a previous failure in addition to stdout or stderr,
            # prioritize showing the previous failure because that's what's
            # useful for debugging flakiness.
            message = attrs.get("last_failure_output") or message
            # no matching status tags were found
            if result is None:
                result = Result.passed

            if attrs.get('quarantined'):
                if result == Result.passed:
                    result = Result.quarantined_passed
                elif result == Result.failed:
                    result = Result.quarantined_failed
                elif result == Result.skipped:
                    result = Result.quarantined_skipped

            owner = attrs.get('owner', None)

            if attrs.get('time'):
                duration = float(attrs['time']) * 1000
            else:
                duration = None

            if not attrs.get('name'):
                raise ArtifactParseError('testcase is missing required "name" property.')

            results.append(TestResult(
                step=step,
                name=attrs['name'],
                package=attrs.get('classname') or None,
                duration=duration,
                result=result,
                # We truncate before deduplication; this gives us a weaker guarantee on maximum size,
                # but ensures that we have at least some message from each test.
                message=_truncate_message(message, message_limit),
                reruns=int(attrs.get('rerun')) if attrs.get('rerun') else None,
                artifacts=self._get_testartifacts(node),
                owner=owner,
            ))

        results = _deduplicate_testresults(results)
        return results

    def _get_testartifacts(self, node):
        test_artifacts_node = node.find('test-artifacts')
        if test_artifacts_node is None:
            return None

        results = []
        for artifact_node in node.iter('artifact'):
            attrs = dict(artifact_node.items())
            results.append(attrs)
        return results

def _deduplicate_testresults(results):
    """Combine TestResult objects until every package+name is unique.

    The traditions surrounding jUnit do not prohibit a single test from
    producing two or more <testcase> elements.  In fact, py.test itself
    will produce two such elements for a single test if the test both
    fails and then hits an error during tear-down.  To impedance-match
    this situation with the Changes constraint of one result per test,
    we combine <testcase> elements that belong to the same test.

    """
    result_dict = {}
    deduped = []

    for result in results:
        key = (result.package, result.name)
        existing_result = result_dict.get(key)

        if existing_result is not None:
            e, r = existing_result, result
            e.duration = _careful(add, e.duration, r.duration)
            e.result = aggregate_result((e.result, r.result))
            if e.message is None:
                e.message = ''
            if r.message is None:
                r.message = ''
            e.message += '\n\n' + r.message
            e.reruns = _careful(max, e.reruns, r.reruns)
            e.artifacts = _careful(add, e.artifacts, r.artifacts)
        else:
            result_dict[key] = result
            deduped.append(result)

    return deduped

def _careful(op, a, b):
    """Return `op(a, b)` if neither is `None`, else the non-`None` value."""
    if a is None:
        return b
    if b is None:
        return a
    return op(a, b)

_TRUNCATION_HEADER = " -- CONTENT TRUNCATED; Look for original XML file for the full data. --\n"

def _truncate_message(msg, limit):
    """Truncate a message if necessary, retaining as many ending lines as possible.
    If truncated, a header line explaining may be prepended to the beginning.

    Args:
        msg (str): The message to potentially truncate.
        limit (Optional[int]): Maximum number of bytes to retain of the message.
    """
    if msg is None or len(msg) <= limit:
        return msg
    nl = msg.find('\n', len(msg) - limit)
    if nl == -1:
        msg = ''  # No line ending? This message is going to be unreadable.
    else:
        msg = msg[nl + 1:]
    return _TRUNCATION_HEADER + msg
<|endoftext|>