<commit_before># -*- coding: utf-8 -*-

from nimp.commands._command import *
from nimp.utilities.ue3 import *
from nimp.utilities.ue4 import *
from nimp.utilities.deployment import *
from nimp.utilities.symbols import *
from nimp.utilities.file_mapper import *

#-------------------------------------------------------------------------------
class PublishCommand(Command):
    def __init__(self):
        Command.__init__(self,
                         'publish',
                         'Publish binaries or symbols')

    #---------------------------------------------------------------------------
    def configure_arguments(self, env, parser):
        parser.add_argument('-m', '--mode',
                            help = 'operating mode (binaries, symbols, version)',
                            metavar = '<mode>')

        parser.add_argument('-p', '--platform',
                            help = 'platforms to publish',
                            metavar = '<platform>')

        parser.add_argument('-c', '--configuration',
                            help = 'configuration to publish',
                            metavar = '<configuration>')

        parser.add_argument('-t', '--target',
                            help = 'target to publish (game, editor, tools)',
                            metavar = '<target>')

        # FIXME: could we avoid this argument? right now we need it for upload_symbols
        parser.add_argument('-r',
                            '--revision',
                            help = 'revision',
                            metavar = '<revision>')

        parser.add_argument('--keep-temp-binaries',
                            help    = 'Don’t delete temporary binaries directory',
                            action  = "store_true",
                            default = False)

        # FIXME: this looks too much like --configuration! do something about it
        parser.add_argument('-l',
                            '--configurations',
                            help    = 'Configurations and targets to deploy',
                            metavar = '<configurations>',
                            nargs = '+')

        return True

    #---------------------------------------------------------------------------
    def run(self, env):

        files_to_publish = env.map_files()

        if not hasattr(env, 'mode') or env.mode == 'binaries':
            log_notification("Publishing binaries…")
            files_to_publish.to(env.publish_binaries).load_set("binaries")
            if not all_map(robocopy, files_to_publish()):
                return False

        elif env.mode == 'symbols':
            log_notification("Publishing symbols…")
            symbols_to_publish = env.map_files()
            symbols_to_publish.load_set("symbols")
            if not upload_symbols(env, symbols_to_publish()):
                return False

        elif env.mode == 'version':
            log_notification("Publishing version…")

            files_to_deploy = env.map_files().to('.')

            for configuration in env.configurations:
                if env.is_ue3:
                    tmp = files_to_deploy.override(configuration = configuration,
                                                   ue3_build_configuration = get_ue3_build_config(configuration))
                elif env.is_ue4:
                    tmp = files_to_deploy.override(configuration = configuration,
                                                   ue4_build_configuration = get_ue4_build_config(configuration))

                tmp = tmp.src(env.publish_binaries)

                # UE4 only? FIXME: the logic here doesn’t seem sane to me…
                #if hasattr(env, 'deploy_version_root'):
                #    tmp.to(env.deploy_version_root).glob("**")

                tmp.glob("**")

            log_notification("Deploying binaries…")
            if not all_map(robocopy, files_to_deploy()):
                return False

            # Only for Unreal Engine 3: build scripts
            if env.is_ue3 and env.is_win64:
                log_notification("Building script…")
                if not ue3_build_script(env.game):
                    log_error("Error while building script")
                    return False

            publish_version_path = env.format(env.publish_version)
            log_notification("Publishing version {0}…", configuration)
            files_to_publish = files_to_publish.to(publish_version_path)
            files_to_publish.load_set("version")
            if not all_map(robocopy, files_to_publish()):
                return False

            # Support UnrealProp if necessary
            if hasattr(env, 'unreal_prop_path') and hasattr(env, 'upms_platform'):
                unreal_prop_path = env.format(env.unreal_prop_path)
                publish_to_unreal_prop = env.map_files().override(is_unreal_prop_version = True).to(unreal_prop_path)
                log_notification("Copying files to unreal prop…")
                publish_to_unreal_prop.load_set("version")

                if not all_map(robocopy, publish_to_unreal_prop()):
                    return False

                # Patoune wants this file
                cl_name = 'CL.txt'
                log_notification("Writing changelist file {0} for {1}…", cl_name, configuration)
                cl_path = os.path.join(unreal_prop_path, cl_name)
                with open(cl_path, "w") as cl_fd:
                    cl_fd.write('%s\n' % env.revision)

                # Many Epic tools only consider the build valid if they find a *TOC.txt file
                # See for instance GetLatestBuildFromUnrealProp() in CIS Build Controller
                # Also, Patoune insists on having a specific TOC name, hence the toc_name part
                if hasattr(env, 'game'):
                    if env.is_ue3:
                        toc_name = '%s%sTOC.txt' % (env.upms_platform, 'FINAL' if env.is_win32 else '')
                        toc_content = ''
                    elif env.is_ue4:
                        toc_name = 'TOC.xml'
                        toc_content = '<lol></lol>\n'

                    log_notification("Writing fake TOC {0} for UnrealProp", toc_name)

                    toc_dir = os.path.join(unreal_prop_path, env.game)
                    safe_makedirs(toc_dir)

                    toc_path = os.path.join(toc_dir, toc_name)
                    with open(toc_path, "w") as toc_fd:
                        toc_fd.write(toc_content)

            # If everything went well, remove temporary binaries
            if not env.keep_temp_binaries:
                for configuration in env.configurations:
                    try:
                        shutil.rmtree(env.format(env.publish_binaries, configuration = configuration))
                    except Exception as ex:
                        log_error("Error while cleaning binaries: {0}", ex)

        return True

<commit_msg>Override the “target” setting when deploying/publishing binaries.<commit_after># -*- coding: utf-8 -*-

from nimp.commands._command import *
from nimp.utilities.ue3 import *
from nimp.utilities.ue4 import *
from nimp.utilities.deployment import *
from nimp.utilities.symbols import *
from nimp.utilities.file_mapper import *

#-------------------------------------------------------------------------------
class PublishCommand(Command):
    def __init__(self):
        Command.__init__(self,
                         'publish',
                         'Publish binaries or symbols')

    #---------------------------------------------------------------------------
    def configure_arguments(self, env, parser):
        parser.add_argument('-m', '--mode',
                            help = 'operating mode (binaries, symbols, version)',
                            metavar = '<mode>')

        parser.add_argument('-p', '--platform',
                            help = 'platforms to publish',
                            metavar = '<platform>')

        parser.add_argument('-c', '--configuration',
                            help = 'configuration to publish',
                            metavar = '<configuration>')

        parser.add_argument('-t', '--target',
                            help = 'target to publish (game, editor, tools)',
                            metavar = '<target>')

        # FIXME: could we avoid this argument? right now we need it for upload_symbols
        parser.add_argument('-r',
                            '--revision',
                            help = 'revision',
                            metavar = '<revision>')

        parser.add_argument('--keep-temp-binaries',
                            help    = 'Don’t delete temporary binaries directory',
                            action  = "store_true",
                            default = False)

        # FIXME: this looks too much like --configuration! do something about it
        parser.add_argument('-l',
                            '--configurations',
                            help    = 'Configurations and targets to deploy',
                            metavar = '<configurations>',
                            nargs = '+')

        return True

    #---------------------------------------------------------------------------
    def run(self, env):

        files_to_publish = env.map_files()

        if not hasattr(env, 'mode') or env.mode == 'binaries':
            log_notification("Publishing binaries…")
            files_to_publish.to(env.publish_binaries).load_set("binaries")
            if not all_map(robocopy, files_to_publish()):
                return False

        elif env.mode == 'symbols':
            log_notification("Publishing symbols…")
            symbols_to_publish = env.map_files()
            symbols_to_publish.load_set("symbols")
            if not upload_symbols(env, symbols_to_publish()):
                return False

        elif env.mode == 'version':
            log_notification("Publishing version…")

            files_to_deploy = env.map_files().to('.')

            for config_or_target in env.configurations:

                c = config_or_target if config_or_target not in ['editor', 'tools'] else 'devel'
                t = config_or_target if config_or_target in ['editor', 'tools'] else 'game'

                if env.is_ue3:
                    tmp = files_to_deploy.override(configuration = c,
                                                   target = t,
                                                   ue3_build_configuration = get_ue3_build_config(c))
                elif env.is_ue4:
                    tmp = files_to_deploy.override(configuration = c,
                                                   target = t,
                                                   ue4_build_configuration = get_ue4_build_config(c))

                tmp = tmp.src(env.publish_binaries)

                # UE4 only? FIXME: the logic here doesn’t seem sane to me…
                #if hasattr(env, 'deploy_version_root'):
                #    tmp.to(env.deploy_version_root).glob("**")

                tmp.glob("**")

            log_notification("Deploying binaries…")
            if not all_map(robocopy, files_to_deploy()):
                return False

            # Only for Unreal Engine 3: build scripts
            if env.is_ue3 and env.is_win64:
                log_notification("Building script…")
                if not ue3_build_script(env.game):
                    log_error("Error while building script")
                    return False

            publish_version_path = env.format(env.publish_version)
            log_notification("Publishing version {0}…", configuration)
            files_to_publish = files_to_publish.to(publish_version_path)
            files_to_publish.load_set("version")
            if not all_map(robocopy, files_to_publish()):
                return False

            # Support UnrealProp if necessary
            if hasattr(env, 'unreal_prop_path') and hasattr(env, 'upms_platform'):
                unreal_prop_path = env.format(env.unreal_prop_path)
                publish_to_unreal_prop = env.map_files().override(is_unreal_prop_version = True).to(unreal_prop_path)
                log_notification("Copying files to unreal prop…")
                publish_to_unreal_prop.load_set("version")

                if not all_map(robocopy, publish_to_unreal_prop()):
                    return False

                # Patoune wants this file
                cl_name = 'CL.txt'
                log_notification("Writing changelist file {0} for {1}…", cl_name, configuration)
                cl_path = os.path.join(unreal_prop_path, cl_name)
                with open(cl_path, "w") as cl_fd:
                    cl_fd.write('%s\n' % env.revision)

                # Many Epic tools only consider the build valid if they find a *TOC.txt file
                # See for instance GetLatestBuildFromUnrealProp() in CIS Build Controller
                # Also, Patoune insists on having a specific TOC name, hence the toc_name part
                if hasattr(env, 'game'):
                    if env.is_ue3:
                        toc_name = '%s%sTOC.txt' % (env.upms_platform, 'FINAL' if env.is_win32 else '')
                        toc_content = ''
                    elif env.is_ue4:
                        toc_name = 'TOC.xml'
                        toc_content = '<lol></lol>\n'

                    log_notification("Writing fake TOC {0} for UnrealProp", toc_name)

                    toc_dir = os.path.join(unreal_prop_path, env.game)
                    safe_makedirs(toc_dir)

                    toc_path = os.path.join(toc_dir, toc_name)
                    with open(toc_path, "w") as toc_fd:
                        toc_fd.write(toc_content)

            # If everything went well, remove temporary binaries
            if not env.keep_temp_binaries:
                for config_or_target in env.configurations:

                    c = config_or_target if config_or_target not in ['editor', 'tools'] else 'devel'
                    t = config_or_target if config_or_target in ['editor', 'tools'] else 'game'

                    try:
                        shutil.rmtree(env.format(env.publish_binaries,
                                                 configuration = c,
                                                 target = t))
                    except Exception as ex:
                        log_error("Error while cleaning binaries: {0}", ex)

        return True

<|endoftext|>


<commit_before>import csv
import math
import os
from matplotlib.pyplot import show, ion, savefig
import pandas
from pandas.util.testing import DataFrame
from util import check_dir_exists
import numpy as np

class PlotOutput:

    @staticmethod
    def plot_output(name, infile_path, model_names, filter, export_pdf):
        graphs = {}
        graphs['SSE'] = {}
        graphs['NLPD'] = {}
        graphs['ER'] = {}
        for m in model_names:
            data_config = PlotOutput.read_config(infile_path + m + '/' + 'config_' + '.csv')
            if filter is None or filter(data_config):
                data_test = pandas.read_csv(infile_path + m + '/' + 'test_' +  '.csv')
                data_train = pandas.read_csv(infile_path + m + '/' + 'train_' + '.csv')
                Y_mean = data_train['Y0'].mean()
                Ypred = data_test['Ypred__0']
                Ytrue = data_test['Ytrue0']
                Yvar = data_test['Yvar_pred__0']

                if data_config['ll'] in ['UnivariateGaussian']:
                    graphs['SSE'][str(data_config)] = (Ypred - Ytrue)**2 / ((Y_mean - Ytrue) **2).mean()
                    graphs['NLPD'][str(data_config)] = 0.5*(Ytrue-Ypred) ** 2./Yvar+np.log(2*math.pi*Yvar)

                if data_config['ll'] in ['LogisticLL']:
                    graphs['ER'][PlotOutput.config_to_str(data_config)] = np.array([(((Ypred > 0.5) & (Ytrue == -1))
                                                                 | ((Ypred < 0.5) & (Ytrue == 1))
                                                                 ).mean()])
                    graphs['NLPD'][PlotOutput.config_to_str(data_config)] = -np.log((-Ytrue + 1) / 2 + Ytrue * Ypred)

        for n, g in graphs.iteritems():
            if g:
                g= DataFrame(g)
                ion()
                if n in ['SSE', 'NLPD']:
                    g.plot(kind='box', title=n)
                if n in ['ER']:
                    ax =g.plot(kind='bar', title=n)
                    patches, labels = ax.get_legend_handles_labels()
                    ax.legend(patches, labels, loc='lower center')
                if export_pdf:
                    check_dir_exists(infile_path + name + '/graphs/')
                    savefig(infile_path + name + '/graphs/'+'n' + '.pdf')
                show(block=True)

    @staticmethod
    def config_to_str(config):
        return config['method'] + str(config['sparsify_factor'])

    @staticmethod
    def plot_output_all(name, path, filter, export_pdf):
        dir = [d for d in os.listdir(path) if os.path.isdir(os.path.join(path, d))]
        PlotOutput.plot_output(name, path, dir, filter, export_pdf)

    @staticmethod
    def read_config(path):
        with open(path) as csvfile:
            reader = csv.DictReader(csvfile)
            return reader.next()

if __name__ == '__main__':
    PlotOutput.plot_output('boston', '../../results/', ['boston_full', 'boston_mix1', 'boston_mix2', 'boston_gp'], None, True)

<commit_msg>corrected lables for univaraite plot<commit_after>import csv
import math
import os
from matplotlib.pyplot import show, ion, savefig
import pandas
from pandas.util.testing import DataFrame
from util import check_dir_exists
import numpy as np

class PlotOutput:

    @staticmethod
    def plot_output(name, infile_path, model_names, filter, export_pdf):
        graphs = {}
        graphs['SSE'] = {}
        graphs['NLPD'] = {}
        graphs['ER'] = {}
        for m in model_names:
            data_config = PlotOutput.read_config(infile_path + m + '/' + 'config_' + '.csv')
            if filter is None or filter(data_config):
                data_test = pandas.read_csv(infile_path + m + '/' + 'test_' +  '.csv')
                data_train = pandas.read_csv(infile_path + m + '/' + 'train_' + '.csv')
                Y_mean = data_train['Y0'].mean()
                Ypred = data_test['Ypred__0']
                Ytrue = data_test['Ytrue0']
                Yvar = data_test['Yvar_pred__0']

                if data_config['ll'] in ['UnivariateGaussian']:
                    graphs['SSE'][PlotOutput.config_to_str(data_config)] = (Ypred - Ytrue)**2 / ((Y_mean - Ytrue) **2).mean()
                    graphs['NLPD'][PlotOutput.config_to_str(data_config)] = 0.5*(Ytrue-Ypred) ** 2./Yvar+np.log(2*math.pi*Yvar)

                if data_config['ll'] in ['LogisticLL']:
                    graphs['ER'][PlotOutput.config_to_str(data_config)] = np.array([(((Ypred > 0.5) & (Ytrue == -1))
                                                                 | ((Ypred < 0.5) & (Ytrue == 1))
                                                                 ).mean()])
                    graphs['NLPD'][PlotOutput.config_to_str(data_config)] = -np.log((-Ytrue + 1) / 2 + Ytrue * Ypred)

        for n, g in graphs.iteritems():
            if g:
                g= DataFrame(g)
                ion()
                if n in ['SSE', 'NLPD']:
                    g.plot(kind='box', title=n)
                if n in ['ER']:
                    ax =g.plot(kind='bar', title=n)
                    patches, labels = ax.get_legend_handles_labels()
                    ax.legend(patches, labels, loc='lower center')
                if export_pdf:
                    check_dir_exists(infile_path + name + '/graphs/')
                    savefig(infile_path + name + '/graphs/'+'n' + '.pdf')
                show(block=True)

    @staticmethod
    def config_to_str(config):
        return config['method'] + str(config['sparsify_factor'])

    @staticmethod
    def plot_output_all(name, path, filter, export_pdf):
        dir = [d for d in os.listdir(path) if os.path.isdir(os.path.join(path, d))]
        PlotOutput.plot_output(name, path, dir, filter, export_pdf)

    @staticmethod
    def read_config(path):
        with open(path) as csvfile:
            reader = csv.DictReader(csvfile)
            return reader.next()

if __name__ == '__main__':
    PlotOutput.plot_output('boston', '../../results/', ['boston_full', 'boston_mix1', 'boston_mix2', 'boston_gp'], None, True)

<|endoftext|>


<commit_before>import os
import shutil
import time
import zipfile
from pathlib import Path

import pandas as pd
import requests
from tqdm import tqdm

BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
DATA_PATH = os.path.join(BASE_DIR, 'data')
YEARS = range(2010, 2017, 2)

class Donation:
    """Context manager to download  a zip, read data them and cleanup"""

    URL = 'http://agencia.tse.jus.br/estatistica/sead/odsele/prestacao_contas'

    DIRNAMES = {
        2010: 'prestacao_contas_2010',
        2012: 'prestacao_final_2010',
        2014: 'prestacao_final_2014',
        2016: 'prestacao_contas_final_2016',
    }

    ZIPNAMES = {key: '{}.zip'.format(value) for key, value in DIRNAMES.items()}

    FILENAMES = {
        2012: (
            'receitas_candidatos_2012_brasil.txt',
            'receitas_partidos_2012_brasil.txt',
            'receitas_comites_2012_brasil.txt'
        ),
        2014: (
            'receitas_candidatos_2014_brasil.txt',
            'receitas_partidos_2014_brasil.txt',
            'receitas_comites_2014_brasil.txt'
        ),
        2016: (
            'receitas_candidatos_prestacao_contas_final_2016_brasil.txt',
            'receitas_partidos_prestacao_contas_final_2016_brasil.txt',
            None
        )
    }

    def __init__(self, year):
        self.year = year
        self.zip_file = self.ZIPNAMES.get(year)
        self.url = '{}/{}'.format(self.URL, self.zip_file)
        self.directory = self.DIRNAMES.get(year)
        self.path = Path(self.directory)

    def _download(self):
        """Saves file from `url` into local `path` showing a progress bar"""
        print('Downloading {}…'.format(self.url))
        request = requests.get(self.url, stream=True)
        total = int(request.headers.get('content-length', 0))
        with open(self.zip_file, 'wb') as file_handler:
            kwargs = dict(total=total, unit='B', unit_scale=True)
            bock_size = 2 ** 19  # ~ 500kB
            for data in tqdm(request.iter_content(bock_size), **kwargs):
                file_handler.write(data)

    def _unzip(self):
        print('Uncompressing {}…'.format(self.zip_file))
        with zipfile.ZipFile(self.zip_file, 'r') as zip_handler:
            zip_handler.extractall(self.directory)

    def _read_csv(self, path, chunksize=None):
        """Wrapper to read CSV with default args and an optional `chunksize`"""
        kwargs = dict(low_memory=False, encoding="ISO-8859-1", sep=';')
        if chunksize:
            kwargs['chunksize'] = 10000

        data = pd.read_csv(path, **kwargs)
        return pd.concat([chunk for chunk in data]) if chunksize else data

    def _data_by_pattern(self, *patterns):
        """
        Given a list of words, loads all files matching these words, and then
        concats them all in a single data frame
        """
        pattern = '*{}*'.format('*'.join(patterns))
        data = [self._read_csv(name) for name in self.path.glob(pattern)]
        return pd.concat(data)

    def data(self):
        """
        Returns a dictionary with data frames for candidates, parties and
        committees
        """
        files = self.FILENAMES.get(year)
        if not files:  # it's 2010, a different file architecture
            return {
                'candidates': self._data_by_pattern('Receitas', 'candidato'),
                'parties': self._data_by_pattern('Receitas', 'partido'),
                'committees': self._data_by_pattern('Receitas', 'comite')
            }

        paths = (os.path.join(self.directory, filename) for filename in files)
        keys = ('candidates', 'parties', 'committees')
        return {
            key: self._read_csv(path, chunksize=10000)
            for path, key in zip(keys, paths)
        }

    def __enter__(self):
        self._download()
        self._extract()
        return self

    def __exit__(self):
        os.remove(self.zip_file)
        shutil.rmtree(self.directory)

def correct_columns(donations_data):
    if 'candidates' in donations_data.keys():
        donations_data['candidates'].rename(columns={'Descricao da receita':
                                                     'Descrição da receita',
                                                     'Especie recurso':
                                                     'Espécie recurso',
                                                     'Numero candidato':
                                                     'Número candidato',
                                                     'Numero do documento':
                                                     'Número do documento',
                                                     'Numero Recibo Eleitoral':
                                                     'Número Recibo Eleitoral',
                                                     'Sigla  Partido':
                                                     'Sigla Partido'},
                                            inplace=True)
    if 'parties' in donations_data.keys():
        donations_data['parties'].rename(columns={'Sigla  Partido':
                                                  'Sigla Partido',
                                                  'Número recibo eleitoral':
                                                  'Número Recibo Eleitoral'},
                                         inplace=True)
    if 'committees' in donations_data.keys():
        donations_data['committees'].rename(columns={'Sigla  Partido':
                                                     'Sigla Partido',
                                                     'Tipo comite':
                                                     'Tipo Comite',
                                                     'Número recibo eleitoral':
                                                     'Número Recibo Eleitoral'},  # noqa
                                            inplace=True)

def translate_columns(donations_data):
    translations = {'Cargo': 'post',
                    'CNPJ Prestador Conta': 'accountable_company_id',
                    'Cod setor econômico do doador': 'donor_economic_setor_id',
                    'Cód. Eleição': 'election_id',
                    'CPF do candidato': 'candidate_cpf',
                    'CPF do vice/suplente': 'substitute_cpf',
                    'CPF/CNPJ do doador': 'donor_cnpj_or_cpf',
                    'CPF/CNPJ do doador originário':
                    'original_donor_cnpj_or_cpf',
                    'Data da receita': 'revenue_date',
                    'Data e hora': 'date_and_time',
                    'Desc. Eleição': 'election_description',
                    'Descrição da receita': 'revenue_description',
                    'Entrega em conjunto?': 'batch',
                    'Espécie recurso': 'type_of_revenue',
                    'Fonte recurso': 'source_of_revenue',
                    'Município': 'city',
                    'Nome candidato': 'candidate_name',
                    'Nome da UE': 'electoral_unit_name',
                    'Nome do doador': 'donor_name',
                    'Nome do doador (Receita Federal)':
                    'donor_name_for_federal_revenue',
                    'Nome do doador originário': 'original_donor_name',
                    'Nome do doador originário (Receita Federal)':
                    'original_donor_name_for_federal_revenue',
                    'Número candidato': 'candidate_number',
                    'Número candidato doador': 'donor_candidate_number',
                    'Número do documento': 'document_number',
                    'Número partido doador': 'donor_party_number',
                    'Número Recibo Eleitoral': 'electoral_receipt_number',
                    'Número UE': 'electoral_unit_number',
                    'Sequencial Candidato': 'candidate_sequence',
                    'Sequencial prestador conta': 'accountable_sequence',
                    'Sequencial comite': 'committee_sequence',
                    'Sequencial Diretorio': 'party_board_sequence',
                    'Setor econômico do doador': 'donor_economic_sector',
                    'Setor econômico do doador originário':
                    'original_donor_economic_sector',
                    'Sigla da UE': 'electoral_unit_abbreviation',
                    'Sigla Partido': 'party_acronym',
                    'Sigla UE doador': 'donor_electoral_unit_abbreviation',
                    'Tipo de documento': 'document_type',
                    'Tipo diretorio': 'party_board_type',
                    'Tipo doador originário': 'original_donor_type',
                    'Tipo partido': 'party_type',
                    'Tipo receita': 'revenue_type',
                    'Tipo comite': 'committee_type',
                    'UF': 'state',
                    'Valor receita': 'revenue_value'}
    if 'candidates' in donations_data.keys():
        donations_data['candidates'].rename(columns=translations,
                                            inplace=True)
    if 'parties' in donations_data.keys():
        donations_data['parties'].rename(columns=translations,
                                         inplace=True)
    if 'committees' in donations_data.keys():
        donations_data['committees'].rename(columns=translations,
                                            inplace=True)

def strip_columns_names(donations_data, key):
    if key in donations_data.keys():
        columns_candidates = donations_data[key].columns.values
        columns_mod_candidates = {}
        for name in columns_candidates:
            columns_mod_candidates[name] = name.strip()
        donations_data[key].rename(columns=columns_mod_candidates,
                                   inplace=True)

def data_by_year(year):
    with Donation(year) as donation:
        data = donation.data()
        for key in ('candidates', 'parties', 'committees'):
            strip_columns_names(data, key)
        return data

if __name__ == '__main__':
    donations_data = [data_by_year(year) for year in YEARS]

    donations_candidates_concatenated = []
    donations_parties_concatenated = []
    donations_committees_concatenated = []

    for data in donations_data:
        correct_columns(data)
        translate_columns(data)
        if 'candidates' in data.keys():
            donations_candidates_concatenated.append(data['candidates'])
        if 'parties' in data.keys():
            donations_parties_concatenated.append(data['parties'])
        if 'committees' in data.keys():
            donations_committees_concatenated.append(data['committees'])

    donations_candidates_concatenated = pd.concat(donations_candidates_concatenated)  # noqa
    donations_parties_concatenated = pd.concat(donations_parties_concatenated)
    donations_committees_concatenated = pd.concat(donations_committees_concatenated)  # noqa

    print("Saving dataframes in csv files (.xz)...")

    if not os.path.exists(DATA_PATH):
        os.makedirs(DATA_PATH)

    donations_candidates_concatenated.to_csv(os.path.join(DATA_PATH,
                                             time.strftime("%Y-%m-%d") +
                                             '-donations_candidates.xz'),
                                             compression='xz')
    donations_parties_concatenated.to_csv(os.path.join(DATA_PATH,
                                          time.strftime("%Y-%m-%d") +
                                          '-donations_parties.xz'),
                                          compression='xz')
    donations_committees_concatenated.to_csv(os.path.join(DATA_PATH,
                                             time.strftime("%Y-%m-%d") +
                                             '-donations_committees.xz'),
                                             compression='xz')
<commit_msg>Clean, normalize and translate column names<commit_after>import os
import shutil
import time
import zipfile
from pathlib import Path

import pandas as pd
import requests
from tqdm import tqdm

BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
DATA_PATH = os.path.join(BASE_DIR, 'data')
KEYS = ('candidates', 'parties', 'committees')
YEARS = range(2010, 2017, 2)

NORMALIZE_COLUMNS = {
    'candidates': {
        'Descricao da receita': 'Descrição da receita',
        'Especie recurso': 'Espécie recurso',
        'Numero candidato': 'Número candidato',
        'Numero do documento': 'Número do documento',
        'Numero Recibo Eleitoral': 'Número Recibo Eleitoral',
        'Sigla  Partido': 'Sigla Partido'
    },
    'parties': {
        'Sigla  Partido': 'Sigla Partido',
        'Número recibo eleitoral': 'Número Recibo Eleitoral'
    },
    'committees': {
        'Sigla  Partido': 'Sigla Partido',
        'Tipo comite': 'Tipo Comite',
        'Número recibo eleitoral': 'Número Recibo Eleitoral'
    }
}

TRANSLATIONS = {
    'Cargo': 'post',
    'CNPJ Prestador Conta': 'accountable_company_id',
    'Cod setor econômico do doador': 'donor_economic_setor_id',
    'Cód. Eleição': 'election_id',
    'CPF do candidato': 'candidate_cpf',
    'CPF do vice/suplente': 'substitute_cpf',
    'CPF/CNPJ do doador': 'donor_cnpj_or_cpf',
    'CPF/CNPJ do doador originário':
    'original_donor_cnpj_or_cpf',
    'Data da receita': 'revenue_date',
    'Data e hora': 'date_and_time',
    'Desc. Eleição': 'election_description',
    'Descrição da receita': 'revenue_description',
    'Entrega em conjunto?': 'batch',
    'Espécie recurso': 'type_of_revenue',
    'Fonte recurso': 'source_of_revenue',
    'Município': 'city',
    'Nome candidato': 'candidate_name',
    'Nome da UE': 'electoral_unit_name',
    'Nome do doador': 'donor_name',
    'Nome do doador (Receita Federal)':
    'donor_name_for_federal_revenue',
    'Nome do doador originário': 'original_donor_name',
    'Nome do doador originário (Receita Federal)':
    'original_donor_name_for_federal_revenue',
    'Número candidato': 'candidate_number',
    'Número candidato doador': 'donor_candidate_number',
    'Número do documento': 'document_number',
    'Número partido doador': 'donor_party_number',
    'Número Recibo Eleitoral': 'electoral_receipt_number',
    'Número UE': 'electoral_unit_number',
    'Sequencial Candidato': 'candidate_sequence',
    'Sequencial prestador conta': 'accountable_sequence',
    'Sequencial comite': 'committee_sequence',
    'Sequencial Diretorio': 'party_board_sequence',
    'Setor econômico do doador': 'donor_economic_sector',
    'Setor econômico do doador originário':
    'original_donor_economic_sector',
    'Sigla da UE': 'electoral_unit_abbreviation',
    'Sigla Partido': 'party_acronym',
    'Sigla UE doador': 'donor_electoral_unit_abbreviation',
    'Tipo de documento': 'document_type',
    'Tipo diretorio': 'party_board_type',
    'Tipo doador originário': 'original_donor_type',
    'Tipo partido': 'party_type',
    'Tipo receita': 'revenue_type',
    'Tipo comite': 'committee_type',
    'UF': 'state',
    'Valor receita': 'revenue_value'
}

class Donation:
    """Context manager to download, read data from a given year and cleanup"""

    URL = 'http://agencia.tse.jus.br/estatistica/sead/odsele/prestacao_contas'

    DIRNAMES = {
        2010: 'prestacao_contas_2010',
        2012: 'prestacao_final_2010',
        2014: 'prestacao_final_2014',
        2016: 'prestacao_contas_final_2016',
    }

    ZIPNAMES = {key: '{}.zip'.format(value) for key, value in DIRNAMES.items()}

    FILENAMES = {
        2012: (
            'receitas_candidatos_2012_brasil.txt',
            'receitas_partidos_2012_brasil.txt',
            'receitas_comites_2012_brasil.txt'
        ),
        2014: (
            'receitas_candidatos_2014_brasil.txt',
            'receitas_partidos_2014_brasil.txt',
            'receitas_comites_2014_brasil.txt'
        ),
        2016: (
            'receitas_candidatos_prestacao_contas_final_2016_brasil.txt',
            'receitas_partidos_prestacao_contas_final_2016_brasil.txt',
            None
        )
    }

    def __init__(self, year):
        self.year = year
        self.zip_file = self.ZIPNAMES.get(year)
        self.url = '{}/{}'.format(self.URL, self.zip_file)
        self.directory = self.DIRNAMES.get(year)
        self.path = Path(self.directory)

    def _download(self):
        """Saves file from `url` into local `path` showing a progress bar"""
        print('Downloading {}…'.format(self.url))
        request = requests.get(self.url, stream=True)
        total = int(request.headers.get('content-length', 0))
        with open(self.zip_file, 'wb') as file_handler:
            kwargs = dict(total=total, unit='B', unit_scale=True)
            bock_size = 2 ** 19  # ~ 500kB
            for data in tqdm(request.iter_content(bock_size), **kwargs):
                file_handler.write(data)

    def _unzip(self):
        print('Uncompressing {}…'.format(self.zip_file))
        with zipfile.ZipFile(self.zip_file, 'r') as zip_handler:
            zip_handler.extractall(self.directory)

    def _read_csv(self, path, chunksize=None):
        """Wrapper to read CSV with default args and an optional `chunksize`"""
        kwargs = dict(low_memory=False, encoding="ISO-8859-1", sep=';')
        if chunksize:
            kwargs['chunksize'] = 10000

        data = pd.read_csv(path, **kwargs)
        return pd.concat([chunk for chunk in data]) if chunksize else data

    def _data_by_pattern(self, *patterns):
        """
        Given a list of words, loads all files matching these words, and then
        concats them all in a single data frame
        """
        pattern = '*{}*'.format('*'.join(patterns))
        data = [self._read_csv(name) for name in self.path.glob(pattern)]
        return pd.concat(data)

    def _data(self):
        """
        Returns a dictionary with data frames for candidates, parties and
        committees
        """
        files = self.FILENAMES.get(year)
        if not files:  # it's 2010, a different file architecture
            return {
                'candidates': self._data_by_pattern('Receitas', 'candidato'),
                'parties': self._data_by_pattern('Receitas', 'partido'),
                'committees': self._data_by_pattern('Receitas', 'comite')
            }

        paths = (os.path.join(self.directory, filename) for filename in files)
        return {
            key: self._read_csv(path, chunksize=10000)
            for path, key in zip(KEYS, paths)
        }

    @property
    def data(self):
        """Takes self._data, clean, normalizes and translate it"""
        data = self._data()
        for key in KEYS:
            normalize_columns = self.NORMALIZE_COLUMNS.get(key)
            if key in data:
                # strip columns names ('foobar ' -> 'foobar')
                names = data[key].columns.values
                cleaned_columns = {name: name.strip() for name in names}
                data[key].rename(columns=cleaned_columns, inplace=True)
                # normalize & translate
                data[key].rename(columns=normalize_columns, inplace=True)
                data[key].rename(columns=TRANSLATIONS, inplace=True)
        return data

    def __enter__(self):
        self._download()
        self._extract()
        return self

    def __exit__(self):
        os.remove(self.zip_file)
        shutil.rmtree(self.directory)

if __name__ == '__main__':
    donations_data = [Donation(year).data for year in YEARS]

    donations_candidates_concatenated = []
    donations_parties_concatenated = []
    donations_committees_concatenated = []

    for data in donations_data:
        if 'candidates' in data.keys():
            donations_candidates_concatenated.append(data['candidates'])
        if 'parties' in data.keys():
            donations_parties_concatenated.append(data['parties'])
        if 'committees' in data.keys():
            donations_committees_concatenated.append(data['committees'])

    donations_candidates_concatenated = pd.concat(donations_candidates_concatenated)  # noqa
    donations_parties_concatenated = pd.concat(donations_parties_concatenated)
    donations_committees_concatenated = pd.concat(donations_committees_concatenated)  # noqa

    print("Saving dataframes in csv files (.xz)...")

    if not os.path.exists(DATA_PATH):
        os.makedirs(DATA_PATH)

    donations_candidates_concatenated.to_csv(os.path.join(DATA_PATH,
                                             time.strftime("%Y-%m-%d") +
                                             '-donations_candidates.xz'),
                                             compression='xz')
    donations_parties_concatenated.to_csv(os.path.join(DATA_PATH,
                                          time.strftime("%Y-%m-%d") +
                                          '-donations_parties.xz'),
                                          compression='xz')
    donations_committees_concatenated.to_csv(os.path.join(DATA_PATH,
                                             time.strftime("%Y-%m-%d") +
                                             '-donations_committees.xz'),
                                             compression='xz')
<|endoftext|>


<commit_before># Natural Language Toolkit: Distance Metrics
#
# Copyright (C) 2001-2018 NLTK Project
# Author: Edward Loper <edloper@gmail.com>
#         Steven Bird <stevenbird1@gmail.com>
#         Tom Lippincott <tom@cs.columbia.edu>
# URL: <http://nltk.org/>
# For license information, see LICENSE.TXT
#

"""
Distance Metrics.

Compute the distance between two items (usually strings).
As metrics, they must satisfy the following three requirements:

1. d(a, a) = 0
2. d(a, b) >= 0
3. d(a, c) <= d(a, b) + d(b, c)
"""

from __future__ import print_function
from __future__ import division
import math

def _edit_dist_init(len1, len2):
    lev = []
    for i in range(len1):
        lev.append([0] * len2)  # initialize 2D array to zero
    for i in range(len1):
        lev[i][0] = i           # column 0: 0,1,2,3,4,...
    for j in range(len2):
        lev[0][j] = j           # row 0: 0,1,2,3,4,...
    return lev

def _edit_dist_step(lev, i, j, s1, s2,
                    substitution_cost=1,
                    transpositions=False):
    c1 = s1[i - 1]
    c2 = s2[j - 1]

    # skipping a character in s1
    a = lev[i - 1][j] + 1
    # skipping a character in s2
    b = lev[i][j - 1] + 1
    # substitution
    c = lev[i - 1][j - 1] + (substitution_cost if c1 != c2 else 0)

    # transposition
    d = c + 1  # never picked by default
    if transpositions and i > 1 and j > 1:
        if s1[i - 2] == c2 and s2[j - 2] == c1:
            d = lev[i - 2][j - 2] + 1

    # pick the cheapest
    lev[i][j] = min(a, b, c, d)

def edit_distance(s1, s2, substitution_cost=1, transpositions=False):
    """
    Calculate the Levenshtein edit-distance between two strings.
    The edit distance is the number of characters that need to be
    substituted, inserted, or deleted, to transform s1 into s2.  For
    example, transforming "rain" to "shine" requires three steps,
    consisting of two substitutions and one insertion:
    "rain" -> "sain" -> "shin" -> "shine".  These operations could have
    been done in other orders, but at least three steps are needed.

    Allows specifying the cost of substitution edits (e.g., "a" -> "b"),
    because sometimes it makes sense to assign greater penalties to
    substitutions.

    This also optionally allows transposition edits (e.g., "ab" -> "ba"),
    though this is disabled by default.

    :param s1, s2: The strings to be analysed
    :param transpositions: Whether to allow transposition edits
    :type s1: str
    :type s2: str
    :type substitution_cost: int
    :type transpositions: bool
    :rtype int
    """
    # set up a 2-D array
    len1 = len(s1)
    len2 = len(s2)
    lev = _edit_dist_init(len1 + 1, len2 + 1)

    # iterate over the array
    for i in range(len1):
        for j in range(len2):
            _edit_dist_step(lev, i + 1, j + 1, s1, s2,
                            substitution_cost=substitution_cost,
                            transpositions=transpositions)
    return lev[len1][len2]

def binary_distance(label1, label2):
    """Simple equality test.

    0.0 if the labels are identical, 1.0 if they are different.

    >>> from nltk.metrics import binary_distance
    >>> binary_distance(1,1)
    0.0

    >>> binary_distance(1,3)
    1.0
    """

    return 0.0 if label1 == label2 else 1.0

def jaccard_distance(label1, label2):
    """Distance metric comparing set-similarity.

    """
    return (len(label1.union(label2)) -
            len(label1.intersection(label2)))/len(label1.union(label2))

def masi_distance(label1, label2):
    """Distance metric that takes into account partial agreement when multiple
    labels are assigned.

    >>> from nltk.metrics import masi_distance
    >>> masi_distance(set([1, 2]), set([1, 2, 3, 4]))
    0.665

    Passonneau 2006, Measuring Agreement on Set-Valued Items (MASI)
    for Semantic and Pragmatic Annotation.
    """

    len_intersection = len(label1.intersection(label2))
    len_union = len(label1.union(label2))
    len_label1 = len(label1)
    len_label2 = len(label2)
    if len_label1 == len_label2 and len_label1 == len_intersection:
        m = 1
    elif len_intersection == min(len_label1, len_label2):
        m = 0.67
    elif len_intersection > 0:
        m = 0.33
    else:
        m = 0

    return 1 - len_intersection / len_union * m

def interval_distance(label1, label2):
    """Krippendorff's interval distance metric

    >>> from nltk.metrics import interval_distance
    >>> interval_distance(1,10)
    81

    Krippendorff 1980, Content Analysis: An Introduction to its Methodology
    """

    try:
        return pow(label1 - label2, 2)
#        return pow(list(label1)[0]-list(label2)[0],2)
    except:
        print("non-numeric labels not supported with interval distance")

def presence(label):
    """Higher-order function to test presence of a given label
    """

    return lambda x, y: 1.0 * ((label in x) == (label in y))

def fractional_presence(label):
    return lambda x, y:\
        abs(((1.0 / len(x)) - (1.0 / len(y)))) * (label in x and label in y) \
        or 0.0 * (label not in x and label not in y) \
        or abs((1.0 / len(x))) * (label in x and label not in y) \
        or ((1.0 / len(y))) * (label not in x and label in y)

def custom_distance(file):
    data = {}
    with open(file, 'r') as infile:
        for l in infile:
            labelA, labelB, dist = l.strip().split("\t")
            labelA = frozenset([labelA])
            labelB = frozenset([labelB])
            data[frozenset([labelA, labelB])] = float(dist)
    return lambda x, y: data[frozenset([x, y])]

def jaro_similarity(s1, s2):
    """
    Computes the Jaro similarity between 2 sequences from:

        Matthew A. Jaro (1989). Advances in record linkage methodology
        as applied to the 1985 census of Tampa Florida. Journal of the
        American Statistical Association. 84 (406): 414-20.

    The Jaro distance between is the min no. of single-character transpositions
    required to change one word into another. The Jaro similarity formula from
    https://en.wikipedia.org/wiki/Jaro%E2%80%93Winkler_distance :

        jaro_sim = 0 if m = 0 else 1/3 * (m/|s_1| + m/s_2 + (m-t)/m)

    where:
        - |s_i| is the length of string s_i
        - m is the no. of matching characters
        - t is the half no. of possible transpositions.
    """
    # First, store the length of the strings
    # because they will be re-used several times.
    len_s1, len_s2 = len(s1), len(s2)

    # The upper bound of the distanc for being a matched character.
    match_bound = math.floor( max(len(s1), len(s2)) / 2 ) - 1

    # Initialize the counts for matches and transpositions.
    matches = 0  # no.of matched characters in s1 and s2
    transpositions = 0  # no. transpositions between s1 and s2

    # Iterate through sequences, check for matches and compute transpositions.
    for ch1 in s1:     # Iterate through each character.
        if ch1 in s2:  # Check whether the
            pos1 = s1.index(ch1)
            pos2 = s2.index(ch1)
            if(abs(pos1-pos2) <= match_bound):
                matches += 1
                if(pos1 != pos2):
                    transpositions += 1

    if matches == 0:
        return 0
    else:
        return 1/3 * ( matches/len_s1 +
                       matches/len_s2 +
                      (matches-transpositions//2) / matches
                     )

def jaro_winkler_similarity(s1, s2, p=0.1, max_l=None):
    """
    The Jaro Winkler distance is an extension of the Jaro similarity in:

        William E. Winkler. 1990. String Comparator Metrics and Enhanced
        Decision Rules in the Fellegi-Sunter Model of Record Linkage.
        Proceedings of the Section on Survey Research Methods.
        American Statistical Association: 354-359.

    such that:

        jaro_winkler_sim = jaro_sim + ( l * p * (1 - jaro_sim) )

    where,

        - jaro_sim is the output from the Jaro Similarity, see jaro_similarity()
        - l is the length of common prefix at the start of the string
            - this implementation provides an upperbound for the l value
              to keep the prefixes
        - p is the constant scaling factor to overweigh common prefixes.
          The Jaro-Winkler similarity will fall within the [0, 1] bound,
          given that max(p)<=0.25 , default is p=0.1 in Winkler (1990)

    """
    # Compute the Jaro similarity
    jaro_sim = jaro_similarity(s1, s2)

    # Initialize the upper bound for the no. of prefixes.
    # if user did not pre-define the upperbound, use length of s1
    max_l = max_l if max_l else len(s1)

    # Compute the prefix matches.
    l = 0
    for i in range(len(s1)):
        if s1[i] == s2[i]:
            l += 1
        else:
            break
        if l == max_l:
            break
    # Return the similarity value as described in docstring.
    return jaro_sim + ( l * p * (1 - jaro_sim) )

def demo():
    string_distance_examples = [
        ("rain", "shine"), ("abcdef", "acbdef"), ("language", "lnaguaeg"),
        ("language", "lnaugage"), ("language", "lngauage")]

    for s1, s2 in string_distance_examples:
        print("Edit distance btwn '%s' and '%s':" % (s1, s2),
              edit_distance(s1, s2))
        print("Edit dist with transpositions btwn '%s' and '%s':" % (s1, s2),
              edit_distance(s1, s2, transpositions=True))
        print("Jaro similarity btwn '%s' and '%s':" % (s1, s2),
              jaro_similarity(s1, s2))
        print("Jaro-Winkler similarity btwn '%s' and '%s':" % (s1, s2),
              jaro_winkler_similarity(s1, s2))
        print("Jaro-Winkler similarity btwn '%s' and '%s':" % (s1, s2),
              1 - jaro_winkler_similarity(s1, s2))
        print()

    s1 = set([1, 2, 3, 4])
    s2 = set([3, 4, 5])
    print("s1:", s1)
    print("s2:", s2)
    print("Binary distance:", binary_distance(s1, s2))
    print("Jaccard distance:", jaccard_distance(s1, s2))
    print("MASI distance:", masi_distance(s1, s2))

if __name__ == '__main__':
    demo()
<commit_msg> Algorithm rechecked and optimized<commit_after># Natural Language Toolkit: Distance Metrics
#
# Copyright (C) 2001-2018 NLTK Project
# Author: Edward Loper <edloper@gmail.com>
#         Steven Bird <stevenbird1@gmail.com>
#         Tom Lippincott <tom@cs.columbia.edu>
# URL: <http://nltk.org/>
# For license information, see LICENSE.TXT
#

"""
Distance Metrics.

Compute the distance between two items (usually strings).
As metrics, they must satisfy the following three requirements:

1. d(a, a) = 0
2. d(a, b) >= 0
3. d(a, c) <= d(a, b) + d(b, c)
"""

from __future__ import print_function
from __future__ import division

def _edit_dist_init(len1, len2):
    lev = []
    for i in range(len1):
        lev.append([0] * len2)  # initialize 2D array to zero
    for i in range(len1):
        lev[i][0] = i           # column 0: 0,1,2,3,4,...
    for j in range(len2):
        lev[0][j] = j           # row 0: 0,1,2,3,4,...
    return lev

def _edit_dist_step(lev, i, j, s1, s2,
                    substitution_cost=1,
                    transpositions=False):
    c1 = s1[i - 1]
    c2 = s2[j - 1]

    # skipping a character in s1
    a = lev[i - 1][j] + 1
    # skipping a character in s2
    b = lev[i][j - 1] + 1
    # substitution
    c = lev[i - 1][j - 1] + (substitution_cost if c1 != c2 else 0)

    # transposition
    d = c + 1  # never picked by default
    if transpositions and i > 1 and j > 1:
        if s1[i - 2] == c2 and s2[j - 2] == c1:
            d = lev[i - 2][j - 2] + 1

    # pick the cheapest
    lev[i][j] = min(a, b, c, d)

def edit_distance(s1, s2, substitution_cost=1, transpositions=False):
    """
    Calculate the Levenshtein edit-distance between two strings.
    The edit distance is the number of characters that need to be
    substituted, inserted, or deleted, to transform s1 into s2.  For
    example, transforming "rain" to "shine" requires three steps,
    consisting of two substitutions and one insertion:
    "rain" -> "sain" -> "shin" -> "shine".  These operations could have
    been done in other orders, but at least three steps are needed.

    Allows specifying the cost of substitution edits (e.g., "a" -> "b"),
    because sometimes it makes sense to assign greater penalties to
    substitutions.

    This also optionally allows transposition edits (e.g., "ab" -> "ba"),
    though this is disabled by default.

    :param s1, s2: The strings to be analysed
    :param transpositions: Whether to allow transposition edits
    :type s1: str
    :type s2: str
    :type substitution_cost: int
    :type transpositions: bool
    :rtype int
    """
    # set up a 2-D array
    len1 = len(s1)
    len2 = len(s2)
    lev = _edit_dist_init(len1 + 1, len2 + 1)

    # iterate over the array
    for i in range(len1):
        for j in range(len2):
            _edit_dist_step(lev, i + 1, j + 1, s1, s2,
                            substitution_cost=substitution_cost,
                            transpositions=transpositions)
    return lev[len1][len2]

def binary_distance(label1, label2):
    """Simple equality test.

    0.0 if the labels are identical, 1.0 if they are different.

    >>> from nltk.metrics import binary_distance
    >>> binary_distance(1,1)
    0.0

    >>> binary_distance(1,3)
    1.0
    """

    return 0.0 if label1 == label2 else 1.0

def jaccard_distance(label1, label2):
    """Distance metric comparing set-similarity.

    """
    return (len(label1.union(label2)) -
            len(label1.intersection(label2)))/len(label1.union(label2))

def masi_distance(label1, label2):
    """Distance metric that takes into account partial agreement when multiple
    labels are assigned.

    >>> from nltk.metrics import masi_distance
    >>> masi_distance(set([1, 2]), set([1, 2, 3, 4]))
    0.665

    Passonneau 2006, Measuring Agreement on Set-Valued Items (MASI)
    for Semantic and Pragmatic Annotation.
    """

    len_intersection = len(label1.intersection(label2))
    len_union = len(label1.union(label2))
    len_label1 = len(label1)
    len_label2 = len(label2)
    if len_label1 == len_label2 and len_label1 == len_intersection:
        m = 1
    elif len_intersection == min(len_label1, len_label2):
        m = 0.67
    elif len_intersection > 0:
        m = 0.33
    else:
        m = 0

    return 1 - len_intersection / len_union * m

def interval_distance(label1, label2):
    """Krippendorff's interval distance metric

    >>> from nltk.metrics import interval_distance
    >>> interval_distance(1,10)
    81

    Krippendorff 1980, Content Analysis: An Introduction to its Methodology
    """

    try:
        return pow(label1 - label2, 2)
#        return pow(list(label1)[0]-list(label2)[0],2)
    except:
        print("non-numeric labels not supported with interval distance")

def presence(label):
    """Higher-order function to test presence of a given label
    """

    return lambda x, y: 1.0 * ((label in x) == (label in y))

def fractional_presence(label):
    return lambda x, y:\
        abs(((1.0 / len(x)) - (1.0 / len(y)))) * (label in x and label in y) \
        or 0.0 * (label not in x and label not in y) \
        or abs((1.0 / len(x))) * (label in x and label not in y) \
        or ((1.0 / len(y))) * (label not in x and label in y)

def custom_distance(file):
    data = {}
    with open(file, 'r') as infile:
        for l in infile:
            labelA, labelB, dist = l.strip().split("\t")
            labelA = frozenset([labelA])
            labelB = frozenset([labelB])
            data[frozenset([labelA, labelB])] = float(dist)
    return lambda x, y: data[frozenset([x, y])]

def jaro_similarity(s1, s2):
    """
    Computes the Jaro similarity between 2 sequences from:

        Matthew A. Jaro (1989). Advances in record linkage methodology
        as applied to the 1985 census of Tampa Florida. Journal of the
        American Statistical Association. 84 (406): 414-20.

    The Jaro distance between is the min no. of single-character transpositions
    required to change one word into another. The Jaro similarity formula from
    https://en.wikipedia.org/wiki/Jaro%E2%80%93Winkler_distance :

        jaro_sim = 0 if m = 0 else 1/3 * (m/|s_1| + m/s_2 + (m-t)/m)

    where:
        - |s_i| is the length of string s_i
        - m is the no. of matching characters
        - t is the half no. of possible transpositions.
    """
    # First, store the length of the strings
    # because they will be re-used several times.
    len_s1, len_s2 = len(s1), len(s2)

    # The upper bound of the distance for being a matched character.
    match_bound = max(len_s1, len_s2) // 2 - 1

    # Initialize the counts for matches and transpositions.
    matches = 0  # no.of matched characters in s1 and s2
    transpositions = 0  # no. of transpositions between s1 and s2
    flagged_1 = [] # positions in s1 which are matches to some character in s2
    flagged_2 = [] # positions in s2 which are matches to some character in s1

    # Iterate through sequences, check for matches and compute transpositions.
    for i in range(len_s1):     # Iterate through each character.
        upperbound = min(i+match_bound, len_s2-1)
        lowerbound = max(0, i-match_bound)
        for j in range(lowerbound, upperbound+1):
            if s1[i] == s2[j] and j not in flagged_2:
                matches += 1
                flagged_1.append(i)
                flagged_2.append(j)
                break
    flagged_2.sort()
    for i,j in zip(flagged_1,flagged_2):
        if s1[i] != s2[j]:
            transpositions += 1

    if matches == 0:
        return 0
    else:
        return 1/3 * ( matches/len_s1 +
                       matches/len_s2 +
                      (matches-transpositions//2) / matches
                     )

def jaro_winkler_similarity(s1, s2, p=0.1, max_l=4):
    """
    The Jaro Winkler distance is an extension of the Jaro similarity in:

        William E. Winkler. 1990. String Comparator Metrics and Enhanced
        Decision Rules in the Fellegi-Sunter Model of Record Linkage.
        Proceedings of the Section on Survey Research Methods.
        American Statistical Association: 354-359.

    such that:

        jaro_winkler_sim = jaro_sim + ( l * p * (1 - jaro_sim) )

    where,

        - jaro_sim is the output from the Jaro Similarity, see jaro_similarity()
        - l is the length of common prefix at the start of the string
            - this implementation provides an upperbound for the l value
              to keep the prefixes.A common value of this upperbound is 4.
        - p is the constant scaling factor to overweigh common prefixes.
          The Jaro-Winkler similarity will fall within the [0, 1] bound,
          given that max(p)<=0.25 , default is p=0.1 in Winkler (1990)

    """
    # Compute the Jaro similarity
    jaro_sim = jaro_similarity(s1, s2)

    # Initialize the upper bound for the no. of prefixes.
    # if user did not pre-define the upperbound, use smaller among length of s1
    # and length of s2

    # Compute the prefix matches.
    l = 0
    for i in range(len(s1)):
        if s1[i] == s2[i]:
            l += 1
        else:
            break
        if l == max_l:
            break
    # Return the similarity value as described in docstring.
    return jaro_sim + ( l * p * (1 - jaro_sim) )

def demo():
    string_distance_examples = [("rain", "shine"), ("abcdef", "acbdef"), ("language", "lnaguaeg"),
                                ("language", "lnaugage"), ("language", "lngauage")]
    for s1, s2 in string_distance_examples:
        print("Edit distance btwn '%s' and '%s':" % (s1, s2),
              edit_distance(s1, s2))
        print("Edit dist with transpositions btwn '%s' and '%s':" % (s1, s2),
              edit_distance(s1, s2, transpositions=True))
        print("Jaro similarity btwn '%s' and '%s':" % (s1, s2),
              jaro_similarity(s1, s2))
        print("Jaro-Winkler similarity btwn '%s' and '%s':" % (s1, s2),
              jaro_winkler_similarity(s1, s2))
        print("Jaro-Winkler distance btwn '%s' and '%s':" % (s1, s2),
              1 - jaro_winkler_similarity(s1, s2))
    s1 = set([1, 2, 3, 4])
    s2 = set([3, 4, 5])
    print("s1:", s1)
    print("s2:", s2)
    print("Binary distance:", binary_distance(s1, s2))
    print("Jaccard distance:", jaccard_distance(s1, s2))
    print("MASI distance:", masi_distance(s1, s2))

if __name__ == '__main__':
    demo()
<|endoftext|>


<commit_before>
from pykml import parser
from pykml.parser import Schema
from pykml.factory import KML_ElementMaker as KML
from os import path
import json
import geojson
from lxml import etree
import sys, os, random
from datetime import datetime
from random import randrange

sampleTrip = '{"date": "20140413", "lastUpdate": "20140414T064442Z", "segments": [], "summary": null}'
sampleSegment = '{"place": {"type": "unknown", "id": 54095848, "location": {"lat": 37.3910149202, "lon": -122.0865010796}}, "endTime": "20140413T234434-0700", "type": "place", "startTime": "20140412T190446-0700", "lastUpdate": "20140414T064442Z"}'

def kml_multiple_to_geojson(infile_path, outdir_path, geojson_properties={}):
        """
        Converts a KML file with multiple Documents into geojson to store in an output file
        """
        data = __read_file(infile_path)
        coord_dict = __get_all_coords(data)
        if not os.path.exists(outdir_path):
                os.makedirs(outdir_path)
        for section_id, coords in coord_dict.items():
                filename = "%s.json" % section_id
                path = os.path.join(outdir_path, filename)
                outfile = file(path,'w')
                dump = __to_geojson(coords)
                outfile.write(dump)
                outfile.close()

def kml_to_geojson(infile_path, outfile_path, geojson_properties={}):
	"""
	Converts a KML file to geojson to store in an output file.
	"""
        data = __read_file(infile_path)
      	coords = __get_coords(data)
	outfile = open(outfile_path, 'w')
	outfile.write(__to_geojson(coords))
	outfile.close()

def __to_geojson(coordinates, geojson_properties={}):
        coordinatePoints = geojson.MultiPoint(coordinates)
	dump = geojson.dumps(geojson.Feature(geometry=coordinatePoints, properties=geojson_properties)) # can add properties and id to feature (perhaps trip/section id?)
        return dump

def kml_to_json(infile_path, outfile_path):
	"""
	Converts a KML file to geojson to store in an output file.
	"""
        data = __read_file(infile_path)
	coordinatePoints = __get_coords(data)
	segmentsList = []
	for coord in coordinatePoints:
		j = json.loads(sampleSegment)
		j["place"]["location"]["lat"] = coord[0] # 0th index might be longitude
		j["place"]["location"]["lon"] = coord[1]
		segmentsList.append(j)
	j = json.loads(sampleTrip)
	j["segments"] = segmentsList
	with open(outfile_path, 'w') as outfile:
		json.dump(j, outfile)

def __read_file(infile_path):
        data = ""
	with open(infile_path, "r") as temp:
		data = temp.read()
	assert(data != "")
        return data

def __get_coords(data):
	doc = parser.fromstring(data)
	dataPoints = doc.Document.Placemark.LineString.coordinates.text.split(" ")
	return [[float(i) for i in x.split(",")[:2]] for x in dataPoints]

def __get_all_coords(data):
        doc = parser.fromstring(data)
        coords = {}
        for folder in doc.Document.getchildren():
               	dataPoints = folder.Placemark.LineString.coordinates.text.split(" ")
                section_id = folder.Placemark.name.text
                coords[section_id] = [[float(i) for i in x.split(",")[:2]] for x in dataPoints]
        return coords

def validate_kml(filepath, schema = "https://developers.google.com/kml/schema/kml21.xsd"):
        """
        Validates kml located at filepath.
        TODO: Even imported kml files do not validate against default xml
        """
        schema_gomaps = Schema(schema)
	data = ""
	with open(filepath, "r") as temp:
		data = temp.read()
	assert(data != "")
	doc = parser.fromstring(data)
        return schema_ogc.assertValid(doc)

def section_to_kml(section, color, outfile_path="", write=True):
        """
        Converts a section into a kml file
        """
        line_style_id = "line-%s-5" % color
        red = "FF1212"
        green = "00B80C"
        start_icon_style_id = "icon-%s" % color
        end_icon_style_id = "icon-%s" % color
        make_coord = lambda p: (",".join(map(lambda x: str(x),
                         p["track_location"]["coordinates"]) + ["0.0"]))
        make_coord_point = lambda p: (",".join(map(lambda x: str(x),
                         p["coordinates"]) + ["0.0"]))
	style_id = "style-%s" % section['section_start_time']
        pm = KML.Placemark(
                KML.styleUrl("#%s" % line_style_id),
                KML.name(section['_id']),
                KML.LineString(
                        KML.tessellate(1),
                        KML.coordinates(" ".join(
                                map(lambda track_point: make_coord(track_point)
                                    ,section['track_points'])))
                )
        )
        start_point = section['section_start_point']
        end_point = section['section_end_point']
        start_time = mongodate_to_datetime(section["section_start_time"])
        end_time = mongodate_to_datetime(section["section_end_time"])
        start_point = KML.Placemark(
                KML.styleUrl("#%s" % start_icon_style_id),
                KML.name("Start: %s" % start_time),
                KML.description("Starting point"),
                KML.Point(KML.coordinates(make_coord_point(start_point)))
        )
        end_point = KML.Placemark(
                KML.styleUrl("#%s" % end_icon_style_id),
                KML.name("End: %s" % end_time),
                KML.description("Ending point"),
                KML.Point(KML.coordinates(make_coord_point(end_point)))
        )
        line_style = KML.Style(
                KML.LineStyle(
                        KML.color("ff%s" % color),
                        KML.width("5")
                )
        )
        line_style.set("id", line_style_id)
        start_icon_style = KML.Style(
                KML.IconStyle(
                        KML.color("ff%s" % color),
                        KML.scale("1.1"),
                        KML.Icon(
                                KML.href("http://www.gstatic.com/mapspro/images/stock/503-wht-blank_maps.png")
                        )
                )
        )
        start_icon_style.set("id", start_icon_style_id)
        end_icon_style = KML.Style(
                KML.IconStyle(
                        KML.color("ff%s" % color),
                        KML.scale("1.1"),
                        KML.Icon(
                                KML.href("http://www.gstatic.com/mapspro/images/stock/503-wht-blank_maps.png")
                        )
                )
        )
        end_icon_style.set("id", end_icon_style_id)
        fld = KML.Folder(
                KML.name(section['_id']),
                KML.description("From %s \nto %s" % (start_time, end_time)),
                pm,
                start_point,
                end_point
        )
        if write:
                kml = KML.kml(KML.Document(fld, section["user_id"]))
                path = os.path.join(outfile_path, str(section['user_id']) +'.kml')
                outfile = file(path,'w')
                outfile.write(etree.tostring(kml, pretty_print=True))
        else:
                return fld, line_style, start_icon_style, end_icon_style

def sections_to_kml(filename, sections, outfile_path=""):
        r = lambda: random.randint(0,255)
        l_tuples = map(lambda section: section_to_kml(section, '%02X%02X%02X' % (r(),r(),r()),  write=False) if len(section['track_points']) > 1 else None ,sections)
        flat = []

        for f,s,i_s,i_e in l_tuples:
                flat.append(f)
                flat.append(s)
                flat.append(i_s)
                flat.append(i_e)
        kml = KML.kml(KML.Document(sections[0]["user_id"],*flat))
        path = os.path.join(outfile_path, filename +'.kml')
        outfile = file(path,'w')
        outfile.write(etree.tostring(kml, pretty_print=True))

def chunks(l,n):
        """
        Generates evenly sized chunks of a list
        """
        for i in xrange(0, len(l), n):
                yield l[i:i+n]

def mongodate_to_datetime(date):
        return datetime.strptime(date[:-5], "%Y%m%dT%H%M%S")

if __name__ == "__main__":
        sys.path.append("%s/../" % os.getcwd())
        from get_database import get_section_db
        from uuid import UUID
        Sections = get_section_db()
        user_id = "__ID_GOES_HERE__"
        sections = Sections.find({'$and': [{'user_id':user_id}]})
        sections_to_kml(user_id,sections)
<commit_msg>update with some error capturing<commit_after>
from pykml import parser
from pykml.parser import Schema
from pykml.factory import KML_ElementMaker as KML
from os import path
import json
import geojson
from lxml import etree
import sys, os, random
from datetime import datetime
from random import randrange

sampleTrip = '{"date": "20140413", "lastUpdate": "20140414T064442Z", "segments": [], "summary": null}'
sampleSegment = '{"place": {"type": "unknown", "id": 54095848, "location": {"lat": 37.3910149202, "lon": -122.0865010796}}, "endTime": "20140413T234434-0700", "type": "place", "startTime": "20140412T190446-0700", "lastUpdate": "20140414T064442Z"}'

def kml_multiple_to_geojson(infile_path, outdir_path, geojson_properties={}):
        """
        Converts a KML file with multiple Documents into geojson to store in an output file
        """
        data = __read_file(infile_path)
        coord_dict = __get_all_coords(data)
        if not os.path.exists(outdir_path):
                os.makedirs(outdir_path)
        for section_id, coords in coord_dict.items():
                filename = "%s.json" % section_id
                path = os.path.join(outdir_path, filename)
                outfile = file(path,'w')
                dump = __to_geojson(coords)
                outfile.write(dump)
                outfile.close()

def kml_to_geojson(infile_path, outfile_path, geojson_properties={}):
	"""
	Converts a KML file to geojson to store in an output file.
	"""
        data = __read_file(infile_path)
      	coords = __get_coords(data)
	outfile = open(outfile_path, 'w')
	outfile.write(__to_geojson(coords))
	outfile.close()

def __to_geojson(coordinates, geojson_properties={}):
        coordinatePoints = geojson.MultiPoint(coordinates)
	dump = geojson.dumps(geojson.Feature(geometry=coordinatePoints, properties=geojson_properties)) # can add properties and id to feature (perhaps trip/section id?)
        return dump

def kml_to_json(infile_path, outfile_path):
	"""
	Converts a KML file to geojson to store in an output file.
	"""
        data = __read_file(infile_path)
	coordinatePoints = __get_coords(data)
	segmentsList = []
	for coord in coordinatePoints:
		j = json.loads(sampleSegment)
		j["place"]["location"]["lat"] = coord[0] # 0th index might be longitude
		j["place"]["location"]["lon"] = coord[1]
		segmentsList.append(j)
	j = json.loads(sampleTrip)
	j["segments"] = segmentsList
	with open(outfile_path, 'w') as outfile:
		json.dump(j, outfile)

def __read_file(infile_path):
        data = ""
	with open(infile_path, "r") as temp:
		data = temp.read()
	assert(data != "")
        return data

def __get_coords(data):
	doc = parser.fromstring(data)
	dataPoints = doc.Document.Placemark.LineString.coordinates.text.split(" ")
	return [[float(i) for i in x.split(",")[:2]] for x in dataPoints]

def __get_all_coords(data):
        doc = parser.fromstring(data)
        coords = {}
        for folder in doc.Document.getchildren():
                try:
                        dataPoints = folder.Placemark.LineString.coordinates.text.split(" ")
                        section_id = folder.Placemark.name.text
                        coords[section_id] = [[float(i) for i in x.split(",")[:2]] for x in dataPoints]
                except (Exception):
                        continue
        return coords

def validate_kml(filepath, schema = "https://developers.google.com/kml/schema/kml21.xsd"):
        """
        Validates kml located at filepath.
        TODO: Even imported kml files do not validate against default xml
        """
        schema_gomaps = Schema(schema)
	data = ""
	with open(filepath, "r") as temp:
		data = temp.read()
	assert(data != "")
	doc = parser.fromstring(data)
        return schema_ogc.assertValid(doc)

def section_to_kml(section, color, outfile_path="", write=True):
        """
        Converts a section into a kml file
        """
        line_style_id = "line-%s-5" % color
        red = "FF1212"
        green = "00B80C"
        start_icon_style_id = "icon-%s" % color
        end_icon_style_id = "icon-%s" % color
        make_coord = lambda p: (",".join(map(lambda x: str(x),
                         p["track_location"]["coordinates"]) + ["0.0"]))
        make_coord_point = lambda p: (",".join(map(lambda x: str(x),
                         p["coordinates"]) + ["0.0"]))
	style_id = "style-%s" % section['section_start_time']
        pm = KML.Placemark(
                KML.styleUrl("#%s" % line_style_id),
                KML.name(section['_id']),
                KML.LineString(
                        KML.tessellate(1),
                        KML.coordinates(" ".join(
                                map(lambda track_point: make_coord(track_point)
                                    ,section['track_points'])))
                )
        )
        start_point = section['section_start_point']
        end_point = section['section_end_point']
        start_time = mongodate_to_datetime(section["section_start_time"])
        end_time = mongodate_to_datetime(section["section_end_time"])
        start_point = KML.Placemark(
                KML.styleUrl("#%s" % start_icon_style_id),
                KML.name("Start: %s" % start_time),
                KML.description("Starting point"),
                KML.Point(KML.coordinates(make_coord_point(start_point)))
        )
        end_point = KML.Placemark(
                KML.styleUrl("#%s" % end_icon_style_id),
                KML.name("End: %s" % end_time),
                KML.description("Ending point"),
                KML.Point(KML.coordinates(make_coord_point(end_point)))
        )
        line_style = KML.Style(
                KML.LineStyle(
                        KML.color("ff%s" % color),
                        KML.width("5")
                )
        )
        line_style.set("id", line_style_id)
        start_icon_style = KML.Style(
                KML.IconStyle(
                        KML.color("ff%s" % color),
                        KML.scale("1.1"),
                        KML.Icon(
                                KML.href("http://www.gstatic.com/mapspro/images/stock/503-wht-blank_maps.png")
                        )
                )
        )
        start_icon_style.set("id", start_icon_style_id)
        end_icon_style = KML.Style(
                KML.IconStyle(
                        KML.color("ff%s" % color),
                        KML.scale("1.1"),
                        KML.Icon(
                                KML.href("http://www.gstatic.com/mapspro/images/stock/503-wht-blank_maps.png")
                        )
                )
        )
        end_icon_style.set("id", end_icon_style_id)
        fld = KML.Folder(
                KML.name(section['_id']),
                KML.description("From %s \nto %s" % (start_time, end_time)),
                pm,
                start_point,
                end_point
        )
        if write:
                kml = KML.kml(KML.Document(fld, section["user_id"]))
                path = os.path.join(outfile_path, str(section['user_id']) +'.kml')
                outfile = file(path,'w')
                outfile.write(etree.tostring(kml, pretty_print=True))
        else:
                return fld, line_style, start_icon_style, end_icon_style

def sections_to_kml(filename, sections, outfile_path=""):
        r = lambda: random.randint(0,255)
        l_tuples = map(lambda section: section_to_kml(section, '%02X%02X%02X' % (r(),r(),r()),  write=False) if len(section['track_points']) > 1 else None ,sections)
        flat = []

        for f,s,i_s,i_e in l_tuples:
                flat.append(f)
                flat.append(s)
                flat.append(i_s)
                flat.append(i_e)
        kml = KML.kml(KML.Document(sections[0]["user_id"],*flat))
        path = os.path.join(outfile_path, filename +'.kml')
        outfile = file(path,'w')
        outfile.write(etree.tostring(kml, pretty_print=True))

def chunks(l,n):
        """
        Generates evenly sized chunks of a list
        """
        for i in xrange(0, len(l), n):
                yield l[i:i+n]

def mongodate_to_datetime(date):
        return datetime.strptime(date[:-5], "%Y%m%dT%H%M%S")

if __name__ == "__main__":
        sys.path.append("%s/../" % os.getcwd())
        from get_database import get_section_db
        from uuid import UUID
        Sections = get_section_db()
        user_id = "__ID_GOES_HERE__"
        sections = Sections.find({'$and': [{'user_id':user_id}]})
        sections_to_kml(user_id,sections)
<|endoftext|>


<commit_before>import numpy as np
import time
from Neural_Networks import NNGPU

def Read_CIFAR_10(path, number_training, number_test, image, target_output):
    file = open(path + "data_batch_1.bin", 'rb')
    filename = ["data_batch_1.bin", "data_batch_2.bin", "data_batch_3.bin", "data_batch_4.bin", "data_batch_5.bin", "test_batch.bin"]
    index = 0

    for h in range(number_training):
        if h % 10000 == 0:
            if not file.closed:
                file.close()

            file = open(path + filename[index], 'rb')
            index += 1

        label = int.from_bytes(file.read(1), byteorder='little')

        for j in range(10):
            target_output[h][j] = (j == label)

        for j in range(3 * 32 * 32):
            image[h][j] = int.from_bytes(file.read(1), byteorder='little') / 255

    file.close()
    file = open(path + filename[5], 'rb')

    for h in range(number_training, number_training + number_test):
        label = int.from_bytes(file.read(1), byteorder='little')

        for j in range(10):
            target_output[h][j] = (j == label)

        for j in range(3 * 32 * 32):
            image[h][j] = int.from_bytes(file.read(1), byteorder='little') / 255

    file.close()

batch_size      = 100
decay_rate      = 0.977
epsilon         = 0.001
learning_rate   = 0.001
number_iterations = 50
number_training = 50000
number_test     = 10000

# train from scratch
NN = NNGPU.Neural_Networks()

NN.Add_Layer("CIFAR-10", 3, 32, 32)
NN.Add_Layer("BN,ReLU", 48, 32, 32) # batch normalization, ReLU activation
NN.Add_Layer("BN,ReLU", 48, 32, 32) # batch normalization, ReLU activation
NN.Add_Layer("BN,ReLU", 48, 32, 32) # batch normalization, ReLU activation
NN.Add_Layer("", 48, 16, 16)
NN.Add_Layer("BN,ReLU", 48, 16, 16) # batch normalization, ReLU activation
NN.Add_Layer("BN,ReLU", 96, 16, 16) # batch normalization, ReLU activation
NN.Add_Layer("BN,ReLU", 96, 16, 16) # batch normalization, ReLU activation
NN.Add_Layer("BN,ReLU", 96, 16, 16) # batch normalization, ReLU activation
NN.Add_Layer("", 96, 8, 8)
NN.Add_Layer("BN,ReLU", 384)        # batch normalization, ReLU activation
NN.Add_Layer("CE,softmax", 10)      # cross-entropy loss, softmax activation

NN.Connect(1, 0, "W,kernel(3x3)")       # 3x3 convolution
NN.Connect(2, 1, "W,kernel(3x3),DS")    # 3x3 depthwise separable convolution
NN.Connect(3, 2, "W")                   # 1x1 convolution
NN.Connect(4, 3, "P,max")               # 2x2 max pooling
NN.Connect(5, 4, "W,kernel(3x3),DS")    # 3x3 depthwise separable convolution
NN.Connect(6, 5, "W")                   # 1x1 convolution
NN.Connect(7, 6, "W,kernel(3x3),DS")    # 3x3 depthwise separable convolution
NN.Connect(8, 7, "W")                   # 1x1 convolution
NN.Connect(9, 8, "P,max")               # 2x2 max pooling
NN.Connect(10, 9, "W")                  # fully connected
NN.Connect(11, 10, "W")                 # fully connected

NN.Initialize(0, 0.01)

# or load pretrained model
# NN = Neural_Networks("CIFAR-10_CNN.txt")

image = np.zeros((number_training + number_test, 3 * 32 * 32), dtype='f')
target_output = np.zeros((number_training + number_test, 10), dtype='f')

path = input("path where CIFAR-10 dataset is : ")
Read_CIFAR_10(path, number_training, number_test, image, target_output)

start = time.time()

for f in range(number_iterations):
    score = [0, 0]

    loss = NN.Train(batch_size, number_training, image, target_output, learning_rate, epsilon)

    h = 0

    for i in range(number_training + number_test):
        h = h + 1

        if h == batch_size or i == number_training + number_test - 1:
            _input = np.array(image[i - h + 1])
            output = np.empty((h, 10), dtype='f')

            for g in range(1, h):
                _input = np.append(_input, image[i - h + g + 1])

            NN.Test(h, _input, output)

            for g in range(h):
                score[0 if i - h + g + 1 < number_training else 1] += int(target_output[i - h + g + 1][np.argmax(output[g])])

            h = 0

    print('.', end='')
    NN.Save("NN.txt")
    print('score: {} / {}, {} / {}, loss = {:.6f}, step {}, {:.2f} sec'.format(score[0], number_training, score[1], number_test, loss, f + 1, time.time() - start))

    learning_rate *= decay_rate

<commit_msg>Update CIFAR-10_CNN.py<commit_after>import numpy as np
import time
from Neural_Networks import *

def Read_CIFAR_10(path, number_training, number_test, image, target_output):
    file = open(path + "data_batch_1.bin", 'rb')
    filename = ["data_batch_1.bin", "data_batch_2.bin", "data_batch_3.bin", "data_batch_4.bin", "data_batch_5.bin", "test_batch.bin"]
    index = 0

    for h in range(number_training):
        if h % 10000 == 0:
            if not file.closed:
                file.close()

            file = open(path + filename[index], 'rb')
            index += 1

        label = int.from_bytes(file.read(1), byteorder='little')

        for j in range(10):
            target_output[h][j] = (j == label)

        for j in range(3 * 32 * 32):
            image[h][j] = int.from_bytes(file.read(1), byteorder='little') / 255

    file.close()
    file = open(path + filename[5], 'rb')

    for h in range(number_training, number_training + number_test):
        label = int.from_bytes(file.read(1), byteorder='little')

        for j in range(10):
            target_output[h][j] = (j == label)

        for j in range(3 * 32 * 32):
            image[h][j] = int.from_bytes(file.read(1), byteorder='little') / 255

    file.close()

batch_size      = 100
decay_rate      = 0.977
epsilon         = 0.001
learning_rate   = 0.001
number_iterations = 50
number_training = 50000
number_test     = 10000

# train from scratch
NN = Neural_Networks()

NN.Add_Layer("CIFAR-10", 3, 32, 32)
NN.Add_Layer("BN,ReLU", 48, 32, 32) # batch normalization, ReLU activation
NN.Add_Layer("BN,ReLU", 48, 32, 32) # batch normalization, ReLU activation
NN.Add_Layer("BN,ReLU", 48, 32, 32) # batch normalization, ReLU activation
NN.Add_Layer("", 48, 16, 16)
NN.Add_Layer("BN,ReLU", 48, 16, 16) # batch normalization, ReLU activation
NN.Add_Layer("BN,ReLU", 96, 16, 16) # batch normalization, ReLU activation
NN.Add_Layer("BN,ReLU", 96, 16, 16) # batch normalization, ReLU activation
NN.Add_Layer("BN,ReLU", 96, 16, 16) # batch normalization, ReLU activation
NN.Add_Layer("", 96, 8, 8)
NN.Add_Layer("BN,ReLU", 384)        # batch normalization, ReLU activation
NN.Add_Layer("CE,softmax", 10)      # cross-entropy loss, softmax activation

NN.Connect(1, 0, "W,kernel(3x3)")       # 3x3 convolution
NN.Connect(2, 1, "W,kernel(3x3),DS")    # 3x3 depthwise separable convolution
NN.Connect(3, 2, "W")                   # 1x1 convolution
NN.Connect(4, 3, "P,max")               # 2x2 max pooling
NN.Connect(5, 4, "W,kernel(3x3),DS")    # 3x3 depthwise separable convolution
NN.Connect(6, 5, "W")                   # 1x1 convolution
NN.Connect(7, 6, "W,kernel(3x3),DS")    # 3x3 depthwise separable convolution
NN.Connect(8, 7, "W")                   # 1x1 convolution
NN.Connect(9, 8, "P,max")               # 2x2 max pooling
NN.Connect(10, 9, "W")                  # fully connected
NN.Connect(11, 10, "W")                 # fully connected

NN.Initialize(0, 0.01)

# or load pretrained model
# NN = Neural_Networks("CIFAR-10_CNN.txt")

image = np.zeros((number_training + number_test, 3 * 32 * 32), dtype='f')
target_output = np.zeros((number_training + number_test, 10), dtype='f')

path = input("path where CIFAR-10 dataset is : ")
Read_CIFAR_10(path, number_training, number_test, image, target_output)

start = time.time()

for f in range(number_iterations):
    score = [0, 0]

    loss = NN.Train(batch_size, number_training, image, target_output, learning_rate, epsilon)

    h = 0

    for i in range(number_training + number_test):
        h = h + 1

        if h == batch_size or i == number_training + number_test - 1:
            _input = np.array(image[i - h + 1])
            output = np.empty((h, 10), dtype='f')

            for g in range(1, h):
                _input = np.append(_input, image[i - h + g + 1])

            NN.Test(h, _input, output)

            for g in range(h):
                score[0 if i - h + g + 1 < number_training else 1] += int(target_output[i - h + g + 1][np.argmax(output[g])])

            h = 0

    print('.', end='')
    NN.Save("NN.txt")
    print('score: {} / {}, {} / {}, loss = {:.6f}, step {}, {:.2f} sec'.format(score[0], number_training, score[1], number_test, loss, f + 1, time.time() - start))

    learning_rate *= decay_rate

<|endoftext|>


<commit_before>from Object import Object
import xml.etree.ElementTree as ElementTree

class GapJunction(Object):

    # TODO: gap junctions can be directional

    def __init__(self, network, neurite1, neurite2, *args, **keywords):
        Object.__init__(self, network, *args, **keywords)
        self.neurites = set([neurite1, neurite2])

    @classmethod
    def fromXMLElement(cls, network, xmlElement):
        object = super(GapJunction, cls).fromXMLElement(network, xmlElement)
        neurite1Id = xmlElement.get('neurite1Id')
        neurite1 = network.objectWithId(neurite1Id)
        neurite2Id = xmlElement.get('neurite2Id')
        neurite2 = network.objectWithId(neurite2Id)
        if neurite1 is None:
            raise ValueError, gettext('Neurite with id "%s" does not exist') % (neurite1Id)
        elif neurite2 is None:
            raise ValueError, gettext('Neurite with id "%s" does not exist') % (neurite2Id)
        else:
            object.neurites = set([neurite1, neurite2])
            neurite1._gapJunctions.append(object)
            neurite2._gapJunctions.append(object)
            return object

    def toXMLElement(self, parentElement):
        gapJunctionElement = Object.toXMLElement(self, parentElement)
        neurites = list(self.neurites)
        gapJunctionElement.set('neurite1Id', str(neurites[0].networkId))
        gapJunctionElement.set('neurite2Id', str(neurites[1].networkId))
        return gapJunctionElement

    def creationScriptCommand(self, scriptRefs):
        neurite1, neurite2 = list(self.neurites)
        if neurite1.networkId not in scriptRefs:
            neurite1 = neurite1.root
        return scriptRefs[neurite1.networkId] + '.gapJunctionWith'

    def creationScriptParams(self, scriptRefs):
        args, keywords = Object.creationScriptParams(self, scriptRefs)
        neurite1, neurite2 = list(self.neurites)
        if neurite2.networkId not in scriptRefs:
            neurite2 = neurite2.root
        args.insert(0, scriptRefs[neurite2.networkId])
        return (args, keywords)
<commit_msg>Added custom class display name.<commit_after>from Object import Object
import xml.etree.ElementTree as ElementTree

class GapJunction(Object):

    # TODO: gap junctions can be directional

    def __init__(self, network, neurite1, neurite2, *args, **keywords):
        Object.__init__(self, network, *args, **keywords)
        self.neurites = set([neurite1, neurite2])

    @classmethod
    def fromXMLElement(cls, network, xmlElement):
        object = super(GapJunction, cls).fromXMLElement(network, xmlElement)
        neurite1Id = xmlElement.get('neurite1Id')
        neurite1 = network.objectWithId(neurite1Id)
        neurite2Id = xmlElement.get('neurite2Id')
        neurite2 = network.objectWithId(neurite2Id)
        if neurite1 is None:
            raise ValueError, gettext('Neurite with id "%s" does not exist') % (neurite1Id)
        elif neurite2 is None:
            raise ValueError, gettext('Neurite with id "%s" does not exist') % (neurite2Id)
        else:
            object.neurites = set([neurite1, neurite2])
            neurite1._gapJunctions.append(object)
            neurite2._gapJunctions.append(object)
            return object

    def toXMLElement(self, parentElement):
        gapJunctionElement = Object.toXMLElement(self, parentElement)
        neurites = list(self.neurites)
        gapJunctionElement.set('neurite1Id', str(neurites[0].networkId))
        gapJunctionElement.set('neurite2Id', str(neurites[1].networkId))
        return gapJunctionElement

    def creationScriptCommand(self, scriptRefs):
        neurite1, neurite2 = list(self.neurites)
        if neurite1.networkId not in scriptRefs:
            neurite1 = neurite1.root
        return scriptRefs[neurite1.networkId] + '.gapJunctionWith'

    def creationScriptParams(self, scriptRefs):
        args, keywords = Object.creationScriptParams(self, scriptRefs)
        neurite1, neurite2 = list(self.neurites)
        if neurite2.networkId not in scriptRefs:
            neurite2 = neurite2.root
        args.insert(0, scriptRefs[neurite2.networkId])
        return (args, keywords)

    @classmethod
    def displayName(cls):
        return gettext('Gap Junction')
<|endoftext|>


<commit_before>def execute():
	import webnotes
	from webnotes.utils.nestedset import rebuild_tree
	rebuild_tree('Account', 'parent_account')

	roots = webnotes.conn.sql("""
		select lft, rgt, debit_or_credit, is_pl_account, company from `tabAccount`
		where ifnull(parent_account, '') = ''
	""", as_dict=1)

	for acc in roots:
		webnotes.conn.sql("""update tabAccount set debit_or_credit = %(debit_or_credit)s,
			is_pl_account = %(is_pl_account)s, company = %(company)s
			where lft > %(lft)s and rgt < %(rgt)s""", acc, debug=1)<commit_msg>[patches] [fix] fix in update_account_property_patch<commit_after>def execute():
	import webnotes
	from webnotes.utils.nestedset import rebuild_tree
	rebuild_tree('Account', 'parent_account')

	roots = webnotes.conn.sql("""
		select lft, rgt, debit_or_credit, is_pl_account, company from `tabAccount`
		where ifnull(parent_account, '') = ''
	""", as_dict=1)

	for acc in roots:
		webnotes.conn.sql("""update tabAccount set debit_or_credit = %(debit_or_credit)s,
			is_pl_account = %(is_pl_account)s, company = %(company)s
			where lft > %(lft)s and rgt < %(rgt)s""", acc)<|endoftext|>


<commit_before>#########
# Copyright (c) 2013 GigaSpaces Technologies Ltd. All rights reserved
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#       http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
#  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#  * See the License for the specific language governing permissions and
#  * limitations under the License.

from multiprocessing import Process
import subprocess
import SimpleHTTPServer
import SocketServer
import os
import sys
import socket
import time

PORT = 53229
FNULL = open(os.devnull, 'w')

class FileServer(object):

    def __init__(self, root_path, use_subprocess=False, port=PORT):
        self.root_path = root_path
        self.process = Process(target=self.start_impl)
        self.use_subprocess = use_subprocess
        self.port = port

    def start(self):
        if self.use_subprocess:
            subprocess.Popen(
                [sys.executable, __file__, self.root_path],
                stdin=FNULL,
                stdout=FNULL,
                stderr=FNULL)
        else:
            self.process.start()

    def stop(self):
        try:
            self.process.terminate()
            while self.is_alive():
                time.sleep(0.1)
        except BaseException:
            pass

    def start_impl(self):
        import logging
        logging.basicConfig(level=logging.DEBUG)
        logging.info('Starting file server and serving files from: %s',
                     self.root_path)
        os.chdir(self.root_path)

        class TCPServer(SocketServer.TCPServer):
            allow_reuse_address = True
        httpd = TCPServer(('0.0.0.0', self.port),
                          SimpleHTTPServer.SimpleHTTPRequestHandler)
        httpd.serve_forever()

    def is_alive(self):
        s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
        try:
            s.connect(('localhost', self.port))
            s.close()
            return True
        except socket.error:
            return False

if __name__ == '__main__':
    FileServer(sys.argv[1]).start_impl()
<commit_msg>testing fileserver stop failure<commit_after>#########
# Copyright (c) 2013 GigaSpaces Technologies Ltd. All rights reserved
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#       http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
#  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#  * See the License for the specific language governing permissions and
#  * limitations under the License.

from multiprocessing import Process
import subprocess
import SimpleHTTPServer
import SocketServer
import os
import sys
import socket
import time

PORT = 53229
FNULL = open(os.devnull, 'w')

class FileServer(object):

    def __init__(self, root_path, use_subprocess=False, port=PORT):
        self.root_path = root_path
        self.process = Process(target=self.start_impl)
        self.use_subprocess = use_subprocess
        self.port = port

    def start(self):
        if self.use_subprocess:
            subprocess.Popen(
                [sys.executable, __file__, self.root_path],
                stdin=FNULL,
                stdout=FNULL,
                stderr=FNULL)
        else:
            self.process.start()

    def stop(self):
        try:
            self.process.terminate()
            while self.is_alive():
                time.sleep(0.1)
        except BaseException as e:
            # pass
            exc_type, exc, traceback = sys.exc_info()
            raise RuntimeError('Failed to stop file_server, error: {1}'
                               .format(e)), None, traceback

    def start_impl(self):
        import logging
        logging.basicConfig(level=logging.DEBUG)
        logging.info('Starting file server and serving files from: %s',
                     self.root_path)
        os.chdir(self.root_path)

        class TCPServer(SocketServer.TCPServer):
            allow_reuse_address = True
        httpd = TCPServer(('0.0.0.0', self.port),
                          SimpleHTTPServer.SimpleHTTPRequestHandler)
        httpd.serve_forever()

    def is_alive(self):
        s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
        try:
            s.connect(('localhost', self.port))
            s.close()
            return True
        except socket.error:
            return False

if __name__ == '__main__':
    FileServer(sys.argv[1]).start_impl()
<|endoftext|>