<commit_before>from django.http import HttpResponse
from django.http import HttpRequest
from django.utils import timezone
import datetime
from .models import Log, Blocked

# block IP when number of events reaches maxAllowed
def add_event(ipAddress, eventName, findTime, maxAllowed):

    # remove events older than findTime
    now = timezone.now()
    dateLim = now - datetime.timedelta(minutes=findTime)
    Log.objects.filter(IpAddress=ipAddress, EventDate__lt=dateLim, EventName=eventName).delete()

    # add new entry
    obj = Log(IpAddress=ipAddress, EventName=eventName)
    obj.save()

    # block IP if counted number equals limit
    num = Log.objects.filter(IpAddress=ipAddress, EventName=eventName).count()
    if num >= maxAllowed:
        # update date of oldest record
        obj = Log.objects.filter(IpAddress=ipAddress, EventName=eventName).order_by('EventDate')[0]
        obj.EventDate = now
        obj.save()

        # add record to banned list, or if it already exists, update the datetime
        obj = Blocked.objects.get_or_create(IpAddress=ipAddress, EventName=eventName)[0]
        obj.BlockDate = now
        obj.save()

def is_ip_blocked(ipAddress, eventName, blockTime):
    # remove blocked IPs older than blockTime
    now = timezone.now()
    dateLim = now - datetime.timedelta(minutes=blockTime)
    Blocked.objects.filter(IpAddress=ipAddress, BlockDate__lt=dateLim, EventName=eventName).delete()

    # check if any entries are still left
    isBlocked = Blocked.objects.filter(IpAddress=ipAddress, EventName=eventName).exists()

    return isBlocked

# logic will not add events once IP has been blocked
# Note: function without an HttpRequest object ought to raise an exception
def filt_req(eventName, blockTime, findTime, maxAllowed, filtFunc = lambda request: True):
    def real_decorator(viewFunc):
        def wrapper(*args):
            # iterate over all arguments to the view function
            for request in args:
                # first make sure that we have found HttpRequest object
                if isinstance (request, HttpRequest):
                    # get IP address of remote client
                    remoteAddress = request.META.get('REMOTE_ADDR')
                    if is_ip_blocked(remoteAddress, eventName, blockTime):
                        result = view_blocked()
                    else:
                        if filtFunc(request):
                            add_event(remoteAddress, eventName, findTime, maxAllowed)
                        result = viewFunc(*args)
                    return result

        return wrapper
    return real_decorator

def view_blocked():
    msg = "Sorry! This page has been locked."
    html = "".join(("
", msg, "
"))
    return HttpResponse(html)
<commit_msg>Remove legacy code and reduce database size.<commit_after>from django.http import HttpResponse
from django.http import HttpRequest
from django.utils import timezone
import datetime
from .models import Log, Blocked

# block IP when number of events reaches maxAllowed
def add_event(ipAddress, eventName, findTime, maxAllowed):

    now = timezone.now()

    # remove events older than findTime
    dateLim = now - datetime.timedelta(minutes=findTime)
    Log.objects.filter(EventName=eventName, EventDate__lt=dateLim).delete()

    # add new entry
    obj = Log(IpAddress=ipAddress, EventName=eventName)
    obj.save()

    # block IP if counted number equals limit
    num = Log.objects.filter(IpAddress=ipAddress, EventName=eventName).count()
    if num >= maxAllowed:
        obj = Blocked.objects.get_or_create(IpAddress=ipAddress, EventName=eventName)[0]
        obj.BlockDate = now
        obj.save()

def is_ip_blocked(ipAddress, eventName, blockTime):
    # remove blocked IPs older than blockTime
    now = timezone.now()
    dateLim = now - datetime.timedelta(minutes=blockTime)
    Blocked.objects.filter(IpAddress=ipAddress, BlockDate__lt=dateLim, EventName=eventName).delete()

    # check if any entries are still left
    isBlocked = Blocked.objects.filter(IpAddress=ipAddress, EventName=eventName).exists()

    return isBlocked

# logic will not add events once IP has been blocked
# Note: function without an HttpRequest object ought to raise an exception
def filt_req(eventName, blockTime, findTime, maxAllowed, filtFunc = lambda request: True):
    def real_decorator(viewFunc):
        def wrapper(*args):
            # iterate to make sure that we have found the HttpRequest object
            for request in args:
                if isinstance (request, HttpRequest):
                    # get IP address of remote client
                    remoteAddress = request.META.get('REMOTE_ADDR')
                    if is_ip_blocked(remoteAddress, eventName, blockTime):
                        result = view_blocked()
                    else:
                        if filtFunc(request):
                            add_event(remoteAddress, eventName, findTime, maxAllowed)
                        result = viewFunc(*args)
                    return result

        return wrapper
    return real_decorator

def view_blocked():
    msg = "Sorry! This page has been locked."
    html = "".join(("
", msg, "
"))
    return HttpResponse(html)
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-

"""Console script for muver."""

import click

from allelic_fraction import get_allelic_fractions
from bias_distribution import calculate_bias_distribution_bam
from call_mutations import call_mutations as _call_mutations
from depth_correction import write_corrected_bedgraph
from depth_distribution import (calculate_depth_distribution_bedgraph,
                                filter_regions_by_depth_bedgraph)
from depth_ratios import calculate_depth_ratios as _calculate_depth_ratios
from pipeline import run_pipeline as _run_pipeline
from reference import create_reference_indices, read_chrom_sizes
from repeat_indels import fit_repeat_indel_rates as _fit_repeat_indel_rates
from repeats import create_repeat_file as _create_repeat_file
from utils import read_repeats
from wrappers.samtools import get_mpileup_depths

@click.group()
def main(args=None):
    pass

@main.command()
@click.option('--processes', '-p', default=1, type=int,
              help='Number of processes to use.')
@click.option('--excluded_regions', default=None, type=click.Path(exists=True),
              help='Regions to exclude in mutation calling (BED format).')
@click.argument('reference_assembly', type=click.Path(exists=True))
@click.argument('fastq_list', type=click.Path(exists=True))
@click.argument('control_sample_name', type=str)
@click.argument('experiment_directory', type=str)
def run_pipeline(reference_assembly, fastq_list, control_sample_name,
                 experiment_directory, processes, excluded_regions):
    '''
    Run MuVer pipeline, starting with FASTQ files.

    FASTQ files and other parameters for each sample are specified in the
    FASTQ_LIST, a TXT file in tab-delimited format. For file specification,
    see tool documentation.

    Accepted fields for the FASTQ_LIST include:

    \b
    "Sample Name"
    "Mate 1 FASTQ"
    "Mate 2 FASTQ"
    "Ploidy"
    "CNV BedGraph"
    '''
    _run_pipeline(
        reference_assembly,
        fastq_list,
        control_sample_name,
        experiment_directory,
        p=processes,
        excluded_regions=excluded_regions,
    )

@main.command()
@click.option('--excluded_regions', default=None, type=click.Path(exists=True),
              help='Regions to exclude in mutation calling (BED format).')
@click.argument('reference_assembly', type=click.Path(exists=True))
@click.argument('control_sample_name', type=str)
@click.argument('sample_list', type=click.Path(exists=True))
@click.argument('input_vcf', type=click.Path(exists=True))
@click.argument('output_header', type=str)
def call_mutations(reference_assembly, control_sample_name, sample_list,
                  input_vcf, output_header, excluded_regions):
    '''
    Call mutations from a HaplotypeCaller VCF file.

    Parameters for each sample are specified in the SAMPLE_LIST, a TXT file in
    tab-delimited format. For file specification, see tool documentation.

    Accepted fields for the SAMPLE_LIST include:

    \b
    "Sample Name"
    "Ploidy"
    "CNV BedGraph"
    "Strand Bias STD"
    "Filtered Sites"
    "Repeat Indel Fits"
    '''
    _call_mutations(
        reference_assembly,
        control_sample_name,
        sample_list,
        input_vcf,
        output_header,
        excluded_regions=excluded_regions,
    )

@main.command()
@click.argument('reference_assembly', type=click.Path(exists=True))
def index_reference(reference_assembly):
    '''
    Index the reference using Bowtie2, picard, and samtools.
    '''
    create_reference_indices(
        reference_assembly,
    )

@main.command()
@click.option('--mean', default=None, type=float,
              help='Manually specify the mean.')
@click.argument('bedgraph_file', type=click.Path(exists=True))
@click.argument('reference_assembly', type=click.Path(exists=True))
@click.argument('output_file', type=str)
def calculate_depth_ratios(bedgraph_file, reference_assembly, output_file,
                           mean):
    '''
    Considering distance from chromosome ends, find the median ratio of depth
    to the average over 500-bp bins.
    '''
    _calculate_depth_ratios(
        bedgraph_file,
        reference_assembly,
        output_file,
        mean=mean,
    )

@main.command()
@click.argument('bam_file', type=click.Path(exists=True))
@click.argument('reference_assembly', type=click.Path(exists=True))
@click.argument('output_file', type=str)
def plot_allelic_fraction(bam_file, reference_assembly, output_file):
    '''
    Plot allelic fractions over a BAM file.

    The OUTPUT_FILE contains a histogram of allelic fractions (tab-delimited
    TXT format).
    '''
    get_allelic_fractions(
        bam_file,
        reference_assembly,
        output_file,
    )

@main.command()
@click.argument('fasta_file', type=click.Path(exists=True))
@click.argument('output_repeat_file', type=str)
def create_repeat_file(fasta_file, output_repeat_file):
    '''
    Create repeat file for the FASTA_FILE sequence.
    '''
    _create_repeat_file(
        fasta_file,
        output_repeat_file,
    )

@main.command()
@click.argument('y_int', type=float)
@click.argument('scalar', type=float)
@click.argument('mean_log', type=float)
@click.argument('sd_log', type=float)
@click.argument('slope', type=float)
@click.argument('reference_assembly', type=click.Path(exists=True))
@click.argument('input_bedgraph', type=click.Path(exists=True))
@click.argument('output_bedgraph', type=str)
def correct_depths(y_int, scalar, mean_log, sd_log, slope, reference_assembly,
                     input_bedgraph, output_bedgraph):
    '''
    Correct values in a depth bedGraph file.

    Correction is performed using the sum of a log-normal cumulative
    distribution function and linear function.
    '''
    chrom_sizes = read_chrom_sizes(reference_assembly)
    write_corrected_bedgraph(
        input_bedgraph,
        chrom_sizes,
        output_bedgraph,
        y_int,
        scalar,
        mean_log,
        sd_log,
        slope,
    )

@main.command()
@click.argument('bam_file', type=click.Path(exists=True))
@click.argument('reference_assembly', type=click.Path(exists=True))
@click.argument('output_bedgraph', type=str)
def calculate_read_depths(bam_file, reference_assembly, output_bedgraph):
    '''
    Give depths in a BAM_FILE in bedGraph format.

    Depth values are calculated using the mpileup function of samtools.
    '''
    get_mpileup_depths(
        bam_file,
        reference_assembly,
        output_bedgraph,
    )

@main.command()
@click.argument('bam_file', type=click.Path(exists=True))
@click.argument('reference_assembly', type=click.Path(exists=True))
@click.argument('output_bias_distribution', type=str)
def calculate_bias_distribution(bam_file, reference_assembly,
                                output_bias_distribution):
    '''
    Calculate distribution of strand biases.
    '''
    calculate_bias_distribution_bam(
        bam_file,
        reference_assembly,
        output_bias_distribution,
    )

@main.command()
@click.option('--output_filtered_regions', type=str, default=None,
              help='If OUTPUT_FILTERED_REGIONS is specified, positions to be '
              'filtered based on abnormal depth will be written to this file.')
@click.argument('bedgraph_file', type=click.Path(exists=True))
@click.argument('reference_assembly', type=click.Path(exists=True))
@click.argument('output_depth_distribution', type=str)
def calculate_depth_distribution(bedgraph_file, reference_assembly,
                                 output_depth_distribution,
                                 output_filtered_regions):
    '''
    Calculate distribution of depths in a bedGraph file.
    '''
    mu, sigma = calculate_depth_distribution_bedgraph(
        bedgraph_file,
        output_depth_distribution,
    )
    chrom_sizes = read_chrom_sizes(reference_assembly)
    if output_filtered_regions:
        filter_regions_by_depth_bedgraph(
            bedgraph_file,
            chrom_sizes,
            mu,
            sigma,
            output_filtered_regions,
        )

@main.command()
@click.option('--output_plot_header', type=str, default=None,
              help='If OUTPUT_PLOT_HEADER is specified, PNGs are written '
              'that show the fit relative to the observed repeat indel rates.')
@click.argument('bam_file', type=click.Path(exists=True))
@click.argument('repeats_file', type=click.Path(exists=True))
@click.argument('output_fits_file', type=str)
def fit_repeat_indel_rates(bam_file, repeats_file, output_fits_file,
                           output_plot_header):
    '''
    Fit a logistic function to log-transformed repeat indel rates.

    The OUTPUT_FITS_FILE gives the parameters for the derived fits.
    '''
    repeats = read_repeats(repeats_file)
    _fit_repeat_indel_rates(
        repeats,
        bam_file,
        output_fits_file,
        output_plot_header,
    )

if __name__ == "__main__":
    main()
<commit_msg>Now generates the repeat file in place<commit_after># -*- coding: utf-8 -*-

"""Console script for muver."""

import click
import os

from allelic_fraction import get_allelic_fractions
from bias_distribution import calculate_bias_distribution_bam
from call_mutations import call_mutations as _call_mutations
from depth_correction import write_corrected_bedgraph
from depth_distribution import (calculate_depth_distribution_bedgraph,
                                filter_regions_by_depth_bedgraph)
from depth_ratios import calculate_depth_ratios as _calculate_depth_ratios
from pipeline import run_pipeline as _run_pipeline
from reference import create_reference_indices, read_chrom_sizes
from repeat_indels import fit_repeat_indel_rates as _fit_repeat_indel_rates
from repeats import create_repeat_file as _create_repeat_file
from utils import read_repeats
from wrappers.samtools import get_mpileup_depths

@click.group()
def main(args=None):
    pass

@main.command()
@click.option('--processes', '-p', default=1, type=int,
              help='Number of processes to use.')
@click.option('--excluded_regions', default=None, type=click.Path(exists=True),
              help='Regions to exclude in mutation calling (BED format).')
@click.argument('reference_assembly', type=click.Path(exists=True))
@click.argument('fastq_list', type=click.Path(exists=True))
@click.argument('control_sample_name', type=str)
@click.argument('experiment_directory', type=str)
def run_pipeline(reference_assembly, fastq_list, control_sample_name,
                 experiment_directory, processes, excluded_regions):
    '''
    Run MuVer pipeline, starting with FASTQ files.

    FASTQ files and other parameters for each sample are specified in the
    FASTQ_LIST, a TXT file in tab-delimited format. For file specification,
    see tool documentation.

    Accepted fields for the FASTQ_LIST include:

    \b
    "Sample Name"
    "Mate 1 FASTQ"
    "Mate 2 FASTQ"
    "Ploidy"
    "CNV BedGraph"
    '''
    _run_pipeline(
        reference_assembly,
        fastq_list,
        control_sample_name,
        experiment_directory,
        p=processes,
        excluded_regions=excluded_regions,
    )

@main.command()
@click.option('--excluded_regions', default=None, type=click.Path(exists=True),
              help='Regions to exclude in mutation calling (BED format).')
@click.argument('reference_assembly', type=click.Path(exists=True))
@click.argument('control_sample_name', type=str)
@click.argument('sample_list', type=click.Path(exists=True))
@click.argument('input_vcf', type=click.Path(exists=True))
@click.argument('output_header', type=str)
def call_mutations(reference_assembly, control_sample_name, sample_list,
                  input_vcf, output_header, excluded_regions):
    '''
    Call mutations from a HaplotypeCaller VCF file.

    Parameters for each sample are specified in the SAMPLE_LIST, a TXT file in
    tab-delimited format. For file specification, see tool documentation.

    Accepted fields for the SAMPLE_LIST include:

    \b
    "Sample Name"
    "Ploidy"
    "CNV BedGraph"
    "Strand Bias STD"
    "Filtered Sites"
    "Repeat Indel Fits"
    '''
    _call_mutations(
        reference_assembly,
        control_sample_name,
        sample_list,
        input_vcf,
        output_header,
        excluded_regions=excluded_regions,
    )

@main.command()
@click.argument('reference_assembly', type=click.Path(exists=True))
def index_reference(reference_assembly):
    '''
    Index the reference using Bowtie2, picard, and samtools.
    '''
    create_reference_indices(
        reference_assembly,
    )

@main.command()
@click.option('--mean', default=None, type=float,
              help='Manually specify the mean.')
@click.argument('bedgraph_file', type=click.Path(exists=True))
@click.argument('reference_assembly', type=click.Path(exists=True))
@click.argument('output_file', type=str)
def calculate_depth_ratios(bedgraph_file, reference_assembly, output_file,
                           mean):
    '''
    Considering distance from chromosome ends, find the median ratio of depth
    to the average over 500-bp bins.
    '''
    _calculate_depth_ratios(
        bedgraph_file,
        reference_assembly,
        output_file,
        mean=mean,
    )

@main.command()
@click.argument('bam_file', type=click.Path(exists=True))
@click.argument('reference_assembly', type=click.Path(exists=True))
@click.argument('output_file', type=str)
def plot_allelic_fraction(bam_file, reference_assembly, output_file):
    '''
    Plot allelic fractions over a BAM file.

    The OUTPUT_FILE contains a histogram of allelic fractions (tab-delimited
    TXT format).
    '''
    get_allelic_fractions(
        bam_file,
        reference_assembly,
        output_file,
    )

@main.command()
@click.option('--output_repeat_file', type=str,
    help='Specify the output file; otherwise generates the file in place.')
@click.argument('fasta_file', type=click.Path(exists=True))
def create_repeat_file(fasta_file, output_repeat_file):
    '''
    Create repeat file for the FASTA_FILE sequence.
    '''
    if output_repeat_file:
        out = output_repeat_file
    else:
        out = '{}.repeats'.format(os.path.splitext(fasta_file)[0])
    _create_repeat_file(
        fasta_file,
        out,
    )

@main.command()
@click.argument('y_int', type=float)
@click.argument('scalar', type=float)
@click.argument('mean_log', type=float)
@click.argument('sd_log', type=float)
@click.argument('slope', type=float)
@click.argument('reference_assembly', type=click.Path(exists=True))
@click.argument('input_bedgraph', type=click.Path(exists=True))
@click.argument('output_bedgraph', type=str)
def correct_depths(y_int, scalar, mean_log, sd_log, slope, reference_assembly,
                     input_bedgraph, output_bedgraph):
    '''
    Correct values in a depth bedGraph file.

    Correction is performed using the sum of a log-normal cumulative
    distribution function and linear function.
    '''
    chrom_sizes = read_chrom_sizes(reference_assembly)
    write_corrected_bedgraph(
        input_bedgraph,
        chrom_sizes,
        output_bedgraph,
        y_int,
        scalar,
        mean_log,
        sd_log,
        slope,
    )

@main.command()
@click.argument('bam_file', type=click.Path(exists=True))
@click.argument('reference_assembly', type=click.Path(exists=True))
@click.argument('output_bedgraph', type=str)
def calculate_read_depths(bam_file, reference_assembly, output_bedgraph):
    '''
    Give depths in a BAM_FILE in bedGraph format.

    Depth values are calculated using the mpileup function of samtools.
    '''
    get_mpileup_depths(
        bam_file,
        reference_assembly,
        output_bedgraph,
    )

@main.command()
@click.argument('bam_file', type=click.Path(exists=True))
@click.argument('reference_assembly', type=click.Path(exists=True))
@click.argument('output_bias_distribution', type=str)
def calculate_bias_distribution(bam_file, reference_assembly,
                                output_bias_distribution):
    '''
    Calculate distribution of strand biases.
    '''
    calculate_bias_distribution_bam(
        bam_file,
        reference_assembly,
        output_bias_distribution,
    )

@main.command()
@click.option('--output_filtered_regions', type=str, default=None,
              help='If OUTPUT_FILTERED_REGIONS is specified, positions to be '
              'filtered based on abnormal depth will be written to this file.')
@click.argument('bedgraph_file', type=click.Path(exists=True))
@click.argument('reference_assembly', type=click.Path(exists=True))
@click.argument('output_depth_distribution', type=str)
def calculate_depth_distribution(bedgraph_file, reference_assembly,
                                 output_depth_distribution,
                                 output_filtered_regions):
    '''
    Calculate distribution of depths in a bedGraph file.
    '''
    mu, sigma = calculate_depth_distribution_bedgraph(
        bedgraph_file,
        output_depth_distribution,
    )
    chrom_sizes = read_chrom_sizes(reference_assembly)
    if output_filtered_regions:
        filter_regions_by_depth_bedgraph(
            bedgraph_file,
            chrom_sizes,
            mu,
            sigma,
            output_filtered_regions,
        )

@main.command()
@click.option('--output_plot_header', type=str, default=None,
              help='If OUTPUT_PLOT_HEADER is specified, PNGs are written '
              'that show the fit relative to the observed repeat indel rates.')
@click.argument('bam_file', type=click.Path(exists=True))
@click.argument('repeats_file', type=click.Path(exists=True))
@click.argument('output_fits_file', type=str)
def fit_repeat_indel_rates(bam_file, repeats_file, output_fits_file,
                           output_plot_header):
    '''
    Fit a logistic function to log-transformed repeat indel rates.

    The OUTPUT_FITS_FILE gives the parameters for the derived fits.
    '''
    repeats = read_repeats(repeats_file)
    _fit_repeat_indel_rates(
        repeats,
        bam_file,
        output_fits_file,
        output_plot_header,
    )

if __name__ == "__main__":
    main()
<|endoftext|>


<commit_before>#!/usr/bin/env python
# Copyright (c) 2016 CNRS
# Author: Joseph Mirabel
#
# This file is part of hpp-corbaserver.
# hpp-corbaserver is free software: you can redistribute it
# and/or modify it under the terms of the GNU Lesser General Public
# License as published by the Free Software Foundation, either version
# 3 of the License, or (at your option) any later version.
#
# hpp-corbaserver is distributed in the hope that it will be
# useful, but WITHOUT ANY WARRANTY; without even the implied warranty
# of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
# General Lesser Public License for more details.  You should have
# received a copy of the GNU Lesser General Public License along with
# hpp-corbaserver.  If not, see
# <http://www.gnu.org/licenses/>.

import numpy as np

## \cond
class _BenchmarkIter (object):
    def __init__ (self, seedI, caseI, iterPerCaseI, case = None):
        self.seedI = seedI
        self.caseI = caseI
        self.case = case
        self.newCase = True;
        self.iterPerCaseI = iterPerCaseI

    def __repr__ (self):
        if self.case is None:
            return "iteration " + str(self.iterPerCaseI) + " for case " + str(self.caseI) + " at seed " + str(self.seedI)
        else:
            return "iteration " + str(self.iterPerCaseI) + " for case " + str(case) + " at seed " + str(self.seedI)

class _BenchmarkIterator (object):
    def __init__(self, seedRange, cases, iterPerCase, initCase, startAt = None):
        self.seedRangeL = len(seedRange)
        self.casesL = len(cases)
        self.iterPerCase = iterPerCase
        if startAt is None:
            self.current = _BenchmarkIter (0, 0, 0)
        else:
            self.current = startAt
        self.start = True

    def __iter__(self):
        return self

    def next(self): # Python 3: def __next__(self)
        if self.start:
            self.newCase = True
            self.start = False
            return self.current

        self.current.newCase = False
        self.current.iterPerCaseI += 1
        if self.current.iterPerCaseI >= self.iterPerCase:
            self.current.seedI += 1
            if self.current.seedI >= self.seedRangeL:
                self.current.newCase = True
                self.current.caseI += 1
                if self.current.caseI >= self.casesL:
                    self.current.caseI = 0
                    raise StopIteration
                self.current.seedI = 0
            self.current.iterPerCaseI = 0
        return self.current
## \endcond

## class to do benchmarking
#
# Basic usage
# \code{.py}
# # Here goes your script to load the problem.
# robot = hpp.corbaserver.robot.Robot (...)
# ps = hpp.corbaserver.problem_solver.ProblemSolver (...)
# ...
#
# from hpp.corbaserver import Benchmark
# benchmark = Benchmark (robot.client, robot, ps)
# benchmark.iterPerCase = 10
# results = benchmark.do()
# \endcode
#
# \sa hpp.corbaserver.benchmark.Benchmark.do
# hpp.corbaserver.benchmark.Benchmark.seedRange
#
# Advanced usage
# \code{.py}
# from hpp.corbaserver import Benchmark
# b = Benchmark (robot.client, robot, ps)
# b.seedRange = range (10)
# b.iterPerCase = 10
#
# b.cases = list()
# for type in ["Progressive", "Global"]:
#     for param in [0.1, 0.2]:
#         b.cases.append((type,param))
# b.cases.append(("None",0.1))
#
# b.tryResumeAndDelete ()
#
# def initialize (bench, case, iter):
#     bench.ps.selectPathProjector (case[0], case[1])
#
# results = b.do(initCase = initialize)
# \endcode
#
# \sa
# hpp.corbaserver.benchmark.Benchmark.cases
# hpp.corbaserver.benchmark.Benchmark.iterPerCase
# hpp.corbaserver.benchmark.Benchmark.tryResumeAndDelete
class Benchmark (object):
    ## Used to transform HPP output into seconds
    toSeconds = np.array ([60*60,60,1,1e-3])
    ## The filename of the crash file.
    crashFile = "/tmp/resume.hpp.corbaserver.benchmark.pickle"

    def __init__ (self, client, robot, problemSolver):
        ## A list of seed to initialize the random generator.
        self.seedRange = range (1)
        ## A list of cases for which benchmarking will be done.
        self.cases = [None]
        ## Number of times one case is repeated (for one seed).
        self.iterPerCase = 1

        self.client = client
        self.robot = robot
        self.ps = problemSolver

        self.current = None
        # internal datas
        self.results = dict ()
        self.results['user'] = []
        self.results['time'] = []
        self.results['pathLength'] = []

    ## Solve the same problem for the specified cases, for various random seed.
    # \param initCase a function of 3 arguments:
    #        - the calling Benchmark instance
    #        - the current element of the list of Benchmark.cases
    #        - the current _BenchmarkIterator (Normally not useful).
    def do(self, initCase = lambda this, case, iter: None):
        for iter in _BenchmarkIterator (self.seedRange, self.cases, self.iterPerCase, initCase, startAt = self.current):
            self.client.problem.clearRoadmap ()
            self.client.problem.setRandomSeed (self.seedRange[iter.seedI])
            try:
                if iter.newCase:
                    print "======================================================="
                    print "Case ", self.getCase (iter)
                    print "======================================================="
                    self.results['user'].append (initCase (self, self.getCase(iter), iter))
                self.results['time'].append (self.client.problem.solve ())
                self.results['pathLength'].append (self.client.problem.pathLength (self.client.problem.numberPaths()-1))
            except:
                # write current data to restart at the same point
                self.current = iter
                self.writeResume ()
                print "\nOops, something went wrong.\nTo resume at the benchmark just before the crash, use method thisobject.tryResumeAndDelete () before calling method do()\n"
                raise

            print "Solved", iter, "in", self.results['time'][-1]
        i = 0
        nb = self.iterPerCase * len(self.seedRange)
        for c in self.cases:
            t = np.array (self.results['time'][i:i+nb]).dot (self.toSeconds)
            pl = np.array (self.results['pathLength'][i:i+nb])
            print "====================================================="
            print "Case", c
            print "Mean time (s):", np.mean(t)
            print "Std dev time (s):",  np.std(t)
            print "Average length:", np.mean(pl)
            print "std dev length:", np.std(pl)
            print "====================================================="
            i += nb
        t = np.array (self.results['time']).dot (self.toSeconds)
        pl = np.array (self.results['pathLength'])
        print "====================================================="
        print "All cases together"
        print "Mean time (s):", np.mean(t)
        print "Std dev time (s):",  np.std(t)
        print "Average length:", np.mean(pl)
        print "std dev length:", np.std(pl)
        print "====================================================="
        return t, pl

    def getCase (self, iter):
        return self.cases[iter.caseI]

    ## Write data to file.
    # \param filename if None, it uses member Benchmark.crashFile
    def writeResume (self, filename = None):
        if filename is None: fname = self.crashFile
        else: fname = filename
        import pickle as pk
        with open (fname, 'w') as f:
            pk.dump(self.cases, f)
            pk.dump(self.seedRange, f)
            pk.dump(self.iterPerCase, f)
            pk.dump(self.current, f)
            pk.dump(self.results, f)

    ## In case of crash of HPP, the benchmark class writes temporary datas to a file.
    #  The method will check if the crash file exists and:
    #  - if it exists, the benchmarking will be initialized at the state before the crash. No datas are lost.
    #    The crash file is deleted after having been loaded.
    #  - if it does not exist, the method does nothing.
    # \param filename if None, it uses member Benchmark.crashFile
    def tryResumeAndDelete (self, filename = None):
        if filename is None: fname = self.crashFile
        else: fname = filename
        import os
        if os.path.isfile (fname):
            print "Retrieving datas from file", fname
            self.resumeFrom (fname)
            os.remove (fname)

    def resumeFrom (self, fname):
        import pickle as pk
        with open (fname, 'r') as f:
            cases = pk.load(f)
            if not cases == self.cases:
                print "Cases are different.\nValue in file is :", cases, "\nValue in this instance was:\n", self.cases
            self.cases = cases

            seedRange = pk.load(f)
            if not seedRange == self.seedRange:
                print "Seed range is different.\nValue in file is :", seedRange, "\nValue in this instance was:\n", self.seedRange

            iterPerCase = pk.load(f)
            if not iterPerCase == self.iterPerCase:
                print "Number of iteration per case is different.\nValue in file is :", iterPerCase, "\nValue in this instance was:\n", self.iterPerCase
            self.iterPerCase = iterPerCase

            self.current = pk.load(f)

            self.results = pk.load(f)
<commit_msg>Do not reset case index in Benchmark class<commit_after>#!/usr/bin/env python
# Copyright (c) 2016 CNRS
# Author: Joseph Mirabel
#
# This file is part of hpp-corbaserver.
# hpp-corbaserver is free software: you can redistribute it
# and/or modify it under the terms of the GNU Lesser General Public
# License as published by the Free Software Foundation, either version
# 3 of the License, or (at your option) any later version.
#
# hpp-corbaserver is distributed in the hope that it will be
# useful, but WITHOUT ANY WARRANTY; without even the implied warranty
# of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
# General Lesser Public License for more details.  You should have
# received a copy of the GNU Lesser General Public License along with
# hpp-corbaserver.  If not, see
# <http://www.gnu.org/licenses/>.

import numpy as np

## \cond
class _BenchmarkIter (object):
    def __init__ (self, seedI, caseI, iterPerCaseI, case = None):
        self.seedI = seedI
        self.caseI = caseI
        self.case = case
        self.newCase = True;
        self.iterPerCaseI = iterPerCaseI

    def __repr__ (self):
        if self.case is None:
            return "iteration " + str(self.iterPerCaseI) + " for case " + str(self.caseI) + " at seed " + str(self.seedI)
        else:
            return "iteration " + str(self.iterPerCaseI) + " for case " + str(case) + " at seed " + str(self.seedI)

class _BenchmarkIterator (object):
    def __init__(self, seedRange, cases, iterPerCase, initCase, startAt = None):
        self.seedRangeL = len(seedRange)
        self.casesL = len(cases)
        self.iterPerCase = iterPerCase
        if startAt is None:
            self.current = _BenchmarkIter (0, 0, 0)
        else:
            self.current = startAt
        self.start = True

    def __iter__(self):
        return self

    def next(self): # Python 3: def __next__(self)
        if self.start:
            self.newCase = True
            self.start = False
            return self.current

        self.current.newCase = False
        self.current.iterPerCaseI += 1
        if self.current.iterPerCaseI >= self.iterPerCase:
            self.current.seedI += 1
            if self.current.seedI >= self.seedRangeL:
                self.current.newCase = True
                self.current.caseI += 1
                if self.current.caseI >= self.casesL:
                    raise StopIteration
                self.current.seedI = 0
            self.current.iterPerCaseI = 0
        return self.current
## \endcond

## class to do benchmarking
#
# Basic usage
# \code{.py}
# # Here goes your script to load the problem.
# robot = hpp.corbaserver.robot.Robot (...)
# ps = hpp.corbaserver.problem_solver.ProblemSolver (...)
# ...
#
# from hpp.corbaserver import Benchmark
# benchmark = Benchmark (robot.client, robot, ps)
# benchmark.iterPerCase = 10
# results = benchmark.do()
# \endcode
#
# \sa hpp.corbaserver.benchmark.Benchmark.do
# hpp.corbaserver.benchmark.Benchmark.seedRange
#
# Advanced usage
# \code{.py}
# from hpp.corbaserver import Benchmark
# b = Benchmark (robot.client, robot, ps)
# b.seedRange = range (10)
# b.iterPerCase = 10
#
# b.cases = list()
# for type in ["Progressive", "Global"]:
#     for param in [0.1, 0.2]:
#         b.cases.append((type,param))
# b.cases.append(("None",0.1))
#
# b.tryResumeAndDelete ()
#
# def initialize (bench, case, iter):
#     bench.ps.selectPathProjector (case[0], case[1])
#
# results = b.do(initCase = initialize)
# \endcode
#
# \sa
# hpp.corbaserver.benchmark.Benchmark.cases
# hpp.corbaserver.benchmark.Benchmark.iterPerCase
# hpp.corbaserver.benchmark.Benchmark.tryResumeAndDelete
class Benchmark (object):
    ## Used to transform HPP output into seconds
    toSeconds = np.array ([60*60,60,1,1e-3])
    ## The filename of the crash file.
    crashFile = "/tmp/resume.hpp.corbaserver.benchmark.pickle"

    def __init__ (self, client, robot, problemSolver):
        ## A list of seed to initialize the random generator.
        self.seedRange = range (1)
        ## A list of cases for which benchmarking will be done.
        self.cases = [None]
        ## Number of times one case is repeated (for one seed).
        self.iterPerCase = 1

        self.client = client
        self.robot = robot
        self.ps = problemSolver

        self.current = None
        # internal datas
        self.results = dict ()
        self.results['user'] = []
        self.results['time'] = []
        self.results['pathLength'] = []

    ## Solve the same problem for the specified cases, for various random seed.
    # \param initCase a function of 3 arguments:
    #        - the calling Benchmark instance
    #        - the current element of the list of Benchmark.cases
    #        - the current _BenchmarkIterator (Normally not useful).
    def do(self, initCase = lambda this, case, iter: None):
        for iter in _BenchmarkIterator (self.seedRange, self.cases, self.iterPerCase, initCase, startAt = self.current):
            self.client.problem.clearRoadmap ()
            self.client.problem.setRandomSeed (self.seedRange[iter.seedI])
            try:
                if iter.newCase:
                    print "======================================================="
                    print "Case ", self.getCase (iter)
                    print "======================================================="
                    self.results['user'].append (initCase (self, self.getCase(iter), iter))
                self.results['time'].append (self.client.problem.solve ())
                self.results['pathLength'].append (self.client.problem.pathLength (self.client.problem.numberPaths()-1))
            except:
                # write current data to restart at the same point
                self.current = iter
                self.writeResume ()
                print "\nOops, something went wrong.\nTo resume at the benchmark just before the crash, use method thisobject.tryResumeAndDelete () before calling method do()\n"
                raise

            print "Solved", iter, "in", self.results['time'][-1]
        i = 0
        nb = self.iterPerCase * len(self.seedRange)
        for c in self.cases:
            t = np.array (self.results['time'][i:i+nb]).dot (self.toSeconds)
            pl = np.array (self.results['pathLength'][i:i+nb])
            print "====================================================="
            print "Case", c
            print "Mean time (s):", np.mean(t)
            print "Std dev time (s):",  np.std(t)
            print "Average length:", np.mean(pl)
            print "std dev length:", np.std(pl)
            print "====================================================="
            i += nb
        t = np.array (self.results['time']).dot (self.toSeconds)
        pl = np.array (self.results['pathLength'])
        print "====================================================="
        print "All cases together"
        print "Mean time (s):", np.mean(t)
        print "Std dev time (s):",  np.std(t)
        print "Average length:", np.mean(pl)
        print "std dev length:", np.std(pl)
        print "====================================================="
        return t, pl

    def getCase (self, iter):
        return self.cases[iter.caseI]

    ## Write data to file.
    # \param filename if None, it uses member Benchmark.crashFile
    def writeResume (self, filename = None):
        if filename is None: fname = self.crashFile
        else: fname = filename
        import pickle as pk
        with open (fname, 'w') as f:
            pk.dump(self.cases, f)
            pk.dump(self.seedRange, f)
            pk.dump(self.iterPerCase, f)
            pk.dump(self.current, f)
            pk.dump(self.results, f)

    ## In case of crash of HPP, the benchmark class writes temporary datas to a file.
    #  The method will check if the crash file exists and:
    #  - if it exists, the benchmarking will be initialized at the state before the crash. No datas are lost.
    #    The crash file is deleted after having been loaded.
    #  - if it does not exist, the method does nothing.
    # \param filename if None, it uses member Benchmark.crashFile
    def tryResumeAndDelete (self, filename = None):
        if filename is None: fname = self.crashFile
        else: fname = filename
        import os
        if os.path.isfile (fname):
            print "Retrieving datas from file", fname
            self.resumeFrom (fname)
            os.remove (fname)

    def resumeFrom (self, fname):
        import pickle as pk
        with open (fname, 'r') as f:
            cases = pk.load(f)
            if not cases == self.cases:
                print "Cases are different.\nValue in file is :", cases, "\nValue in this instance was:\n", self.cases
            self.cases = cases

            seedRange = pk.load(f)
            if not seedRange == self.seedRange:
                print "Seed range is different.\nValue in file is :", seedRange, "\nValue in this instance was:\n", self.seedRange

            iterPerCase = pk.load(f)
            if not iterPerCase == self.iterPerCase:
                print "Number of iteration per case is different.\nValue in file is :", iterPerCase, "\nValue in this instance was:\n", self.iterPerCase
            self.iterPerCase = iterPerCase

            self.current = pk.load(f)

            self.results = pk.load(f)
<|endoftext|>


<commit_before>import re
import gettext
_ = gettext.gettext

from BeautifulSoup import BeautifulSoup, Declaration, Comment, Tag

import _base

class TreeWalker(_base.NonRecursiveTreeWalker):
    doctype_regexp = re.compile(
        r'(?P<name>[^\s]*)(\s*PUBLIC\s*"(?P<publicId>.*)"\s*"(?P<systemId1>.*)"|\s*SYSTEM\s*"(?P<systemId2>.*)")?')
    def getNodeDetails(self, node):
        if isinstance(node, BeautifulSoup): # Document or DocumentFragment
            return (_base.DOCUMENT,)

        elif isinstance(node, Declaration): # DocumentType
            string = unicode(node.string)
            #Slice needed to remove markup added during unicode conversion,
            #but only in some versions of BeautifulSoup/Python
            if string.startswith('<!') and string.endswith('>'):
                string = string[2:-1]
            m = self.doctype_regexp.match(string)
            #This regexp approach seems wrong and fragile
            #but beautiful soup stores the doctype as a single thing and we want the seperate bits
            #It should work as long as the tree is created by html5lib itself but may be wrong if it's
            #been modified at all
            #We could just feed to it a html5lib tokenizer, I guess...
            assert m is not None, "DOCTYPE did not match expected format"
            name = m.group('name')
            publicId = m.group('publicId')
            if publicId is not None:
                systemId = m.group('systemId1')
            else:
                systemId = m.group('systemId2')
            return _base.DOCTYPE, name, publicId or "", systemId or ""

        elif isinstance(node, Comment):
            string = unicode(node.string)
            if string.startswith('<!--') and string.endswith('-->'):
                string = string[4:-3]
            return _base.COMMENT, string

        elif isinstance(node, unicode): # TextNode
            return _base.TEXT, node

        elif isinstance(node, Tag): # Element
            return _base.ELEMENT, node.name, \
                dict(node.attrs).items(), node.contents
        else:
            return _base.UNKNOWN, node.__class__.__name__

    def getFirstChild(self, node):
        return node.contents[0]

    def getNextSibling(self, node):
        return node.nextSibling

    def getParentNode(self, node):
        return node.parent
<commit_msg>Fix beautifulsoup treewalker<commit_after>import re
import gettext
_ = gettext.gettext

from BeautifulSoup import BeautifulSoup, Declaration, Comment, Tag
from html5lib.constants import namespaces
import _base

class TreeWalker(_base.NonRecursiveTreeWalker):
    doctype_regexp = re.compile(
        r'(?P<name>[^\s]*)(\s*PUBLIC\s*"(?P<publicId>.*)"\s*"(?P<systemId1>.*)"|\s*SYSTEM\s*"(?P<systemId2>.*)")?')
    def getNodeDetails(self, node):
        if isinstance(node, BeautifulSoup): # Document or DocumentFragment
            return (_base.DOCUMENT,)

        elif isinstance(node, Declaration): # DocumentType
            string = unicode(node.string)
            #Slice needed to remove markup added during unicode conversion,
            #but only in some versions of BeautifulSoup/Python
            if string.startswith('<!') and string.endswith('>'):
                string = string[2:-1]
            m = self.doctype_regexp.match(string)
            #This regexp approach seems wrong and fragile
            #but beautiful soup stores the doctype as a single thing and we want the seperate bits
            #It should work as long as the tree is created by html5lib itself but may be wrong if it's
            #been modified at all
            #We could just feed to it a html5lib tokenizer, I guess...
            assert m is not None, "DOCTYPE did not match expected format"
            name = m.group('name')
            publicId = m.group('publicId')
            if publicId is not None:
                systemId = m.group('systemId1')
            else:
                systemId = m.group('systemId2')
            return _base.DOCTYPE, name, publicId or "", systemId or ""

        elif isinstance(node, Comment):
            string = unicode(node.string)
            if string.startswith('<!--') and string.endswith('-->'):
                string = string[4:-3]
            return _base.COMMENT, string

        elif isinstance(node, unicode): # TextNode
            return _base.TEXT, node

        elif isinstance(node, Tag): # Element
            return (_base.ELEMENT, namespaces["html"], node.name,
                    dict(node.attrs).items(), node.contents)
        else:
            return _base.UNKNOWN, node.__class__.__name__

    def getFirstChild(self, node):
        return node.contents[0]

    def getNextSibling(self, node):
        return node.nextSibling

    def getParentNode(self, node):
        return node.parent
<|endoftext|>


<commit_before>#!/usr/bin/env python

import sys
import math
import random
from textwrap import dedent

defaults = {
    'mysql_dir' : "/var/lib/mysql",

    'log_error' : "/var/log/mysqld.log",
    'slow_query_log_file' : "/var/log/mysqld-slow.log",

    'pid_file' : "/var/run/mysqld/mysqld.pid",

    'mysql_ram_gb' : 1,

    'query_cache_type' : 0,
    'query_cache_size' : 0,

    'long_query_time' : 2,
    'max_connections' : 500,

    'server_id' : random.randint(100000, 999999)
}

def output_my_cnf(_metaconf):

    print dedent("""

    [mysql]
    # CLIENT #
    port                           = 3306
    socket                         = {mysql_dir}/mysql.sock

    [mysqld]

    # GENERAL #
    user                           = mysql
    default-storage-engine         = InnoDB
    socket                         = {mysql_dir}/mysql.sock
    pid-file                       = {pid_file}

    # MyISAM #
    key-buffer-size                = 32M
    myisam-recover                 = FORCE,BACKUP

    # SAFETY #
    max-allowed-packet             = 16M
    max-connect-errors             = 1000000
    skip-name-resolve
    sql-mode                       = NO_ENGINE_SUBSTITUTION,NO_AUTO_CREATE_USER
    sysdate-is-now                 = 1
    innodb                         = FORCE
    innodb-strict-mode             = 1

    # DATA STORAGE #
    datadir                        = {mysql_dir}

    # SERVER ID #
    server-id                      = {server_id}

    # BINARY LOGGING #
    log-bin                        = {mysql_dir}/mysql-bin
    expire-logs-days               = 14
    sync-binlog                    = 1
    binlog-format                  = MIXED

    # CACHES AND LIMITS #
    max-connections                = {max_connections}
    tmp-table-size                 = 32M
    max-heap-table-size            = 32M
    query-cache-type               = {query_cache_type}
    query-cache-size               = {query_cache_size}
    thread-cache-size              = 50
    open-files-limit               = 65535
    table-definition-cache         = 1024
    table-open-cache               = 2048

    # INNODB #
    innodb-flush-method            = O_DIRECT
    innodb-log-files-in-group      = 2
    innodb-log-file-size           = {innodb_log_file_size}
    innodb-flush-log-at-trx-commit = 1
    innodb-file-per-table          = 1
    innodb-buffer-pool-size        = {innodb_buffer_pool_size}

    # LOGGING #
    log-error                      = {log_error}
    log-queries-not-using-indexes  = 1
    slow-query-log                 = 1
    slow-query-log-file            = {slow_query_log_file}
    log-queries-not-using-indexes  = OFF
    long_query_time                = 2

    [mysqldump]
    max-allowed-packet             = 16M

    """.format(**mycnf_make(_metaconf)))

def mycnf_innodb_log_file_size_MB(innodb_buffer_pool_size_GB):
    if innodb_buffer_pool_size_GB > 64:
        return '768M'
    if innodb_buffer_pool_size_GB > 24:
        return '512M'
    if innodb_buffer_pool_size_GB > 8:
        return '256M'
    if innodb_buffer_pool_size_GB > 2:
        return '128M'

    return '64M'

def output_memory_gb(gb):

    if math.fabs(math.ceil(gb) - gb) < 0.01:
        return str(int(gb))+'G'

    return str(int(gb*1024))+'M'

def mycnf_make(m):

    m['innodb_buffer_pool_size'] = output_memory_gb(float(m['mysql_ram_gb']) *  0.75)
    m['innodb_log_file_size'] = mycnf_innodb_log_file_size_MB(m['mysql_ram_gb'])
    return m

def main(argv):
    actual_conf = defaults
    for arg in argv:
        kv = arg.split('=')
        if len(kv) == 2:
            actual_conf[kv[0]] = kv[1]

    output_my_cnf(actual_conf)
    return 0

if __name__ == "__main__":
    sys.exit(main(sys.argv))

<commit_msg>- more changes<commit_after>#!/usr/bin/env python

import sys
import math
import random
from textwrap import dedent

defaults = {
    'mysql_dir' : "/var/lib/mysql",

    'log_error' : "/var/log/mysqld.log",
    'slow_query_log_file' : "/var/log/mysqld-slow.log",

    'pid_file' : "/var/run/mysqld/mysqld.pid",

    'mysql_ram_gb' : 1,

    'query_cache_type' : 0,
    'query_cache_size' : 0,

    'long_query_time' : 2,
    'max_connections' : 500,

    'server_id' : random.randint(100000, 999999)
}

def output_my_cnf(_metaconf):

    print dedent("""

    [mysql]
    # CLIENT #
    port                           = 3306
    socket                         = {mysql_dir}/mysql.sock

    [mysqld]

    # GENERAL #
    user                           = mysql
    default-storage-engine         = InnoDB
    socket                         = {mysql_dir}/mysql.sock
    pid-file                       = {pid_file}

    # MyISAM #
    # key-buffer-size                = 32M
    # myisam-recover                 = FORCE,BACKUP

    # SAFETY #
    max-allowed-packet             = 16M
    max-connect-errors             = 1000000
    skip-name-resolve
    sql-mode                       = NO_ENGINE_SUBSTITUTION,NO_AUTO_CREATE_USER
    sysdate-is-now                 = 1
    innodb-strict-mode             = 1

    # DATA STORAGE #
    datadir                        = {mysql_dir}

    # SERVER ID #
    server-id                      = {server_id}

    # BINARY LOGGING #
    log-bin

    # CACHES AND LIMITS #
    max-connections                = {max_connections}
    tmp-table-size                 = 32M
    max-heap-table-size            = 32M
    query-cache-type               = {query_cache_type}
    query-cache-size               = {query_cache_size}
    thread-cache-size              = 50
    open-files-limit               = 65535
    table-definition-cache         = 1024
    table-open-cache               = 2048

    # INNODB #
    innodb-flush-method            = O_DIRECT
    innodb-log-files-in-group      = 2
    innodb-log-file-size           = {innodb_log_file_size}
    innodb-flush-log-at-trx-commit = 1
    innodb-file-per-table          = 1
    innodb-buffer-pool-size        = {innodb_buffer_pool_size}

    # LOGGING #
    log-error                      = {log_error}
    slow-query-log                 = 1
    slow-query-log-file            = {slow_query_log_file}
    log-queries-not-using-indexes  = OFF
    long_query_time                = 30

    [mysqldump]
    max-allowed-packet             = 16M

    """.format(**mycnf_make(_metaconf)))

def mycnf_innodb_log_file_size_MB(innodb_buffer_pool_size_GB):
    if innodb_buffer_pool_size_GB > 64:
        return '768M'
    if innodb_buffer_pool_size_GB > 24:
        return '512M'
    if innodb_buffer_pool_size_GB > 8:
        return '256M'
    if innodb_buffer_pool_size_GB > 2:
        return '128M'

    return '64M'

def output_memory_gb(gb):

    if math.fabs(math.ceil(gb) - gb) < 0.01:
        return str(int(gb))+'G'

    return str(int(gb*1024))+'M'

def mycnf_make(m):

    m['innodb_buffer_pool_size'] = output_memory_gb(float(m['mysql_ram_gb']) *  0.75)
    m['innodb_log_file_size'] = mycnf_innodb_log_file_size_MB(m['mysql_ram_gb'])
    return m

def main(argv):
    actual_conf = defaults
    for arg in argv:
        kv = arg.split('=')
        if len(kv) == 2:
            actual_conf[kv[0]] = kv[1]

    output_my_cnf(actual_conf)
    return 0

if __name__ == "__main__":
    sys.exit(main(sys.argv))

<|endoftext|>


<commit_before># -*- coding: utf-8 -*-

"""
"""

import os
import datetime

from PyQt5 import QtWidgets
from PyQt5 import QtGui
from PyQt5 import QtCore

from . import mainwindow_ui
from . import audiorecorder
from . import recordsmanager
from . import helperutils

from . import __app_name__
from . import __version__

class MainWindow(QtWidgets.QMainWindow):

    def __init__(self, parent=None, *args, **kwargs):
        super().__init__(parent, *args, **kwargs)

        self.__ui = mainwindow_ui.Ui_MainWindow()
        self.__ui.setupUi(self)

        self.setWindowTitle(f'{__app_name__} - {__version__}')
        self.ui.labelRecordDuration.setVisible(False)

        self.ui.tableRecords.horizontalHeader().setFixedHeight(37)

        self.ui.tableRecords.horizontalHeader().setSectionResizeMode(
            0, QtWidgets.QHeaderView.Stretch)
        self.ui.tableRecords.horizontalHeader().setSectionResizeMode(
            1, QtWidgets.QHeaderView.ResizeToContents)

        settings_fname = os.path.normpath(
            os.path.join(helperutils.get_app_config_dir(), __app_name__+'.ini'))

        self.__settings = QtCore.QSettings(
            settings_fname, QtCore.QSettings.IniFormat, self)
        self.__settings_group = helperutils.qsettings_group(self.__settings)

        self.__audio_recorder = audiorecorder.AudioRecorder(parent=self)
        self.__records_manager = recordsmanager.RecordsManager(parent=self)

        self.ui.pbRecordingStartAndStop.toggled.connect(self.__on_start_stop)
        self.ui.pbRecordingPause.toggled.connect(self.__on_pause)
        self.ui.pbRemoveRecords.clicked.connect(self.__remove_selected_records)
        self.ui.tableRecords.cellDoubleClicked.connect(self.__on_play_record)
        self.ui.tableRecords.itemSelectionChanged.connect(
            self.__on_change_selected_records)
        self.ui.tableRecords.installEventFilter(self)

        self.__audio_recorder.record_updated.connect(
            self.__on_update_duration_time)

        self.__read_settings()
        self.__update_records_info()

    def closeEvent(self, event):
        self.__write_settings()

    def eventFilter(self, obj, event: QtGui.QKeyEvent):
        if obj is not self.ui.tableRecords:
            return False
        if event.type() != QtCore.QEvent.KeyPress:
            return False
        if event.key() != QtCore.Qt.Key_Delete:
            return False

        self.__remove_selected_records()
        return True

    @property
    def ui(self):
        return self.__ui

    def __on_start_stop(self, is_checked):
        pb_title_text = {
            True: self.tr('Stop'),
            False: self.tr('Record'),
        }

        self.ui.pbRecordingStartAndStop.setText(pb_title_text[is_checked])
        self.ui.labelRecordDuration.setVisible(is_checked)

        if is_checked:
            self.__start_recording()
        else:
            self.__stop_recording()

    def __on_pause(self, is_checked):
        if is_checked:
            self.__pause_recording()
        else:
            self.__start_recording()

    def __on_update_duration_time(self):
        duration_delta = datetime.timedelta(
            seconds=int(self.__audio_recorder.duration))
        self.ui.labelRecordDuration.setText(str(duration_delta))

    def __on_play_record(self, index):
        record_info = self.ui.tableRecords.item(index, 0).data(
            QtCore.Qt.UserRole)

        record_url = QtCore.QUrl(record_info.filename.replace('\\', '/'))
        QtGui.QDesktopServices.openUrl(record_url)

    def __on_change_selected_records(self):
        is_selected = len(self.ui.tableRecords.selectedItems()) > 0
        self.ui.pbRemoveRecords.setEnabled(is_selected)

    def __start_recording(self):
        self.__audio_recorder.record()

    def __pause_recording(self):
        self.__audio_recorder.stop()

    def __stop_recording(self):
        self.__audio_recorder.stop()
        self.__save_record()
        self.__audio_recorder.clear()
        self.__on_update_duration_time()

    def __save_record(self):
        record_info = self.__records_manager.save_record(
            self.__audio_recorder.get_record())

        self.__add_record_info_to_table(0, record_info)

    def __read_settings(self):
        with self.__settings_group('UI'):
            self.restoreGeometry(self.__settings.value(
                'WindowGeometry', self.saveGeometry()))
            self.restoreState(self.__settings.value(
                'WindowState', self.saveState()))

        self.__records_manager.read_settings(self.__settings)

    def __write_settings(self):
        with self.__settings_group('UI'):
            self.__settings.setValue('WindowGeometry', self.saveGeometry())
            self.__settings.setValue('WindowState', self.saveState())

        self.__records_manager.write_settings(self.__settings)

    def __add_record_info_to_table(self, index, record_info):
        self.ui.tableRecords.insertRow(index)

        date_item = QtWidgets.QTableWidgetItem(str(record_info.date))
        date_item.setData(QtCore.Qt.UserRole, record_info)

        dur_item = QtWidgets.QTableWidgetItem(str(record_info.duration))
        dur_item.setTextAlignment(QtCore.Qt.AlignCenter)

        self.ui.tableRecords.setItem(index, 0, date_item)
        self.ui.tableRecords.setItem(index, 1, dur_item)

        self.ui.tableRecords.verticalHeader().setSectionResizeMode(
            index, QtWidgets.QHeaderView.ResizeToContents)

    def __update_records_info(self):
        records_info = self.__records_manager.get_records_info()
        self.ui.tableRecords.clearContents()
        self.ui.tableRecords.setRowCount(0)

        for i, record_info in enumerate(records_info):
            self.__add_record_info_to_table(i, record_info)

    def __remove_selected_records(self):
        selected_items = self.ui.tableRecords.selectedItems()

        if not selected_items:
            return False

        records_for_remove = [
            (self.ui.tableRecords.row(item), item.data(QtCore.Qt.UserRole))
            for item in selected_items if item.data(QtCore.Qt.UserRole)]

        for row, record_info in records_for_remove:
            self.__records_manager.remove_record(record_info)
            self.ui.tableRecords.removeRow(row)
<commit_msg>Refactoring<commit_after># -*- coding: utf-8 -*-

"""
"""

import os
import datetime

from PyQt5 import QtWidgets
from PyQt5 import QtGui
from PyQt5 import QtCore

from . import mainwindow_ui
from . import audiorecorder
from . import recordsmanager
from . import helperutils

from . import __app_name__
from . import __version__

class MainWindow(QtWidgets.QMainWindow):

    def __init__(self, parent=None, *args, **kwargs):
        super().__init__(parent, *args, **kwargs)

        self.__ui = mainwindow_ui.Ui_MainWindow()
        self.__ui.setupUi(self)

        self.setWindowTitle(f'{__app_name__} - {__version__}')
        self.ui.labelRecordDuration.setVisible(False)

        self.ui.tableRecords.horizontalHeader().setFixedHeight(37)

        self.ui.tableRecords.horizontalHeader().setSectionResizeMode(
            0, QtWidgets.QHeaderView.Stretch)
        self.ui.tableRecords.horizontalHeader().setSectionResizeMode(
            1, QtWidgets.QHeaderView.ResizeToContents)

        settings_fname = os.path.normpath(
            os.path.join(helperutils.get_app_config_dir(), __app_name__+'.ini'))

        self.__settings = QtCore.QSettings(
            settings_fname, QtCore.QSettings.IniFormat, self)
        self.__settings_group = helperutils.qsettings_group(self.__settings)

        self.__audio_recorder = audiorecorder.AudioRecorder(parent=self)
        self.__records_manager = recordsmanager.RecordsManager(parent=self)

        self.ui.pbRecordingStartAndStop.toggled.connect(self.__on_start_stop)
        self.ui.pbRecordingPause.toggled.connect(self.__on_pause)
        self.ui.pbRemoveRecords.clicked.connect(self.__remove_selected_records)

        self.ui.tableRecords.cellDoubleClicked.connect(self.__on_play_record)
        self.ui.tableRecords.itemSelectionChanged.connect(
            self.__on_change_selected_records)
        self.ui.tableRecords.installEventFilter(self)

        self.__audio_recorder.record_updated.connect(
            self.__on_update_duration_time)

        self.__read_settings()
        self.__update_records_info()

    def closeEvent(self, event):
        self.__write_settings()

    def eventFilter(self, obj, event: QtGui.QKeyEvent):
        if obj is not self.ui.tableRecords:
            return False
        if event.type() != QtCore.QEvent.KeyPress:
            return False
        if event.key() != QtCore.Qt.Key_Delete:
            return False

        self.__remove_selected_records()
        return True

    @property
    def ui(self):
        return self.__ui

    def __on_start_stop(self, is_checked):
        pb_title_text = {
            True: self.tr('Stop'),
            False: self.tr('Record'),
        }

        self.ui.pbRecordingStartAndStop.setText(pb_title_text[is_checked])
        self.ui.labelRecordDuration.setVisible(is_checked)

        if is_checked:
            self.__start_recording()
        else:
            self.__stop_recording()

    def __on_pause(self, is_checked):
        if is_checked:
            self.__pause_recording()
        else:
            self.__start_recording()

    def __on_update_duration_time(self):
        duration_delta = datetime.timedelta(
            seconds=int(self.__audio_recorder.duration))
        self.ui.labelRecordDuration.setText(str(duration_delta))

    def __on_play_record(self, index):
        record_info = self.ui.tableRecords.item(index, 0).data(
            QtCore.Qt.UserRole)

        record_url = QtCore.QUrl(record_info.filename.replace('\\', '/'))
        QtGui.QDesktopServices.openUrl(record_url)

    def __on_change_selected_records(self):
        is_selected = len(self.ui.tableRecords.selectedItems()) > 0
        self.ui.pbRemoveRecords.setEnabled(is_selected)

    def __start_recording(self):
        self.__audio_recorder.record()

    def __pause_recording(self):
        self.__audio_recorder.stop()

    def __stop_recording(self):
        self.__audio_recorder.stop()
        self.__save_record()
        self.__audio_recorder.clear()
        self.__on_update_duration_time()

    def __save_record(self):
        record_info = self.__records_manager.save_record(
            self.__audio_recorder.get_record())

        self.__add_record_info_to_table(0, record_info)

    def __read_settings(self):
        with self.__settings_group('UI'):
            self.restoreGeometry(self.__settings.value(
                'WindowGeometry', self.saveGeometry()))
            self.restoreState(self.__settings.value(
                'WindowState', self.saveState()))

        self.__records_manager.read_settings(self.__settings)

    def __write_settings(self):
        with self.__settings_group('UI'):
            self.__settings.setValue('WindowGeometry', self.saveGeometry())
            self.__settings.setValue('WindowState', self.saveState())

        self.__records_manager.write_settings(self.__settings)

    def __add_record_info_to_table(self, index, record_info):
        self.ui.tableRecords.insertRow(index)

        date_item = QtWidgets.QTableWidgetItem(str(record_info.date))
        date_item.setData(QtCore.Qt.UserRole, record_info)

        dur_item = QtWidgets.QTableWidgetItem(str(record_info.duration))
        dur_item.setTextAlignment(QtCore.Qt.AlignCenter)

        self.ui.tableRecords.setItem(index, 0, date_item)
        self.ui.tableRecords.setItem(index, 1, dur_item)

        self.ui.tableRecords.verticalHeader().setSectionResizeMode(
            index, QtWidgets.QHeaderView.ResizeToContents)

    def __update_records_info(self):
        records_info = self.__records_manager.get_records_info()
        self.ui.tableRecords.clearContents()
        self.ui.tableRecords.setRowCount(0)

        for i, record_info in enumerate(records_info):
            self.__add_record_info_to_table(i, record_info)

    def __remove_selected_records(self):
        selected_items = self.ui.tableRecords.selectedItems()

        if not selected_items:
            return False

        records_for_remove = [
            (self.ui.tableRecords.row(item), item.data(QtCore.Qt.UserRole))
            for item in selected_items if item.data(QtCore.Qt.UserRole)]

        for row, record_info in records_for_remove:
            self.__records_manager.remove_record(record_info)
            self.ui.tableRecords.removeRow(row)
<|endoftext|>


<commit_before>"""
============================
Faces dataset decompositions
============================

This example compares different unsupervised matrix decomposition (dimension
reduction) methods on the Olivetti faces dataset.

"""
print __doc__

# Authors: Vlad Niculae, Alexandre Gramfort
# License: BSD

import logging
from time import time

import pylab as pl

from scikits.learn.cluster import MiniBatchKMeans
from scikits.learn.datasets import fetch_olivetti_faces
from scikits.learn.decomposition import FastICA
from scikits.learn.decomposition import MiniBatchSparsePCA
from scikits.learn.decomposition import NMF
from scikits.learn.decomposition import RandomizedPCA

# Display progress logs on stdout
logging.basicConfig(level=logging.INFO,
                    format='%(asctime)s %(levelname)s %(message)s')
n_row, n_col = 2, 3
n_components = n_row * n_col
image_shape = (64, 64)

###############################################################################
# Load faces data
dataset = fetch_olivetti_faces(shuffle=True)
faces = dataset.data

n_samples, n_features = faces.shape

# global centering
faces_centered = faces - faces.mean(axis=0)

# local centering
faces_centered -= faces_centered.mean(axis=1).reshape(n_samples, -1)

print "Dataset consists of %d faces" % n_samples

###############################################################################
def plot_gallery(title, images):
    pl.figure(figsize=(2. * n_col, 2.26 * n_row))
    pl.suptitle(title, size=16)
    for i, comp in enumerate(images):
        pl.subplot(n_row, n_col, i + 1)
        vmax = max(comp.max(), -comp.min())
        pl.imshow(comp.reshape(image_shape), cmap=pl.cm.gray,
                  interpolation='nearest',
                  vmin=-vmax, vmax=vmax)
        pl.xticks(())
        pl.yticks(())
    pl.subplots_adjust(0.01, 0.05, 0.99, 0.93, 0.04, 0.)

###############################################################################
# List of the different estimators, whether to center and transpose the
# problem, and whether the transformer uses the clustering API.
estimators = [
    ('Eigenfaces - RandomizedPCA',
     RandomizedPCA(n_components=n_components, whiten=True),
     True, False),

    ('Non-negative components - NMF',
     NMF(n_components=n_components, init='nndsvda', beta=5.0, tol=5e-3,
         sparseness='components'),
     False, False),

    ('Independent components - FastICA',
     FastICA(n_components=n_components, whiten=True, max_iter=10),
     True, True),

    ('Sparse comp. - MiniBatchSparsePCA',
     MiniBatchSparsePCA(n_components=n_components, alpha=1e-3, n_iter=100,
                        chunk_size=3),
     True, False),

    ('Cluster centers - MiniBatchKMeans',
     MiniBatchKMeans(k=n_components, tol=1e-3, chunk_size=20, max_iter=50),
     True, False)
]

###############################################################################
# Plot a sample of the input data

plot_gallery("First centered Olivetti faces", faces_centered[:n_components])

###############################################################################
# Do the estimation and plot it

for name, estimator, center, transpose in estimators:
    print "Extracting the top %d %s..." % (n_components, name)
    t0 = time()
    data = faces
    if center:
        data = faces_centered
    if transpose:
        data = data.T
    estimator.fit(data)
    train_time = (time() - t0)
    print "done in %0.3fs" % train_time
    if hasattr(estimator, 'cluster_centers_'):
        components_ = estimator.cluster_centers_
    else:
        components_ = estimator.components_
    if transpose:
        components_ = components_.T
    plot_gallery('%s - Train time %.1fs' % (name, train_time), components_)

pl.show()
<commit_msg>add refs to example<commit_after>"""
============================
Faces dataset decompositions
============================

This example compares different unsupervised matrix decomposition (dimension
reduction) methods  from the module :py:mod:`scikits.learn.decomposition` on
:doc:`/datasets/olivetti_faces`.

"""
print __doc__

# Authors: Vlad Niculae, Alexandre Gramfort
# License: BSD

import logging
from time import time

import pylab as pl

from scikits.learn.cluster import MiniBatchKMeans
from scikits.learn.datasets import fetch_olivetti_faces
from scikits.learn.decomposition import FastICA
from scikits.learn.decomposition import MiniBatchSparsePCA
from scikits.learn.decomposition import NMF
from scikits.learn.decomposition import RandomizedPCA

# Display progress logs on stdout
logging.basicConfig(level=logging.INFO,
                    format='%(asctime)s %(levelname)s %(message)s')
n_row, n_col = 2, 3
n_components = n_row * n_col
image_shape = (64, 64)

###############################################################################
# Load faces data
dataset = fetch_olivetti_faces(shuffle=True)
faces = dataset.data

n_samples, n_features = faces.shape

# global centering
faces_centered = faces - faces.mean(axis=0)

# local centering
faces_centered -= faces_centered.mean(axis=1).reshape(n_samples, -1)

print "Dataset consists of %d faces" % n_samples

###############################################################################
def plot_gallery(title, images):
    pl.figure(figsize=(2. * n_col, 2.26 * n_row))
    pl.suptitle(title, size=16)
    for i, comp in enumerate(images):
        pl.subplot(n_row, n_col, i + 1)
        vmax = max(comp.max(), -comp.min())
        pl.imshow(comp.reshape(image_shape), cmap=pl.cm.gray,
                  interpolation='nearest',
                  vmin=-vmax, vmax=vmax)
        pl.xticks(())
        pl.yticks(())
    pl.subplots_adjust(0.01, 0.05, 0.99, 0.93, 0.04, 0.)

###############################################################################
# List of the different estimators, whether to center and transpose the
# problem, and whether the transformer uses the clustering API.
estimators = [
    ('Eigenfaces - RandomizedPCA',
     RandomizedPCA(n_components=n_components, whiten=True),
     True, False),

    ('Non-negative components - NMF',
     NMF(n_components=n_components, init='nndsvda', beta=5.0, tol=5e-3,
         sparseness='components'),
     False, False),

    ('Independent components - FastICA',
     FastICA(n_components=n_components, whiten=True, max_iter=10),
     True, True),

    ('Sparse comp. - MiniBatchSparsePCA',
     MiniBatchSparsePCA(n_components=n_components, alpha=1e-3, n_iter=100,
                        chunk_size=3),
     True, False),

    ('Cluster centers - MiniBatchKMeans',
     MiniBatchKMeans(k=n_components, tol=1e-3, chunk_size=20, max_iter=50),
     True, False)
]

###############################################################################
# Plot a sample of the input data

plot_gallery("First centered Olivetti faces", faces_centered[:n_components])

###############################################################################
# Do the estimation and plot it

for name, estimator, center, transpose in estimators:
    print "Extracting the top %d %s..." % (n_components, name)
    t0 = time()
    data = faces
    if center:
        data = faces_centered
    if transpose:
        data = data.T
    estimator.fit(data)
    train_time = (time() - t0)
    print "done in %0.3fs" % train_time
    if hasattr(estimator, 'cluster_centers_'):
        components_ = estimator.cluster_centers_
    else:
        components_ = estimator.components_
    if transpose:
        components_ = components_.T
    plot_gallery('%s - Train time %.1fs' % (name, train_time), components_)

pl.show()
<|endoftext|>