<commit_before>from __future__ import division
from __future__ import print_function

COLUMNS_ZONE_GEOMETRY = ['Name', 'floors_bg', 'floors_ag', 'height_bg', 'height_ag']
COLUMNS_DISTRICT_GEOMETRY = ['Name', 'height_ag', 'floors_ag', 'floors_bg']
COLUMNS_ZONE_AGE = ['built', 'roof', 'windows', 'partitions', 'basement', 'HVAC', 'envelope']
COLUMNS_ZONE_OCCUPANCY = ['MULTI_RES', 'OFFICE', 'RETAIL', 'SCHOOL', 'HOTEL', 'GYM', 'HOSPITAL', 'INDUSTRIAL',
                          'RESTAURANT','SINGLE_RES', 'SERVERROOM', 'SWIMMING', 'FOODSTORE', 'LIBRARY', 'COOLROOM', 'PARKING']

def assert_columns_names(zone_df, columns):

    try:
        zone_test = zone_df[columns]
    except ValueError:
        print("one or more columns in the Zone or District input files is not compatible with CEA, please ensure the column" +
              " names comply with:", columns)

def assert_input_geometry_acceptable_values_floor_height(zone_df):

    # Rule 0. nothing can be negative
    rule0 = zone_df.where(zone_df < 0).any().any()
    if rule0:
        raise Exception('There are negative values in your geometry. This is not possible to simulate in CEA at the moment'
                        ' Please verify your Zone or District shapefile file')

    # Rule 1. Floors above ground cannot be less than 1 or negative.
    rule1_1 = zone_df['floors_ag'].where(zone_df['floors_ag'] < 1).any()
    if rule1_1:
        raise Exception('one of more buildings have less than one floor above ground. This is not possible'
                        'to simulate in CEA at the moment. Please verify your Zone or District shapefile file')

    # Rule 2. Where floor height is less than 1m on average above ground.
    zone_df['rule2'] = zone_df['height_ag'] / zone_df['floors_ag']
    rule2 = zone_df['rule2'].where(zone_df['rule2'] <= 1).any()
    if rule2:
        raise Exception('one of more buildings have less report less than 1m height per floor. This is not possible'
                        'to simulate in CEA at the moment. Please verify your Zone or District shapefile file')

    # Rule 3. floors below ground cannot be negative
    rule3 = zone_df['floors_bg'].where(zone_df['floors_bg'] < 0).any()
    if rule3:
        raise Exception('one of more buildings have a negative floor below ground. This is not possible'
                        'to simulate in CEA at the moment. Please verify your Zone or District file')

def verify_input_geometry_zone(zone_df):

    # Verification 1. verify if all the column names are correct
    assert_columns_names(zone_df, COLUMNS_ZONE_GEOMETRY)

    # Verification 2. verify if the floor_height ratio is correct
    assert_input_geometry_acceptable_values_floor_height(zone_df)

def verify_input_geometry_district(district_df):

    # Verification 1. verify if all the column names are correct
    assert_columns_names(district_df, COLUMNS_DISTRICT_GEOMETRY)

    # Verification 2. verify if the floor_height ratio is correct
    assert_input_geometry_acceptable_values_floor_height(district_df)

def verify_input_occupancy(occupancy_df):

    # Verification 1. verify if all the column names are correct
    assert_columns_names(occupancy_df, COLUMNS_ZONE_OCCUPANCY)

def verify_input_age(age_df):

    # Verification 1. verify if all the column names are correct
    assert_columns_names(age_df, COLUMNS_ZONE_AGE)
<commit_msg>last detail<commit_after>from __future__ import division
from __future__ import print_function

COLUMNS_ZONE_GEOMETRY = ['Name', 'floors_bg', 'floors_ag', 'height_bg', 'height_ag']
COLUMNS_DISTRICT_GEOMETRY = ['Name', 'height_ag', 'floors_ag', 'floors_bg']
COLUMNS_ZONE_AGE = ['built', 'roof', 'windows', 'partitions', 'basement', 'HVAC', 'envelope']
COLUMNS_ZONE_OCCUPANCY = ['MULTI_RES', 'OFFICE', 'RETAIL', 'SCHOOL', 'HOTEL', 'GYM', 'HOSPITAL', 'INDUSTRIAL',
                          'RESTAURANT','SINGLE_RES', 'SERVERROOM', 'SWIMMING', 'FOODSTORE', 'LIBRARY', 'COOLROOM', 'PARKING']

def assert_columns_names(zone_df, columns):

    try:
        zone_test = zone_df[columns]
    except ValueError:
        print("one or more columns in the Zone or District input files is not compatible with CEA, please ensure the column" +
              " names comply with:", columns)

def assert_input_geometry_acceptable_values_floor_height(zone_df):

    # Rule 0. nothing can be negative
    rule0 = zone_df.where(zone_df < 0).any().any()
    if rule0:
        raise Exception('There are negative values in your geometry. This is not possible to simulate in CEA at the moment'
                        ' Please verify your Zone or District shapefile file')

    # Rule 1. Floors above ground cannot be less than 1 or negative.
    rule1_1 = zone_df['floors_ag'].where(zone_df['floors_ag'] < 1).any()
    rule1_2 = zone_df['height_ag'].where(zone_df['height_ag'] < 1).any()
    if rule1_1 or rule1_2:
        raise Exception('one of more buildings have less than one floor above ground or the height above ground is less than 1 meter.'
                        ' This is not possible to simulate in CEA at the moment. Please verify your Zone or District shapefile file')

    # Rule 2. Where floor height is less than 1m on average above ground.
    zone_df['rule2'] = zone_df['height_ag'] / zone_df['floors_ag']
    rule2 = zone_df['rule2'].where(zone_df['rule2'] <= 1).any()
    if rule2:
        raise Exception('one of more buildings have less report less than 1m height per floor. This is not possible'
                        'to simulate in CEA at the moment. Please verify your Zone or District shapefile file')

    # Rule 3. floors below ground cannot be negative
    rule3 = zone_df['floors_bg'].where(zone_df['floors_bg'] < 0).any()
    if rule3:
        raise Exception('one of more buildings have a negative floor below ground. This is not possible'
                        'to simulate in CEA at the moment. Please verify your Zone or District file')

def verify_input_geometry_zone(zone_df):

    # Verification 1. verify if all the column names are correct
    assert_columns_names(zone_df, COLUMNS_ZONE_GEOMETRY)

    # Verification 2. verify if the floor_height ratio is correct
    assert_input_geometry_acceptable_values_floor_height(zone_df)

def verify_input_geometry_district(district_df):

    # Verification 1. verify if all the column names are correct
    assert_columns_names(district_df, COLUMNS_DISTRICT_GEOMETRY)

    # Verification 2. verify if the floor_height ratio is correct
    assert_input_geometry_acceptable_values_floor_height(district_df)

def verify_input_occupancy(occupancy_df):

    # Verification 1. verify if all the column names are correct
    assert_columns_names(occupancy_df, COLUMNS_ZONE_OCCUPANCY)

def verify_input_age(age_df):

    # Verification 1. verify if all the column names are correct
    assert_columns_names(age_df, COLUMNS_ZONE_AGE)
<|endoftext|>


<commit_before>
import numpy as np
from scipy.spatial.distance import pdist, squareform
from scipy.cluster.hierarchy import fcluster, fclusterdata, linkage, dendrogram
from functools import partial

try:
    from sklearn.metrics import pairwise_distances
    from multiprocessing import cpu_count
    _sklearn_flag = True
except ImportError:
    Warning("sklearn must be installed to parallelize distance calculations.")
    _sklearn_flag = False

from log import overlap_metric
from utils import mode

def cluster_2D_regions(twod_region_props, metric='position', cut_val=18,
                       multiprocess=False, n_jobs=None, verbose=False):
    '''
    Cluster 2D Bubble regions by their postion or overlap.
    '''

    # Cluster on the x, y centre positions
    if metric is "position":

        cluster_idx = fclusterdata(twod_region_props[:, :2], cut_val,
                                   criterion='distance', method='complete')

    # Cluster on the spatial overlap of regions
    elif metric is "overlap":

        if cut_val > 1.0:
            raise ValueError("cut_val <= 1 when metric is overlap.")

        overlap_func = partial(overlap_metric, return_corr=True)

        if multiprocess and _sklearn_flag:
            if n_jobs is None:
                n_jobs = cpu_count()
            sims = pairwise_distances(twod_region_props, metric=overlap_func,
                                      n_jobs=n_jobs)
            # It's minorly not symmetric (~10^-10 differences), probably due
            # to pixelization. Force it to be symmetric.
            sym_sims = np.triu(sims) + np.triu(sims).T
            np.fill_diagonal(sym_sims, 0.0)
            # Convert to condensed form. linkage doesn't handle nxn dist
            # matrices properly
            sims = squareform(sym_sims)
        else:
            sims = pdist(twod_region_props, metric=overlap_func)
        sims[sims < 0] = 0.0

        link_mat = linkage(1 - sims, 'complete')

        if verbose:
            import matplotlib.pyplot as p
            dendrogram(link_mat)
            p.show()

        cluster_idx = fcluster(link_mat, cut_val, criterion='distance')

    # Cluster on the connectivity of regions in the spectral dimension
    elif metric is 'channel':

        Warning("cut_val forced to 1.0 for channel clustering.")

        cluster_idx = fclusterdata(twod_region_props[:, -1:], 1.0,
                                   criterion='distance', method='single',
                                   metric='cityblock')

    else:
        raise ValueError("metric must be 'position', 'channel', or 'overlap'.")

    return cluster_idx

def cluster_and_clean(twod_region_props, min_scatter=9, cut_val=None):
    '''
    Clean-up clusters of 2D regions. Real bubbles must be connected in
    velocity space. This function also looks for clusters that are closely
    related, and combines them.

    Parameters
    ----------
    twod_region_props : np.ndarray
        Array with the channel, y position, x position, semi-major radius,
        semi-minor radius, and position angle of the detected 2D blobs.
    min_scatter : float, optional
        Minimum distance to allow between the centres of regions in a cluster.

    Returns
    -------
    cluster_idx : np.ndarray
        The cluster ID for each of the given regions.
    '''

    # Create initial clusters based on the overlap correlation
    cluster_idx = cluster_2D_regions(twod_region_props,
                                     metric='overlap',
                                     cut_val=0.5, multiprocess=True)

    # Now we split clusters based on spectral connectivity.
    for clust in np.unique(cluster_idx[cluster_idx > 0]):

        posns = np.where(cluster_idx == clust)[0]

        if posns.size == 1:
            continue

        props = twod_region_props[posns]

        # Cluster on channel and split.
        spec_idx = cluster_2D_regions(props, metric='channel')

        # If not split is found, continue on
        if spec_idx.max() == 1:
            continue

        for idx in np.unique(spec_idx[spec_idx > 1]):
            cluster_idx[posns[spec_idx == idx]] = cluster_idx.max() + 1

    return cluster_idx

def cluster_brute_force(twod_region_props, min_corr=0.5, min_overlap=0.7,
                        multiprocess=True, n_jobs=None, min_multi_size=100):
    '''
    Do a brute force clustering of the regions
    '''

    cluster_idx = np.zeros_like(twod_region_props[:, 0])

    # Determine the channels which have regions defined in them
    chans = np.unique(twod_region_props[:, 5])

    overlap_func = partial(overlap_metric, return_corr=True)

    # Now loop through each channel's regions looking for significant overlap
    # with a region before it.
    for chan in chans[1:]:
        chan_regions_idx = np.where(twod_region_props[:, 5] == chan)[0]
        prev_regions_idx = np.where(twod_region_props[:, 5] == chan - 1)[0]

        all_overlaps = np.zeros((2, len(prev_regions_idx),
                                 len(chan_regions_idx)),
                                dtype=np.float)

        multi_conds = (multiprocess and _sklearn_flag and
                       all_overlaps[0].size >= min_multi_size)
        if multi_conds:
            if n_jobs is None:
                n_jobs = cpu_count()

            all_overlaps[0] = \
                pairwise_distances(twod_region_props[prev_regions_idx],
                                   twod_region_props[chan_regions_idx],
                                   metric=overlap_func,
                                   n_jobs=n_jobs)
            all_overlaps[1] = \
                pairwise_distances(twod_region_props[prev_regions_idx],
                                   twod_region_props[chan_regions_idx],
                                   metric=overlap_metric,
                                   n_jobs=n_jobs)

        else:
            for i, prev_idx in enumerate(prev_regions_idx):
                for j, idx in enumerate(chan_regions_idx):
                    # Area correlation
                    all_overlaps[0, i, j] = \
                        overlap_func(twod_region_props[prev_idx],
                                     twod_region_props[idx])
                    # Area fractional overlap
                    all_overlaps[1, i, j] = \
                        overlap_metric(twod_region_props[prev_idx],
                                       twod_region_props[idx])

        any_corr = np.any(all_overlaps[0] >= min_corr)
        any_frac = np.any(all_overlaps[1] >= min_overlap)
        if not any_corr or not any_frac:
            continue

        # Blank all positions that don't satisfy the criteria
        good_mask = np.logical_and(all_overlaps[0] >= min_corr,
                                   all_overlaps[1] >= min_overlap)

        all_overlaps = all_overlaps * good_mask

        # The max number of matches is the smallest number of regions in the
        # two channels.
        for _ in range(min(len(chan_regions_idx), len(prev_regions_idx))):

            # They don't all have to match, so break when nothing is left.
            if not np.any(all_overlaps):
                break

            i, j = np.unravel_index(all_overlaps[0].argmax(),
                                    all_overlaps.shape[1:])

            idx = prev_regions_idx[i]
            join_idx = chan_regions_idx[j]

            # Create a new cluster, or add to the existing one.
            if cluster_idx[idx] == 0:
                # Create a new cluster
                new_idx = cluster_idx.max() + 1
                cluster_idx[idx] = new_idx
                cluster_idx[join_idx] = new_idx
            else:
                cluster_idx[join_idx] = cluster_idx[idx]

            # Set that row and column to 0
            all_overlaps[:, i, :] = 0.0
            all_overlaps[:, :, j] = 0.0

    return cluster_idx
<commit_msg>Global overlap condition for clustering<commit_after>
import numpy as np
from scipy.spatial.distance import pdist, squareform
from scipy.cluster.hierarchy import fcluster, fclusterdata, linkage, dendrogram
from functools import partial

try:
    from sklearn.metrics import pairwise_distances
    from multiprocessing import cpu_count
    _sklearn_flag = True
except ImportError:
    Warning("sklearn must be installed to parallelize distance calculations.")
    _sklearn_flag = False

from log import overlap_metric
from utils import mode

def cluster_2D_regions(twod_region_props, metric='position', cut_val=18,
                       multiprocess=False, n_jobs=None, verbose=False):
    '''
    Cluster 2D Bubble regions by their postion or overlap.
    '''

    # Cluster on the x, y centre positions
    if metric is "position":

        cluster_idx = fclusterdata(twod_region_props[:, :2], cut_val,
                                   criterion='distance', method='complete')

    # Cluster on the spatial overlap of regions
    elif metric is "overlap":

        if cut_val > 1.0:
            raise ValueError("cut_val <= 1 when metric is overlap.")

        overlap_func = partial(overlap_metric, return_corr=True)

        if multiprocess and _sklearn_flag:
            if n_jobs is None:
                n_jobs = cpu_count()
            sims = pairwise_distances(twod_region_props, metric=overlap_func,
                                      n_jobs=n_jobs)
            # It's minorly not symmetric (~10^-10 differences), probably due
            # to pixelization. Force it to be symmetric.
            sym_sims = np.triu(sims) + np.triu(sims).T
            np.fill_diagonal(sym_sims, 0.0)
            # Convert to condensed form. linkage doesn't handle nxn dist
            # matrices properly
            sims = squareform(sym_sims)
        else:
            sims = pdist(twod_region_props, metric=overlap_func)
        sims[sims < 0] = 0.0

        link_mat = linkage(1 - sims, 'complete')

        if verbose:
            import matplotlib.pyplot as p
            dendrogram(link_mat)
            p.show()

        cluster_idx = fcluster(link_mat, cut_val, criterion='distance')

    # Cluster on the connectivity of regions in the spectral dimension
    elif metric is 'channel':

        Warning("cut_val forced to 1.0 for channel clustering.")

        cluster_idx = fclusterdata(twod_region_props[:, -1:], 1.0,
                                   criterion='distance', method='single',
                                   metric='cityblock')

    else:
        raise ValueError("metric must be 'position', 'channel', or 'overlap'.")

    return cluster_idx

def cluster_and_clean(twod_region_props, min_scatter=9, cut_val=None):
    '''
    Clean-up clusters of 2D regions. Real bubbles must be connected in
    velocity space. This function also looks for clusters that are closely
    related, and combines them.

    Parameters
    ----------
    twod_region_props : np.ndarray
        Array with the channel, y position, x position, semi-major radius,
        semi-minor radius, and position angle of the detected 2D blobs.
    min_scatter : float, optional
        Minimum distance to allow between the centres of regions in a cluster.

    Returns
    -------
    cluster_idx : np.ndarray
        The cluster ID for each of the given regions.
    '''

    # Create initial clusters based on the overlap correlation
    cluster_idx = cluster_2D_regions(twod_region_props,
                                     metric='overlap',
                                     cut_val=0.5, multiprocess=True)

    # Now we split clusters based on spectral connectivity.
    for clust in np.unique(cluster_idx[cluster_idx > 0]):

        posns = np.where(cluster_idx == clust)[0]

        if posns.size == 1:
            continue

        props = twod_region_props[posns]

        # Cluster on channel and split.
        spec_idx = cluster_2D_regions(props, metric='channel')

        # If not split is found, continue on
        if spec_idx.max() == 1:
            continue

        for idx in np.unique(spec_idx[spec_idx > 1]):
            cluster_idx[posns[spec_idx == idx]] = cluster_idx.max() + 1

    return cluster_idx

def cluster_brute_force(twod_region_props, min_corr=0.5, min_overlap=0.7,
                        global_corr=0.5,
                        multiprocess=True, n_jobs=None, min_multi_size=100):
    '''
    Do a brute force clustering of the regions
    '''

    cluster_idx = np.zeros_like(twod_region_props[:, 0])

    # Determine the channels which have regions defined in them
    chans = np.unique(twod_region_props[:, 5])

    overlap_func = partial(overlap_metric, return_corr=True)

    # Now loop through each channel's regions looking for significant overlap
    # with a region before it.
    for chan in chans[1:]:
        chan_regions_idx = np.where(twod_region_props[:, 5] == chan)[0]
        prev_regions_idx = np.where(twod_region_props[:, 5] == chan - 1)[0]

        all_overlaps = np.zeros((2, len(prev_regions_idx),
                                 len(chan_regions_idx)),
                                dtype=np.float)

        multi_conds = (multiprocess and _sklearn_flag and
                       all_overlaps[0].size >= min_multi_size)
        if multi_conds:
            if n_jobs is None:
                n_jobs = cpu_count()

            all_overlaps[0] = \
                pairwise_distances(twod_region_props[prev_regions_idx],
                                   twod_region_props[chan_regions_idx],
                                   metric=overlap_func,
                                   n_jobs=n_jobs)
            all_overlaps[1] = \
                pairwise_distances(twod_region_props[prev_regions_idx],
                                   twod_region_props[chan_regions_idx],
                                   metric=overlap_metric,
                                   n_jobs=n_jobs)

        else:
            for i, prev_idx in enumerate(prev_regions_idx):
                for j, idx in enumerate(chan_regions_idx):
                    # Area correlation
                    all_overlaps[0, i, j] = \
                        overlap_func(twod_region_props[prev_idx],
                                     twod_region_props[idx])
                    # Area fractional overlap
                    all_overlaps[1, i, j] = \
                        overlap_metric(twod_region_props[prev_idx],
                                       twod_region_props[idx])

        any_corr = np.any(all_overlaps[0] >= min_corr)
        any_frac = np.any(all_overlaps[1] >= min_overlap)
        if not any_corr or not any_frac:
            continue

        # Blank all positions that don't satisfy the criteria
        good_mask = np.logical_and(all_overlaps[0] >= min_corr,
                                   all_overlaps[1] >= min_overlap)

        all_overlaps = all_overlaps * good_mask

        # The max number of matches is the smallest number of regions in the
        # two channels.
        for _ in range(min(len(chan_regions_idx), len(prev_regions_idx))):

            # They don't all have to match, so break when nothing is left.
            if not np.any(all_overlaps):
                break

            i, j = np.unravel_index(all_overlaps[0].argmax(),
                                    all_overlaps.shape[1:])

            idx = prev_regions_idx[i]
            join_idx = chan_regions_idx[j]

            # Create a new cluster, or add to the existing one.
            if cluster_idx[idx] == 0:
                # Create a new cluster
                new_idx = cluster_idx.max() + 1
                cluster_idx[idx] = new_idx
                cluster_idx[join_idx] = new_idx
            else:
                # Global connectivity check
                clust_overlaps = []
                for memb in np.where(cluster_idx == cluster_idx[idx])[0]:
                    # We already know this one is good.
                    if memb == idx:
                        continue
                    clust_overlaps.append(
                        overlap_func(twod_region_props[memb],
                                     twod_region_props[join_idx]))
                clust_overlaps = np.array(clust_overlaps)
                if np.any(clust_overlaps < global_corr):
                    all_overlaps[:, i, j] = 0.0

                cluster_idx[join_idx] = cluster_idx[idx]

            # Set that row and column to 0
            all_overlaps[:, i, :] = 0.0
            all_overlaps[:, :, j] = 0.0

    return cluster_idx

def join_bubbles():
    pass

def threeD_overlaps():
    pass
<|endoftext|>


<commit_before>import os
from datetime import datetime
import urllib2
from copy import deepcopy
import unicodedata
import httplib

from flask import Flask, jsonify, json, request, redirect, abort, make_response
from flask import render_template
from flask.views import View, MethodView
from flaskext.login import login_user, current_user

import bibserver.dao
from bibserver.config import config
import bibserver.iomanager
import bibserver.importer
from bibserver.core import app, login_manager
from bibserver.view.account import blueprint as account

app.register_blueprint(account, url_prefix='/account')

# NB: the decorator appears to kill the function for normal usage
@login_manager.user_loader
def load_account_for_login_manager(userid):
    out = bibserver.dao.Account.get(userid)
    return out

@app.context_processor
def set_current_user():
    """ Set some template context globals. """
    return dict(current_user=current_user)

@app.route('/')
def home():
    # get list of available collections
    try:
        result = bibserver.dao.Record.query(q="type.exact:collection",sort={"received.exact":{"order":"desc"}})
        if result["hits"]["total"] != 0:
            colls = [i["_source"]["collection"] for i in result["hits"]["hits"]]
    except:
        colls = None
    return render_template('home/index.html', colls=colls, upload=config["allow_upload"] )

@app.route('/content/<path:path>')
def content(path):
    return render_template('home/content.html', page=path)

@app.route('/collection/<collid>/<path:path>')
def record(collid,path):
    JSON = False
    if path.endswith(".json") or path.endswith(".bibjson") or request.values.get('format',"") == "json" or request.values.get('format',"") == "bibjson":
        path = path.replace(".bibjson","").replace(".json","")
        JSON = True

    res = bibserver.dao.Record.query(q='collection:"' + collid + '" AND citekey:"' + path + '"')
    if res["hits"]["total"] == 0:
        abort(404)
    if res["hits"]["total"] > 1:
        return render_template('record.html', msg="hmmm... there is more than one record in this collection with that id...")
    recorddict = res["hits"]["hits"][0]["_source"]

    if JSON:
        resp = make_response( json.dumps( recorddict, indent=4 ) )
        resp.mimetype = "application/json"
        return resp

    return render_template('record.html', record=recorddict)

@app.route('/query', methods=['GET','POST'])
def query():
    if request.method == "GET":
        resp = make_response( bibserver.dao.Record.raw_query(request.query_string) )
        if request.values.get('delete','') and request.values.get('q',''):
            resp = make_response( bibserver.dao.Record.delete_by_query(request.values.get('q')) )
        resp.mimetype = "application/json"
        return resp

    if request.method == "POST":
        data = json.dumps(dict(request.form).keys()[0])
        host = str(config['ELASTIC_SEARCH_HOST']).rstrip('/')
        db_name = config['ELASTIC_SEARCH_DB']
        fullpath = '/' + db_name + '/record/_search'
        c =  httplib.HTTPConnection(host)
        c.request('POST', fullpath, data)
        resp = make_response(c.getresponse().read())
        resp.mimetype = "application/json"
        return resp

class UploadView(MethodView):
    '''The upload view.

    upload from URL provided in source, or from file upload button, or from
    POST default format is bibtex, but accept other format specifications via
    format default upload is a collection, but could also be person or group
    record
    '''
    def get(self):
        if request.values.get("source") is not None:
            return self.post()
        return render_template('upload.html')

    def post(self):
        pkg = self.package()
        if self.validate(pkg):
            importer = bibserver.importer.Importer()
#            try:
            res = importer.upload(pkg)
            if res != "DUPLICATE":
                if "collection" in pkg:
                    return redirect('/collection/' + pkg["collection"])
                msg = "Your records were uploaded but no collection name could be discerned."
            elif res == "DUPLICATE":
                msg = "The collection name you specified is already in use."
                msg += "
Please use another collection name."
            else:
                msg = "Sorry! There was an indexing error. Please try again."
#            except:
#                msg = 'Sorry! There was an error indexing your collection. Please try again.'
        else:
            msg = 'Your upload failed to validate. Please try again.'
        return render_template('upload.html', msg=msg)

    def validate(self, pkg):
        '''validate the submission before proceeding'''
        if "source" in pkg or "upfile" in pkg or "data" in pkg:
            return True
        return False

    def package(self):
        '''make package with:
            format of upload (default to bibtex)
            collection name (default to version of filename)
            email address (for a file upload)
            source, upfile, or data, depending on if URL, upload, or POST
            date received
        '''
        pkg = dict()
        if request.values.get("source"):
            pkg["source"] = urllib2.unquote(request.values.get("source"))
            pkg["format"] = self.findformat(pkg["source"])
        if request.files.get('upfile'):
            pkg["upfile"] = request.files.get('upfile')
            pkg["format"] = self.findformat(str(pkg["upfile"].filename))
        if request.values.get('format'):
            pkg["format"] = request.values.get('format')
        if request.values.get('data'):
            pkg["data"] = request.values['data']
        if request.values.get("collection"):
            pkg["collection"] = request.values.get("collection")
        pkg["email"] = request.values.get("email", None)
        pkg["received"] = str(datetime.now())
        return pkg

    def findformat(self,filename):
        if filename.endswith(".json"): return "json"
        if filename.endswith(".bibjson"): return "bibjson"
        if filename.endswith(".bibtex"): return "bibtex"
        if filename.endswith(".bib"): return "bibtex"
        if filename.endswith(".csv"): return "csv"
        return "bibtex"

# enable upload unless not allowed in config
if config["allow_upload"] == "YES":
    app.add_url_rule('/upload', view_func=UploadView.as_view('upload'))

@app.route('/search')
@app.route('/<path:path>')
def search(path=''):

    JSON = False
    if path.endswith(".json") or path.endswith(".bibjson") or request.values.get('format',"") == "json" or request.values.get('format',"") == "bibjson":
        path = path.replace(".bibjson","").replace(".json","")
        JSON = True

    # read args from config and params
    args = {"terms":{},"facet_fields" : [i + config["facet_field"] for i in config["facet_fields"]]}
    if 'from' in request.values:
        args['start'] = request.values.get('from')
    if 'size' in request.values:
        args['size'] = request.values.get('size')
    if 'sort' in request.values:
        if request.values.get("sort") != "..." and request.values.get("sort") != "":
            args['sort'] = {request.values.get('sort')+config["facet_field"] : {"order" : request.values.get('order','asc')}}
    if 'q' in request.values:
        if len(request.values.get('q')) > 0:
            args['q'] = request.values.get('q')
    for param in request.values:
        if param in config["facet_fields"]:
            vals = json.loads(unicodedata.normalize('NFKD',urllib2.unquote(request.values.get(param))).encode('utf-8','ignore'))
            args["terms"][param + config["facet_field"]] = vals

    # get implicit facet
    c = {'implicit_facet': {}}
    if path != '' and not path.startswith("search"):
        path = path.strip()
        if path.endswith("/"):
            path = path[:-1]
        bits = path.split('/')
        if len(bits) == 2:
            # its an implicit facet
            args['terms'][bits[0]+config["facet_field"]] = [bits[1]]
            c['implicit_facet'][bits[0]] = bits[1]
        elif len(bits) == 1:
            # send request through as an implicit facet on type, if said type exists
            qry = 'type' + config["facet_field"] + ':' + bits[0]
            check = bibserver.dao.Record.query(q=qry,size=1)
            if check["hits"]["total"] != 0:
                c['implicit_facet']["type"] = bits[0]
                args['terms']["type"+config["facet_field"]] = [bits[0]]
            else:
                # otherwise just show a listing of the facet values for that key
                if 'q' in args:
                    qryval = args['q']
                else:
                    qryval = "*:*"
                result = bibserver.dao.Record.query(q=qryval,facet_fields=[bits[0]+config["facet_field"]])
                vals = result["facets"][bits[0]+config["facet_field"]]["terms"]
                #return render_template('search/listing.html', vals=vals)

    # get results and render
    results = bibserver.dao.Record.query(**args)
    args['path'] = path
    c['io'] = bibserver.iomanager.IOManager(results, args)

    if JSON:
        resp = make_response( json.dumps(c['io'].set(), indent=4 ) )
        resp.mimetype = "application/json"
        return resp

    return render_template('search/index.html', c=c)

if __name__ == "__main__":
    bibserver.dao.init_db()
    app.run(host='0.0.0.0', debug=True)

<commit_msg>testing query endpoint changes<commit_after>import os
from datetime import datetime
import urllib2
from copy import deepcopy
import unicodedata
import httplib

from flask import Flask, jsonify, json, request, redirect, abort, make_response
from flask import render_template
from flask.views import View, MethodView
from flaskext.login import login_user, current_user

import bibserver.dao
from bibserver.config import config
import bibserver.iomanager
import bibserver.importer
from bibserver.core import app, login_manager
from bibserver.view.account import blueprint as account

app.register_blueprint(account, url_prefix='/account')

# NB: the decorator appears to kill the function for normal usage
@login_manager.user_loader
def load_account_for_login_manager(userid):
    out = bibserver.dao.Account.get(userid)
    return out

@app.context_processor
def set_current_user():
    """ Set some template context globals. """
    return dict(current_user=current_user)

@app.route('/')
def home():
    # get list of available collections
    try:
        result = bibserver.dao.Record.query(q="type.exact:collection",sort={"received.exact":{"order":"desc"}})
        if result["hits"]["total"] != 0:
            colls = [i["_source"]["collection"] for i in result["hits"]["hits"]]
    except:
        colls = None
    return render_template('home/index.html', colls=colls, upload=config["allow_upload"] )

@app.route('/content/<path:path>')
def content(path):
    return render_template('home/content.html', page=path)

@app.route('/collection/<collid>/<path:path>')
def record(collid,path):
    JSON = False
    if path.endswith(".json") or path.endswith(".bibjson") or request.values.get('format',"") == "json" or request.values.get('format',"") == "bibjson":
        path = path.replace(".bibjson","").replace(".json","")
        JSON = True

    res = bibserver.dao.Record.query(q='collection:"' + collid + '" AND citekey:"' + path + '"')
    if res["hits"]["total"] == 0:
        abort(404)
    if res["hits"]["total"] > 1:
        return render_template('record.html', msg="hmmm... there is more than one record in this collection with that id...")
    recorddict = res["hits"]["hits"][0]["_source"]

    if JSON:
        resp = make_response( json.dumps( recorddict, indent=4 ) )
        resp.mimetype = "application/json"
        return resp

    return render_template('record.html', record=recorddict)

@app.route('/query', methods=['GET','POST'])
def query():
    if request.method == "GET":
        resp = make_response( bibserver.dao.Record.raw_query(request.query_string) )
        if request.values.get('delete','') and request.values.get('q',''):
            resp = make_response( bibserver.dao.Record.delete_by_query(request.values.get('q')) )
        resp.mimetype = "application/json"
        return resp

    if request.method == "POST":
        if request.values.get('callback',''):
            print request.values["callback"]
        data = json.dumps(dict(request.form).keys()[0])
        print data
        host = str(config['ELASTIC_SEARCH_HOST']).rstrip('/')
        db_name = config['ELASTIC_SEARCH_DB']
        fullpath = '/' + db_name + '/record/_search'
        c =  httplib.HTTPConnection(host)
        c.request('POST', fullpath, data)
        resp = make_response(c.getresponse().read())
        resp.mimetype = "application/json"
        return resp

class UploadView(MethodView):
    '''The upload view.

    upload from URL provided in source, or from file upload button, or from
    POST default format is bibtex, but accept other format specifications via
    format default upload is a collection, but could also be person or group
    record
    '''
    def get(self):
        if request.values.get("source") is not None:
            return self.post()
        return render_template('upload.html')

    def post(self):
        pkg = self.package()
        if self.validate(pkg):
            importer = bibserver.importer.Importer()
#            try:
            res = importer.upload(pkg)
            if res != "DUPLICATE":
                if "collection" in pkg:
                    return redirect('/collection/' + pkg["collection"])
                msg = "Your records were uploaded but no collection name could be discerned."
            elif res == "DUPLICATE":
                msg = "The collection name you specified is already in use."
                msg += "
Please use another collection name."
            else:
                msg = "Sorry! There was an indexing error. Please try again."
#            except:
#                msg = 'Sorry! There was an error indexing your collection. Please try again.'
        else:
            msg = 'Your upload failed to validate. Please try again.'
        return render_template('upload.html', msg=msg)

    def validate(self, pkg):
        '''validate the submission before proceeding'''
        if "source" in pkg or "upfile" in pkg or "data" in pkg:
            return True
        return False

    def package(self):
        '''make package with:
            format of upload (default to bibtex)
            collection name (default to version of filename)
            email address (for a file upload)
            source, upfile, or data, depending on if URL, upload, or POST
            date received
        '''
        pkg = dict()
        if request.values.get("source"):
            pkg["source"] = urllib2.unquote(request.values.get("source"))
            pkg["format"] = self.findformat(pkg["source"])
        if request.files.get('upfile'):
            pkg["upfile"] = request.files.get('upfile')
            pkg["format"] = self.findformat(str(pkg["upfile"].filename))
        if request.values.get('format'):
            pkg["format"] = request.values.get('format')
        if request.values.get('data'):
            pkg["data"] = request.values['data']
        if request.values.get("collection"):
            pkg["collection"] = request.values.get("collection")
        pkg["email"] = request.values.get("email", None)
        pkg["received"] = str(datetime.now())
        return pkg

    def findformat(self,filename):
        if filename.endswith(".json"): return "json"
        if filename.endswith(".bibjson"): return "bibjson"
        if filename.endswith(".bibtex"): return "bibtex"
        if filename.endswith(".bib"): return "bibtex"
        if filename.endswith(".csv"): return "csv"
        return "bibtex"

# enable upload unless not allowed in config
if config["allow_upload"] == "YES":
    app.add_url_rule('/upload', view_func=UploadView.as_view('upload'))

@app.route('/search')
@app.route('/<path:path>')
def search(path=''):

    JSON = False
    if path.endswith(".json") or path.endswith(".bibjson") or request.values.get('format',"") == "json" or request.values.get('format',"") == "bibjson":
        path = path.replace(".bibjson","").replace(".json","")
        JSON = True

    # read args from config and params
    args = {"terms":{},"facet_fields" : [i + config["facet_field"] for i in config["facet_fields"]]}
    if 'from' in request.values:
        args['start'] = request.values.get('from')
    if 'size' in request.values:
        args['size'] = request.values.get('size')
    if 'sort' in request.values:
        if request.values.get("sort") != "..." and request.values.get("sort") != "":
            args['sort'] = {request.values.get('sort')+config["facet_field"] : {"order" : request.values.get('order','asc')}}
    if 'q' in request.values:
        if len(request.values.get('q')) > 0:
            args['q'] = request.values.get('q')
    for param in request.values:
        if param in config["facet_fields"]:
            vals = json.loads(unicodedata.normalize('NFKD',urllib2.unquote(request.values.get(param))).encode('utf-8','ignore'))
            args["terms"][param + config["facet_field"]] = vals

    # get implicit facet
    c = {'implicit_facet': {}}
    if path != '' and not path.startswith("search"):
        path = path.strip()
        if path.endswith("/"):
            path = path[:-1]
        bits = path.split('/')
        if len(bits) == 2:
            # its an implicit facet
            args['terms'][bits[0]+config["facet_field"]] = [bits[1]]
            c['implicit_facet'][bits[0]] = bits[1]
        elif len(bits) == 1:
            # send request through as an implicit facet on type, if said type exists
            qry = 'type' + config["facet_field"] + ':' + bits[0]
            check = bibserver.dao.Record.query(q=qry,size=1)
            if check["hits"]["total"] != 0:
                c['implicit_facet']["type"] = bits[0]
                args['terms']["type"+config["facet_field"]] = [bits[0]]
            else:
                # otherwise just show a listing of the facet values for that key
                if 'q' in args:
                    qryval = args['q']
                else:
                    qryval = "*:*"
                result = bibserver.dao.Record.query(q=qryval,facet_fields=[bits[0]+config["facet_field"]])
                vals = result["facets"][bits[0]+config["facet_field"]]["terms"]
                #return render_template('search/listing.html', vals=vals)

    # get results and render
    results = bibserver.dao.Record.query(**args)
    args['path'] = path
    c['io'] = bibserver.iomanager.IOManager(results, args)

    if JSON:
        resp = make_response( json.dumps(c['io'].set(), indent=4 ) )
        resp.mimetype = "application/json"
        return resp

    return render_template('search/index.html', c=c)

if __name__ == "__main__":
    bibserver.dao.init_db()
    app.run(host='0.0.0.0', debug=True)

<|endoftext|>


<commit_before>"""\
@file tpool.py

Copyright (c) 2007, Linden Research, Inc.
Copyright (c) 2007, IBM Corp.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

  http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
"""

import os, threading
import Queue

from sys import stdout
from Queue import Empty, Queue

from eventlet import api, coros, greenio
from eventlet.api import trampoline, get_hub

try:
    set
except NameError:  # python 2.3 compatibility
    from sets import Set as set

QUIET=False

_rpipe, _wpipe = os.pipe()
_rfile = os.fdopen(_rpipe,"r",0)
## Work whether or not wrap_pipe_with_coroutine_pipe was called
if not isinstance(_rfile, greenio.GreenPipe):
    _rfile = greenio.GreenPipe(_rfile)

def _signal_t2e():
    from eventlet import util
    nwritten = util.__original_write__(_wpipe, ' ')

_reqq = Queue(maxsize=-1)
_rspq = Queue(maxsize=-1)

def tpool_trampoline():
    global _reqq, _rspq
    while(True):
        _c = _rfile.recv(1)
        assert(_c != "")
        while not _rspq.empty():
            try:
                (e,rv) = _rspq.get(block=False)
                e.send(rv)
            except Empty:
                pass

def esend(meth,*args, **kwargs):
    global _reqq, _rspq
    e = coros.event()
    _reqq.put((e,meth,args,kwargs))
    return e

def tworker():
    global _reqq, _rspq
    while(True):
        msg = _reqq.get()
        if msg is None:
            return
        (e,meth,args,kwargs) = msg
        rv = None
        try:
            rv = meth(*args,**kwargs)
        except Exception,exn:
            import sys, traceback
            (a,b,tb) = sys.exc_info()
            rv = (exn,a,b,tb)
        _rspq.put((e,rv))
        _signal_t2e()

def erecv(e):
    rv = e.wait()
    if isinstance(rv,tuple) and len(rv) == 4 and isinstance(rv[0],Exception):
        import sys, traceback
        (e,a,b,tb) = rv
        if not QUIET:
            traceback.print_exception(Exception,e,tb)
            traceback.print_stack()
        raise e
    return rv

def execute(meth,*args, **kwargs):
    """Execute method in a thread, blocking the current
    coroutine until the method completes.
    """
    e = esend(meth,*args,**kwargs)
    rv = erecv(e)
    return rv

## TODO deprecate
erpc = execute

def proxy_call(autowrap, f, *args, **kwargs):
    """ Call a function *f* and returns the value.  If the type of the
    return value is in the *autowrap* collection, then it is wrapped in
    a Proxy object before return.  Normally *f* will be called
    nonblocking with the execute method; if the keyword argument
    "nonblocking" is set to true, it will simply be executed directly."""
    if kwargs.pop('nonblocking',False):
        rv = f(*args, **kwargs)
    else:
        rv = execute(f,*args,**kwargs)
    if type(rv) in autowrap:
        return Proxy(rv, autowrap)
    else:
        return rv

class Proxy(object):
    """ a simple proxy-wrapper of any object that comes with a methods-only interface,
    in order to forward every method invocation onto a thread in the native-thread pool.
    A key restriction is that the object's methods cannot call into eventlets, since the
    eventlet dispatcher runs on a different native thread.  This is for running native-threaded
    code only. """
    def __init__(self, obj,autowrap=()):
        self._obj = obj
        if isinstance(autowrap, (list, tuple)):
            autowrap = set(autowrap)
        self._autowrap = autowrap

    def __getattr__(self,attr_name):
        f = getattr(self._obj,attr_name)
        if not callable(f):
            return f
        def doit(*args, **kwargs):
            return proxy_call(self._autowrap, f, *args, **kwargs)
        return doit

    # the following are a buncha methods that the python interpeter
    # doesn't use getattr to retrieve and therefore have to be defined
    # explicitly
    def __getitem__(self, key):
        return proxy_call(self._autowrap, self._obj.__getitem__, key)
    def __setitem__(self, key, value):
        return proxy_call(self._autowrap, self._obj.__setitem__, key, value)
    def __deepcopy__(self, memo=None):
        return proxy_call(self._autowrap, self._obj.__deepcopy__, memo)
    def __copy__(self, memo=None):
        return proxy_call(self._autowrap, self._obj.__copy__, memo)
    # these don't go through a proxy call, because they're likely to
    # be called often, and are unlikely to be implemented on the
    # wrapped object in such a way that they would block
    def __eq__(self, rhs):
        return self._obj.__eq__(rhs)
    def __repr__(self):
        return self._obj.__repr__()
    def __str__(self):
        return self._obj.__str__()
    def __len__(self):
        return len(self._obj)
    def __nonzero__(self):
        return bool(self._obj)

_nthreads = int(os.environ.get('EVENTLET_THREADPOOL_SIZE', 20))
_threads = {}
def setup():
    global _threads
    for i in range(0,_nthreads):
        _threads[i] = threading.Thread(target=tworker)
        _threads[i].setDaemon(True)
        _threads[i].start()

    api.spawn(tpool_trampoline)

setup()

def killall():
    for i in _threads:
        _reqq.put(None)
    for thr in _threads.values():
        thr.join()

<commit_msg>https://bitbucket.org/which_linden/eventlet/<commit_after>"""\
@file tpool.py

Copyright (c) 2007, Linden Research, Inc.
Copyright (c) 2007, IBM Corp.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

  http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
"""

import os
import threading

from Queue import Empty, Queue

from eventlet import api, coros, greenio

QUIET=False

_rpipe, _wpipe = os.pipe()
_rfile = os.fdopen(_rpipe,"r",0)
## Work whether or not wrap_pipe_with_coroutine_pipe was called
if not isinstance(_rfile, greenio.GreenPipe):
    _rfile = greenio.GreenPipe(_rfile)

def _signal_t2e():
    from eventlet import util
    nwritten = util.__original_write__(_wpipe, ' ')

_reqq = Queue(maxsize=-1)
_rspq = Queue(maxsize=-1)

def tpool_trampoline():
    global _reqq, _rspq
    while(True):
        _c = _rfile.recv(1)
        assert(_c != "")
        while not _rspq.empty():
            try:
                (e,rv) = _rspq.get(block=False)
                e.send(rv)
                rv = None
            except Empty:
                pass

def esend(meth,*args, **kwargs):
    global _reqq, _rspq
    e = coros.event()
    _reqq.put((e,meth,args,kwargs))
    return e

SYS_EXCS = (KeyboardInterrupt, SystemExit)

def tworker():
    global _reqq, _rspq
    while(True):
        msg = _reqq.get()
        if msg is None:
            return
        (e,meth,args,kwargs) = msg
        rv = None
        try:
            rv = meth(*args,**kwargs)
        except SYS_EXCS:
            raise
        except Exception,exn:
            import sys, traceback
            (a,b,tb) = sys.exc_info()
            rv = (exn,a,b,tb)
        _rspq.put((e,rv))
        meth = args = kwargs = e = rv = None
        _signal_t2e()

def erecv(e):
    rv = e.wait()
    if isinstance(rv,tuple) and len(rv) == 4 and isinstance(rv[0],Exception):
        import sys, traceback
        (e,a,b,tb) = rv
        if not QUIET:
            traceback.print_exception(Exception,e,tb)
            traceback.print_stack()
        raise e
    return rv

def execute(meth,*args, **kwargs):
    """Execute method in a thread, blocking the current
    coroutine until the method completes.
    """
    e = esend(meth,*args,**kwargs)
    rv = erecv(e)
    return rv

## TODO deprecate
erpc = execute

def proxy_call(autowrap, f, *args, **kwargs):
    """ Call a function *f* and returns the value.  If the type of the
    return value is in the *autowrap* collection, then it is wrapped in
    a Proxy object before return.  Normally *f* will be called
    nonblocking with the execute method; if the keyword argument
    "nonblocking" is set to true, it will simply be executed directly."""
    if kwargs.pop('nonblocking',False):
        rv = f(*args, **kwargs)
    else:
        rv = execute(f,*args,**kwargs)
    if isinstance(rv, autowrap):
        return Proxy(rv, autowrap)
    else:
        return rv

class Proxy(object):
    """ a simple proxy-wrapper of any object that comes with a methods-only interface,
    in order to forward every method invocation onto a thread in the native-thread pool.
    A key restriction is that the object's methods cannot call into eventlets, since the
    eventlet dispatcher runs on a different native thread.  This is for running native-threaded
    code only. """
    def __init__(self, obj,autowrap=()):
        self._obj = obj
        self._autowrap = autowrap

    def __getattr__(self,attr_name):
        f = getattr(self._obj,attr_name)
        if not callable(f):
            return f
        def doit(*args, **kwargs):
            return proxy_call(self._autowrap, f, *args, **kwargs)
        return doit

    # the following are a buncha methods that the python interpeter
    # doesn't use getattr to retrieve and therefore have to be defined
    # explicitly
    def __getitem__(self, key):
        return proxy_call(self._autowrap, self._obj.__getitem__, key)
    def __setitem__(self, key, value):
        return proxy_call(self._autowrap, self._obj.__setitem__, key, value)
    def __deepcopy__(self, memo=None):
        return proxy_call(self._autowrap, self._obj.__deepcopy__, memo)
    def __copy__(self, memo=None):
        return proxy_call(self._autowrap, self._obj.__copy__, memo)
    # these don't go through a proxy call, because they're likely to
    # be called often, and are unlikely to be implemented on the
    # wrapped object in such a way that they would block
    def __eq__(self, rhs):
        return self._obj.__eq__(rhs)
    def __repr__(self):
        return self._obj.__repr__()
    def __str__(self):
        return self._obj.__str__()
    def __len__(self):
        return len(self._obj)
    def __nonzero__(self):
        return bool(self._obj)

_nthreads = int(os.environ.get('EVENTLET_THREADPOOL_SIZE', 20))
_threads = {}
def setup():
    global _threads
    for i in range(0,_nthreads):
        _threads[i] = threading.Thread(target=tworker)
        _threads[i].setDaemon(True)
        _threads[i].start()

    api.spawn(tpool_trampoline)

setup()

def killall():
    for i in _threads:
        _reqq.put(None)
    for thr in _threads.values():
        thr.join()

<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
from django.shortcuts import render
from django.http import HttpResponse
from django.contrib.auth.decorators import login_required
from reportlab.pdfgen import canvas
from reportlab.lib.pagesizes import A4
from reportlab.lib.units import cm
from reportlab.platypus import Table
from core import settings
from io import BytesIO
from models import Bill

@login_required
def generate_pdf(request, id):
    bill = Bill.objects.get(id=id)
    response = HttpResponse(content_type='application/pdf')
    response['Content-Disposition'] = 'attachment; filename="%s.pdf"' % bill.number

    # Create a buffer
    buffer = BytesIO()
    pdf = canvas.Canvas(buffer, pagesize=A4)
    # define new 0,0 bottom left with cm as margin
    pdf.translate(cm,cm)
    # define document width and height with cm as margin
    width, height = A4
    width = width - 2*cm
    height = height - 2*cm

    # if debug draw lines for document limit
    if settings.DEBUG is True:
        pdf.setStrokeColorRGB(1,0,0)
        pdf.line(0,0,width,0)
        pdf.line(0,0,0,height)
        pdf.line(0,height,width,height)
        pdf.line(width,height,width,0)

    # Put logo on top of pdf original image size is 570px/250px
    pdf.drawImage(settings.STATICFILES_DIRS[0]+'/logo-coworking.jpg', 0, height-75, width=138, height=75)
    # billing information
    lh = 15 #define a line height
    pdf.setFillColorRGB(0.3,0.3,0.3)
    pdf.setFont("Helvetica-Bold", 14)
    pdf.drawRightString(width, height-lh, 'Facture');
    pdf.setFont("Helvetica-Bold", 10)
    pdf.drawRightString(width, height-2*lh, u'Numéro : %s' % bill.number)
    pdf.setFont("Helvetica", 10)
    pdf.drawRightString(width, height-3*lh, u'Date facturation : %s' % bill.billing_date.strftime('%d/%m/%Y'))

    # define new heght
    nh = height - 90

    # seller
    pdf.setFillColorRGB(0.95,0.95,0.95)
    # rect(x,y,width,height)
    pdf.rect(0, nh-8*lh, width/2-40, 6.4*lh, fill=1)
    # reset fill for text color
    pdf.setFillColorRGB(0.3,0.3,0.3)
    pdf.drawString(10, nh-lh, 'Emetteur')
    pdf.drawString(20, nh-3*lh, 'Cowork\'in Montpellier')
    pdf.drawString(20, nh-4*lh, '19 rue de l\'école de droit')
    pdf.drawString(20, nh-5*lh, '34000 Montpellier')

    # customer
    pdf.drawString(width/2, nh-lh, 'Adressé à')
    customer = pdf.beginText()
    customer.setTextOrigin(width/2+20, nh-3*lh)
    # create text with \n and remove \r
    text = '%s %s\n%s' % (bill.user.first_name, bill.user.last_name,
                bill.user.userprofile.billing_address.replace('\r',''))
    # get each line
    for line in text.split('\n'):
        customer.textOut(line)
        customer.moveCursor(0,lh)
    pdf.drawText(customer)
    pdf.setStrokeColorRGB(0,0,0)
    # rect(x,y,width,height)
    pdf.rect(width/2, nh-8*lh, width/2, 6.4*lh, fill=0)

    # define new heght
    nh = nh - 10*lh

    data = [['Désignation', 'Prix HT', 'Quantité', 'Total HT']]

    for line in bill.billline_set.all():
        description = '%s - %s' % (line.service.reference, line.service.name)
        line = (description, line.service.price, line.quantity, line.total)
        data.append(line)

    table = Table(data)
    table.wrapOn(pdf, width, height)
    table.drawOn(pdf, 0, nh)

    pdf.showPage()
    pdf.save()
    # get pdf from buffer and return it to response
    genpdf = buffer.getvalue()
    buffer.close()
    response.write(genpdf)
    return response
<commit_msg>add table footer<commit_after># -*- coding: utf-8 -*-
from django.shortcuts import render
from django.http import HttpResponse
from django.contrib.auth.decorators import login_required
from reportlab.pdfgen import canvas
from reportlab.lib import colors
from reportlab.lib.pagesizes import A4
from reportlab.lib.units import cm
from reportlab.platypus import Table
from core import settings
from io import BytesIO
from models import Bill

@login_required
def generate_pdf(request, id):
    bill = Bill.objects.get(id=id)
    response = HttpResponse(content_type='application/pdf')
    response['Content-Disposition'] = 'attachment; filename="%s.pdf"' % bill.number

    # Create a buffer
    buffer = BytesIO()
    pdf = canvas.Canvas(buffer, pagesize=A4)
    # define new 0,0 bottom left with cm as margin
    pdf.translate(cm,cm)
    # define document width and height with cm as margin
    width, height = A4
    width = width - 2*cm
    height = height - 2*cm

    # if debug draw lines for document limit
    if settings.DEBUG is True:
        pdf.setStrokeColorRGB(1,0,0)
        pdf.line(0,0,width,0)
        pdf.line(0,0,0,height)
        pdf.line(0,height,width,height)
        pdf.line(width,height,width,0)

    # Put logo on top of pdf original image size is 570px/250px
    pdf.drawImage(settings.STATICFILES_DIRS[0]+'/logo-coworking.jpg', 0, height-75, width=138, height=75)
    # billing information
    lh = 15 #define a line height
    pdf.setFillColorRGB(0.3,0.3,0.3)
    pdf.setFont("Helvetica-Bold", 14)
    pdf.drawRightString(width, height-lh, 'Facture');
    pdf.setFont("Helvetica-Bold", 10)
    pdf.drawRightString(width, height-2*lh, u'Numéro : %s' % bill.number)
    pdf.setFont("Helvetica", 10)
    pdf.drawRightString(width, height-3*lh, u'Date facturation : %s' % bill.billing_date.strftime('%d/%m/%Y'))

    # define new heght
    nh = height - 90

    # seller
    pdf.setFillColorRGB(0.95,0.95,0.95)
    # rect(x,y,width,height)
    pdf.rect(0, nh-8*lh, width/2-40, 6.4*lh, fill=1)
    # reset fill for text color
    pdf.setFillColorRGB(0.3,0.3,0.3)
    pdf.drawString(10, nh-lh, 'Emetteur')
    pdf.drawString(20, nh-3*lh, 'Cowork\'in Montpellier')
    pdf.drawString(20, nh-4*lh, '19 rue de l\'école de droit')
    pdf.drawString(20, nh-5*lh, '34000 Montpellier')

    # customer
    pdf.drawString(width/2, nh-lh, 'Adressé à')
    customer = pdf.beginText()
    customer.setTextOrigin(width/2+20, nh-3*lh)
    # create text with \n and remove \r
    text = '%s %s\n%s' % (bill.user.first_name, bill.user.last_name,
                bill.user.userprofile.billing_address.replace('\r',''))
    # get each line
    for line in text.split('\n'):
        customer.textOut(line)
        customer.moveCursor(0,lh)
    pdf.drawText(customer)
    pdf.setStrokeColorRGB(0,0,0)
    # rect(x,y,width,height)
    pdf.rect(width/2, nh-8*lh, width/2, 6.4*lh, fill=0)

    # define new heght
    nh = nh - 10*lh

    data = [['Désignation', 'Prix unit. HT', 'Quantité', 'Total HT']]

    for line in bill.billline_set.all():
        description = '%s - %s\n%s' % (line.service.reference, line.service.name, line.service.description)
        line = (description, line.service.price, line.quantity, line.total)
        data.append(line)

    data.append(('', '', 'Total HT', '%s €' % bill.amount))
    data.append(('', '', 'TVA 0%', '0'))
    data.append(('', '', 'Total TTC', '%s €' % bill.amount))

    # widths in percent of pdf width
    colWidths = (width*0.55, width*0.15, width*0.15, width*0.15)
    style = [('GRID', (0,0), (-1,0),1,colors.black),
            ('ALIGN',(0,0),(0,-1),'LEFT'),
            ('ALIGN',(1,0),(-1,-1),'CENTER'),
            ('ALIGN',(-1,0),(-1,-1),'RIGHT'),
            ]
    table = Table(data, colWidths=colWidths, style=style)
    table.wrapOn(pdf, width, height)
    table.drawOn(pdf, 0, nh-2*lh)

    pdf.showPage()
    pdf.save()
    # get pdf from buffer and return it to response
    genpdf = buffer.getvalue()
    buffer.close()
    response.write(genpdf)
    return response
<|endoftext|>


<commit_before>'''
agent_groups
============

The following methods allow for interaction into the Tenable.io
`agent groups <https://cloud.tenable.com/api#/resources/agent-groups>`_
API endpoints.

Methods available on ``tio.agent_groups``:

.. rst-class:: hide-signature
.. autoclass:: AgentGroupsAPI

    .. automethod:: add_agent
    .. automethod:: configure
    .. automethod:: create
    .. automethod:: delete
    .. automethod:: delete_agent
    .. automethod:: details
    .. automethod:: task_status
'''
from .base import TIOEndpoint

class AgentGroupsAPI(TIOEndpoint):
    def add_agent(self, group_id, *agent_ids, **kw):
        '''
        Adds an agent or multiple agents to the agent group specified.

        `agent-groups: add-agent <https://cloud.tenable.com/api#/resources/agent-groups/add-agent>`_

        Args:
            group_id (int): The id of the group
            *agent_ids (int): The id of the agent
            scanner_id (int, optional): The id of the scanner

        Returns:
            None: Single Agent added successfully
            dict: Task resource if multiple Agents were added.

        Examples:
            Adding a singular agent:

            >>> tio.agent_groups.add_agent(1, 1)

            Adding multiple agents:

            >>> tio.agent_groups.add_agent(1, 1, 2, 3)
        '''
        scanner_id = 1
        if 'scanner_id' in kw:
            scanner_id = kw['scanner_id']

        if len(agent_ids) <= 1:
            # if there is only 1 agent id, we will perform a singular add.
            self._api.put('scanners/{}/agent-groups/{}/agents/{}'.format(
                self._check('scanner_id', scanner_id, int),
                self._check('group_id', group_id, int),
                self._check('agent_id', agent_ids[0], int)
            ))
        else:
            # If there are many agent_ids, then we will want to perform a bulk
            # operation.
            return self._api.post(
                'scanners/{}/agent-groups/{}/agents/_bulk/add'.format(
                    self._check('scanner_id', scanner_id, int),
                    self._check('group_id', group_id, int)),
                json={'items': [self._check('agent_id', i, int) for i in agent_ids]}).json()

    def configure(self, group_id, name, scanner_id=1):
        '''
        Renames an existing agent group.

        `agent-groups: configure <https://cloud.tenable.com/api#/resources/agent-groups/configure>`_

        Args:
            group_id (int): The id of the group
            name (str): The new name for the agent group
            scanner_id (int, optional): The id of the scanner

        Returns:
            None: Agent group updated successfully

        Examples:
            >>> tio.agent_groups.configure(1, 'New Name')
        '''
        self._api.put('scanners/{}/agent-groups/{}'.format(
            self._check('scanner_id', scanner_id, int),
            self._check('group_id', group_id, int)
        ), json={'name': self._check('name', name, str)}).json()

    def create(self, name, scanner_id=1):
        '''
        Creates a new agent group.

        `agent-groups: create <https://cloud.tenable.com/api#/resources/agent-groups/create>`_

        Args:
            name (str): The name of the agent group
            scanner_id (int, optional):
                The id of the scanner to add the agent group to

        Returns:
            dict:
                The dictionary object representing the newly minted agent group

        Examples:
            >>> group = tio.agent_groups.create('New Agent Group')
        '''
        return self._api.post(
            'scanners/{}/agent-groups'.format(
                self._check('scanner_id', scanner_id, int)
            ), json={'name': self. _check('name', name, str)}).json()

    def delete(self, group_id, scanner_id=1):
        '''
        Delete an agent group.

        `agent-groups: delete <https://cloud.tenable.com/api#/resources/agent-groups/delete>`_

        Args:
            group_id (int): The id of the agent group to delete
            scanner_id (int, optional): The id of the scanner

        Returns:
            None: Agent group deleted successfully

        Examples:
            >>> tio.agent_groups.delete(1)
        '''
        self._api.delete('scanners/{}/agent-groups/{}'.format(
            self._check('scanner_id', scanner_id, int),
            self._check('group_id', group_id, int)
        ))

    def delete_agent(self, group_id, *agent_ids, **kw):
        '''
        Delete one or many agents from an agent group.

        `agent-groups: delete-agent <https://cloud.tenable.com/api#/resources/agent-groups/delete-agent>`_

        Args:
            group_id (int): The id of the agent group to remove the agent from
            *agent_ids (int): The id of the agent to be removed
            scanner_id (int, optional): The id of the scanner

        Returns:
            None: A singular agent was deleted from the group successfully.
            dict: Task resource if multiple agents were requested to be deleted.

        Examples:
            Delete a singular agent from an agent group:

            >>> tio.agent_groups.delete(1, 1)

            Delete multiple agents from an agent group:

            >>> tio.agent_groups.delete(1, 1, 2, 3)
        '''
        scanner_id = 1
        if 'scanner_id' in kw:
            scanner_id = kw['scanner_id']

        if len(agent_ids) <= 1:
            # if only a singular agent_id was passed, then we will want to
            self._api.delete('scanners/{}/agent-groups/{}/agents/{}'.format(
                self._check('scanner_id', scanner_id, int),
                self._check('group_id', group_id, int),
                self._check('agent_id', agent_ids[0], int)
            ))
        else:
            # if multiple agent ids were requested to be deleted, then we will
            # call the bulk deletion API.
            return self._api.post(
                'scanners/{}/agent-groups/{}/agents/_bulk/add'.format(
                    self._check('scanner_id', scanner_id, int),
                    self._check('group_id', group_id, int)),
                json={'items': [self._check('agent_ids', i, int) for i in agent_ids]}).json()

    def details(self, group_id, scanner_id=1):
        '''
        Retrieve the details about the specified agent group.

        `agent-groups: details <https://cloud.tenable.com/api#/resources/agent-groups/details>`_

        Args:
            group_id (int): The id of the agent group to remove the agent from
            scanner_id (int, optional): The id of the scanner

        Returns:
            dict:
                The dictionary object representing the requested agent group

        Examples:
            >>> group = tio.agent_groups.details(1)
            >>> pprint(group)
        '''
        return self._api.get(
            'scanners/{}/agent-groups/{}'.format(
                self._check('scanner_id', scanner_id, int),
                self._check('group_id', group_id, int)
            )).json()

    def task_status(self, group_id, task_uuid, scanner_id=1):
        '''
        Retrieves the current status of a bulk task.

        `bulk-operations: bulk-agent-group-status <https://cloud.tenable.com/api#/resources/bulk-operations/bulk-agent-group-status>`_

        Args:
            group_id (int): The id of the group
            task_uuid (str): The id of the task
            scanner_id (int, optional): The id of the scanner

        Returns:
            dict: Task resource

        Examples:
            >>> item = tio.agent_groups.add_agent(1, 21, 22, 23)
            >>> task = tio.agent_groups.task_status(item['task_uuid'])
            >>> pprint(task)
        '''
        return self._api.get(
            'scanners/{}/agent-groups/{}/agents/_bulk/{}'.format(
                self._check('scanner_id', scanner_id, int),
                self._check('group_id', group_id, int),
                self._check('task_uuid', task_uuid, 'uuid')
            )).json()<commit_msg>Issue 59: updated comments and endpoint for delete_agent<commit_after>'''
agent_groups
============

The following methods allow for interaction into the Tenable.io
`agent groups <https://cloud.tenable.com/api#/resources/agent-groups>`_
API endpoints.

Methods available on ``tio.agent_groups``:

.. rst-class:: hide-signature
.. autoclass:: AgentGroupsAPI

    .. automethod:: add_agent
    .. automethod:: configure
    .. automethod:: create
    .. automethod:: delete
    .. automethod:: delete_agent
    .. automethod:: details
    .. automethod:: task_status
'''
from .base import TIOEndpoint

class AgentGroupsAPI(TIOEndpoint):
    def add_agent(self, group_id, *agent_ids, **kw):
        '''
        Adds an agent or multiple agents to the agent group specified.

        `agent-groups: add-agent <https://cloud.tenable.com/api#/resources/agent-groups/add-agent>`_

        Args:
            group_id (int): The id of the group
            *agent_ids (int): The id of the agent
            scanner_id (int, optional): The id of the scanner

        Returns:
            None: Single Agent added successfully
            dict: Task resource if multiple Agents were added.

        Examples:
            Adding a singular agent:

            >>> tio.agent_groups.add_agent(1, 1)

            Adding multiple agents:

            >>> tio.agent_groups.add_agent(1, 1, 2, 3)
        '''
        scanner_id = 1
        if 'scanner_id' in kw:
            scanner_id = kw['scanner_id']

        if len(agent_ids) <= 1:
            # if there is only 1 agent id, we will perform a singular add.
            self._api.put('scanners/{}/agent-groups/{}/agents/{}'.format(
                self._check('scanner_id', scanner_id, int),
                self._check('group_id', group_id, int),
                self._check('agent_id', agent_ids[0], int)
            ))
        else:
            # If there are many agent_ids, then we will want to perform a bulk
            # operation.
            return self._api.post(
                'scanners/{}/agent-groups/{}/agents/_bulk/add'.format(
                    self._check('scanner_id', scanner_id, int),
                    self._check('group_id', group_id, int)),
                json={'items': [self._check('agent_id', i, int) for i in agent_ids]}).json()

    def configure(self, group_id, name, scanner_id=1):
        '''
        Renames an existing agent group.

        `agent-groups: configure <https://cloud.tenable.com/api#/resources/agent-groups/configure>`_

        Args:
            group_id (int): The id of the group
            name (str): The new name for the agent group
            scanner_id (int, optional): The id of the scanner

        Returns:
            None: Agent group updated successfully

        Examples:
            >>> tio.agent_groups.configure(1, 'New Name')
        '''
        self._api.put('scanners/{}/agent-groups/{}'.format(
            self._check('scanner_id', scanner_id, int),
            self._check('group_id', group_id, int)
        ), json={'name': self._check('name', name, str)}).json()

    def create(self, name, scanner_id=1):
        '''
        Creates a new agent group.

        `agent-groups: create <https://cloud.tenable.com/api#/resources/agent-groups/create>`_

        Args:
            name (str): The name of the agent group
            scanner_id (int, optional):
                The id of the scanner to add the agent group to

        Returns:
            dict:
                The dictionary object representing the newly minted agent group

        Examples:
            >>> group = tio.agent_groups.create('New Agent Group')
        '''
        return self._api.post(
            'scanners/{}/agent-groups'.format(
                self._check('scanner_id', scanner_id, int)
            ), json={'name': self. _check('name', name, str)}).json()

    def delete(self, group_id, scanner_id=1):
        '''
        Delete an agent group.

        `agent-groups: delete <https://cloud.tenable.com/api#/resources/agent-groups/delete>`_

        Args:
            group_id (int): The id of the agent group to delete
            scanner_id (int, optional): The id of the scanner

        Returns:
            None: Agent group deleted successfully

        Examples:
            >>> tio.agent_groups.delete(1)
        '''
        self._api.delete('scanners/{}/agent-groups/{}'.format(
            self._check('scanner_id', scanner_id, int),
            self._check('group_id', group_id, int)
        ))

    def delete_agent(self, group_id, *agent_ids, **kw):
        '''
        Delete one or many agents from an agent group.

        `agent-groups: delete-agent <https://cloud.tenable.com/api#/resources/agent-groups/delete-agent>`_

        Args:
            group_id (int): The id of the agent group to remove the agent from
            *agent_ids (int): The id of the agent to be removed
            scanner_id (int, optional): The id of the scanner

        Returns:
            None: A singular agent was deleted from the group successfully.
            dict: Task resource if multiple agents were requested to be deleted.

        Examples:
            Delete a singular agent from an agent group:

            >>> tio.agent_groups.delete_agent(1, 1)

            Delete multiple agents from an agent group:

            >>> tio.agent_groups.delete_agent(1, 1, 2, 3)
        '''
        scanner_id = 1
        if 'scanner_id' in kw:
            scanner_id = kw['scanner_id']

        if len(agent_ids) <= 1:
            # if only a singular agent_id was passed, then we will want to
            self._api.delete('scanners/{}/agent-groups/{}/agents/{}'.format(
                self._check('scanner_id', scanner_id, int),
                self._check('group_id', group_id, int),
                self._check('agent_id', agent_ids[0], int)
            ))
        else:
            # if multiple agent ids were requested to be deleted, then we will
            # call the bulk deletion API.
            return self._api.post(
                'scanners/{}/agent-groups/{}/agents/_bulk/remove'.format(
                    self._check('scanner_id', scanner_id, int),
                    self._check('group_id', group_id, int)),
                json={'items': [self._check('agent_ids', i, int) for i in agent_ids]}).json()

    def details(self, group_id, scanner_id=1):
        '''
        Retrieve the details about the specified agent group.

        `agent-groups: details <https://cloud.tenable.com/api#/resources/agent-groups/details>`_

        Args:
            group_id (int): The id of the agent group to remove the agent from
            scanner_id (int, optional): The id of the scanner

        Returns:
            dict:
                The dictionary object representing the requested agent group

        Examples:
            >>> group = tio.agent_groups.details(1)
            >>> pprint(group)
        '''
        return self._api.get(
            'scanners/{}/agent-groups/{}'.format(
                self._check('scanner_id', scanner_id, int),
                self._check('group_id', group_id, int)
            )).json()

    def task_status(self, group_id, task_uuid, scanner_id=1):
        '''
        Retrieves the current status of a bulk task.

        `bulk-operations: bulk-agent-group-status <https://cloud.tenable.com/api#/resources/bulk-operations/bulk-agent-group-status>`_

        Args:
            group_id (int): The id of the group
            task_uuid (str): The id of the task
            scanner_id (int, optional): The id of the scanner

        Returns:
            dict: Task resource

        Examples:
            >>> item = tio.agent_groups.add_agent(1, 21, 22, 23)
            >>> task = tio.agent_groups.task_status(item['task_uuid'])
            >>> pprint(task)
        '''
        return self._api.get(
            'scanners/{}/agent-groups/{}/agents/_bulk/{}'.format(
                self._check('scanner_id', scanner_id, int),
                self._check('group_id', group_id, int),
                self._check('task_uuid', task_uuid, 'uuid')
            )).json()
<|endoftext|>


<commit_before># This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.

from nose.plugins.attrib import attr

from socorro.external import MissingArgumentError, DatabaseError
from socorro.external.postgresql.skiplist import SkipList

from unittestbase import PostgreSQLTestCase

#==============================================================================
@attr(integration='postgres')  # for nosetests
class IntegrationTestSkipList(PostgreSQLTestCase):
    """Test socorro.external.postgresql.crashes.Crashes class. """

    #--------------------------------------------------------------------------
    def setUp(self):
        """Set up this test class by populating the reports table with fake
        data. """
        super(IntegrationTestSkipList, self).setUp()

        cursor = self.connection.cursor()
        cursor.execute("""
            TRUNCATE skiplist
            CASCADE
        """)
        cursor.execute("""
            INSERT INTO skiplist (category, rule)
            VALUES
            ('prefix', 'arena_.*'),
            ('prefix', 'CrashInJS'),
            ('irrelevant', 'ashmem'),
            ('irrelevant', 'CxThrowException')
            ;
        """)
        self.connection.commit()
        cursor.close()

    #--------------------------------------------------------------------------
    def tearDown(self):
        """Clean up the database, delete tables and functions. """
        cursor = self.connection.cursor()
        cursor.execute("""
            TRUNCATE skiplist
            CASCADE
        """)
        self.connection.commit()
        cursor.close()
        super(IntegrationTestSkipList, self).tearDown()

    #--------------------------------------------------------------------------
    def test_get(self):
        skiplist = SkipList(config=self.config)

        params = {
        }
        res_expected = {
            "hits": [
                # Note: sort is case-insensitive
                {
                    "category": "irrelevant",
                    "rule": "ashmem",
                },
                {
                    "category": "irrelevant",
                    "rule": "CxThrowException",
                },
                {
                    "category": "prefix",
                    "rule": "arena_.*",
                },
                {
                    "category": "prefix",
                    "rule": "CrashInJS",
                },
            ],
            "total": 4
        }

        res = skiplist.get(**params)
        self.assertEqual(res, res_expected)

    def test_get_with_optional_filtering(self):
        skiplist = SkipList(config=self.config)

        # filter by category
        params = {
            'category': 'irrelevant'
        }
        res_expected = {
            "hits": [
                {
                    "category": "irrelevant",
                    "rule": "ashmem",
                },
                {
                    "category": "irrelevant",
                    "rule": "CxThrowException",
                },
            ],
            "total": 2
        }
        res = skiplist.get(**params)
        self.assertEqual(res, res_expected)

        # filter by rule
        params = {
            'rule': 'ashmem'
        }
        res_expected = {
            "hits": [
                {
                    "category": "irrelevant",
                    "rule": "ashmem",
                },
            ],
            "total": 1
        }
        res = skiplist.get(**params)
        self.assertEqual(res, res_expected)

        # filter by both
        params = {
            'category': 'irrelevant',
            'rule': 'ashmem'
        }
        res_expected = {
            "hits": [
                {
                    "category": "irrelevant",
                    "rule": "ashmem",
                },
            ],
            "total": 1
        }
        res = skiplist.get(**params)
        self.assertEqual(res, res_expected)

    def test_post(self):
        skiplist = SkipList(config=self.config)
        self.assertRaises(MissingArgumentError, skiplist.post)
        self.assertRaises(
            MissingArgumentError,
            skiplist.post,
            category='something'
        )
        self.assertRaises(
            MissingArgumentError,
            skiplist.post,
            rule='something'
        )

        # because of an integrity error since it already exists
        self.assertRaises(
            DatabaseError,
            skiplist.post,
            category='prefix', rule='CrashInJS'
        )

        self.assertTrue(
            skiplist.post(category='suffix', rule='Erik*tiny*font')
        )

        cursor = self.connection.cursor()
        cursor.execute("""
        select * from skiplist where category=%s and rule=%s
        """, ('suffix', 'Erik*tiny*font'))
        first, = cursor.fetchall()
        self.assertEqual(first[0], 'suffix')
        self.assertEqual(first[1], 'Erik*tiny*font')

    def test_delete(self):
        skiplist = SkipList(config=self.config)
        self.assertRaises(MissingArgumentError, skiplist.delete)
        self.assertRaises(
            MissingArgumentError,
            skiplist.delete,
            category='something'
        )
        self.assertRaises(
            MissingArgumentError,
            skiplist.delete,
            rule='something'
        )

        cursor = self.connection.cursor()
        cursor.execute("select count(*) from skiplist")
        first, = cursor.fetchall()
        count = first[0]
        self.assertEqual(count, 4)

        self.assertTrue(skiplist.delete(category='irrelevant', rule='ashmem'))

        cursor.execute("select count(*) from skiplist")
        first, = cursor.fetchall()
        count = first[0]
        self.assertEqual(count, 3)

        cursor.execute("""
        select count(*) from skiplist
        where category=%s and rule=%s
        """, ('irrelevant', 'ashmem'))
        first, = cursor.fetchall()
        count = first[0]
        self.assertEqual(count, 0)

        self.assertTrue(not skiplist.delete(category='neverheard', rule='of'))
<commit_msg>fixes bug 875106 - test for skiplist<commit_after># This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.

from nose.plugins.attrib import attr

from socorro.external import MissingArgumentError, DatabaseError
from socorro.external.postgresql.skiplist import SkipList

from unittestbase import PostgreSQLTestCase

#==============================================================================
@attr(integration='postgres')  # for nosetests
class IntegrationTestSkipList(PostgreSQLTestCase):
    """Test socorro.external.postgresql.crashes.Crashes class. """

    #--------------------------------------------------------------------------
    def setUp(self):
        """Set up this test class by populating the reports table with fake
        data. """
        super(IntegrationTestSkipList, self).setUp()

        cursor = self.connection.cursor()
        cursor.execute("""
            TRUNCATE skiplist
            CASCADE
        """)
        cursor.execute("""
            INSERT INTO skiplist (category, rule)
            VALUES
            ('prefix', 'arena_.*'),
            ('prefix', 'CrashInJS'),
            ('irrelevant', 'ashmem'),
            ('irrelevant', 'CxThrowException'),
            ('line_number', 'signatures_with_line_numbers_re')
            ;
        """)
        self.connection.commit()
        cursor.close()

    #--------------------------------------------------------------------------
    def tearDown(self):
        """Clean up the database, delete tables and functions. """
        cursor = self.connection.cursor()
        cursor.execute("""
            TRUNCATE skiplist
            CASCADE
        """)
        self.connection.commit()
        cursor.close()
        super(IntegrationTestSkipList, self).tearDown()

    #--------------------------------------------------------------------------
    def test_get(self):
        skiplist = SkipList(config=self.config)

        params = {
        }
        res_expected = {
            "hits": [
                # Note: sort is case-insensitive
                {
                    "category": "irrelevant",
                    "rule": "ashmem",
                },
                {
                    "category": "irrelevant",
                    "rule": "CxThrowException",
                },
                {
                    "category": "line_number",
                    "rule": "signatures_with_line_numbers_re",
                },
                {
                    "category": "prefix",
                    "rule": "arena_.*",
                },
                {
                    "category": "prefix",
                    "rule": "CrashInJS",
                },
            ],
            "total": 5
        }

        res = skiplist.get(**params)
        self.assertEqual(res, res_expected)

    def test_get_with_optional_filtering(self):
        skiplist = SkipList(config=self.config)

        # filter by category
        params = {
            'category': 'irrelevant'
        }
        res_expected = {
            "hits": [
                {
                    "category": "irrelevant",
                    "rule": "ashmem",
                },
                {
                    "category": "irrelevant",
                    "rule": "CxThrowException",
                },
            ],
            "total": 2
        }
        res = skiplist.get(**params)
        self.assertEqual(res, res_expected)

        # filter by rule
        params = {
            'rule': 'ashmem'
        }
        res_expected = {
            "hits": [
                {
                    "category": "irrelevant",
                    "rule": "ashmem",
                },
            ],
            "total": 1
        }
        res = skiplist.get(**params)
        self.assertEqual(res, res_expected)

        # filter by both
        params = {
            'category': 'irrelevant',
            'rule': 'ashmem'
        }
        res_expected = {
            "hits": [
                {
                    "category": "irrelevant",
                    "rule": "ashmem",
                },
            ],
            "total": 1
        }
        res = skiplist.get(**params)
        self.assertEqual(res, res_expected)

    def test_post(self):
        skiplist = SkipList(config=self.config)
        self.assertRaises(MissingArgumentError, skiplist.post)
        self.assertRaises(
            MissingArgumentError,
            skiplist.post,
            category='something'
        )
        self.assertRaises(
            MissingArgumentError,
            skiplist.post,
            rule='something'
        )

        # because of an integrity error since it already exists
        self.assertRaises(
            DatabaseError,
            skiplist.post,
            category='prefix', rule='CrashInJS'
        )

        self.assertTrue(
            skiplist.post(category='suffix', rule='Erik*tiny*font')
        )

        cursor = self.connection.cursor()
        cursor.execute("""
        select * from skiplist where category=%s and rule=%s
        """, ('suffix', 'Erik*tiny*font'))
        first, = cursor.fetchall()
        self.assertEqual(first[0], 'suffix')
        self.assertEqual(first[1], 'Erik*tiny*font')

    def test_delete(self):
        skiplist = SkipList(config=self.config)
        self.assertRaises(MissingArgumentError, skiplist.delete)
        self.assertRaises(
            MissingArgumentError,
            skiplist.delete,
            category='something'
        )
        self.assertRaises(
            MissingArgumentError,
            skiplist.delete,
            rule='something'
        )

        cursor = self.connection.cursor()
        cursor.execute("select count(*) from skiplist")
        first, = cursor.fetchall()
        count = first[0]
        self.assertEqual(count, 5)

        self.assertTrue(skiplist.delete(category='irrelevant', rule='ashmem'))

        cursor.execute("select count(*) from skiplist")
        first, = cursor.fetchall()
        count = first[0]
        self.assertEqual(count, 4)

        cursor.execute("""
        select count(*) from skiplist
        where category=%s and rule=%s
        """, ('irrelevant', 'ashmem'))
        first, = cursor.fetchall()
        count = first[0]
        self.assertEqual(count, 0)

        self.assertTrue(not skiplist.delete(category='neverheard', rule='of'))
<|endoftext|>