<commit_before>from __future__ import print_function, division, absolute_import

import os
import hashlib
import datetime
import time

import uuid
import hashlib
import hmac
import base64

import urlparse

import simplejson as json

import redis
Redis = redis.StrictRedis()

from flask import (Flask, g, render_template, url_for, request, make_response,
                   redirect, flash, jsonify)

from wqflask import app

from pprint import pformat as pf

from wqflask.database import db_session

from wqflask import model
from wqflask import user_manager

from utility import Bunch, Struct
from utility.formatting import numify

from base import trait
from base.data_set import create_dataset

import logging
from utility.logger import getLogger
logger = getLogger(__name__)

class AnonCollection(object):
    """User is not logged in"""
    def __init__(self, collection_name):
        anon_user = user_manager.AnonUser()
        self.key = anon_user.key
        self.name = collection_name
        self.id = None
        self.created_timestamp = datetime.datetime.utcnow().strftime('%b %d %Y %I:%M%p')
        self.changed_timestamp = self.created_timestamp #ZS: will be updated when changes are made

        #ZS: Find id and set it if the collection doesn't already exist
        if Redis.get(self.key) == "None" or Redis.get(self.key) == None:
            Redis.set(self.key, None) #ZS: For some reason I get the error "Operation against a key holding the wrong kind of value" if I don't do this
        else:
            collections_list = json.loads(Redis.get(self.key))
            collection_position = 0 #ZS: Position of collection in collection_list, if it exists
            collection_exists = False
            for i, collection in enumerate(collections_list):
                if collection['name'] == self.name:
                    collection_position = i
                    collection_exists = True
                    self.id = collection['id']
                    break

        if self.id == None:
            self.id = str(uuid.uuid4())

    def get_members(self):
        traits = []
        collections_list = json.loads(Redis.get(self.key))
        for collection in collections_list:
            if collection['id'] == self.id:
                traits = collection['members']
        return traits

    @property
    def num_members(self):
        num_members = 0
        collections_list = json.loads(Redis.get(self.key))
        for collection in collections_list:
            if collection['id'] == self.id:
                num_members = collection['num_members']
        return num_members

    def add_traits(self, unprocessed_traits):
        #assert collection_name == "Default", "Unexpected collection name for anonymous user"
        self.traits = list(process_traits(unprocessed_traits))
        existing_collections = Redis.get(self.key)
        logger.debug("existing_collections:", existing_collections)
        if existing_collections != None and existing_collections != "None":
            collections_list = json.loads(existing_collections)
            collection_position = 0 #ZS: Position of collection in collection_list, if it exists
            collection_exists = False
            for i, collection in enumerate(collections_list):
                if collection['id'] == self.id:
                    collection_position = i
                    collection_exists = True
                    break
            if collection_exists:
                collections_list[collection_position]['members'].extend(self.traits)
                collections_list[collection_position]['num_members'] = len(collections_list[collection_position]['members'])
                collections_list[collection_position]['changed_timestamp'] = datetime.datetime.utcnow().strftime('%b %d %Y %I:%M%p')
            else:
                collection_dict = {"id" : self.id,
                                   "name" : self.name,
                                   "created_timestamp" : self.created_timestamp,
                                   "changed_timestamp" : self.changed_timestamp,
                                   "num_members" : len(self.traits),
                                   "members" : self.traits}
                collections_list.append(collection_dict)
        else:
            collections_list = []
            collection_dict = {"id" : self.id,
                               "name" : self.name,
                               "created_timestamp" : self.created_timestamp,
                               "changed_timestamp" : self.changed_timestamp,
                               "num_members" : len(self.traits),
                               "members" : self.traits}
            collections_list.append(collection_dict)

        Redis.set(self.key, json.dumps(collections_list))

    def remove_traits(self, params):
        traits_to_remove = [(":").join(trait.split(":")[:2]) for trait in params.getlist('traits[]')]
        existing_collections = Redis.get(self.key)
        collection_position = 0
        collections_list = json.loads(existing_collections)
        for i, collection in enumerate(collections_list):
            if collection['id'] == self.id:
                collection_position = i
                collection_exists = True
                break
        collections_list[collection_position]['members'] = [trait for trait in collections_list[collection_position]['members'] if trait not in traits_to_remove]
        collections_list[collection_position]['num_members'] = len(collections_list[collection_position]['members'])
        collections_list[collection_position]['changed_timestamp'] = datetime.datetime.utcnow().strftime('%b %d %Y %I:%M%p')
        len_now = collections_list[collection_position]['num_members']
        Redis.set(self.key, json.dumps(collections_list))

        # We need to return something so we'll return this...maybe in the future
        # we can use it to check the results
        return str(len_now)

def process_traits(unprocessed_traits):
    if isinstance(unprocessed_traits, basestring):
        unprocessed_traits = unprocessed_traits.split(",")
    traits = set()
    for trait in unprocessed_traits:
        #print("trait is:", trait)
        data, _separator, hmac = trait.rpartition(':')
        data = data.strip()
        if g.user_session.logged_in:
          assert hmac==user_manager.actual_hmac_creation(data), "Data tampering?"
        traits.add(str(data))

    return traits

def report_change(len_before, len_now):
    new_length = len_now - len_before
    if new_length:
        logger.debug("We've added {} to your collection.".format(
            numify(new_length, 'new trait', 'new traits')))
        flash("We've added {} to your collection.".format(
            numify(new_length, 'new trait', 'new traits')))
    else:
        logger.debug("No new traits were added.")

@app.route("/collections/store_trait_list", methods=('POST',))
def store_traits_list():
   params = request.form

   traits = params['traits']
   hash = params['hash']

   Redis.set(hash, traits)

   return hash

@app.route("/collections/add")
def collections_add():
    if g.user_session.logged_in:
        collections = g.user_session.user_collections
        if len(collections) < 1:
            collection_name = "Default Collection"
            uc_id = g.user_session.add_collection(collection_name, set())
            collections = g.user_session.user_collections
    else:
        anon_collections = user_manager.AnonUser().get_collections()
        collections = []
        for collection in anon_collections:
            collections.append({'id':collection['id'], 'name':collection['name']})

    if 'traits' in request.args:
        traits=request.args['traits']
        return render_template("collections/add.html",
                                traits = traits,
                                collections = collections,
                              )
    else:
        hash = request.args['hash']
        return render_template("collections/add.html",
                                hash = hash,
                                collections = collections,
                              )

@app.route("/collections/new")
def collections_new():
    params = request.args

    if "sign_in" in params:
        return redirect(url_for('login'))
    if "create_new" in params:
        logger.debug("in create_new")
        collection_name = params['new_collection']
        if collection_name.strip() == "":
            collection_name = datetime.datetime.utcnow().strftime('Collection_%b_%d_%H:%M')
        return create_new(collection_name)
    elif "add_to_existing" in params:
        logger.debug("in add to existing")
        if 'existing_collection' not in params:
            default_collection_exists = False
            if g.user_session.logged_in:
                collections = g.user_session.user_collections
            else:
                collections = user_manager.AnonUser().get_collections()
            for collection in collections:
                if collection["name"] == "Default Collection":
                    collection_id = collection["id"]
                    collection_name = collection["name"]
                    default_collection_exists = True
            if not default_collection_exists:
                return create_new("Default Collection")
        else:
            collection_id = params['existing_collection'].split(":")[0]
            collection_name = params['existing_collection'].split(":")[1]
        if g.user_session.logged_in:
            if "hash" in params:
                unprocessed_traits = Redis.get(params['hash'])
            else:
                unprocessed_traits = params['traits']
            traits = list(process_traits(unprocessed_traits))
            g.user_session.add_traits_to_collection(collection_id, traits)
            return redirect(url_for('view_collection', uc_id=collection_id))
        else:
            ac = AnonCollection(collection_name)
            if "hash" in params:
                unprocessed_traits = Redis.get(params['hash'])
            else:
                unprocessed_traits = params['traits']
            ac.add_traits(unprocessed_traits)
            return redirect(url_for('view_collection', collection_id=ac.id))
    else:
        CauseAnError

def create_new(collection_name):
    params = request.args

    if "hash" in params:
        unprocessed_traits = Redis.get(params['hash'])
        Redis.delete(hash)
    else:
        unprocessed_traits = params['traits']

    traits = process_traits(unprocessed_traits)

    if g.user_session.logged_in:
        uc_id = g.user_session.add_collection(collection_name, traits)

        return redirect(url_for('view_collection', uc_id=uc_id))
    else:
        ac = AnonCollection(collection_name)
        ac.changed_timestamp = datetime.datetime.utcnow().strftime('%b %d %Y %I:%M%p')
        ac.add_traits(unprocessed_traits)
        return redirect(url_for('view_collection', collection_id=ac.id))

@app.route("/collections/list")
def list_collections():
    params = request.args

    if g.user_session.logged_in:
        user_collections = list(g.user_session.user_collections)
        #logger.debug("user_collections are:", user_collections)
        return render_template("collections/list.html",
                               params = params,
                               collections = user_collections,
                               )
    else:
        anon_collections = user_manager.AnonUser().get_collections()
        #logger.debug("anon_collections are:", anon_collections)
        return render_template("collections/list.html",
                               params = params,
                               collections = anon_collections)

@app.route("/collections/remove", methods=('POST',))
def remove_traits():
    params = request.form

    if "uc_id" in params:
        uc_id = params['uc_id']
        traits_to_remove = params.getlist('traits[]')
        traits_to_remove = process_traits(traits_to_remove)
        logger.debug("\n\n  after processing, traits_to_remove:", traits_to_remove)

        members_now = g.user_session.remove_traits_from_collection(uc_id, traits_to_remove)
    else:
        collection_name = params['collection_name']
        members_now = AnonCollection(collection_name).remove_traits(params)

    # We need to return something so we'll return this...maybe in the future
    # we can use it to check the results
    return str(len(members_now))

@app.route("/collections/delete", methods=('POST',))
def delete_collection():
    params = request.form
    uc_id = ""
    if g.user_session.logged_in:
        uc_id = params['uc_id']
        if len(uc_id.split(":")) > 1:
            for this_uc_id in uc_id.split(":"):
                collection_name = g.user_session.delete_collection(this_uc_id)
        else:
            collection_name = g.user_session.delete_collection(uc_id)
    else:
        if "collection_name" in params:
            collection_name = params['collection_name']
            user_manager.AnonUser().delete_collection(collection_name)
        else:
            uc_id = params['uc_id']
            for this_collection in uc_id.split(":"):
                user_manager.AnonUser().delete_collection(this_collection)

    if uc_id != "":
        if len(uc_id.split(":")) > 1:
            flash("We've deleted the selected collections.", "alert-info")
        else:
            flash("We've deleted the collection: {}.".format(uc_id), "alert-info")
    else:
        flash("We've deleted the collection: {}.".format(collection_name), "alert-info")

    return redirect(url_for('list_collections'))

@app.route("/collections/view")
def view_collection():
    params = request.args

    if g.user_session.logged_in and "uc_id" in params:
        uc_id = params['uc_id']
        uc = (collection for collection in g.user_session.user_collections if collection["id"] == uc_id).next()
        traits = uc["members"]
    else:
        user_collections = json.loads(Redis.get(user_manager.AnonUser().key))
        this_collection = {}
        for collection in user_collections:
            if collection['id'] == params['collection_id']:
                this_collection = collection
                break

        traits = this_collection['members']

    trait_obs = []
    json_version = []

    for atrait in traits:
        name, dataset_name = atrait.split(':')
        if dataset_name == "Temp":
            group = name.split("_")[2]
            dataset = create_dataset(dataset_name, dataset_type = "Temp", group_name = group)
            trait_ob = trait.GeneralTrait(name=name, dataset=dataset)
        else:
            dataset = create_dataset(dataset_name)
            trait_ob = trait.GeneralTrait(name=name, dataset=dataset)
            trait_ob = trait.retrieve_trait_info(trait_ob, dataset, get_qtl_info=True)
        trait_obs.append(trait_ob)

        json_version.append(trait.jsonable(trait_ob))

    if "uc_id" in params:
        collection_info = dict(trait_obs=trait_obs,
                               uc = uc)
    else:
        collection_info = dict(trait_obs=trait_obs,
                               collection_name=this_collection['name'])
    if "json" in params:
        return json.dumps(json_version)
    else:
        return render_template("collections/view.html",
                           **collection_info
                           )<commit_msg>Fixed an issue that caused Redis to throw an error when creating collections<commit_after>from __future__ import print_function, division, absolute_import

import os
import hashlib
import datetime
import time

import uuid
import hashlib
import hmac
import base64

import urlparse

import simplejson as json

import redis
Redis = redis.StrictRedis()

from flask import (Flask, g, render_template, url_for, request, make_response,
                   redirect, flash, jsonify)

from wqflask import app

from pprint import pformat as pf

from wqflask.database import db_session

from wqflask import model
from wqflask import user_manager

from utility import Bunch, Struct
from utility.formatting import numify

from base import trait
from base.data_set import create_dataset

import logging
from utility.logger import getLogger
logger = getLogger(__name__)

class AnonCollection(object):
    """User is not logged in"""
    def __init__(self, collection_name):
        anon_user = user_manager.AnonUser()
        self.key = anon_user.key
        self.name = collection_name
        self.id = None
        self.created_timestamp = datetime.datetime.utcnow().strftime('%b %d %Y %I:%M%p')
        self.changed_timestamp = self.created_timestamp #ZS: will be updated when changes are made

        #ZS: Find id and set it if the collection doesn't already exist
        if Redis.get(self.key) == "None" or Redis.get(self.key) == None:
            Redis.set(self.key, None) #ZS: For some reason I get the error "Operation against a key holding the wrong kind of value" if I don't do this
        else:
            collections_list = json.loads(Redis.get(self.key))
            collection_position = 0 #ZS: Position of collection in collection_list, if it exists
            collection_exists = False
            for i, collection in enumerate(collections_list):
                if collection['name'] == self.name:
                    collection_position = i
                    collection_exists = True
                    self.id = collection['id']
                    break

        if self.id == None:
            self.id = str(uuid.uuid4())

    def get_members(self):
        traits = []
        collections_list = json.loads(Redis.get(self.key))
        for collection in collections_list:
            if collection['id'] == self.id:
                traits = collection['members']
        return traits

    @property
    def num_members(self):
        num_members = 0
        collections_list = json.loads(Redis.get(self.key))
        for collection in collections_list:
            if collection['id'] == self.id:
                num_members = collection['num_members']
        return num_members

    def add_traits(self, unprocessed_traits):
        #assert collection_name == "Default", "Unexpected collection name for anonymous user"
        self.traits = list(process_traits(unprocessed_traits))
        existing_collections = Redis.get(self.key)
        logger.debug("existing_collections:", existing_collections)
        if existing_collections != None and existing_collections != "None":
            collections_list = json.loads(existing_collections)
            collection_position = 0 #ZS: Position of collection in collection_list, if it exists
            collection_exists = False
            for i, collection in enumerate(collections_list):
                if collection['id'] == self.id:
                    collection_position = i
                    collection_exists = True
                    break
            if collection_exists:
                collections_list[collection_position]['members'].extend(self.traits)
                collections_list[collection_position]['num_members'] = len(collections_list[collection_position]['members'])
                collections_list[collection_position]['changed_timestamp'] = datetime.datetime.utcnow().strftime('%b %d %Y %I:%M%p')
            else:
                collection_dict = {"id" : self.id,
                                   "name" : self.name,
                                   "created_timestamp" : self.created_timestamp,
                                   "changed_timestamp" : self.changed_timestamp,
                                   "num_members" : len(self.traits),
                                   "members" : self.traits}
                collections_list.append(collection_dict)
        else:
            collections_list = []
            collection_dict = {"id" : self.id,
                               "name" : self.name,
                               "created_timestamp" : self.created_timestamp,
                               "changed_timestamp" : self.changed_timestamp,
                               "num_members" : len(self.traits),
                               "members" : self.traits}
            collections_list.append(collection_dict)

        Redis.set(self.key, json.dumps(collections_list))

    def remove_traits(self, params):
        traits_to_remove = [(":").join(trait.split(":")[:2]) for trait in params.getlist('traits[]')]
        existing_collections = Redis.get(self.key)
        collection_position = 0
        collections_list = json.loads(existing_collections)
        for i, collection in enumerate(collections_list):
            if collection['id'] == self.id:
                collection_position = i
                collection_exists = True
                break
        collections_list[collection_position]['members'] = [trait for trait in collections_list[collection_position]['members'] if trait not in traits_to_remove]
        collections_list[collection_position]['num_members'] = len(collections_list[collection_position]['members'])
        collections_list[collection_position]['changed_timestamp'] = datetime.datetime.utcnow().strftime('%b %d %Y %I:%M%p')
        len_now = collections_list[collection_position]['num_members']
        Redis.set(self.key, json.dumps(collections_list))

        # We need to return something so we'll return this...maybe in the future
        # we can use it to check the results
        return str(len_now)

def process_traits(unprocessed_traits):
    if isinstance(unprocessed_traits, basestring):
        unprocessed_traits = unprocessed_traits.split(",")
    traits = set()
    for trait in unprocessed_traits:
        #print("trait is:", trait)
        data, _separator, hmac = trait.rpartition(':')
        data = data.strip()
        if g.user_session.logged_in:
          assert hmac==user_manager.actual_hmac_creation(data), "Data tampering?"
        traits.add(str(data))

    return traits

def report_change(len_before, len_now):
    new_length = len_now - len_before
    if new_length:
        logger.debug("We've added {} to your collection.".format(
            numify(new_length, 'new trait', 'new traits')))
        flash("We've added {} to your collection.".format(
            numify(new_length, 'new trait', 'new traits')))
    else:
        logger.debug("No new traits were added.")

@app.route("/collections/store_trait_list", methods=('POST',))
def store_traits_list():
   params = request.form

   traits = params['traits']
   hash = params['hash']

   Redis.set(hash, traits)

   return hash

@app.route("/collections/add")
def collections_add():
    if g.user_session.logged_in:
        collections = g.user_session.user_collections
        if len(collections) < 1:
            collection_name = "Default Collection"
            uc_id = g.user_session.add_collection(collection_name, set())
            collections = g.user_session.user_collections
    else:
        anon_collections = user_manager.AnonUser().get_collections()
        collections = []
        for collection in anon_collections:
            collections.append({'id':collection['id'], 'name':collection['name']})

    if 'traits' in request.args:
        traits=request.args['traits']
        return render_template("collections/add.html",
                                traits = traits,
                                collections = collections,
                              )
    else:
        hash = request.args['hash']
        return render_template("collections/add.html",
                                hash = hash,
                                collections = collections,
                              )

@app.route("/collections/new")
def collections_new():
    params = request.args

    if "sign_in" in params:
        return redirect(url_for('login'))
    if "create_new" in params:
        logger.debug("in create_new")
        collection_name = params['new_collection']
        if collection_name.strip() == "":
            collection_name = datetime.datetime.utcnow().strftime('Collection_%b_%d_%H:%M')
        return create_new(collection_name)
    elif "add_to_existing" in params:
        logger.debug("in add to existing")
        if 'existing_collection' not in params:
            default_collection_exists = False
            if g.user_session.logged_in:
                collections = g.user_session.user_collections
            else:
                collections = user_manager.AnonUser().get_collections()
            for collection in collections:
                if collection["name"] == "Default Collection":
                    collection_id = collection["id"]
                    collection_name = collection["name"]
                    default_collection_exists = True
            if not default_collection_exists:
                return create_new("Default Collection")
        else:
            collection_id = params['existing_collection'].split(":")[0]
            collection_name = params['existing_collection'].split(":")[1]
        if g.user_session.logged_in:
            if "hash" in params:
                unprocessed_traits = Redis.get(params['hash'])
            else:
                unprocessed_traits = params['traits']
            traits = list(process_traits(unprocessed_traits))
            g.user_session.add_traits_to_collection(collection_id, traits)
            return redirect(url_for('view_collection', uc_id=collection_id))
        else:
            ac = AnonCollection(collection_name)
            if "hash" in params:
                unprocessed_traits = Redis.get(params['hash'])
            else:
                unprocessed_traits = params['traits']
            ac.add_traits(unprocessed_traits)
            return redirect(url_for('view_collection', collection_id=ac.id))
    else:
        CauseAnError

def create_new(collection_name):
    params = request.args

    if "hash" in params:
        unprocessed_traits = Redis.get(params['hash'])
        Redis.delete(params['hash'])
    else:
        unprocessed_traits = params['traits']

    traits = process_traits(unprocessed_traits)

    if g.user_session.logged_in:
        uc_id = g.user_session.add_collection(collection_name, traits)

        return redirect(url_for('view_collection', uc_id=uc_id))
    else:
        ac = AnonCollection(collection_name)
        ac.changed_timestamp = datetime.datetime.utcnow().strftime('%b %d %Y %I:%M%p')
        ac.add_traits(unprocessed_traits)
        return redirect(url_for('view_collection', collection_id=ac.id))

@app.route("/collections/list")
def list_collections():
    params = request.args

    if g.user_session.logged_in:
        user_collections = list(g.user_session.user_collections)
        #logger.debug("user_collections are:", user_collections)
        return render_template("collections/list.html",
                               params = params,
                               collections = user_collections,
                               )
    else:
        anon_collections = user_manager.AnonUser().get_collections()
        #logger.debug("anon_collections are:", anon_collections)
        return render_template("collections/list.html",
                               params = params,
                               collections = anon_collections)

@app.route("/collections/remove", methods=('POST',))
def remove_traits():
    params = request.form

    if "uc_id" in params:
        uc_id = params['uc_id']
        traits_to_remove = params.getlist('traits[]')
        traits_to_remove = process_traits(traits_to_remove)
        logger.debug("\n\n  after processing, traits_to_remove:", traits_to_remove)

        members_now = g.user_session.remove_traits_from_collection(uc_id, traits_to_remove)
    else:
        collection_name = params['collection_name']
        members_now = AnonCollection(collection_name).remove_traits(params)

    # We need to return something so we'll return this...maybe in the future
    # we can use it to check the results
    return str(len(members_now))

@app.route("/collections/delete", methods=('POST',))
def delete_collection():
    params = request.form
    uc_id = ""
    if g.user_session.logged_in:
        uc_id = params['uc_id']
        if len(uc_id.split(":")) > 1:
            for this_uc_id in uc_id.split(":"):
                collection_name = g.user_session.delete_collection(this_uc_id)
        else:
            collection_name = g.user_session.delete_collection(uc_id)
    else:
        if "collection_name" in params:
            collection_name = params['collection_name']
            user_manager.AnonUser().delete_collection(collection_name)
        else:
            uc_id = params['uc_id']
            for this_collection in uc_id.split(":"):
                user_manager.AnonUser().delete_collection(this_collection)

    if uc_id != "":
        if len(uc_id.split(":")) > 1:
            flash("We've deleted the selected collections.", "alert-info")
        else:
            flash("We've deleted the collection: {}.".format(uc_id), "alert-info")
    else:
        flash("We've deleted the collection: {}.".format(collection_name), "alert-info")

    return redirect(url_for('list_collections'))

@app.route("/collections/view")
def view_collection():
    params = request.args

    if g.user_session.logged_in and "uc_id" in params:
        uc_id = params['uc_id']
        uc = (collection for collection in g.user_session.user_collections if collection["id"] == uc_id).next()
        traits = uc["members"]
    else:
        user_collections = json.loads(Redis.get(user_manager.AnonUser().key))
        this_collection = {}
        for collection in user_collections:
            if collection['id'] == params['collection_id']:
                this_collection = collection
                break

        traits = this_collection['members']

    trait_obs = []
    json_version = []

    for atrait in traits:
        name, dataset_name = atrait.split(':')
        if dataset_name == "Temp":
            group = name.split("_")[2]
            dataset = create_dataset(dataset_name, dataset_type = "Temp", group_name = group)
            trait_ob = trait.GeneralTrait(name=name, dataset=dataset)
        else:
            dataset = create_dataset(dataset_name)
            trait_ob = trait.GeneralTrait(name=name, dataset=dataset)
            trait_ob = trait.retrieve_trait_info(trait_ob, dataset, get_qtl_info=True)
        trait_obs.append(trait_ob)

        json_version.append(trait.jsonable(trait_ob))

    if "uc_id" in params:
        collection_info = dict(trait_obs=trait_obs,
                               uc = uc)
    else:
        collection_info = dict(trait_obs=trait_obs,
                               collection_name=this_collection['name'])
    if "json" in params:
        return json.dumps(json_version)
    else:
        return render_template("collections/view.html",
                           **collection_info
                           )<|endoftext|>


<commit_before>#!/usr/bin/python
# coding=utf-8

import requests
import certifi
import httplib
import time
from system.logging import Logger

from utils import Utils

'''
   Copyright 2017 Gareth Williams

   Licensed under the Apache License, Version 2.0 (the "License");
   you may not use this file except in compliance with the License.
   You may obtain a copy of the License at

       http://www.apache.org/licenses/LICENSE-2.0

   Unless required by applicable law or agreed to in writing, software
   distributed under the License is distributed on an "AS IS" BASIS,
   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
   See the License for the specific language governing permissions and
   limitations under the License.
'''

'''

   Incident Statuses

   0 = Scheduled
   1 = Investigating
   2 = Identified
   3 = Watching
   4 = Fixed

   Component Statuses

   1 = Operational
   2 = Performance Issues
   3 = Partial Outage
   4 = Major Outage

'''

class Cachet(object):

    httpErrors = {
        # Cloudflare Errors
        520: "Web server is returning an unknown error (Cloudflare)",
        521: "Web server is down (Cloudflare)",
        522: "Connection timed out (Cloudflare)",
        523: "Origin is unreachable (Cloudflare)",
        524: "A timeout occurred (Cloudflare)",
        525: "SSL handshake failed (Cloudflare)",
        526: "Invalid SSL certificate (Cloudflare)",
        # Nginx Errors
        444: "No Response (Nginx)",
        494: "Request Header Too Large (Nginx)",
        495: "Cert Error (Nginx)",
        496: "No Cert (Nginx)",
        497: "HTTP to HTTPS (Nginx)",
        499: "Client Closed Request (Nginx)",
        # Other
        # 1xx
        102: "Server has received and is processing the request",
        103: "resume aborted PUT or POST requests",
        122: "URI is longer than a maximum of 2083 characters",
        # 2xx
        207: "XML return with possible multiple seperate responses.",
        208: "The results are previously returned.",
        226: "The request has been fulfilled amd response is in instance manipulations.",
        # 3xx
        308: "Please connect again to the different URL using the same method.",
        # 4xx
        418: "I'm a teapot.",
        420: "Method Failure",
        421: "Enhance Your Calm",
        422: "Unprocessable Entity",
        423: "Locked",
        424: "Failed Dependency",
        426: "Upgrade Required",
        428: "Precondition Required",
        429: "Too Many Requests",
        431: "Request Header Fields Too Large",
        440: "Login Timeout (Microsoft)",
        449: "Retry With (Microsoft)",
        450: "Blocked by Windows Parental Controls",
        451: "Unavailable For Legal Reasons",
        # 5xx
        506: "Variant Also Negotiates (RFC 2295)",
        507: "Insufficient Storage (WebDAV; RFC 4918)",
        508: "Loop Detected (WebDAV; RFC 5842)",
        509: "Bandwidth Limit Exceeded (Apache bw/limited extension)",
        510: "Not Extended (RFC 2774)",
        511: "Network Authentication Required"

    }

    def __init__(self):
        self.logs = Logger()
        self.utils = Utils()
        self.config = self.utils.readConfig()
        self.base_url = self.config['api_url']
        self.api_token = self.config['api_token']
        self.maxRetries = self.config['retries']

        try:
            if self.utils.ping().status_code == 200 and self.utils.ping().json()['data'] == "Pong!":
                self.checkSites()
            else:
                exit(1)
        except Exception as e:
            self.logs.error(e)
            exit(1)

    def checkSites(self):
        # Count how many sites to monitor
        monitor_count = len(self.config['monitoring'])
        x = 0

        # Loop through sites to monitor
        while x < monitor_count:

            isEnabled = self.config['monitoring'][x]['enabled']
            status_codes = self.config['monitoring'][x]['expected_status_code']
            url = self.config['monitoring'][x]['url']
            request_method = self.config['monitoring'][x]['method']
            c_id = self.config['monitoring'][x]['component_id']
            localtime = time.asctime(time.localtime(time.time()))
            current_status = self.utils.getComponentsByID(c_id).json()['data']['status']
            load_time = 0
            incident_id = self.checkForIncident(c_id)

            try:
                if isEnabled:
                    if request_method.lower() == "get":
                        r = requests.get(url, verify=True)
                        if r.status_code not in status_codes and r.status_code not in self.httpErrors:
                            error_code = '%s check **failed** - %s \n\n`%s %s HTTP Error: %s`' % (url, localtime, request_method, url, httplib.responses[r.status_code])
                            if not incident_id:
                                self.utils.postIncidents('%s: HTTP Error' % url, error_code, 1, 1, component_id=c_id,
                                                         component_status=4)
                            if current_status is not 4:
                                self.utils.putComponentsByID(c_id, status=4)
                            self.logs.warn("%s" % error_code.strip('\n').strip('`'))
                        elif r.status_code not in status_codes and r.status_code in self.httpErrors:
                            error_code = '%s check **failed** - %s \n\n`%s %s HTTP Error: %s`' % (url, localtime, request_method, url, self.httpErrors[r.status_code])
                            if not incident_id:
                                self.utils.postIncidents('%s: HTTP Error' % url, error_code, 1, 1, component_id=c_id,
                                                         component_status=4)
                            if current_status is not 4:
                                self.utils.putComponentsByID(c_id, status=4)
                            self.logs.warn("%s" % error_code.replace('\n', '').replace('`', ''))
                    elif request_method.lower() == "post":
                        r = requests.get(url, verify=True)
                        if r.status_code not in status_codes and r.status_code not in self.httpErrors:
                            error_code = '%s check **failed** - %s \n\n`%s %s HTTP Error: %s`' % (url, localtime, request_method, url, httplib.responses[r.status_code])
                            if not incident_id:
                                self.utils.postIncidents('%s: HTTP Error' % url, error_code, 1, 1, component_id=c_id,
                                                         component_status=4)
                            if current_status is not 4:
                                self.utils.putComponentsByID(c_id, status=4)
                            self.logs.warn("%s" % error_code.strip('\n').strip('`'))
                        elif r.status_code not in status_codes and r.status_code in self.httpErrors:
                            error_code = '%s check **failed** - %s \n\n`%s %s HTTP Error: %s`' % (url, localtime, request_method, url, self.httpErrors[r.status_code])
                            if not incident_id:
                                self.utils.postIncidents('%s: HTTP Error' % url, error_code, 1, 1, component_id=c_id,
                                                         component_status=4)
                            if current_status is not 4:
                                self.utils.putComponentsByID(c_id, status=4)
                            self.logs.warn("%s" % error_code.replace('\n', '').replace('`', ''))
            except requests.exceptions.HTTPError as e:
                error_code = '%s check **failed** - %s \n\n`%s %s HTTP Error: %s`' % (url, localtime, request_method, url, e)
                if not incident_id:
                    self.utils.postIncidents('%s: HTTP Error' % url, error_code, 1, 1, component_id=c_id, component_status=4)
                if current_status is not 4:
                    self.utils.putComponentsByID(c_id, status=4)
                self.logs.warn(error_code.replace('\n', '').replace('`', ''))
            except requests.exceptions.SSLError as e:
                error_code = '%s check **failed** - %s \n\n`%s %s SSL Error: %s`' % (url, localtime, request_method, url, e)
                if not incident_id:
                    self.utils.postIncidents('%s: SSL Error' % url, error_code, 1, 1, component_id=c_id, component_status=4)
                if current_status is not 4:
                    self.utils.putComponentsByID(c_id, status=4)
                self.logs.warn(error_code.replace('\n', '').replace('`', ''))
            except requests.exceptions.ConnectionError as e:
                error_code = '%s check **failed** - %s \n\n`%s %s Connection Error: %s`' % (url, localtime, request_method, url, e)
                if not incident_id:
                    self.utils.postIncidents('%s: Connection Error' % url, error_code, 1, 1, component_id=c_id, component_status=4)
                if current_status is not 4:
                    self.utils.putComponentsByID(c_id, status=4)
                self.logs.warn(error_code.replace('\n', '').replace('`', ''))
            except requests.exceptions.Timeout as e:
                error_code = '%s check **failed** - %s \n\n`%s %s Request Timeout: %s`' % (url, localtime, request_method, url, e)
                if not incident_id:
                    self.utils.postIncidents('%s: Request Timeout' % url, error_code, 1, 1, component_id=c_id, component_status=2)
                if current_status is not 4:
                    self.utils.putComponentsByID(c_id, status=4)
                self.logs.warn(error_code.replace('\n', '').replace('`', ''))
            except requests.exceptions.TooManyRedirects as e:
                error_code = '%s check **failed** - %s \n\n`%s %s Too Many Redirects: %s`' % (url, localtime, request_method, url, e)
                if not incident_id:
                    self.utils.postIncidents('%s: Too Many Redirects' % url, error_code, 1, 1, component_id=c_id, component_status=4)
                if current_status is not 4:
                    self.utils.putComponentsByID(c_id, status=4)
                self.logs.warn(error_code.replace('\n', '').replace('`', ''))
            except requests.exceptions.RetryError as e:
                error_code = '%s check **failed** - %s \n\n`%s %s Retry Error: %s`' % (url, localtime, request_method, url, e)
                if not incident_id:
                    self.utils.postIncidents('%s: Retry Error' % url, error_code, 1, 1, component_id=c_id, component_status=4)
                if current_status is not 4:
                    self.utils.putComponentsByID(c_id, status=4)
                self.logs.warn(error_code.replace('\n', '').replace('`', ''))
            except Exception as e:
                error_code = '%s check **failed** - %s \n\n`%s %s Unexpected Error: %s`' % (url, localtime, request_method, url, e)
                if not incident_id:
                    self.utils.postIncidents('%s: Unexpected Error' % url, error_code, 1, 1, component_id=c_id, component_status=4)
                if current_status is not 4:
                    self.utils.putComponentsByID(c_id, status=4)
                self.logs.error(error_code.replace('\n', '').replace('`', ''))
            else:
                if r.status_code in status_codes:
                    if current_status is not 1 and not incident_id:
                        self.utils.putComponentsByID(c_id, status=1)
                        self.logs.info("Issue with %s has been resolved" % url)
                    elif current_status is not 1 and incident_id:
                        incident_description = "Resolved at %s\n\n***\n\n%s" % (localtime, self.getIncidentInfo(incident_id))
                        self.utils.putIncidentsByID(incident_id, message=incident_description, status=4, component_id=c_id, component_status=1)
                    else:
                        self.logs.info("%s no issues found" % url)

            x += 1
        self.logs.info("############################")

    def checkForIncident(self, component_id):
        current_incidents = self.utils.getIncidents().json()
        incidents = len(current_incidents['data'])
        x = 0

        while x < incidents:
            incident_id = current_incidents['data'][x]['id']
            incident_component_id = current_incidents['data'][x]['component_id']
            incident_status = current_incidents['data'][x]['status']

            if component_id == incident_component_id and incident_status is not 4:
                # self.logs.debug("c_id: %s i_id: %s" % (component_id, incident_id))
                return incident_id
            x += 1

    def getIncidentInfo(self, i_id):
        incident = self.utils.getIncidentsByID(i_id).json()
        i_description = incident['data']['message']
        return i_description

<commit_msg>added metrics<commit_after>#!/usr/bin/python
# coding=utf-8

import requests
import certifi
import httplib
import time
from system.logging import Logger

from utils import Utils

'''
   Copyright 2017 Gareth Williams

   Licensed under the Apache License, Version 2.0 (the "License");
   you may not use this file except in compliance with the License.
   You may obtain a copy of the License at

       http://www.apache.org/licenses/LICENSE-2.0

   Unless required by applicable law or agreed to in writing, software
   distributed under the License is distributed on an "AS IS" BASIS,
   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
   See the License for the specific language governing permissions and
   limitations under the License.
'''

'''

   Incident Statuses

   0 = Scheduled
   1 = Investigating
   2 = Identified
   3 = Watching
   4 = Fixed

   Component Statuses

   1 = Operational
   2 = Performance Issues
   3 = Partial Outage
   4 = Major Outage

'''

class Cachet(object):

    httpErrors = {
        # Cloudflare Errors
        520: "Web server is returning an unknown error (Cloudflare)",
        521: "Web server is down (Cloudflare)",
        522: "Connection timed out (Cloudflare)",
        523: "Origin is unreachable (Cloudflare)",
        524: "A timeout occurred (Cloudflare)",
        525: "SSL handshake failed (Cloudflare)",
        526: "Invalid SSL certificate (Cloudflare)",
        # Nginx Errors
        444: "No Response (Nginx)",
        494: "Request Header Too Large (Nginx)",
        495: "Cert Error (Nginx)",
        496: "No Cert (Nginx)",
        497: "HTTP to HTTPS (Nginx)",
        499: "Client Closed Request (Nginx)",
        # Other
        # 1xx
        102: "Server has received and is processing the request",
        103: "resume aborted PUT or POST requests",
        122: "URI is longer than a maximum of 2083 characters",
        # 2xx
        207: "XML return with possible multiple seperate responses.",
        208: "The results are previously returned.",
        226: "The request has been fulfilled amd response is in instance manipulations.",
        # 3xx
        308: "Please connect again to the different URL using the same method.",
        # 4xx
        418: "I'm a teapot.",
        420: "Method Failure",
        421: "Enhance Your Calm",
        422: "Unprocessable Entity",
        423: "Locked",
        424: "Failed Dependency",
        426: "Upgrade Required",
        428: "Precondition Required",
        429: "Too Many Requests",
        431: "Request Header Fields Too Large",
        440: "Login Timeout (Microsoft)",
        449: "Retry With (Microsoft)",
        450: "Blocked by Windows Parental Controls",
        451: "Unavailable For Legal Reasons",
        # 5xx
        506: "Variant Also Negotiates (RFC 2295)",
        507: "Insufficient Storage (WebDAV; RFC 4918)",
        508: "Loop Detected (WebDAV; RFC 5842)",
        509: "Bandwidth Limit Exceeded (Apache bw/limited extension)",
        510: "Not Extended (RFC 2774)",
        511: "Network Authentication Required"

    }

    def __init__(self):
        self.logs = Logger()
        self.utils = Utils()
        self.config = self.utils.readConfig()
        self.base_url = self.config['api_url']
        self.api_token = self.config['api_token']
        self.maxRetries = self.config['retries']

        try:
            if self.utils.ping().status_code == 200 and self.utils.ping().json()['data'] == "Pong!":
                self.checkSites()
            else:
                exit(1)
        except Exception as e:
            self.logs.error(e)
            exit(1)

    def checkSites(self):
        # Count how many sites to monitor
        monitor_count = len(self.config['monitoring'])
        x = 0

        # Loop through sites to monitor
        while x < monitor_count:

            isEnabled = self.config['monitoring'][x]['enabled']
            status_codes = self.config['monitoring'][x]['expected_status_code']
            url = self.config['monitoring'][x]['url']
            request_method = self.config['monitoring'][x]['method']
            c_id = self.config['monitoring'][x]['component_id']
            localtime = time.asctime(time.localtime(time.time()))
            current_status = self.utils.getComponentsByID(c_id).json()['data']['status']
            load_time = 0
            incident_id = self.checkForIncident(c_id)

            try:
                if isEnabled:
                    if request_method.lower() == "get":
                        r = requests.get(url, verify=True)
                        self.utils.postMetricsPointsByID(1, r.elapsed.microseconds)
                        if r.status_code not in status_codes and r.status_code not in self.httpErrors:
                            error_code = '%s check **failed** - %s \n\n`%s %s HTTP Error: %s`' % (url, localtime, request_method, url, httplib.responses[r.status_code])
                            if not incident_id:
                                self.utils.postIncidents('%s: HTTP Error' % url, error_code, 1, 1, component_id=c_id,
                                                         component_status=4)
                            if current_status is not 4:
                                self.utils.putComponentsByID(c_id, status=4)
                            self.logs.warn("%s" % error_code.strip('\n').strip('`'))
                        elif r.status_code not in status_codes and r.status_code in self.httpErrors:
                            error_code = '%s check **failed** - %s \n\n`%s %s HTTP Error: %s`' % (url, localtime, request_method, url, self.httpErrors[r.status_code])
                            if not incident_id:
                                self.utils.postIncidents('%s: HTTP Error' % url, error_code, 1, 1, component_id=c_id,
                                                         component_status=4)
                            if current_status is not 4:
                                self.utils.putComponentsByID(c_id, status=4)
                            self.logs.warn("%s" % error_code.replace('\n', '').replace('`', ''))
                    elif request_method.lower() == "post":
                        r = requests.get(url, verify=True)
                        if r.status_code not in status_codes and r.status_code not in self.httpErrors:
                            error_code = '%s check **failed** - %s \n\n`%s %s HTTP Error: %s`' % (url, localtime, request_method, url, httplib.responses[r.status_code])
                            if not incident_id:
                                self.utils.postIncidents('%s: HTTP Error' % url, error_code, 1, 1, component_id=c_id,
                                                         component_status=4)
                            if current_status is not 4:
                                self.utils.putComponentsByID(c_id, status=4)
                            self.logs.warn("%s" % error_code.strip('\n').strip('`'))
                        elif r.status_code not in status_codes and r.status_code in self.httpErrors:
                            error_code = '%s check **failed** - %s \n\n`%s %s HTTP Error: %s`' % (url, localtime, request_method, url, self.httpErrors[r.status_code])
                            if not incident_id:
                                self.utils.postIncidents('%s: HTTP Error' % url, error_code, 1, 1, component_id=c_id,
                                                         component_status=4)
                            if current_status is not 4:
                                self.utils.putComponentsByID(c_id, status=4)
                            self.logs.warn("%s" % error_code.replace('\n', '').replace('`', ''))
            except requests.exceptions.HTTPError as e:
                error_code = '%s check **failed** - %s \n\n`%s %s HTTP Error: %s`' % (url, localtime, request_method, url, e)
                if not incident_id:
                    self.utils.postIncidents('%s: HTTP Error' % url, error_code, 1, 1, component_id=c_id, component_status=4)
                if current_status is not 4:
                    self.utils.putComponentsByID(c_id, status=4)
                self.logs.warn(error_code.replace('\n', '').replace('`', ''))
            except requests.exceptions.SSLError as e:
                error_code = '%s check **failed** - %s \n\n`%s %s SSL Error: %s`' % (url, localtime, request_method, url, e)
                if not incident_id:
                    self.utils.postIncidents('%s: SSL Error' % url, error_code, 1, 1, component_id=c_id, component_status=4)
                if current_status is not 4:
                    self.utils.putComponentsByID(c_id, status=4)
                self.logs.warn(error_code.replace('\n', '').replace('`', ''))
            except requests.exceptions.ConnectionError as e:
                error_code = '%s check **failed** - %s \n\n`%s %s Connection Error: %s`' % (url, localtime, request_method, url, e)
                if not incident_id:
                    self.utils.postIncidents('%s: Connection Error' % url, error_code, 1, 1, component_id=c_id, component_status=4)
                if current_status is not 4:
                    self.utils.putComponentsByID(c_id, status=4)
                self.logs.warn(error_code.replace('\n', '').replace('`', ''))
            except requests.exceptions.Timeout as e:
                error_code = '%s check **failed** - %s \n\n`%s %s Request Timeout: %s`' % (url, localtime, request_method, url, e)
                if not incident_id:
                    self.utils.postIncidents('%s: Request Timeout' % url, error_code, 1, 1, component_id=c_id, component_status=2)
                if current_status is not 4:
                    self.utils.putComponentsByID(c_id, status=4)
                self.logs.warn(error_code.replace('\n', '').replace('`', ''))
            except requests.exceptions.TooManyRedirects as e:
                error_code = '%s check **failed** - %s \n\n`%s %s Too Many Redirects: %s`' % (url, localtime, request_method, url, e)
                if not incident_id:
                    self.utils.postIncidents('%s: Too Many Redirects' % url, error_code, 1, 1, component_id=c_id, component_status=4)
                if current_status is not 4:
                    self.utils.putComponentsByID(c_id, status=4)
                self.logs.warn(error_code.replace('\n', '').replace('`', ''))
            except requests.exceptions.RetryError as e:
                error_code = '%s check **failed** - %s \n\n`%s %s Retry Error: %s`' % (url, localtime, request_method, url, e)
                if not incident_id:
                    self.utils.postIncidents('%s: Retry Error' % url, error_code, 1, 1, component_id=c_id, component_status=4)
                if current_status is not 4:
                    self.utils.putComponentsByID(c_id, status=4)
                self.logs.warn(error_code.replace('\n', '').replace('`', ''))
            except Exception as e:
                error_code = '%s check **failed** - %s \n\n`%s %s Unexpected Error: %s`' % (url, localtime, request_method, url, e)
                if not incident_id:
                    self.utils.postIncidents('%s: Unexpected Error' % url, error_code, 1, 1, component_id=c_id, component_status=4)
                if current_status is not 4:
                    self.utils.putComponentsByID(c_id, status=4)
                self.logs.error(error_code.replace('\n', '').replace('`', ''))
            else:
                if r.status_code in status_codes:
                    if current_status is not 1 and not incident_id:
                        self.utils.putComponentsByID(c_id, status=1)
                        self.logs.info("Issue with %s has been resolved" % url)
                    elif current_status is not 1 and incident_id:
                        incident_description = "Resolved at %s\n\n***\n\n%s" % (localtime, self.getIncidentInfo(incident_id))
                        self.utils.putIncidentsByID(incident_id, message=incident_description, status=4, component_id=c_id, component_status=1)
                    else:
                        self.logs.info("%s no issues found" % url)

            x += 1
        self.logs.info("############################")

    def checkForIncident(self, component_id):
        current_incidents = self.utils.getIncidents().json()
        incidents = len(current_incidents['data'])
        x = 0

        while x < incidents:
            incident_id = current_incidents['data'][x]['id']
            incident_component_id = current_incidents['data'][x]['component_id']
            incident_status = current_incidents['data'][x]['status']

            if component_id == incident_component_id and incident_status is not 4:
                # self.logs.debug("c_id: %s i_id: %s" % (component_id, incident_id))
                return incident_id
            x += 1

    def getIncidentInfo(self, i_id):
        incident = self.utils.getIncidentsByID(i_id).json()
        i_description = incident['data']['message']
        return i_description

<|endoftext|>


<commit_before>import json
from types import SimpleNamespace

from pymonad.maybe import Just, Maybe, Nothing
from pymonad.tools import curry
import xapian

from base import webqtlConfig
from utility.authentication_tools import check_resource_availability
from utility.monads import MonadicDict
from wqflask.database import xapian_database

def is_permitted_for_listing(trait, search_type):
    """Check if it is permissible to list trait in search results."""
    dataset_type = {"gene": "ProbeSet", "phenotype": "Publish"}
    dataset_ob = (Maybe.apply(curry(2, lambda id, species:
                                    SimpleNamespace(id=id,
                                                    type=dataset_type[search_type],
                                                    name=trait["dataset"],
                                                    species=species)))
                  .to_arguments(trait["dataset_id"], trait["species"]))
    return (Maybe.apply(curry(2, check_resource_availability))
            .to_arguments(dataset_ob, trait["name"])
            .map(lambda permissions:
                 ((isinstance(permissions["data"], list)) and ("view" in permissions["data"]))
                 or (permissions["data"] != 'no-access'))
            .maybe(False, lambda x: x))

class GSearch:
    def __init__(self, kwargs):
        if ("type" not in kwargs) or ("terms" not in kwargs):
            raise ValueError
        self.type = kwargs["type"]
        self.terms = kwargs["terms"]

        queryparser = xapian.QueryParser()
        queryparser.set_stemmer(xapian.Stem("en"))
        queryparser.set_stemming_strategy(queryparser.STEM_SOME)
        queryparser.add_prefix("author", "A")
        queryparser.add_prefix("species", "XS")
        queryparser.add_prefix("group", "XG")
        queryparser.add_prefix("tissue", "XI")
        queryparser.add_prefix("description", "XD")
        queryparser.add_prefix("dataset", "XDS")
        queryparser.add_prefix("symbol", "XY")
        queryparser.add_prefix("chr", "XC")
        queryparser.add_prefix("peakchr", "XPC")
        for i, prefix in enumerate(["mean:", "peak:", "mb:", "peakmb:", "additive:", "year:"]):
            queryparser.add_rangeprocessor(xapian.NumberRangeProcessor(i, prefix))
        querystring = self.terms
        query = queryparser.parse_query(querystring)
        # FIXME: Handle presentation (that is, formatting strings for
        # display) in the template rendering, not when retrieving
        # search results.
        chr_mb = curry(2, lambda chr, mb: f"Chr{chr}: {mb:.6f}")
        format3f = lambda x: f"{x:.3f}"
        hmac = curry(2, lambda dataset, dataset_fullname: f"{dataset_fullname}:{dataset}")
        self.trait_list = []
        # pylint: disable=invalid-name
        with xapian_database() as db:
            enquire = xapian.Enquire(db)
            # Filter documents by type.
            enquire.set_query(xapian.Query(xapian.Query.OP_FILTER,
                                           query,
                                           xapian.Query(f"XT{self.type}")))
            for i, trait in enumerate(
                    [trait for xapian_match in enquire.get_mset(0, db.get_doccount())
                     if is_permitted_for_listing(
                             trait := MonadicDict(json.loads(xapian_match.document.get_data())),
                             search_type=self.type)]):
                trait["index"] = Just(i)
                trait["location_repr"] = (Maybe.apply(chr_mb)
                                          .to_arguments(trait.pop("chr"), trait.pop("mb")))
                trait["LRS_score_repr"] = trait.pop("lrs").map(format3f)
                trait["additive"] = trait["additive"].map(format3f)
                trait["mean"] = trait["mean"].map(format3f)
                trait["max_lrs_text"] = (Maybe.apply(chr_mb)
                                         .to_arguments(trait.pop("geno_chr"), trait.pop("geno_mb")))
                if self.type == "gene":
                    trait["hmac"] = (Maybe.apply(hmac)
                                     .to_arguments(trait["dataset"], trait["dataset_fullname"]))
                elif self.type == "phenotype":
                    trait["display_name"] = trait["name"]
                    inbredsetcode = trait.pop("inbredsetcode")
                    if inbredsetcode.map(len) == Just(3):
                        trait["display_name"] = (Maybe.apply(lambda inbredsetcode, name:
                                                             f"{inbredsetcode}_{name}")
                                                 .to_arguments(inbredsetcode, trait["name"]))
                    trait["hmac"] = (Maybe.apply(hmac)
                                     .to_arguments(trait.pop("dataset_fullname"), trait["name"]))
                    trait["authors_display"] = (trait.pop("authors").map(
                        lambda authors:
                        ", ".join(authors[:2] + ["et al."] if len(authors) >=2 else authors)))
                    trait["pubmed_text"] = trait["year"].map(str)
                    trait["pubmed_link"] = (trait["pubmed_id"].map(
                        lambda pubmedid: webqtlConfig.PUBMEDLINK_URL % pubmedid))
                self.trait_list.append(trait.data)
            self.trait_count = len(self.trait_list)
<commit_msg>Hard-limit the number of search results.<commit_after>import json
from types import SimpleNamespace

from pymonad.maybe import Just, Maybe, Nothing
from pymonad.tools import curry
import xapian

from base import webqtlConfig
from utility.authentication_tools import check_resource_availability
from utility.monads import MonadicDict
from wqflask.database import xapian_database

# KLUDGE: Due to the lack of pagination, we hard-limit the maximum
# number of search results.
MAX_SEARCH_RESULTS = 1000

def is_permitted_for_listing(trait, search_type):
    """Check if it is permissible to list trait in search results."""
    dataset_type = {"gene": "ProbeSet", "phenotype": "Publish"}
    dataset_ob = (Maybe.apply(curry(2, lambda id, species:
                                    SimpleNamespace(id=id,
                                                    type=dataset_type[search_type],
                                                    name=trait["dataset"],
                                                    species=species)))
                  .to_arguments(trait["dataset_id"], trait["species"]))
    return (Maybe.apply(curry(2, check_resource_availability))
            .to_arguments(dataset_ob, trait["name"])
            .map(lambda permissions:
                 ((isinstance(permissions["data"], list)) and ("view" in permissions["data"]))
                 or (permissions["data"] != 'no-access'))
            .maybe(False, lambda x: x))

class GSearch:
    def __init__(self, kwargs):
        if ("type" not in kwargs) or ("terms" not in kwargs):
            raise ValueError
        self.type = kwargs["type"]
        self.terms = kwargs["terms"]

        queryparser = xapian.QueryParser()
        queryparser.set_stemmer(xapian.Stem("en"))
        queryparser.set_stemming_strategy(queryparser.STEM_SOME)
        queryparser.add_prefix("author", "A")
        queryparser.add_prefix("species", "XS")
        queryparser.add_prefix("group", "XG")
        queryparser.add_prefix("tissue", "XI")
        queryparser.add_prefix("description", "XD")
        queryparser.add_prefix("dataset", "XDS")
        queryparser.add_prefix("symbol", "XY")
        queryparser.add_prefix("chr", "XC")
        queryparser.add_prefix("peakchr", "XPC")
        for i, prefix in enumerate(["mean:", "peak:", "mb:", "peakmb:", "additive:", "year:"]):
            queryparser.add_rangeprocessor(xapian.NumberRangeProcessor(i, prefix))
        querystring = self.terms
        query = queryparser.parse_query(querystring)
        # FIXME: Handle presentation (that is, formatting strings for
        # display) in the template rendering, not when retrieving
        # search results.
        chr_mb = curry(2, lambda chr, mb: f"Chr{chr}: {mb:.6f}")
        format3f = lambda x: f"{x:.3f}"
        hmac = curry(2, lambda dataset, dataset_fullname: f"{dataset_fullname}:{dataset}")
        self.trait_list = []
        # pylint: disable=invalid-name
        with xapian_database() as db:
            enquire = xapian.Enquire(db)
            # Filter documents by type.
            enquire.set_query(xapian.Query(xapian.Query.OP_FILTER,
                                           query,
                                           xapian.Query(f"XT{self.type}")))
            for i, trait in enumerate(
                    [trait for xapian_match in enquire.get_mset(0, MAX_SEARCH_RESULTS)
                     if is_permitted_for_listing(
                             trait := MonadicDict(json.loads(xapian_match.document.get_data())),
                             search_type=self.type)]):
                trait["index"] = Just(i)
                trait["location_repr"] = (Maybe.apply(chr_mb)
                                          .to_arguments(trait.pop("chr"), trait.pop("mb")))
                trait["LRS_score_repr"] = trait.pop("lrs").map(format3f)
                trait["additive"] = trait["additive"].map(format3f)
                trait["mean"] = trait["mean"].map(format3f)
                trait["max_lrs_text"] = (Maybe.apply(chr_mb)
                                         .to_arguments(trait.pop("geno_chr"), trait.pop("geno_mb")))
                if self.type == "gene":
                    trait["hmac"] = (Maybe.apply(hmac)
                                     .to_arguments(trait["dataset"], trait["dataset_fullname"]))
                elif self.type == "phenotype":
                    trait["display_name"] = trait["name"]
                    inbredsetcode = trait.pop("inbredsetcode")
                    if inbredsetcode.map(len) == Just(3):
                        trait["display_name"] = (Maybe.apply(lambda inbredsetcode, name:
                                                             f"{inbredsetcode}_{name}")
                                                 .to_arguments(inbredsetcode, trait["name"]))
                    trait["hmac"] = (Maybe.apply(hmac)
                                     .to_arguments(trait.pop("dataset_fullname"), trait["name"]))
                    trait["authors_display"] = (trait.pop("authors").map(
                        lambda authors:
                        ", ".join(authors[:2] + ["et al."] if len(authors) >=2 else authors)))
                    trait["pubmed_text"] = trait["year"].map(str)
                    trait["pubmed_link"] = (trait["pubmed_id"].map(
                        lambda pubmedid: webqtlConfig.PUBMEDLINK_URL % pubmedid))
                self.trait_list.append(trait.data)
            self.trait_count = len(self.trait_list)
<|endoftext|>


<commit_before>import json
from types import SimpleNamespace

from pymonad.maybe import Just, Maybe, Nothing
from pymonad.tools import curry
import xapian

from base import webqtlConfig
from utility.authentication_tools import check_resource_availability
from utility.monads import MonadicDict
from wqflask.database import xapian_database

def is_permitted_for_listing(trait, search_type):
    """Check if it is permissible to list trait in search results."""
    dataset_type = {"gene": "ProbeSet", "phenotype": "Publish"}
    dataset_ob = (Maybe.apply(curry(2, lambda id, species:
                                    SimpleNamespace(id=id,
                                                    type=dataset_type[search_type],
                                                    name=trait["dataset"],
                                                    species=species)))
                  .to_arguments(trait["dataset_id"], trait["species"]))
    return (Maybe.apply(curry(2, check_resource_availability))
            .to_arguments(dataset_ob, trait["name"])
            .map(lambda permissions:
                 ((isinstance(permissions["data"], list)) and ("view" in permissions["data"]))
                 or (permissions["data"] != 'no-access'))
            .maybe(False, lambda x: x))

class GSearch:
    def __init__(self, kwargs):
        if ("type" not in kwargs) or ("terms" not in kwargs):
            raise ValueError
        self.type = kwargs["type"]
        self.terms = kwargs["terms"]

        queryparser = xapian.QueryParser()
        queryparser.set_stemmer(xapian.Stem("en"))
        queryparser.set_stemming_strategy(queryparser.STEM_SOME)
        querystring = self.terms
        query = queryparser.parse_query(querystring)
        # FIXME: Handle presentation (that is, formatting strings for
        # display) in the template rendering, not when retrieving
        # search results.
        chr_mb = curry(2, lambda chr, mb: f"Chr{chr}: {mb:.6f}")
        format3f = lambda x: f"{x:.3f}"
        hmac = curry(2, lambda dataset, dataset_fullname: f"{dataset_fullname}:{dataset}")
        self.trait_list = []
        # pylint: disable=invalid-name
        with xapian_database() as db:
            enquire = xapian.Enquire(db)
            # Filter documents by type.
            enquire.set_query(xapian.Query(xapian.Query.OP_FILTER,
                                           query,
                                           xapian.Query(f"XT{self.type}")))
            for i, trait in enumerate(
                    [trait for xapian_match in enquire.get_mset(0, db.get_doccount())
                     if is_permitted_for_listing(
                             trait := MonadicDict(json.loads(xapian_match.document.get_data())),
                             search_type=self.type)]):
                trait["index"] = Just(i)
                trait["location_repr"] = (Maybe.apply(chr_mb)
                                          .to_arguments(trait.pop("chr"), trait.pop("mb")))
                trait["LRS_score_repr"] = trait.pop("lrs").map(format3f)
                trait["additive"] = trait["additive"].map(format3f)
                trait["mean"] = trait["mean"].map(format3f)
                trait["max_lrs_text"] = (Maybe.apply(chr_mb)
                                         .to_arguments(trait.pop("geno_chr"), trait.pop("geno_mb")))
                if self.type == "gene":
                    trait["hmac"] = (Maybe.apply(hmac)
                                     .to_arguments(trait["dataset"], trait["dataset_fullname"]))
                elif self.type == "phenotype":
                    trait["display_name"] = trait["name"]
                    inbredsetcode = trait.pop("inbredsetcode")
                    if inbredsetcode.map(len) == Just(3):
                        trait["display_name"] = (Maybe.apply(lambda inbredsetcode, name:
                                                             f"{inbredsetcode}_{name}")
                                                 .to_arguments(inbredsetcode, trait["name"]))
                    trait["hmac"] = (Maybe.apply(hmac)
                                     .to_arguments(trait.pop("dataset_fullname"), trait["name"]))
                    trait["authors_display"] = (trait.pop("authors").map(
                        lambda authors:
                        ", ".join(authors[:2] + ["et al."] if len(authors) >=2 else authors)))
                    trait["pubmed_text"] = trait["year"].map(str)
                    trait["pubmed_link"] = (trait["pubmed_id"].map(
                        lambda pubmedid: webqtlConfig.PUBMEDLINK_URL % pubmedid))
                self.trait_list.append(trait.data)
            self.trait_count = len(self.trait_list)
<commit_msg>Support prefix search and range queries.<commit_after>import json
from types import SimpleNamespace

from pymonad.maybe import Just, Maybe, Nothing
from pymonad.tools import curry
import xapian

from base import webqtlConfig
from utility.authentication_tools import check_resource_availability
from utility.monads import MonadicDict
from wqflask.database import xapian_database

def is_permitted_for_listing(trait, search_type):
    """Check if it is permissible to list trait in search results."""
    dataset_type = {"gene": "ProbeSet", "phenotype": "Publish"}
    dataset_ob = (Maybe.apply(curry(2, lambda id, species:
                                    SimpleNamespace(id=id,
                                                    type=dataset_type[search_type],
                                                    name=trait["dataset"],
                                                    species=species)))
                  .to_arguments(trait["dataset_id"], trait["species"]))
    return (Maybe.apply(curry(2, check_resource_availability))
            .to_arguments(dataset_ob, trait["name"])
            .map(lambda permissions:
                 ((isinstance(permissions["data"], list)) and ("view" in permissions["data"]))
                 or (permissions["data"] != 'no-access'))
            .maybe(False, lambda x: x))

class GSearch:
    def __init__(self, kwargs):
        if ("type" not in kwargs) or ("terms" not in kwargs):
            raise ValueError
        self.type = kwargs["type"]
        self.terms = kwargs["terms"]

        queryparser = xapian.QueryParser()
        queryparser.set_stemmer(xapian.Stem("en"))
        queryparser.set_stemming_strategy(queryparser.STEM_SOME)
        queryparser.add_prefix("author", "A")
        queryparser.add_prefix("species", "XS")
        queryparser.add_prefix("group", "XG")
        queryparser.add_prefix("tissue", "XI")
        queryparser.add_prefix("description", "XD")
        queryparser.add_prefix("dataset", "XDS")
        queryparser.add_prefix("symbol", "XY")
        queryparser.add_prefix("chr", "XC")
        queryparser.add_prefix("peakchr", "XPC")
        for i, prefix in enumerate(["mean:", "peak:", "mb:", "peakmb:", "additive:", "year:"]):
            queryparser.add_rangeprocessor(xapian.NumberRangeProcessor(i, prefix))
        querystring = self.terms
        query = queryparser.parse_query(querystring)
        # FIXME: Handle presentation (that is, formatting strings for
        # display) in the template rendering, not when retrieving
        # search results.
        chr_mb = curry(2, lambda chr, mb: f"Chr{chr}: {mb:.6f}")
        format3f = lambda x: f"{x:.3f}"
        hmac = curry(2, lambda dataset, dataset_fullname: f"{dataset_fullname}:{dataset}")
        self.trait_list = []
        # pylint: disable=invalid-name
        with xapian_database() as db:
            enquire = xapian.Enquire(db)
            # Filter documents by type.
            enquire.set_query(xapian.Query(xapian.Query.OP_FILTER,
                                           query,
                                           xapian.Query(f"XT{self.type}")))
            for i, trait in enumerate(
                    [trait for xapian_match in enquire.get_mset(0, db.get_doccount())
                     if is_permitted_for_listing(
                             trait := MonadicDict(json.loads(xapian_match.document.get_data())),
                             search_type=self.type)]):
                trait["index"] = Just(i)
                trait["location_repr"] = (Maybe.apply(chr_mb)
                                          .to_arguments(trait.pop("chr"), trait.pop("mb")))
                trait["LRS_score_repr"] = trait.pop("lrs").map(format3f)
                trait["additive"] = trait["additive"].map(format3f)
                trait["mean"] = trait["mean"].map(format3f)
                trait["max_lrs_text"] = (Maybe.apply(chr_mb)
                                         .to_arguments(trait.pop("geno_chr"), trait.pop("geno_mb")))
                if self.type == "gene":
                    trait["hmac"] = (Maybe.apply(hmac)
                                     .to_arguments(trait["dataset"], trait["dataset_fullname"]))
                elif self.type == "phenotype":
                    trait["display_name"] = trait["name"]
                    inbredsetcode = trait.pop("inbredsetcode")
                    if inbredsetcode.map(len) == Just(3):
                        trait["display_name"] = (Maybe.apply(lambda inbredsetcode, name:
                                                             f"{inbredsetcode}_{name}")
                                                 .to_arguments(inbredsetcode, trait["name"]))
                    trait["hmac"] = (Maybe.apply(hmac)
                                     .to_arguments(trait.pop("dataset_fullname"), trait["name"]))
                    trait["authors_display"] = (trait.pop("authors").map(
                        lambda authors:
                        ", ".join(authors[:2] + ["et al."] if len(authors) >=2 else authors)))
                    trait["pubmed_text"] = trait["year"].map(str)
                    trait["pubmed_link"] = (trait["pubmed_id"].map(
                        lambda pubmedid: webqtlConfig.PUBMEDLINK_URL % pubmedid))
                self.trait_list.append(trait.data)
            self.trait_count = len(self.trait_list)
<|endoftext|>


<commit_before>from pprint import pprint

from django.shortcuts import render
from django.urls import reverse
from django.views import View

from fo2.connections import db_cursor_so

import comercial.forms as forms
import comercial.queries as queries

class TabelaDePreco(View):
    Form_class = forms.TabelaDePrecoForm
    template_name = 'comercial/tabela_de_preco.html'
    title_name = 'Tabela de preços'

    def mount_context(self, cursor, tabela):
        context = {'tabela': tabela}
        codigo_tabela_chunks = tabela.split('.')
        if len(codigo_tabela_chunks) != 3:
            context.update({
                'erro': 'Código inválido. '
                        '3 números inteiros separados por ".".'
            })

            data = queries.get_tabela_preco(cursor)
            if len(data) == 0:
                context.update({'erro': 'Sem tabelas de preço'})
                return context

            for row in data:
                row['tabela'] = "{:02d}.{:02d}.{:02d}".format(
                    row['col_tabela_preco'],
                    row['mes_tabela_preco'],
                    row['seq_tabela_preco'],
                )
                row['tabela|LINK'] = reverse(
                    'comercial:tabela_de_preco__get',
                    args=[row['tabela']]
                )
                row['data_ini_tabela'] = row['data_ini_tabela'].date()
                row['data_fim_tabela'] = row['data_fim_tabela'].date()

            context.update({
                'headers': [
                    'Tabela',
                    'Descrição',
                    'Início',
                    'Fim',
                ],
                'fields': [
                    'tabela',
                    'descricao',
                    'data_ini_tabela',
                    'data_fim_tabela',
                ],
                'data': data,
            })
            return context

        for subcodigo_tabela in codigo_tabela_chunks:
            if not subcodigo_tabela.isdigit():
                context.update({
                    'erro': 'Cada subcódigo deve ser um número inteiro.'
                })
                return context
        codigo_tabela_ints = list(map(int, codigo_tabela_chunks))
        tabela = "{:02d}.{:02d}.{:02d}".format(*codigo_tabela_ints)
        context = {'tabela': tabela}

        data = queries.get_tabela_preco(cursor, *codigo_tabela_ints)
        if len(data) == 0:
            context.update({'erro': 'Tabela não encontrada'})
            return context

        for row in data:
            row['data_ini_tabela'] = row['data_ini_tabela'].date()
            row['data_fim_tabela'] = row['data_fim_tabela'].date()

        context.update({
            'headers': [
                'Descrição',
                'Início',
                'Fim',
            ],
            'fields': [
                'descricao',
                'data_ini_tabela',
                'data_fim_tabela',
            ],
            'data': data,
        })

        i_data = queries.itens_tabela_preco(cursor, *codigo_tabela_ints)
        if len(i_data) == 0:
            context.update({'erro': 'Tabela vazia'})
            return context

        context.update({
            'i_headers': [
                'Referência',
                'Descrição',
                'Valor',
            ],
            'i_fields': [
                'grupo_estrutura',
                'descr_referencia',
                'val_tabela_preco'
            ],
            'i_data': i_data,
        })

        return context

    def get(self, request, *args, **kwargs):
        if 'tabela' in kwargs:
            return self.post(request, *args, **kwargs)
        else:
            context = {'titulo': self.title_name}
            form = self.Form_class()
            context['form'] = form
            return render(request, self.template_name, context)

    def post(self, request, *args, **kwargs):
        context = {'titulo': self.title_name}
        form = self.Form_class(request.POST)
        form.data = form.data.copy()
        if 'tabela' in kwargs:
            form.data['tabela'] = kwargs['tabela']
        if form.is_valid():
            tabela = form.cleaned_data['tabela']
            cursor = db_cursor_so(request)
            context.update(self.mount_context(cursor, tabela))
        context['form'] = form
        return render(request, self.template_name, context)
<commit_msg>view TabelaDePreco list de tabelas ordenada<commit_after>from pprint import pprint

from django.shortcuts import render
from django.urls import reverse
from django.views import View

from fo2.connections import db_cursor_so

import comercial.forms as forms
import comercial.queries as queries

class TabelaDePreco(View):
    Form_class = forms.TabelaDePrecoForm
    template_name = 'comercial/tabela_de_preco.html'
    title_name = 'Tabela de preços'

    def mount_context(self, cursor, tabela):
        context = {'tabela': tabela}
        codigo_tabela_chunks = tabela.split('.')
        if len(codigo_tabela_chunks) != 3:
            context.update({
                'erro': 'Código inválido. '
                        '3 números inteiros separados por ".".'
            })

            data = queries.get_tabela_preco(cursor, order='a')
            if len(data) == 0:
                context.update({'erro': 'Sem tabelas de preço'})
                return context

            for row in data:
                row['tabela'] = "{:02d}.{:02d}.{:02d}".format(
                    row['col_tabela_preco'],
                    row['mes_tabela_preco'],
                    row['seq_tabela_preco'],
                )
                row['tabela|LINK'] = reverse(
                    'comercial:tabela_de_preco__get',
                    args=[row['tabela']]
                )
                row['data_ini_tabela'] = row['data_ini_tabela'].date()
                row['data_fim_tabela'] = row['data_fim_tabela'].date()

            context.update({
                'headers': [
                    'Tabela',
                    'Descrição',
                    'Início',
                    'Fim',
                ],
                'fields': [
                    'tabela',
                    'descricao',
                    'data_ini_tabela',
                    'data_fim_tabela',
                ],
                'data': data,
            })
            return context

        for subcodigo_tabela in codigo_tabela_chunks:
            if not subcodigo_tabela.isdigit():
                context.update({
                    'erro': 'Cada subcódigo deve ser um número inteiro.'
                })
                return context
        codigo_tabela_ints = list(map(int, codigo_tabela_chunks))
        tabela = "{:02d}.{:02d}.{:02d}".format(*codigo_tabela_ints)
        context = {'tabela': tabela}

        data = queries.get_tabela_preco(cursor, *codigo_tabela_ints)
        if len(data) == 0:
            context.update({'erro': 'Tabela não encontrada'})
            return context

        for row in data:
            row['data_ini_tabela'] = row['data_ini_tabela'].date()
            row['data_fim_tabela'] = row['data_fim_tabela'].date()

        context.update({
            'headers': [
                'Descrição',
                'Início',
                'Fim',
            ],
            'fields': [
                'descricao',
                'data_ini_tabela',
                'data_fim_tabela',
            ],
            'data': data,
        })

        i_data = queries.itens_tabela_preco(cursor, *codigo_tabela_ints)
        if len(i_data) == 0:
            context.update({'erro': 'Tabela vazia'})
            return context

        context.update({
            'i_headers': [
                'Referência',
                'Descrição',
                'Valor',
            ],
            'i_fields': [
                'grupo_estrutura',
                'descr_referencia',
                'val_tabela_preco'
            ],
            'i_data': i_data,
        })

        return context

    def get(self, request, *args, **kwargs):
        if 'tabela' in kwargs:
            return self.post(request, *args, **kwargs)
        else:
            context = {'titulo': self.title_name}
            form = self.Form_class()
            context['form'] = form
            return render(request, self.template_name, context)

    def post(self, request, *args, **kwargs):
        context = {'titulo': self.title_name}
        form = self.Form_class(request.POST)
        form.data = form.data.copy()
        if 'tabela' in kwargs:
            form.data['tabela'] = kwargs['tabela']
        if form.is_valid():
            tabela = form.cleaned_data['tabela']
            cursor = db_cursor_so(request)
            context.update(self.mount_context(cursor, tabela))
        context['form'] = form
        return render(request, self.template_name, context)
<|endoftext|>