<commit_before>## Copyright (C) University of Tennessee Health Science Center, Memphis, TN.
#
# This program is free software: you can redistribute it and/or modify it
# under the terms of the GNU Affero General Public License
# as published by the Free Software Foundation, either version 3 of the
# License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
# See the GNU Affero General Public License for more details.
#
# This program is available from Source Forge: at GeneNetwork Project
# (sourceforge.net/projects/genenetwork/).
#
# Contact Dr. Robert W. Williams at rwilliams@uthsc.edu
#
#
# This module is used by GeneNetwork project (www.genenetwork.org)

import datetime
import math
import numpy as np
import scipy

from rpy2.robjects.packages import importr
import rpy2.robjects as robjects

from pprint import pformat as pf

from utility.redis_tools import get_redis_conn
try:  # Only import this for Python3
    from functools import reduce
except:
    pass
Redis = get_redis_conn()
THIRTY_DAYS = 60 * 60 * 24 * 30

from utility.THCell import THCell
from utility.TDCell import TDCell
from base.trait import GeneralTrait
from base import data_set
from utility import webqtlUtil, helper_functions, corr_result_helpers

from db import webqtlDatabaseFunction
import utility.webqtlUtil #this is for parallel computing only.
from wqflask.correlation import correlation_functions
from utility.benchmark import Bench

from flask import Flask, g, url_for

import utility.logger
logger = utility.logger.getLogger(__name__ )

class CorrelationMatrix(object):

    def __init__(self, start_vars):
        trait_db_list = [trait.strip() for trait in start_vars['trait_list'].split(',')]

        helper_functions.get_trait_db_obs(self, trait_db_list)

        self.all_sample_list = []
        self.traits = []
        self.insufficient_shared_samples = False
        self.do_PCA = True
        this_group = self.trait_list[0][1].group.name #ZS: Getting initial group name before verifying all traits are in the same group in the following loop
        for trait_db in self.trait_list:
            if trait_db[1].group.name != this_group:
                self.insufficient_shared_samples = True
                break
            else:
                this_group = trait_db[1].group.name
            this_trait = trait_db[0]
            self.traits.append(this_trait)
            this_sample_data = this_trait.data

            for sample in this_sample_data:
                if sample not in self.all_sample_list:
                    self.all_sample_list.append(sample)

        if self.insufficient_shared_samples:
            pass
        else:
            self.sample_data = []
            for trait_db in self.trait_list:
                this_trait = trait_db[0]
                this_sample_data = this_trait.data

                this_trait_vals = []
                for sample in self.all_sample_list:
                    if sample in this_sample_data:
                        this_trait_vals.append(this_sample_data[sample].value)
                    else:
                        this_trait_vals.append('')
                self.sample_data.append(this_trait_vals)

            if len(this_trait_vals) < len(self.trait_list): #Shouldn't do PCA if there are more traits than observations/samples
                self.do_PCA = False

            self.lowest_overlap = 8 #ZS: Variable set to the lowest overlapping samples in order to notify user, or 8, whichever is lower (since 8 is when we want to display warning)

            self.corr_results = []
            self.pca_corr_results = []
            self.shared_samples_list = self.all_sample_list
            for trait_db in self.trait_list:
                this_trait = trait_db[0]
                this_db = trait_db[1]

                this_db_samples = this_db.group.all_samples_ordered()
                this_sample_data = this_trait.data

                corr_result_row = []
                pca_corr_result_row = []
                is_spearman = False #ZS: To determine if it's above or below the diagonal
                for target in self.trait_list:
                    target_trait = target[0]
                    target_db = target[1]
                    target_samples = target_db.group.all_samples_ordered()
                    target_sample_data = target_trait.data

                    this_trait_vals = []
                    target_vals = []
                    for index, sample in enumerate(target_samples):
                        if (sample in this_sample_data) and (sample in target_sample_data):
                            sample_value = this_sample_data[sample].value
                            target_sample_value = target_sample_data[sample].value
                            this_trait_vals.append(sample_value)
                            target_vals.append(target_sample_value)
                        else:
                            if sample in self.shared_samples_list:
                                self.shared_samples_list.remove(sample)

                    this_trait_vals, target_vals, num_overlap = corr_result_helpers.normalize_values(this_trait_vals, target_vals)

                    if num_overlap < self.lowest_overlap:
                        self.lowest_overlap = num_overlap
                    if num_overlap == 0:
                        corr_result_row.append([target_trait, 0, num_overlap])
                        pca_corr_result_row.append(0)
                    else:
                        pearson_r, pearson_p = scipy.stats.pearsonr(this_trait_vals, target_vals)
                        if is_spearman == False:
                            sample_r, sample_p = pearson_r, pearson_p
                            if sample_r == 1:
                                is_spearman = True
                        else:
                            sample_r, sample_p = scipy.stats.spearmanr(this_trait_vals, target_vals)

                        corr_result_row.append([target_trait, sample_r, num_overlap])
                        pca_corr_result_row.append(pearson_r)

                self.corr_results.append(corr_result_row)
                self.pca_corr_results.append(pca_corr_result_row)

            self.trait_data_array = []
            for trait_db in self.trait_list:
                this_trait = trait_db[0]
                this_db = trait_db[1]
                this_db_samples = this_db.group.all_samples_ordered()
                this_sample_data = this_trait.data

                this_trait_vals = []
                for index, sample in enumerate(this_db_samples):
                    if (sample in this_sample_data) and (sample in self.shared_samples_list):
                        sample_value = this_sample_data[sample].value
                        this_trait_vals.append(sample_value)
                self.trait_data_array.append(this_trait_vals)

            corr_result_eigen = np.linalg.eig(np.array(self.pca_corr_results))
            corr_eigen_value, corr_eigen_vectors = sortEigenVectors(corr_result_eigen)

            groups = []
            for sample in self.all_sample_list:
                groups.append(1)

            try:
                if self.do_PCA == True:
                    self.pca_works = "True"
                    self.pca_trait_ids = []
                    pca = self.calculate_pca(list(range(len(self.traits))), corr_eigen_value, corr_eigen_vectors)
                    self.loadings_array = self.process_loadings()
                else:
                    self.pca_works = "False"
            except:
                self.pca_works = "False"

            self.js_data = dict(traits = [trait.name for trait in self.traits],
                                groups = groups,
                                cols = list(range(len(self.traits))),
                                rows = list(range(len(self.traits))),
                                samples = self.all_sample_list,
                                sample_data = self.sample_data,)
            #                    corr_results = [result[1] for result in result_row for result_row in self.corr_results])

    def calculate_pca(self, cols, corr_eigen_value, corr_eigen_vectors):
        base = importr('base')
        stats = importr('stats')

        corr_results_to_list = robjects.FloatVector([item for sublist in self.pca_corr_results for item in sublist])

        m = robjects.r.matrix(corr_results_to_list, nrow=len(cols))
        eigen = base.eigen(m)
        pca = stats.princomp(m, cor = "TRUE")
        self.loadings = pca.rx('loadings')
        self.scores = pca.rx('scores')
        self.scale = pca.rx('scale')

        trait_array = zScore(self.trait_data_array)
        trait_array_vectors = np.dot(corr_eigen_vectors, trait_array)

        pca_traits = []
        for i, vector in enumerate(trait_array_vectors):
            #ZS: Check if below check is necessary
            #if corr_eigen_value[i-1] > 100.0/len(self.trait_list):
            pca_traits.append((vector*-1.0).tolist())

        this_group_name = self.trait_list[0][1].group.name
        temp_dataset = data_set.create_dataset(dataset_name = "Temp", dataset_type = "Temp", group_name = this_group_name)
        temp_dataset.group.get_samplelist()
        for i, pca_trait in enumerate(pca_traits):
            trait_id = "PCA" + str(i+1) + "_" + temp_dataset.group.species + "_" + this_group_name + "_" + datetime.datetime.now().strftime("%m%d%H%M%S")
            this_vals_string = ""
            position = 0
            for sample in temp_dataset.group.all_samples_ordered():
                if sample in self.shared_samples_list:
                    this_vals_string += str(pca_trait[position])
                    this_vals_string += " "
                    position += 1
                else:
                    this_vals_string += "x "
            this_vals_string = this_vals_string[:-1]

            Redis.set(trait_id, this_vals_string, ex=THIRTY_DAYS)
            self.pca_trait_ids.append(trait_id)

        return pca

    def process_loadings(self):
        loadings_array = []
        loadings_row = []
        for i in range(len(self.trait_list)):
            loadings_row = []
            if len(self.trait_list) > 2:
               the_range = 3
            else:
               the_range = 2
            for j in range(the_range):
                position = i + len(self.trait_list)*j
                loadings_row.append(self.loadings[0][position])
            loadings_array.append(loadings_row)
        return loadings_array

def zScore(trait_data_array):
    NN = len(trait_data_array[0])
    if NN < 10:
        return trait_data_array
    else:
        i = 0
        for data in trait_data_array:
            N = len(data)
            S = reduce(lambda x, y: x+y, data, 0.)
            SS = reduce(lambda x, y: x+y*y, data, 0.)
            mean = S/N
            var = SS - S*S/N
            stdev = math.sqrt(var/(N-1))
            if stdev == 0:
                stdev = 1e-100
            data2 = [(x-mean)/stdev for x in data]
            trait_data_array[i] = data2
            i += 1
        return trait_data_array

def sortEigenVectors(vector):
    try:
        eigenValues = vector[0].tolist()
        eigenVectors = vector[1].T.tolist()
        combines = []
        i = 0
        for item in eigenValues:
            combines.append([eigenValues[i], eigenVectors[i]])
            i += 1
        combines.sort(webqtlUtil.cmpEigenValue)
        A = []
        B = []
        for item in combines:
            A.append(item[0])
            B.append(item[1])
        sum = reduce(lambda x, y: x+y, A, 0.0)
        A = [x*100.0/sum for x in A]
        return [A, B]
    except:
        return []
<commit_msg>Import "reduce" without a try block<commit_after>## Copyright (C) University of Tennessee Health Science Center, Memphis, TN.
#
# This program is free software: you can redistribute it and/or modify it
# under the terms of the GNU Affero General Public License
# as published by the Free Software Foundation, either version 3 of the
# License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
# See the GNU Affero General Public License for more details.
#
# This program is available from Source Forge: at GeneNetwork Project
# (sourceforge.net/projects/genenetwork/).
#
# Contact Dr. Robert W. Williams at rwilliams@uthsc.edu
#
#
# This module is used by GeneNetwork project (www.genenetwork.org)

import datetime
import math
import numpy as np
import scipy

from rpy2.robjects.packages import importr
import rpy2.robjects as robjects

from pprint import pformat as pf

from utility.redis_tools import get_redis_conn
from functools import reduce

Redis = get_redis_conn()
THIRTY_DAYS = 60 * 60 * 24 * 30

from utility.THCell import THCell
from utility.TDCell import TDCell
from base.trait import GeneralTrait
from base import data_set
from utility import webqtlUtil, helper_functions, corr_result_helpers

from db import webqtlDatabaseFunction
import utility.webqtlUtil #this is for parallel computing only.
from wqflask.correlation import correlation_functions
from utility.benchmark import Bench

from flask import Flask, g, url_for

import utility.logger
logger = utility.logger.getLogger(__name__ )

class CorrelationMatrix(object):

    def __init__(self, start_vars):
        trait_db_list = [trait.strip() for trait in start_vars['trait_list'].split(',')]

        helper_functions.get_trait_db_obs(self, trait_db_list)

        self.all_sample_list = []
        self.traits = []
        self.insufficient_shared_samples = False
        self.do_PCA = True
        this_group = self.trait_list[0][1].group.name #ZS: Getting initial group name before verifying all traits are in the same group in the following loop
        for trait_db in self.trait_list:
            if trait_db[1].group.name != this_group:
                self.insufficient_shared_samples = True
                break
            else:
                this_group = trait_db[1].group.name
            this_trait = trait_db[0]
            self.traits.append(this_trait)
            this_sample_data = this_trait.data

            for sample in this_sample_data:
                if sample not in self.all_sample_list:
                    self.all_sample_list.append(sample)

        if self.insufficient_shared_samples:
            pass
        else:
            self.sample_data = []
            for trait_db in self.trait_list:
                this_trait = trait_db[0]
                this_sample_data = this_trait.data

                this_trait_vals = []
                for sample in self.all_sample_list:
                    if sample in this_sample_data:
                        this_trait_vals.append(this_sample_data[sample].value)
                    else:
                        this_trait_vals.append('')
                self.sample_data.append(this_trait_vals)

            if len(this_trait_vals) < len(self.trait_list): #Shouldn't do PCA if there are more traits than observations/samples
                self.do_PCA = False

            self.lowest_overlap = 8 #ZS: Variable set to the lowest overlapping samples in order to notify user, or 8, whichever is lower (since 8 is when we want to display warning)

            self.corr_results = []
            self.pca_corr_results = []
            self.shared_samples_list = self.all_sample_list
            for trait_db in self.trait_list:
                this_trait = trait_db[0]
                this_db = trait_db[1]

                this_db_samples = this_db.group.all_samples_ordered()
                this_sample_data = this_trait.data

                corr_result_row = []
                pca_corr_result_row = []
                is_spearman = False #ZS: To determine if it's above or below the diagonal
                for target in self.trait_list:
                    target_trait = target[0]
                    target_db = target[1]
                    target_samples = target_db.group.all_samples_ordered()
                    target_sample_data = target_trait.data

                    this_trait_vals = []
                    target_vals = []
                    for index, sample in enumerate(target_samples):
                        if (sample in this_sample_data) and (sample in target_sample_data):
                            sample_value = this_sample_data[sample].value
                            target_sample_value = target_sample_data[sample].value
                            this_trait_vals.append(sample_value)
                            target_vals.append(target_sample_value)
                        else:
                            if sample in self.shared_samples_list:
                                self.shared_samples_list.remove(sample)

                    this_trait_vals, target_vals, num_overlap = corr_result_helpers.normalize_values(this_trait_vals, target_vals)

                    if num_overlap < self.lowest_overlap:
                        self.lowest_overlap = num_overlap
                    if num_overlap == 0:
                        corr_result_row.append([target_trait, 0, num_overlap])
                        pca_corr_result_row.append(0)
                    else:
                        pearson_r, pearson_p = scipy.stats.pearsonr(this_trait_vals, target_vals)
                        if is_spearman == False:
                            sample_r, sample_p = pearson_r, pearson_p
                            if sample_r == 1:
                                is_spearman = True
                        else:
                            sample_r, sample_p = scipy.stats.spearmanr(this_trait_vals, target_vals)

                        corr_result_row.append([target_trait, sample_r, num_overlap])
                        pca_corr_result_row.append(pearson_r)

                self.corr_results.append(corr_result_row)
                self.pca_corr_results.append(pca_corr_result_row)

            self.trait_data_array = []
            for trait_db in self.trait_list:
                this_trait = trait_db[0]
                this_db = trait_db[1]
                this_db_samples = this_db.group.all_samples_ordered()
                this_sample_data = this_trait.data

                this_trait_vals = []
                for index, sample in enumerate(this_db_samples):
                    if (sample in this_sample_data) and (sample in self.shared_samples_list):
                        sample_value = this_sample_data[sample].value
                        this_trait_vals.append(sample_value)
                self.trait_data_array.append(this_trait_vals)

            corr_result_eigen = np.linalg.eig(np.array(self.pca_corr_results))
            corr_eigen_value, corr_eigen_vectors = sortEigenVectors(corr_result_eigen)

            groups = []
            for sample in self.all_sample_list:
                groups.append(1)

            try:
                if self.do_PCA == True:
                    self.pca_works = "True"
                    self.pca_trait_ids = []
                    pca = self.calculate_pca(list(range(len(self.traits))), corr_eigen_value, corr_eigen_vectors)
                    self.loadings_array = self.process_loadings()
                else:
                    self.pca_works = "False"
            except:
                self.pca_works = "False"

            self.js_data = dict(traits = [trait.name for trait in self.traits],
                                groups = groups,
                                cols = list(range(len(self.traits))),
                                rows = list(range(len(self.traits))),
                                samples = self.all_sample_list,
                                sample_data = self.sample_data,)
            #                    corr_results = [result[1] for result in result_row for result_row in self.corr_results])

    def calculate_pca(self, cols, corr_eigen_value, corr_eigen_vectors):
        base = importr('base')
        stats = importr('stats')

        corr_results_to_list = robjects.FloatVector([item for sublist in self.pca_corr_results for item in sublist])

        m = robjects.r.matrix(corr_results_to_list, nrow=len(cols))
        eigen = base.eigen(m)
        pca = stats.princomp(m, cor = "TRUE")
        self.loadings = pca.rx('loadings')
        self.scores = pca.rx('scores')
        self.scale = pca.rx('scale')

        trait_array = zScore(self.trait_data_array)
        trait_array_vectors = np.dot(corr_eigen_vectors, trait_array)

        pca_traits = []
        for i, vector in enumerate(trait_array_vectors):
            #ZS: Check if below check is necessary
            #if corr_eigen_value[i-1] > 100.0/len(self.trait_list):
            pca_traits.append((vector*-1.0).tolist())

        this_group_name = self.trait_list[0][1].group.name
        temp_dataset = data_set.create_dataset(dataset_name = "Temp", dataset_type = "Temp", group_name = this_group_name)
        temp_dataset.group.get_samplelist()
        for i, pca_trait in enumerate(pca_traits):
            trait_id = "PCA" + str(i+1) + "_" + temp_dataset.group.species + "_" + this_group_name + "_" + datetime.datetime.now().strftime("%m%d%H%M%S")
            this_vals_string = ""
            position = 0
            for sample in temp_dataset.group.all_samples_ordered():
                if sample in self.shared_samples_list:
                    this_vals_string += str(pca_trait[position])
                    this_vals_string += " "
                    position += 1
                else:
                    this_vals_string += "x "
            this_vals_string = this_vals_string[:-1]

            Redis.set(trait_id, this_vals_string, ex=THIRTY_DAYS)
            self.pca_trait_ids.append(trait_id)

        return pca

    def process_loadings(self):
        loadings_array = []
        loadings_row = []
        for i in range(len(self.trait_list)):
            loadings_row = []
            if len(self.trait_list) > 2:
               the_range = 3
            else:
               the_range = 2
            for j in range(the_range):
                position = i + len(self.trait_list)*j
                loadings_row.append(self.loadings[0][position])
            loadings_array.append(loadings_row)
        return loadings_array

def zScore(trait_data_array):
    NN = len(trait_data_array[0])
    if NN < 10:
        return trait_data_array
    else:
        i = 0
        for data in trait_data_array:
            N = len(data)
            S = reduce(lambda x, y: x+y, data, 0.)
            SS = reduce(lambda x, y: x+y*y, data, 0.)
            mean = S/N
            var = SS - S*S/N
            stdev = math.sqrt(var/(N-1))
            if stdev == 0:
                stdev = 1e-100
            data2 = [(x-mean)/stdev for x in data]
            trait_data_array[i] = data2
            i += 1
        return trait_data_array

def sortEigenVectors(vector):
    try:
        eigenValues = vector[0].tolist()
        eigenVectors = vector[1].T.tolist()
        combines = []
        i = 0
        for item in eigenValues:
            combines.append([eigenValues[i], eigenVectors[i]])
            i += 1
        combines.sort(webqtlUtil.cmpEigenValue)
        A = []
        B = []
        for item in combines:
            A.append(item[0])
            B.append(item[1])
        sum = reduce(lambda x, y: x+y, A, 0.0)
        A = [x*100.0/sum for x in A]
        return [A, B]
    except:
        return []
<|endoftext|>


<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-

"""
Elementary Math Library. (For demonstration purpose only)
"""

__version__ = "0.0.1"
__short_description__ = "Elementary Mathematics."
__license__ = "MIT"
<commit_msg>2016-06-03<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-

"""
Elementary Math Library. (For demonstration purpose only)
"""

__version__ = "0.0.2"
__short_description__ = "Elementary Mathematics."
__license__ = "MIT"
<|endoftext|>


<commit_before>from os.path import isfile, exists, sep as path_separator
import logging, logging.config
import traceback

from django.utils.itercompat import is_iterable
from django.conf import settings

INSTALLED_APPS_REGISTER = {}

def register(app_name, modules):
    """
    simple module registering for later usage
    we don't want to import admin.py in models.py
    """
    global INSTALLED_APPS_REGISTER
    mod_list = INSTALLED_APPS_REGISTER.get(app_name, [])

    if isinstance(modules, basestring):
        mod_list.append(modules)
    elif is_iterable(modules):
        mod_list.extend(modules)

    INSTALLED_APPS_REGISTER[app_name] = mod_list

def call_modules(auto_discover=()):
    """
    this is called in project urls.py
    for registering desired modules (eg.: admin.py)
    """
    log = logging.getLogger('ella.utils.installedapps.call_modules')

    for app in settings.INSTALLED_APPS:
        modules = set(auto_discover)
        if app in INSTALLED_APPS_REGISTER:
            modules.update(INSTALLED_APPS_REGISTER[app])
        for module in modules:
            try:
                imp = '%s.%s' % (app, module)
                mod = __import__(imp, {}, {}, [''])
                inst = getattr(mod, '__install__', lambda:None)
                inst()
            except ImportError, e:
                msg = 'problem during discovering %s - %s\n%s' % (imp, e, traceback.format_exc())
                # check if problem is inside autodiscovered file (i.e. mispelled module name) OR autodiscovered file does not exists
                mod = __import__(app, {}, {}, [''])
                app_path = mod.__file__.split(path_separator)[:-1]
                app_path.append('%s.py' % module)
                mod_path = path_separator.join(app_path)
                if not exists(mod_path):
                    # autodiscovered file does not exist
                    log.debug(msg)
                else:
                    # ImportError inside autodiscovered file
                    log.error(msg)
                    logging.error(msg)

def init_logger():
    """init logger with LOGGING_CONFIG_FILE settings option"""
    LOGGING_CONFIG_FILE = getattr(settings, 'LOGGING_CONFIG_FILE', None)
    if isinstance(LOGGING_CONFIG_FILE, basestring) and isfile(LOGGING_CONFIG_FILE):
        logging.config.fileConfig(LOGGING_CONFIG_FILE)

<commit_msg>use django's import_module functionality<commit_after>from os.path import isfile, exists, sep as path_separator
import logging, logging.config
import traceback

from django.utils.itercompat import is_iterable
from django.conf import settings
from django.utils.importlib import import_module

INSTALLED_APPS_REGISTER = {}

def register(app_name, modules):
    """
    simple module registering for later usage
    we don't want to import admin.py in models.py
    """
    global INSTALLED_APPS_REGISTER
    mod_list = INSTALLED_APPS_REGISTER.get(app_name, [])

    if isinstance(modules, basestring):
        mod_list.append(modules)
    elif is_iterable(modules):
        mod_list.extend(modules)

    INSTALLED_APPS_REGISTER[app_name] = mod_list

def call_modules(auto_discover=()):
    """
    this is called in project urls.py
    for registering desired modules (eg.: admin.py)
    """
    log = logging.getLogger('ella.utils.installedapps.call_modules')

    for app in settings.INSTALLED_APPS:
        modules = set(auto_discover)
        if app in INSTALLED_APPS_REGISTER:
            modules.update(INSTALLED_APPS_REGISTER[app])
        for module in modules:
            try:
                imp = '%s.%s' % (app, module)
                mod = import_module(imp)
                inst = getattr(mod, '__install__', lambda:None)
                inst()
            except ImportError, e:
                msg = 'problem during discovering %s - %s\n%s' % (imp, e, traceback.format_exc())
                # check if problem is inside autodiscovered file (i.e. mispelled module name) OR autodiscovered file does not exists
                mod = __import__(app, {}, {}, [''])
                app_path = mod.__file__.split(path_separator)[:-1]
                app_path.append('%s.py' % module)
                mod_path = path_separator.join(app_path)
                if not exists(mod_path):
                    # autodiscovered file does not exist
                    log.debug(msg)
                else:
                    # ImportError inside autodiscovered file
                    log.error(msg)
                    logging.error(msg)

def init_logger():
    """init logger with LOGGING_CONFIG_FILE settings option"""
    LOGGING_CONFIG_FILE = getattr(settings, 'LOGGING_CONFIG_FILE', None)
    if isinstance(LOGGING_CONFIG_FILE, basestring) and isfile(LOGGING_CONFIG_FILE):
        logging.config.fileConfig(LOGGING_CONFIG_FILE)

<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
from django.db import models

class WebUrl(models.Model):
    raw_url         = models.CharField(max_length=512)
    domain          = models.ForeignKey("Domain")

class Domain(models.Model):
    name         = models.CharField(max_length=64)

class UrlTime(models.Model):
    userid          = models.IntegerField()
    start_time      = models.DateTimeField()
<<<<<<< HEAD
    seconds         = models .IntegerField()
=======
    milli_seconds   = models.IntegerField()
    end_time        = models.DateTimeField()
>>>>>>> 6dc97a5b8d6ca5754ace0b6f3101e58ad914492a
    web_url         = models.ForeignKey("WebUrl") # maybe useful
    domain          = models.ForeignKey("Domain")

# class WebLabelCategory(models.Model):
#     name            = models.CharField(max_length=32)

class WebUrlLabel(models.Model):
    name            = models.CharField(max_length=32)
    domain          = models.ForeignKey("Domain")
    # category        = models.ForeignKey("WebLabelCategory", null=True)

class UserLabel(models.Model):
    name            = models.CharField(max_length=32)

<commit_msg>wait for debug<commit_after># -*- coding: utf-8 -*-
from django.db import models

class WebUrl(models.Model):
    raw_url         = models.CharField(max_length=512)
    domain          = models.ForeignKey("Domain")

class Domain(models.Model):
    name         = models.CharField(max_length=64)

class UrlTime(models.Model):
    userid          = models.IntegerField()
    start_time      = models.DateTimeField()
    milli_seconds   = models.IntegerField()
    end_time        = models.DateTimeField()
    web_url         = models.ForeignKey("WebUrl") # maybe useful
    domain          = models.ForeignKey("Domain")

# class WebLabelCategory(models.Model):
#     name            = models.CharField(max_length=32)

class WebUrlLabel(models.Model):
    name            = models.CharField(max_length=32)
    domain          = models.ForeignKey("Domain")
    # category        = models.ForeignKey("WebLabelCategory", null=True)

class UserLabel(models.Model):
    name            = models.CharField(max_length=32)

<|endoftext|>


<commit_before>import sys
import os
import files.io
from StringIO import StringIO
from constants import *

added_path = os.path.join(os.path.dirname(os.path.abspath(os.path.dirname(__file__))), "script")
sys.path.append(added_path)
import script
import mess
sys.path.pop()

def full_file(pointer):
    pointer.seek(0)
    return pointer.read()

def send_confirmation_email(recipient, contents):
    connection = script.get_mailserver_connection()
    email_headers = {
        "Subject": CONFIRMATION_EMAIL_SUBJECT,
        "To": recipient,
        "From": CONFIRMATION_EMAIL_FROM
    }
    mess.Message(email_headers, contents).send(connection)
    connection.quit()

def run_on(key, actually_send=False, confirmation_email=CONFIRMATION_EMAIL_FROM, confirmation_email_on_success=True, confirmation_email_on_fail=True, special_text="", send_default_confirmation=False):
    input_files = files.io.get(key)
    assert "template" in input_files
    output_location = StringIO()
    kwargs = {}
    if send_default_confirmation:
        pass
    else:
        kwargs["send_confirmation_to"] = None
    if "constants" in input_files:
        kwargs["constants"] = StringIO(input_files["constants"])
    success = script.run_script(
        StringIO(input_files["template"]),
        StringIO(input_files["spreadsheet"]),
        output=output_location,
        actually_send=actually_send,
        **kwargs
    )
    script_results = special_text + full_file(output_location)
    if confirmation_email and ((confirmation_email_on_success and success) or (confirmation_email_on_fail and not success)):
        send_confirmation_email(confirmation_email, script_results)
    return script_results, success
<commit_msg>added more helpful errors<commit_after>import sys
import os
import files.io
from StringIO import StringIO
from constants import *

added_path = os.path.join(os.path.dirname(os.path.abspath(os.path.dirname(__file__))), "script")
sys.path.append(added_path)
import script
import mess
sys.path.pop()

def full_file(pointer):
    pointer.seek(0)
    return pointer.read()

def send_confirmation_email(recipient, contents):
    connection = script.get_mailserver_connection()
    email_headers = {
        "Subject": CONFIRMATION_EMAIL_SUBJECT,
        "To": recipient,
        "From": CONFIRMATION_EMAIL_FROM
    }
    mess.Message(email_headers, contents).send(connection)
    connection.quit()

def run_on(key, actually_send=False, confirmation_email=CONFIRMATION_EMAIL_FROM, confirmation_email_on_success=True, confirmation_email_on_fail=True, special_text="", send_default_confirmation=False):
    input_files = files.io.get(key)
    assert "template" in input_files, "Need to submit non-empty template"
    assert "spreadsheet" in input_files, "Need to submit non-empty spreadsheet"
    output_location = StringIO()
    kwargs = {}
    if send_default_confirmation:
        pass
    else:
        kwargs["send_confirmation_to"] = None
    if "constants" in input_files:
        kwargs["constants"] = StringIO(input_files["constants"])
    success = script.run_script(
        StringIO(input_files["template"]),
        StringIO(input_files["spreadsheet"]),
        output=output_location,
        actually_send=actually_send,
        **kwargs
    )
    script_results = special_text + full_file(output_location)
    if confirmation_email and ((confirmation_email_on_success and success) or (confirmation_email_on_fail and not success)):
        send_confirmation_email(confirmation_email, script_results)
    return script_results, success
<|endoftext|>


<commit_before>'''
VMController is a general purpose virtual machine controller written in Python.\n
vmcontroller.host is the module that runs on the host operating system.
'''

try:
    from setuptools import setup, find_packages
except ImportError:
    from distribute_setup import use_setuptools
    use_setuptools()
    from setuptools import setup, find_packages

from vmcontroller.host import __version__

name = 'vmcontroller.host'
version = __version__

setup(
    name=name,
    version=version,
    description='Virtual Machine Controller for host operating system',
    long_description=__doc__,
    keywords='hypervisor virtual-machine controller',
    license='BSD',
    author='David Garcia Quintas, Rohit Yadav',
    author_email='dgquintas@gmail.com, rohityadav89@gmail.com',
    url='http://code.google.com/p/vmcontroller',
    packages=find_packages(exclude=['ez_setup', 'distribute_setup', 'tests', 'tests.*']),
    package_data={'vmcontroller.host': ['config/*.cfg*']},
    namespace_packages=['vmcontroller'],
    zip_safe=False,
    include_package_data=True,
    install_requires=[
        'distribute',
        'setuptools',
        'vmcontroller.common>=0.1.0',
        'twisted',
        'stomper==0.2.2',
        'netifaces',
        'coilmq',
        'morbid',
        'simplejson',
        'inject',
        'uuid'
    ],
    classifiers=[
        "Development Status :: 2 - Pre-Alpha",
        "Environment :: Console",
        "Intended Audience :: Developers",
        "License :: OSI Approved :: BSD License",
        "Operating System :: OS Independent",
        "Programming Language :: Python :: 2.6",
        "Topic :: Software Development :: Libraries :: Python Modules",
    ],
    entry_points="""
    [console_scripts]
    vmcontroller-host = vmcontroller.host:main
    """,
)
<commit_msg>Setup: Host version import in setup may cause problem... Commented out for the time being<commit_after>'''
VMController is a general purpose virtual machine controller written in Python.\n
vmcontroller.host is the module that runs on the host operating system.
'''

try:
    from setuptools import setup, find_packages
except ImportError:
    from distribute_setup import use_setuptools
    use_setuptools()
    from setuptools import setup, find_packages

#from vmcontroller.host import __version__

name = 'vmcontroller.host'
version = '0.1.0' #__version__

setup(
    name=name,
    version=version,
    description='Virtual Machine Controller for host operating system',
    long_description=__doc__,
    keywords='hypervisor virtual-machine controller',
    license='BSD',
    author='David Garcia Quintas, Rohit Yadav',
    author_email='dgquintas@gmail.com, rohityadav89@gmail.com',
    url='http://code.google.com/p/vmcontroller',
    packages=find_packages(exclude=['ez_setup', 'distribute_setup', 'tests', 'tests.*']),
    package_data={'vmcontroller.host': ['config/*.cfg*']},
    namespace_packages=['vmcontroller'],
    zip_safe=False,
    include_package_data=True,
    install_requires=[
        'distribute',
        'setuptools',
        'vmcontroller.common>=0.1.0',
        'twisted',
        'stomper==0.2.2',
        'netifaces',
        'coilmq',
        'morbid',
        'simplejson',
        'inject',
        'uuid'
    ],
    classifiers=[
        "Development Status :: 2 - Pre-Alpha",
        "Environment :: Console",
        "Intended Audience :: Developers",
        "License :: OSI Approved :: BSD License",
        "Operating System :: OS Independent",
        "Programming Language :: Python :: 2.6",
        "Topic :: Software Development :: Libraries :: Python Modules",
    ],
    entry_points="""
    [console_scripts]
    vmcontroller-host = vmcontroller.host:main
    """,
)
<|endoftext|>


<commit_before>"""
Tests for the storage service.
"""

# TODO tests for getting all users

import json

from requests import codes

from .testtools import InMemoryStorageTests

USER_DATA = {'email': 'alice@example.com', 'password_hash': '123abc'}

class CreateUserTests(InMemoryStorageTests):
    """
    Tests for the user creation endpoint at ``POST /users``.
    """

    def test_success_response(self):
        """
        A ``POST /users`` request with an email address and password hash
        returns a JSON response with user details and a CREATED status.
        """
        response = self.storage_app.post(
            '/users',
            content_type='application/json',
            data=json.dumps(USER_DATA))
        self.assertEqual(response.headers['Content-Type'], 'application/json')
        self.assertEqual(response.status_code, codes.CREATED)
        self.assertEqual(json.loads(response.data.decode('utf8')), USER_DATA)

    def test_missing_email(self):
        """
        A ``POST /users`` request without an email address returns a
        BAD_REQUEST status code and an error message.
        """
        data = USER_DATA.copy()
        data.pop('email')

        response = self.storage_app.post(
            '/users',
            content_type='application/json',
            data=json.dumps(data))
        self.assertEqual(response.headers['Content-Type'], 'application/json')
        self.assertEqual(response.status_code, codes.BAD_REQUEST)
        expected = {
            'title': 'There was an error validating the given arguments.',
            'detail': "'email' is a required property",
        }
        self.assertEqual(json.loads(response.data.decode('utf8')), expected)

    def test_missing_password_hash(self):
        """
        A ``POST /users`` request without a password hash returns a BAD_REQUEST
        status code and an error message.
        """
        data = USER_DATA.copy()
        data.pop('password_hash')

        response = self.storage_app.post(
            '/users',
            content_type='application/json',
            data=json.dumps({'email': USER_DATA['email']}))
        self.assertEqual(response.headers['Content-Type'], 'application/json')
        self.assertEqual(response.status_code, codes.BAD_REQUEST)
        expected = {
            'title': 'There was an error validating the given arguments.',
            'detail': "'password_hash' is a required property",
        }
        self.assertEqual(json.loads(response.data.decode('utf8')), expected)

    def test_existing_user(self):
        """
        A ``POST /users`` request for an email address which already exists
        returns a CONFLICT status code and error details.
        """
        self.storage_app.post(
            '/users',
            content_type='application/json',
            data=json.dumps(USER_DATA))
        data = USER_DATA.copy()
        data['password'] = 'different'
        response = self.storage_app.post(
            '/users',
            content_type='application/json',
            data=json.dumps(USER_DATA))
        self.assertEqual(response.headers['Content-Type'], 'application/json')
        self.assertEqual(response.status_code, codes.CONFLICT)
        expected = {
            'title': 'There is already a user with the given email address.',
            'detail': 'A user already exists with the email "{email}"'.format(
                email=USER_DATA['email']),
        }
        self.assertEqual(json.loads(response.data.decode('utf8')), expected)

    def test_incorrect_content_type(self):
        """
        If a Content-Type header other than 'application/json' is given, an
        UNSUPPORTED_MEDIA_TYPE status code is given.
        """
        response = self.storage_app.post('/users', content_type='text/html')
        self.assertEqual(response.status_code, codes.UNSUPPORTED_MEDIA_TYPE)

class GetUserTests(InMemoryStorageTests):
    """
    Tests for getting a user at ``GET /users/{email}``.
    """

    def test_success(self):
        """
        A ``GET`` request for an existing user an OK status code and the user's
        details.
        """
        self.storage_app.post(
            '/users',
            content_type='application/json',
            data=json.dumps(USER_DATA))
        response = self.storage_app.get(
            '/users/{email}'.format(email=USER_DATA['email']),
            content_type='application/json')
        self.assertEqual(response.status_code, codes.OK)
        self.assertEqual(json.loads(response.data.decode('utf8')), USER_DATA)

    def test_non_existant_user(self):
        """
        A ``GET`` request for a user which does not exist returns a NOT_FOUND
        status code and error details.
        """
        response = self.storage_app.get(
            '/users/{email}'.format(email=USER_DATA['email']),
            content_type='application/json')
        self.assertEqual(response.headers['Content-Type'], 'application/json')
        self.assertEqual(response.status_code, codes.NOT_FOUND)
        expected = {
            'title': 'The requested user does not exist.',
            'detail': 'No user exists with the email "{email}"'.format(
                email=USER_DATA['email']),
        }
        self.assertEqual(json.loads(response.data.decode('utf8')), expected)

    def test_incorrect_content_type(self):
        """
        If a Content-Type header other than 'application/json' is given, an
        UNSUPPORTED_MEDIA_TYPE status code is given.
        """
        response = self.storage_app.get(
            '/users/{email}'.format(email=USER_DATA['email']),
            content_type='text/html',
        )

        self.assertEqual(response.status_code, codes.UNSUPPORTED_MEDIA_TYPE)

class GetUsersTests(InMemoryStorageTests):
    """
    Tests for getting all users at ``GET /users/``.
    """

    def test_incorrect_content_type(self):
        """
        If a Content-Type header other than 'application/json' is given, an
        UNSUPPORTED_MEDIA_TYPE status code is given.
        """
        response = self.storage_app.get('/users/', content_type='text/html')
        self.assertEqual(response.status_code, codes.UNSUPPORTED_MEDIA_TYPE)
<commit_msg>Stub tests for getting /users/<commit_after>"""
Tests for the storage service.
"""

# TODO tests for getting all users

import json

from requests import codes

from .testtools import InMemoryStorageTests

USER_DATA = {'email': 'alice@example.com', 'password_hash': '123abc'}

class CreateUserTests(InMemoryStorageTests):
    """
    Tests for the user creation endpoint at ``POST /users``.
    """

    def test_success_response(self):
        """
        A ``POST /users`` request with an email address and password hash
        returns a JSON response with user details and a CREATED status.
        """
        response = self.storage_app.post(
            '/users',
            content_type='application/json',
            data=json.dumps(USER_DATA))
        self.assertEqual(response.headers['Content-Type'], 'application/json')
        self.assertEqual(response.status_code, codes.CREATED)
        self.assertEqual(json.loads(response.data.decode('utf8')), USER_DATA)

    def test_missing_email(self):
        """
        A ``POST /users`` request without an email address returns a
        BAD_REQUEST status code and an error message.
        """
        data = USER_DATA.copy()
        data.pop('email')

        response = self.storage_app.post(
            '/users',
            content_type='application/json',
            data=json.dumps(data))
        self.assertEqual(response.headers['Content-Type'], 'application/json')
        self.assertEqual(response.status_code, codes.BAD_REQUEST)
        expected = {
            'title': 'There was an error validating the given arguments.',
            'detail': "'email' is a required property",
        }
        self.assertEqual(json.loads(response.data.decode('utf8')), expected)

    def test_missing_password_hash(self):
        """
        A ``POST /users`` request without a password hash returns a BAD_REQUEST
        status code and an error message.
        """
        data = USER_DATA.copy()
        data.pop('password_hash')

        response = self.storage_app.post(
            '/users',
            content_type='application/json',
            data=json.dumps({'email': USER_DATA['email']}))
        self.assertEqual(response.headers['Content-Type'], 'application/json')
        self.assertEqual(response.status_code, codes.BAD_REQUEST)
        expected = {
            'title': 'There was an error validating the given arguments.',
            'detail': "'password_hash' is a required property",
        }
        self.assertEqual(json.loads(response.data.decode('utf8')), expected)

    def test_existing_user(self):
        """
        A ``POST /users`` request for an email address which already exists
        returns a CONFLICT status code and error details.
        """
        self.storage_app.post(
            '/users',
            content_type='application/json',
            data=json.dumps(USER_DATA))
        data = USER_DATA.copy()
        data['password'] = 'different'
        response = self.storage_app.post(
            '/users',
            content_type='application/json',
            data=json.dumps(USER_DATA))
        self.assertEqual(response.headers['Content-Type'], 'application/json')
        self.assertEqual(response.status_code, codes.CONFLICT)
        expected = {
            'title': 'There is already a user with the given email address.',
            'detail': 'A user already exists with the email "{email}"'.format(
                email=USER_DATA['email']),
        }
        self.assertEqual(json.loads(response.data.decode('utf8')), expected)

    def test_incorrect_content_type(self):
        """
        If a Content-Type header other than 'application/json' is given, an
        UNSUPPORTED_MEDIA_TYPE status code is given.
        """
        response = self.storage_app.post('/users', content_type='text/html')
        self.assertEqual(response.status_code, codes.UNSUPPORTED_MEDIA_TYPE)

class GetUserTests(InMemoryStorageTests):
    """
    Tests for getting a user at ``GET /users/{email}``.
    """

    def test_success(self):
        """
        A ``GET`` request for an existing user an OK status code and the user's
        details.
        """
        self.storage_app.post(
            '/users',
            content_type='application/json',
            data=json.dumps(USER_DATA))
        response = self.storage_app.get(
            '/users/{email}'.format(email=USER_DATA['email']),
            content_type='application/json')
        self.assertEqual(response.status_code, codes.OK)
        self.assertEqual(json.loads(response.data.decode('utf8')), USER_DATA)

    def test_non_existant_user(self):
        """
        A ``GET`` request for a user which does not exist returns a NOT_FOUND
        status code and error details.
        """
        response = self.storage_app.get(
            '/users/{email}'.format(email=USER_DATA['email']),
            content_type='application/json')
        self.assertEqual(response.headers['Content-Type'], 'application/json')
        self.assertEqual(response.status_code, codes.NOT_FOUND)
        expected = {
            'title': 'The requested user does not exist.',
            'detail': 'No user exists with the email "{email}"'.format(
                email=USER_DATA['email']),
        }
        self.assertEqual(json.loads(response.data.decode('utf8')), expected)

    def test_incorrect_content_type(self):
        """
        If a Content-Type header other than 'application/json' is given, an
        UNSUPPORTED_MEDIA_TYPE status code is given.
        """
        response = self.storage_app.get(
            '/users/{email}'.format(email=USER_DATA['email']),
            content_type='text/html',
        )

        self.assertEqual(response.status_code, codes.UNSUPPORTED_MEDIA_TYPE)

class GetUsersTests(InMemoryStorageTests):
    """
    Tests for getting information about all users at ``GET /users/``.
    """

    def test_no_users(self):
        """
        A ``GET`` request for information about all users returns an OK status
        code and an empty array when there are no users.
        """

    def test_with_users(self):
        """
        A ``GET`` request for information about all users returns an OK status
        code and an array of user information.
        """

    def test_incorrect_content_type(self):
        """
        If a Content-Type header other than 'application/json' is given, an
        UNSUPPORTED_MEDIA_TYPE status code is given.
        """
        response = self.storage_app.get('/users/', content_type='text/html')
        self.assertEqual(response.status_code, codes.UNSUPPORTED_MEDIA_TYPE)
<|endoftext|>


<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-
"""
Base class for proxies to OCD models.
"""
import textwrap

class OCDProxyModelMixin(object):
    """
    Properties and methods shared by all OCD proxy models.
    """

    @property
    def model(self):
        """
        Returns the model class that is being proxied.
        """
        return self.__class__.__bases__[0]

    @property
    def object_name(self):
        """
        Return the model's object name as a string.
        """
        return self.model._meta.object_name

    @property
    def doc(self):
        """
        Returns doc string of the model corresponding to the ProcessedDataFile.
        """
        if self.model.__doc__.startswith(self.model._meta.object_name):
            return ''
        return textwrap.dedent(self.model.__doc__).strip()

    @property
    def klass_group(self):
        """
        Return the model's group.
        """
        return "CCDC"

    def get_field_list(self):
        """
        Return all the fields on the model as a list.
        """
        return self.model._meta.fields
<commit_msg>Prefix flat proxy models with 'Flat'<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-
"""
Base class for proxies to OCD models.
"""
import textwrap

class OCDProxyModelMixin(object):
    """
    Properties and methods shared by all OCD proxy models.
    """

    @property
    def model(self):
        """
        Returns the model class that is being proxied.
        """
        return self.__class__.__bases__[0]

    @property
    def object_name(self):
        """
        Return the model's object name as a string.

        If the model is flat model proxy, include the prefix "Flat". Otherwise,
        just use the object name of the model that is being proxied.
        """
        if 'Flat' in self._meta.object_name:
            object_name = 'Flat%s' % self.model._meta.object_name
        else:
            object_name = self.model._meta.object_name
        return object_name

    @property
    def doc(self):
        """
        Returns doc string of the model corresponding to the ProcessedDataFile.
        """
        if self.model.__doc__.startswith(self.model._meta.object_name):
            return ''
        return textwrap.dedent(self.model.__doc__).strip()

    @property
    def klass_group(self):
        """
        Return the model's group.
        """
        return "CCDC"

    def get_field_list(self):
        """
        Return all the fields on the model as a list.
        """
        return self.model._meta.fields
<|endoftext|>


<commit_before>from qutebrowser.config.configfiles import ConfigAPI  # noqa: F401
from qutebrowser.config.config import ConfigContainer  # noqa: F401

config: ConfigAPI = config  # noqa: F821 pylint: disable=E0602,C0103
c: ConfigContainer = c  # noqa: F821 pylint: disable=E0602,C0103

class Colors:
    black = "#282a36"
    light_black = "#686868"
    red = "#ff5c57"
    green = "#5af78e"
    yellow = "#f3f99d"
    blue = "#57c7ff"
    magenta = "#ff6ac1"
    cyan = "#9aedfe"
    white = "#eff0eb"

class Fonts:
    sans = "Cantarell"
    fixed = "OperatorMonoLig Nerd Font"
    medium = 12
    large = 14
    web = 17

config.load_autoconfig(False)

# Key bindings
config.unbind("d")
config.unbind("J")
config.unbind("K")
config.bind("gc", "tab-close")
config.bind("g[", "tab-prev")
config.bind("g]", "tab-next")

config.set("auto_save.session", True)
config.set("completion.shrink", True)
config.set("confirm_quit", ["multiple-tabs", "downloads"])
config.set("content.notifications.enabled", True, "https://web.whatsapp.com/")
config.set(
    "content.tls.certificate_errors", "load-insecurely", "https://127.0.0.1:8384/"
)
config.set("editor.command", ["foot", "-e", "nvim", "{file}"])
config.set(
    "url.searchengines",
    {
        "DEFAULT": "https://duckduckgo.com/?q={}",
        "arch": "https://wiki.archlinux.org/?search={}",
        "crunch": "https://www.crunchbase.com/textsearch?q={}",
        "define": "https://dictionary.cambridge.org/search/direct/?datasetsearch=english&q={}",
    },
)

# Fonts
fonts = Fonts()

config.set("fonts.default_family", fonts.fixed)
config.set("fonts.default_size", f"{fonts.medium}pt")
config.set("fonts.tabs.selected", f"{fonts.large}pt {fonts.sans}")
config.set("fonts.tabs.unselected", f"{fonts.large}pt {fonts.sans}")
config.set("fonts.web.family.fixed", fonts.fixed)
config.set("fonts.web.family.sans_serif", fonts.sans)
config.set("fonts.web.family.serif", fonts.sans)
config.set("fonts.web.family.standard", fonts.sans)
config.set("fonts.web.size.default", fonts.web)
config.set("fonts.web.size.default_fixed", fonts.medium)

# Colors
colors = Colors()

c.colors.completion.fg = colors.white
c.colors.completion.odd.bg = colors.black
c.colors.completion.even.bg = colors.black

c.colors.completion.category.fg = colors.magenta
c.colors.completion.category.bg = colors.black
c.colors.completion.category.border.top = colors.black
c.colors.completion.category.border.bottom = colors.black

c.colors.completion.item.selected.fg = colors.black
c.colors.completion.item.selected.bg = colors.yellow
c.colors.completion.item.selected.border.top = colors.yellow
c.colors.completion.item.selected.border.bottom = colors.yellow
c.colors.completion.item.selected.match.fg = colors.magenta

c.colors.completion.match.fg = colors.magenta
c.colors.completion.scrollbar.fg = colors.magenta
c.colors.completion.scrollbar.bg = colors.black

c.colors.contextmenu.disabled.fg = colors.light_black
c.colors.contextmenu.disabled.bg = colors.black
c.colors.contextmenu.menu.fg = colors.white
c.colors.contextmenu.menu.bg = colors.black
c.colors.contextmenu.selected.fg = colors.black
c.colors.contextmenu.selected.bg = colors.blue

c.colors.downloads.bar.bg = colors.black
c.colors.downloads.start.fg = colors.black
c.colors.downloads.start.bg = colors.magenta
c.colors.downloads.stop.fg = colors.black
c.colors.downloads.stop.bg = colors.blue
c.colors.downloads.error.fg = colors.black

c.hints.border = f"1px solid {colors.yellow}"
c.colors.hints.fg = colors.black
c.colors.hints.bg = colors.yellow
c.colors.hints.match.fg = colors.red

c.colors.keyhint.suffix.fg = colors.magenta
c.colors.keyhint.fg = colors.magenta
c.colors.keyhint.bg = colors.black

c.colors.messages.error.fg = colors.black
c.colors.messages.error.bg = colors.red
c.colors.messages.error.border = colors.red

c.colors.messages.warning.fg = colors.black
c.colors.messages.warning.bg = colors.yellow
c.colors.messages.warning.border = colors.yellow

c.colors.messages.info.fg = colors.magenta
c.colors.messages.info.bg = colors.black
c.colors.messages.info.border = colors.black

c.colors.prompts.fg = colors.white
c.colors.prompts.bg = colors.black
c.colors.prompts.border = colors.black
c.colors.prompts.selected.fg = colors.magenta
c.colors.prompts.selected.bg = colors.green

c.colors.statusbar.normal.fg = colors.magenta
c.colors.statusbar.normal.bg = colors.black
c.colors.statusbar.insert.fg = colors.blue
c.colors.statusbar.insert.bg = colors.black
c.colors.statusbar.passthrough.fg = colors.green
c.colors.statusbar.passthrough.bg = colors.black
c.colors.statusbar.private.fg = colors.cyan
c.colors.statusbar.private.bg = colors.black

c.colors.statusbar.command.fg = colors.white
c.colors.statusbar.command.bg = colors.black
c.colors.statusbar.command.private.fg = colors.cyan
c.colors.statusbar.command.private.bg = colors.red

c.colors.statusbar.caret.fg = colors.magenta
c.colors.statusbar.caret.bg = colors.black
c.colors.statusbar.caret.selection.fg = colors.magenta
c.colors.statusbar.caret.selection.bg = colors.black

c.colors.statusbar.progress.bg = colors.magenta

c.colors.statusbar.url.fg = colors.magenta
c.colors.statusbar.url.hover.fg = colors.green
c.colors.statusbar.url.success.http.fg = colors.yellow
c.colors.statusbar.url.success.https.fg = colors.white
c.colors.statusbar.url.warn.fg = colors.yellow
c.colors.statusbar.url.error.fg = colors.red

c.colors.tabs.bar.bg = colors.black

c.colors.tabs.indicator.start = colors.magenta
c.colors.tabs.indicator.stop = colors.blue
c.colors.tabs.indicator.error = colors.red

c.colors.tabs.odd.fg = colors.white
c.colors.tabs.odd.bg = colors.black
c.colors.tabs.even.fg = colors.white
c.colors.tabs.even.bg = colors.black

c.colors.tabs.pinned.even.fg = colors.white
c.colors.tabs.pinned.even.bg = colors.black
c.colors.tabs.pinned.odd.fg = colors.white
c.colors.tabs.pinned.odd.bg = colors.black

c.colors.tabs.pinned.selected.even.fg = colors.black
c.colors.tabs.pinned.selected.even.bg = colors.blue
c.colors.tabs.pinned.selected.odd.fg = colors.black
c.colors.tabs.pinned.selected.odd.bg = colors.blue

c.colors.tabs.selected.odd.fg = colors.black
c.colors.tabs.selected.odd.bg = colors.blue
c.colors.tabs.selected.even.fg = colors.black
c.colors.tabs.selected.even.bg = colors.blue

c.colors.webpage.bg = colors.white
<commit_msg>Update dot_config/qutebrowser/config.py<commit_after>from qutebrowser.config.configfiles import ConfigAPI

config: ConfigAPI = config

class Colors:
    black = "#282a36"
    light_black = "#686868"
    red = "#ff5c57"
    green = "#5af78e"
    yellow = "#f3f99d"
    blue = "#57c7ff"
    magenta = "#ff6ac1"
    cyan = "#9aedfe"
    white = "#eff0eb"

class Fonts:
    sans = "Cantarell"
    fixed = "OperatorMonoLig Nerd Font"
    medium = 12
    large = 14
    web = 17

config.load_autoconfig(False)

# Key bindings
config.unbind("d")
config.unbind("J")
config.unbind("K")
config.bind("gc", "tab-close")
config.bind("g[", "tab-prev")
config.bind("g]", "tab-next")

config.set("auto_save.session", True)
config.set("completion.shrink", True)
config.set("confirm_quit", ["multiple-tabs", "downloads"])
config.set("content.notifications.enabled", True, "https://web.whatsapp.com/")
config.set(
    "content.tls.certificate_errors", "load-insecurely", "https://127.0.0.1:8384/"
)
config.set("editor.command", ["foot", "-e", "nvim", "{file}"])
config.set(
    "url.searchengines",
    {
        "DEFAULT": "https://duckduckgo.com/?q={}",
        "arch": "https://wiki.archlinux.org/?search={}",
        "crunch": "https://www.crunchbase.com/textsearch?q={}",
        "define": "https://dictionary.cambridge.org/search/direct/?datasetsearch=english&q={}",
    },
)

# Fonts
fonts = Fonts()

config.set("fonts.default_family", fonts.fixed)
config.set("fonts.default_size", f"{fonts.medium}pt")
config.set("fonts.tabs.selected", f"{fonts.large}pt {fonts.sans}")
config.set("fonts.tabs.unselected", f"{fonts.large}pt {fonts.sans}")
config.set("fonts.web.family.fixed", fonts.fixed)
config.set("fonts.web.family.sans_serif", fonts.sans)
config.set("fonts.web.family.serif", fonts.sans)
config.set("fonts.web.family.standard", fonts.sans)
config.set("fonts.web.size.default", fonts.web)
config.set("fonts.web.size.default_fixed", fonts.medium)

# Colors
colors = Colors()

config.set("colors.completion.fg", colors.white)
config.set("colors.completion.odd.bg", colors.black)
config.set("colors.completion.even.bg", colors.black)

config.set("colors.completion.category.fg", colors.magenta)
config.set("colors.completion.category.bg", colors.black)
config.set("colors.completion.category.border.top", colors.black)
config.set("colors.completion.category.border.bottom", colors.black)

config.set("colors.completion.item.selected.fg", colors.black)
config.set("colors.completion.item.selected.bg", colors.yellow)
config.set("colors.completion.item.selected.border.top", colors.yellow)
config.set("colors.completion.item.selected.border.bottom", colors.yellow)
config.set("colors.completion.item.selected.match.fg", colors.magenta)

config.set("colors.completion.match.fg", colors.magenta)
config.set("colors.completion.scrollbar.fg", colors.magenta)
config.set("colors.completion.scrollbar.bg", colors.black)

config.set("colors.contextmenu.disabled.fg", colors.light_black)
config.set("colors.contextmenu.disabled.bg", colors.black)
config.set("colors.contextmenu.menu.fg", colors.white)
config.set("colors.contextmenu.menu.bg", colors.black)
config.set("colors.contextmenu.selected.fg", colors.black)
config.set("colors.contextmenu.selected.bg", colors.blue)

config.set("colors.downloads.bar.bg", colors.black)
config.set("colors.downloads.start.fg", colors.black)
config.set("colors.downloads.start.bg", colors.magenta)
config.set("colors.downloads.stop.fg", colors.black)
config.set("colors.downloads.stop.bg", colors.blue)
config.set("colors.downloads.error.fg", colors.black)

config.set("hints.border", f"1px solid {colors.yellow}")
config.set("colors.hints.fg", colors.black)
config.set("colors.hints.bg", colors.yellow)
config.set("colors.hints.match.fg", colors.red)

config.set("colors.keyhint.suffix.fg", colors.magenta)
config.set("colors.keyhint.fg", colors.magenta)
config.set("colors.keyhint.bg", colors.black)

config.set("colors.messages.error.fg", colors.black)
config.set("colors.messages.error.bg", colors.red)
config.set("colors.messages.error.border", colors.red)

config.set("colors.messages.warning.fg", colors.black)
config.set("colors.messages.warning.bg", colors.yellow)
config.set("colors.messages.warning.border", colors.yellow)

config.set("colors.messages.info.fg", colors.magenta)
config.set("colors.messages.info.bg", colors.black)
config.set("colors.messages.info.border", colors.black)

config.set("colors.prompts.fg", colors.white)
config.set("colors.prompts.bg", colors.black)
config.set("colors.prompts.border", colors.black)
config.set("colors.prompts.selected.fg", colors.magenta)
config.set("colors.prompts.selected.bg", colors.green)

config.set("colors.statusbar.normal.fg", colors.magenta)
config.set("colors.statusbar.normal.bg", colors.black)
config.set("colors.statusbar.insert.fg", colors.blue)
config.set("colors.statusbar.insert.bg", colors.black)
config.set("colors.statusbar.passthrough.fg", colors.green)
config.set("colors.statusbar.passthrough.bg", colors.black)
config.set("colors.statusbar.private.fg", colors.cyan)
config.set("colors.statusbar.private.bg", colors.black)

config.set("colors.statusbar.command.fg", colors.white)
config.set("colors.statusbar.command.bg", colors.black)
config.set("colors.statusbar.command.private.fg", colors.cyan)
config.set("colors.statusbar.command.private.bg", colors.red)

config.set("colors.statusbar.caret.fg", colors.magenta)
config.set("colors.statusbar.caret.bg", colors.black)
config.set("colors.statusbar.caret.selection.fg", colors.magenta)
config.set("colors.statusbar.caret.selection.bg", colors.black)

config.set("colors.statusbar.progress.bg", colors.magenta)

config.set("colors.statusbar.url.fg", colors.magenta)
config.set("colors.statusbar.url.hover.fg", colors.green)
config.set("colors.statusbar.url.success.http.fg", colors.yellow)
config.set("colors.statusbar.url.success.https.fg", colors.white)
config.set("colors.statusbar.url.warn.fg", colors.yellow)
config.set("colors.statusbar.url.error.fg", colors.red)

config.set("colors.tabs.bar.bg", colors.black)

config.set("colors.tabs.indicator.start", colors.magenta)
config.set("colors.tabs.indicator.stop", colors.blue)
config.set("colors.tabs.indicator.error", colors.red)

config.set("colors.tabs.odd.fg", colors.white)
config.set("colors.tabs.odd.bg", colors.black)
config.set("colors.tabs.even.fg", colors.white)
config.set("colors.tabs.even.bg", colors.black)

config.set("colors.tabs.pinned.even.fg", colors.white)
config.set("colors.tabs.pinned.even.bg", colors.black)
config.set("colors.tabs.pinned.odd.fg", colors.white)
config.set("colors.tabs.pinned.odd.bg", colors.black)

config.set("colors.tabs.pinned.selected.even.fg", colors.black)
config.set("colors.tabs.pinned.selected.even.bg", colors.blue)
config.set("colors.tabs.pinned.selected.odd.fg", colors.black)
config.set("colors.tabs.pinned.selected.odd.bg", colors.blue)

config.set("colors.tabs.selected.odd.fg", colors.black)
config.set("colors.tabs.selected.odd.bg", colors.blue)
config.set("colors.tabs.selected.even.fg", colors.black)
config.set("colors.tabs.selected.even.bg", colors.blue)

config.set("colors.webpage.bg", colors.white)
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
from __future__ import unicode_literals
import json
import math
import sys
from rasterio import features
from affine import Affine

class OGRError(Exception):
    pass

def bbox_to_pixel_offsets(gt, bbox, rshape):
    originX = gt[0]
    originY = gt[3]
    pixel_width = gt[1]
    pixel_height = gt[5]

    col1 = int(math.floor((bbox[0] - originX) / pixel_width))
    col2 = int(math.ceil((bbox[2] - originX) / pixel_width))

    row1 = int(math.floor((bbox[3] - originY) / pixel_height))
    row2 = int(math.ceil((bbox[1] - originY) / pixel_height))

    # "Clip" the geometry bounds to the overall raster bounding box
    # This should avoid any rasterIO errors for partially overlapping polys
    if col1 < 0:
        col1 = 0
    if col2 > rshape[0]:
        col2 = rshape[0]
    if row1 < 0:
        row1 = 0
    if row2 > rshape[1]:
        row2 = rshape[1]

    cols = col2 - col1
    rows = row2 - row1

    return (col1, row1, cols, rows)

def pixel_offsets_to_window(offsets):
    """
    Convert (col1, row1, cols, rows)
    to a rasterio-compatible window
    https://github.com/mapbox/rasterio/blob/master/docs/windowed-rw.rst#windows
    """
    if len(offsets) != 4:
        raise ValueError("offset should be a 4-element tuple")
    col1, row1, cols, rows = offsets
    return ((row1, row1 + rows), (col1, col1 + cols))

def raster_extent_as_bounds(gt, shape):
    x1 = gt[0]
    x2 = gt[0] + (gt[1] * shape[0])
    y1 = gt[3] + (gt[5] * shape[1])
    y2 = gt[3]
    return (x1, y1, x2, y2)

def get_percentile(stat):
    if not stat.startswith('percentile_'):
        raise ValueError("must start with 'percentile_'")
    qstr = stat.replace("percentile_", '')
    q = float(qstr)
    if q > 100.0:
        raise ValueError('percentiles must be <= 100')
    if q < 0.0:
        raise ValueError('percentiles must be >= 0')
    return q

def feature_to_geojson(feature):
    """ This duplicates the feature.ExportToJson ogr method
    but is safe across gdal versions since it was fixed only in 1.8+
    see http://trac.osgeo.org/gdal/ticket/3870"""

    geom = feature.GetGeometryRef()
    if geom is not None:
        geom_json_string = geom.ExportToJson()
        geom_json_object = json.loads(geom_json_string)
    else:
        geom_json_object = None

    output = {
        'type': 'Feature',
        'geometry': geom_json_object,
        'properties': {}
    }

    fid = feature.GetFID()
    if fid:
        output['fid'] = fid

    for key in list(feature.keys()):
        output['properties'][key] = feature.GetField(key)

    return output

def parse_geo(thing):
    """ Given a python object, try to get a geo-json like mapping from it
    """
    from shapely.geos import ReadingError
    from shapely import wkt, wkb

    # object implementing geo_interface
    try:
        geo = thing.__geo_interface__
        return geo
    except AttributeError:
        pass

    # wkb
    try:
        shape = wkb.loads(thing)
        return shape.__geo_interface__
    except (ReadingError, TypeError):
        pass

    # wkt
    try:
        shape = wkt.loads(thing)
        return shape.__geo_interface__
    except (ReadingError, TypeError, AttributeError):
        pass

    # geojson-like python mapping
    valid_types = ["Feature", "Point", "LineString", "Polygon",
                   "MultiPoint", "MultiLineString", "MultiPolygon"]
    try:
        assert thing['type'] in valid_types
        return thing
    except (AssertionError, TypeError):
        pass

    # geojson string
    try:
        maybe_geo = json.loads(thing)
        assert maybe_geo['type'] in valid_types + ["FeatureCollection"]
        return maybe_geo
    except (ValueError, AssertionError, TypeError):
        pass

    raise ValueError("Can't parse %s as a geo-like object" % thing)

def get_ogr_ds(vds):
    from osgeo import ogr
    if not isinstance(vds, str):
        raise OGRError("OGR cannot open %r: not a string" % vds)

    ds = ogr.Open(vds)
    if not ds:
        raise OGRError("OGR cannot open %r" % vds)

    return ds

def ogr_srs(vector, layer_num):
    ds = get_ogr_ds(vector)
    layer = ds.GetLayer(layer_num)
    return layer.GetSpatialRef()

def ogr_records(vector, layer_num=0):
    ds = get_ogr_ds(vector)
    layer = ds.GetLayer(layer_num)
    if layer.GetFeatureCount() == 0:
        raise OGRError("No Features")
    feature = layer.GetNextFeature()
    while feature is not None:
        yield feature_to_geojson(feature)
        feature = layer.GetNextFeature()

def geo_records(vectors):
    for vector in vectors:
        yield parse_geo(vector)

def get_features(vectors, layer_num=0):
    if isinstance(vectors, str):
        try:
            # either an OGR layer ...
            get_ogr_ds(vectors)
            features_iter = ogr_records(vectors, layer_num)
            strategy = "ogr"
        except (OGRError, AttributeError):
            # ... or a single string to be parsed as wkt/wkb/json
            feat = parse_geo(vectors)
            features_iter = [feat]
            strategy = "single_geo"
    elif isinstance(vectors, bytes):
        # wkb
        feat = parse_geo(vectors)
        features_iter = [feat]
        strategy = "single_geo"
    elif hasattr(vectors, '__geo_interface__'):
        geotype = vectors.__geo_interface__['type']
        if geotype.lower() == 'featurecollection':
            # ... a featurecollection
            features_iter = geo_records(vectors.__geo_interface__['features'])
            strategy = "geo_featurecollection"
        else:
            # ... or an single object
            feat = parse_geo(vectors)
            features_iter = [feat]
            strategy = "single_geo"
    elif isinstance(vectors, dict):
        # ... or an python mapping
        feat = parse_geo(vectors)
        features_iter = [feat]
        strategy = "single_geo"
    else:
        # ... or an iterable of objects
        features_iter = geo_records(vectors)
        strategy = "iter_geo"

    return features_iter, strategy

def rasterize_geom(geom, src_offset, new_gt, all_touched):
    geoms = [(geom, 1)]
    affinetrans = Affine.from_gdal(*new_gt)
    rv_array = features.rasterize(
        geoms,
        out_shape=(src_offset[3], src_offset[2]),
        transform=affinetrans,
        fill=0,
        all_touched=all_touched)
    return rv_array

def is_nan(x):
    return isinstance(x, float) and math.isnan(x)

def combine_features_results(features, results, prefix, nan_to_None=True):
    """
    Given a list of geojson features and a list of zonal stats results
    Append the zonal stats to the feature's properties and yield a new feature
    """
    assert len(features) == len(results)
    for feat, res in zip(features, results):
        for key, val in res.items():
            if key == "__fid__":
                continue
            prefixed_key = "{}{}".format(prefix, key)

            # normalize
            if nan_to_None and is_nan(val):
                val = None

            feat['properties'][prefixed_key] = val
        yield feat

def stats_to_csv(stats):
    if sys.version_info[0] >= 3:
        from io import StringIO as IO
    else:
        from cStringIO import StringIO as IO
    import csv

    csv_fh = IO()

    keys = set()
    for stat in stats:
        for key in list(stat.keys()):
            keys.add(key)

    fieldnames = sorted(list(keys), key=str)

    csvwriter = csv.DictWriter(csv_fh, delimiter=str(","), fieldnames=fieldnames)
    csvwriter.writerow(dict((fn, fn) for fn in fieldnames))
    for row in stats:
        csvwriter.writerow(row)
    contents = csv_fh.getvalue()
    csv_fh.close()
    return contents
<commit_msg>no longer need ogr_srs<commit_after># -*- coding: utf-8 -*-
from __future__ import unicode_literals
import json
import math
import sys
from rasterio import features
from affine import Affine

class OGRError(Exception):
    pass

def bbox_to_pixel_offsets(gt, bbox, rshape):
    originX = gt[0]
    originY = gt[3]
    pixel_width = gt[1]
    pixel_height = gt[5]

    col1 = int(math.floor((bbox[0] - originX) / pixel_width))
    col2 = int(math.ceil((bbox[2] - originX) / pixel_width))

    row1 = int(math.floor((bbox[3] - originY) / pixel_height))
    row2 = int(math.ceil((bbox[1] - originY) / pixel_height))

    # "Clip" the geometry bounds to the overall raster bounding box
    # This should avoid any rasterIO errors for partially overlapping polys
    if col1 < 0:
        col1 = 0
    if col2 > rshape[0]:
        col2 = rshape[0]
    if row1 < 0:
        row1 = 0
    if row2 > rshape[1]:
        row2 = rshape[1]

    cols = col2 - col1
    rows = row2 - row1

    return (col1, row1, cols, rows)

def pixel_offsets_to_window(offsets):
    """
    Convert (col1, row1, cols, rows)
    to a rasterio-compatible window
    https://github.com/mapbox/rasterio/blob/master/docs/windowed-rw.rst#windows
    """
    if len(offsets) != 4:
        raise ValueError("offset should be a 4-element tuple")
    col1, row1, cols, rows = offsets
    return ((row1, row1 + rows), (col1, col1 + cols))

def raster_extent_as_bounds(gt, shape):
    x1 = gt[0]
    x2 = gt[0] + (gt[1] * shape[0])
    y1 = gt[3] + (gt[5] * shape[1])
    y2 = gt[3]
    return (x1, y1, x2, y2)

def get_percentile(stat):
    if not stat.startswith('percentile_'):
        raise ValueError("must start with 'percentile_'")
    qstr = stat.replace("percentile_", '')
    q = float(qstr)
    if q > 100.0:
        raise ValueError('percentiles must be <= 100')
    if q < 0.0:
        raise ValueError('percentiles must be >= 0')
    return q

def feature_to_geojson(feature):
    """ This duplicates the feature.ExportToJson ogr method
    but is safe across gdal versions since it was fixed only in 1.8+
    see http://trac.osgeo.org/gdal/ticket/3870"""

    geom = feature.GetGeometryRef()
    if geom is not None:
        geom_json_string = geom.ExportToJson()
        geom_json_object = json.loads(geom_json_string)
    else:
        geom_json_object = None

    output = {
        'type': 'Feature',
        'geometry': geom_json_object,
        'properties': {}
    }

    fid = feature.GetFID()
    if fid:
        output['fid'] = fid

    for key in list(feature.keys()):
        output['properties'][key] = feature.GetField(key)

    return output

def parse_geo(thing):
    """ Given a python object, try to get a geo-json like mapping from it
    """
    from shapely.geos import ReadingError
    from shapely import wkt, wkb

    # object implementing geo_interface
    try:
        geo = thing.__geo_interface__
        return geo
    except AttributeError:
        pass

    # wkb
    try:
        shape = wkb.loads(thing)
        return shape.__geo_interface__
    except (ReadingError, TypeError):
        pass

    # wkt
    try:
        shape = wkt.loads(thing)
        return shape.__geo_interface__
    except (ReadingError, TypeError, AttributeError):
        pass

    # geojson-like python mapping
    valid_types = ["Feature", "Point", "LineString", "Polygon",
                   "MultiPoint", "MultiLineString", "MultiPolygon"]
    try:
        assert thing['type'] in valid_types
        return thing
    except (AssertionError, TypeError):
        pass

    # geojson string
    try:
        maybe_geo = json.loads(thing)
        assert maybe_geo['type'] in valid_types + ["FeatureCollection"]
        return maybe_geo
    except (ValueError, AssertionError, TypeError):
        pass

    raise ValueError("Can't parse %s as a geo-like object" % thing)

def get_ogr_ds(vds):
    from osgeo import ogr
    if not isinstance(vds, str):
        raise OGRError("OGR cannot open %r: not a string" % vds)

    ds = ogr.Open(vds)
    if not ds:
        raise OGRError("OGR cannot open %r" % vds)

    return ds

def ogr_records(vector, layer_num=0):
    ds = get_ogr_ds(vector)
    layer = ds.GetLayer(layer_num)
    if layer.GetFeatureCount() == 0:
        raise OGRError("No Features")
    feature = layer.GetNextFeature()
    while feature is not None:
        yield feature_to_geojson(feature)
        feature = layer.GetNextFeature()

def geo_records(vectors):
    for vector in vectors:
        yield parse_geo(vector)

def get_features(vectors, layer_num=0):
    if isinstance(vectors, str):
        try:
            # either an OGR layer ...
            get_ogr_ds(vectors)
            features_iter = ogr_records(vectors, layer_num)
            strategy = "ogr"
        except (OGRError, AttributeError):
            # ... or a single string to be parsed as wkt/wkb/json
            feat = parse_geo(vectors)
            features_iter = [feat]
            strategy = "single_geo"
    elif isinstance(vectors, bytes):
        # wkb
        feat = parse_geo(vectors)
        features_iter = [feat]
        strategy = "single_geo"
    elif hasattr(vectors, '__geo_interface__'):
        geotype = vectors.__geo_interface__['type']
        if geotype.lower() == 'featurecollection':
            # ... a featurecollection
            features_iter = geo_records(vectors.__geo_interface__['features'])
            strategy = "geo_featurecollection"
        else:
            # ... or an single object
            feat = parse_geo(vectors)
            features_iter = [feat]
            strategy = "single_geo"
    elif isinstance(vectors, dict):
        # ... or an python mapping
        feat = parse_geo(vectors)
        features_iter = [feat]
        strategy = "single_geo"
    else:
        # ... or an iterable of objects
        features_iter = geo_records(vectors)
        strategy = "iter_geo"

    return features_iter, strategy

def rasterize_geom(geom, src_offset, new_gt, all_touched):
    geoms = [(geom, 1)]
    affinetrans = Affine.from_gdal(*new_gt)
    rv_array = features.rasterize(
        geoms,
        out_shape=(src_offset[3], src_offset[2]),
        transform=affinetrans,
        fill=0,
        all_touched=all_touched)
    return rv_array

def is_nan(x):
    return isinstance(x, float) and math.isnan(x)

def combine_features_results(features, results, prefix, nan_to_None=True):
    """
    Given a list of geojson features and a list of zonal stats results
    Append the zonal stats to the feature's properties and yield a new feature
    """
    assert len(features) == len(results)
    for feat, res in zip(features, results):
        for key, val in res.items():
            if key == "__fid__":
                continue
            prefixed_key = "{}{}".format(prefix, key)

            # normalize
            if nan_to_None and is_nan(val):
                val = None

            feat['properties'][prefixed_key] = val
        yield feat

def stats_to_csv(stats):
    if sys.version_info[0] >= 3:
        from io import StringIO as IO
    else:
        from cStringIO import StringIO as IO
    import csv

    csv_fh = IO()

    keys = set()
    for stat in stats:
        for key in list(stat.keys()):
            keys.add(key)

    fieldnames = sorted(list(keys), key=str)

    csvwriter = csv.DictWriter(csv_fh, delimiter=str(","), fieldnames=fieldnames)
    csvwriter.writerow(dict((fn, fn) for fn in fieldnames))
    for row in stats:
        csvwriter.writerow(row)
    contents = csv_fh.getvalue()
    csv_fh.close()
    return contents
<|endoftext|>


<commit_before>import logging, os
from glob import glob

from send2trash import send2trash

from .dbguy import TVDatabase
from .dirscanner import get_episodes, get_file_from_single_ep_dir, is_rar
from .parser import reverse_parse_episode
from .texceptions import IncompleteEpisodeError
from .texceptions import ShowNotFoundError, SeasonNotFoundError, EpisodeNotFoundError
from .texceptions import InvalidDirectoryError
from .tvdbwrapper import lookup
from .quality import quality_battle
from .unrarman import unrar_file
from . import appconfig as cfg
from . import __pkgname__

log = logging.getLogger(__pkgname__)

class Importer(object):

    scrape_errors = (
        ShowNotFoundError,
        SeasonNotFoundError,
        EpisodeNotFoundError,
        IncompleteEpisodeError
        )

    def __init__(self, directory, **kwargs):
        #TODO: take options from cfg and do update and reset functions
        """
        the kwargs must be the command line options thingies, right..?
        """
        self.db = TVDatabase(directory)
        self.directory = directory
        #Episodes which weren't found on the web
        self._not_found = []
        self.scraped_count = 0 #deprecate
        self.success_count = 0

    def start_import(self):
        if self.db.db_file_exists():
            if cfg.get('database', 'reset', bool):
                os.unlink(self.db.dbfile)
                self.db.create_database(
                    force=True
                    )
        else:
            self.db.create_database()
        self.wrap()

    def episodes(self):
        """
        Yield Episodes in directory.
        Episodes returned from here are result of ez_parse_episode.
        """
        for ep in get_episodes(self.directory):
            yield ep

    def _scrape_episode(self, episode):
        """
        scrape_episode(Episode) -> Episode
        Fully parse it, look it up, fill it with info and return.
        Ideally results in an Episode with full info, ready for the database.
        Can raise ShowNotFoundError, SeasonNotFoundError
        or EpisodeNotFoundError
        """
        #TODO: be smarter (all eps in same dir
        #should be the same series, right, unless
        #there's a mix thing like incoming)
        if not episode.is_fully_parsed():
            episode = reverse_parse_episode(episode['file_path'], self.directory)
        scrapedep = lookup(episode)
        return scrapedep

    def _fallback_scrape_episode(self, path):
        """
        hard_scrape_path(path) -> Episode
        Do a reverse_parse and scrape given path.
        """
        ep = reverse_parse_episode(path, self.directory)
        return self._scrape_episode(ep)

    def fill_episode(self, ep):
        """
        fill_episode(Episode) -> Episode
        Parses and looks up given episode info and returns.
        May raise exceptions in self.scrape_errors.
        """
        fileindb = self.db.path_exists(ep['file_path'])
        try:
            ep = self._scrape_episode(ep)
        except self.scrape_errors as e:
            log.debug(
                'Failed lookup for episode %s.\nMessage:%s',
                ep,
                e.message
                )
            ep = self._fallback_scrape_episode(ep['file_path'])
        return ep

    def _wrap_single(self, ep):
        """
        _wrap_single(Episode)
        Wrapper method to handle single episode, from filename to db.
        """
        fileindb = self.db.path_exists(ep['file_path'])
        update = cfg.get('database', 'update', bool)
        if fileindb and update:
            #don't need to scrape cause we ain't gunna do nuthin'
            log.debug(
                '"%s" already in database and update option set, skipping',
                ep['file_path']
                )
            return
        try:
            ep = self.fill_episode(ep)
        except self.scrape_errors as e:
            self._not_found.append(ep)
            self.db.add_unparsed_child(
                os.path.relpath(
                    ep['file_path'],
                    self.db.directory)
                )
            return
        if fileindb:
            return self.db.upsert_episode(ep) #we update it

        idindb = self.db.episode_exists(ep)
        if idindb:
            oldep = self.db.get_episodes(
                'WHERE id = ?', params=(ep['id'],)).next()
            log.info(
                'Found duplicates. Original: "%s". Contender: "%s".',
                oldep['file_path'],
                ep['file_path']
                )
            #can't be having same episode twice, thems ids be primary keys, yo
            better = quality_battle(ep, oldep, self.db.directory)
            if better: bpath = better['file_path']
            else: bpath = None
            log.info(
                'Quality battle between "%s" and "%s". "%s" wins.',
                ep['file_path'], oldep['file_path'], None
                )
            if not better:
                #neither is better, it ain't no thang, do nuthin'
                return
            elif better is oldep:
                return
            else:
                log.info('Replacing "%s" with "%s" in db.',
                         oldep['file_path'], ep['file_path'])
                return self.db.upsert_episode(ep)
        else:
            return self.db.upsert_episode(ep)

    def trash_rars_in_dir(self, directory):
        """
        trash_rars_in_dir(directory)
        Send rar files in given directory to trash.
        """
        log.info('Sending rar files in "%s" to trash.', directory)
        rnfiles = glob(
            os.path.join(directory, '*.r[0-9][0-9]'))
        rarfiles = glob(
            os.path.join(directory, '*.rar'))
        for f in rnfiles+rarfiles:
            send2trash(f)

    def unrar_episode(self, ep, out_dir=None):
        """
        unrar_episode(Episode)
        """
        p = os.path.join(
            self.db.directory,
            ep['file_path']
            )
        if not os.path.isdir(ep['file_path']):
            raise InvalidDirectoryError(
                'Episode path must be a directory. "%s" is not.' % p
                )
        log.info('Extracting "%s" from rar files.', p)
        unrar_file(p, out_dir=out_dir)
        #get new path to episode

        ep['file_path'] = get_file_from_single_ep_dir(p)
        delr = cfg.get('importer', 'delete-rar', bool)
        if delr:
            self.trash_rars_in_dir(p)
        return ep

    def wrap(self):
        for ep in self.episodes():
            if cfg.get('importer', 'unrar', bool) and is_rar(ep['file_path']):
                ep = self.unrar_episode(ep)
            res = self._wrap_single(ep)
            if isinstance(res, int):
                self.success_count+=1
        log.warning(
            '%s episodes were scraped and added to the database.',
            self.success_count
            )
        log.warning(
            '%s "episodes" were not fully parsed or not found the tvdb',
            len(self._not_found)
            )

class DifficultEpisodeHandler(object):
    """
    Handler of difficult episodes which can't be scraped with ordinary methods.
    """
    pass
<commit_msg>fix crash when oldep does not exist anymore in quality_battle<commit_after>import logging, os
from glob import glob

from send2trash import send2trash

from .dbguy import TVDatabase
from .dirscanner import get_episodes, get_file_from_single_ep_dir, is_rar
from .parser import reverse_parse_episode
from .texceptions import IncompleteEpisodeError
from .texceptions import ShowNotFoundError, SeasonNotFoundError, EpisodeNotFoundError
from .texceptions import InvalidDirectoryError
from .tvdbwrapper import lookup
from .quality import quality_battle
from .unrarman import unrar_file
from . import appconfig as cfg
from . import __pkgname__

log = logging.getLogger(__pkgname__)

class Importer(object):

    scrape_errors = (
        ShowNotFoundError,
        SeasonNotFoundError,
        EpisodeNotFoundError,
        IncompleteEpisodeError
        )

    def __init__(self, directory, **kwargs):
        #TODO: take options from cfg and do update and reset functions
        """
        the kwargs must be the command line options thingies, right..?
        """
        self.db = TVDatabase(directory)
        self.directory = directory
        #Episodes which weren't found on the web
        self._not_found = []
        self.scraped_count = 0 #deprecate
        self.success_count = 0

    def start_import(self):
        if self.db.db_file_exists():
            if cfg.get('database', 'reset', bool):
                os.unlink(self.db.dbfile)
                self.db.create_database(
                    force=True
                    )
        else:
            self.db.create_database()
        self.wrap()

    def episodes(self):
        """
        Yield Episodes in directory.
        Episodes returned from here are result of ez_parse_episode.
        """
        for ep in get_episodes(self.directory):
            yield ep

    def _scrape_episode(self, episode):
        """
        scrape_episode(Episode) -> Episode
        Fully parse it, look it up, fill it with info and return.
        Ideally results in an Episode with full info, ready for the database.
        Can raise ShowNotFoundError, SeasonNotFoundError
        or EpisodeNotFoundError
        """
        #TODO: be smarter (all eps in same dir
        #should be the same series, right, unless
        #there's a mix thing like incoming)
        if not episode.is_fully_parsed():
            episode = reverse_parse_episode(episode['file_path'], self.directory)
        scrapedep = lookup(episode)
        return scrapedep

    def _fallback_scrape_episode(self, path):
        """
        hard_scrape_path(path) -> Episode
        Do a reverse_parse and scrape given path.
        """
        ep = reverse_parse_episode(path, self.directory)
        return self._scrape_episode(ep)

    def fill_episode(self, ep):
        """
        fill_episode(Episode) -> Episode
        Parses and looks up given episode info and returns.
        May raise exceptions in self.scrape_errors.
        """
        fileindb = self.db.path_exists(ep['file_path'])
        try:
            ep = self._scrape_episode(ep)
        except self.scrape_errors as e:
            log.debug(
                'Failed lookup for episode %s.\nMessage:%s',
                ep,
                e.message
                )
            ep = self._fallback_scrape_episode(ep['file_path'])
        return ep

    def full_path(self, path):
        """
        full_path(path) -> db.directory + path
        Gets full path concatenated with source directory.
        convenience function.
        """
        return os.path.join(self.db.directory, path)

    def _wrap_single(self, ep):
        """
        _wrap_single(Episode)
        Wrapper method to handle single episode, from filename to db.
        """
        fileindb = self.db.path_exists(ep['file_path'])
        update = cfg.get('database', 'update', bool)
        if fileindb and update:
            #don't need to scrape cause we ain't gunna do nuthin'
            log.debug(
                '"%s" already in database and update option set, skipping',
                ep['file_path']
                )
            return
        try:
            ep = self.fill_episode(ep)
        except self.scrape_errors as e:
            self._not_found.append(ep)
            self.db.add_unparsed_child(
                os.path.relpath(
                    ep['file_path'],
                    self.db.directory)
                )
            return
        if fileindb:
            return self.db.upsert_episode(ep) #we update it

        idindb = self.db.episode_exists(ep)
        if idindb:
            oldep = self.db.get_episodes(
                'WHERE id = ?', params=(ep['id'],)).next()
            log.info(
                'Found duplicates. Original: "%s". Contender: "%s".',
                oldep['file_path'],
                ep['file_path']
                )
            if not os.path.exists(self.full_path(oldep['file_path'])):
                log.info(
                    'Original: "%s" does not exist anymore".'\
                    +' Replacing with contender: "%s".',
                    oldep['file_path'],
                    ep['file_path']
                    )
                return self.db.upsert_episode(ep)
            #can't be having same episode twice, thems ids be primary keys, yo
            better = quality_battle(ep, oldep, self.db.directory)
            if better: bpath = better['file_path']
            else: bpath = None
            log.info(
                'Quality battle between "%s" and "%s". "%s" wins.',
                ep['file_path'], oldep['file_path'], None
                )
            if not better:
                #neither is better, it ain't no thang, do nuthin'
                return
            elif better is oldep:
                return
            else:
                log.info('Replacing "%s" with "%s" in db.',
                         oldep['file_path'], ep['file_path'])
                return self.db.upsert_episode(ep)
        else:
            return self.db.upsert_episode(ep)

    def trash_rars_in_dir(self, directory):
        """
        trash_rars_in_dir(directory)
        Send rar files in given directory to trash.
        """
        log.info('Sending rar files in "%s" to trash.', directory)
        rnfiles = glob(
            os.path.join(directory, '*.r[0-9][0-9]'))
        rarfiles = glob(
            os.path.join(directory, '*.rar'))
        for f in rnfiles+rarfiles:
            send2trash(f)

    def unrar_episode(self, ep, out_dir=None):
        """
        unrar_episode(Episode)
        """
        p = os.path.join(
            self.db.directory,
            ep['file_path']
            )
        if not os.path.isdir(ep['file_path']):
            raise InvalidDirectoryError(
                'Episode path must be a directory. "%s" is not.' % p
                )
        log.info('Extracting "%s" from rar files.', p)
        unrar_file(p, out_dir=out_dir)
        #get new path to episode

        ep['file_path'] = get_file_from_single_ep_dir(p)
        delr = cfg.get('importer', 'delete-rar', bool)
        if delr:
            self.trash_rars_in_dir(p)
        return ep

    def wrap(self):
        for ep in self.episodes():
            if cfg.get('importer', 'unrar', bool) and is_rar(ep['file_path']):
                ep = self.unrar_episode(ep)
            res = self._wrap_single(ep)
            if isinstance(res, int):
                self.success_count+=1
        log.warning(
            '%s episodes were scraped and added to the database.',
            self.success_count
            )
        log.warning(
            '%s "episodes" were not fully parsed or not found the tvdb',
            len(self._not_found)
            )

class DifficultEpisodeHandler(object):
    """
    Handler of difficult episodes which can't be scraped with ordinary methods.
    """
    pass
<|endoftext|>