<commit_before># -*- coding: utf-8 -*-
__author__ = 'Lukasz Augustyniak'

import memory_profiler
import logging
import pickle
import sys
from os.path import join, basename
from glob import glob
from datetime import datetime
import time
from pprint import pprint
import pandas as pd
from textlytics.processing.sentiment.document_preprocessing import \
    DocumentPreprocessor
from textlytics.processing.sentiment.io_sentiment import results_to_pickle
from textlytics.processing.sentiment.sentiment import Sentiment
from textlytics.processing.sentiment.lexicons import SentimentLexicons
from textlytics.utils import LEXICONS_PATH

log = logging.getLogger()
log.setLevel(logging.DEBUG)

ch = logging.StreamHandler(sys.stdout)
ch.setLevel(logging.DEBUG)
formatter = logging.Formatter(
    '%(asctime)s - %(name)s - %(levelname)s - %(message)s')
ch.setFormatter(formatter)
log.addHandler(ch)

@memory_profiler.profile
def lexicons_amazon(base_path='/home/engine/csv/', dataset_filter='',
                    lexicons=None, n_reviews=2000, train=False,
                    norm_freq=False):
    # ############################# LEXICONS ##################################
    results = {}

    # manually created lexicons
    lexicons_files = [
        'AFINN-96.txt',
        'AFINN-111.txt',
        'Bing-Liu.txt',
        'enchantedlearning.com.txt',
        'past_future_list.txt',
        'past_future_list_plus.txt',
        'simple_list.txt',
        'simple_list_plus.txt',
        'simplest.txt'
    ]

    # auto generated lexicons - frequentiment based
    cat_lex_paths = glob(join(LEXICONS_PATH, '*.csv'))
    if norm_freq:
        cat_lex_names = [basename(x) for x in cat_lex_paths]
    else:
        cat_lex_names = [basename(x) for x in cat_lex_paths if
                         'normalized' not in x]

    # get all datasets and cv
    train_test_path = join(base_path, 'train_test_subsets')
    datasets = glob(join(base_path, '*%s*.txt.gz.csv' % dataset_filter))
    print 'Datasets:'
    pprint(datasets)

    for dataset in datasets:
        print dataset
        lexicons_frequentiment = []
        dataset_file_name = basename(dataset)
        dataset_name = dataset_file_name.split('.')[0]
        log.info('Dataset name: %s' % dataset_name)
        dp = DocumentPreprocessor()
        df = pd.DataFrame.from_csv(dataset, sep=';', index_col=False)
        df, _ = dp.star_score_to_sentiment(df,
                                           score_column='review/score')
        # extract only Document and Sentiment columns
        df['Document'] = df['review/text']
        df = df[['Document', 'Sentiment']]

        # get lexicons only for specific category of chosen dataset
        # add them to list of all lexicons in experiment
        for cl in cat_lex_names:
            if cl.split('-')[1] in dataset_name.lower():
                lexicons_frequentiment.append(cl)

        try:
            # load train/test sets folds
            f_path = join(train_test_path, 'train-test-%s-%s.pkl' % (
                n_reviews, dataset_file_name))
            with open(f_path, 'rb') as f:
                train_test_indexes = pickle.load(f)
                log.info('Pickle has been loaded, %s' % f_path)

            # results = []
            results[dataset_name] = []
            for index, cv in enumerate(train_test_indexes[:1]):
                print index, datetime.now()
                lexicons = [x for x in lexicons_frequentiment if
                            'cv-%s' % index in x]
                lexicons.extend(lexicons_files)
                pprint(lexicons)
                log.info('%s fold from cv has been started!' % index)
                print '%sth fold %s' % (index, datetime.now())
                # start computations
                if train:
                    ind = cv[0]
                    f_n = 'train-%s-fold-%s' % (dataset_name, index)
                else:
                    ind = cv[1]
                    f_n = 'test-%s-fold-%s' % (dataset_name, index)
                s = Sentiment()
                df_lex, lexicon_prediction, lexicon_result, classes = \
                    s.lexicon_based_sentiment_simplified(
                        dataset=df.ix[ind],
                        lexs_files=lexicons,
                        words_stem=False,
                        dataset_name=f_n)
                results[dataset_name].append(lexicon_result)
        except IOError as err:
            log.error('%s not loaded' % dataset_name)
            raise IOError(str(err))
    pprint(results)
    results_to_pickle(dataset='', f_name='all-lex-train', obj=results)

def sentiment_lexicons_amazon_cv(datasets_path='', dataset_filter=None,
                                 lexs_files=None, n_reviews=2000, train=False,
                                 norm_freq=False, lex_path=None, f_name='',
                                 n_cv=10, stars=None,
                                 frequentiment_lexicons=False):
    """
    Counting sentiment analysis tasks for lexicon-based approach.
    :param datasets_path: path to the datasets directory, it must contain
        folder /train_test_subsets with cross-validation information - data
        frame indexes for each fold. Datasets are in csv files - converted from
        Web Amazon dataset structure.
    :param dataset_filter: list of substring for choosing datasets
    :param lexs_files: list of path/file names for lexicons loading
    :param n_reviews: number of reviews from each star score
    :param train: if True you are counting sentiment for train subsets,
        otherwise counting sentiment for testing subsets
    :param norm_freq: tuple, i.e., (-1, 1) fot threshold cutting, lower than
        first value of tuples will be negative, between -1 and 1 will be neutral
        more than 1 will be positive
    :param lex_path: path to the directory with lexicon's files
    :param f_name: additional part of output files name (results, predictions)
    :param n_cv: number of Cross-Validation's folds to performed
    :param stars: star scores that will be used in experiment, as default all
    :return: nothing, all necessary files will be saved
    """
    results = {}
    predictions = {}
    datasets = []

    lexs_names = [x.split('.')[0] for x in lexs_files]
    train_test_path = join(datasets_path, 'train_test_subsets')

    # get all datasets and cv
    if dataset_filter is not None:
        for df in dataset_filter:
            datasets.extend(glob(join(datasets_path,
                                      '%s*.txt.gz.csv' % df)))
    else:
        datasets = glob(join(datasets_path, '*.txt.gz.csv'))
    pprint(datasets)

    # if frequentiment_lexicons:

    for dataset in datasets:
        dataset_file_name = basename(dataset)
        dataset_name = dataset_file_name.split('.')[0]
        log.info('Dataset name: %s' % dataset_name)
        dp = DocumentPreprocessor()
        df = pd.DataFrame.from_csv(dataset, sep=';', index_col=False)
        if stars is not None:
            df = df[df['review/score'].isin(stars)]
        df, _ = dp.star_score_to_sentiment(df, score_column='review/score')
        # extract only Document and Sentiment columns
        df['Document'] = df['review/text']
        df = df[['Document', 'Sentiment']]

        try:
            # load train/test sets folds
            f_path = join(train_test_path, 'train-test-%s-%s.pkl'
                                           '' % (
                              n_reviews, dataset_file_name))
            with open(f_path, 'rb') as f:
                train_test_indexes = pickle.load(f)
                log.info('Pickle has been loaded, %s' % f_path)

            results[dataset_name] = []
            predictions[dataset_name] = []

            for index, cv in enumerate(train_test_indexes[:n_cv]):
                log.info('CV: %s/%s ' % (index + 1, n_cv))
                # lexicons = [x for x in lexicons_frequentiment if
                #             'cv-%s' % index in x]
                # lexicons.extend(lexicons_files)
                if stars is not None:
                    cv = (set(cv[0]).intersection(df.index.values),
                          set(cv[1]).intersection(df.index.values))

                if train:
                    ind = cv[0]
                    f_name = 'train-%s-fold-%s-%s' % (dataset_name, index,
                                                      n_cv - 1)
                else:
                    ind = cv[1]
                    f_name = 'test-%s-fold-%s-%s' % (dataset_name, index,
                                                     n_cv - 1)
                s = Sentiment()
                df_lex, lexicon_prediction, lexicon_result, classes = \
                    s.lexicon_based_sentiment_simplified(
                        dataset=df.ix[ind],
                        lexs_files=lexs_files,
                        words_stem=False,
                        dataset_name=dataset_name,
                        lex_path=lex_path)
                results[dataset_name].append(lexicon_result)
                predictions[dataset_name].append(lexicon_prediction)

                # save predictions
                results_to_pickle(dataset=dataset_name,
                                  f_name='Part-predictions-%s' % f_name,
                                  obj=lexicon_prediction)
        except IOError as err:
            log.error('%s not loaded' % dataset_name)
            raise IOError(str(err))

    # saving predictions and results
    logging.info(results)
    results_to_pickle(dataset='',
                      f_name='Results-%s' % f_name,
                      obj=results)
    results_to_pickle(dataset='',
                      f_name='Predictions-%s' % f_name,
                      obj=predictions)

# manually created lexicons
lexicons_files = [
    'AFINN-96.txt',
    'AFINN-111.txt',
    'Bing-Liu.txt',
    'enchantedlearning.com.txt',
    'past_future_list.txt',
    'past_future_list_plus.txt',
    'simple_list.txt',
    'simple_list_plus.txt',
    'simplest.txt',
    'nrcEmotion.txt',
    'mpaa.txt',
    'nrcHashtag.txt',
    'nrcHashtagBigrams.txt',
    'sentiment140.txt',
    'sentiment140Bigrams.txt',
    'MSOL-lexicon.txt',
    'Amazon-laptops-electronics-reviews-unigrams.txt',
    'Amazon-laptops-electronics-reviews-bigrams.txt',
    'Yelp-restaurant-reviews-unigrams.txt',
    'Yelp-restaurant-reviews-bigrams.txt',
]

sentiment_lexicons_amazon_cv(lexs_files=lexicons_files,
                             lex_path='/home/engine/cn-data/lexicons/',
                             # lex_path='C:/GitHub/cn-data/lexicons/',
                             dataset_filter=['Automotive'],
                             # dataset_filter=['Automotive', 'Books'],
                             datasets_path='/datasets/amazon-data/csv/',
                             # datasets_path='C:/Datasets/amazon-data/csv/',
                             # n_reviews=20,
                             f_name='all-folds-w2v-{}'.format(
                                 time.strftime("%Y-%m-%d_%H-%M-%S")),
                             # f_name='____',
                             # train=True,
                             n_cv=10,
                             # stars=[1, 3, 5]
                             # stars=[1, 5],
                             # frequentiment_lexicons=True
                             )
<commit_msg>multiprocessing for lexicon-based approach<commit_after># -*- coding: utf-8 -*-
__author__ = 'Lukasz Augustyniak'

import multiprocessing
import memory_profiler
import logging
import pickle
import sys
from os.path import join, basename, exists
from os import makedirs
from glob import glob
from datetime import datetime
import time
from pprint import pprint
import pandas as pd
from textlytics.processing.sentiment.document_preprocessing import \
    DocumentPreprocessor
from textlytics.processing.sentiment.io_sentiment import results_to_pickle, to_pickle
from textlytics.processing.sentiment.sentiment import Sentiment
from textlytics.processing.sentiment.lexicons import SentimentLexicons
from textlytics.utils import LEXICONS_PATH

log = logging.getLogger()
log.setLevel(logging.DEBUG)

ch = logging.StreamHandler(sys.stdout)
ch.setLevel(logging.DEBUG)
formatter = logging.Formatter(
    '%(asctime)s - %(name)s - %(levelname)s - %(message)s')
ch.setFormatter(formatter)
log.addHandler(ch)

@memory_profiler.profile
def lexicons_amazon(base_path='/home/engine/csv/', dataset_filter='',
                    lexicons=None, n_reviews=2000, train=False,
                    norm_freq=False):
    # ############################# LEXICONS ##################################
    results = {}

    # manually created lexicons
    lexicons_files = [
        'AFINN-96.txt',
        'AFINN-111.txt',
        'Bing-Liu.txt',
        'enchantedlearning.com.txt',
        'past_future_list.txt',
        'past_future_list_plus.txt',
        'simple_list.txt',
        'simple_list_plus.txt',
        'simplest.txt'
    ]

    # auto generated lexicons - frequentiment based
    cat_lex_paths = glob(join(LEXICONS_PATH, '*.csv'))
    if norm_freq:
        cat_lex_names = [basename(x) for x in cat_lex_paths]
    else:
        cat_lex_names = [basename(x) for x in cat_lex_paths if
                         'normalized' not in x]

    # get all datasets and cv
    train_test_path = join(base_path, 'train_test_subsets')
    datasets = glob(join(base_path, '*%s*.txt.gz.csv' % dataset_filter))
    print 'Datasets:'
    pprint(datasets)

    for dataset in datasets:
        print dataset
        lexicons_frequentiment = []
        dataset_file_name = basename(dataset)
        dataset_name = dataset_file_name.split('.')[0]
        log.info('Dataset name: %s' % dataset_name)
        dp = DocumentPreprocessor()
        df = pd.DataFrame.from_csv(dataset, sep=';', index_col=False)
        df, _ = dp.star_score_to_sentiment(df,
                                           score_column='review/score')
        # extract only Document and Sentiment columns
        df['Document'] = df['review/text']
        df = df[['Document', 'Sentiment']]

        # get lexicons only for specific category of chosen dataset
        # add them to list of all lexicons in experiment
        for cl in cat_lex_names:
            if cl.split('-')[1] in dataset_name.lower():
                lexicons_frequentiment.append(cl)

        try:
            # load train/test sets folds
            f_path = join(train_test_path, 'train-test-%s-%s.pkl' % (
                n_reviews, dataset_file_name))
            with open(f_path, 'rb') as f:
                train_test_indexes = pickle.load(f)
                log.info('Pickle has been loaded, %s' % f_path)

            # results = []
            results[dataset_name] = []
            for index, cv in enumerate(train_test_indexes[:1]):
                print index, datetime.now()
                lexicons = [x for x in lexicons_frequentiment if
                            'cv-%s' % index in x]
                lexicons.extend(lexicons_files)
                pprint(lexicons)
                log.info('%s fold from cv has been started!' % index)
                print '%sth fold %s' % (index, datetime.now())
                # start computations
                if train:
                    ind = cv[0]
                    f_n = 'train-%s-fold-%s' % (dataset_name, index)
                else:
                    ind = cv[1]
                    f_n = 'test-%s-fold-%s' % (dataset_name, index)
                s = Sentiment()
                df_lex, lexicon_prediction, lexicon_result, classes = \
                    s.lexicon_based_sentiment_simplified(
                        dataset=df.ix[ind],
                        lexs_files=lexicons,
                        words_stem=False,
                        dataset_name=f_n)
                results[dataset_name].append(lexicon_result)
        except IOError as err:
            log.error('%s not loaded' % dataset_name)
            raise IOError(str(err))
    pprint(results)
    results_to_pickle(dataset='', f_name='all-lex-train', obj=results)

def sentiment_lexicons_amazon_cv(datasets_path='', dataset_filter=None,
                                 lexs_names=None, n_reviews=2000, train=False,
                                 norm_freq=False, lex_path=None, f_name='',
                                 n_cv=10, stars=None,
                                 frequentiment_lexicons='',
                                 output_folder=None,
                                 evaluate=True):
    """
    Counting sentiment analysis tasks for lexicon-based approach.
    :param datasets_path: path to the datasets directory, it must contain
        folder /train_test_subsets with cross-validation information - data
        frame indexes for each fold. Datasets are in csv files - converted from
        Web Amazon dataset structure.
    :param dataset_filter: list of substring for choosing datasets
    :param lexs_names: list of path/file names for lexicons loading
    :param n_reviews: number of reviews from each star score
    :param train: if True you are counting sentiment for train subsets,
        otherwise counting sentiment for testing subsets
    :param norm_freq: tuple, i.e., (-1, 1) fot threshold cutting, lower than
        first value of tuples will be negative, between -1 and 1 will be neutral
        more than 1 will be positive
    :param lex_path: path to the directory with lexicon's files
    :param f_name: additional part of output files name (results, predictions)
    :param n_cv: number of Cross-Validation's folds to performed
    :param stars: star scores that will be used in experiment, as default all
    :return: nothing, all necessary files will be saved
    """
    results = {}
    predictions = {}
    datasets = []
    predictions_directory = join(output_folder, 'predictions')

    if not exists(output_folder):
        makedirs(output_folder)
        log.info('New directory has been created in: {}'.format(output_folder))

    if not exists(predictions_directory):
        makedirs(predictions_directory)
        log.info('Directory for predictions has been created: {}'.format(predictions_directory))

    # lexs_names = [x.split('.')[0] for x in lexs_files]
    train_test_path = join(datasets_path, 'train_test_subsets')

    # get all datasets and cv
    if dataset_filter is not None:
        for df in dataset_filter:
            datasets.extend(glob(join(datasets_path, '%s*.txt.gz.csv' % df)))
    else:
        datasets = glob(join(datasets_path, '*.txt.gz.csv'))
    log.debug('Datasets to process: {}'.format(datasets))

    if frequentiment_lexicons:
        log.debug(
            'Freq lexs path: {}'.format(join(frequentiment_lexicons, '*.csv')))
        freq_lexs = glob(join(frequentiment_lexicons, '*.csv'))
    else:
        freq_lexs = []

    for dataset in datasets:
        dataset_file_name = basename(dataset)
        dataset_name = dataset_file_name.split('.')[0]
        log.info('Dataset name: %s' % dataset_name)
        dp = DocumentPreprocessor()
        df = pd.DataFrame.from_csv(dataset, sep=';', index_col=False)
        if stars is not None:
            df = df[df['review/score'].isin(stars)]
        df, _ = dp.star_score_to_sentiment(df, score_column='review/score')
        # extract only Document and Sentiment columns
        df['Document'] = df['review/text']
        # df['Document'] = df['review/summary']
        df = df[['Document', 'Sentiment']]

        try:
            # load train/test sets folds
            f_path = join(train_test_path, 'train-test-%s-%s.pkl' % (n_reviews, dataset_file_name))
            with open(f_path, 'rb') as f:
                train_test_indexes = pickle.load(f)
                log.info('Pickle has been loaded, %s' % f_path)

            results[dataset_name] = []
            predictions[dataset_name] = []

            for cv_idx, cv in enumerate(train_test_indexes[:n_cv]):
                log.info('Start for {}: CV: {}/{} '.format(dataset_name, cv_idx + 1, n_cv))
                freq_lexs_ = [basename(x) for x in freq_lexs if '{}-{}'.format(dataset_name, cv_idx) in x]
                log.info('Dataset: {}, CV: {} => frequentiment lexicons: {}'.format(dataset_name, cv_idx, freq_lexs_))
                lexs_names.extend(freq_lexs_)
                if stars is not None:
                    cv = (set(cv[0]).intersection(df.index.values),
                          set(cv[1]).intersection(df.index.values))

                if train:
                    ind = cv[0]
                    f_name = 'train-%s-fold-%s' % (dataset_name, cv_idx)
                else:
                    ind = cv[1]
                    f_name = 'test-%s-fold-%s' % (dataset_name, cv_idx)

                s = Sentiment()
                df_lex, lexicon_prediction, lexicon_result, classes = \
                    s.lexicon_based_sentiment_simplified(
                        dataset=df.ix[ind],
                        lexs_files=lexs_names,
                        words_stem=False,
                        dataset_name=dataset_name,
                        lex_path=lex_path,
                        evaluate=evaluate)
                results[dataset_name].append(lexicon_result)
                predictions[dataset_name].append(lexicon_prediction)

                to_pickle(p=output_folder, dataset='', f_name=f_name, obj=lexicon_prediction)
                # df_lex.to_excel(join(output_folder, 'predictions', 'predictions-%s.xls' % f_name))
                df_lex.to_pickle(join(output_folder, 'predictions', 'predictions-%s.pkl' % f_name))

                # save predictions
                # results_to_pickle(dataset=dataset_name,
                #                   f_name='Part-predictions-%s' % f_name,
                #                   obj=lexicon_prediction)
        except IOError as err:
            log.error('%s not loaded' % dataset_name)
            raise IOError(str(err))

    # saving predictions and results
    # logging.info(results)

    to_pickle(p=output_folder, dataset='', f_name='Results', obj=results)
    to_pickle(p=output_folder, dataset='', f_name='Predictions', obj=predictions)
    # results_to_pickle(dataset='',
    #                   f_name='Results-%s' % f_name,
    #                   obj=results)
    # results_to_pickle(dataset='',
    #                   f_name='Predictions-%s' % f_name,
    #                   obj=predictions)

# manually created lexicons
lexicons_files = [
    'AFINN-96.txt',
    'AFINN-111.txt',
    'Bing-Liu.txt',
    'enchantedlearning.com.txt',
    'past_future_list.txt',
    'past_future_list_plus.txt',
    'simple_list.txt',
    'simple_list_plus.txt',
    'simplest.txt',
    # 'nrcEmotion.txt',
    # 'mpaa.txt',
    # 'nrcHashtag.txt',
    # 'nrcHashtagBigrams.txt',
    # 'sentiment140.txt',
    # 'sentiment140Bigrams.txt',
    # 'MSOL-lexicon.txt',
    # 'Amazon-laptops-electronics-reviews-unigrams.txt',
    # 'Amazon-laptops-electronics-reviews-bigrams.txt',
    # 'Yelp-restaurant-reviews-unigrams.txt',
    # 'Yelp-restaurant-reviews-bigrams.txt',
]

def run_multi(d):
    sentiment_lexicons_amazon_cv(lexs_names=lexicons_files,
                                 lex_path='/home/engine/cn-data/lexicons/',
                                 # lex_path='C:/GitHub/cn-data/lexicons/',
                                 # dataset_filter=['Automotive'],
                                 # dataset_filter=['Automotive', 'Books'],
                                 dataset_filter=[d],
                                 datasets_path='/datasets/amazon-data/csv/',
                                 # datasets_path='C:/Datasets/amazon-data/csv/',
                                 # n_reviews=20,
                                 # f_name='all-folds-15stars-summary'),
                                 # f_name='____',
                                 # train=True,
                                 n_cv=10,
                                 # stars=[1, 3, 5]
                                 stars=[1, 5],
                                 frequentiment_lexicons='/datasets/amazon-data/csv/lexicons/',
                                 # output_folder='/datasets/amazon-data/csv/acl-train-data-continuous',
                                 output_folder='/datasets/amazon-data/csv/acl-test-data-continuous',
                                 evaluate=False,
                                 )

datasets = ['Automotive',
            'Book',
            'Clot',
            'Electro',
            'Healt',
            'Movi',
            'Music',
            'Video',
            'Toys',
            'Sport',
            ]

jobs = []
for dataset in datasets:
    log.info('Add process for {}'.format(dataset))
    p = multiprocessing.Process(target=run_multi, args=(dataset,))
    p.start()
    jobs.append(p)

# wait for all processes to end
log.info('Waiting for all processes')
[j.join() for j in jobs]
log.info('All processes joined')
<|endoftext|>


<commit_before>from __future__ import absolute_import

import base64
import datetime
import functools
import logging
import sys
import time
import traceback
import urllib2
import uuid

from django.core.cache import cache
from django.template import TemplateSyntaxError
from django.template.loader import LoaderOrigin
from django.views.debug import ExceptionReporter

import sentry
from sentry.conf import settings
from sentry.utils import json
from sentry.utils import construct_checksum, varmap, transform, get_installed_apps, force_unicode, \
                           get_versions, shorten, get_signature, get_auth_header

logger = logging.getLogger('sentry.errors')

def fail_silently(default=None):
    def wrapped(func):
        @functools.wraps(func)
        def _wrapped(*args, **kwargs):
            try:
                return func(*args, **kwargs)
            except Exception, e:
                logger.exception(e)
                return default
        return _wrapped
    return wrapped

class SentryClient(object):
    @fail_silently((False, None))
    def check_throttle(self, checksum):
        if not (settings.THRASHING_TIMEOUT and settings.THRASHING_LIMIT):
            return (False, None)

        cache_key = 'sentry:%s' % (checksum,)
        # We MUST do a get first to avoid re-setting the timeout when doing .add
        added = cache.get(cache_key) is None
        if added:
            # Use add to avoid race conditions
            added = cache.add(cache_key, 1, settings.THRASHING_TIMEOUT)

        if added:
            return (False, None)

        try:
            thrash_count = cache.incr(cache_key)
        except (KeyError, ValueError):
            # cache.incr can fail. Assume we aren't thrashing yet, and
            # if we are, hope that the next error has a successful
            # cache.incr call.
            thrash_count = 0

        if thrash_count > settings.THRASHING_LIMIT:
             return (True, self.get_last_message_id(checksum))

        return (False, None)

    @fail_silently()
    def get_last_message_id(self, checksum):
        cache_key = 'sentry:%s:last_message_id' % (checksum,)

        return cache.get(cache_key)

    @fail_silently()
    def set_last_message_id(self, checksum, message_id):
        if settings.THRASHING_TIMEOUT and settings.THRASHING_LIMIT:
            cache_key = 'sentry:%s:last_message_id' % (checksum,)

            cache.set(cache_key, message_id, settings.THRASHING_LIMIT + 5)

    def process(self, **kwargs):
        "Processes the message before passing it on to the server"
        from sentry.utils import get_filters

        if kwargs.get('data'):
            # Ensure we're not changing the original data which was passed
            # to Sentry
            kwargs['data'] = kwargs['data'].copy()

        request = kwargs.pop('request', None)
        if request:
            if not kwargs.get('data'):
                kwargs['data'] = {}

            if not request.POST and request.raw_post_data:
                post_data = request.raw_post_data
            else:
                post_data = request.POST

            kwargs['data'].update(dict(
                META=request.META,
                POST=post_data,
                GET=request.GET,
                COOKIES=request.COOKIES,
            ))

            if not kwargs.get('url'):
                kwargs['url'] = request.build_absolute_uri()

        kwargs.setdefault('level', logging.ERROR)
        kwargs.setdefault('server_name', settings.NAME)

        # save versions of all installed apps
        if 'data' not in kwargs or '__sentry__' not in (kwargs['data'] or {}):
            if kwargs.get('data') is None:
                kwargs['data'] = {}
            kwargs['data']['__sentry__'] = {}

        versions = get_versions()
        kwargs['data']['__sentry__']['versions'] = versions

        # Shorten lists/strings
        for k, v in kwargs['data'].iteritems():
            if k == '__sentry__':
                continue
            kwargs['data'][k] = shorten(v)

        if kwargs.get('view'):
            # get list of modules from right to left
            parts = kwargs['view'].split('.')
            module_list = ['.'.join(parts[:idx]) for idx in xrange(1, len(parts)+1)][::-1]
            version = None
            module = None
            for m in module_list:
                if m in versions:
                    module = m
                    version = versions[m]

            # store our "best guess" for application version
            if version:
                kwargs['data']['__sentry__'].update({
                    'version': version,
                    'module': module,
                })

        if 'checksum' not in kwargs:
            checksum = construct_checksum(**kwargs)
        else:
            checksum = kwargs['checksum']

        (is_thrashing, message_id) = self.check_throttle(checksum)

        if is_thrashing:
            if request and message_id:
                # attach the sentry object to the request
                request.sentry = {
                    'id': message_id,
                    'thrashed': True,
                }

            return message_id

        for filter_ in get_filters():
            kwargs = filter_(None).process(kwargs) or kwargs

        # create ID client-side so that it can be passed to application
        message_id = uuid.uuid4().hex
        kwargs['message_id'] = message_id

        # Make sure all data is coerced
        kwargs['data'] = transform(kwargs['data'])

        if 'timestamp' not in kwargs:
            kwargs['timestamp'] = datetime.datetime.now()

        self.send(**kwargs)

        if request:
            # attach the sentry object to the request
            request.sentry = {
                'id': message_id,
                'trashed': False,
            }

        # store the last message_id incase we hit thrashing limits
        self.set_last_message_id(checksum, message_id)

        return message_id

    def send_remote(self, url, data, headers={}):
        req = urllib2.Request(url, headers=headers)
        try:
            response = urllib2.urlopen(req, data, settings.REMOTE_TIMEOUT).read()
        except:
            response = urllib2.urlopen(req, data).read()
        return response

    def send(self, **kwargs):
        "Sends the message to the server."
        if settings.REMOTE_URL:
            message = base64.b64encode(json.dumps(kwargs).encode('zlib'))
            for url in settings.REMOTE_URL:
                timestamp = time.time()
                signature = get_signature(message, timestamp)
                headers={
                    'Authorization': get_auth_header(signature, timestamp, '%s/%s' % (self.__class__.__name__, sentry.VERSION)),
                    'Content-Type': 'application/octet-stream',
                }

                try:
                    return self.send_remote(url=url, data=message, headers=headers)
                except urllib2.HTTPError, e:
                    body = e.read()
                    logger.error('Unable to reach Sentry log server: %s (url: %%s, body: %%s)' % (e,), url, body,
                                 exc_info=True, extra={'data':{'body': body, 'remote_url': url}})
                    logger.log(kwargs.pop('level', None) or logging.ERROR, kwargs.pop('message', None))
                except urllib2.URLError, e:
                    logger.error('Unable to reach Sentry log server: %s (url: %%s)' % (e,), url,
                                 exc_info=True, extra={'data':{'remote_url': url}})
                    logger.log(kwargs.pop('level', None) or logging.ERROR, kwargs.pop('message', None))
        else:
            from sentry.models import GroupedMessage

            return GroupedMessage.objects.from_kwargs(**kwargs)

    def create_from_record(self, record, **kwargs):
        """
        Creates an error log for a ``logging`` module ``record`` instance.
        """
        for k in ('url', 'view', 'request', 'data'):
            if not kwargs.get(k):
                kwargs[k] = record.__dict__.get(k)

        kwargs.update({
            'logger': record.name,
            'level': record.levelno,
            'message': force_unicode(record.msg),
            'server_name': settings.NAME,
        })

        # construct the checksum with the unparsed message
        kwargs['checksum'] = construct_checksum(**kwargs)

        # save the message with included formatting
        kwargs['message'] = record.getMessage()

        # If there's no exception being processed, exc_info may be a 3-tuple of None
        # http://docs.python.org/library/sys.html#sys.exc_info
        if record.exc_info and all(record.exc_info):
            return self.create_from_exception(record.exc_info, **kwargs)

        return self.process(
            traceback=record.exc_text,
            **kwargs
        )

    def create_from_text(self, message, **kwargs):
        """
        Creates an error log for from ``message``.
        """
        return self.process(
            message=message,
            **kwargs
        )

    def create_from_exception(self, exc_info=None, **kwargs):
        """
        Creates an error log from an exception.
        """
        new_exc = bool(exc_info)
        if not exc_info:
            exc_info = sys.exc_info()

        try:
            exc_type, exc_value, exc_traceback = exc_info

            reporter = ExceptionReporter(None, exc_type, exc_value, exc_traceback)
            frames = varmap(shorten, reporter.get_traceback_frames())

            if not kwargs.get('view'):
                # This should be cached
                modules = get_installed_apps()
                if settings.INCLUDE_PATHS:
                    modules = set(list(modules) + settings.INCLUDE_PATHS)

                def iter_tb_frames(tb):
                    while tb:
                        yield tb.tb_frame
                        tb = tb.tb_next

                def contains(iterator, value):
                    for k in iterator:
                        if value.startswith(k):
                            return True
                    return False

                # We iterate through each frame looking for an app in INSTALLED_APPS
                # When one is found, we mark it as last "best guess" (best_guess) and then
                # check it against SENTRY_EXCLUDE_PATHS. If it isnt listed, then we
                # use this option. If nothing is found, we use the "best guess".
                best_guess = None
                view = None
                for frame in iter_tb_frames(exc_traceback):
                    try:
                        view = '.'.join([frame.f_globals['__name__'], frame.f_code.co_name])
                    except:
                        continue
                    if contains(modules, view):
                        if not (contains(settings.EXCLUDE_PATHS, view) and best_guess):
                            best_guess = view
                    elif best_guess:
                        break
                if best_guess:
                    view = best_guess

                if view:
                    kwargs['view'] = view

            data = kwargs.pop('data', {}) or {}
            if hasattr(exc_type, '__class__'):
                exc_module = exc_type.__class__.__module__
            else:
                exc_module = None
            data['__sentry__'] = {
                'exc': map(transform, [exc_module, exc_value.args, frames]),
            }

            if (isinstance(exc_value, TemplateSyntaxError) and \
                isinstance(getattr(exc_value, 'source', None), (tuple, list)) and isinstance(exc_value.source[0], LoaderOrigin)):
                origin, (start, end) = exc_value.source
                data['__sentry__'].update({
                    'template': (origin.reload(), start, end, origin.name),
                })
                kwargs['view'] = origin.loadname

            tb_message = '\n'.join(traceback.format_exception(exc_type, exc_value, exc_traceback))

            kwargs.setdefault('message', transform(force_unicode(exc_value)))

            return self.process(
                class_name=exc_type.__name__,
                traceback=tb_message,
                data=data,
                **kwargs
            )
        finally:
            if new_exc:
                del exc_info

class DummyClient(SentryClient):
    "Sends messages into an empty void"
    def send(self, **kwargs):
        return None
<commit_msg>passing True to exc_info in create_from_exception works the same as passing None<commit_after>from __future__ import absolute_import

import base64
import datetime
import functools
import logging
import sys
import time
import traceback
import urllib2
import uuid

from django.core.cache import cache
from django.template import TemplateSyntaxError
from django.template.loader import LoaderOrigin
from django.views.debug import ExceptionReporter

import sentry
from sentry.conf import settings
from sentry.utils import json
from sentry.utils import construct_checksum, varmap, transform, get_installed_apps, force_unicode, \
                           get_versions, shorten, get_signature, get_auth_header

logger = logging.getLogger('sentry.errors')

def fail_silently(default=None):
    def wrapped(func):
        @functools.wraps(func)
        def _wrapped(*args, **kwargs):
            try:
                return func(*args, **kwargs)
            except Exception, e:
                logger.exception(e)
                return default
        return _wrapped
    return wrapped

class SentryClient(object):
    @fail_silently((False, None))
    def check_throttle(self, checksum):
        if not (settings.THRASHING_TIMEOUT and settings.THRASHING_LIMIT):
            return (False, None)

        cache_key = 'sentry:%s' % (checksum,)
        # We MUST do a get first to avoid re-setting the timeout when doing .add
        added = cache.get(cache_key) is None
        if added:
            # Use add to avoid race conditions
            added = cache.add(cache_key, 1, settings.THRASHING_TIMEOUT)

        if added:
            return (False, None)

        try:
            thrash_count = cache.incr(cache_key)
        except (KeyError, ValueError):
            # cache.incr can fail. Assume we aren't thrashing yet, and
            # if we are, hope that the next error has a successful
            # cache.incr call.
            thrash_count = 0

        if thrash_count > settings.THRASHING_LIMIT:
             return (True, self.get_last_message_id(checksum))

        return (False, None)

    @fail_silently()
    def get_last_message_id(self, checksum):
        cache_key = 'sentry:%s:last_message_id' % (checksum,)

        return cache.get(cache_key)

    @fail_silently()
    def set_last_message_id(self, checksum, message_id):
        if settings.THRASHING_TIMEOUT and settings.THRASHING_LIMIT:
            cache_key = 'sentry:%s:last_message_id' % (checksum,)

            cache.set(cache_key, message_id, settings.THRASHING_LIMIT + 5)

    def process(self, **kwargs):
        "Processes the message before passing it on to the server"
        from sentry.utils import get_filters

        if kwargs.get('data'):
            # Ensure we're not changing the original data which was passed
            # to Sentry
            kwargs['data'] = kwargs['data'].copy()

        request = kwargs.pop('request', None)
        if request:
            if not kwargs.get('data'):
                kwargs['data'] = {}

            if not request.POST and request.raw_post_data:
                post_data = request.raw_post_data
            else:
                post_data = request.POST

            kwargs['data'].update(dict(
                META=request.META,
                POST=post_data,
                GET=request.GET,
                COOKIES=request.COOKIES,
            ))

            if not kwargs.get('url'):
                kwargs['url'] = request.build_absolute_uri()

        kwargs.setdefault('level', logging.ERROR)
        kwargs.setdefault('server_name', settings.NAME)

        # save versions of all installed apps
        if 'data' not in kwargs or '__sentry__' not in (kwargs['data'] or {}):
            if kwargs.get('data') is None:
                kwargs['data'] = {}
            kwargs['data']['__sentry__'] = {}

        versions = get_versions()
        kwargs['data']['__sentry__']['versions'] = versions

        # Shorten lists/strings
        for k, v in kwargs['data'].iteritems():
            if k == '__sentry__':
                continue
            kwargs['data'][k] = shorten(v)

        if kwargs.get('view'):
            # get list of modules from right to left
            parts = kwargs['view'].split('.')
            module_list = ['.'.join(parts[:idx]) for idx in xrange(1, len(parts)+1)][::-1]
            version = None
            module = None
            for m in module_list:
                if m in versions:
                    module = m
                    version = versions[m]

            # store our "best guess" for application version
            if version:
                kwargs['data']['__sentry__'].update({
                    'version': version,
                    'module': module,
                })

        if 'checksum' not in kwargs:
            checksum = construct_checksum(**kwargs)
        else:
            checksum = kwargs['checksum']

        (is_thrashing, message_id) = self.check_throttle(checksum)

        if is_thrashing:
            if request and message_id:
                # attach the sentry object to the request
                request.sentry = {
                    'id': message_id,
                    'thrashed': True,
                }

            return message_id

        for filter_ in get_filters():
            kwargs = filter_(None).process(kwargs) or kwargs

        # create ID client-side so that it can be passed to application
        message_id = uuid.uuid4().hex
        kwargs['message_id'] = message_id

        # Make sure all data is coerced
        kwargs['data'] = transform(kwargs['data'])

        if 'timestamp' not in kwargs:
            kwargs['timestamp'] = datetime.datetime.now()

        self.send(**kwargs)

        if request:
            # attach the sentry object to the request
            request.sentry = {
                'id': message_id,
                'trashed': False,
            }

        # store the last message_id incase we hit thrashing limits
        self.set_last_message_id(checksum, message_id)

        return message_id

    def send_remote(self, url, data, headers={}):
        req = urllib2.Request(url, headers=headers)
        try:
            response = urllib2.urlopen(req, data, settings.REMOTE_TIMEOUT).read()
        except:
            response = urllib2.urlopen(req, data).read()
        return response

    def send(self, **kwargs):
        "Sends the message to the server."
        if settings.REMOTE_URL:
            message = base64.b64encode(json.dumps(kwargs).encode('zlib'))
            for url in settings.REMOTE_URL:
                timestamp = time.time()
                signature = get_signature(message, timestamp)
                headers={
                    'Authorization': get_auth_header(signature, timestamp, '%s/%s' % (self.__class__.__name__, sentry.VERSION)),
                    'Content-Type': 'application/octet-stream',
                }

                try:
                    return self.send_remote(url=url, data=message, headers=headers)
                except urllib2.HTTPError, e:
                    body = e.read()
                    logger.error('Unable to reach Sentry log server: %s (url: %%s, body: %%s)' % (e,), url, body,
                                 exc_info=True, extra={'data':{'body': body, 'remote_url': url}})
                    logger.log(kwargs.pop('level', None) or logging.ERROR, kwargs.pop('message', None))
                except urllib2.URLError, e:
                    logger.error('Unable to reach Sentry log server: %s (url: %%s)' % (e,), url,
                                 exc_info=True, extra={'data':{'remote_url': url}})
                    logger.log(kwargs.pop('level', None) or logging.ERROR, kwargs.pop('message', None))
        else:
            from sentry.models import GroupedMessage

            return GroupedMessage.objects.from_kwargs(**kwargs)

    def create_from_record(self, record, **kwargs):
        """
        Creates an error log for a ``logging`` module ``record`` instance.
        """
        for k in ('url', 'view', 'request', 'data'):
            if not kwargs.get(k):
                kwargs[k] = record.__dict__.get(k)

        kwargs.update({
            'logger': record.name,
            'level': record.levelno,
            'message': force_unicode(record.msg),
            'server_name': settings.NAME,
        })

        # construct the checksum with the unparsed message
        kwargs['checksum'] = construct_checksum(**kwargs)

        # save the message with included formatting
        kwargs['message'] = record.getMessage()

        # If there's no exception being processed, exc_info may be a 3-tuple of None
        # http://docs.python.org/library/sys.html#sys.exc_info
        if record.exc_info and all(record.exc_info):
            return self.create_from_exception(record.exc_info, **kwargs)

        return self.process(
            traceback=record.exc_text,
            **kwargs
        )

    def create_from_text(self, message, **kwargs):
        """
        Creates an error log for from ``message``.
        """
        return self.process(
            message=message,
            **kwargs
        )

    def create_from_exception(self, exc_info=None, **kwargs):
        """
        Creates an error log from an exception.
        """
        new_exc = bool(exc_info)
        if not exc_info or exc_info is True:
            exc_info = sys.exc_info()

        try:
            exc_type, exc_value, exc_traceback = exc_info

            reporter = ExceptionReporter(None, exc_type, exc_value, exc_traceback)
            frames = varmap(shorten, reporter.get_traceback_frames())

            if not kwargs.get('view'):
                # This should be cached
                modules = get_installed_apps()
                if settings.INCLUDE_PATHS:
                    modules = set(list(modules) + settings.INCLUDE_PATHS)

                def iter_tb_frames(tb):
                    while tb:
                        yield tb.tb_frame
                        tb = tb.tb_next

                def contains(iterator, value):
                    for k in iterator:
                        if value.startswith(k):
                            return True
                    return False

                # We iterate through each frame looking for an app in INSTALLED_APPS
                # When one is found, we mark it as last "best guess" (best_guess) and then
                # check it against SENTRY_EXCLUDE_PATHS. If it isnt listed, then we
                # use this option. If nothing is found, we use the "best guess".
                best_guess = None
                view = None
                for frame in iter_tb_frames(exc_traceback):
                    try:
                        view = '.'.join([frame.f_globals['__name__'], frame.f_code.co_name])
                    except:
                        continue
                    if contains(modules, view):
                        if not (contains(settings.EXCLUDE_PATHS, view) and best_guess):
                            best_guess = view
                    elif best_guess:
                        break
                if best_guess:
                    view = best_guess

                if view:
                    kwargs['view'] = view

            data = kwargs.pop('data', {}) or {}
            if hasattr(exc_type, '__class__'):
                exc_module = exc_type.__class__.__module__
            else:
                exc_module = None
            data['__sentry__'] = {
                'exc': map(transform, [exc_module, exc_value.args, frames]),
            }

            if (isinstance(exc_value, TemplateSyntaxError) and \
                isinstance(getattr(exc_value, 'source', None), (tuple, list)) and isinstance(exc_value.source[0], LoaderOrigin)):
                origin, (start, end) = exc_value.source
                data['__sentry__'].update({
                    'template': (origin.reload(), start, end, origin.name),
                })
                kwargs['view'] = origin.loadname

            tb_message = '\n'.join(traceback.format_exception(exc_type, exc_value, exc_traceback))

            kwargs.setdefault('message', transform(force_unicode(exc_value)))

            return self.process(
                class_name=exc_type.__name__,
                traceback=tb_message,
                data=data,
                **kwargs
            )
        finally:
            if new_exc:
                del exc_info

class DummyClient(SentryClient):
    "Sends messages into an empty void"
    def send(self, **kwargs):
        return None
<|endoftext|>


<commit_before>#!/usr/bin/env python3

from connexion import App
from opwen_email_server.utils.imports import can_import

_servers = list(filter(can_import, ('tornado', 'gevent', 'flask')))
_hosts = ['127.0.0.1', '0.0.0.0']

_server = _servers[0]
_host = _hosts[0]
_port = 8080
_ui = False
_waitress = False

def build_app(apis, host=_host, port=_port, server=_server, ui=_ui):
    app = App(__name__, host=host, port=port, server=server, swagger_ui=ui)

    for api in apis:
        app.add_api(api)

    return app

if __name__ == '__main__':
    from argparse import ArgumentParser
    from argparse import FileType
    from os.path import dirname
    from os.path import join

    try:
        # noinspection PyUnresolvedReferences
        from dotenv import load_dotenv
        load_dotenv(join(dirname(__file__), '.env'))
    except ImportError:
        pass

    parser = ArgumentParser()
    parser.add_argument('--host', choices=_hosts, default=_host)
    parser.add_argument('--port', type=int, default=_port)
    parser.add_argument('--server', choices=_servers, default=_server)
    parser.add_argument('--ui', action='store_true', default=_ui)
    if can_import('waitress'):
        parser.add_argument('--waitress', action='store_true', default=_waitress)
    parser.add_argument('apis', nargs='+', type=FileType('r'))
    args = parser.parse_args()

    app = build_app([api.name for api in args.apis], args.host,
                    args.port, args.server, args.ui)

    if can_import('waitress') and args.waitress:
        from waitress import serve
        serve(app.app, host=args.host, port=args.port)
    else:
        app.run()
<commit_msg>Remove waitress support<commit_after>#!/usr/bin/env python3

from connexion import App
from opwen_email_server.utils.imports import can_import

_servers = list(filter(can_import, ('tornado', 'gevent', 'flask')))
_hosts = ['127.0.0.1', '0.0.0.0']

_server = _servers[0]
_host = _hosts[0]
_port = 8080
_ui = False

def build_app(apis, host=_host, port=_port, server=_server, ui=_ui):
    app = App(__name__, host=host, port=port, server=server, swagger_ui=ui)

    for api in apis:
        app.add_api(api)

    return app

if __name__ == '__main__':
    from argparse import ArgumentParser
    from argparse import FileType
    from os.path import dirname
    from os.path import join

    try:
        # noinspection PyUnresolvedReferences
        from dotenv import load_dotenv
        load_dotenv(join(dirname(__file__), '.env'))
    except ImportError:
        pass

    parser = ArgumentParser()
    parser.add_argument('--host', choices=_hosts, default=_host)
    parser.add_argument('--port', type=int, default=_port)
    parser.add_argument('--server', choices=_servers, default=_server)
    parser.add_argument('--ui', action='store_true', default=_ui)
    parser.add_argument('apis', nargs='+', type=FileType('r'))
    args = parser.parse_args()

    app = build_app([api.name for api in args.apis], args.host,
                    args.port, args.server, args.ui)

    app.run()
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
#
#  This file is part of Sequana software
#
#  Copyright (c) 2016 - Sequana Development Team
#
#  File author(s):
#      Thomas Cokelaer <thomas.cokelaer@pasteur.fr>
#      Dimitri Desvillechabrol <dimitri.desvillechabrol@pasteur.fr>,
#          <d.desvillechabrol@gmail.com>
#
#  Distributed under the terms of the 3-clause BSD license.
#  The full license is in the LICENSE file, distributed with this software.
#
#  website: https://github.com/sequana/sequana
#  documentation: http://sequana.readthedocs.io
#
##############################################################################
"""Analysis of VCF file generated by freebayes.

"""

import sys

import vcf
import pandas as pd

class VCF(vcf.Reader):
    """VCF class (Variant Calling Format)

    This class is a wrapping of vcf.Reader class from the pyVCF package

    ::

        from sequana import sequana_data, VCF
        vcf_filename = sequana_data("test.vcf", "testing")

        # Read the data
        v = VCF(vcf_filename)

        # Filter the data
        filter_dict = {"QUAL": 10000, "FREQ": 0.85,
            "INFO": {"DP": ">10", "AO": ">200", "SRP": "<100"}}
        v.filter_vcf(filter_dict)

    """
    def __init__(self, filename, **kwargs):
        """
        Filter vcf file with a dictionnary.

        :param str filename: a vcf file.
        :param kwargs: any arguments accepted by vcf.Reader
        """
        try:
            self.filename = filename
            filin = open(filename, "r")
            vcf.Reader.__init__(self, fsock=filin, **kwargs)
        except IOError as e:
            print("I/O error({0}): {1}".format(e.errno, e.strerror))

    def _strand_rate(self, number1, number2):
        try:
            division = float(number1) / (number1 + number2)
        except ZeroDivisionError:
            return 0
        return division

    def _compute_freq(self, vcf_line):
        alt_freq = [float(count)/vcf_line.INFO["DP"] for count in \
                vcf_line.INFO["AO"]]
        return alt_freq

    def _compute_strand_bal(self, vcf_line):
        strand_bal = [self._strand_rate(vcf_line.INFO["SAF"][i],
            vcf_line.INFO["SAR"][i]) for i in range(len(vcf_line.INFO["SAF"]))]
        return strand_bal

    def _filter_info_field(self, info_value, threshold):
        if(threshold.startswith("<")):
            if(threshold.startswith("<=")):
                if(info_value <= int(threshold[2:])):
                    return False
            elif(info_value < int(threshold[1:])):
                return False
        else:
            if(threshold.startswith(">=")):
                if(info_value >= int(threshold[2:])):
                    return False
            elif(info_value > int(threshold[1:])):
                return False
        return True

    def _filter_line(self, vcf_line, filter_dict):
        # dictionary must have QUAL/FREQ/INFO keys

        if(vcf_line.QUAL < filter_dict["QUAL"]):
            return False

        alt_freq = self._compute_freq(vcf_line)
        strand_bal = self._compute_strand_bal(vcf_line)
        if(alt_freq[0] < filter_dict["FREQ"]):
            return False

        for key, value in filter_dict["INFO"].items():
            try:
                if(type(vcf_line.INFO[key]) != list):
                    if(self._filter_info_field(vcf_line.INFO[key], value)):
                        return False
                else:
                    if(self._filter_info_field(vcf_line.INFO[key][0], value)):
                        return False
            except KeyError:
                print("The key {0} does not exist in VCF file".format(key))
                print("Please, remove it in your config file")
                sys.exit(1)
        return True

    def filter_vcf(self, filter_dict, output):
        """ Read the VCF file and write the filter vcf file and return a data
        frame.

        """
        with open(output, "w") as fp:
            vcf_writer = vcf.Writer(fp, self)
            for variant in self:
                keep_line = self._filter_line(variant, filter_dict)
                if keep_line:
                    vcf_writer.write_record(variant)

    def _vcf_line_to_csv_line(self, vcf_line):
        alt_freq = self._compute_freq(vcf_line)
        strand_bal = self._compute_strand_bal(vcf_line)
        line_dict = {"chr": vcf_line.CHROM, "position": str(vcf_line.POS),
                "depth": vcf_line.INFO["DP"], "reference": vcf_line.REF,
                "alternative": "; ".join(str(x) for x in vcf_line.ALT),
                "freebayes_score": vcf_line.QUAL,
                "strand_balance": "; ".join("{0:.2f}".format(x) for x in \
                        strand_bal),
                "frequency": "; ".join("{0:.2f}".format(x) for x in alt_freq)}
        try:
            annotation = vcf_line.INFO["EFF"][0].split("|")
            effect_type, effect_lvl = annotation[0].split("(")
            try:
                prot_effect, cds_effect = annotation[3].split("/")
            except ValueError:
                cds_effect = annotation[3]
                prot_effect = ""
            ann_dict = {"CDS_position": cds_effect[2:],
                        "annotation": effect_type,
                        "codon_change": annotation[2],
                        "gene_name": annotation[5],
                        "mutation_type": annotation[1],
                        "prot_effect": prot_effect[2:],
                        "prot_size": annotation[4],
                        "putative_impact": effect_lvl}
            line_dict = dict(line_dict, **ann_dict)
        except KeyError:
            pass
        return line_dict

    def vcf_to_csv(self, output):
        df = pd.DataFrame(columns=["chr", "position", "reference",
            "alternative", "depth", "frequency", "strand_balance",
            "freebayes_score"])
        for variant in self:
            line_dict = self._vcf_line_to_csv_line(variant)
            df = df.append(line_dict, ignore_index=True)

        try:
            cols = df.columns.tolist()
            df = df[cols[:8] + [cols[9], cols[10], cols[15], cols[11], cols[8],
                cols[10], cols[13], cols[14]]]
        except (ValueError, IndexError):
            pass
        df.to_csv(output, index=False)
        return df
<commit_msg>Add mutation type<commit_after># -*- coding: utf-8 -*-
#
#  This file is part of Sequana software
#
#  Copyright (c) 2016 - Sequana Development Team
#
#  File author(s):
#      Thomas Cokelaer <thomas.cokelaer@pasteur.fr>
#      Dimitri Desvillechabrol <dimitri.desvillechabrol@pasteur.fr>,
#          <d.desvillechabrol@gmail.com>
#
#  Distributed under the terms of the 3-clause BSD license.
#  The full license is in the LICENSE file, distributed with this software.
#
#  website: https://github.com/sequana/sequana
#  documentation: http://sequana.readthedocs.io
#
##############################################################################
"""Analysis of VCF file generated by freebayes.

"""

import sys

import vcf
import pandas as pd

class VCF(vcf.Reader):
    """VCF class (Variant Calling Format)

    This class is a wrapping of vcf.Reader class from the pyVCF package

    ::

        from sequana import sequana_data, VCF
        vcf_filename = sequana_data("test.vcf", "testing")

        # Read the data
        v = VCF(vcf_filename)

        # Filter the data
        filter_dict = {"QUAL": 10000, "FREQ": 0.85,
            "INFO": {"DP": ">10", "AO": ">200", "SRP": "<100"}}
        v.filter_vcf(filter_dict)

    """
    def __init__(self, filename, **kwargs):
        """
        Filter vcf file with a dictionnary.

        :param str filename: a vcf file.
        :param kwargs: any arguments accepted by vcf.Reader
        """
        try:
            self.filename = filename
            filin = open(filename, "r")
            vcf.Reader.__init__(self, fsock=filin, **kwargs)
        except IOError as e:
            print("I/O error({0}): {1}".format(e.errno, e.strerror))

    def _strand_rate(self, number1, number2):
        try:
            division = float(number1) / (number1 + number2)
        except ZeroDivisionError:
            return 0
        return division

    def _compute_freq(self, vcf_line):
        alt_freq = [float(count)/vcf_line.INFO["DP"] for count in \
                vcf_line.INFO["AO"]]
        return alt_freq

    def _compute_strand_bal(self, vcf_line):
        strand_bal = [self._strand_rate(vcf_line.INFO["SAF"][i],
            vcf_line.INFO["SAR"][i]) for i in range(len(vcf_line.INFO["SAF"]))]
        return strand_bal

    def _filter_info_field(self, info_value, threshold):
        if(threshold.startswith("<")):
            if(threshold.startswith("<=")):
                if(info_value <= int(threshold[2:])):
                    return False
            elif(info_value < int(threshold[1:])):
                return False
        else:
            if(threshold.startswith(">=")):
                if(info_value >= int(threshold[2:])):
                    return False
            elif(info_value > int(threshold[1:])):
                return False
        return True

    def _filter_line(self, vcf_line, filter_dict):
        # dictionary must have QUAL/FREQ/INFO keys

        if(vcf_line.QUAL < filter_dict["QUAL"]):
            return False

        alt_freq = self._compute_freq(vcf_line)
        strand_bal = self._compute_strand_bal(vcf_line)
        if(alt_freq[0] < filter_dict["FREQ"]):
            return False

        for key, value in filter_dict["INFO"].items():
            try:
                if(type(vcf_line.INFO[key]) != list):
                    if(self._filter_info_field(vcf_line.INFO[key], value)):
                        return False
                else:
                    if(self._filter_info_field(vcf_line.INFO[key][0], value)):
                        return False
            except KeyError:
                print("The key {0} does not exist in VCF file".format(key))
                print("Please, remove it in your config file")
                sys.exit(1)
        return True

    def filter_vcf(self, filter_dict, output):
        """ Read the VCF file and write the filter vcf file and return a data
        frame.

        """
        with open(output, "w") as fp:
            vcf_writer = vcf.Writer(fp, self)
            for variant in self:
                keep_line = self._filter_line(variant, filter_dict)
                if keep_line:
                    vcf_writer.write_record(variant)

    def _vcf_line_to_csv_line(self, vcf_line):
        alt_freq = self._compute_freq(vcf_line)
        strand_bal = self._compute_strand_bal(vcf_line)
        line_dict = {"chr": vcf_line.CHROM, "position": str(vcf_line.POS),
                "depth": vcf_line.INFO["DP"], "reference": vcf_line.REF,
                "alternative": "; ".join(str(x) for x in vcf_line.ALT),
                "freebayes_score": vcf_line.QUAL,
                "strand_balance": "; ".join("{0:.2f}".format(x) for x in \
                        strand_bal),
                "frequency": "; ".join("{0:.2f}".format(x) for x in alt_freq)}
        try:
            annotation = vcf_line.INFO["EFF"][0].split("|")
            effect_type, effect_lvl = annotation[0].split("(")
            try:
                prot_effect, cds_effect = annotation[3].split("/")
            except ValueError:
                cds_effect = annotation[3]
                prot_effect = ""
            ann_dict = {"CDS_position": cds_effect[2:],
                        "annotation": effect_type,
                        "codon_change": annotation[2],
                        "gene_name": annotation[5],
                        "mutation_type": annotation[1],
                        "prot_effect": prot_effect[2:],
                        "prot_size": annotation[4],
                        "putative_impact": effect_lvl}
            line_dict = dict(line_dict, **ann_dict)
        except KeyError:
            pass
        return line_dict

    def vcf_to_csv(self, output):
        df = pd.DataFrame(columns=["chr", "position", "reference",
            "alternative", "depth", "frequency", "strand_balance",
            "freebayes_score"])
        for variant in self:
            line_dict = self._vcf_line_to_csv_line(variant)
            df = df.append(line_dict, ignore_index=True)

        try:
            cols = df.columns.tolist()
            df = df[cols[:8] + [cols[9], cols[12], cols[15], cols[11], cols[8],
                cols[10], cols[13], cols[14]]]
        except (ValueError, IndexError):
            pass
        df.to_csv(output, index=False)
        return df
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
from brightway_projects import projects, Project, backend_mapping
from brightway_projects.testing import bwtest
import os
import platform
import pytest

windows = platform.system() == "Windows"

def test_setup(bwtest):
    assert not projects.current
    projects.create(name="foo", backends=["tests"])
    p = Project.get(name="foo")
    assert not p.data
    assert "projects.test.db" in os.listdir(bwtest)
    assert "__logs__" in os.listdir(bwtest)

def test_select_project(bwtest):
    backend = backend_mapping["tests"]
    projects.create("foo", backends=["tests"], switch=False)
    assert backend.created.name == "foo"
    assert not backend.activated
    projects.select("foo")
    assert backend.activated.name == "foo"
    assert projects.current.name == "foo"

def test_create_and_switch_project(bwtest):
    backend = backend_mapping["tests"]
    assert not backend.activated
    projects.create("foo", backends=["tests"])
    assert backend.activated.name == "foo"
    assert backend.created.name == "foo"
    assert projects.current.name == "foo"

def test_create_project_creates_dir(bwtest):
    projects.create("foo", backends=["tests"])
    assert os.path.isdir(projects.dir)

@pytest.mark.skipif(windows, reason="Windows hates fun")
def test_really_funny_project_names(bwtest):
    NAMES = [
        "Roses are red, violets are blue. Hope you enjoy terminal hue",
        "👾 🙇 💁 🙅 🙆 🙋 🙎 🙍 ",
    ]
    error_found = False
    for name in NAMES:
        try:
            projects.create(name, backends=["tests"])
            assert os.path.isdir(projects.dir)
            print("This is OK:", name)
        except:
            print("This is not OK:", name)
            error_found = True
    if error_found:
        raise ValueError("Invaid project name")

def test_funny_project_names(bwtest):
    NAMES = [
        "Powerلُلُصّبُلُلصّبُررً ॣ ॣh ॣ ॣ冗",
        "True",
        "None",
        "1.0/0.0",
        "0xabad1dea",
        "!@#$%^&*()`~",
        "<>?:'{}|_+",
        r",./;'[]\-=",
        "Ω≈ç√∫˜µ≤≥÷",
        "田中さんにあげて下さい",
        "｀ｨ(´∀｀∩",
        "הָיְתָהtestالصفحات التّحول",
        "　",
    ]
    for name in NAMES:
        projects.create(name, backends=["tests"])
        assert os.path.isdir(projects.dir)

def test_project_report(bwtest):
    projects.create("foo", backends=["tests"])
    assert projects.report()
<commit_msg>Compat for projects changes<commit_after># -*- coding: utf-8 -*-
from brightway_projects import projects, Project, backend_mapping
from brightway_projects.testing import bwtest
import os
import platform
import pytest

windows = platform.system() == "Windows"

def test_setup(bwtest):
    assert not projects.current
    projects.create_project(name="foo", backends=["tests"])
    p = Project.get(name="foo")
    assert not p.data
    assert "projects.test.db" in os.listdir(bwtest)
    assert "__logs__" in os.listdir(bwtest)

def test_select_project(bwtest):
    backend = backend_mapping["tests"]
    projects.create_project("foo", backends=["tests"], switch=False)
    assert backend.created.name == "foo"
    assert not backend.activated
    projects.select("foo")
    assert backend.activated.name == "foo"
    assert projects.current.name == "foo"

def test_create_and_switch_project(bwtest):
    backend = backend_mapping["tests"]
    assert not backend.activated
    projects.create_project("foo", backends=["tests"])
    assert backend.activated.name == "foo"
    assert backend.created.name == "foo"
    assert projects.current.name == "foo"

def test_create_project_creates_dir(bwtest):
    projects.create_project("foo", backends=["tests"])
    assert os.path.isdir(projects.dir)

@pytest.mark.skipif(windows, reason="Windows hates fun")
def test_really_funny_project_names(bwtest):
    NAMES = [
        "Roses are red, violets are blue. Hope you enjoy terminal hue",
        "👾 🙇 💁 🙅 🙆 🙋 🙎 🙍 ",
    ]
    error_found = False
    for name in NAMES:
        try:
            projects.create_project(name, backends=["tests"])
            assert os.path.isdir(projects.dir)
            print("This is OK:", name)
        except:
            print("This is not OK:", name)
            error_found = True
    if error_found:
        raise ValueError("Invaid project name")

def test_funny_project_names(bwtest):
    NAMES = [
        "Powerلُلُصّبُلُلصّبُررً ॣ ॣh ॣ ॣ冗",
        "True",
        "None",
        "1.0/0.0",
        "0xabad1dea",
        "!@#$%^&*()`~",
        "<>?:'{}|_+",
        r",./;'[]\-=",
        "Ω≈ç√∫˜µ≤≥÷",
        "田中さんにあげて下さい",
        "｀ｨ(´∀｀∩",
        "הָיְתָהtestالصفحات التّحول",
        "　",
    ]
    for name in NAMES:
        projects.create_project(name, backends=["tests"])
        assert os.path.isdir(projects.dir)

def test_project_report(bwtest):
    projects.create_project("foo", backends=["tests"])
    assert projects.report()
<|endoftext|>


<commit_before># encoding: utf-8
"""
Utilities for path handling.
"""

# Copyright (c) IPython Development Team.
# Distributed under the terms of the Modified BSD License.

import os
import sys
import errno
import shutil
import random
import tempfile
import glob
from warnings import warn
from hashlib import md5

from IPython.utils.process import system
from IPython.utils import py3compat
from IPython.utils.decorators import undoc

#-----------------------------------------------------------------------------
# Code
#-----------------------------------------------------------------------------

fs_encoding = sys.getfilesystemencoding()

def _writable_dir(path):
    """Whether `path` is a directory, to which the user has write access."""
    return os.path.isdir(path) and os.access(path, os.W_OK)

if sys.platform == 'win32':
    def _get_long_path_name(path):
        """Get a long path name (expand ~) on Windows using ctypes.

        Examples
        --------

        >>> get_long_path_name('c:\\docume~1')
        u'c:\\\\Documents and Settings'

        """
        try:
            import ctypes
        except ImportError:
            raise ImportError('you need to have ctypes installed for this to work')
        _GetLongPathName = ctypes.windll.kernel32.GetLongPathNameW
        _GetLongPathName.argtypes = [ctypes.c_wchar_p, ctypes.c_wchar_p,
            ctypes.c_uint ]

        buf = ctypes.create_unicode_buffer(260)
        rv = _GetLongPathName(path, buf, 260)
        if rv == 0 or rv > 260:
            return path
        else:
            return buf.value
else:
    def _get_long_path_name(path):
        """Dummy no-op."""
        return path

def get_long_path_name(path):
    """Expand a path into its long form.

    On Windows this expands any ~ in the paths. On other platforms, it is
    a null operation.
    """
    return _get_long_path_name(path)

def unquote_filename(name, win32=(sys.platform=='win32')):
    """ On Windows, remove leading and trailing quotes from filenames.
    """
    if win32:
        if name.startswith(("'", '"')) and name.endswith(("'", '"')):
            name = name[1:-1]
    return name

def compress_user(path):
    """Reverse of :func:`os.path.expanduser`
    """
    home = os.path.expanduser('~')
    if path.startswith(home):
        path =  "~" + path[len(home):]
    return path

def get_py_filename(name, force_win32=None):
    """Return a valid python filename in the current directory.

    If the given name is not a file, it adds '.py' and searches again.
    Raises IOError with an informative message if the file isn't found.

    On Windows, apply Windows semantics to the filename. In particular, remove
    any quoting that has been applied to it. This option can be forced for
    testing purposes.
    """

    name = os.path.expanduser(name)
    if force_win32 is None:
        win32 = (sys.platform == 'win32')
    else:
        win32 = force_win32
    name = unquote_filename(name, win32=win32)
    if not os.path.isfile(name) and not name.endswith('.py'):
        name += '.py'
    if os.path.isfile(name):
        return name
    else:
        raise IOError('File `%r` not found.' % name)

def filefind(filename, path_dirs=None):
    """Find a file by looking through a sequence of paths.

    This iterates through a sequence of paths looking for a file and returns
    the full, absolute path of the first occurence of the file.  If no set of
    path dirs is given, the filename is tested as is, after running through
    :func:`expandvars` and :func:`expanduser`.  Thus a simple call::

        filefind('myfile.txt')

    will find the file in the current working dir, but::

        filefind('~/myfile.txt')

    Will find the file in the users home directory.  This function does not
    automatically try any paths, such as the cwd or the user's home directory.

    Parameters
    ----------
    filename : str
        The filename to look for.
    path_dirs : str, None or sequence of str
        The sequence of paths to look for the file in.  If None, the filename
        need to be absolute or be in the cwd.  If a string, the string is
        put into a sequence and the searched.  If a sequence, walk through
        each element and join with ``filename``, calling :func:`expandvars`
        and :func:`expanduser` before testing for existence.

    Returns
    -------
    Raises :exc:`IOError` or returns absolute path to file.
    """

    # If paths are quoted, abspath gets confused, strip them...
    filename = filename.strip('"').strip("'")
    # If the input is an absolute path, just check it exists
    if os.path.isabs(filename) and os.path.isfile(filename):
        return filename

    if path_dirs is None:
        path_dirs = ("",)
    elif isinstance(path_dirs, py3compat.string_types):
        path_dirs = (path_dirs,)

    for path in path_dirs:
        if path == '.': path = py3compat.getcwd()
        testname = expand_path(os.path.join(path, filename))
        if os.path.isfile(testname):
            return os.path.abspath(testname)

    raise IOError("File %r does not exist in any of the search paths: %r" %
                  (filename, path_dirs) )

class HomeDirError(Exception):
    pass

def get_home_dir(require_writable=False):
    """Return the 'home' directory, as a unicode string.

    Uses os.path.expanduser('~'), and checks for writability.

    See stdlib docs for how this is determined.
    $HOME is first priority on *ALL* platforms.

    Parameters
    ----------

    require_writable : bool [default: False]
        if True:
            guarantees the return value is a writable directory, otherwise
            raises HomeDirError
        if False:
            The path is resolved, but it is not guaranteed to exist or be writable.
    """

    homedir = os.path.expanduser('~')
    # Next line will make things work even when /home/ is a symlink to
    # /usr/home as it is on FreeBSD, for example
    homedir = os.path.realpath(homedir)

    if not _writable_dir(homedir) and os.name == 'nt':
        # expanduser failed, use the registry to get the 'My Documents' folder.
        try:
            try:
                import winreg as wreg  # Py 3
            except ImportError:
                import _winreg as wreg  # Py 2
            key = wreg.OpenKey(
                wreg.HKEY_CURRENT_USER,
                "Software\Microsoft\Windows\CurrentVersion\Explorer\Shell Folders"
            )
            homedir = wreg.QueryValueEx(key,'Personal')[0]
            key.Close()
        except:
            pass

    if (not require_writable) or _writable_dir(homedir):
        return py3compat.cast_unicode(homedir, fs_encoding)
    else:
        raise HomeDirError('%s is not a writable dir, '
                'set $HOME environment variable to override' % homedir)

def get_xdg_dir():
    """Return the XDG_CONFIG_HOME, if it is defined and exists, else None.

    This is only for non-OS X posix (Linux,Unix,etc.) systems.
    """

    env = os.environ

    if os.name == 'posix' and sys.platform != 'darwin':
        # Linux, Unix, AIX, etc.
        # use ~/.config if empty OR not set
        xdg = env.get("XDG_CONFIG_HOME", None) or os.path.join(get_home_dir(), '.config')
        if xdg and _writable_dir(xdg):
            return py3compat.cast_unicode(xdg, fs_encoding)

    return None

def get_xdg_cache_dir():
    """Return the XDG_CACHE_HOME, if it is defined and exists, else None.

    This is only for non-OS X posix (Linux,Unix,etc.) systems.
    """

    env = os.environ

    if os.name == 'posix' and sys.platform != 'darwin':
        # Linux, Unix, AIX, etc.
        # use ~/.cache if empty OR not set
        xdg = env.get("XDG_CACHE_HOME", None) or os.path.join(get_home_dir(), '.cache')
        if xdg and _writable_dir(xdg):
            return py3compat.cast_unicode(xdg, fs_encoding)

    return None

@undoc
def get_ipython_dir():
    warn("get_ipython_dir has moved to the IPython.paths module")
    from IPython.paths import get_ipython_dir
    return get_ipython_dir()

@undoc
def get_ipython_cache_dir():
    warn("get_ipython_cache_dir has moved to the IPython.paths module")
    from IPython.paths import get_ipython_cache_dir
    return get_ipython_cache_dir()

@undoc
def get_ipython_package_dir():
    warn("get_ipython_package_dir has moved to the IPython.paths module")
    from IPython.paths import get_ipython_package_dir
    return get_ipython_package_dir()

@undoc
def get_ipython_module_path(module_str):
    warn("get_ipython_module_path has moved to the IPython.paths module")
    from IPython.paths import get_ipython_module_path
    return get_ipython_module_path(module_str)

@undoc
def locate_profile(profile='default'):
    warn("locate_profile has moved to the IPython.paths module")
    from IPython.paths import locate_profile
    return locate_profile(profile=profile)

def expand_path(s):
    """Expand $VARS and ~names in a string, like a shell

    :Examples:

       In [2]: os.environ['FOO']='test'

       In [3]: expand_path('variable FOO is $FOO')
       Out[3]: 'variable FOO is test'
    """
    # This is a pretty subtle hack. When expand user is given a UNC path
    # on Windows (\\server\share$\%username%), os.path.expandvars, removes
    # the $ to get (\\server\share\%username%). I think it considered $
    # alone an empty var. But, we need the $ to remains there (it indicates
    # a hidden share).
    if os.name=='nt':
        s = s.replace('$\\', 'IPYTHON_TEMP')
    s = os.path.expandvars(os.path.expanduser(s))
    if os.name=='nt':
        s = s.replace('IPYTHON_TEMP', '$\\')
    return s

def unescape_glob(string):
    """Unescape glob pattern in `string`."""
    def unescape(s):
        for pattern in '*[]!?':
            s = s.replace(r'\{0}'.format(pattern), pattern)
        return s
    return '\\'.join(map(unescape, string.split('\\\\')))

def shellglob(args):
    """
    Do glob expansion for each element in `args` and return a flattened list.

    Unmatched glob pattern will remain as-is in the returned list.

    """
    expanded = []
    # Do not unescape backslash in Windows as it is interpreted as
    # path separator:
    unescape = unescape_glob if sys.platform != 'win32' else lambda x: x
    for a in args:
        expanded.extend(glob.glob(a) or [unescape(a)])
    return expanded

def target_outdated(target,deps):
    """Determine whether a target is out of date.

    target_outdated(target,deps) -> 1/0

    deps: list of filenames which MUST exist.
    target: single filename which may or may not exist.

    If target doesn't exist or is older than any file listed in deps, return
    true, otherwise return false.
    """
    try:
        target_time = os.path.getmtime(target)
    except os.error:
        return 1
    for dep in deps:
        dep_time = os.path.getmtime(dep)
        if dep_time > target_time:
            #print "For target",target,"Dep failed:",dep # dbg
            #print "times (dep,tar):",dep_time,target_time # dbg
            return 1
    return 0

def target_update(target,deps,cmd):
    """Update a target with a given command given a list of dependencies.

    target_update(target,deps,cmd) -> runs cmd if target is outdated.

    This is just a wrapper around target_outdated() which calls the given
    command if target is outdated."""

    if target_outdated(target,deps):
        system(cmd)

@undoc
def filehash(path):
    """Make an MD5 hash of a file, ignoring any differences in line
    ending characters."""
    warn("filehash() is deprecated")
    with open(path, "rU") as f:
        return md5(py3compat.str_to_bytes(f.read())).hexdigest()

ENOLINK = 1998

def link(src, dst):
    """Hard links ``src`` to ``dst``, returning 0 or errno.

    Note that the special errno ``ENOLINK`` will be returned if ``os.link`` isn't
    supported by the operating system.
    """

    if not hasattr(os, "link"):
        return ENOLINK
    link_errno = 0
    try:
        os.link(src, dst)
    except OSError as e:
        link_errno = e.errno
    return link_errno

def link_or_copy(src, dst):
    """Attempts to hardlink ``src`` to ``dst``, copying if the link fails.

    Attempts to maintain the semantics of ``shutil.copy``.

    Because ``os.link`` does not overwrite files, a unique temporary file
    will be used if the target already exists, then that file will be moved
    into place.
    """

    if os.path.isdir(dst):
        dst = os.path.join(dst, os.path.basename(src))

    link_errno = link(src, dst)
    if link_errno == errno.EEXIST:
        if os.stat(src).st_ino == os.stat(dst).st_ino:
            # dst is already a hard link to the correct file, so we don't need
            # to do anything else. If we try to link and rename the file
            # anyway, we get duplicate files - see http://bugs.python.org/issue21876
            return

        new_dst = dst + "-temp-%04X" %(random.randint(1, 16**4), )
        try:
            link_or_copy(src, new_dst)
        except:
            try:
                os.remove(new_dst)
            except OSError:
                pass
            raise
        os.rename(new_dst, dst)
    elif link_errno != 0:
        # Either link isn't supported, or the filesystem doesn't support
        # linking, or 'src' and 'dst' are on different filesystems.
        shutil.copy(src, dst)

def ensure_dir_exists(path, mode=0o755):
    """ensure that a directory exists

    If it doesn't exist, try to create it and protect against a race condition
    if another process is doing the same.

    The default permissions are 755, which differ from os.makedirs default of 777.
    """
    if not os.path.exists(path):
        try:
            os.makedirs(path, mode=mode)
        except OSError as e:
            if e.errno != errno.EEXIST:
                raise
    elif not os.path.isdir(path):
        raise IOError("%r exists but is not a directory" % path)
<commit_msg>Safely encode paths in compress_user - Avoids a `UnicodeDecodeError` when a tooltip that includes a non-ascii pathname is shown on Python 2 - Related to #8543<commit_after># encoding: utf-8
"""
Utilities for path handling.
"""

# Copyright (c) IPython Development Team.
# Distributed under the terms of the Modified BSD License.

import os
import sys
import errno
import shutil
import random
import tempfile
import glob
from warnings import warn
from hashlib import md5

from IPython.utils.process import system
from IPython.utils import py3compat
from IPython.utils.decorators import undoc

#-----------------------------------------------------------------------------
# Code
#-----------------------------------------------------------------------------

fs_encoding = sys.getfilesystemencoding()

def _writable_dir(path):
    """Whether `path` is a directory, to which the user has write access."""
    return os.path.isdir(path) and os.access(path, os.W_OK)

if sys.platform == 'win32':
    def _get_long_path_name(path):
        """Get a long path name (expand ~) on Windows using ctypes.

        Examples
        --------

        >>> get_long_path_name('c:\\docume~1')
        u'c:\\\\Documents and Settings'

        """
        try:
            import ctypes
        except ImportError:
            raise ImportError('you need to have ctypes installed for this to work')
        _GetLongPathName = ctypes.windll.kernel32.GetLongPathNameW
        _GetLongPathName.argtypes = [ctypes.c_wchar_p, ctypes.c_wchar_p,
            ctypes.c_uint ]

        buf = ctypes.create_unicode_buffer(260)
        rv = _GetLongPathName(path, buf, 260)
        if rv == 0 or rv > 260:
            return path
        else:
            return buf.value
else:
    def _get_long_path_name(path):
        """Dummy no-op."""
        return path

def get_long_path_name(path):
    """Expand a path into its long form.

    On Windows this expands any ~ in the paths. On other platforms, it is
    a null operation.
    """
    return _get_long_path_name(path)

def unquote_filename(name, win32=(sys.platform=='win32')):
    """ On Windows, remove leading and trailing quotes from filenames.
    """
    if win32:
        if name.startswith(("'", '"')) and name.endswith(("'", '"')):
            name = name[1:-1]
    return name

def compress_user(path):
    """Reverse of :func:`os.path.expanduser`
    """
    path = py3compat.unicode_to_str(path, sys.getfilesystemencoding())
    home = os.path.expanduser('~')
    if path.startswith(home):
        path =  "~" + path[len(home):]
    return path

def get_py_filename(name, force_win32=None):
    """Return a valid python filename in the current directory.

    If the given name is not a file, it adds '.py' and searches again.
    Raises IOError with an informative message if the file isn't found.

    On Windows, apply Windows semantics to the filename. In particular, remove
    any quoting that has been applied to it. This option can be forced for
    testing purposes.
    """

    name = os.path.expanduser(name)
    if force_win32 is None:
        win32 = (sys.platform == 'win32')
    else:
        win32 = force_win32
    name = unquote_filename(name, win32=win32)
    if not os.path.isfile(name) and not name.endswith('.py'):
        name += '.py'
    if os.path.isfile(name):
        return name
    else:
        raise IOError('File `%r` not found.' % name)

def filefind(filename, path_dirs=None):
    """Find a file by looking through a sequence of paths.

    This iterates through a sequence of paths looking for a file and returns
    the full, absolute path of the first occurence of the file.  If no set of
    path dirs is given, the filename is tested as is, after running through
    :func:`expandvars` and :func:`expanduser`.  Thus a simple call::

        filefind('myfile.txt')

    will find the file in the current working dir, but::

        filefind('~/myfile.txt')

    Will find the file in the users home directory.  This function does not
    automatically try any paths, such as the cwd or the user's home directory.

    Parameters
    ----------
    filename : str
        The filename to look for.
    path_dirs : str, None or sequence of str
        The sequence of paths to look for the file in.  If None, the filename
        need to be absolute or be in the cwd.  If a string, the string is
        put into a sequence and the searched.  If a sequence, walk through
        each element and join with ``filename``, calling :func:`expandvars`
        and :func:`expanduser` before testing for existence.

    Returns
    -------
    Raises :exc:`IOError` or returns absolute path to file.
    """

    # If paths are quoted, abspath gets confused, strip them...
    filename = filename.strip('"').strip("'")
    # If the input is an absolute path, just check it exists
    if os.path.isabs(filename) and os.path.isfile(filename):
        return filename

    if path_dirs is None:
        path_dirs = ("",)
    elif isinstance(path_dirs, py3compat.string_types):
        path_dirs = (path_dirs,)

    for path in path_dirs:
        if path == '.': path = py3compat.getcwd()
        testname = expand_path(os.path.join(path, filename))
        if os.path.isfile(testname):
            return os.path.abspath(testname)

    raise IOError("File %r does not exist in any of the search paths: %r" %
                  (filename, path_dirs) )

class HomeDirError(Exception):
    pass

def get_home_dir(require_writable=False):
    """Return the 'home' directory, as a unicode string.

    Uses os.path.expanduser('~'), and checks for writability.

    See stdlib docs for how this is determined.
    $HOME is first priority on *ALL* platforms.

    Parameters
    ----------

    require_writable : bool [default: False]
        if True:
            guarantees the return value is a writable directory, otherwise
            raises HomeDirError
        if False:
            The path is resolved, but it is not guaranteed to exist or be writable.
    """

    homedir = os.path.expanduser('~')
    # Next line will make things work even when /home/ is a symlink to
    # /usr/home as it is on FreeBSD, for example
    homedir = os.path.realpath(homedir)

    if not _writable_dir(homedir) and os.name == 'nt':
        # expanduser failed, use the registry to get the 'My Documents' folder.
        try:
            try:
                import winreg as wreg  # Py 3
            except ImportError:
                import _winreg as wreg  # Py 2
            key = wreg.OpenKey(
                wreg.HKEY_CURRENT_USER,
                "Software\Microsoft\Windows\CurrentVersion\Explorer\Shell Folders"
            )
            homedir = wreg.QueryValueEx(key,'Personal')[0]
            key.Close()
        except:
            pass

    if (not require_writable) or _writable_dir(homedir):
        return py3compat.cast_unicode(homedir, fs_encoding)
    else:
        raise HomeDirError('%s is not a writable dir, '
                'set $HOME environment variable to override' % homedir)

def get_xdg_dir():
    """Return the XDG_CONFIG_HOME, if it is defined and exists, else None.

    This is only for non-OS X posix (Linux,Unix,etc.) systems.
    """

    env = os.environ

    if os.name == 'posix' and sys.platform != 'darwin':
        # Linux, Unix, AIX, etc.
        # use ~/.config if empty OR not set
        xdg = env.get("XDG_CONFIG_HOME", None) or os.path.join(get_home_dir(), '.config')
        if xdg and _writable_dir(xdg):
            return py3compat.cast_unicode(xdg, fs_encoding)

    return None

def get_xdg_cache_dir():
    """Return the XDG_CACHE_HOME, if it is defined and exists, else None.

    This is only for non-OS X posix (Linux,Unix,etc.) systems.
    """

    env = os.environ

    if os.name == 'posix' and sys.platform != 'darwin':
        # Linux, Unix, AIX, etc.
        # use ~/.cache if empty OR not set
        xdg = env.get("XDG_CACHE_HOME", None) or os.path.join(get_home_dir(), '.cache')
        if xdg and _writable_dir(xdg):
            return py3compat.cast_unicode(xdg, fs_encoding)

    return None

@undoc
def get_ipython_dir():
    warn("get_ipython_dir has moved to the IPython.paths module")
    from IPython.paths import get_ipython_dir
    return get_ipython_dir()

@undoc
def get_ipython_cache_dir():
    warn("get_ipython_cache_dir has moved to the IPython.paths module")
    from IPython.paths import get_ipython_cache_dir
    return get_ipython_cache_dir()

@undoc
def get_ipython_package_dir():
    warn("get_ipython_package_dir has moved to the IPython.paths module")
    from IPython.paths import get_ipython_package_dir
    return get_ipython_package_dir()

@undoc
def get_ipython_module_path(module_str):
    warn("get_ipython_module_path has moved to the IPython.paths module")
    from IPython.paths import get_ipython_module_path
    return get_ipython_module_path(module_str)

@undoc
def locate_profile(profile='default'):
    warn("locate_profile has moved to the IPython.paths module")
    from IPython.paths import locate_profile
    return locate_profile(profile=profile)

def expand_path(s):
    """Expand $VARS and ~names in a string, like a shell

    :Examples:

       In [2]: os.environ['FOO']='test'

       In [3]: expand_path('variable FOO is $FOO')
       Out[3]: 'variable FOO is test'
    """
    # This is a pretty subtle hack. When expand user is given a UNC path
    # on Windows (\\server\share$\%username%), os.path.expandvars, removes
    # the $ to get (\\server\share\%username%). I think it considered $
    # alone an empty var. But, we need the $ to remains there (it indicates
    # a hidden share).
    if os.name=='nt':
        s = s.replace('$\\', 'IPYTHON_TEMP')
    s = os.path.expandvars(os.path.expanduser(s))
    if os.name=='nt':
        s = s.replace('IPYTHON_TEMP', '$\\')
    return s

def unescape_glob(string):
    """Unescape glob pattern in `string`."""
    def unescape(s):
        for pattern in '*[]!?':
            s = s.replace(r'\{0}'.format(pattern), pattern)
        return s
    return '\\'.join(map(unescape, string.split('\\\\')))

def shellglob(args):
    """
    Do glob expansion for each element in `args` and return a flattened list.

    Unmatched glob pattern will remain as-is in the returned list.

    """
    expanded = []
    # Do not unescape backslash in Windows as it is interpreted as
    # path separator:
    unescape = unescape_glob if sys.platform != 'win32' else lambda x: x
    for a in args:
        expanded.extend(glob.glob(a) or [unescape(a)])
    return expanded

def target_outdated(target,deps):
    """Determine whether a target is out of date.

    target_outdated(target,deps) -> 1/0

    deps: list of filenames which MUST exist.
    target: single filename which may or may not exist.

    If target doesn't exist or is older than any file listed in deps, return
    true, otherwise return false.
    """
    try:
        target_time = os.path.getmtime(target)
    except os.error:
        return 1
    for dep in deps:
        dep_time = os.path.getmtime(dep)
        if dep_time > target_time:
            #print "For target",target,"Dep failed:",dep # dbg
            #print "times (dep,tar):",dep_time,target_time # dbg
            return 1
    return 0

def target_update(target,deps,cmd):
    """Update a target with a given command given a list of dependencies.

    target_update(target,deps,cmd) -> runs cmd if target is outdated.

    This is just a wrapper around target_outdated() which calls the given
    command if target is outdated."""

    if target_outdated(target,deps):
        system(cmd)

@undoc
def filehash(path):
    """Make an MD5 hash of a file, ignoring any differences in line
    ending characters."""
    warn("filehash() is deprecated")
    with open(path, "rU") as f:
        return md5(py3compat.str_to_bytes(f.read())).hexdigest()

ENOLINK = 1998

def link(src, dst):
    """Hard links ``src`` to ``dst``, returning 0 or errno.

    Note that the special errno ``ENOLINK`` will be returned if ``os.link`` isn't
    supported by the operating system.
    """

    if not hasattr(os, "link"):
        return ENOLINK
    link_errno = 0
    try:
        os.link(src, dst)
    except OSError as e:
        link_errno = e.errno
    return link_errno

def link_or_copy(src, dst):
    """Attempts to hardlink ``src`` to ``dst``, copying if the link fails.

    Attempts to maintain the semantics of ``shutil.copy``.

    Because ``os.link`` does not overwrite files, a unique temporary file
    will be used if the target already exists, then that file will be moved
    into place.
    """

    if os.path.isdir(dst):
        dst = os.path.join(dst, os.path.basename(src))

    link_errno = link(src, dst)
    if link_errno == errno.EEXIST:
        if os.stat(src).st_ino == os.stat(dst).st_ino:
            # dst is already a hard link to the correct file, so we don't need
            # to do anything else. If we try to link and rename the file
            # anyway, we get duplicate files - see http://bugs.python.org/issue21876
            return

        new_dst = dst + "-temp-%04X" %(random.randint(1, 16**4), )
        try:
            link_or_copy(src, new_dst)
        except:
            try:
                os.remove(new_dst)
            except OSError:
                pass
            raise
        os.rename(new_dst, dst)
    elif link_errno != 0:
        # Either link isn't supported, or the filesystem doesn't support
        # linking, or 'src' and 'dst' are on different filesystems.
        shutil.copy(src, dst)

def ensure_dir_exists(path, mode=0o755):
    """ensure that a directory exists

    If it doesn't exist, try to create it and protect against a race condition
    if another process is doing the same.

    The default permissions are 755, which differ from os.makedirs default of 777.
    """
    if not os.path.exists(path):
        try:
            os.makedirs(path, mode=mode)
        except OSError as e:
            if e.errno != errno.EEXIST:
                raise
    elif not os.path.isdir(path):
        raise IOError("%r exists but is not a directory" % path)
<|endoftext|>


<commit_before># This is a stab file to mock `requests` package and some of its contents for tests.

class ResponseMock(object):

    def __init__(self, text):
        self.text = text

    def raise_for_status(self):
        if self.text is None:
            raise ImportError('Mock exception')

class RequestException(Exception):
    pass
<commit_msg>requests for test updated<commit_after># This is a stab file to mock `requests` package and some of its contents for tests.

class ResponseMock(object):

    def __init__(self, text):
        self.text = text

    def raise_for_status(self):
        if self.text is None:
            raise ImportError('Mock exception')

class RequestException(Exception):
    pass

def get():
    pass

def post():
    pass
<|endoftext|>