<commit_before>import os
BASE_DIR = os.path.dirname(os.path.dirname(__file__))

SECRET_KEY = '4h=6tp37*3c&92f$y00%!r4+s!l*w*iij07n-cjqk&tuf=%3wa'

DEBUG = False
TEMPLATE_DEBUG = False

ALLOWED_HOSTS = [
    '.chosak.org',
]

INSTALLED_APPS = (
    'django.contrib.contenttypes',
    'django.contrib.staticfiles',
    'reports',
)

MIDDLEWARE_CLASSES = (
    'django.middleware.common.CommonMiddleware',
    'django.middleware.clickjacking.XFrameOptionsMiddleware',
)

ROOT_URLCONF = 'reports.urls'
WSGI_APPLICATION = 'wsgi.application'

DATABASES = {}

LANGUAGE_CODE = 'en-us'
TIME_ZONE = 'UTC'
USE_I18N = True
USE_L10N = True
USE_TZ = True

STATIC_URL = '/static/'

LOGGING = {
    'version': 1,
    'disable_existing_loggers': True,
    'formatters': {
        'standard': {
            'format': '%(asctime)s [%(levelname)s] %(name)s: %(message)s'
        },
    },
    'handlers': {
        'console': {
            'level': 'DEBUG',
            'class': 'logging.StreamHandler',
            'formatter': 'standard',
        },
        'file': {
            'level':'INFO',
            'class':'logging.handlers.RotatingFileHandler',
            'filename': '/var/log/wsgi/app.log',
            'maxBytes': 1024*1024*5, # 5 MB
            'backupCount': 5,
            'formatter':'standard',
        },
        'error': {
            'level': 'ERROR',
            'class':'logging.handlers.RotatingFileHandler',
            'filename': '/var/log/wsgi/error.log',
            'maxBytes': 1024*1024*5, # 5 MB
            'backupCount': 5,
            'formatter':'standard',
        },
    },
    'loggers': {
        '': {
            'handlers': ['error'],
            'level': 'ERROR',
            'propagate': True,
        },
        'reports': {
            'handlers': ['console', 'file'],
            'level': 'DEBUG',
            'propagate': True,
        }
    },
}

CACHES = {
    'default': {
        'BACKEND': 'django.core.cache.backends.memcached.MemcachedCache',
        'LOCATION': [
            'cache.kez3tf.cfg.use1.cache.amazonaws.com:11211',
        ]
    }
}
<commit_msg>redact prod info<commit_after>import os
BASE_DIR = os.path.dirname(os.path.dirname(__file__))

SECRET_KEY = '4h=6tp37*3c&92f$y00%!r4+s!l*w*iij07n-cjqk&tuf=%3wa'

DEBUG = False
TEMPLATE_DEBUG = False

ALLOWED_HOSTS = [
    '.chosak.org',
]

INSTALLED_APPS = (
    'django.contrib.contenttypes',
    'django.contrib.staticfiles',
    'reports',
)

MIDDLEWARE_CLASSES = (
    'django.middleware.common.CommonMiddleware',
    'django.middleware.clickjacking.XFrameOptionsMiddleware',
)

ROOT_URLCONF = 'reports.urls'
WSGI_APPLICATION = 'wsgi.application'

DATABASES = {}

LANGUAGE_CODE = 'en-us'
TIME_ZONE = 'UTC'
USE_I18N = True
USE_L10N = True
USE_TZ = True

STATIC_URL = '/static/'

LOGGING = {
    'version': 1,
    'disable_existing_loggers': True,
    'formatters': {
        'standard': {
            'format': '%(asctime)s [%(levelname)s] %(name)s: %(message)s'
        },
    },
    'handlers': {
        'console': {
            'level': 'DEBUG',
            'class': 'logging.StreamHandler',
            'formatter': 'standard',
        },
        'file': {
            'level':'INFO',
            'class':'logging.handlers.RotatingFileHandler',
            'filename': '/var/log/wsgi/app.log',
            'maxBytes': 1024*1024*5, # 5 MB
            'backupCount': 5,
            'formatter':'standard',
        },
        'error': {
            'level': 'ERROR',
            'class':'logging.handlers.RotatingFileHandler',
            'filename': '/var/log/wsgi/error.log',
            'maxBytes': 1024*1024*5, # 5 MB
            'backupCount': 5,
            'formatter':'standard',
        },
    },
    'loggers': {
        '': {
            'handlers': ['error'],
            'level': 'ERROR',
            'propagate': True,
        },
        'reports': {
            'handlers': ['console', 'file'],
            'level': 'DEBUG',
            'propagate': True,
        }
    },
}

memcached_server = os.environ.get('MEMCACHED_SERVER')
if memcached_server:
    CACHES = {
        'default': {
            'BACKEND': 'django.core.cache.backends.memcached.MemcachedCache',
            'LOCATION': [
                memcached_server,
            ]
        }
    }
<|endoftext|>


<commit_before>from rerankfun import *
from compute_bleu_function import *
from collections import defaultdict
import os
import sys

feat_norm = 'data/train/norm/'
feat_unnorm = 'data/train/unnorm/'
features = ['diag','ibm','lex','tm','diag_rev','ibm_rev','lm','untranslated']

##############################
#       Challenge
##############################
'''
We should use the rerankfun.rerank function, and pass in the following:
rerank(sentences, scores, weights, outname)
'''

class PRO(object):

    def __init__(self,
                  train_location='train/',
                  test_location='dev+test/',
                  num_data=400,
                  tps=100
                  tau=5000,
                  alpha=0.1,
                  sample_number=100
                  eta=0.1
                  epochs=5):

        self.train_location = train_location
        self.test_location = test_location
        self.tps = tps
        self.num_train = 40000
        self.num_test = 80000
        self.tau = tau
        self.alpha = alpha
        self.sample_number = sample_number
        self.eta = eta
        self.epochs = epochs

        # load the training data: translations, references and source
        translations_trn = self.structure(train_location)
        references_trn = open(train_location+'train.ref','r')
        source_trn = open(train_location+'src.out','r')

        # build the feature vectors for each translation
        scores_trn = self.build_features(train_location, num_train)

        # collect the BLEU scores for each translation
        bleu_list = self.collect_bleu_scores()

    def build_features(self, data_location, num_data, norm=True):
        '''
        Given a dataset, build feature vectors for each of its translations.
        Requirements: a directory with the name dataset, which contains folders called 'norm' and 'unnorm'

        @data_location either 'train' or 'dev+test' (the name of any directory containing data in the correct format)
        @num_data the number of data in the dataset
        @norm True if we want to use the normalised data, False otherwise

        @return a list of lists of feature vectors. Each Russian sentence has a list of feature vectors, one for each
        of its translations.
        '''
        normalised = '/norm' if norm else '/unnorm'
        feature_locations = os.listdir(data_location+normalised)
        print feature_locations
        features = []
        for location in feature_locations:
            feature_file = open(data_location+normalised+'/'+location,'r')
            feature = [line.strip() for line in feature_file]
            print len(feature), location
            assert len(feature) == num_data * self.tps
            features.append(feature)

        all_features = []
        for i in range(num_data):
            all_features.append([])
            for j in range(i*self.tps, i*self.tps + self.tps):
                feat_vec = tuple([float(feature[j]) for feature in features])
                all_features[i].append(feat_vec)
        return all_features

    def structure(self, data_location):
        '''
        Convert the unstructured datasets which aren't grouped by Russian sentence to a structured list, indexed by Russian sentence.
        '''
        unstructured = [line.strip() for line in open(data_location+'/100best_clean.out','r')]
        structured = [[unstructured[j] for j in range(i*self.tps, (i+1)*self.tps)] for i in range(len(unstructured)/self.tps)]
        return structured

    def collect_bleu_scores(self):
        '''
        Compute smoothed BLEU scores for each translation in the list of translations
        @translations_list a structured list of lists of translations, one list for each Russian sentence
        @return the BLEU scores in the same format, i.e. a list of lists
        '''
        bleu_list = [[0]*self.tps for _ in self.num_data]
        for rus_ind, translations in enumerate(translations_list):
            ref = references[rus_ind]
            for tr_ind, translation in enumerate(translation):
                bleu_score = individual_bleu(translation, ref)
                bleu_list[rus_ind].append(bleu_score)
        return bleu_list

    def bleu_difference(self,s1, s2, reference, alpha):
        stats_s1 = bleu.bleu_stats(s1, reference)
        stats_s2 = bleu.bleu_stats(s2, reference)

        s1_score = bleu.bleu(stats_s1)
        s2_score = bleu.bleu(stats_s2)

        return fabs(s1_score-s2_score)

    def getSamples(self, best_candidates, ref_sent):
        sample = []
        for i in range(self.tau):
            s1 = best_candidates[randint(0, len(nbests))]
            s2 = best_candidates[randint(0, len(nbests))]
            if (bleu_difference(s1, s2, ref) > self.alpha):
                sample.append(s1, s2)
            elif s1 != s2:
                sample.append(s2, s1)
            else:
                continue
        sample.sort(key=lambda t: t[1]-t[2]) #in ascending order
        return sample[-self.sample_number:]

translations_list = structure(dataset='dev+test')
scores = build_features(dataset='dev+test', num_data=800, norm=False)
weights = [1,-1,1]
outname = 'dev+test/delete.out'
rerank(translations_list, scores, weights, outname)
print compute_bleu(outname)

def collect_bleu_scores():
    pass

>>>>>>> 7fedbb64a25ba6a31c743459237088ec18cb0a49
<commit_msg>added data object<commit_after>from rerankfun import *
from compute_bleu_function import *
from collections import defaultdict
import os
import sys
from namedlist import namedlist

feat_norm = 'data/train/norm/'
feat_unnorm = 'data/train/unnorm/'
features = ['diag','ibm','lex','tm','diag_rev','ibm_rev','lm','untranslated']

##############################
#       Challenge
##############################
'''
We should use the rerankfun.rerank function, and pass in the following:
rerank(sentences, scores, weights, outname)
'''

class PRO(object):

    def __init__(self,
                  train_location='train/',
                  test_location='dev+test/',
                  num_data=400,
                  tps=100
                  tau=5000,
                  alpha=0.1,
                  sample_number=100
                  eta=0.1
                  epochs=5):

        self.train_location = train_location
        self.test_location = test_location
        self.tps = tps
        self.num_train = 40000
        self.num_test = 80000
        self.tau = tau
        self.alpha = alpha
        self.sample_number = sample_number
        self.eta = eta
        self.epochs = epochs

        # load the training data: translations, references and source
        self.data = build_data()
        data_point = namedlist('blue', 'features')

    def build_data(self):
        self.data = defaultdict()
        references_trn = open(self.train_location+'train.ref','r')
        source_trn = open(self.train_location+'src.out','r')

        translations = self.structure(self.train_location)

        feature_vectors = self.build_features(self.train_location, self.num_train)
        bleu_list = self.collect_bleu_scores()

        for r, ref in enumerate(references_trn):
            self.data[ref] = defaultdict()
            for t, trans in enumerate(translations[r]):
                blue_score = bleu_list[r][t]
                feature_vec = feature_vectors[r][t]
                self.data[ref][trans] = data_point(blue_score, feature_vec)

    def build_features(self, data_location, num_data, norm=True):
        '''
        Given a dataset, build feature vectors for each of its translations.
        Requirements: a directory with the name dataset, which contains folders called 'norm' and 'unnorm'

        @data_location either 'train' or 'dev+test' (the name of any directory containing data in the correct format)
        @num_data the number of data in the dataset
        @norm True if we want to use the normalised data, False otherwise

        @return a list of lists of feature vectors. Each Russian sentence has a list of feature vectors, one for each
        of its translations.
        '''
        normalised = '/norm' if norm else '/unnorm'
        feature_locations = os.listdir(data_location+normalised)
        print feature_locations
        features = []
        for location in feature_locations:
            feature_file = open(data_location+normalised+'/'+location,'r')
            feature = [line.strip() for line in feature_file]
            print len(feature), location
            assert len(feature) == num_data * self.tps
            features.append(feature)

        all_features = []
        for i in range(num_data):
            all_features.append([])
            for j in range(i*self.tps, i*self.tps + self.tps):
                feat_vec = tuple([float(feature[j]) for feature in features])
                all_features[i].append(feat_vec)
        return all_features

    def structure(self, data_location):
        '''
        Convert the unstructured datasets which aren't grouped by Russian sentence to a structured list, indexed by Russian sentence.
        '''
        unstructured = [line.strip() for line in open(data_location+'/100best_clean.out','r')]
        structured = [[unstructured[j] for j in range(i*self.tps, (i+1)*self.tps)] for i in range(len(unstructured)/self.tps)]
        return structured

    def collect_bleu_scores(self):
        '''
        Compute smoothed BLEU scores for each translation in the list of translations
        @translations_list a structured list of lists of translations, one list for each Russian sentence
        @return the BLEU scores in the same format, i.e. a list of lists
        '''
        bleu_list = [[0]*self.tps for _ in self.num_data]
        for rus_ind, translations in enumerate(translations_list):
            ref = references[rus_ind]
            for tr_ind, translation in enumerate(translation):
                bleu_score = individual_bleu(translation, ref)
                bleu_list[rus_ind].append(bleu_score)
        return bleu_list

    def bleu_difference(self,s1, s2, reference, alpha):
        stats_s1 = bleu.bleu_stats(s1, reference)
        stats_s2 = bleu.bleu_stats(s2, reference)

        s1_score = bleu.bleu(stats_s1)
        s2_score = bleu.bleu(stats_s2)

        return fabs(s1_score-s2_score)

    def getSamples(self, best_candidates, ref_sent):
        sample = []
        for i in range(self.tau):
            s1 = best_candidates[randint(0, len(nbests))]
            s2 = best_candidates[randint(0, len(nbests))]
            if (bleu_difference(s1, s2, ref) > self.alpha):
                sample.append(s1, s2)
            elif s1 != s2:
                sample.append(s2, s1)
            else:
                continue
        sample.sort(key=lambda t: t[1]-t[2]) #in ascending order
        return sample[-self.sample_number:]

    #def perceptron(self, theta, s1, s2):

do a perceptron update of the parameters theta:
   if theta * s1.features <= theta * s2.features:
       mistakes += 1
       theta += eta * (s1.features - s2.features) # this is vector addition!

''' Test
translations_list = structure(dataset='dev+test')
scores = build_features(dataset='dev+test', num_data=800, norm=False)
weights = [1,-1,1]
outname = 'dev+test/delete.out'
rerank(translations_list, scores, weights, outname)
print compute_bleu(outname)

'''

<|endoftext|>


<commit_before>from abc import ABCMeta, abstractmethod
import typing

from .stats import statistic_factory

__all__ = ['Feature', 'BaseFeature']

class BaseFeature(metaclass=ABCMeta):
    @abstractmethod
    def __call__(self, item):
        pass

# TODO: Statistic should only ever be a callable, no more "or"s in arguments
class Feature(BaseFeature):
    def __init__(self, data, statistic, name, is_nominal=False):
        """

        Args:
            data (``numpy.array``):
            statistic (``str`` or callable):
            name (``str``):
            is_nominal (``bool``):
        """

        self.data = data
        self.name = name
        self.is_nominal = is_nominal
        if isinstance(statistic, str):
            self.statistic = statistic_factory(statistic)
        elif isinstance(statistic, typing.Callable):
            self.statistic = statistic
        else:
            raise TypeError('statistic must be a string name or a callable')

    def __call__(self, item):
        """

        Args:
            item (``slice``):

        Returns:

        """
        return self.statistic(self.data[item])
<commit_msg>Simplify Feature class<commit_after>import abc
import typing

from .stats import statistic_factory

__all__ = ['Feature', 'feature']

class BaseFeature(metaclass=abc.ABCMeta):
    @abc.abstractmethod
    def __call__(self, item):
        pass

class Feature(BaseFeature):
    def __init__(self, data, statistic, name, is_categorical=False):
        """

        Args:
            data (``numpy.ndarray``):
            statistic (``Callable``):
            name (``str``):
            is_categorical (``bool``):
        """

        self.data = data
        self.name = name
        self.is_nominal = is_categorical
        self.statistic = statistic

    def __call__(self, item):
        """

        Args:
            item (``slice``):

        Returns:

        """
        return self.statistic(self.data[item])
<|endoftext|>


<commit_before>__author__    = 'Viktor Kerkez <alefnula@gmail.com>'
__date__      = '07 January 2009'
__copyright__ = 'Copyright (c) 2009 Viktor Kerkez'

import os
import zipfile
import logging
# tea imports
from tea.system import platform
from tea.process import execute_and_report as er

logger = logging.getLogger(__name__)

def _extract_file(archive, destination, filename):
    try:
        output_path = os.path.join(destination, filename)
        output_dir = os.path.dirname(output_path)
        if not os.path.isdir(output_dir):
            os.makedirs(output_dir)
        # Cannot write big chunks of data to windows shares
        MAX_BYTES = 5242880 # 5MB
        reader = archive.open(file)
        writer = open(output_path, 'wb')
        while True:
            data = reader.read(MAX_BYTES)
            if len(data) > 0:
                writer.write(data)
            else: break
        return True
    except:
        logger.exception('Error while unzipping filename %s from archive %s' % (filename, archive.filename))
        return False

def unzip(archive, destination, filenames=None):
    '''Unzip the a complete zip archive into destination directory,
    or unzip a specific file(s) from the archive.

    Usage:
        >>> output = os.path.join(os.getcwd(), 'output')
        >>> # Archive can be an instance of a ZipFile class
        >>> archive = zipfile.ZipFile('test.zip', 'r')
        >>> # Or just a filename
        >>> archive = 'test.zip'
        >>> # Extracts all files
        >>> unzip(archive, output)
        >>> # Extract only one file
        >>> unzip(archive, output, 'my_file.txt')
        >>> # Extract a list of files
        >>> unzip(archive, output, ['my_file1.txt', 'my_file2.txt'])
        >>> unzip_file('test.zip', 'my_file.txt', output)

    @type  archive: L{zipfile.ZipFile} or string
    @param archive: Zipfile object to extract from or path to the zip archive.
    @type  destination: string
    @param destination: Path to the output directory
    @type  filenames: string, list or None
    @param filenames: Path(s) to the filename(s) inside the zip archive that you want to extract.
    '''
    close = False
    try:
        if not isinstance(archive, zipfile.ZipFile):
            archive = zipfile.ZipFile(archive, 'r', allowZip64=True)
            close = True
        logger.info('Extracting: %s -> %s' % (archive.filename, destination))
        if isinstance(filenames, str):
            filenames = [filenames]
        if filenames is None: # extract all
            filenames = archive.namelist()
        for filename in filenames:
            if filename.endswith('/'): # it's a directory
                os.makedirs(os.path.join(destination, filename))
            else:
                if not _extract_file(archive, destination, filename):
                    raise Exception()
        logger.info('Extracting zip archive "%s" succeeded' % archive.filename)
        return True
    except:
        logger.exception('Error while unzipping archive %s' % archive.filename)
        return False
    finally:
        if close: archive.close()

def mkzip(archive, items, mode='w', save_full_paths=False):
    '''Recursively zip a directory

    @type  archive: L{zipfile.ZipFile} or string
    @param archive: ZipFile object add to or path to the output zip archive.
    @type  items: string or list
    @param items: Single item or list of items (files and directories) to be added to zipfile
    @type  mode: string
    @param mode: w for create new and write a for append to
    @type  save_full_paths: boolean
    @param save_full_paths: preserve full paths
    '''
    close = False
    try:
        if not isinstance(archive, zipfile.ZipFile):
            archive = zipfile.ZipFile(archive, mode, allowZip64=True)
            close = True
        logger.info('mkdzip: Creating %s, from: %s' % (archive.filename, items))
        if isinstance(items, str): items = [items]
        for item in items:
            item = os.path.abspath(item)
            basename = os.path.basename(item)
            if os.path.isdir(item):
                for root, directoires, filenames in os.walk(item): #@UnusedVariable
                    for filename in filenames:
                        path = os.path.join(root, filename)
                        if save_full_paths:
                            archive_path = path.encode('utf-8')
                        else:
                            archive_path = os.path.join(basename, path.replace(item, '').strip('\\/')).encode('utf-8')
                        archive.write(path, archive_path) #, zipfile.ZIP_DEFLATED)
            elif os.path.isfile(item):
                if save_full_paths:
                    archive_name = item.encode('utf-8')
                else:
                    archive_name = basename.encode('utf-8')
                archive.write(item, archive_name) #, zipfile.ZIP_DEFLATED)
        return True
    except Exception as e:
        logger.error('Error occurred during mkzip: %s' % e)
        return False
    finally:
        if close: archive.close()

_SZ_EXECUTABLE = None

def _get_sz():
    global _SZ_EXECUTABLE
    if _SZ_EXECUTABLE is None:
        _SZ_EXECUTABLE = '7z'
        if platform.is_a(platform.WINDOWS):
            for pf in ('ProgramFiles', 'ProgramFiles(x86)', 'ProgramW6432'):
                executable = os.path.join(os.environ.get(pf, ''), '7-Zip', '7z.exe')
                if os.path.exists(executable):
                    _SZ_EXECUTABLE = executable
                    break
    return _SZ_EXECUTABLE

def seven_zip(archive, items, self_extracting=False):
    '''Create a 7z archive'''
    if not isinstance(items, (list, tuple)):
        items = [items]
    if self_extracting:
        return er(_get_sz(), 'a', '-sfx', archive, *items)
    else:
        return er(_get_sz(), 'a', archive, *items)

def seven_unzip(archive, output):
    '''Extract a 7z archive'''
    return er(_get_sz(), 'x', archive, '-o%s' % output, '-aoa')
<commit_msg>7zip switch for compressing open files<commit_after>__author__    = 'Viktor Kerkez <alefnula@gmail.com>'
__date__      = '07 January 2009'
__copyright__ = 'Copyright (c) 2009 Viktor Kerkez'

import os
import zipfile
import logging
# tea imports
from tea.system import platform
from tea.process import execute_and_report as er

logger = logging.getLogger(__name__)

def _extract_file(archive, destination, filename):
    try:
        output_path = os.path.join(destination, filename)
        output_dir = os.path.dirname(output_path)
        if not os.path.isdir(output_dir):
            os.makedirs(output_dir)
        # Cannot write big chunks of data to windows shares
        MAX_BYTES = 5242880 # 5MB
        reader = archive.open(file)
        writer = open(output_path, 'wb')
        while True:
            data = reader.read(MAX_BYTES)
            if len(data) > 0:
                writer.write(data)
            else: break
        return True
    except:
        logger.exception('Error while unzipping filename %s from archive %s' % (filename, archive.filename))
        return False

def unzip(archive, destination, filenames=None):
    '''Unzip the a complete zip archive into destination directory,
    or unzip a specific file(s) from the archive.

    Usage:
        >>> output = os.path.join(os.getcwd(), 'output')
        >>> # Archive can be an instance of a ZipFile class
        >>> archive = zipfile.ZipFile('test.zip', 'r')
        >>> # Or just a filename
        >>> archive = 'test.zip'
        >>> # Extracts all files
        >>> unzip(archive, output)
        >>> # Extract only one file
        >>> unzip(archive, output, 'my_file.txt')
        >>> # Extract a list of files
        >>> unzip(archive, output, ['my_file1.txt', 'my_file2.txt'])
        >>> unzip_file('test.zip', 'my_file.txt', output)

    @type  archive: L{zipfile.ZipFile} or string
    @param archive: Zipfile object to extract from or path to the zip archive.
    @type  destination: string
    @param destination: Path to the output directory
    @type  filenames: string, list or None
    @param filenames: Path(s) to the filename(s) inside the zip archive that you want to extract.
    '''
    close = False
    try:
        if not isinstance(archive, zipfile.ZipFile):
            archive = zipfile.ZipFile(archive, 'r', allowZip64=True)
            close = True
        logger.info('Extracting: %s -> %s' % (archive.filename, destination))
        if isinstance(filenames, str):
            filenames = [filenames]
        if filenames is None: # extract all
            filenames = archive.namelist()
        for filename in filenames:
            if filename.endswith('/'): # it's a directory
                os.makedirs(os.path.join(destination, filename))
            else:
                if not _extract_file(archive, destination, filename):
                    raise Exception()
        logger.info('Extracting zip archive "%s" succeeded' % archive.filename)
        return True
    except:
        logger.exception('Error while unzipping archive %s' % archive.filename)
        return False
    finally:
        if close: archive.close()

def mkzip(archive, items, mode='w', save_full_paths=False):
    '''Recursively zip a directory

    @type  archive: L{zipfile.ZipFile} or string
    @param archive: ZipFile object add to or path to the output zip archive.
    @type  items: string or list
    @param items: Single item or list of items (files and directories) to be added to zipfile
    @type  mode: string
    @param mode: w for create new and write a for append to
    @type  save_full_paths: boolean
    @param save_full_paths: preserve full paths
    '''
    close = False
    try:
        if not isinstance(archive, zipfile.ZipFile):
            archive = zipfile.ZipFile(archive, mode, allowZip64=True)
            close = True
        logger.info('mkdzip: Creating %s, from: %s' % (archive.filename, items))
        if isinstance(items, str): items = [items]
        for item in items:
            item = os.path.abspath(item)
            basename = os.path.basename(item)
            if os.path.isdir(item):
                for root, directoires, filenames in os.walk(item): #@UnusedVariable
                    for filename in filenames:
                        path = os.path.join(root, filename)
                        if save_full_paths:
                            archive_path = path.encode('utf-8')
                        else:
                            archive_path = os.path.join(basename, path.replace(item, '').strip('\\/')).encode('utf-8')
                        archive.write(path, archive_path) #, zipfile.ZIP_DEFLATED)
            elif os.path.isfile(item):
                if save_full_paths:
                    archive_name = item.encode('utf-8')
                else:
                    archive_name = basename.encode('utf-8')
                archive.write(item, archive_name) #, zipfile.ZIP_DEFLATED)
        return True
    except Exception as e:
        logger.error('Error occurred during mkzip: %s' % e)
        return False
    finally:
        if close: archive.close()

_SZ_EXECUTABLE = None

def _get_sz():
    global _SZ_EXECUTABLE
    if _SZ_EXECUTABLE is None:
        _SZ_EXECUTABLE = '7z'
        if platform.is_a(platform.WINDOWS):
            for pf in ('ProgramFiles', 'ProgramFiles(x86)', 'ProgramW6432'):
                executable = os.path.join(os.environ.get(pf, ''), '7-Zip', '7z.exe')
                if os.path.exists(executable):
                    _SZ_EXECUTABLE = executable
                    break
    return _SZ_EXECUTABLE

def seven_zip(archive, items, self_extracting=False):
    '''Create a 7z archive'''
    if not isinstance(items, (list, tuple)):
        items = [items]
    if self_extracting:
        return er(_get_sz(), 'a', '-ssw', '-sfx', archive, *items)
    else:
        return er(_get_sz(), 'a', '-ssw', archive, *items)

def seven_unzip(archive, output):
    '''Extract a 7z archive'''
    return er(_get_sz(), 'x', archive, '-o%s' % output, '-aoa')
<|endoftext|>


<commit_before># -*- coding: utf-8 -
from datetime import timedelta
from faker import Factory
from munch import munchify
from tempfile import NamedTemporaryFile
from .local_time import get_now
from .op_faker import OP_Provider
import random

fake = Factory.create('uk_UA')
fake_ru = Factory.create('ru')
fake_en = Factory.create()
fake.add_provider(OP_Provider)

def create_fake_sentence():
    return fake.sentence(nb_words=10, variable_nb_words=True)

def field_with_id(prefix, sentence):
    return u"{}-{}: {}".format(prefix, fake.uuid4()[:8], sentence)

def translate_country_en(country):
    if country == u"Україна":
        return "Ukraine"
    else:
        raise Exception(u"Cannot translate country to english: {}".format(country))

def translate_country_ru(country):
    if country == u"Україна":
        return u"Украина"
    else:
        raise Exception(u"Cannot translate country to russian: {}".format(country))

def create_fake_doc():
    content = fake.text()
    suffix = fake.random_element(('.doc', '.docx', '.pdf'))
    prefix = "{}-{}{}".format("d", fake.uuid4()[:8], fake_en.word())
    tf = NamedTemporaryFile(delete=False, suffix=suffix, prefix=prefix)
    tf.write(content)
    tf.close()
    return tf.name

def test_tender_data(intervals, periods=("enquiry", "tender")):
    now = get_now()
    value_amount = round(random.uniform(3000, 250000000000), 2)  # max value equals to budget of Ukraine in hryvnias
    data = {
        "mode": "test",
        "submissionMethodDetails": "quick",
        "description": fake.description(),
        "description_en": fake_en.sentence(nb_words=10, variable_nb_words=True),
        "description_ru": fake_ru.sentence(nb_words=10, variable_nb_words=True),
        "title": fake.title(),
        "title_en": fake_en.catch_phrase(),
        "title_ru": fake_ru.catch_phrase(),
        "procuringEntity": fake.procuringEntity(),
        "value": {
            "amount": value_amount,
            "currency": u"UAH",
            "valueAddedTaxIncluded": True
        },
        "minimalStep": {
            "amount": round(random.uniform(0.005, 0.03) * value_amount, 2),
            "currency": u"UAH"
        },
        "items": []
    }
    data["procuringEntity"]["kind"] = "other"
    new_item = test_item_data()
    data["items"].append(new_item)
    if data.get("mode") == "test":
        data["title"] = u"[ТЕСТУВАННЯ] {}".format(data["title"])
        data["title_en"] = u"[TESTING] {}".format(data["title_en"])
        data["title_ru"] = u"[ТЕСТИРОВАНИЕ] {}".format(data["title_ru"])
    period_dict = {}
    inc_dt = now
    for period_name in periods:
        period_dict[period_name + "Period"] = {}
        for i, j in zip(range(2), ("start", "end")):
            inc_dt += timedelta(minutes=intervals[period_name][i])
            period_dict[period_name + "Period"][j + "Date"] = inc_dt.isoformat()
    data.update(period_dict)
    return munchify(data)

def test_tender_data_limited(intervals, procurement_method_type):
    data = test_tender_data(intervals)
    del data["submissionMethodDetails"]
    del data["minimalStep"]
    del data["enquiryPeriod"]
    del data["tenderPeriod"]
    data["procuringEntity"]["kind"] = "general"
    data.update({"procurementMethodType": procurement_method_type, "procurementMethod": "limited"})
    if procurement_method_type == "negotiation":
        cause_variants = (
            "artContestIP",
            "noCompetition",
            "twiceUnsuccessful",
            "additionalPurchase",
            "additionalConstruction",
            "stateLegalServices"
        )
        cause = fake.random_element(cause_variants)
        data.update({"cause": cause})
    if procurement_method_type == "negotiation.quick":
        cause_variants = ('quick',)
        cause = fake.random_element(cause_variants)
        data.update({"cause": cause})
    if procurement_method_type in ("negotiation", "negotiation.quick"):
        data.update({
            "procurementMethodDetails": "quick, accelerator=1440",
            "causeDescription": fake.description()
        })
    return munchify(data)

def test_tender_data_multiple_items(intervals):
    t_data = test_tender_data(intervals)
    for _ in range(4):
        new_item = test_item_data()
        t_data['items'].append(new_item)
    return munchify(t_data)

def test_tender_data_multiple_lots(intervals):
    tender = test_tender_data(intervals)
    first_lot_id = "3c8f387879de4c38957402dbdb8b31af"
    tender['items'][0]['relatedLot'] = first_lot_id
    tender['lots'] = [test_lot_data(tender['value']['amount'])]
    tender['lots'][0]['id'] = first_lot_id
    max_lot_value_amount = max([lot['value']['amount'] for lot in tender['lots']])
    tender['value']['amount'] = max_lot_value_amount
    return munchify(tender)

def test_tender_data_meat(intervals):
    tender = munchify(test_tender_data(intervals))
    item_id = "edd0032574bf4402877ad5f362df225a"
    tender['items'][0].id = item_id
    tender.features = [
        {
            "code": "ee3e24bc17234a41bd3e3a04cc28e9c6",
            "featureOf": "tenderer",
            "title": field_with_id("f", fake.title()),
            "description": fake.description(),
            "enum": [
                {
                    "value": 0.15,
                    "title": fake.word()
                },
                {
                    "value": 0.1,
                    "title": fake.word()
                },
                {
                    "value": 0.05,
                    "title": fake.word()
                },
                {
                    "value": 0,
                    "title": fake.word()
                }
            ]
        },
        {
            "code": "48cfd91612c04125ab406374d7cc8d93",
            "featureOf": "item",
            "relatedItem": item_id,
            "title": field_with_id("f", fake.title()),
            "description": fake.description(),
            "enum": [
                {
                    "value": 0.05,
                    "title": fake.word()
                },
                {
                    "value": 0.01,
                    "title": fake.word()
                },
                {
                    "value": 0,
                    "title": fake.word()
                }
            ]
        }
    ]
    return munchify(tender)

def test_question_data():
    return munchify({
        "data": {
            "author": fake.procuringEntity(),
            "description": fake.description(),
            "title": field_with_id("q", fake.title())
        }
    })

def test_question_answer_data():
    return munchify({
        "data": {
            "answer": fake.sentence(nb_words=40, variable_nb_words=True)
        }
    })

def test_complaint_data(lot=False):
    data = munchify({
        "data": {
            "author": fake.procuringEntity(),
            "description": fake.description(),
            "title": fake.title()
        }
    })
    if lot:
        data = test_lot_complaint_data(data)
    return data

test_claim_data = test_complaint_data

def test_claim_answer_data():
    return munchify({
        "data": {
            "status": "answered",
            "resolutionType": "resolved",
            "tendererAction": fake.sentence(nb_words=10, variable_nb_words=True),
            "resolution": fake.sentence(nb_words=15, variable_nb_words=True)
        }
    })

def test_confirm_data(id):
    return munchify({
        "data": {
            "status": "active",
            "id": id
        }
    })

def test_submit_claim_data(claim_id):
    return munchify({
        "data": {
            "id": claim_id,
            "status": "claim"
        }
    })

def test_complaint_reply_data():
    return munchify({
        "data": {
            "status": "resolved"
        }
    })

def test_bid_data(mode, max_value_amount):
    bid = munchify({
        "data": {
            "tenderers": [
                fake.procuringEntity()
            ]
        }
    })
    bid.data.tenderers[0].address.countryName_en = translate_country_en(bid.data.tenderers[0].address.countryName)
    bid.data.tenderers[0].address.countryName_ru = translate_country_ru(bid.data.tenderers[0].address.countryName)
    if 'open' in mode:
        bid.data['selfEligible'] = True
        bid.data['selfQualified'] = True
    if mode == 'multiLot':
        bid.data.lotValues = list()
        for _ in range(2):
            bid.data.lotValues.append(test_bid_value(max_value_amount))
    else:
        bid.data.update(test_bid_value(max_value_amount))
    if mode == 'meat':
        bid.data.update(test_bid_params())
    return bid

def test_bid_params():
    return munchify({
        "parameters": [
            {
                "code": "ee3e24bc17234a41bd3e3a04cc28e9c6",
                "value": fake.random_element(elements=(0.15, 0.1, 0.05, 0))
            },
            {
                "code": "48cfd91612c04125ab406374d7cc8d93",
                "value": fake.random_element(elements=(0.05, 0.01, 0))
            }
        ]
    })

def test_bid_value(max_value_amount):
    return munchify({
        "value": {
            "currency": "UAH",
            "amount": round(random.uniform(1, max_value_amount), 2),
            "valueAddedTaxIncluded": True
        }
    })

def test_supplier_data():
    return munchify({
        "data": {
            "suppliers": [
                fake.procuringEntity()
            ],
            "value": {
                "amount": fake.random_int(min=1),
                "currency": "UAH",
                "valueAddedTaxIncluded": True
            }
        }
    })

def test_item_data(cpv=None):
    data = fake.fake_item(cpv)
    data["description"] = field_with_id("i", data["description"])
    days = fake.random_int(min=1, max=30)
    data["deliveryDate"] = {"endDate": (get_now() + timedelta(days=days)).isoformat()}
    data["deliveryAddress"]["countryName_en"] = translate_country_en(data["deliveryAddress"]["countryName"])
    data["deliveryAddress"]["countryName_ru"] = translate_country_ru(data["deliveryAddress"]["countryName"])
    return munchify(data)

def test_invalid_features_data():
    return [
        {
            "code": "ee3e24bc17234a41bd3e3a04cc28e9c6",
            "featureOf": "tenderer",
            "title": fake.title(),
            "description": fake.description(),
            "enum": [
                {
                    "value": 0.35,
                    "title": fake.word()
                },
                {
                    "value": 0,
                    "title": fake.word()
                }
            ]
        },
        {
            "code": "48cfd91612c04125ab406374d7cc8d93",
            "featureOf": "item",
            "relatedItem": "edd0032574bf4402877ad5f362df225a",
            "title": fake.title(),
            "description": fake.description(),
            "enum": [
                {
                    "value": 0.35,
                    "title": fake.word()
                },
                {
                    "value": 0,
                    "title": fake.word()
                }
            ]
        }
    ]

def test_lot_data(max_value_amount):
    value_amount = round(random.uniform(1, max_value_amount), 2)
    return munchify(
        {
            "description": fake.description(),
            "title": field_with_id('l', fake.title()),
            "value": {
                "currency": "UAH",
                "amount": value_amount,
                "valueAddedTaxIncluded": True
            },
            "minimalStep": {
                "currency": "UAH",
                "amount": round(random.uniform(0.005, 0.03) * value_amount, 2),
                "valueAddedTaxIncluded": True
            },
            "status": "active"
        })

def test_lot_document_data(document, lot_id):
    document.data.update({"documentOf": "lot", "relatedItem": lot_id})
    return munchify(document)

def test_lot_question_data(question, lot_id):
    question.data.update({"questionOf": "lot", "relatedItem": lot_id})
    return munchify(question)

def test_lot_complaint_data(complaint, lot_id):
    complaint.data.update({"complaintOf": "lot", "relatedItem": lot_id})
    return munchify(complaint)

def test_tender_data_openua(intervals):
    accelerator = intervals['accelerator']
    # Since `accelerator` field is not really a list containing timings
    # for a period called `acceleratorPeriod`, let's remove it :)
    del intervals['accelerator']
    # We should not provide any values for `enquiryPeriod` when creating
    # an openUA or openEU procedure. That field should not be present at all.
    # Therefore, we pass a nondefault list of periods to `test_tender_data()`.
    t_data = test_tender_data(intervals, periods=('tender',))
    t_data['procurementMethodType'] = 'aboveThresholdUA'
    t_data['procurementMethodDetails'] = 'quick, ' \
        'accelerator={}'.format(accelerator)
    t_data['procuringEntity']['kind'] = 'general'
    return t_data

def test_tender_data_openeu(intervals):
    accelerator = intervals['accelerator']
    # Since `accelerator` field is not really a list containing timings
    # for a period called `acceleratorPeriod`, let's remove it :)
    del intervals['accelerator']
    # We should not provide any values for `enquiryPeriod` when creating
    # an openUA or openEU procedure. That field should not be present at all.
    # Therefore, we pass a nondefault list of periods to `test_tender_data()`.
    t_data = test_tender_data(intervals, periods=('tender',))
    t_data['procurementMethodType'] = 'aboveThresholdEU'
    t_data['procurementMethodDetails'] = 'quick, ' \
        'accelerator={}'.format(accelerator)
    t_data['title_en'] = "[TESTING]"
    for item_number, item in enumerate(t_data['items']):
        item['description_en'] = "Test item #{}".format(item_number)
    t_data['procuringEntity']['contactPoint']['name_en'] = fake_en.name()
    t_data['procuringEntity']['contactPoint']['availableLanguage'] = "en"
    t_data['procuringEntity']['identifier']['legalName_en'] = "Institution \"Vinnytsia City Council primary and secondary general school № 10\""
    t_data['procuringEntity']['kind'] = 'general'
    return t_data
<commit_msg>Change max value of tender.value.amount<commit_after># -*- coding: utf-8 -
from datetime import timedelta
from faker import Factory
from munch import munchify
from tempfile import NamedTemporaryFile
from .local_time import get_now
from .op_faker import OP_Provider
import random

fake = Factory.create('uk_UA')
fake_ru = Factory.create('ru')
fake_en = Factory.create()
fake.add_provider(OP_Provider)

def create_fake_sentence():
    return fake.sentence(nb_words=10, variable_nb_words=True)

def field_with_id(prefix, sentence):
    return u"{}-{}: {}".format(prefix, fake.uuid4()[:8], sentence)

def translate_country_en(country):
    if country == u"Україна":
        return "Ukraine"
    else:
        raise Exception(u"Cannot translate country to english: {}".format(country))

def translate_country_ru(country):
    if country == u"Україна":
        return u"Украина"
    else:
        raise Exception(u"Cannot translate country to russian: {}".format(country))

def create_fake_doc():
    content = fake.text()
    suffix = fake.random_element(('.doc', '.docx', '.pdf'))
    prefix = "{}-{}{}".format("d", fake.uuid4()[:8], fake_en.word())
    tf = NamedTemporaryFile(delete=False, suffix=suffix, prefix=prefix)
    tf.write(content)
    tf.close()
    return tf.name

def test_tender_data(intervals, periods=("enquiry", "tender")):
    now = get_now()
    value_amount = round(random.uniform(3000, 99999999999.99), 2)  # max value equals to budget of Ukraine in hryvnias
    data = {
        "mode": "test",
        "submissionMethodDetails": "quick",
        "description": fake.description(),
        "description_en": fake_en.sentence(nb_words=10, variable_nb_words=True),
        "description_ru": fake_ru.sentence(nb_words=10, variable_nb_words=True),
        "title": fake.title(),
        "title_en": fake_en.catch_phrase(),
        "title_ru": fake_ru.catch_phrase(),
        "procuringEntity": fake.procuringEntity(),
        "value": {
            "amount": value_amount,
            "currency": u"UAH",
            "valueAddedTaxIncluded": True
        },
        "minimalStep": {
            "amount": round(random.uniform(0.005, 0.03) * value_amount, 2),
            "currency": u"UAH"
        },
        "items": []
    }
    data["procuringEntity"]["kind"] = "other"
    new_item = test_item_data()
    data["items"].append(new_item)
    if data.get("mode") == "test":
        data["title"] = u"[ТЕСТУВАННЯ] {}".format(data["title"])
        data["title_en"] = u"[TESTING] {}".format(data["title_en"])
        data["title_ru"] = u"[ТЕСТИРОВАНИЕ] {}".format(data["title_ru"])
    period_dict = {}
    inc_dt = now
    for period_name in periods:
        period_dict[period_name + "Period"] = {}
        for i, j in zip(range(2), ("start", "end")):
            inc_dt += timedelta(minutes=intervals[period_name][i])
            period_dict[period_name + "Period"][j + "Date"] = inc_dt.isoformat()
    data.update(period_dict)
    return munchify(data)

def test_tender_data_limited(intervals, procurement_method_type):
    data = test_tender_data(intervals)
    del data["submissionMethodDetails"]
    del data["minimalStep"]
    del data["enquiryPeriod"]
    del data["tenderPeriod"]
    data["procuringEntity"]["kind"] = "general"
    data.update({"procurementMethodType": procurement_method_type, "procurementMethod": "limited"})
    if procurement_method_type == "negotiation":
        cause_variants = (
            "artContestIP",
            "noCompetition",
            "twiceUnsuccessful",
            "additionalPurchase",
            "additionalConstruction",
            "stateLegalServices"
        )
        cause = fake.random_element(cause_variants)
        data.update({"cause": cause})
    if procurement_method_type == "negotiation.quick":
        cause_variants = ('quick',)
        cause = fake.random_element(cause_variants)
        data.update({"cause": cause})
    if procurement_method_type in ("negotiation", "negotiation.quick"):
        data.update({
            "procurementMethodDetails": "quick, accelerator=1440",
            "causeDescription": fake.description()
        })
    return munchify(data)

def test_tender_data_multiple_items(intervals):
    t_data = test_tender_data(intervals)
    for _ in range(4):
        new_item = test_item_data()
        t_data['items'].append(new_item)
    return munchify(t_data)

def test_tender_data_multiple_lots(intervals):
    tender = test_tender_data(intervals)
    first_lot_id = "3c8f387879de4c38957402dbdb8b31af"
    tender['items'][0]['relatedLot'] = first_lot_id
    tender['lots'] = [test_lot_data(tender['value']['amount'])]
    tender['lots'][0]['id'] = first_lot_id
    max_lot_value_amount = max([lot['value']['amount'] for lot in tender['lots']])
    tender['value']['amount'] = max_lot_value_amount
    return munchify(tender)

def test_tender_data_meat(intervals):
    tender = munchify(test_tender_data(intervals))
    item_id = "edd0032574bf4402877ad5f362df225a"
    tender['items'][0].id = item_id
    tender.features = [
        {
            "code": "ee3e24bc17234a41bd3e3a04cc28e9c6",
            "featureOf": "tenderer",
            "title": field_with_id("f", fake.title()),
            "description": fake.description(),
            "enum": [
                {
                    "value": 0.15,
                    "title": fake.word()
                },
                {
                    "value": 0.1,
                    "title": fake.word()
                },
                {
                    "value": 0.05,
                    "title": fake.word()
                },
                {
                    "value": 0,
                    "title": fake.word()
                }
            ]
        },
        {
            "code": "48cfd91612c04125ab406374d7cc8d93",
            "featureOf": "item",
            "relatedItem": item_id,
            "title": field_with_id("f", fake.title()),
            "description": fake.description(),
            "enum": [
                {
                    "value": 0.05,
                    "title": fake.word()
                },
                {
                    "value": 0.01,
                    "title": fake.word()
                },
                {
                    "value": 0,
                    "title": fake.word()
                }
            ]
        }
    ]
    return munchify(tender)

def test_question_data():
    return munchify({
        "data": {
            "author": fake.procuringEntity(),
            "description": fake.description(),
            "title": field_with_id("q", fake.title())
        }
    })

def test_question_answer_data():
    return munchify({
        "data": {
            "answer": fake.sentence(nb_words=40, variable_nb_words=True)
        }
    })

def test_complaint_data(lot=False):
    data = munchify({
        "data": {
            "author": fake.procuringEntity(),
            "description": fake.description(),
            "title": fake.title()
        }
    })
    if lot:
        data = test_lot_complaint_data(data)
    return data

test_claim_data = test_complaint_data

def test_claim_answer_data():
    return munchify({
        "data": {
            "status": "answered",
            "resolutionType": "resolved",
            "tendererAction": fake.sentence(nb_words=10, variable_nb_words=True),
            "resolution": fake.sentence(nb_words=15, variable_nb_words=True)
        }
    })

def test_confirm_data(id):
    return munchify({
        "data": {
            "status": "active",
            "id": id
        }
    })

def test_submit_claim_data(claim_id):
    return munchify({
        "data": {
            "id": claim_id,
            "status": "claim"
        }
    })

def test_complaint_reply_data():
    return munchify({
        "data": {
            "status": "resolved"
        }
    })

def test_bid_data(mode, max_value_amount):
    bid = munchify({
        "data": {
            "tenderers": [
                fake.procuringEntity()
            ]
        }
    })
    bid.data.tenderers[0].address.countryName_en = translate_country_en(bid.data.tenderers[0].address.countryName)
    bid.data.tenderers[0].address.countryName_ru = translate_country_ru(bid.data.tenderers[0].address.countryName)
    if 'open' in mode:
        bid.data['selfEligible'] = True
        bid.data['selfQualified'] = True
    if mode == 'multiLot':
        bid.data.lotValues = list()
        for _ in range(2):
            bid.data.lotValues.append(test_bid_value(max_value_amount))
    else:
        bid.data.update(test_bid_value(max_value_amount))
    if mode == 'meat':
        bid.data.update(test_bid_params())
    return bid

def test_bid_params():
    return munchify({
        "parameters": [
            {
                "code": "ee3e24bc17234a41bd3e3a04cc28e9c6",
                "value": fake.random_element(elements=(0.15, 0.1, 0.05, 0))
            },
            {
                "code": "48cfd91612c04125ab406374d7cc8d93",
                "value": fake.random_element(elements=(0.05, 0.01, 0))
            }
        ]
    })

def test_bid_value(max_value_amount):
    return munchify({
        "value": {
            "currency": "UAH",
            "amount": round(random.uniform(1, max_value_amount), 2),
            "valueAddedTaxIncluded": True
        }
    })

def test_supplier_data():
    return munchify({
        "data": {
            "suppliers": [
                fake.procuringEntity()
            ],
            "value": {
                "amount": fake.random_int(min=1),
                "currency": "UAH",
                "valueAddedTaxIncluded": True
            }
        }
    })

def test_item_data(cpv=None):
    data = fake.fake_item(cpv)
    data["description"] = field_with_id("i", data["description"])
    days = fake.random_int(min=1, max=30)
    data["deliveryDate"] = {"endDate": (get_now() + timedelta(days=days)).isoformat()}
    data["deliveryAddress"]["countryName_en"] = translate_country_en(data["deliveryAddress"]["countryName"])
    data["deliveryAddress"]["countryName_ru"] = translate_country_ru(data["deliveryAddress"]["countryName"])
    return munchify(data)

def test_invalid_features_data():
    return [
        {
            "code": "ee3e24bc17234a41bd3e3a04cc28e9c6",
            "featureOf": "tenderer",
            "title": fake.title(),
            "description": fake.description(),
            "enum": [
                {
                    "value": 0.35,
                    "title": fake.word()
                },
                {
                    "value": 0,
                    "title": fake.word()
                }
            ]
        },
        {
            "code": "48cfd91612c04125ab406374d7cc8d93",
            "featureOf": "item",
            "relatedItem": "edd0032574bf4402877ad5f362df225a",
            "title": fake.title(),
            "description": fake.description(),
            "enum": [
                {
                    "value": 0.35,
                    "title": fake.word()
                },
                {
                    "value": 0,
                    "title": fake.word()
                }
            ]
        }
    ]

def test_lot_data(max_value_amount):
    value_amount = round(random.uniform(1, max_value_amount), 2)
    return munchify(
        {
            "description": fake.description(),
            "title": field_with_id('l', fake.title()),
            "value": {
                "currency": "UAH",
                "amount": value_amount,
                "valueAddedTaxIncluded": True
            },
            "minimalStep": {
                "currency": "UAH",
                "amount": round(random.uniform(0.005, 0.03) * value_amount, 2),
                "valueAddedTaxIncluded": True
            },
            "status": "active"
        })

def test_lot_document_data(document, lot_id):
    document.data.update({"documentOf": "lot", "relatedItem": lot_id})
    return munchify(document)

def test_lot_question_data(question, lot_id):
    question.data.update({"questionOf": "lot", "relatedItem": lot_id})
    return munchify(question)

def test_lot_complaint_data(complaint, lot_id):
    complaint.data.update({"complaintOf": "lot", "relatedItem": lot_id})
    return munchify(complaint)

def test_tender_data_openua(intervals):
    accelerator = intervals['accelerator']
    # Since `accelerator` field is not really a list containing timings
    # for a period called `acceleratorPeriod`, let's remove it :)
    del intervals['accelerator']
    # We should not provide any values for `enquiryPeriod` when creating
    # an openUA or openEU procedure. That field should not be present at all.
    # Therefore, we pass a nondefault list of periods to `test_tender_data()`.
    t_data = test_tender_data(intervals, periods=('tender',))
    t_data['procurementMethodType'] = 'aboveThresholdUA'
    t_data['procurementMethodDetails'] = 'quick, ' \
        'accelerator={}'.format(accelerator)
    t_data['procuringEntity']['kind'] = 'general'
    return t_data

def test_tender_data_openeu(intervals):
    accelerator = intervals['accelerator']
    # Since `accelerator` field is not really a list containing timings
    # for a period called `acceleratorPeriod`, let's remove it :)
    del intervals['accelerator']
    # We should not provide any values for `enquiryPeriod` when creating
    # an openUA or openEU procedure. That field should not be present at all.
    # Therefore, we pass a nondefault list of periods to `test_tender_data()`.
    t_data = test_tender_data(intervals, periods=('tender',))
    t_data['procurementMethodType'] = 'aboveThresholdEU'
    t_data['procurementMethodDetails'] = 'quick, ' \
        'accelerator={}'.format(accelerator)
    t_data['title_en'] = "[TESTING]"
    for item_number, item in enumerate(t_data['items']):
        item['description_en'] = "Test item #{}".format(item_number)
    t_data['procuringEntity']['contactPoint']['name_en'] = fake_en.name()
    t_data['procuringEntity']['contactPoint']['availableLanguage'] = "en"
    t_data['procuringEntity']['identifier']['legalName_en'] = "Institution \"Vinnytsia City Council primary and secondary general school № 10\""
    t_data['procuringEntity']['kind'] = 'general'
    return t_data
<|endoftext|>


<commit_before>"""
Static file finders for Django.
https://docs.djangoproject.com/en/1.8/ref/settings/#std:setting-STATICFILES_FINDERS
Yes, this interface is private and undocumented, but we need to access it anyway.

In order to deploy Open edX in production, it's important to be able to collect
and process static assets: images, CSS, JS, fonts, etc. Django's collectstatic
system is the accepted way to do that in Django-based projects, but that system
doesn't handle every kind of collection and processing that web developers need.
Other open source projects like `Django-Pipeline`_ and `Django-Require`_ hook
into Django's collectstatic system to provide features like minification,
compression, Sass pre-processing, and require.js optimization for assets before
they are pushed to production. To make sure that themed assets are collected
and served by the system (in addition to core assets), we need to extend this
interface, as well.

.. _Django-Pipeline: http://django-pipeline.readthedocs.org/
.. _Django-Require: https://github.com/etianen/django-require
"""
import os
from collections import OrderedDict

from django.contrib.staticfiles import utils
from django.contrib.staticfiles.finders import BaseFinder
from django.utils import six

from openedx.core.djangoapps.theming.helpers import get_themes
from openedx.core.djangoapps.theming.storage import ThemeStorage

class ThemeFilesFinder(BaseFinder):
    """
    A static files finder that looks in the directory of each theme as
    specified in the source_dir attribute.
    """
    storage_class = ThemeStorage
    source_dir = 'static'

    def __init__(self, *args, **kwargs):
        # The list of themes that are handled
        self.themes = []
        # Mapping of theme names to storage instances
        self.storages = OrderedDict()
        self.customer_storages = OrderedDict()

        themes = get_themes()
        for theme in themes:
            customer_theme_storage = self.storage_class(
                os.path.join(theme.customer_specific_path, self.source_dir),
                prefix=theme.theme_dir_name,
            )
            theme_storage = self.storage_class(
                os.path.join(theme.path, self.source_dir),
                prefix=theme.theme_dir_name,
            )
            self.customer_storages[theme.theme_dir_name] = customer_theme_storage
            self.storages[theme.theme_dir_name] = theme_storage
            if theme.theme_dir_name not in self.themes:
                self.themes.append(theme.theme_dir_name)

        super(ThemeFilesFinder, self).__init__(*args, **kwargs)

    def list(self, ignore_patterns):
        """
        List all files in all app storages.
        """
        for storage in six.itervalues(self.customer_storagesstorages):
            if storage.exists(''):  # check if storage location exists
                for path in utils.get_files(storage, ignore_patterns):
                    yield path, storage

        for storage in six.itervalues(self.storages):
            if storage.exists(''):  # check if storage location exists
                for path in utils.get_files(storage, ignore_patterns):
                    yield path, storage

    def find(self, path, all=False):  # pylint: disable=redefined-builtin
        """
        Looks for files in the theme directories.
        """
        matches = []
        theme_dir_name = path.split("/", 1)[0]

        themes = {t.theme_dir_name: t for t in get_themes()}
        # if path is prefixed by theme name then search in the corresponding storage other wise search all storages.
        if theme_dir_name in themes:
            theme = themes[theme_dir_name]
            path = "/".join(path.split("/")[1:])
            match = self.find_in_theme(theme.theme_dir_name, path)
            if match:
                if not all:
                    return match
                matches.append(match)
        return matches

    def find_in_theme(self, theme, path):
        """
        Find a requested static file in an theme's static locations.
        """
        customer_storage = self.customer_storages.get(theme, None)
        if customer_storage:
            # only try to find a file if the source dir actually exists
            if customer_storage.exists(path):
                matched_path = customer_storage.path(path)
                if matched_path:
                    return matched_path

        storage = self.storages.get(theme, None)
        if storage:
            # only try to find a file if the source dir actually exists
            if storage.exists(path):
                matched_path = storage.path(path)
                if matched_path:
                    return matched_path
<commit_msg>fix typo<commit_after>"""
Static file finders for Django.
https://docs.djangoproject.com/en/1.8/ref/settings/#std:setting-STATICFILES_FINDERS
Yes, this interface is private and undocumented, but we need to access it anyway.

In order to deploy Open edX in production, it's important to be able to collect
and process static assets: images, CSS, JS, fonts, etc. Django's collectstatic
system is the accepted way to do that in Django-based projects, but that system
doesn't handle every kind of collection and processing that web developers need.
Other open source projects like `Django-Pipeline`_ and `Django-Require`_ hook
into Django's collectstatic system to provide features like minification,
compression, Sass pre-processing, and require.js optimization for assets before
they are pushed to production. To make sure that themed assets are collected
and served by the system (in addition to core assets), we need to extend this
interface, as well.

.. _Django-Pipeline: http://django-pipeline.readthedocs.org/
.. _Django-Require: https://github.com/etianen/django-require
"""
import os
from collections import OrderedDict

from django.contrib.staticfiles import utils
from django.contrib.staticfiles.finders import BaseFinder
from django.utils import six

from openedx.core.djangoapps.theming.helpers import get_themes
from openedx.core.djangoapps.theming.storage import ThemeStorage

class ThemeFilesFinder(BaseFinder):
    """
    A static files finder that looks in the directory of each theme as
    specified in the source_dir attribute.
    """
    storage_class = ThemeStorage
    source_dir = 'static'

    def __init__(self, *args, **kwargs):
        # The list of themes that are handled
        self.themes = []
        # Mapping of theme names to storage instances
        self.storages = OrderedDict()
        self.customer_storages = OrderedDict()

        themes = get_themes()
        for theme in themes:
            customer_theme_storage = self.storage_class(
                os.path.join(theme.customer_specific_path, self.source_dir),
                prefix=theme.theme_dir_name,
            )
            theme_storage = self.storage_class(
                os.path.join(theme.path, self.source_dir),
                prefix=theme.theme_dir_name,
            )
            self.customer_storages[theme.theme_dir_name] = customer_theme_storage
            self.storages[theme.theme_dir_name] = theme_storage
            if theme.theme_dir_name not in self.themes:
                self.themes.append(theme.theme_dir_name)

        super(ThemeFilesFinder, self).__init__(*args, **kwargs)

    def list(self, ignore_patterns):
        """
        List all files in all app storages.
        """
        for storage in six.itervalues(self.customer_storages):
            if storage.exists(''):  # check if storage location exists
                for path in utils.get_files(storage, ignore_patterns):
                    yield path, storage

        for storage in six.itervalues(self.storages):
            if storage.exists(''):  # check if storage location exists
                for path in utils.get_files(storage, ignore_patterns):
                    yield path, storage

    def find(self, path, all=False):  # pylint: disable=redefined-builtin
        """
        Looks for files in the theme directories.
        """
        matches = []
        theme_dir_name = path.split("/", 1)[0]

        themes = {t.theme_dir_name: t for t in get_themes()}
        # if path is prefixed by theme name then search in the corresponding storage other wise search all storages.
        if theme_dir_name in themes:
            theme = themes[theme_dir_name]
            path = "/".join(path.split("/")[1:])
            match = self.find_in_theme(theme.theme_dir_name, path)
            if match:
                if not all:
                    return match
                matches.append(match)
        return matches

    def find_in_theme(self, theme, path):
        """
        Find a requested static file in an theme's static locations.
        """
        customer_storage = self.customer_storages.get(theme, None)
        if customer_storage:
            # only try to find a file if the source dir actually exists
            if customer_storage.exists(path):
                matched_path = customer_storage.path(path)
                if matched_path:
                    return matched_path

        storage = self.storages.get(theme, None)
        if storage:
            # only try to find a file if the source dir actually exists
            if storage.exists(path):
                matched_path = storage.path(path)
                if matched_path:
                    return matched_path
<|endoftext|>


<commit_before>import os
import json

from openpassword import abstract
from openpassword.exceptions import KeyValidationException, IncorrectPasswordException
from openpassword.agile_keychain._key_manager import KeyManager

AGILE_KEYCHAIN_BASE_FILES = ['1password.keys', 'contents.js', 'encryptionKeys.js']
DEFAULT_ITERATIONS = 25000

class DataSource(abstract.DataSource):
    BUILDNUM = '32009'
    def __init__(self, path, key_manager=KeyManager):
        self._base_path = path
        self._default_folder = os.path.join(self._base_path, "data", "default")
        self._config_folder = os.path.join(self._base_path, "config")
        self._key_manager = key_manager(self._base_path)
        self._keys = []

    def create_buildnum_file(self):
        buildnum_file = os.path.join(self._config_folder, 'buildnum')
        buildnum_file = open(buildnum_file, "w+")
        buildnum_file.write(self.BUILDNUM)
        buildnum_file.close()

    def initialise(self, password):
        os.makedirs(self._default_folder)
        os.makedirs(self._config_folder)

        self.create_buildnum_file()

        for agile_keychain_base_file in AGILE_KEYCHAIN_BASE_FILES:
            open(os.path.join(self._default_folder, agile_keychain_base_file), "w+").close()

        self._initialise_key_files(password)

        self.set_password(password)

    def is_keychain_initialised(self):
        return self._validate_agile_keychain_base_files() and self._is_valid_folder(self._default_folder)

    def add_item(self, item):
        file_handle = open(os.path.join(self._default_folder, "{0}.1password".format(item['id'])), "w")
        json.dump(item, file_handle)
        file_handle.close()

    def authenticate(self, password):
        keys = self._key_manager.get_keys()

        for key in keys:
            try:
                key.decrypt_with_password(password)
            except KeyValidationException:
                raise IncorrectPasswordException

            self._keys.append(key)

    def set_password(self, password):
        for key in self._keys:
            key.encrypt_with_password(password)
            self._key_manager.save_key(key)

    def _validate_agile_keychain_base_files(self):
        is_initialised = True
        for base_file in AGILE_KEYCHAIN_BASE_FILES:
            current_file = os.path.join(self._default_folder, base_file)
            is_initialised = is_initialised and self._is_valid_file(current_file)
        return is_initialised

    def _is_valid_file(self, file):
        return os.path.exists(file) and os.path.isfile(file)

    def _is_valid_folder(self, folder):
        return os.path.exists(folder) and os.path.isdir(folder)

    def _initialise_key_files(self, password, iterations=DEFAULT_ITERATIONS):
        level3_key = self._key_manager.create_key(password, security_level='SL3', iterations=iterations)
        level5_key = self._key_manager.create_key(password, security_level='SL5', iterations=iterations)

        self._key_manager.save_key(level3_key)
        self._key_manager.save_key(level5_key)
<commit_msg>Change constants naming<commit_after>import os
import json

from openpassword import abstract
from openpassword.exceptions import KeyValidationException, IncorrectPasswordException
from openpassword.agile_keychain._key_manager import KeyManager

AGILE_KEYCHAIN_BASE_FILES = ['1password.keys', 'contents.js', 'encryptionKeys.js']
DEFAULT_ITERATIONS = 25000

class DataSource(abstract.DataSource):

    BUILD_NUMBER_FILE = 'buildnum'
    BUILD_NUMBER = '32009'

    def __init__(self, path, key_manager=KeyManager):
        self._base_path = path
        self._default_folder = os.path.join(self._base_path, "data", "default")
        self._config_folder = os.path.join(self._base_path, "config")
        self._key_manager = key_manager(self._base_path)
        self._keys = []

    def create_buildnum_file(self):
        buildnum_file = os.path.join(self._config_folder, self.BUILD_NUMBER_FILE)
        buildnum_file = open(buildnum_file, "w+")
        buildnum_file.write(self.BUILD_NUMBER)
        buildnum_file.close()

    def initialise(self, password):
        os.makedirs(self._default_folder)
        os.makedirs(self._config_folder)

        self.create_buildnum_file()

        for agile_keychain_base_file in AGILE_KEYCHAIN_BASE_FILES:
            open(os.path.join(self._default_folder, agile_keychain_base_file), "w+").close()

        self._initialise_key_files(password)

        self.set_password(password)

    def is_keychain_initialised(self):
        return self._validate_agile_keychain_base_files() and self._is_valid_folder(self._default_folder)

    def add_item(self, item):
        file_handle = open(os.path.join(self._default_folder, "{0}.1password".format(item['id'])), "w")
        json.dump(item, file_handle)
        file_handle.close()

    def authenticate(self, password):
        keys = self._key_manager.get_keys()

        for key in keys:
            try:
                key.decrypt_with_password(password)
            except KeyValidationException:
                raise IncorrectPasswordException

            self._keys.append(key)

    def set_password(self, password):
        for key in self._keys:
            key.encrypt_with_password(password)
            self._key_manager.save_key(key)

    def _validate_agile_keychain_base_files(self):
        is_initialised = True
        for base_file in AGILE_KEYCHAIN_BASE_FILES:
            current_file = os.path.join(self._default_folder, base_file)
            is_initialised = is_initialised and self._is_valid_file(current_file)
        return is_initialised

    def _is_valid_file(self, file):
        return os.path.exists(file) and os.path.isfile(file)

    def _is_valid_folder(self, folder):
        return os.path.exists(folder) and os.path.isdir(folder)

    def _initialise_key_files(self, password, iterations=DEFAULT_ITERATIONS):
        level3_key = self._key_manager.create_key(password, security_level='SL3', iterations=iterations)
        level5_key = self._key_manager.create_key(password, security_level='SL5', iterations=iterations)

        self._key_manager.save_key(level3_key)
        self._key_manager.save_key(level5_key)
<|endoftext|>


<commit_before>from gevent import monkey
monkey.patch_all()

import time
from collections import deque
from Cookie import SimpleCookie
from couchdb import Database, Session
from datetime import datetime
from design import sync_design, endDate_view
from flask import Flask, render_template, request, abort, url_for, redirect, Response
from flask.ext.assets import Environment, Bundle
from flask_redis import Redis
from http_parser.util import IOrderedDict
from memoize import Memoizer
from paste.proxy import make_proxy
from pkg_resources import parse_version
from pytz import timezone as tz
from restkit.conn import Connection
from restkit.contrib.wsgi_proxy import HostProxy
from socketpool import ConnectionPool
from sse import Sse as PySse
from urlparse import urlparse, urljoin
from werkzeug.exceptions import NotFound

from .utils import StreamWrapper

def start_response_decorated(start_response_decorated):
    def inner(status, headers):
        headers_obj = IOrderedDict(headers)
        if 'Set-Cookie' in headers_obj and ', ' in headers_obj['Set-Cookie']:
            cookie = SimpleCookie()
            cookie.load(headers_obj['Set-Cookie'])
            del headers_obj['Set-Cookie']
            headers_list = headers_obj.items()
            for key in ("auctions_loggedin", "auction_session"):
                if key in cookie:
                    headers_list += [
                        ('Set-Cookie', cookie[key].output(header="").lstrip().rstrip(','))
                    ]
            headers = headers_list
        return start_response_decorated(status, headers)
    return inner

class StreamProxy(HostProxy):
    def __init__(self, uri, event_sources_pool,
                 auction_doc_id="",
                 event_source_connection_limit=1000,
                 **kwargs):
        super(StreamProxy, self).__init__(uri, **kwargs)
        self.auction_doc_id = auction_doc_id
        self.event_source_connection_limit = event_source_connection_limit
        self.event_sources = event_sources_pool

    def add_event_source(self, stream_response):
        self.event_sources.append(stream_response)
        while len(self.event_sources) > self.event_source_connection_limit:
            ev_connection = self.event_sources.popleft()
            if not ev_connection._closed:
                ev_connection.close()

    def __call__(self, environ, start_response):
        header_map = {
            'HTTP_HOST': 'X_FORWARDED_SERVER',
            'SCRIPT_NAME': 'X_FORWARDED_SCRIPT_NAME',
            'wsgi.url_scheme': 'X_FORWARDED_SCHEME'
        }
        for key, dest in header_map.items():
            value = environ.get(key)
            if value:
                environ['HTTP_%s' % dest] = value
        environ['HTTP_X-Forwarded-Path'] = request.url
        if 'HTTP_X_FORWARDED_FOR' in environ:
            environ['HTTP_X_FORWARDED_FOR'] = ", ".join(
                [ip
                 for ip in environ['HTTP_X_FORWARDED_FOR'].split(", ")
                 if not ip.startswith("172.")]
            )
        else:
            environ['HTTP_X_FORWARDED_FOR'] = environ['REMOTE_ADDR']
        try:
            response = super(StreamProxy, self).__call__(
                environ, start_response_decorated(start_response)
            )
            stream_response = StreamWrapper(response.resp, response.connection)
            if 'event_source' in stream_response.resp.request.url:
                self.add_event_source(stream_response)
            return stream_response
        except Exception, e:
            auctions_server.logger.warning(
                "Error on request to {} with msg {}".format(request.url, e)
            )
            auctions_server.proxy_mappings.expire(str(self.auction_doc_id), 0)
            return NotFound()(environ, start_response)

auctions_server = Flask(
    __name__,
    static_url_path='',
    template_folder='templates'
)

################################################################################
assets = Environment(auctions_server)
assets.manifest = "json:manifest.json"

css = Bundle("vendor/angular-growl-2/build/angular-growl.min.css",
             "static/css/starter-template.css",
             filters='cssmin,datauri', output='min/styles_%(version)s.css')
assets.register('all_css', css)
js = Bundle("vendor/event-source-polyfill/eventsource.min.js",
            "vendor/angular-cookies/angular-cookies.min.js",
            "vendor/angular-ellipses/src/truncate.js",
            "vendor/angular-timer/dist/angular-timer.min.js",
            "vendor/angular-translate/angular-translate.min.js",
            "vendor/angular-translate-storage-cookie/angular-translate-storage-cookie.min.js",
            "vendor/angular-translate-storage-local/angular-translate-storage-local.min.js",
            "vendor/angular-growl-2/build/angular-growl.js",
            "static/js/app.js",
            "static/js/utils.js",
            "static/js/translations.js",
            "static/js/controllers.js",
            "vendor/moment/locale/uk.js",
            "vendor/moment/locale/ru.js",
            filters='jsmin', output='min/all_js_%(version)s.js')
assets.register('all_js', js)

################################################################################

@auctions_server.before_request
def before_request():
    auctions_server.logger.debug('Start {0.method}: {0.url}'.format(request))

@auctions_server.after_request
def after_request(response):
    auctions_server.logger.debug(
        'End {1.status_code} : {0.method} : {0.url} '.format(request, response)
    )
    return response

@auctions_server.route('/tenders/<auction_doc_id>')
def auction_url(auction_doc_id):
    unsupported_browser = False
    if request.user_agent.browser == 'msie':
        if parse_version(request.user_agent.version) <= parse_version('9'):
            unsupported_browser = True
        # Add to blacklist IE11
        if parse_version(request.user_agent.version) >= parse_version('11'):
            unsupported_browser = True
    elif request.user_agent.browser == 'opera':
        if 'Opera Mini' in request.user_agent.string:
            unsupported_browser = True
    url_obj = urlparse(request.url)
    request_base = u'//' + url_obj.netloc + url_obj.path + u'/'
    return render_template(
        'index.html',
        db_url=auctions_server.config.get('EXT_COUCH_DB'),
        auction_doc_id=auction_doc_id,
        unsupported_browser=unsupported_browser,
        request_base=request_base
    )

@auctions_server.route('/')
def archive_tenders_list_index():
    return render_template(
        'list.html',
        documents=reversed(
            [auction.doc
             for auction in endDate_view(auctions_server.db,
                                         startkey=time.time() * 1000,
                                         include_docs=True)
             ])
    )

@auctions_server.route('/health')
def health():
    if auctions_server.db.info():
        return 'Ok'
    abort(500)

@auctions_server.route('/archive')
def auction_list_index():
    return render_template(
        'list.html',
        documents=[auction.doc
                   for auction in endDate_view(auctions_server.db,
                                               endkey=time.time() * 1000,
                                               include_docs=True)
                   ]
    )

@auctions_server.route('/tenders/<auction_doc_id>/<path:path>',
                       methods=['GET', 'POST'])
def auctions_proxy(auction_doc_id, path):
    auctions_server.logger.debug('Auction_doc_id: {}'.format(auction_doc_id))
    proxy_path = auctions_server.proxy_mappings.get(
        str(auction_doc_id),
        auctions_server.redis.get,
        (str(auction_doc_id), ), max_age=60
    )
    auctions_server.logger.debug('Proxy path: {}'.format(proxy_path))
    if proxy_path:
        request.environ['PATH_INFO'] = '/' + path
        auctions_server.logger.debug('Start proxy to path: {}'.format(path))
        return StreamProxy(
            proxy_path,
            auction_doc_id=str(auction_doc_id),
            event_sources_pool=auctions_server.event_sources_pool,
            event_source_connection_limit=auctions_server.config['event_source_connection_limit'],
            pool=auctions_server.proxy_connection_pool,
            backend="gevent"
        )
    elif path == 'login' and auction_doc_id in auctions_server.db:
        return redirect((
            url_for('auction_url', auction_doc_id=auction_doc_id,
                    wait=1, **request.args)
        ))
    elif path == 'event_source':
        events_close = PySse()
        events_close.add_message("Close", "Disable")
        return Response(
            events_close,
            mimetype='text/event-stream',
            content_type='text/event-stream'
        )

    return abort(404)

@auctions_server.route('/get_current_server_time')
def auctions_server_current_server_time():
    response = Response(datetime.now(auctions_server.config['TIMEZONE']).isoformat())
    response.headers['Cache-Control'] = 'public, max-age=0'
    return response

def couch_server_proxy(path):
    """USED FOR DEBUG ONLY"""
    return make_proxy(
        {}, auctions_server.config['PROXY_COUCH_URL'], allowed_request_methods="",
        suppress_http_headers="")

def auth_couch_server_proxy(path):
    """USED FOR DEBUG ONLY"""
    return make_proxy(
        {}, auctions_server.config['PROXY_COUCH_URL'], allowed_request_methods="",
        suppress_http_headers="")

def make_auctions_app(global_conf,
                      redis_url='redis://localhost:7777/0',
                      external_couch_url='http://localhost:5000/auction',
                      internal_couch_url='http://localhost:9000/',
                      proxy_internal_couch_url='http://localhost:9000/',
                      auctions_db='auctions',
                      hash_secret_key='',
                      timezone='Europe/Kiev',
                      preferred_url_scheme='http',
                      debug=False,
                      auto_build=False,
                      event_source_connection_limit=1000
                      ):
    """
    [app:main]
    use = egg:openprocurement.auction#auctions_server
    redis_url = redis://:passwod@localhost:1111/0
    external_couch_url = http://localhost:1111/auction
    internal_couch_url = http://localhost:9011/
    auctions_db = auction
    timezone = Europe/Kiev
    """
    auctions_server.proxy_connection_pool = ConnectionPool(
        factory=Connection, max_size=20, backend="gevent"
    )
    auctions_server.proxy_mappings = Memoizer({})
    auctions_server.event_sources_pool = deque([])
    auctions_server.config['PREFERRED_URL_SCHEME'] = preferred_url_scheme
    auctions_server.config['REDIS_URL'] = redis_url
    auctions_server.config['event_source_connection_limit'] = int(event_source_connection_limit)
    auctions_server.config['EXT_COUCH_DB'] = urljoin(
        external_couch_url,
        auctions_db
    )
    auctions_server.add_url_rule(
        '/' + auctions_db + '/<path:path>',
        'couch_server_proxy',
        couch_server_proxy,
        methods=['GET'])
    auctions_server.add_url_rule(
        '/' + auctions_db + '/',
        'couch_server_proxy',
        couch_server_proxy,
        methods=['GET'], defaults={'path': ''})

    auctions_server.add_url_rule(
        '/' + auctions_db + '_secured/<path:path>',
        'auth_couch_server_proxy',
        auth_couch_server_proxy,
        methods=['GET'])
    auctions_server.add_url_rule(
        '/' + auctions_db + '_secured/',
        'auth_couch_server_proxy',
        auth_couch_server_proxy,
        methods=['GET'], defaults={'path': ''})

    auctions_server.config['INT_COUCH_URL'] = internal_couch_url
    auctions_server.config['PROXY_COUCH_URL'] = proxy_internal_couch_url
    auctions_server.config['COUCH_DB'] = auctions_db
    auctions_server.config['TIMEZONE'] = tz(timezone)
    auctions_server.redis = Redis(auctions_server)
    auctions_server.db = Database(
        urljoin(auctions_server.config.get('INT_COUCH_URL'),
                auctions_server.config['COUCH_DB']),
        session=Session(retry_delays=range(10))
    )
    auctions_server.config['HASH_SECRET_KEY'] = hash_secret_key
    sync_design(auctions_server.db)
    auctions_server.config['ASSETS_DEBUG'] = True if debug else False
    assets.auto_build = True if auto_build else False
    return auctions_server
<commit_msg>Improved health view<commit_after>from gevent import monkey
monkey.patch_all()

import time
from collections import deque
from Cookie import SimpleCookie
from json import dumps
from couchdb import Server, Session
from datetime import datetime
from design import sync_design, endDate_view
from flask import Flask, render_template, request, abort, url_for, redirect, Response
from flask.ext.assets import Environment, Bundle
from flask_redis import Redis
from http_parser.util import IOrderedDict
from memoize import Memoizer
from paste.proxy import make_proxy
from pkg_resources import parse_version
from pytz import timezone as tz
from restkit.conn import Connection
from restkit.contrib.wsgi_proxy import HostProxy
from socketpool import ConnectionPool
from sse import Sse as PySse
from urlparse import urlparse, urljoin
from werkzeug.exceptions import NotFound

from .utils import StreamWrapper

def start_response_decorated(start_response_decorated):
    def inner(status, headers):
        headers_obj = IOrderedDict(headers)
        if 'Set-Cookie' in headers_obj and ', ' in headers_obj['Set-Cookie']:
            cookie = SimpleCookie()
            cookie.load(headers_obj['Set-Cookie'])
            del headers_obj['Set-Cookie']
            headers_list = headers_obj.items()
            for key in ("auctions_loggedin", "auction_session"):
                if key in cookie:
                    headers_list += [
                        ('Set-Cookie', cookie[key].output(header="").lstrip().rstrip(','))
                    ]
            headers = headers_list
        return start_response_decorated(status, headers)
    return inner

class StreamProxy(HostProxy):
    def __init__(self, uri, event_sources_pool,
                 auction_doc_id="",
                 event_source_connection_limit=1000,
                 **kwargs):
        super(StreamProxy, self).__init__(uri, **kwargs)
        self.auction_doc_id = auction_doc_id
        self.event_source_connection_limit = event_source_connection_limit
        self.event_sources = event_sources_pool

    def add_event_source(self, stream_response):
        self.event_sources.append(stream_response)
        while len(self.event_sources) > self.event_source_connection_limit:
            ev_connection = self.event_sources.popleft()
            if not ev_connection._closed:
                ev_connection.close()

    def __call__(self, environ, start_response):
        header_map = {
            'HTTP_HOST': 'X_FORWARDED_SERVER',
            'SCRIPT_NAME': 'X_FORWARDED_SCRIPT_NAME',
            'wsgi.url_scheme': 'X_FORWARDED_SCHEME'
        }
        for key, dest in header_map.items():
            value = environ.get(key)
            if value:
                environ['HTTP_%s' % dest] = value
        environ['HTTP_X-Forwarded-Path'] = request.url
        if 'HTTP_X_FORWARDED_FOR' in environ:
            environ['HTTP_X_FORWARDED_FOR'] = ", ".join(
                [ip
                 for ip in environ['HTTP_X_FORWARDED_FOR'].split(", ")
                 if not ip.startswith("172.")]
            )
        else:
            environ['HTTP_X_FORWARDED_FOR'] = environ['REMOTE_ADDR']
        try:
            response = super(StreamProxy, self).__call__(
                environ, start_response_decorated(start_response)
            )
            stream_response = StreamWrapper(response.resp, response.connection)
            if 'event_source' in stream_response.resp.request.url:
                self.add_event_source(stream_response)
            return stream_response
        except Exception, e:
            auctions_server.logger.warning(
                "Error on request to {} with msg {}".format(request.url, e)
            )
            auctions_server.proxy_mappings.expire(str(self.auction_doc_id), 0)
            return NotFound()(environ, start_response)

auctions_server = Flask(
    __name__,
    static_url_path='',
    template_folder='templates'
)

################################################################################
assets = Environment(auctions_server)
assets.manifest = "json:manifest.json"

css = Bundle("vendor/angular-growl-2/build/angular-growl.min.css",
             "static/css/starter-template.css",
             filters='cssmin,datauri', output='min/styles_%(version)s.css')
assets.register('all_css', css)
js = Bundle("vendor/event-source-polyfill/eventsource.min.js",
            "vendor/angular-cookies/angular-cookies.min.js",
            "vendor/angular-ellipses/src/truncate.js",
            "vendor/angular-timer/dist/angular-timer.min.js",
            "vendor/angular-translate/angular-translate.min.js",
            "vendor/angular-translate-storage-cookie/angular-translate-storage-cookie.min.js",
            "vendor/angular-translate-storage-local/angular-translate-storage-local.min.js",
            "vendor/angular-growl-2/build/angular-growl.js",
            "static/js/app.js",
            "static/js/utils.js",
            "static/js/translations.js",
            "static/js/controllers.js",
            "vendor/moment/locale/uk.js",
            "vendor/moment/locale/ru.js",
            filters='jsmin', output='min/all_js_%(version)s.js')
assets.register('all_js', js)

################################################################################

@auctions_server.before_request
def before_request():
    auctions_server.logger.debug('Start {0.method}: {0.url}'.format(request))

@auctions_server.after_request
def after_request(response):
    auctions_server.logger.debug(
        'End {1.status_code} : {0.method} : {0.url} '.format(request, response)
    )
    return response

@auctions_server.route('/tenders/<auction_doc_id>')
def auction_url(auction_doc_id):
    unsupported_browser = False
    if request.user_agent.browser == 'msie':
        if parse_version(request.user_agent.version) <= parse_version('9'):
            unsupported_browser = True
        # Add to blacklist IE11
        if parse_version(request.user_agent.version) >= parse_version('11'):
            unsupported_browser = True
    elif request.user_agent.browser == 'opera':
        if 'Opera Mini' in request.user_agent.string:
            unsupported_browser = True
    url_obj = urlparse(request.url)
    request_base = u'//' + url_obj.netloc + url_obj.path + u'/'
    return render_template(
        'index.html',
        db_url=auctions_server.config.get('EXT_COUCH_DB'),
        auction_doc_id=auction_doc_id,
        unsupported_browser=unsupported_browser,
        request_base=request_base
    )

@auctions_server.route('/')
def archive_tenders_list_index():
    return render_template(
        'list.html',
        documents=reversed(
            [auction.doc
             for auction in endDate_view(auctions_server.db,
                                         startkey=time.time() * 1000,
                                         include_docs=True)
             ])
    )

@auctions_server.route('/health')
def health():
    data = auctions_server.couch_server.tasks()
    response = Response(dumps(data))
    if not(data and data[0]['progress'] > 95):
        response.status_code = 503
    return response

@auctions_server.route('/archive')
def auction_list_index():
    return render_template(
        'list.html',
        documents=[auction.doc
                   for auction in endDate_view(auctions_server.db,
                                               endkey=time.time() * 1000,
                                               include_docs=True)
                   ]
    )

@auctions_server.route('/tenders/<auction_doc_id>/<path:path>',
                       methods=['GET', 'POST'])
def auctions_proxy(auction_doc_id, path):
    auctions_server.logger.debug('Auction_doc_id: {}'.format(auction_doc_id))
    proxy_path = auctions_server.proxy_mappings.get(
        str(auction_doc_id),
        auctions_server.redis.get,
        (str(auction_doc_id), ), max_age=60
    )
    auctions_server.logger.debug('Proxy path: {}'.format(proxy_path))
    if proxy_path:
        request.environ['PATH_INFO'] = '/' + path
        auctions_server.logger.debug('Start proxy to path: {}'.format(path))
        return StreamProxy(
            proxy_path,
            auction_doc_id=str(auction_doc_id),
            event_sources_pool=auctions_server.event_sources_pool,
            event_source_connection_limit=auctions_server.config['event_source_connection_limit'],
            pool=auctions_server.proxy_connection_pool,
            backend="gevent"
        )
    elif path == 'login' and auction_doc_id in auctions_server.db:
        return redirect((
            url_for('auction_url', auction_doc_id=auction_doc_id,
                    wait=1, **request.args)
        ))
    elif path == 'event_source':
        events_close = PySse()
        events_close.add_message("Close", "Disable")
        return Response(
            events_close,
            mimetype='text/event-stream',
            content_type='text/event-stream'
        )

    return abort(404)

@auctions_server.route('/get_current_server_time')
def auctions_server_current_server_time():
    response = Response(datetime.now(auctions_server.config['TIMEZONE']).isoformat())
    response.headers['Cache-Control'] = 'public, max-age=0'
    return response

def couch_server_proxy(path):
    """USED FOR DEBUG ONLY"""
    return make_proxy(
        {}, auctions_server.config['PROXY_COUCH_URL'], allowed_request_methods="",
        suppress_http_headers="")

def auth_couch_server_proxy(path):
    """USED FOR DEBUG ONLY"""
    return make_proxy(
        {}, auctions_server.config['PROXY_COUCH_URL'], allowed_request_methods="",
        suppress_http_headers="")

def make_auctions_app(global_conf,
                      redis_url='redis://localhost:7777/0',
                      external_couch_url='http://localhost:5000/auction',
                      internal_couch_url='http://localhost:9000/',
                      proxy_internal_couch_url='http://localhost:9000/',
                      auctions_db='auctions',
                      hash_secret_key='',
                      timezone='Europe/Kiev',
                      preferred_url_scheme='http',
                      debug=False,
                      auto_build=False,
                      event_source_connection_limit=1000
                      ):
    """
    [app:main]
    use = egg:openprocurement.auction#auctions_server
    redis_url = redis://:passwod@localhost:1111/0
    external_couch_url = http://localhost:1111/auction
    internal_couch_url = http://localhost:9011/
    auctions_db = auction
    timezone = Europe/Kiev
    """
    auctions_server.proxy_connection_pool = ConnectionPool(
        factory=Connection, max_size=20, backend="gevent"
    )
    auctions_server.proxy_mappings = Memoizer({})
    auctions_server.event_sources_pool = deque([])
    auctions_server.config['PREFERRED_URL_SCHEME'] = preferred_url_scheme
    auctions_server.config['REDIS_URL'] = redis_url
    auctions_server.config['event_source_connection_limit'] = int(event_source_connection_limit)
    auctions_server.config['EXT_COUCH_DB'] = urljoin(
        external_couch_url,
        auctions_db
    )
    auctions_server.add_url_rule(
        '/' + auctions_db + '/<path:path>',
        'couch_server_proxy',
        couch_server_proxy,
        methods=['GET'])
    auctions_server.add_url_rule(
        '/' + auctions_db + '/',
        'couch_server_proxy',
        couch_server_proxy,
        methods=['GET'], defaults={'path': ''})

    auctions_server.add_url_rule(
        '/' + auctions_db + '_secured/<path:path>',
        'auth_couch_server_proxy',
        auth_couch_server_proxy,
        methods=['GET'])
    auctions_server.add_url_rule(
        '/' + auctions_db + '_secured/',
        'auth_couch_server_proxy',
        auth_couch_server_proxy,
        methods=['GET'], defaults={'path': ''})

    auctions_server.config['INT_COUCH_URL'] = internal_couch_url
    auctions_server.config['PROXY_COUCH_URL'] = proxy_internal_couch_url
    auctions_server.config['COUCH_DB'] = auctions_db
    auctions_server.config['TIMEZONE'] = tz(timezone)
    auctions_server.redis = Redis(auctions_server)
    auctions_server.couch_server = Server(
        auctions_server.config.get('INT_COUCH_URL'),
        session=Session(retry_delays=range(10))
    )
    if auctions_server.config['COUCH_DB'] not in auctions_server.couch_server:
        auctions_server.couch_server.create(auctions_server.config['COUCH_DB'])

    auctions_server.db = auctions_server.couch_server[auctions_server.config['COUCH_DB']]
    auctions_server.config['HASH_SECRET_KEY'] = hash_secret_key
    sync_design(auctions_server.db)
    auctions_server.config['ASSETS_DEBUG'] = True if debug else False
    assets.auto_build = True if auto_build else False
    return auctions_server
<|endoftext|>