<commit_before>"""

Parser of text intended to obtain IOCs from tweets.
First substitutions are performed and then words in the text are compared with
'(/|^)([a-z0-9.-]+\.[a-z0-9]+?)([/:]|$)'
In the case of a match it is checked whether this can be a valid domain using get_tld
There is also a whitelist for filtering out good domains.

Parameters:
    domain_whitelist : domains that will be ignored in parsing

    substitutions : semicolon separated list of pairs
                    substitutions that will be made in the text,
                    for example " .com,.com" enables parsing of one fuzzy format
                                "[.];." enables the parsing of another fuzzy format

    classification_type : string with a valid classificationtype
"""

from intelmq.lib.bot import ParserBot
from intelmq.lib.bot import utils
from intelmq.lib.exceptions import InvalidArgument
from intelmq.lib.harmonization import ClassificationType
import re

try:
    from url_normalize import url_normalize
except ImportError:
    url_normalize = None

try:
    from tld import get_tld
    from tld.utils import update_tld_names
except ImportError:
    get_tld = None
    update_tld_names = None

class TwitterParserBot(ParserBot):
    def init(self):
        if url_normalize is None:
            raise ValueError("Could not import 'url-normalize'. Please install it.")
        if get_tld is None:
            raise ValueError("Could not import 'tld'. Please install it.")
        update_tld_names()
        self.domain_whitelist = []
        if getattr(self.parameters, "domain_whitelist", '') != '':
            self.domain_whitelist.extend(self.parameters.domain_whitelist.split(','))
        self.substitutions = []
        if getattr(self.parameters, "substitutions", '') != '':
            temp = self.parameters.substitutions.split(';')
            if len(temp) % 2 != 0:
                raise InvalidArgument(
                    'substitutions',
                    got=self.parameters.substitutions,
                    expected="even number of ; separeted strings")
            for i in range(int(len(temp) / 2)):
                self.substitutions.append([temp[2 * i], temp[2 * i + 1]])
        self.classification_type = getattr(self.parameters, "classification_type", "unknown")
        if not ClassificationType.is_valid(self.classification_type):
            self.classification_type = 'unknown'

    def get_domain(self, address):
        try:
            dom = re.search('(//|^)([a-z0-9.-]*[a-z]\.[a-z]+?(?:[/:].*|$))', address).group(2)
            if not self.in_whitelist(dom):
                if get_tld(url_normalize(dom), fail_silently=True):
                    return url_normalize(dom)
            return None
        except AttributeError:
            return None
        except UnicodeError:  # url_normalize's error, happens when something weird matches regex
            self.logger.info("Caught UnicodeError on %r.", address)
            return None

    def in_whitelist(self, domain: str) -> bool:
        for dom_clean in self.domain_whitelist:
            if re.match(dom_clean, domain):
                return True
        return False

    def get_data_from_text(self, text) -> list:
        data = []
        for sub in self.substitutions:  # allows for custom matches
            text = text.replace(sub[0], sub[1])
        tweet_text = text.split()
        tweet_text = [x.strip().lower() for x in tweet_text]
        for part in tweet_text:
            domain = self.get_domain(part)
            if domain:
                data.append(domain)
        return data

    def process(self):
        report = self.receive_message()
        text = utils.base64_decode(report["raw"])
        data = self.get_data_from_text(text)
        for url in data:
            event = self.new_event(report)
            event.add('raw', text)
            event.add('source.url', url)
            event.add('classification.type', self.classification_type)
            self.send_message(event)
        self.acknowledge_message()

BOT = TwitterParserBot
<commit_msg>BUG: Fixed twitter parser whitelist, added support for internationalized TLDs<commit_after>"""

Parser of text intended to obtain IOCs from tweets.
First substitutions are performed and then words in the text are compared with
'(/|^)([a-z0-9.-]+\.[a-z0-9]+?)([/:]|$)'
In the case of a match it is checked whether this can be a valid domain using get_tld
There is also a whitelist for filtering out good domains.

Parameters:
    domain_whitelist : domains that will be ignored in parsing

    substitutions : semicolon separated list of pairs
                    substitutions that will be made in the text,
                    for example " .com,.com" enables parsing of one fuzzy format
                                "[.];." enables the parsing of another fuzzy format

    classification_type : string with a valid classificationtype
"""

from intelmq.lib.bot import ParserBot
from intelmq.lib.bot import utils
from intelmq.lib.exceptions import InvalidArgument
from intelmq.lib.harmonization import ClassificationType
import re

try:
    from url_normalize import url_normalize
except ImportError:
    url_normalize = None

try:
    from tld import get_tld
    from tld.utils import update_tld_names
except ImportError:
    get_tld = None
    update_tld_names = None

class TwitterParserBot(ParserBot):
    def init(self):
        if url_normalize is None:
            raise ValueError("Could not import 'url-normalize'. Please install it.")
        if get_tld is None:
            raise ValueError("Could not import 'tld'. Please install it.")
        update_tld_names()
        self.domain_whitelist = []
        if getattr(self.parameters, "domain_whitelist", '') != '':
            self.domain_whitelist.extend(self.parameters.domain_whitelist.split(','))
        self.substitutions = []
        if getattr(self.parameters, "substitutions", '') != '':
            temp = self.parameters.substitutions.split(';')
            if len(temp) % 2 != 0:
                raise InvalidArgument(
                    'substitutions',
                    got=self.parameters.substitutions,
                    expected="even number of ; separeted strings")
            for i in range(int(len(temp) / 2)):
                self.substitutions.append([temp[2 * i], temp[2 * i + 1]])
        self.classification_type = getattr(self.parameters, "classification_type", "unknown")
        if not ClassificationType.is_valid(self.classification_type):
            self.classification_type = 'unknown'

    def get_domain(self, address):
        try:
            dom = re.search('(//|^)([a-z0-9.-]*[a-z]\.[a-z][a-z-]*?(?:[/:].*|$))', address).group(2)
            if not self.in_whitelist(dom):
                if get_tld(url_normalize(dom), fail_silently=True):
                    return url_normalize(dom)
            return None
        except AttributeError:
            return None
        except UnicodeError:  # url_normalize's error, happens when something weird matches regex
            self.logger.info("Caught UnicodeError on %r.", address)
            return None

    def in_whitelist(self, domain: str) -> bool:
        for dom_clean in self.domain_whitelist:
            if re.search(dom_clean, domain):
                return True
        return False

    def get_data_from_text(self, text) -> list:
        data = []
        for sub in self.substitutions:  # allows for custom matches
            text = text.replace(sub[0], sub[1])
        tweet_text = text.split()
        tweet_text = [x.strip().lower() for x in tweet_text]
        for part in tweet_text:
            domain = self.get_domain(part)
            if domain:
                data.append(domain)
        return data

    def process(self):
        report = self.receive_message()
        text = utils.base64_decode(report["raw"])
        data = self.get_data_from_text(text)
        for url in data:
            event = self.new_event(report)
            event.add('raw', text)
            event.add('source.url', url)
            event.add('classification.type', self.classification_type)
            self.send_message(event)
        self.acknowledge_message()

BOT = TwitterParserBot
<|endoftext|>


<commit_before># -*- coding: UTF-8 -*-
"""
This module is used to preprocess corpora.
"""
import os
import pickle

import regex

from langdist.transliterator import get_transliterator
from langdist.util import CorpusParser
from langdist.constant import CORPUS_DIR, NON_ALPHABET_LOCALES, TRANSLITERATION_CODE, LOCALES

__author__ = 'kensk8er1017@gmail.com'

_SENTENCE_BORDER_REGEX = regex.compile(r'(?<!\w\.\w.)(?<![A-Z][a-z]\.)(?<=\.|\!|\?)\s')
_SENTENCE_BORDER_REGEX_ZH = regex.compile(r'\.|\?|\!|。|．|！|？')
_MAX_SENTENCE_LEN = 500

def _sent_tokenize(paragraph, locale):
    """Tokenize paragraph into sentences using a simple regex rule."""
    if locale in ['zh', 'ja']:
        index = 0
        sentences = list()
        for match in _SENTENCE_BORDER_REGEX_ZH.finditer(paragraph):
            sentences.append(paragraph[index: match.end(0)])
            index = match.end(0)

        if index < len(paragraph):
            sentences.append(paragraph[index:])
        return sentences
    else:
        return _SENTENCE_BORDER_REGEX.split(paragraph)

def _preprocess(paragraph, locale):
    """Preprocess corpus text."""
    # for some reason, zh text has white spaces between characters, which isn't normal for zh texts
    if locale == 'zh':
        paragraph = regex.sub(r'\s', '', paragraph)

    return (sentence.strip() for sentence in _sent_tokenize(paragraph, locale))

def preprocess_corpus(locale):
    """
    Preprocess the corpus of the specified locale and store it into a pickle file.

    :param locale: locale of the corpus to preprocess
    """
    corpus = list()
    parser = CorpusParser(locale)
    for paragraph in parser.gen_paragraphs():
        sentences = _preprocess(paragraph, locale)
        for sentence in sentences:
            if sentence and len(sentence) < _MAX_SENTENCE_LEN:
                corpus.append(sentence)

    processed_filepath = os.path.join(CORPUS_DIR, '{}.pkl'.format(locale))
    with open(processed_filepath, 'wb') as processed_file:
        pickle.dump(corpus, processed_file)

    if locale in NON_ALPHABET_LOCALES:
        transliterator = get_transliterator(locale)
        transliterated_corpus = transliterator.transliterate_corpus(corpus)
        transliterated_filepath = os.path.join(
            CORPUS_DIR, '{}-{}.pkl'.format(locale, TRANSLITERATION_CODE))
        with open(transliterated_filepath, 'wb') as processed_file:
            pickle.dump(transliterated_corpus, processed_file)

def preprocess_corpora():
    """
    Preprocess all the corpora and store them into pickle files.
    """
    for locale in LOCALES:
        if locale in NON_ALPHABET_LOCALES:
            continue
        preprocess_corpus(locale)

def load_corpus(locale):
    """Load corpus for the locale and return sentences (list of sentences (str))."""
    processed_corpus_path = os.path.join(CORPUS_DIR, '{}.pkl'.format(locale))

    if not os.path.exists(processed_corpus_path):
        preprocess_corpus(locale)

    with open(processed_corpus_path, 'rb') as corpus_file:
        sentences = pickle.load(corpus_file)
    return sentences
<commit_msg>minor debug<commit_after># -*- coding: UTF-8 -*-
"""
This module is used to preprocess corpora.
"""
import os
import pickle

import regex

from langdist.transliterator import get_transliterator
from langdist.util import CorpusParser
from langdist.constant import CORPUS_DIR, NON_ALPHABET_LOCALES, TRANSLITERATION_CODE, LOCALES

__author__ = 'kensk8er1017@gmail.com'

_SENTENCE_BORDER_REGEX = regex.compile(r'(?<!\w\.\w.)(?<![A-Z][a-z]\.)(?<=\.|\!|\?)\s')
_SENTENCE_BORDER_REGEX_ZH = regex.compile(r'\.|\?|\!|。|．|！|？')
_MAX_SENTENCE_LEN = 500

def _sent_tokenize(paragraph, locale):
    """Tokenize paragraph into sentences using a simple regex rule."""
    if locale in ['zh', 'ja']:
        index = 0
        sentences = list()
        for match in _SENTENCE_BORDER_REGEX_ZH.finditer(paragraph):
            sentences.append(paragraph[index: match.end(0)])
            index = match.end(0)

        if index < len(paragraph):
            sentences.append(paragraph[index:])
        return sentences
    else:
        return _SENTENCE_BORDER_REGEX.split(paragraph)

def _preprocess(paragraph, locale):
    """Preprocess corpus text."""
    # for some reason, zh text has white spaces between characters, which isn't normal for zh texts
    if locale == 'zh':
        paragraph = regex.sub(r'\s', '', paragraph)

    return (sentence.strip() for sentence in _sent_tokenize(paragraph, locale))

def preprocess_corpus(locale):
    """
    Preprocess the corpus of the specified locale and store it into a pickle file.

    :param locale: locale of the corpus to preprocess
    """
    corpus = list()
    parser = CorpusParser(locale)
    for paragraph in parser.gen_paragraphs():
        sentences = _preprocess(paragraph, locale)
        for sentence in sentences:
            if sentence and len(sentence) < _MAX_SENTENCE_LEN:
                corpus.append(sentence)

    processed_filepath = os.path.join(CORPUS_DIR, '{}.pkl'.format(locale))
    with open(processed_filepath, 'wb') as processed_file:
        pickle.dump(corpus, processed_file)

    if locale in NON_ALPHABET_LOCALES:
        transliterator = get_transliterator(locale)
        transliterated_corpus = transliterator.transliterate_corpus(corpus)
        transliterated_filepath = os.path.join(
            CORPUS_DIR, '{}-{}.pkl'.format(locale, TRANSLITERATION_CODE))
        with open(transliterated_filepath, 'wb') as processed_file:
            pickle.dump(transliterated_corpus, processed_file)

def preprocess_corpora():
    """
    Preprocess all the corpora and store them into pickle files.
    """
    for locale in LOCALES:
        if locale.endswith(TRANSLITERATION_CODE):
            continue
        preprocess_corpus(locale)

def load_corpus(locale):
    """Load corpus for the locale and return sentences (list of sentences (str))."""
    processed_corpus_path = os.path.join(CORPUS_DIR, '{}.pkl'.format(locale))

    if not os.path.exists(processed_corpus_path):
        preprocess_corpus(locale)

    with open(processed_corpus_path, 'rb') as corpus_file:
        sentences = pickle.load(corpus_file)
    return sentences
<|endoftext|>


<commit_before>"""

Parser of text intended to obtain IOCs from tweets.
First substitutions are performed and then words in the text are compared with
'(/|^)([a-z0-9.-]+\.[a-z0-9]+?)([/:]|$)'
In the case of a match it is checked whether this can be a valid domain using get_tld
There is also a whitelist for filtering out good domains.

Parameters:
    domain_whitelist : domains that will be ignored in parsing

    substitutions : semicolon separated list of pairs
                    substitutions that will be made in the text,
                    for example " .com,.com" enables parsing of one fuzzy format
                                "[.];." enables the parsing of another fuzzy format

    classification_type : string with a valid classificationtype
"""

from intelmq.lib.bot import ParserBot
from intelmq.lib.bot import utils
from intelmq.lib.exceptions import InvalidArgument
from intelmq.lib.harmonization import ClassificationType
import re

try:
    from url_normalize import url_normalize
except ImportError:
    url_normalize = None

try:
    from tld import get_tld
    from tld.utils import update_tld_names
except ImportError:
    get_tld = None
    update_tld_names = None

class TwitterParserBot(ParserBot):
    def init(self):
        if url_normalize is None:
            raise ValueError("Could not import 'url-normalize'. Please install it.")
        if get_tld is None:
            raise ValueError("Could not import 'tld'. Please install it.")
        update_tld_names()
        self.domain_whitelist = []
        if getattr(self.parameters, "domain_whitelist", '') != '':
            self.domain_whitelist.extend(self.parameters.domain_whitelist.split(','))
        self.substitutions = []
        if getattr(self.parameters, "substitutions", '') != '':
            temp = self.parameters.substitutions.split(';')
            if len(temp) % 2 != 0:
                raise InvalidArgument(
                    'substitutions',
                    got=self.parameters.substitutions,
                    expected="even number of ; separeted strings")
            for i in range(int(len(temp) / 2)):
                self.substitutions.append([temp[2 * i], temp[2 * i + 1]])
        self.classification_type = getattr(self.parameters, "classification_type", "unknown")
        if not ClassificationType.is_valid(self.classification_type):
            self.classification_type = 'unknown'

    def get_domain(self, address):
        try:
            dom = re.search('(//|^)([a-z0-9.-]*[a-z]\.[a-z][a-z-]*?(?:[/:].*|$))', address).group(2)
            if not self.in_whitelist(dom):
                if get_tld(url_normalize(dom), fail_silently=True):
                    return url_normalize(dom)
            return None
        except AttributeError:
            return None
        except UnicodeError:  # url_normalize's error, happens when something weird matches regex
            self.logger.info("Caught UnicodeError on %r.", address)
            return None

    def in_whitelist(self, domain: str) -> bool:
        for dom_clean in self.domain_whitelist:
            if re.search(dom_clean, domain):
                return True
        return False

    def get_data_from_text(self, text) -> list:
        data = []
        for sub in self.substitutions:  # allows for custom matches
            text = text.replace(sub[0], sub[1])
        tweet_text = text.split()
        tweet_text = [x.strip().lower() for x in tweet_text]
        for part in tweet_text:
            domain = self.get_domain(part)
            if domain:
                data.append(domain)
        return data

    def process(self):
        report = self.receive_message()
        text = utils.base64_decode(report["raw"])
        data = self.get_data_from_text(text)
        for url in data:
            event = self.new_event(report)
            event.add('raw', text)
            event.add('source.url', url)
            event.add('classification.type', self.classification_type)
            self.send_message(event)
        self.acknowledge_message()

BOT = TwitterParserBot
<commit_msg>STY: twitter parser<commit_after>r"""

Parser of text intended to obtain IOCs from tweets.
First substitutions are performed and then words in the text are compared with
'(/|^)([a-z0-9.-]+\.[a-z0-9]+?)([/:]|$)'
In the case of a match it is checked whether this can be a valid domain using get_tld
There is also a whitelist for filtering out good domains.

Parameters:
    domain_whitelist : domains that will be ignored in parsing

    substitutions : semicolon separated list of pairs
                    substitutions that will be made in the text,
                    for example " .com,.com" enables parsing of one fuzzy format
                                "[.];." enables the parsing of another fuzzy format

    classification_type : string with a valid classificationtype
"""

from intelmq.lib.bot import ParserBot
from intelmq.lib.bot import utils
from intelmq.lib.exceptions import InvalidArgument
from intelmq.lib.harmonization import ClassificationType
import re

try:
    from url_normalize import url_normalize
except ImportError:
    url_normalize = None

try:
    from tld import get_tld
    from tld.utils import update_tld_names
except ImportError:
    get_tld = None
    update_tld_names = None

class TwitterParserBot(ParserBot):
    def init(self):
        if url_normalize is None:
            raise ValueError("Could not import 'url-normalize'. Please install it.")
        if get_tld is None:
            raise ValueError("Could not import 'tld'. Please install it.")
        update_tld_names()
        self.domain_whitelist = []
        if getattr(self.parameters, "domain_whitelist", '') != '':
            self.domain_whitelist.extend(self.parameters.domain_whitelist.split(','))
        self.substitutions = []
        if getattr(self.parameters, "substitutions", '') != '':
            temp = self.parameters.substitutions.split(';')
            if len(temp) % 2 != 0:
                raise InvalidArgument(
                    'substitutions',
                    got=self.parameters.substitutions,
                    expected="even number of ; separeted strings")
            for i in range(int(len(temp) / 2)):
                self.substitutions.append([temp[2 * i], temp[2 * i + 1]])
        self.classification_type = getattr(self.parameters, "classification_type", "unknown")
        if not ClassificationType.is_valid(self.classification_type):
            self.classification_type = 'unknown'

    def get_domain(self, address):
        try:
            dom = re.search(r'(//|^)([a-z0-9.-]*[a-z]\.[a-z][a-z-]*?(?:[/:].*|$))', address).group(2)
            if not self.in_whitelist(dom):
                if get_tld(url_normalize(dom), fail_silently=True):
                    return url_normalize(dom)
            return None
        except AttributeError:
            return None
        except UnicodeError:  # url_normalize's error, happens when something weird matches regex
            self.logger.info("Caught UnicodeError on %r.", address)
            return None

    def in_whitelist(self, domain: str) -> bool:
        for dom_clean in self.domain_whitelist:
            if re.search(dom_clean, domain):
                return True
        return False

    def get_data_from_text(self, text) -> list:
        data = []
        for sub in self.substitutions:  # allows for custom matches
            text = text.replace(sub[0], sub[1])
        tweet_text = text.split()
        tweet_text = [x.strip().lower() for x in tweet_text]
        for part in tweet_text:
            domain = self.get_domain(part)
            if domain:
                data.append(domain)
        return data

    def process(self):
        report = self.receive_message()
        text = utils.base64_decode(report["raw"])
        data = self.get_data_from_text(text)
        for url in data:
            event = self.new_event(report)
            event.add('raw', text)
            event.add('source.url', url)
            event.add('classification.type', self.classification_type)
            self.send_message(event)
        self.acknowledge_message()

BOT = TwitterParserBot
<|endoftext|>


<commit_before>import argparse
import torch
from torch.autograd import Variable
import model
import data
import time
import shutil

model_names = sorted(name for name in model.__dict__
    if name.startswith("Planet")
    and callable(model.__dict__[name]))

print model_names

# def weighted_binary_cross_entropy(output, target, weights=None):
#
#     if weights is not None:
#         assert len(weights) == 2
#
#         loss = weights[1] * (target * torch.log(output)) + \
#                weights[0] * ((1 - target) * torch.log(1 - output))
#     else:
#         loss = target * torch.log(output) + (1 - target) * torch.log(1 - output)
#
#     return torch.neg(torch.mean(loss))

def weighted_multi_label_loss(p,y,w):
    return torch.neg(torch.mean(torch.sum(y*torch.log(p)*w
                                +(1.-y)*torch.log(1.-p),1)))

# class WeightedMultiLabelLoss(torch.nn.modules.loss._WeightedLoss):
#
#     def forward(self, input, target):
#         #_assert_no_grad(target)
#         weight = Variable(torch.zeros(input.size()))#self.weight.repeat(input.size(0),1))
#         return weighted_multi_label_loss(torch.sigmoid(input), target,
#                                weight)

def train(net,loader,criterion,optimizer,weight):
    net.train()
    avg_loss = 0.
    start = time.time()
    for i, (X, y) in enumerate(loader):
        input_var = torch.autograd.Variable(X)
        target_var = torch.autograd.Variable(y)
        weights = torch.autograd.Variable(weight.repeat(X.size(0),1),requires_grad=False)

        output = net(input_var)
        loss = weighted_multi_label_loss(torch.sigmoid(output),target_var,weights)#criterion(output, target_var)
        avg_loss += loss.data[0]
        optimizer.zero_grad()
        loss.backward()
        optimizer.step()
        if i%2 == 0:
            dt = time.time()-start
            pct = float(i+1)/len(loader)
            curr_loss = avg_loss / (i+1)
            print('%fs elapsed \t'
                  '%f  done \t'
                  '%f loss \t'
                  '%fs remaining'%(dt,pct*100,curr_loss,dt/pct*(1.-pct)))
    return avg_loss / len(loader)

def validate(net,loader,criterion,weight):
    net.eval()
    avg_loss = 0.
    for i, (X, y) in enumerate(loader):
        input_var = torch.autograd.Variable(X)
        target_var = torch.autograd.Variable(y)
        output = net(input_var)
        weights = torch.autograd.Variable(weight.repeat(X.size(0),1),requires_grad=False)

        output = net(input_var)
        loss = weighted_multi_label_loss(torch.sigmoid(output),target_var,weights)
        #loss = criterion(output, target_var)

        avg_loss += loss.data[0]
    return avg_loss/len(loader)

def save_model(model_state,filename='checkpoint.pth.tar',is_best=False):
    torch.save(model_state, model_state['arch']+'-'+filename)
    if is_best:
        shutil.copyfile(filename, model_state['arch']+'-best.pth.tar')

def main(args):
    # create model and optimizer
    net = model.__dict__[args.model](17)
    optimizer = torch.optim.Adam(net.parameters())
    stats = torch.load('positive.pth.tar')
    weights = (1.-stats['positive'])/stats['positive']
    #criterion = WeightedMultiLabelLoss(weight = weights)
    criterion = torch.nn.MultiLabelSoftMarginLoss(weight = weights)

    #optionally restore weights
    if args.resume is not None:
        print("=> loading checkpoint '{}'".format(args.resume))
        checkpoint = torch.load(args.resume)
        best_loss = checkpoint['score']
        net.load_state_dict(checkpoint['state_dict'])
        optimizer.load_state_dict(checkpoint['optimizer'])
    else:
        best_loss = 1e10

    # load data
    train_data = data.create_dataset(args.datapath +'/train',
                                     args.datapath + '/img_labels.csv',
                                     args.datapath + '/labels.txt',
                                     crop=224)
    val_data = data.create_dataset(args.datapath +'/val',
                                   args.datapath + '/img_labels.csv',
                                   args.datapath + '/labels.txt',
                                   crop=224)

    train_loader = torch.utils.data.DataLoader(
        train_data,
        batch_size=args.batch_size, shuffle=True,
        num_workers=args.workers)

    val_loader = torch.utils.data.DataLoader(
        val_data,
        batch_size=args.batch_size, shuffle=False,
        num_workers=args.workers)

    #run training
    patience = args.patience
    for e in range(args.nepochs):
        start = time.time()
        # run 1 training epoch
        train_loss = train(net,train_loader, criterion, optimizer,weights)
        # validate
        val_loss = validate(net, val_loader, criterion,weights)
        end = time.time()
        #checkpoint
        print ('epoch %d \t'
               'time %f \t'
               'train loss %f \t'
               'val loss %f \t'%(e,end-start,train_loss, val_loss)
        )
        model_state = {
            'epoch': e,
            'score': val_loss,
            'cfg': net.cfg,
            'arch': args.model,
            'state_dict': net.state_dict(),
            'optimizer': optimizer.state_dict()
        }
        save_model(model_state,'checkpoint.pth.tar', val_loss < best_loss)
        #early stopping
        if val_loss < best_loss:
            best_loss = val_loss
            patience = args.patience
        else:
            patience -= 1
            if patience == 0:
                print('early_stopping')
                break

if __name__ == '__main__':
    parser = argparse.ArgumentParser()
    parser.add_argument("-model", type=str, default='PlanetNet', help="model name")
    parser.add_argument("-patience", type=int, default=5, help="early stopping patience")
    parser.add_argument("-batch_size", type=int, default=128, help="batch size")
    parser.add_argument("-resume", type=str, default=None, help="resume training model file")
    parser.add_argument("-nepochs", type=int, default=100, help="max epochs")
    parser.add_argument("-workers", type=int, default=2, help="number of data loaders")
    parser.add_argument("datapath", type=str, help="data path")

    args = parser.parse_args()

    main(args)
<commit_msg>bgfix saving<commit_after>import argparse
import torch
from torch.autograd import Variable
import model
import data
import time
import shutil

model_names = sorted(name for name in model.__dict__
    if name.startswith("Planet")
    and callable(model.__dict__[name]))

print model_names

# def weighted_binary_cross_entropy(output, target, weights=None):
#
#     if weights is not None:
#         assert len(weights) == 2
#
#         loss = weights[1] * (target * torch.log(output)) + \
#                weights[0] * ((1 - target) * torch.log(1 - output))
#     else:
#         loss = target * torch.log(output) + (1 - target) * torch.log(1 - output)
#
#     return torch.neg(torch.mean(loss))

def weighted_multi_label_loss(p,y,w):
    return torch.neg(torch.mean(torch.sum(y*torch.log(p)*w
                                +(1.-y)*torch.log(1.-p),1)))

# class WeightedMultiLabelLoss(torch.nn.modules.loss._WeightedLoss):
#
#     def forward(self, input, target):
#         #_assert_no_grad(target)
#         weight = Variable(torch.zeros(input.size()))#self.weight.repeat(input.size(0),1))
#         return weighted_multi_label_loss(torch.sigmoid(input), target,
#                                weight)

def train(net,loader,criterion,optimizer,weight):
    net.train()
    avg_loss = 0.
    start = time.time()
    for i, (X, y) in enumerate(loader):
        input_var = torch.autograd.Variable(X)
        target_var = torch.autograd.Variable(y)
        weights = torch.autograd.Variable(weight.repeat(X.size(0),1),requires_grad=False)

        output = net(input_var)
        loss = weighted_multi_label_loss(torch.sigmoid(output),target_var,weights)#criterion(output, target_var)
        avg_loss += loss.data[0]
        optimizer.zero_grad()
        loss.backward()
        optimizer.step()
        if i%20 == 0:
            dt = time.time()-start
            pct = float(i+1)/len(loader)
            curr_loss = avg_loss / (i+1)
            print('%fs elapsed \t'
                  '%f  done \t'
                  '%f loss \t'
                  '%fs remaining'%(dt,pct*100,curr_loss,dt/pct*(1.-pct)))
    return avg_loss / len(loader)

def validate(net,loader,criterion,weight):
    net.eval()
    avg_loss = 0.
    for i, (X, y) in enumerate(loader):
        input_var = torch.autograd.Variable(X)
        target_var = torch.autograd.Variable(y)
        output = net(input_var)
        weights = torch.autograd.Variable(weight.repeat(X.size(0),1),requires_grad=False)

        output = net(input_var)
        loss = weighted_multi_label_loss(torch.sigmoid(output),target_var,weights)
        #loss = criterion(output, target_var)

        avg_loss += loss.data[0]
    return avg_loss/len(loader)

def save_model(model_state,filename='checkpoint.pth.tar',is_best=False):
    fname = model_state['arch']+'-'+filename
    torch.save(model_state, fname)
    if is_best:
        shutil.copyfile( fname, model_state['arch']+'-best.pth.tar')

def main(args):
    # create model and optimizer
    net = model.__dict__[args.model](17)
    optimizer = torch.optim.Adam(net.parameters())
    stats = torch.load('positive.pth.tar')
    weights = (1.-stats['positive'])/stats['positive']
    #criterion = WeightedMultiLabelLoss(weight = weights)
    criterion = torch.nn.MultiLabelSoftMarginLoss(weight = weights)

    #optionally restore weights
    if args.resume is not None:
        print("=> loading checkpoint '{}'".format(args.resume))
        checkpoint = torch.load(args.resume)
        best_loss = checkpoint['score']
        net.load_state_dict(checkpoint['state_dict'])
        optimizer.load_state_dict(checkpoint['optimizer'])
    else:
        best_loss = 1e10

    # load data
    train_data = data.create_dataset(args.datapath +'/train',
                                     args.datapath + '/img_labels.csv',
                                     args.datapath + '/labels.txt',
                                     crop=224)
    val_data = data.create_dataset(args.datapath +'/val',
                                   args.datapath + '/img_labels.csv',
                                   args.datapath + '/labels.txt',
                                   crop=224)

    train_loader = torch.utils.data.DataLoader(
        train_data,
        batch_size=args.batch_size, shuffle=True,
        num_workers=args.workers)

    val_loader = torch.utils.data.DataLoader(
        val_data,
        batch_size=args.batch_size, shuffle=False,
        num_workers=args.workers)

    #run training
    patience = args.patience
    for e in range(args.nepochs):
        start = time.time()
        # run 1 training epoch
        train_loss = train(net,train_loader, criterion, optimizer,weights)
        # validate
        val_loss = validate(net, val_loader, criterion,weights)
        end = time.time()
        #checkpoint
        print ('epoch %d \t'
               'time %f \t'
               'train loss %f \t'
               'val loss %f \t'%(e,end-start,train_loss, val_loss)
        )
        model_state = {
            'epoch': e,
            'score': val_loss,
            'cfg': net.cfg,
            'arch': args.model,
            'state_dict': net.state_dict(),
            'optimizer': optimizer.state_dict()
        }
        save_model(model_state,'checkpoint.pth.tar', val_loss < best_loss)
        #early stopping
        if val_loss < best_loss:
            best_loss = val_loss
            patience = args.patience
        else:
            patience -= 1
            if patience == 0:
                print('early_stopping')
                break

if __name__ == '__main__':
    parser = argparse.ArgumentParser()
    parser.add_argument("-model", type=str, default='PlanetNet', help="model name")
    parser.add_argument("-patience", type=int, default=5, help="early stopping patience")
    parser.add_argument("-batch_size", type=int, default=128, help="batch size")
    parser.add_argument("-resume", type=str, default=None, help="resume training model file")
    parser.add_argument("-nepochs", type=int, default=100, help="max epochs")
    parser.add_argument("-workers", type=int, default=2, help="number of data loaders")
    parser.add_argument("datapath", type=str, help="data path")

    args = parser.parse_args()

    main(args)
<|endoftext|>


<commit_before>#!/usr/bin/env python

# Build a bunch of SRPMs

import argparse
import getopt
import sys
import os
import glob
import subprocess
import shutil
import rpm
import hashlib
import time

from planex.globals import (BUILD_ROOT_DIR, SRPMS_DIR, RPMS_DIR, BUILD_DIR,
                            SPECS_GLOB)

from planex.util import (bcolours, print_col, run, dump_cmds)

TMP_RPM_PATH = "/tmp/RPMS"
RPM_TOP_DIR = os.path.join(os.getcwd(), BUILD_ROOT_DIR)
CACHE_DIR = "rpmcache"
DEFAULT_ARCH = "x86_64"

def doexec(args, inputtext=None, check=True):
    """Execute a subprocess, then return its return code, stdout and stderr"""
    myenv = os.environ.copy()
    myenv['HOME'] = RPM_TOP_DIR
    return run(args, check=check, env=myenv, inputtext=inputtext)

def get_srpm_info(srpm):
    for spec_path in glob.glob(SPECS_GLOB):
        os.unlink(spec_path)
    doexec(["rpm", "-i", srpm])
    myspecfile = glob.glob(SPECS_GLOB)[0]
    spec = rpm.ts().parseSpec(myspecfile)
    info = {}
    info['deps'] = spec.sourceHeader["requires"]
    info['arch'] = DEFAULT_ARCH
    info['packages'] = [{'name':p.header['name']} for p in spec.packages]
    info['srcrpm'] = srpm
    content_file = open(myspecfile,'r')
    info['spec'] = content_file.read()
    content_file.close()
    return info

def extract_target(srpm_infos, srpm_filename):
    """
    Given a list of srpm_info and an srpm filename, return the target arch
    """
    for srpm_info in srpm_infos:
        if srpm_info["srcrpm"] == srpm_filename:
            return srpm_info["arch"]

def get_package_to_srpm_map(srpm_info):
    pkg_map = {}
    for srpm in srpm_info:
        for package in srpm['packages']:
            pkg_map[package['name']] = srpm['srcrpm']
    return pkg_map

def get_deps(srpm_infos):
    p_to_s_map = get_package_to_srpm_map(srpm_infos)
    deps = {}
    for srpm_info in srpm_infos:
        deps[srpm_info['srcrpm']] = set()
        for dep in srpm_info['deps']:
            if dep in p_to_s_map:
                deps[srpm_info['srcrpm']].add(p_to_s_map[dep])
    return deps

def toposort2(data):
    # Ignore self dependencies.
    for key, val in data.items():
        val.discard(key)
    # Find all items that don't depend on anything.
    extra_items_in_deps = reduce(set.union,
                                 data.itervalues()) - set(data.iterkeys())
    # Add empty dependences where needed
    extra = {}
    for item in extra_items_in_deps:
        extra[item] = set()
    data.update(extra)
    result = []
    while True:
        ordered = set(item for item, dep in data.iteritems() if not dep)
        if not ordered:
            break
        result.append(ordered)
        newdata = {}
        for item, dep in data.iteritems():
            if item not in ordered:
                newdata[item] = (dep - ordered)
        data = newdata
    assert not data, ("Cyclic dependencies exist among these items:\n%s" %
                      '\n'.join(repr(x) for x in data.iteritems()))
    return result

def write_rpmmacros():
    rpmmacros = open(os.path.join(RPM_TOP_DIR, '.rpmmacros'), 'w')
    rpmmacros.write('%%_topdir %s\n' % RPM_TOP_DIR)
    rpmmacros.write('%%_rpmdir %s\n' % TMP_RPM_PATH)
    rpmmacros.close()

def find_pkg(srpm_infos, srpm):
    for srpm_info in srpm_infos:
        if srpm_info["srcrpm"] == srpm:
            return srpm_info

def get_pkg_ddeps(deps, srpm):
    if srpm in deps:
        ddeps = []
        for dep in deps[srpm]:
            ddeps.append(dep)
            for ddep in get_pkg_ddeps(deps, dep):
                ddeps.append(ddep)
        return ddeps
    else:
        return []

def get_srpm_hash(srpm_infos, external, deps, srpm):
    allpkgs = get_pkg_ddeps(deps, srpm)
    allpkgs.append(srpm)
    allpkgs.sort()
    srpm_hash = hashlib.md5()
    for mypkg in allpkgs:
        srpm_info = find_pkg(srpm_infos, mypkg)
        srpm_hash.update(srpm_info['spec'])
    srpm_hash.update(external)
    return srpm_hash.hexdigest()

def get_external_hash(external_deps):
    external_deps.sort()
    external_hash = hashlib.md5()
    for dep in external_deps:
        with open(dep, "rb") as f:
            for block in iter(lambda: f.read(1024), ""):
                external_hash.update(block)
    return external_hash.hexdigest()

def get_cache_dir(srpm_infos, external, deps, srpm):
    if not os.path.exists(CACHE_DIR):
        return None
    myhash = get_srpm_hash(srpm_infos, external, deps, srpm)
    dst_dir = os.path.join(CACHE_DIR, myhash)
    return dst_dir

def need_to_build(srpm_infos, external, deps, srpm):
    dst_dir = get_cache_dir(srpm_infos, external, deps, srpm)
    if not dst_dir:
        return True
    return (not os.path.exists(dst_dir))

def get_new_number(srpm, cache_dir):
    if cache_dir == None:
        return 1
    latest_path = os.path.join(CACHE_DIR, srpm, "latest")
    if os.path.exists(latest_path):
        latest = int(os.readlink(latest_path))
        os.remove(latest_path)
        build_number = latest+1
    else:
        try:
            os.makedirs(os.path.join(CACHE_DIR, srpm))
        except os.error:
            pass
        build_number = 1

    os.symlink("%d" % build_number, latest_path)
    num_file_path = os.path.join(CACHE_DIR, srpm, "%d" % build_number)
    num_file = open(num_file_path, 'w')
    num_file.write(cache_dir)
    num_file.close()
    return build_number

def createrepo():
    doexec(["createrepo", "--update", RPMS_DIR])

def do_build(srpm, target, build_number, use_mock, xs_build_sys):
    if xs_build_sys:
	mock = "/usr/bin/mock"
    else:
	mock = "mock"
    if use_mock:
        cmd = [mock, "--configdir=mock",
               "--resultdir=%s" % TMP_RPM_PATH, "--rebuild",
               "--target", target,
#               "--enable-plugin=tmpfs",
               "--define", "extrarelease .%d" % build_number,
               "-v", srpm]
        if not xs_build_sys:
            cmd = ["sudo"] + cmd + ["--disable-plugin=package_state"]
    else:
        cmd = ["rpmbuild", "--rebuild", "-v", "%s" % srpm,
               "--target", target, "--define",
               "_build_name_fmt %%{NAME}-%%{VERSION}-%%{RELEASE}.%%{ARCH}.rpm"]

    doexec(cmd)

    srpms = glob.glob(os.path.join(TMP_RPM_PATH, "*.src.rpm"))
    for srpm in srpms:
        print_col(bcolours.WARNING,"Removing SRPM %s" % srpm)
        os.unlink(srpm)

    return glob.glob(os.path.join(TMP_RPM_PATH, "*.rpm"))

def build_srpm(srpm, srpm_infos, external, deps, use_mock, xs_build_sys):
    cache_dir = get_cache_dir(srpm_infos, external, deps, srpm)

    if(need_to_build(srpm_infos, external, deps, srpm)):
        target = extract_target(srpm_infos, srpm)
        build_number = get_new_number(srpm, cache_dir)
        print_col(bcolours.OKGREEN, "CACHE MISS: Building %s (%d)" % (srpm, build_number))
        createrepo()

        pkgs = do_build(srpm, target, build_number, use_mock, xs_build_sys)
        if cache_dir:
            try:
                os.makedirs(cache_dir+".tmp")
                print "Archiving result in cache"
                for pkg in pkgs:
                    shutil.copy(pkg, cache_dir+".tmp")
                os.rename(cache_dir+".tmp",cache_dir)
            except:
                print bcolours.WARNING + "FAILED TO PUT BUILD RESULTS INTO CACHE"

    else:
        print_col(bcolours.OKGREEN,"CACHE HIT: Not building %s" % srpm)
        pkgs = glob.glob(os.path.join(cache_dir, "*.rpm"))
        for pkg in pkgs:
            shutil.copy(pkg, TMP_RPM_PATH)
        mytime=time.time()
        os.utime(cache_dir,(mytime,mytime))
        pkgs = glob.glob(os.path.join(TMP_RPM_PATH, "*.rpm"))

    if not use_mock:
        result = doexec(["rpm", "-U", "--force", "--nodeps"] + pkgs, check=False)
        if result['rc'] != 0:
            print "Ignoring failure installing rpm batch: %s" % pkgs
            print result['stderr']

    for pkg in pkgs:
        shutil.move(pkg, RPMS_DIR)

def parse_cmdline(argv=None):
    """
    Parse command line options
    """
    parser = argparse.ArgumentParser()
    parser.add_argument(
        '--no-mock', help="Don't use mock", action='store_true')
    parser.add_argument(
        '--xs-build-sys', help='Assume XenServer build system',
        action='store_true')
    parser.add_argument('--i686', help='Build for i686',
        action='store_true')
    parser.add_argument('--external-dependencies',
        help='External dependencies to include in the package hash',
        metavar="file", nargs="+", default=[])
    parser.add_argument('--cache-dir',
        help='Root directory of the RPM cache',
        metavar="directory", default=None)
    return parser.parse_args(argv)

def main():
    global DEFAULT_ARCH
    global CACHE_DIR

    args = parse_cmdline()
    use_mock = not args.no_mock
    xs_build_sys = args.xs_build_sys
    if args.i686:
        DEFAULT_ARCH = "i686"
    if args.cache_dir:
        CACHE_DIR = args.cache_dir

    if not os.path.isdir(SRPMS_DIR) or not os.listdir(SRPMS_DIR):
        print ("Error: No srpms found in %s; First run configure.py." %
               SRPMS_DIR)
        sys.exit(1)

    packages = glob.glob(os.path.join(SRPMS_DIR, '*.src.rpm'))
    write_rpmmacros()
    srpm_infos = [get_srpm_info(pkg) for pkg in packages]
    deps = get_deps(srpm_infos)
    order = toposort2(deps)
    external = get_external_hash(args.external_dependencies)

    for path in (TMP_RPM_PATH, BUILD_DIR, RPMS_DIR):
        if os.path.exists(path):
            print "Cleaning out directory: %s" % path
            shutil.rmtree(path)
        os.makedirs(path)
        os.chmod(path, 0777)

    createrepo()

    for batch in order:
        for srpm in batch:
            build_srpm(srpm, srpm_infos, external, deps, use_mock, xs_build_sys)

    createrepo()

if __name__ == '__main__':
    main()
<commit_msg>Use planex-cache in favour of mock<commit_after>#!/usr/bin/env python

# Build a bunch of SRPMs

import argparse
import getopt
import sys
import os
import glob
import subprocess
import shutil
import rpm
import hashlib
import time

from planex.globals import (BUILD_ROOT_DIR, SRPMS_DIR, RPMS_DIR, BUILD_DIR,
                            SPECS_GLOB)

from planex.util import (bcolours, print_col, run, dump_cmds)

TMP_RPM_PATH = "/tmp/RPMS"
RPM_TOP_DIR = os.path.join(os.getcwd(), BUILD_ROOT_DIR)
CACHE_DIR = "rpmcache"
DEFAULT_ARCH = "x86_64"

def doexec(args, inputtext=None, check=True):
    """Execute a subprocess, then return its return code, stdout and stderr"""
    myenv = os.environ.copy()
    myenv['HOME'] = RPM_TOP_DIR
    return run(args, check=check, env=myenv, inputtext=inputtext)

def get_srpm_info(srpm):
    for spec_path in glob.glob(SPECS_GLOB):
        os.unlink(spec_path)
    doexec(["rpm", "-i", srpm])
    myspecfile = glob.glob(SPECS_GLOB)[0]
    spec = rpm.ts().parseSpec(myspecfile)
    info = {}
    info['deps'] = spec.sourceHeader["requires"]
    info['arch'] = DEFAULT_ARCH
    info['packages'] = [{'name':p.header['name']} for p in spec.packages]
    info['srcrpm'] = srpm
    content_file = open(myspecfile,'r')
    info['spec'] = content_file.read()
    content_file.close()
    return info

def extract_target(srpm_infos, srpm_filename):
    """
    Given a list of srpm_info and an srpm filename, return the target arch
    """
    for srpm_info in srpm_infos:
        if srpm_info["srcrpm"] == srpm_filename:
            return srpm_info["arch"]

def get_package_to_srpm_map(srpm_info):
    pkg_map = {}
    for srpm in srpm_info:
        for package in srpm['packages']:
            pkg_map[package['name']] = srpm['srcrpm']
    return pkg_map

def get_deps(srpm_infos):
    p_to_s_map = get_package_to_srpm_map(srpm_infos)
    deps = {}
    for srpm_info in srpm_infos:
        deps[srpm_info['srcrpm']] = set()
        for dep in srpm_info['deps']:
            if dep in p_to_s_map:
                deps[srpm_info['srcrpm']].add(p_to_s_map[dep])
    return deps

def toposort2(data):
    # Ignore self dependencies.
    for key, val in data.items():
        val.discard(key)
    # Find all items that don't depend on anything.
    extra_items_in_deps = reduce(set.union,
                                 data.itervalues()) - set(data.iterkeys())
    # Add empty dependences where needed
    extra = {}
    for item in extra_items_in_deps:
        extra[item] = set()
    data.update(extra)
    result = []
    while True:
        ordered = set(item for item, dep in data.iteritems() if not dep)
        if not ordered:
            break
        result.append(ordered)
        newdata = {}
        for item, dep in data.iteritems():
            if item not in ordered:
                newdata[item] = (dep - ordered)
        data = newdata
    assert not data, ("Cyclic dependencies exist among these items:\n%s" %
                      '\n'.join(repr(x) for x in data.iteritems()))
    return result

def write_rpmmacros():
    rpmmacros = open(os.path.join(RPM_TOP_DIR, '.rpmmacros'), 'w')
    rpmmacros.write('%%_topdir %s\n' % RPM_TOP_DIR)
    rpmmacros.write('%%_rpmdir %s\n' % TMP_RPM_PATH)
    rpmmacros.close()

def find_pkg(srpm_infos, srpm):
    for srpm_info in srpm_infos:
        if srpm_info["srcrpm"] == srpm:
            return srpm_info

def get_pkg_ddeps(deps, srpm):
    if srpm in deps:
        ddeps = []
        for dep in deps[srpm]:
            ddeps.append(dep)
            for ddep in get_pkg_ddeps(deps, dep):
                ddeps.append(ddep)
        return ddeps
    else:
        return []

def get_srpm_hash(srpm_infos, external, deps, srpm):
    allpkgs = get_pkg_ddeps(deps, srpm)
    allpkgs.append(srpm)
    allpkgs.sort()
    srpm_hash = hashlib.md5()
    for mypkg in allpkgs:
        srpm_info = find_pkg(srpm_infos, mypkg)
        srpm_hash.update(srpm_info['spec'])
    srpm_hash.update(external)
    return srpm_hash.hexdigest()

def get_external_hash(external_deps):
    external_deps.sort()
    external_hash = hashlib.md5()
    for dep in external_deps:
        with open(dep, "rb") as f:
            for block in iter(lambda: f.read(1024), ""):
                external_hash.update(block)
    return external_hash.hexdigest()

def get_cache_dir(srpm_infos, external, deps, srpm):
    if not os.path.exists(CACHE_DIR):
        return None
    myhash = get_srpm_hash(srpm_infos, external, deps, srpm)
    dst_dir = os.path.join(CACHE_DIR, myhash)
    return dst_dir

def need_to_build(srpm_infos, external, deps, srpm):
    dst_dir = get_cache_dir(srpm_infos, external, deps, srpm)
    if not dst_dir:
        return True
    return (not os.path.exists(dst_dir))

def get_new_number(srpm, cache_dir):
    if cache_dir == None:
        return 1
    latest_path = os.path.join(CACHE_DIR, srpm, "latest")
    if os.path.exists(latest_path):
        latest = int(os.readlink(latest_path))
        os.remove(latest_path)
        build_number = latest+1
    else:
        try:
            os.makedirs(os.path.join(CACHE_DIR, srpm))
        except os.error:
            pass
        build_number = 1

    os.symlink("%d" % build_number, latest_path)
    num_file_path = os.path.join(CACHE_DIR, srpm, "%d" % build_number)
    num_file = open(num_file_path, 'w')
    num_file.write(cache_dir)
    num_file.close()
    return build_number

def createrepo():
    doexec(["createrepo", "--update", RPMS_DIR])

def do_build(srpm, target, build_number, use_mock, xs_build_sys):
    if xs_build_sys:
	mock = "/usr/bin/mock"
    else:
	mock = "planex-cache"
    if use_mock:
        cmd = [mock, "--configdir=mock",
               "--resultdir=%s" % TMP_RPM_PATH, "--rebuild",
               "--target", target,
#               "--enable-plugin=tmpfs",
               "--define", "extrarelease .%d" % build_number,
               "-v", srpm]
        if not xs_build_sys:
            cmd = cmd
    else:
        cmd = ["rpmbuild", "--rebuild", "-v", "%s" % srpm,
               "--target", target, "--define",
               "_build_name_fmt %%{NAME}-%%{VERSION}-%%{RELEASE}.%%{ARCH}.rpm"]

    doexec(cmd)

    srpms = glob.glob(os.path.join(TMP_RPM_PATH, "*.src.rpm"))
    for srpm in srpms:
        print_col(bcolours.WARNING,"Removing SRPM %s" % srpm)
        os.unlink(srpm)

    return glob.glob(os.path.join(TMP_RPM_PATH, "*.rpm"))

def build_srpm(srpm, srpm_infos, external, deps, use_mock, xs_build_sys):
    cache_dir = get_cache_dir(srpm_infos, external, deps, srpm)

    if(need_to_build(srpm_infos, external, deps, srpm)):
        target = extract_target(srpm_infos, srpm)
        build_number = get_new_number(srpm, cache_dir)
        print_col(bcolours.OKGREEN, "CACHE MISS: Building %s (%d)" % (srpm, build_number))
        createrepo()

        pkgs = do_build(srpm, target, build_number, use_mock, xs_build_sys)
        if cache_dir:
            try:
                os.makedirs(cache_dir+".tmp")
                print "Archiving result in cache"
                for pkg in pkgs:
                    shutil.copy(pkg, cache_dir+".tmp")
                os.rename(cache_dir+".tmp",cache_dir)
            except:
                print bcolours.WARNING + "FAILED TO PUT BUILD RESULTS INTO CACHE"

    else:
        print_col(bcolours.OKGREEN,"CACHE HIT: Not building %s" % srpm)
        pkgs = glob.glob(os.path.join(cache_dir, "*.rpm"))
        for pkg in pkgs:
            shutil.copy(pkg, TMP_RPM_PATH)
        mytime=time.time()
        os.utime(cache_dir,(mytime,mytime))
        pkgs = glob.glob(os.path.join(TMP_RPM_PATH, "*.rpm"))

    if not use_mock:
        result = doexec(["rpm", "-U", "--force", "--nodeps"] + pkgs, check=False)
        if result['rc'] != 0:
            print "Ignoring failure installing rpm batch: %s" % pkgs
            print result['stderr']

    for pkg in pkgs:
        shutil.move(pkg, RPMS_DIR)

def parse_cmdline(argv=None):
    """
    Parse command line options
    """
    parser = argparse.ArgumentParser()
    parser.add_argument(
        '--no-mock', help="Don't use mock", action='store_true')
    parser.add_argument(
        '--xs-build-sys', help='Assume XenServer build system',
        action='store_true')
    parser.add_argument('--i686', help='Build for i686',
        action='store_true')
    parser.add_argument('--external-dependencies',
        help='External dependencies to include in the package hash',
        metavar="file", nargs="+", default=[])
    parser.add_argument('--cache-dir',
        help='Root directory of the RPM cache',
        metavar="directory", default=None)
    return parser.parse_args(argv)

def main():
    global DEFAULT_ARCH
    global CACHE_DIR

    args = parse_cmdline()
    use_mock = not args.no_mock
    xs_build_sys = args.xs_build_sys
    if args.i686:
        DEFAULT_ARCH = "i686"
    if args.cache_dir:
        CACHE_DIR = args.cache_dir

    if not os.path.isdir(SRPMS_DIR) or not os.listdir(SRPMS_DIR):
        print ("Error: No srpms found in %s; First run configure.py." %
               SRPMS_DIR)
        sys.exit(1)

    packages = glob.glob(os.path.join(SRPMS_DIR, '*.src.rpm'))
    write_rpmmacros()
    srpm_infos = [get_srpm_info(pkg) for pkg in packages]
    deps = get_deps(srpm_infos)
    order = toposort2(deps)
    external = get_external_hash(args.external_dependencies)

    for path in (TMP_RPM_PATH, BUILD_DIR, RPMS_DIR):
        if os.path.exists(path):
            print "Cleaning out directory: %s" % path
            shutil.rmtree(path)
        os.makedirs(path)
        os.chmod(path, 0777)

    createrepo()

    for batch in order:
        for srpm in batch:
            build_srpm(srpm, srpm_infos, external, deps, use_mock, xs_build_sys)

    createrepo()

if __name__ == '__main__':
    main()
<|endoftext|>


<commit_before>from __future__ import unicode_literals
__author__ = 'saguas'

import frappe
import frappe.utils
from frappe.website import render, statics

def before_install():
	frappe.db.sql_ddl("""CREATE TABLE IF NOT EXISTS tabJasperSessions(
		user varchar(255) DEFAULT NULL,
		sessiondata longtext,
		lastupdate datetime(6) DEFAULT NULL,
		status varchar(20) DEFAULT NULL
		)""")

	frappe.db.sql_ddl("""CREATE TABLE IF NOT EXISTS tabJasperReqids(
		reqid varchar(255) DEFAULT NULL,
		data longtext,
		lastupdate datetime(6) DEFAULT NULL,
		KEY reqid (reqid)
		)""")

def after_install(rebuild_website=False):
	version = frappe.utils.cint(frappe.__version__.split(".", 1)[0])
	if version >= 5:
		return
	if rebuild_website:
		render.clear_cache()
		statics.sync().start()

	init_singles()
	frappe.db.commit()
	frappe.clear_cache()

def init_singles():
	singles = [single['name'] for single in frappe.get_all("DocType", filters={'issingle': True})]
	for single in singles:
		if not frappe.db.get_singles_dict(single):
			doc = frappe.new_doc(single)
			doc.ignore_mandatory=True
			doc.ignore_validate=True
			doc.save()<commit_msg>Fix list of reports.<commit_after>from __future__ import unicode_literals
__author__ = 'saguas'

import frappe
import frappe.utils
from frappe.website import render, statics

def before_install():
	frappe.db.sql_ddl("""CREATE TABLE IF NOT EXISTS tabJasperSessions(
		user varchar(255) DEFAULT NULL,
		sessiondata longtext,
		lastupdate datetime(6) DEFAULT NULL,
		status varchar(20) DEFAULT NULL
		)""")

	frappe.db.sql_ddl("""CREATE TABLE IF NOT EXISTS tabJasperReqids(
		reqid varchar(150) DEFAULT NULL,
		data longtext,
		lastupdate datetime(6) DEFAULT NULL,
		KEY reqid (reqid)
		)""")

def after_install(rebuild_website=False):
	version = frappe.utils.cint(frappe.__version__.split(".", 1)[0])
	if version >= 5:
		return
	if rebuild_website:
		render.clear_cache()
		statics.sync().start()

	init_singles()
	frappe.db.commit()
	frappe.clear_cache()

def init_singles():
	singles = [single['name'] for single in frappe.get_all("DocType", filters={'issingle': True})]
	for single in singles:
		if not frappe.db.get_singles_dict(single):
			doc = frappe.new_doc(single)
			doc.ignore_mandatory=True
			doc.ignore_validate=True
			doc.save()<|endoftext|>


<commit_before>import httplib
import os
import re
import time

import pyprometheus
import pyprometheus.contrib.uwsgi_features
import pyprometheus.registry

from flask import request, Response

from pyprometheus.utils.exposition import registry_to_text

if 'PROMETHEUS_UWSGI_SHAREDAREA' in os.environ:
    storage = pyprometheus.contrib.uwsgi_features.UWSGIStorage()
else:
    storage = pyprometheus.LocalMemoryStorage()

registry = pyprometheus.registry.BaseRegistry(storage=storage)

TIME_BUCKETS = (0.001, 0.003, 0.005, 0.010, 0.020, 0.030, 0.050, 0.075, 0.100,
                0.250, 0.500, 0.750, 1.0, 2.5, 5, 7.5, 10, 15, float('inf'))

BYTE_BUCKETS = (0, 8, 16, 64, 256, 512, 1024, 2048, 4096, float('inf'))

RETRY_BUCKETS = (1, 2, 3, 4, 5, 6, 7, 8, 9, 10, float('inf'))

# Rest of these get populated lazily with http_%d as fallback.
HTTP_STATUS = {429: 'http_too_many_requests'}

DBErrorCounter = pyprometheus.Counter(
    'oauth_database_error_total', 'Database errors.',
    ['query', 'error'], registry=registry)

DBLatencyHistorgram = pyprometheus.Histogram(
    'oauth_database_latency_seconds', 'Database query latency.',
    ['query'], buckets=TIME_BUCKETS, registry=registry)

TokenGauge = pyprometheus.Gauge(
    'oauth_tokens_count', 'Number of tokens.',
    ['state'], registry=registry)

ServerErrorCounter = pyprometheus.Counter(
    'oauth_server_error_total', 'OAuth errors returned to users.',
    ['endpoint', 'status', 'error'], registry=registry)

ServerLatencyHistogram = pyprometheus.Histogram(
    'oauth_server_latency_seconds', 'Overall request latency.',
    ['endpoint', 'status'], buckets=TIME_BUCKETS, registry=registry)

ServerRequestSizeHistogram = pyprometheus.Histogram(
    'oauth_server_request_bytes', 'Overall request size.',
    ['endpoint', 'status'], buckets=BYTE_BUCKETS, registry=registry)

ServerResponseSizeHistogram = pyprometheus.Histogram(
    'oauth_server_response_bytes', 'Overall response size.',
    ['endpoint', 'status'], buckets=BYTE_BUCKETS, registry=registry)

ClientErrorCounter = pyprometheus.Counter(
    'oauth_client_error_total', 'OAuth errors from upstream provider.',
    ['endpoint', 'status', 'error'], registry=registry)

ClientRetryHistogram = pyprometheus.Histogram(
    'oauth_client_retries', 'OAuth fetch retries.',
    ['endpoint', 'status'], buckets=RETRY_BUCKETS, registry=registry)

ClientLatencyHistogram = pyprometheus.Histogram(
    'oauth_client_latency_seconds', 'Overall request latency.',
    ['endpoint', 'status'], buckets=TIME_BUCKETS, registry=registry)

ClientResponseSizeHistogram = pyprometheus.Histogram(
    'oauth_client_response_bytes', 'Overall response size.',
    ['endpoint', 'status'], buckets=BYTE_BUCKETS, registry=registry)

def status(code):
    if code not in HTTP_STATUS:
        text = httplib.responses.get(code, str(code)).lower()
        HTTP_STATUS[code] = 'http_%s' % re.sub(r'[ -]', '_', text)
    return HTTP_STATUS[code]

def endpoint():
    return getattr(request.url_rule, 'endpoint', 'notfound')

def before_request():
    request._stats_latency_start_time = time.time()

def after_request(response):
    request_latency = time.time() - request._stats_latency_start_time
    labels = {'endpoint': endpoint(), 'status': status(response.status_code)}

    ServerLatencyHistogram.labels(**labels).observe(request_latency)
    if response.content_length >= 0:
        ServerResponseSizeHistogram.labels(**labels).observe(
            response.content_length)
    if request.content_length >= 0:
        ServerRequestSizeHistogram.labels(**labels).observe(
            request.content_length)
    return response

def export_metrics():
    return Response(registry_to_text(registry), mimetype="text/plain")
<commit_msg>Rebucket stats<commit_after>import httplib
import os
import re
import time

import pyprometheus
import pyprometheus.contrib.uwsgi_features
import pyprometheus.registry

from flask import request, Response

from pyprometheus.utils.exposition import registry_to_text

if 'PROMETHEUS_UWSGI_SHAREDAREA' in os.environ:
    storage = pyprometheus.contrib.uwsgi_features.UWSGIStorage()
else:
    storage = pyprometheus.LocalMemoryStorage()

registry = pyprometheus.registry.BaseRegistry(storage=storage)

TIME_BUCKETS = (0.0001, 0.00055,
                0.001, 0.0028, 0.0046, 0.0064, 0.0082,
                0.01, 0.028, 0.046, 0.064, 0.082,
                0.1, 0.4, 0.7,
                1.0, 4.0, 7.0,
                10.0, float('inf'))

BYTE_BUCKETS = (8, 22, 36, 50,
                64, 176, 288, 400,
                512, 1408, 2304, 3200,
                4096, 11264, 18432, 25600,
                32768, 90112, 147456, 204800,
                float('inf'))

RETRY_BUCKETS = (1, 2, 3, 4, 5, 6, 7, 8, 9, 10, float('inf'))

# Rest of these get populated lazily with http_%d as fallback.
HTTP_STATUS = {429: 'http_too_many_requests'}

DBErrorCounter = pyprometheus.Counter(
    'oauth_database_error_total', 'Database errors.',
    ['query', 'error'], registry=registry)

DBLatencyHistorgram = pyprometheus.Histogram(
    'oauth_database_latency_seconds', 'Database query latency.',
    ['query'], buckets=TIME_BUCKETS, registry=registry)

TokenGauge = pyprometheus.Gauge(
    'oauth_tokens_count', 'Number of tokens.',
    ['state'], registry=registry)

ServerErrorCounter = pyprometheus.Counter(
    'oauth_server_error_total', 'OAuth errors returned to users.',
    ['endpoint', 'status', 'error'], registry=registry)

ServerLatencyHistogram = pyprometheus.Histogram(
    'oauth_server_latency_seconds', 'Overall request latency.',
    ['endpoint', 'status'], buckets=TIME_BUCKETS, registry=registry)

ServerRequestSizeHistogram = pyprometheus.Histogram(
    'oauth_server_request_bytes', 'Overall request size.',
    ['endpoint', 'status'], buckets=BYTE_BUCKETS, registry=registry)

ServerResponseSizeHistogram = pyprometheus.Histogram(
    'oauth_server_response_bytes', 'Overall response size.',
    ['endpoint', 'status'], buckets=BYTE_BUCKETS, registry=registry)

ClientErrorCounter = pyprometheus.Counter(
    'oauth_client_error_total', 'OAuth errors from upstream provider.',
    ['endpoint', 'status', 'error'], registry=registry)

ClientRetryHistogram = pyprometheus.Histogram(
    'oauth_client_retries', 'OAuth fetch retries.',
    ['endpoint', 'status'], buckets=RETRY_BUCKETS, registry=registry)

ClientLatencyHistogram = pyprometheus.Histogram(
    'oauth_client_latency_seconds', 'Overall request latency.',
    ['endpoint', 'status'], buckets=TIME_BUCKETS, registry=registry)

ClientResponseSizeHistogram = pyprometheus.Histogram(
    'oauth_client_response_bytes', 'Overall response size.',
    ['endpoint', 'status'], buckets=BYTE_BUCKETS, registry=registry)

def status(code):
    if code not in HTTP_STATUS:
        text = httplib.responses.get(code, str(code)).lower()
        HTTP_STATUS[code] = 'http_%s' % re.sub(r'[ -]', '_', text)
    return HTTP_STATUS[code]

def endpoint():
    return getattr(request.url_rule, 'endpoint', 'notfound')

def before_request():
    request._stats_latency_start_time = time.time()

def after_request(response):
    request_latency = time.time() - request._stats_latency_start_time
    labels = {'endpoint': endpoint(), 'status': status(response.status_code)}

    ServerLatencyHistogram.labels(**labels).observe(request_latency)
    if response.content_length >= 0:
        ServerResponseSizeHistogram.labels(**labels).observe(
            response.content_length)
    if request.content_length >= 0:
        ServerRequestSizeHistogram.labels(**labels).observe(
            request.content_length)
    return response

def export_metrics():
    return Response(registry_to_text(registry), mimetype="text/plain")
<|endoftext|>


<commit_before>#!/usr/bin/python3

from utils.kodiBasics import *
from utils.youtube import *

itemCount = int(sys.argv[1]) # number of results to add for each search query

for query in sys.argv[2:]:
    # search youtube and create a list of urls
    searchResultsJson = ytSearch(query);
    urls = []
    for idx, res in enumerate(searchResultsJson['items']):
        id = res['id']['videoId']
        if idx < itemCount:
            print(str(idx+1) + '. ' + res['snippet']['title'])
            urls.append("plugin://plugin.video.youtube/?action=play_video&videoid={}".format(id))
        else:
            break
    clearPlaylist()
    createPlaylist(urls)
setShuffle(False)
playPlaylist()
<commit_msg>added failsafe to searching for videos<commit_after>#!/usr/bin/python3

from utils.kodiBasics import *
from utils.youtube import *

itemCount = int(sys.argv[1]) # number of results to add for each search query

for query in sys.argv[2:]:
    # search youtube and create a list of urls
    searchResultsJson = ytSearch(query);
    urls = []
    for idx, res in enumerate(searchResultsJson['items']):
        try:
            id = res['id']['videoId']
            if idx < itemCount:
                print(str(idx+1) + '. ' + res['snippet']['title'])
                urls.append("plugin://plugin.video.youtube/?action=play_video&videoid={}".format(id))
            else:
                break
        except:
            print('an error occured when processing {}th item'.format(idx))

clearPlaylist()
createPlaylist(urls)
setShuffle(False)
playPlaylist()
<|endoftext|>


<commit_before>from . import ocookie

def parse_cookies(cpwt_cookies):
    '''Parses self.cookies after a getPage call.
    '''

    cookies = [ocookie.CookieParser.parse_set_cookie_value(cookie_header[1]) for cookie_header in cpwt_cookies]
    return cookies

class CpwtCookieJar(ocookie.CookieJar):
    def update(self, cpwt_cookies):
        cookies = parse_cookies(cpwt_cookies)
        for cookie in cookies:
            self.add(cookie)
<commit_msg>cherrypywebtest import chase<commit_after>from . import CookieParser, CookieJar

def parse_cookies(cpwt_cookies):
    '''Parses self.cookies after a getPage call.
    '''

    cookies = [CookieParser.parse_set_cookie_value(cookie_header[1]) for cookie_header in cpwt_cookies]
    return cookies

class CpwtCookieJar(CookieJar):
    def update(self, cpwt_cookies):
        cookies = parse_cookies(cpwt_cookies)
        for cookie in cookies:
            self.add(cookie)
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
##############################################################################
#    Copyright (C) 2004-2014 OpenERP s.a. (<http://www.openerp.com>).
#
#    This program is free software: you can redistribute it and/or modify
#    it under the terms of the GNU Affero General Public License as
#    published by the Free Software Foundation, either version 3 of the
#    License, or (at your option) any later version.
#
#    This program is distributed in the hope that it will be useful,
#    but WITHOUT ANY WARRANTY; without even the implied warranty of
#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#    GNU Affero General Public License for more details.
#
#    You should have received a copy of the GNU Affero General Public License
#    along with this program.  If not, see <http://www.gnu.org/licenses/>.
#
##############################################################################

"""
safe_eval module - methods intended to provide more restricted alternatives to
                   evaluate simple and/or untrusted code.

Methods in this module are typically used as alternatives to eval() to parse
OpenERP domain strings, conditions and expressions, mostly based on locals
condition/math builtins.
"""

# Module partially ripped from/inspired by several different sources:
#  - http://code.activestate.com/recipes/286134/
#  - safe_eval in lp:~xrg/openobject-server/optimize-5.0
#  - safe_eval in tryton http://hg.tryton.org/hgwebdir.cgi/trytond/rev/bbb5f73319ad

from opcode import HAVE_ARGUMENT, opmap, opname
from psycopg2 import OperationalError
from types import CodeType
import logging

from .misc import ustr

import openerp

__all__ = ['test_expr', 'safe_eval', 'const_eval']

# The time module is usually already provided in the safe_eval environment
# but some code, e.g. datetime.datetime.now() (Windows/Python 2.5.2, bug
# lp:703841), does import time.
_ALLOWED_MODULES = ['_strptime', 'math', 'time']

_UNSAFE_ATTRIBUTES = ['f_builtins', 'f_globals', 'f_locals', 'gi_frame',
                      'co_code', 'func_globals']

_CONST_OPCODES = set(opmap[x] for x in [
    'POP_TOP', 'ROT_TWO', 'ROT_THREE', 'ROT_FOUR', 'DUP_TOP', 'DUP_TOPX',
    'POP_BLOCK','SETUP_LOOP', 'BUILD_LIST', 'BUILD_MAP', 'BUILD_TUPLE',
    'LOAD_CONST', 'RETURN_VALUE', 'STORE_SUBSCR', 'STORE_MAP'] if x in opmap)

_EXPR_OPCODES = _CONST_OPCODES.union(set(opmap[x] for x in [
    'UNARY_POSITIVE', 'UNARY_NEGATIVE', 'UNARY_NOT',
    'UNARY_INVERT', 'BINARY_POWER', 'BINARY_MULTIPLY',
    'BINARY_DIVIDE', 'BINARY_FLOOR_DIVIDE', 'BINARY_TRUE_DIVIDE',
    'BINARY_MODULO', 'BINARY_ADD', 'BINARY_SUBTRACT', 'BINARY_SUBSCR',
    'BINARY_LSHIFT', 'BINARY_RSHIFT', 'BINARY_AND', 'BINARY_XOR',
    'BINARY_OR', 'INPLACE_ADD', 'INPLACE_SUBTRACT', 'INPLACE_MULTIPLY',
    'INPLACE_DIVIDE', 'INPLACE_REMAINDER', 'INPLACE_POWER',
    'INPLACE_LEFTSHIFT', 'INPLACE_RIGHTSHIFT', 'INPLACE_AND',
    'INPLACE_XOR','INPLACE_OR'
    ] if x in opmap))

_SAFE_OPCODES = _EXPR_OPCODES.union(set(opmap[x] for x in [
    'LOAD_NAME', 'CALL_FUNCTION', 'COMPARE_OP', 'LOAD_ATTR',
    'STORE_NAME', 'GET_ITER', 'FOR_ITER', 'LIST_APPEND', 'DELETE_NAME',
    'JUMP_FORWARD', 'JUMP_IF_TRUE', 'JUMP_IF_FALSE', 'JUMP_ABSOLUTE',
    'MAKE_FUNCTION', 'SLICE+0', 'SLICE+1', 'SLICE+2', 'SLICE+3', 'BREAK_LOOP',
    'CONTINUE_LOOP', 'RAISE_VARARGS', 'YIELD_VALUE',
    # New in Python 2.7 - http://bugs.python.org/issue4715 :
    'JUMP_IF_FALSE_OR_POP', 'JUMP_IF_TRUE_OR_POP', 'POP_JUMP_IF_FALSE',
    'POP_JUMP_IF_TRUE', 'SETUP_EXCEPT', 'END_FINALLY',
    'LOAD_FAST', 'STORE_FAST', 'DELETE_FAST', 'UNPACK_SEQUENCE',
    'LOAD_GLOBAL', # Only allows access to restricted globals
    ] if x in opmap))

_logger = logging.getLogger(__name__)

def _get_opcodes(codeobj):
    """_get_opcodes(codeobj) -> [opcodes]

    Extract the actual opcodes as a list from a code object

    >>> c = compile("[1 + 2, (1,2)]", "", "eval")
    >>> _get_opcodes(c)
    [100, 100, 23, 100, 100, 102, 103, 83]
    """
    i = 0
    byte_codes = codeobj.co_code
    while i < len(byte_codes):
        code = ord(byte_codes[i])
        yield code

        if code >= HAVE_ARGUMENT:
            i += 3
        else:
            i += 1

def assert_no_dunder_name(code_obj, expr):
    """ assert_no_dunder_name(code_obj, expr) -> None

    Asserts that the code object does not refer to any "dunder name"
    (__$name__), so that safe_eval prevents access to any internal-ish Python
    attribute or method (both are loaded via LOAD_ATTR which uses a name, not a
    const or a var).

    Checks that no such name exists in the provided code object (co_names).

    :param code_obj: code object to name-validate
    :type code_obj: CodeType
    :param str expr: expression corresponding to the code object, for debugging
                     purposes
    :raises NameError: in case a forbidden name (containing two underscores)
                       is found in ``code_obj``

    .. note:: actually forbids every name containing 2 underscores
    """
    for name in code_obj.co_names:
        if "__" in name or name in _UNSAFE_ATTRIBUTES:
            raise NameError('Access to forbidden name %r (%r)' % (name, expr))

def assert_valid_codeobj(allowed_codes, code_obj, expr):
    """ Asserts that the provided code object validates against the bytecode
    and name constraints.

    Recursively validates the code objects stored in its co_consts in case
    lambdas are being created/used (lambdas generate their own separated code
    objects and don't live in the root one)

    :param allowed_codes: list of permissible bytecode instructions
    :type allowed_codes: set(int)
    :param code_obj: code object to name-validate
    :type code_obj: CodeType
    :param str expr: expression corresponding to the code object, for debugging
                     purposes
    :raises ValueError: in case of forbidden bytecode in ``code_obj``
    :raises NameError: in case a forbidden name (containing two underscores)
                       is found in ``code_obj``
    """
    assert_no_dunder_name(code_obj, expr)
    for opcode in _get_opcodes(code_obj):
        if opcode not in allowed_codes:
            raise ValueError(
                "opcode %s not allowed (%r)" % (opname[opcode], expr))
    for const in code_obj.co_consts:
        if isinstance(const, CodeType):
            assert_valid_codeobj(allowed_codes, const, 'lambda')

def test_expr(expr, allowed_codes, mode="eval"):
    """test_expr(expression, allowed_codes[, mode]) -> code_object

    Test that the expression contains only the allowed opcodes.
    If the expression is valid and contains only allowed codes,
    return the compiled code object.
    Otherwise raise a ValueError, a Syntax Error or TypeError accordingly.
    """
    try:
        if mode == 'eval':
            # eval() does not like leading/trailing whitespace
            expr = expr.strip()
        code_obj = compile(expr, "", mode)
    except (SyntaxError, TypeError, ValueError):
        raise
    except Exception, e:
        import sys
        exc_info = sys.exc_info()
        raise ValueError, '"%s" while compiling\n%r' % (ustr(e), expr), exc_info[2]
    assert_valid_codeobj(allowed_codes, code_obj, expr)
    return code_obj

def const_eval(expr):
    """const_eval(expression) -> value

    Safe Python constant evaluation

    Evaluates a string that contains an expression describing
    a Python constant. Strings that are not valid Python expressions
    or that contain other code besides the constant raise ValueError.

    >>> const_eval("10")
    10
    >>> const_eval("[1,2, (3,4), {'foo':'bar'}]")
    [1, 2, (3, 4), {'foo': 'bar'}]
    >>> const_eval("1+2")
    Traceback (most recent call last):
    ...
    ValueError: opcode BINARY_ADD not allowed
    """
    c = test_expr(expr, _CONST_OPCODES)
    return eval(c)

def expr_eval(expr):
    """expr_eval(expression) -> value

    Restricted Python expression evaluation

    Evaluates a string that contains an expression that only
    uses Python constants. This can be used to e.g. evaluate
    a numerical expression from an untrusted source.

    >>> expr_eval("1+2")
    3
    >>> expr_eval("[1,2]*2")
    [1, 2, 1, 2]
    >>> expr_eval("__import__('sys').modules")
    Traceback (most recent call last):
    ...
    ValueError: opcode LOAD_NAME not allowed
    """
    c = test_expr(expr, _EXPR_OPCODES)
    return eval(c)

def _import(name, globals=None, locals=None, fromlist=None, level=-1):
    if globals is None:
        globals = {}
    if locals is None:
        locals = {}
    if fromlist is None:
        fromlist = []
    if name in _ALLOWED_MODULES:
        return __import__(name, globals, locals, level)
    raise ImportError(name)

def safe_eval(expr, globals_dict=None, locals_dict=None, mode="eval", nocopy=False, locals_builtins=False):
    """safe_eval(expression[, globals[, locals[, mode[, nocopy]]]]) -> result

    System-restricted Python expression evaluation

    Evaluates a string that contains an expression that mostly
    uses Python constants, arithmetic expressions and the
    objects directly provided in context.

    This can be used to e.g. evaluate
    an OpenERP domain expression from an untrusted source.

    :throws TypeError: If the expression provided is a code object
    :throws SyntaxError: If the expression provided is not valid Python
    :throws NameError: If the expression provided accesses forbidden names
    :throws ValueError: If the expression provided uses forbidden bytecode
    """
    if isinstance(expr, CodeType):
        raise TypeError("safe_eval does not allow direct evaluation of code objects.")

    if globals_dict is None:
        globals_dict = {}

    # prevent altering the globals/locals from within the sandbox
    # by taking a copy.
    if not nocopy:
        # isinstance() does not work below, we want *exactly* the dict class
        if (globals_dict is not None and type(globals_dict) is not dict) \
            or (locals_dict is not None and type(locals_dict) is not dict):
            _logger.warning(
                "Looks like you are trying to pass a dynamic environment, "
                "you should probably pass nocopy=True to safe_eval().")

        globals_dict = dict(globals_dict)
        if locals_dict is not None:
            locals_dict = dict(locals_dict)

    globals_dict.update(
        __builtins__={
            '__import__': _import,
            'True': True,
            'False': False,
            'None': None,
            'str': str,
            'unicode': unicode,
            'bool': bool,
            'int': int,
            'float': float,
            'long': long,
            'enumerate': enumerate,
            'dict': dict,
            'list': list,
            'tuple': tuple,
            'map': map,
            'abs': abs,
            'min': min,
            'max': max,
            'sum': sum,
            'reduce': reduce,
            'filter': filter,
            'round': round,
            'len': len,
            'repr': repr,
            'set': set,
            'all': all,
            'any': any,
            'ord': ord,
            'chr': chr,
            'cmp': cmp,
            'divmod': divmod,
            'isinstance': isinstance,
            'range': range,
            'xrange': xrange,
            'zip': zip,
            'Exception': Exception,
        }
    )
    if locals_builtins:
        if locals_dict is None:
            locals_dict = {}
        locals_dict.update(globals_dict.get('__builtins__'))
    c = test_expr(expr, _SAFE_OPCODES, mode=mode)
    try:
        return eval(c, globals_dict, locals_dict)
    except openerp.exceptions.except_orm:
        raise
    except openerp.exceptions.Warning:
        raise
    except openerp.exceptions.RedirectWarning:
        raise
    except openerp.exceptions.AccessDenied:
        raise
    except openerp.exceptions.AccessError:
        raise
    except OperationalError:
        # Do not hide PostgreSQL low-level exceptions, to let the auto-replay
        # of serialized transactions work its magic
        raise
    except openerp.exceptions.MissingError:
        raise
    except Exception, e:
        import sys
        exc_info = sys.exc_info()
        raise ValueError, '"%s" while evaluating\n%r' % (ustr(e), expr), exc_info[2]
<commit_msg>[IMP] safe_eval: speed up evaluation<commit_after># -*- coding: utf-8 -*-
##############################################################################
#    Copyright (C) 2004-2014 OpenERP s.a. (<http://www.openerp.com>).
#
#    This program is free software: you can redistribute it and/or modify
#    it under the terms of the GNU Affero General Public License as
#    published by the Free Software Foundation, either version 3 of the
#    License, or (at your option) any later version.
#
#    This program is distributed in the hope that it will be useful,
#    but WITHOUT ANY WARRANTY; without even the implied warranty of
#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#    GNU Affero General Public License for more details.
#
#    You should have received a copy of the GNU Affero General Public License
#    along with this program.  If not, see <http://www.gnu.org/licenses/>.
#
##############################################################################

"""
safe_eval module - methods intended to provide more restricted alternatives to
                   evaluate simple and/or untrusted code.

Methods in this module are typically used as alternatives to eval() to parse
OpenERP domain strings, conditions and expressions, mostly based on locals
condition/math builtins.
"""

# Module partially ripped from/inspired by several different sources:
#  - http://code.activestate.com/recipes/286134/
#  - safe_eval in lp:~xrg/openobject-server/optimize-5.0
#  - safe_eval in tryton http://hg.tryton.org/hgwebdir.cgi/trytond/rev/bbb5f73319ad

from opcode import HAVE_ARGUMENT, opmap, opname
from psycopg2 import OperationalError
from types import CodeType
import logging

from .misc import ustr

import openerp

__all__ = ['test_expr', 'safe_eval', 'const_eval']

# The time module is usually already provided in the safe_eval environment
# but some code, e.g. datetime.datetime.now() (Windows/Python 2.5.2, bug
# lp:703841), does import time.
_ALLOWED_MODULES = ['_strptime', 'math', 'time']

_UNSAFE_ATTRIBUTES = ['f_builtins', 'f_globals', 'f_locals', 'gi_frame',
                      'co_code', 'func_globals']

_CONST_OPCODES = set(opmap[x] for x in [
    'POP_TOP', 'ROT_TWO', 'ROT_THREE', 'ROT_FOUR', 'DUP_TOP', 'DUP_TOPX',
    'POP_BLOCK','SETUP_LOOP', 'BUILD_LIST', 'BUILD_MAP', 'BUILD_TUPLE',
    'LOAD_CONST', 'RETURN_VALUE', 'STORE_SUBSCR', 'STORE_MAP'] if x in opmap)

_EXPR_OPCODES = _CONST_OPCODES.union(set(opmap[x] for x in [
    'UNARY_POSITIVE', 'UNARY_NEGATIVE', 'UNARY_NOT',
    'UNARY_INVERT', 'BINARY_POWER', 'BINARY_MULTIPLY',
    'BINARY_DIVIDE', 'BINARY_FLOOR_DIVIDE', 'BINARY_TRUE_DIVIDE',
    'BINARY_MODULO', 'BINARY_ADD', 'BINARY_SUBTRACT', 'BINARY_SUBSCR',
    'BINARY_LSHIFT', 'BINARY_RSHIFT', 'BINARY_AND', 'BINARY_XOR',
    'BINARY_OR', 'INPLACE_ADD', 'INPLACE_SUBTRACT', 'INPLACE_MULTIPLY',
    'INPLACE_DIVIDE', 'INPLACE_REMAINDER', 'INPLACE_POWER',
    'INPLACE_LEFTSHIFT', 'INPLACE_RIGHTSHIFT', 'INPLACE_AND',
    'INPLACE_XOR','INPLACE_OR'
    ] if x in opmap))

_SAFE_OPCODES = _EXPR_OPCODES.union(set(opmap[x] for x in [
    'LOAD_NAME', 'CALL_FUNCTION', 'COMPARE_OP', 'LOAD_ATTR',
    'STORE_NAME', 'GET_ITER', 'FOR_ITER', 'LIST_APPEND', 'DELETE_NAME',
    'JUMP_FORWARD', 'JUMP_IF_TRUE', 'JUMP_IF_FALSE', 'JUMP_ABSOLUTE',
    'MAKE_FUNCTION', 'SLICE+0', 'SLICE+1', 'SLICE+2', 'SLICE+3', 'BREAK_LOOP',
    'CONTINUE_LOOP', 'RAISE_VARARGS', 'YIELD_VALUE',
    # New in Python 2.7 - http://bugs.python.org/issue4715 :
    'JUMP_IF_FALSE_OR_POP', 'JUMP_IF_TRUE_OR_POP', 'POP_JUMP_IF_FALSE',
    'POP_JUMP_IF_TRUE', 'SETUP_EXCEPT', 'END_FINALLY',
    'LOAD_FAST', 'STORE_FAST', 'DELETE_FAST', 'UNPACK_SEQUENCE',
    'LOAD_GLOBAL', # Only allows access to restricted globals
    ] if x in opmap))

_logger = logging.getLogger(__name__)

def _get_opcodes(codeobj):
    """_get_opcodes(codeobj) -> [opcodes]

    Extract the actual opcodes as a list from a code object

    >>> c = compile("[1 + 2, (1,2)]", "", "eval")
    >>> _get_opcodes(c)
    [100, 100, 23, 100, 100, 102, 103, 83]
    """
    i = 0
    byte_codes = codeobj.co_code
    while i < len(byte_codes):
        code = ord(byte_codes[i])
        yield code

        if code >= HAVE_ARGUMENT:
            i += 3
        else:
            i += 1

def assert_no_dunder_name(code_obj, expr):
    """ assert_no_dunder_name(code_obj, expr) -> None

    Asserts that the code object does not refer to any "dunder name"
    (__$name__), so that safe_eval prevents access to any internal-ish Python
    attribute or method (both are loaded via LOAD_ATTR which uses a name, not a
    const or a var).

    Checks that no such name exists in the provided code object (co_names).

    :param code_obj: code object to name-validate
    :type code_obj: CodeType
    :param str expr: expression corresponding to the code object, for debugging
                     purposes
    :raises NameError: in case a forbidden name (containing two underscores)
                       is found in ``code_obj``

    .. note:: actually forbids every name containing 2 underscores
    """
    for name in code_obj.co_names:
        if "__" in name or name in _UNSAFE_ATTRIBUTES:
            raise NameError('Access to forbidden name %r (%r)' % (name, expr))

def assert_valid_codeobj(allowed_codes, code_obj, expr):
    """ Asserts that the provided code object validates against the bytecode
    and name constraints.

    Recursively validates the code objects stored in its co_consts in case
    lambdas are being created/used (lambdas generate their own separated code
    objects and don't live in the root one)

    :param allowed_codes: list of permissible bytecode instructions
    :type allowed_codes: set(int)
    :param code_obj: code object to name-validate
    :type code_obj: CodeType
    :param str expr: expression corresponding to the code object, for debugging
                     purposes
    :raises ValueError: in case of forbidden bytecode in ``code_obj``
    :raises NameError: in case a forbidden name (containing two underscores)
                       is found in ``code_obj``
    """
    assert_no_dunder_name(code_obj, expr)

    # almost twice as fast as a manual iteration + condition when loading
    # /web according to line_profiler
    if set(_get_opcodes(code_obj)) - allowed_codes:
        raise ValueError("forbidden opcode(s) in %r" % expr)

    for const in code_obj.co_consts:
        if isinstance(const, CodeType):
            assert_valid_codeobj(allowed_codes, const, 'lambda')

def test_expr(expr, allowed_codes, mode="eval"):
    """test_expr(expression, allowed_codes[, mode]) -> code_object

    Test that the expression contains only the allowed opcodes.
    If the expression is valid and contains only allowed codes,
    return the compiled code object.
    Otherwise raise a ValueError, a Syntax Error or TypeError accordingly.
    """
    try:
        if mode == 'eval':
            # eval() does not like leading/trailing whitespace
            expr = expr.strip()
        code_obj = compile(expr, "", mode)
    except (SyntaxError, TypeError, ValueError):
        raise
    except Exception, e:
        import sys
        exc_info = sys.exc_info()
        raise ValueError, '"%s" while compiling\n%r' % (ustr(e), expr), exc_info[2]
    assert_valid_codeobj(allowed_codes, code_obj, expr)
    return code_obj

def const_eval(expr):
    """const_eval(expression) -> value

    Safe Python constant evaluation

    Evaluates a string that contains an expression describing
    a Python constant. Strings that are not valid Python expressions
    or that contain other code besides the constant raise ValueError.

    >>> const_eval("10")
    10
    >>> const_eval("[1,2, (3,4), {'foo':'bar'}]")
    [1, 2, (3, 4), {'foo': 'bar'}]
    >>> const_eval("1+2")
    Traceback (most recent call last):
    ...
    ValueError: opcode BINARY_ADD not allowed
    """
    c = test_expr(expr, _CONST_OPCODES)
    return eval(c)

def expr_eval(expr):
    """expr_eval(expression) -> value

    Restricted Python expression evaluation

    Evaluates a string that contains an expression that only
    uses Python constants. This can be used to e.g. evaluate
    a numerical expression from an untrusted source.

    >>> expr_eval("1+2")
    3
    >>> expr_eval("[1,2]*2")
    [1, 2, 1, 2]
    >>> expr_eval("__import__('sys').modules")
    Traceback (most recent call last):
    ...
    ValueError: opcode LOAD_NAME not allowed
    """
    c = test_expr(expr, _EXPR_OPCODES)
    return eval(c)

def _import(name, globals=None, locals=None, fromlist=None, level=-1):
    if globals is None:
        globals = {}
    if locals is None:
        locals = {}
    if fromlist is None:
        fromlist = []
    if name in _ALLOWED_MODULES:
        return __import__(name, globals, locals, level)
    raise ImportError(name)
_BUILTINS = {
    '__import__': _import,
    'True': True,
    'False': False,
    'None': None,
    'str': str,
    'unicode': unicode,
    'bool': bool,
    'int': int,
    'float': float,
    'long': long,
    'enumerate': enumerate,
    'dict': dict,
    'list': list,
    'tuple': tuple,
    'map': map,
    'abs': abs,
    'min': min,
    'max': max,
    'sum': sum,
    'reduce': reduce,
    'filter': filter,
    'round': round,
    'len': len,
    'repr': repr,
    'set': set,
    'all': all,
    'any': any,
    'ord': ord,
    'chr': chr,
    'cmp': cmp,
    'divmod': divmod,
    'isinstance': isinstance,
    'range': range,
    'xrange': xrange,
    'zip': zip,
    'Exception': Exception,
}
def safe_eval(expr, globals_dict=None, locals_dict=None, mode="eval", nocopy=False, locals_builtins=False):
    """safe_eval(expression[, globals[, locals[, mode[, nocopy]]]]) -> result

    System-restricted Python expression evaluation

    Evaluates a string that contains an expression that mostly
    uses Python constants, arithmetic expressions and the
    objects directly provided in context.

    This can be used to e.g. evaluate
    an OpenERP domain expression from an untrusted source.

    :throws TypeError: If the expression provided is a code object
    :throws SyntaxError: If the expression provided is not valid Python
    :throws NameError: If the expression provided accesses forbidden names
    :throws ValueError: If the expression provided uses forbidden bytecode
    """
    if type(expr) is CodeType:
        raise TypeError("safe_eval does not allow direct evaluation of code objects.")

    # prevent altering the globals/locals from within the sandbox
    # by taking a copy.
    if not nocopy:
        # isinstance() does not work below, we want *exactly* the dict class
        if (globals_dict is not None and type(globals_dict) is not dict) \
            or (locals_dict is not None and type(locals_dict) is not dict):
            _logger.warning(
                "Looks like you are trying to pass a dynamic environment, "
                "you should probably pass nocopy=True to safe_eval().")
        if globals_dict is not None:
            globals_dict = dict(globals_dict)
        if locals_dict is not None:
            locals_dict = dict(locals_dict)

    if globals_dict is None:
        globals_dict = {}

    globals_dict['__builtins__'] = _BUILTINS
    if locals_builtins:
        if locals_dict is None:
            locals_dict = {}
        locals_dict.update(_BUILTINS)
    c = test_expr(expr, _SAFE_OPCODES, mode=mode)
    try:
        return eval(c, globals_dict, locals_dict)
    except openerp.exceptions.except_orm:
        raise
    except openerp.exceptions.Warning:
        raise
    except openerp.exceptions.RedirectWarning:
        raise
    except openerp.exceptions.AccessDenied:
        raise
    except openerp.exceptions.AccessError:
        raise
    except OperationalError:
        # Do not hide PostgreSQL low-level exceptions, to let the auto-replay
        # of serialized transactions work its magic
        raise
    except openerp.exceptions.MissingError:
        raise
    except Exception, e:
        import sys
        exc_info = sys.exc_info()
        raise ValueError, '"%s" while evaluating\n%r' % (ustr(e), expr), exc_info[2]
<|endoftext|>