<commit_before>#!/usr/bin/env python3
# Copyright (c) 2017 The Bitcoin Gold developers
# Distributed under the MIT software license, see the accompanying
# file COPYING or http://www.opensource.org/licenses/mit-license.php.
"""Test cases for Bitcoin Gold CLTV wallet."""

import io

from test_framework.address import *
from test_framework.key import CECKey, CPubKey
from test_framework.script import *
from test_framework.test_framework import BitcoinTestFramework
from test_framework.util import *

EPS = 1e-6

def sig(x):
    if x < -EPS:
        return -1
    elif x > EPS:
        return 1
    else:
        return 0

class BTGTimeLockTest(BitcoinTestFramework):

    def __init__(self):
        super().__init__()
        self.num_nodes = 2
        self.setup_clean_chain = False

    def sign_tx(self, tx, spend_tx, spend_n, redeem_script, in_n, keys):
        """Sign a P2SH transaction by privkeys."""
        vout = spend_tx.vout[spend_n]
        sighash = SegwitVersion1SignatureHash(
            redeem_script, tx, in_n, SIGHASH_ALL|SIGHASH_FORKID, vout.nValue)
        sigs = [key.sign(sighash) + bytes(bytearray([SIGHASH_ALL|SIGHASH_FORKID])) for key in keys]
        tx.vin[0].scriptSig = CScript([OP_0] + sigs + [redeem_script])
        tx.rehash()

    def run_test(self):
        node = self.nodes[0]

        # Generate 6 keys.
        rawkeys = []
        pubkeys = []
        for i in range(6):
            raw_key = CECKey()
            raw_key.set_secretbytes(b'privkey%d' % i)
            rawkeys.append(raw_key)
        pubkeys = [CPubKey(key.get_pubkey()) for key in rawkeys]

        # Create a 4-of-6 multi-sig wallet with CLTV.
        height = 210
        redeem_script = CScript(
            [CScriptNum(height), OP_CHECKLOCKTIMEVERIFY, OP_DROP] +  # CLTV (lock_time >= 210)
            [OP_4] + pubkeys +  [OP_6, OP_CHECKMULTISIG])  # multi-sig
        hex_redeem_script = bytes_to_hex_str(redeem_script)
        p2sh_address = script_to_p2sh(redeem_script, main=False)

        # Send 1 coin to the mult-sig wallet.
        txid = node.sendtoaddress(p2sh_address, 1.0)
        raw_tx = node.getrawtransaction(txid, True)
        try:
            node.importaddress(hex_redeem_script, 'cltv', True, True)
        except Exception as err:
            pass
        assert_equal(sig(node.getreceivedbyaddress(p2sh_address, 0) - Decimal(1.0)), 0)

        # Mine one block to confirm the transaction.
        node.generate(1)  # block 201
        assert_equal(sig(node.getreceivedbyaddress(p2sh_address, 1) - Decimal(1.0)), 0)

        # Try to spend the coin.
        addr_to = node.getnewaddress('')

        # (1) Find the UTXO
        for vout in raw_tx['vout']:
            if vout['scriptPubKey']['addresses'] == [p2sh_address]:
                vout_n = vout['n']
        hex_script_pubkey = raw_tx['vout'][vout_n]['scriptPubKey']['hex']
        value = raw_tx['vout'][vout_n]['value']

        # (2) Create a tx
        inputs = [{
            "txid": txid,
            "vout": vout_n,
            "scriptPubKey": hex_script_pubkey,
            "redeemScript": hex_redeem_script,
            "amount": value,
        }]
        outputs = {addr_to: 0.999}
        lock_time = height
        hex_spend_raw_tx = node.createrawtransaction(inputs, outputs, lock_time)
        hex_funding_raw_tx = node.getrawtransaction(txid, False)

        # (3) Try to sign the spending tx.
        tx0 = CTransaction()
        tx0.deserialize(io.BytesIO(hex_str_to_bytes(hex_funding_raw_tx)))
        tx1 = CTransaction()
        tx1.deserialize(io.BytesIO(hex_str_to_bytes(hex_spend_raw_tx)))
        self.sign_tx(tx1, tx0, vout_n, redeem_script, 0, rawkeys[:4])  # Sign with key[0:4]

        # Mine some blocks to pass the lock time.
        node.generate(10)

        # Spend the CLTV multi-sig coins.
        raw_tx1 = tx1.serialize()
        hex_raw_tx1 = bytes_to_hex_str(raw_tx1)
        node.sendrawtransaction(hex_raw_tx1)

        # Check the tx is accepted by mempool but not confirmed.
        assert_equal(sig(node.getreceivedbyaddress(addr_to, 0) - Decimal(0.999)), 0)
        assert_equal(sig(node.getreceivedbyaddress(addr_to, 1)), 0)

        # Mine a block to confirm the tx.
        node.generate(1)
        assert_equal(sig(node.getreceivedbyaddress(addr_to, 1) - Decimal(0.999)), 0)

if __name__ == '__main__':
    BTGTimeLockTest().main()
<commit_msg>Fix nits in regtest btg-timelock.py.<commit_after>#!/usr/bin/env python3
# Copyright (c) 2017 The Bitcoin Gold developers
# Distributed under the MIT software license, see the accompanying
# file COPYING or http://www.opensource.org/licenses/mit-license.php.
"""Test cases for Bitcoin Gold CLTV wallet."""

import io

from test_framework.address import *
from test_framework.key import CECKey, CPubKey
from test_framework.script import *
from test_framework.test_framework import BitcoinTestFramework
from test_framework.util import *

EPS = 1e-6

def sig(x):
    if x < -EPS:
        return -1
    elif x > EPS:
        return 1
    else:
        return 0

class BTGTimeLockTest(BitcoinTestFramework):

    def __init__(self):
        super().__init__()
        self.num_nodes = 2
        self.setup_clean_chain = False

    def sign_tx(self, tx, spend_tx, spend_n, redeem_script, in_n, keys):
        """Sign a P2SH transaction by privkeys."""
        vout = spend_tx.vout[spend_n]
        sighash = SegwitVersion1SignatureHash(
            redeem_script, tx, in_n, SIGHASH_ALL|SIGHASH_FORKID, vout.nValue)
        sigs = [key.sign(sighash) + bytes(bytearray([SIGHASH_ALL|SIGHASH_FORKID])) for key in keys]
        tx.vin[0].scriptSig = CScript([OP_0] + sigs + [redeem_script])
        tx.rehash()

    def run_test(self):
        node = self.nodes[0]

        # Generate 6 keys.
        rawkeys = []
        pubkeys = []
        for i in range(6):
            raw_key = CECKey()
            raw_key.set_secretbytes(('privkey%d' % i).encode('ascii'))
            rawkeys.append(raw_key)
        pubkeys = [CPubKey(key.get_pubkey()) for key in rawkeys]

        # Create a 4-of-6 multi-sig wallet with CLTV.
        height = 210
        redeem_script = CScript(
            [CScriptNum(height), OP_CHECKLOCKTIMEVERIFY, OP_DROP] +  # CLTV (lock_time >= 210)
            [OP_4] + pubkeys +  [OP_6, OP_CHECKMULTISIG])  # multi-sig
        hex_redeem_script = bytes_to_hex_str(redeem_script)
        p2sh_address = script_to_p2sh(redeem_script, main=False)

        # Send 1 coin to the mult-sig wallet.
        txid = node.sendtoaddress(p2sh_address, 1.0)
        raw_tx = node.getrawtransaction(txid, True)
        try:
            node.importaddress(hex_redeem_script, 'cltv', True, True)
        except Exception as err:
            pass
        assert_equal(sig(node.getreceivedbyaddress(p2sh_address, 0) - Decimal(1.0)), 0)

        # Mine one block to confirm the transaction.
        node.generate(1)  # block 201
        assert_equal(sig(node.getreceivedbyaddress(p2sh_address, 1) - Decimal(1.0)), 0)

        # Try to spend the coin.
        addr_to = node.getnewaddress('')

        # (1) Find the UTXO
        for vout in raw_tx['vout']:
            if vout['scriptPubKey']['addresses'] == [p2sh_address]:
                vout_n = vout['n']
        hex_script_pubkey = raw_tx['vout'][vout_n]['scriptPubKey']['hex']
        value = raw_tx['vout'][vout_n]['value']

        # (2) Create a tx
        inputs = [{
            "txid": txid,
            "vout": vout_n,
            "scriptPubKey": hex_script_pubkey,
            "redeemScript": hex_redeem_script,
            "amount": value,
        }]
        outputs = {addr_to: 0.999}
        lock_time = height
        hex_spend_raw_tx = node.createrawtransaction(inputs, outputs, lock_time)
        hex_funding_raw_tx = node.getrawtransaction(txid, False)

        # (3) Try to sign the spending tx.
        tx0 = CTransaction()
        tx0.deserialize(io.BytesIO(hex_str_to_bytes(hex_funding_raw_tx)))
        tx1 = CTransaction()
        tx1.deserialize(io.BytesIO(hex_str_to_bytes(hex_spend_raw_tx)))
        self.sign_tx(tx1, tx0, vout_n, redeem_script, 0, rawkeys[:4])  # Sign with key[0:4]

        # Mine some blocks to pass the lock time.
        node.generate(10)

        # Spend the CLTV multi-sig coins.
        raw_tx1 = tx1.serialize()
        hex_raw_tx1 = bytes_to_hex_str(raw_tx1)
        node.sendrawtransaction(hex_raw_tx1)

        # Check the tx is accepted by mempool but not confirmed.
        assert_equal(sig(node.getreceivedbyaddress(addr_to, 0) - Decimal(0.999)), 0)
        assert_equal(sig(node.getreceivedbyaddress(addr_to, 1)), 0)

        # Mine a block to confirm the tx.
        node.generate(1)
        assert_equal(sig(node.getreceivedbyaddress(addr_to, 1) - Decimal(0.999)), 0)

if __name__ == '__main__':
    BTGTimeLockTest().main()
<|endoftext|>


<commit_before>"""
Name: Dimitri Somoff
Final Project: Flappy bird remake "Crappy Bird"
"""
from ggame import App, Color, LineStyle, Sprite, ImageAsset, Frame, SoundAsset
import random

wals = 0
bottomwals = 0

class TopWall(Sprite):
    top = ImageAsset("images/top.png")
    def __init__(self, position):
        super().__init__(TopWall.top, position)

    def step(self):
        if self.x <= -100 and self.y == -201:
            self.y = -420
            self.x = 400
        elif self.x <= -100 and self.y == -420:
            self.y = -25
            self.x = 400
        elif self.x <= -100 and self.y == -25:
            self.y = -290
            self.x = 400
        elif self.x <= -100 and self.y == -290:
            self.y = -135
            self.x = 400
        elif self.x <= -100 and self.y == -135:
            self.y = -400
            self.x = 400
        elif self.x <= -100 and self.y == -400:
            self.y = -200
            self.x = 400
        elif self.x <= -100 and self.y == -200:
            self.y = -420
            self.x = 400
        self.x -= 3

class BottomWall(Sprite):
    bottom = ImageAsset("images/bottom.png")
    def __init__(self, position):
        super().__init__(BottomWall.bottom, position)

    def step(self):
        if self.x <= -100 and self.y == 424:
            self.y = 205
            self.x = 400
        elif self.x <= -100 and self.y == 205:
            self.y = 600
            self.x = 400
        elif self.x <= -100 and self.y == 600:
            self.y = 335
            self.x = 400
        elif self.x <= -100 and self.y == 335:
            self.y = 490
            self.x = 400
        elif self.x <= -100 and self.y == 490:
            self.y = 225
            self.x = 400
        elif self.x <= -100 and self.y == 225:
            self.y = 425
            self.x = 400
        elif self.x <= -100 and self.y == 425:
            self.y = 205
            self.x = 400
        self.x -= 3

class Bird(Sprite):
    asset1 = ImageAsset("images/0.png")
    asset2 = ImageAsset("images/1.png")
    asset3 = ImageAsset("images/2.png")

    def __init__(self, position):
        super().__init__(Bird.asset1, position)
        self.birdy = 350
        self.gravity = .1
        self.wallx = 400
        self.wally = random.randint(-420, 0)
        self.bottomwally = self.wally + 625
        CrappyApp.listenKeyEvent("keydown", "space", self.Jump)

    def step(self):
        self.gravity += .15
        self.birdy += self.gravity
        self.y = self.birdy
        self.spawnwalls()
        self.wallx -= 3
        if self.visible == True:
            collides = self.collidingWithSprites(TopWall)
            collides1 = self.collidingWithSprites(BottomWall)
            if len(collides):
                self.die()
            if len(collides1):
                self.die()
            if self.y <= -30:
                self.die()

    def Jump(self, event):
        self.birdy -= 50
        self.gravity = .1

    def spawnwalls(self):
        if self.wallx == 400:
            TopWall((self.wallx, -201))
            BottomWall((self.wallx, 424))

    def die(self):
        self.visible = False
        DeadBird(self.position)

class DeadBird(Sprite):

    asset = ImageAsset("images/dead.png")
    boomasset = SoundAsset("sounds/explosion2.mp3")

    def __init__(self, position):
        super().__init__(ExplosionBig.asset, position)
        #self.image = 0
        #self.center = (0.5, 0.5)
        self.boom = Sound(ExplosionBig.boomasset)
        self.boom.play()

class CrappyApp(App):
    def __init__(self, width, height):
        super().__init__(width, height)
        bg_asset = ImageAsset("images/background.png")
        bg = Sprite(bg_asset, (0,0))
        Bird((20, 350))

    def step(self):
        for bird in self.getSpritesbyClass(Bird):
            bird.step()
        for birds in self.getSpritesbyClass(TopWall):
            birds.step()
        for bottoms in self.getSpritesbyClass(BottomWall):
            bottoms.step()

myapp = CrappyApp(400, 708)
myapp.run()<commit_msg>Updated from Brython Server: 1/5/2016 4:31:48 PM<commit_after>"""
Name: Dimitri Somoff
Final Project: Flappy bird remake "Crappy Bird"
"""
from ggame import App, Color, LineStyle, Sprite, ImageAsset, Frame, SoundAsset
import random

wals = 0
bottomwals = 0

class TopWall(Sprite):
    top = ImageAsset("images/top.png")
    def __init__(self, position):
        super().__init__(TopWall.top, position)

    def step(self):
        if self.x <= -100 and self.y == -201:
            self.y = -420
            self.x = 400
        elif self.x <= -100 and self.y == -420:
            self.y = -25
            self.x = 400
        elif self.x <= -100 and self.y == -25:
            self.y = -290
            self.x = 400
        elif self.x <= -100 and self.y == -290:
            self.y = -135
            self.x = 400
        elif self.x <= -100 and self.y == -135:
            self.y = -400
            self.x = 400
        elif self.x <= -100 and self.y == -400:
            self.y = -200
            self.x = 400
        elif self.x <= -100 and self.y == -200:
            self.y = -420
            self.x = 400
        self.x -= 3

class BottomWall(Sprite):
    bottom = ImageAsset("images/bottom.png")
    def __init__(self, position):
        super().__init__(BottomWall.bottom, position)

    def step(self):
        if self.x <= -100 and self.y == 424:
            self.y = 205
            self.x = 400
        elif self.x <= -100 and self.y == 205:
            self.y = 600
            self.x = 400
        elif self.x <= -100 and self.y == 600:
            self.y = 335
            self.x = 400
        elif self.x <= -100 and self.y == 335:
            self.y = 490
            self.x = 400
        elif self.x <= -100 and self.y == 490:
            self.y = 225
            self.x = 400
        elif self.x <= -100 and self.y == 225:
            self.y = 425
            self.x = 400
        elif self.x <= -100 and self.y == 425:
            self.y = 205
            self.x = 400
        self.x -= 3

class Bird(Sprite):
    asset1 = ImageAsset("images/0.png")
    asset2 = ImageAsset("images/1.png")
    asset3 = ImageAsset("images/2.png")

    def __init__(self, position):
        super().__init__(Bird.asset1, position)
        self.birdy = 350
        self.gravity = .1
        self.wallx = 400
        self.wally = random.randint(-420, 0)
        self.bottomwally = self.wally + 625
        CrappyApp.listenKeyEvent("keydown", "space", self.Jump)

    def step(self):
        self.gravity += .15
        self.birdy += self.gravity
        self.y = self.birdy
        self.spawnwalls()
        self.wallx -= 3
        if self.visible == True:
            collides = self.collidingWithSprites(TopWall)
            collides1 = self.collidingWithSprites(BottomWall)
            if len(collides):
                self.die()
            if len(collides1):
                self.die()
            if self.y <= -30:
                self.die()
            if self.y >= 708:
                self.die()

    def Jump(self, event):
        self.birdy -= 50
        self.gravity = .1

    def spawnwalls(self):
        if self.wallx == 400:
            TopWall((self.wallx, -201))
            BottomWall((self.wallx, 424))

    def die(self):
        self.visible = False
        DeadBird(self.position)

class DeadBird(Sprite):

    asset = ImageAsset("images/dead.png")
    boomasset = SoundAsset("sounds/explosion2.mp3")

    def __init__(self, position):
        super().__init__(ExplosionBig.asset, position)
        #self.image = 0
        #self.center = (0.5, 0.5)
        self.boom = Sound(ExplosionBig.boomasset)
        self.boom.play()

class CrappyApp(App):
    def __init__(self, width, height):
        super().__init__(width, height)
        bg_asset = ImageAsset("images/background.png")
        bg = Sprite(bg_asset, (0,0))
        Bird((20, 350))

    def step(self):
        for bird in self.getSpritesbyClass(Bird):
            bird.step()
        for birds in self.getSpritesbyClass(TopWall):
            birds.step()
        for bottoms in self.getSpritesbyClass(BottomWall):
            bottoms.step()

myapp = CrappyApp(400, 708)
myapp.run()<|endoftext|>


<commit_before><commit_msg>add a test or two<commit_after><|endoftext|>


<commit_before>'''
This module is part of ngs_backbone. This module performs analyses related to
sequence cleaning.

Created on 15/03/2010

@author: peio
'''

# Copyright 2009 Jose Blanca, Peio Ziarsolo, COMAV-Univ. Politecnica Valencia
# This file is part of franklin.
# franklin is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as
# published by the Free Software Foundation, either version 3 of the
# License, or (at your option) any later version.

# franklin is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR  PURPOSE.  See the
# GNU Affero General Public License for more details.

# You should have received a copy of the GNU Affero General Public License
# along with franklin. If not, see <http://www.gnu.org/licenses/>.

import os, itertools, logging
from tempfile import NamedTemporaryFile
from franklin.backbone.analysis import Analyzer, scrape_info_from_fname
from franklin.pipelines.pipelines import seq_pipeline_runner
from franklin.backbone.specifications import BACKBONE_DIRECTORIES
from franklin.statistics import (seq_distrib, general_seq_statistics,
                                 seq_distrib_diff)
from franklin.seq.readers import guess_seq_file_format, num_seqs_in_file
from franklin.utils.seqio_utils import seqs_in_file
from franklin.seq.seq_cleaner import MIN_LONG_ADAPTOR_LENGTH
from franklin.seq.writers import SequenceWriter

class CleanReadsAnalyzer(Analyzer):
    'It does a cleaning reads analysis'

    @staticmethod
    def _guess_cleaning_pipepile(file_info):
        'It returns the pipeline suited to clean the given file'
        if file_info['format'] == 'fasta':
            return 'sanger_without_qual'
        elif file_info['pl'] == 'illumina':
            return 'solexa'
        elif file_info['pl'] == '454':
            return 'sanger_with_qual'
        elif file_info['pl'] == 'sanger':
            return 'sanger_with_qual'
        else:
            raise ValueError('Unable to guess the cleaning pipeline: %s' %
                             str(file_info))

    @staticmethod
    def _purge_short_adaptors(adaptors_fpath):
        '''It returns a file without short adaptors and the list of adaptors
        removed'''
        long_adap_fhand = NamedTemporaryFile(delete=False, mode='a')
        seq_writer = SequenceWriter(long_adap_fhand, file_format='fasta')
        short_adaptors = []
        for seq in seqs_in_file(open(adaptors_fpath)):
            if len(seq) < MIN_LONG_ADAPTOR_LENGTH:
                short_adaptors.append(str(seq.seq))
            else:
                seq_writer.write(seq)
        if not short_adaptors:
            #not worth the new file, we remove it
            long_adap_fhand.close()
            os.remove(long_adap_fhand.name)
            return adaptors_fpath, short_adaptors
        elif not seq_writer.num_features:   #no long adaptors remaining
            os.remove(long_adap_fhand.name)
            return None, short_adaptors     #we do not return the adaptor file
        else:
            return long_adap_fhand.name, short_adaptors

    def create_cleaning_configuration(self, platform, library=None):
        'It returns the pipeline configuration looking at the project settings'
        settings = self._project_settings['Cleaning']
        configuration = {}

        configuration['remove_vectors'] = {}
        configuration['remove_vectors']['vectors'] = settings['vector_database']

        # adaptors settings
        adap_param = 'adaptors_file_%s' % platform
        adaptors_fpath = settings[adap_param]

        #we have to remove the short adaptors from the file and treat them
        #as short adaptors
        if adaptors_fpath:
            adaptors_fpath, words = self._purge_short_adaptors(adaptors_fpath)
        else:
            words = []
        configuration['remove_adaptors'] = {}
        configuration['remove_adaptors']['vectors'] = adaptors_fpath

        # Words settings
        word_param = 'short_adaptors_%s' % platform
        if settings[word_param] is not None:
            words.extend(settings[word_param])
        configuration['remove_short_adaptors'] = {}
        configuration['remove_short_adaptors']['words'] = words

        #edge_remover
        left =  settings['edge_removal']['%s_left' % platform]
        right = settings['edge_removal']['%s_right' % platform]
        configuration['edge_removal'] = {}
        configuration['edge_removal']['left_length'] = left
        configuration['edge_removal']['right_length'] = right

        # lucy settings.
        lucy_settings = settings['lucy']
        lucy_vector_settings = lucy_settings['vector_settings']
        if lucy_vector_settings is not None:
            lucy_settings_dir = os.path.dirname(lucy_vector_settings)
            lucy_settings_fhand = open(lucy_vector_settings)
            lucy_libraries = eval(lucy_settings_fhand.read())
            lucy_settings_fhand.close()
            if library in lucy_libraries:
                vector = os.path.join(lucy_settings_dir,
                                      lucy_libraries[library]['vector_file'])
                splice = os.path.join(lucy_settings_dir,
                                      lucy_libraries[library]['splice_file'])
                vector_settings = [vector, splice]
            else:
                vector_settings = None
        else:
            vector_settings = None

        parameters = {'bracket':lucy_settings['bracket'],
                      'window':lucy_settings['window'],
                      'error':lucy_settings['error']}
        if vector_settings:
            parameters['vector'] = vector_settings

        configuration['strip_lucy'] = {}
        configuration['strip_lucy']['parameters'] = parameters

        # min length settings
        min_length = settings['min_seq_length'][platform]
        configuration['remove_short'] = {}
        configuration['remove_short']['length'] = min_length

        return configuration

    def run(self):
        '''It runs the analysis. It checks if the analysis is already done per
        input file'''
        logger = logging.getLogger("franklin")
        self._log({'analysis_started':True})
        input_paths = self._get_input_fpaths()['reads']
        output_dir = self._create_output_dirs()['reads']
        for input_path in input_paths:
            input_fpath = str(input_path)
            fname = os.path.split(input_fpath)[-1]
            output_fpath = os.path.join(output_dir, fname)
            if os.path.exists(output_fpath):
                msg = '%s already cleaned. Not cleaned again'  % output_fpath
                logger.info(msg)
                continue
            file_info = scrape_info_from_fname(input_path)
            input_fhand = open(input_fpath)
            output_fhand = open(output_fpath, 'w')
            pipeline = self._guess_cleaning_pipepile(file_info)
            iofhands = {'in_seq':input_fhand,
                        'outputs':{'sequence': output_fhand }}
            configuration = self.create_cleaning_configuration(
                                                       platform=file_info['pl'],
                                                       library=file_info['lb'])
            seq_pipeline_runner(pipeline, configuration, iofhands,
                                file_info['format'], processes=self.threads)
            input_fhand.close()
            output_fhand.close()
        self._log({'analysis_finished':True})
        return

class ReadsStatsAnalyzer(Analyzer):
    '''It calculates stats for original and cleaned reads.
    It calculates the distribution between both type off reads'''

    def run(self):
        'It runs the analysis'
        self._log({'analysis_started':True})
        self._create_output_dirs()['original_reads']
        self._create_output_dirs()['cleaned_reads']

        clean_paths = self._get_input_fpaths()['cleaned_reads']
        original_paths = self._get_input_fpaths()['original_reads']
        reads = {'cleaned': clean_paths, 'original': original_paths}

        sample_size = self._project_settings['Read_stats' ]['sampling_size']

        # first per file stats
        for seq_type, paths in reads.items():
            for path in paths:
                stats_dir = self._get_stats_dir(seq_type)
                basename = path.basename
                fpath = path.last_version
                seqs = seqs_in_file(open(fpath), sample_size=sample_size)
                file_format = guess_seq_file_format(open(fpath))
                analyses = ['seq_length_distrib', 'qual_distrib']
                if file_format == 'fasta':
                    analyses = ['seq_length_distrib']

                self._do_seq_stats(seqs, basename, stats_dir, analyses)

        # now the difference between the cleaned and the original per file
        for clean_path in reads['cleaned']:
            original_fpath = os.path.join(self._get_project_path(),
                                         BACKBONE_DIRECTORIES['original_reads'],
                                         clean_path.basename)
            clean_fpath = clean_path.last_version
            if os.path.exists(clean_fpath) and os.path.exists(original_fpath):
                stats_dir = self._get_stats_dir('cleaned')

                clean_seqs = seqs_in_file(open(clean_fpath),
                                          sample_size=sample_size)
                original_seqs = seqs_in_file(open(original_fpath),
                                             sample_size=sample_size)
                file_format = guess_seq_file_format(open(clean_fpath))
                analyses = ['seq_length_distrib', 'qual_distrib']
                if file_format == 'fasta':
                    analyses = ['seq_length_distrib']
                self._do_diff_seq_stats(clean_seqs, original_seqs,
                                        clean_path.basename,
                                        stats_dir, analyses)

        # stats per seq file. All files together
        if clean_paths:
            clean_fpaths = [path.last_version for path in clean_paths]
            clean_seqs = self._seqs_in_files(clean_fpaths, sample_size)
            clean_seqs, clean_seqs2 = itertools.tee(clean_seqs, 2)
        else:
            clean_seqs, clean_seqs2 = None, None
        if original_paths:
            original_fpaths = [path.last_version for path in original_paths]
            original_seqs = self._seqs_in_files(original_fpaths, sample_size)
            original_seqs, original_seqs2 = itertools.tee(original_seqs, 2)
        else:
            original_seqs, original_seqs2 = None, None
        all_reads = {'cleaned': clean_seqs, 'original': original_seqs}
        basename = 'global'
        for seqtype, seqs in all_reads.items():
            if seqs is None:
                continue
            stats_dir = self._get_stats_dir(seqtype)
            analyses = ['seq_length_distrib', 'qual_distrib']
            self._do_seq_stats(seqs, basename, stats_dir, analyses)

        # global stats. Diff fistributions
        stats_dir = self._get_stats_dir('cleaned')
        analyses = ['seq_length_distrib', 'qual_distrib']
        if clean_seqs2 is not None and original_seqs2 is not None:
            self._do_diff_seq_stats(clean_seqs2, original_seqs2, basename,
                                    stats_dir, analyses)
        self._log({'analysis_finished':True})

    def _get_stats_dir(self, seqtype):
        'It gets the stats dir for each seqtype'
        return os.path.join(self._get_project_path(),
                             BACKBONE_DIRECTORIES['%s_reads_stats' % seqtype])

    @staticmethod
    def _seqs_in_files(fpaths, sample_size):
        'It yields SeqRecords from a list of files'
        if sample_size is None:
            samples = [None] * len(fpaths)
        else:
            #we count the seqs in every file
            num_seqs = []
            for fpath in fpaths:
                num_seqs.append(num_seqs_in_file(open(fpath)))
            total_seqs = float(sum(num_seqs))
            #how many seqs to sample for each file?
            samples = [int(num / total_seqs * sample_size) for num in num_seqs]

        if sample_size is None:
            for fpath in fpaths:
                for seq in seqs_in_file(open(fpath), sample_size=samples):
                    yield seq

    def _do_seq_stats(self, seqs, basename, stats_dir, analyses):
        'It performs all kind of stats for a fpath'
        # Some distributions
        for analysis in analyses:
            analysis_basename = '%s.%s' % (basename, analysis)
            seqs, seqs2 = itertools.tee(seqs, 2)
            self._do_distrib(seqs2, analysis, analysis_basename, stats_dir)

        # now general stats
        analysis_basename = '%s.general_stats' % basename
        self._do_general_stats(seqs, analysis_basename, stats_dir)

    def _do_general_stats(self, seqs, basename, stats_dir):
        'It performs the  general stats analysis'
        general_fpath = os.path.join(self._get_project_path(), stats_dir,
                                     basename + '.txt')
        if os.path.exists(general_fpath):
            return

        general_fhand = open(general_fpath, 'w')
        stats = general_seq_statistics(seqs)
        for key, value in stats.items():
            if value is not None:
                to_print = '%-19s : %d\n' % (key, value)
                general_fhand.write(to_print)

    @staticmethod
    def _do_distrib(seqs, analysis, basename, stats_dir):
        'I actually do the distrib'
        plot_fpath = os.path.join(stats_dir, basename + '.png')
        distrib_fpath = os.path.join(stats_dir, basename + '.dat')
        if os.path.exists(distrib_fpath) and os.path.exists(plot_fpath):
            return
        try:
            plot_fhand = open(plot_fpath, 'w')
            distrib_fhand = open(distrib_fpath, 'w')

            seq_distrib(sequences=seqs, kind=analysis,
                        distrib_fhand=distrib_fhand, plot_fhand=plot_fhand)
        except TypeError:
            plot_fhand.close()
            distrib_fhand.close()
            os.remove(plot_fpath)
            os.remove(distrib_fpath)
            raise

    def _do_diff_seq_stats(self, seqs1, seqs2, basename, stats_dir, analyses):
        'It performs the differential distribution'

        for analysis in analyses:
            seqs1, seqs1_use = itertools.tee(seqs1, 2)
            seqs2, seqs2_use = itertools.tee(seqs2, 2)
            analysis_basename = '%s.diff_%s' % (basename, analysis)
            self._do_diff_distrib(seqs1_use, seqs2_use, analysis,
                                  analysis_basename, stats_dir)

    @staticmethod
    def _do_diff_distrib(clean_seqs, original_seqs, analysis, basename,
                         stats_dir):
        'It performs the differential distribution'
        plot_fpath = os.path.join(stats_dir, basename + '.png')
        distrib_fpath = os.path.join(stats_dir, basename + '.dat')

        if os.path.exists(distrib_fpath) and os.path.exists(plot_fpath):
            return

        seq_distrib_diff(clean_seqs, original_seqs, analysis,
                         distrib_fhand=open(distrib_fpath, 'w'),
                         plot_fhand=open(plot_fpath, 'w'))

DEFINITIONS = {
    'clean_reads':
        {'inputs':{
            'reads':
                {'directory': 'original_reads',
                 'file_kinds': 'sequence_files'}
            },
         'outputs':{'reads':{'directory': 'cleaned_reads'}},
         'analyzer': CleanReadsAnalyzer,
        },
    'read_stats':
        {'inputs':{
            'original_reads':
                {'directory': 'original_reads',
                 'file_kinds': 'sequence_files'},
            'cleaned_reads':
                {'directory': 'cleaned_reads',
                 'file_kinds': 'sequence_files'}
                },
         'outputs':{'original_reads':{'directory':'original_reads_stats'},
                    'cleaned_reads'   :{'directory':'cleaned_reads_stats'}},
         'analyzer': ReadsStatsAnalyzer,
        },
     }
<commit_msg>read stats refactoring<commit_after>'''
This module is part of ngs_backbone. This module performs analyses related to
sequence cleaning.

Created on 15/03/2010

@author: peio
'''

# Copyright 2009 Jose Blanca, Peio Ziarsolo, COMAV-Univ. Politecnica Valencia
# This file is part of franklin.
# franklin is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as
# published by the Free Software Foundation, either version 3 of the
# License, or (at your option) any later version.

# franklin is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR  PURPOSE.  See the
# GNU Affero General Public License for more details.

# You should have received a copy of the GNU Affero General Public License
# along with franklin. If not, see <http://www.gnu.org/licenses/>.

import os, itertools, logging
from tempfile import NamedTemporaryFile
from franklin.backbone.analysis import Analyzer, scrape_info_from_fname
from franklin.pipelines.pipelines import seq_pipeline_runner
from franklin.backbone.specifications import BACKBONE_DIRECTORIES
from franklin.statistics import (seq_distrib, general_seq_statistics,
                                 seq_distrib_diff)
from franklin.seq.readers import guess_seq_file_format, num_seqs_in_file
from franklin.utils.seqio_utils import seqs_in_file
from franklin.seq.seq_cleaner import MIN_LONG_ADAPTOR_LENGTH
from franklin.seq.writers import SequenceWriter

class CleanReadsAnalyzer(Analyzer):
    'It does a cleaning reads analysis'

    @staticmethod
    def _guess_cleaning_pipepile(file_info):
        'It returns the pipeline suited to clean the given file'
        if file_info['format'] == 'fasta':
            return 'sanger_without_qual'
        elif file_info['pl'] == 'illumina':
            return 'solexa'
        elif file_info['pl'] == '454':
            return 'sanger_with_qual'
        elif file_info['pl'] == 'sanger':
            return 'sanger_with_qual'
        else:
            raise ValueError('Unable to guess the cleaning pipeline: %s' %
                             str(file_info))

    @staticmethod
    def _purge_short_adaptors(adaptors_fpath):
        '''It returns a file without short adaptors and the list of adaptors
        removed'''
        long_adap_fhand = NamedTemporaryFile(delete=False, mode='a')
        seq_writer = SequenceWriter(long_adap_fhand, file_format='fasta')
        short_adaptors = []
        for seq in seqs_in_file(open(adaptors_fpath)):
            if len(seq) < MIN_LONG_ADAPTOR_LENGTH:
                short_adaptors.append(str(seq.seq))
            else:
                seq_writer.write(seq)
        if not short_adaptors:
            #not worth the new file, we remove it
            long_adap_fhand.close()
            os.remove(long_adap_fhand.name)
            return adaptors_fpath, short_adaptors
        elif not seq_writer.num_features:   #no long adaptors remaining
            os.remove(long_adap_fhand.name)
            return None, short_adaptors     #we do not return the adaptor file
        else:
            return long_adap_fhand.name, short_adaptors

    def create_cleaning_configuration(self, platform, library=None):
        'It returns the pipeline configuration looking at the project settings'
        settings = self._project_settings['Cleaning']
        configuration = {}

        configuration['remove_vectors'] = {}
        configuration['remove_vectors']['vectors'] = settings['vector_database']

        # adaptors settings
        adap_param = 'adaptors_file_%s' % platform
        adaptors_fpath = settings[adap_param]

        #we have to remove the short adaptors from the file and treat them
        #as short adaptors
        if adaptors_fpath:
            adaptors_fpath, words = self._purge_short_adaptors(adaptors_fpath)
        else:
            words = []
        configuration['remove_adaptors'] = {}
        configuration['remove_adaptors']['vectors'] = adaptors_fpath

        # Words settings
        word_param = 'short_adaptors_%s' % platform
        if settings[word_param] is not None:
            words.extend(settings[word_param])
        configuration['remove_short_adaptors'] = {}
        configuration['remove_short_adaptors']['words'] = words

        #edge_remover
        left =  settings['edge_removal']['%s_left' % platform]
        right = settings['edge_removal']['%s_right' % platform]
        configuration['edge_removal'] = {}
        configuration['edge_removal']['left_length'] = left
        configuration['edge_removal']['right_length'] = right

        # lucy settings.
        lucy_settings = settings['lucy']
        lucy_vector_settings = lucy_settings['vector_settings']
        if lucy_vector_settings is not None:
            lucy_settings_dir = os.path.dirname(lucy_vector_settings)
            lucy_settings_fhand = open(lucy_vector_settings)
            lucy_libraries = eval(lucy_settings_fhand.read())
            lucy_settings_fhand.close()
            if library in lucy_libraries:
                vector = os.path.join(lucy_settings_dir,
                                      lucy_libraries[library]['vector_file'])
                splice = os.path.join(lucy_settings_dir,
                                      lucy_libraries[library]['splice_file'])
                vector_settings = [vector, splice]
            else:
                vector_settings = None
        else:
            vector_settings = None

        parameters = {'bracket':lucy_settings['bracket'],
                      'window':lucy_settings['window'],
                      'error':lucy_settings['error']}
        if vector_settings:
            parameters['vector'] = vector_settings

        configuration['strip_lucy'] = {}
        configuration['strip_lucy']['parameters'] = parameters

        # min length settings
        min_length = settings['min_seq_length'][platform]
        configuration['remove_short'] = {}
        configuration['remove_short']['length'] = min_length

        return configuration

    def run(self):
        '''It runs the analysis. It checks if the analysis is already done per
        input file'''
        logger = logging.getLogger("franklin")
        self._log({'analysis_started':True})
        input_paths = self._get_input_fpaths()['reads']
        output_dir = self._create_output_dirs()['reads']
        for input_path in input_paths:
            input_fpath = str(input_path)
            fname = os.path.split(input_fpath)[-1]
            output_fpath = os.path.join(output_dir, fname)
            if os.path.exists(output_fpath):
                msg = '%s already cleaned. Not cleaned again'  % output_fpath
                logger.info(msg)
                continue
            file_info = scrape_info_from_fname(input_path)
            input_fhand = open(input_fpath)
            output_fhand = open(output_fpath, 'w')
            pipeline = self._guess_cleaning_pipepile(file_info)
            iofhands = {'in_seq':input_fhand,
                        'outputs':{'sequence': output_fhand }}
            configuration = self.create_cleaning_configuration(
                                                       platform=file_info['pl'],
                                                       library=file_info['lb'])
            seq_pipeline_runner(pipeline, configuration, iofhands,
                                file_info['format'], processes=self.threads)
            input_fhand.close()
            output_fhand.close()
        self._log({'analysis_finished':True})
        return

BASENAME_FOR_ALL_TOGETHER = 'all'

class ReadsStatsAnalyzer(Analyzer):
    '''It calculates stats for original and cleaned reads.
    It calculates the distribution between both type off reads'''

    def run(self):
        'It runs the analysis'
        self._log({'analysis_started':True})
        self._create_output_dirs()['original_reads']
        self._create_output_dirs()['cleaned_reads']

        clean_paths = self._get_input_fpaths()['cleaned_reads']
        original_paths = self._get_input_fpaths()['original_reads']
        reads = {'cleaned': clean_paths, 'original': original_paths}

        sample_size = self._project_settings['Read_stats' ]['sampling_size']

        # first per file stats
        for seq_type, paths in reads.items():
            for path in paths:
                stats_dir = self._get_stats_dir(seq_type)
                file_format = guess_seq_file_format(open(path.last_version))
                if file_format == 'fasta':
                    analyses = ['seq_length_distrib']
                else:
                    analyses = ['seq_length_distrib', 'qual_distrib']
                self._do_seq_stats(path, stats_dir, analyses,
                                   sample_size)

        # now the difference between the cleaned and the original per file
        for clean_path in reads['cleaned']:
            original_fpath = os.path.join(self._get_project_path(),
                                         BACKBONE_DIRECTORIES['original_reads'],
                                         clean_path.basename)
            clean_fpath = clean_path.last_version
            if os.path.exists(clean_fpath) and os.path.exists(original_fpath):
                stats_dir = self._get_stats_dir('cleaned')

                file_format = guess_seq_file_format(open(clean_fpath))
                analyses = ['seq_length_distrib', 'qual_distrib']
                if file_format == 'fasta':
                    analyses = ['seq_length_distrib']
                self._do_diff_seq_stats(clean_fpath, original_fpath,
                                        clean_path.basename, stats_dir,
                                        analyses, sample_size)

        # stats per seq file. All files together
        all_reads = {'cleaned': clean_paths, 'original': original_paths}
        for seqtype, seq_paths in all_reads.items():
            if seq_paths is None:
                continue
            stats_dir = self._get_stats_dir(seqtype)
            analyses = ['seq_length_distrib', 'qual_distrib']
            self._do_seq_stats(seq_paths, stats_dir, analyses, sample_size)

        # global stats. Diff fistributions
        if clean_paths:
            clean_fpaths = [path.last_version for path in clean_paths]
        else:
            clean_fpaths = None
        if original_paths:
            original_fpaths = [path.last_version for path in original_paths]
        else:
            original_fpaths = None

        stats_dir = self._get_stats_dir('cleaned')
        analyses = ['seq_length_distrib', 'qual_distrib']
        if clean_fpaths is not None and original_fpaths is not None:
            self._do_diff_seq_stats(clean_fpaths, original_fpaths,
                                    BASENAME_FOR_ALL_TOGETHER,
                                    stats_dir, analyses, sample_size)
        self._log({'analysis_finished':True})

    def _get_stats_dir(self, seqtype):
        'It gets the stats dir for each seqtype'
        return os.path.join(self._get_project_path(),
                             BACKBONE_DIRECTORIES['%s_reads_stats' % seqtype])

    @staticmethod
    def _seqs_in_files(fpaths, sample_size):
        'It yields SeqRecords from a list of files'
        if sample_size is None:
            samples = [None] * len(fpaths)
        else:
            #we count the seqs in every file
            num_seqs = []
            for fpath in fpaths:
                num_seqs.append(num_seqs_in_file(open(fpath)))
            total_seqs = float(sum(num_seqs))
            #how many seqs to sample for each file?
            samples = [int(num / total_seqs * sample_size) for num in num_seqs]

        if sample_size is None:
            for fpath in fpaths:
                for seq in seqs_in_file(open(fpath), sample_size=samples):
                    yield seq

    def _do_seq_stats(self, seqs_path, stats_dir, analyses,
                      sample_size):
        'It performs all kind of stats for a path'
        if isinstance(seqs_path, list):
            seqs_fpaths = [path.last_version for path in seqs_path]
            seqs = self._seqs_in_files(seqs_fpaths, sample_size)
            basename = BASENAME_FOR_ALL_TOGETHER
        else:
            basename = seqs_path.basename
            fpath = seqs_path.last_version
            seqs = seqs_in_file(open(fpath), sample_size=sample_size)

        for analysis in analyses:
            analysis_basename = '%s.%s' % (basename, analysis)
            seqs, seqs2 = itertools.tee(seqs, 2)
            self._do_distrib(seqs2, analysis, analysis_basename, stats_dir)

        # now general stats
        analysis_basename = '%s.general_stats' % basename
        self._do_general_stats(seqs, analysis_basename, stats_dir)

    def _do_general_stats(self, seqs, basename, stats_dir):
        'It performs the  general stats analysis'
        general_fpath = os.path.join(self._get_project_path(), stats_dir,
                                     basename + '.txt')
        if os.path.exists(general_fpath):
            return

        general_fhand = open(general_fpath, 'w')
        stats = general_seq_statistics(seqs)
        for key, value in stats.items():
            if value is not None:
                to_print = '%-19s : %d\n' % (key, value)
                general_fhand.write(to_print)

    @staticmethod
    def _do_distrib(seqs, analysis, basename, stats_dir):
        'I actually do the distrib'
        plot_fpath = os.path.join(stats_dir, basename + '.png')
        distrib_fpath = os.path.join(stats_dir, basename + '.dat')
        if os.path.exists(distrib_fpath) and os.path.exists(plot_fpath):
            return
        try:
            plot_fhand = open(plot_fpath, 'w')
            distrib_fhand = open(distrib_fpath, 'w')

            seq_distrib(sequences=seqs, kind=analysis,
                        distrib_fhand=distrib_fhand, plot_fhand=plot_fhand)
        except TypeError:
            plot_fhand.close()
            distrib_fhand.close()
            os.remove(plot_fpath)
            os.remove(distrib_fpath)
            raise

    def _do_diff_seq_stats(self, seqs1_fpath, seqs2_fpath, basename,
                           stats_dir, analyses, sample_size):
        'It performs the differential distribution'

        for analysis in analyses:
            if isinstance(seqs1_fpath, list):
                seqs1 = self._seqs_in_files(seqs1_fpath,
                                            sample_size=sample_size)
                seqs2 = self._seqs_in_files(seqs2_fpath,
                                            sample_size=sample_size)
            else:
                seqs1 = seqs_in_file(open(seqs1_fpath),
                                          sample_size=sample_size)
                seqs2 = seqs_in_file(open(seqs2_fpath),
                                          sample_size=sample_size)

            analysis_basename = '%s.diff_%s' % (basename, analysis)
            self._do_diff_distrib(seqs1, seqs2, analysis,
                                  analysis_basename, stats_dir)

    @staticmethod
    def _do_diff_distrib(clean_seqs, original_seqs, analysis, basename,
                         stats_dir):
        'It performs the differential distribution'
        plot_fpath = os.path.join(stats_dir, basename + '.png')
        distrib_fpath = os.path.join(stats_dir, basename + '.dat')

        if os.path.exists(distrib_fpath) and os.path.exists(plot_fpath):
            return

        seq_distrib_diff(clean_seqs, original_seqs, analysis,
                         distrib_fhand=open(distrib_fpath, 'w'),
                         plot_fhand=open(plot_fpath, 'w'))

DEFINITIONS = {
    'clean_reads':
        {'inputs':{
            'reads':
                {'directory': 'original_reads',
                 'file_kinds': 'sequence_files'}
            },
         'outputs':{'reads':{'directory': 'cleaned_reads'}},
         'analyzer': CleanReadsAnalyzer,
        },
    'read_stats':
        {'inputs':{
            'original_reads':
                {'directory': 'original_reads',
                 'file_kinds': 'sequence_files'},
            'cleaned_reads':
                {'directory': 'cleaned_reads',
                 'file_kinds': 'sequence_files'}
                },
         'outputs':{'original_reads':{'directory':'original_reads_stats'},
                    'cleaned_reads'   :{'directory':'cleaned_reads_stats'}},
         'analyzer': ReadsStatsAnalyzer,
        },
     }
<|endoftext|>


<commit_before># Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors
# MIT License. See license.txt
from __future__ import unicode_literals

import os
import unittest

import frappe
from frappe.utils import cint, add_to_date, now
from frappe.model.naming import revert_series_if_last, make_autoname, parse_naming_series
from frappe.exceptions import DoesNotExistError

class TestDocument(unittest.TestCase):
	def test_get_return_empty_list_for_table_field_if_none(self):
		d = frappe.get_doc({"doctype":"User"})
		self.assertEqual(d.get("roles"), [])

	def test_load(self):
		d = frappe.get_doc("DocType", "User")
		self.assertEqual(d.doctype, "DocType")
		self.assertEqual(d.name, "User")
		self.assertEqual(d.allow_rename, 1)
		self.assertTrue(isinstance(d.fields, list))
		self.assertTrue(isinstance(d.permissions, list))
		self.assertTrue(filter(lambda d: d.fieldname=="email", d.fields))

	def test_load_single(self):
		d = frappe.get_doc("Website Settings", "Website Settings")
		self.assertEqual(d.name, "Website Settings")
		self.assertEqual(d.doctype, "Website Settings")
		self.assertTrue(d.disable_signup in (0, 1))

	def test_insert(self):
		d = frappe.get_doc({
			"doctype":"Event",
			"subject":"test-doc-test-event 1",
			"starts_on": "2014-01-01",
			"event_type": "Public"
		})
		d.insert()
		self.assertTrue(d.name.startswith("EV"))
		self.assertEqual(frappe.db.get_value("Event", d.name, "subject"),
			"test-doc-test-event 1")

		# test if default values are added
		self.assertEqual(d.send_reminder, 1)
		return d

	def test_insert_with_child(self):
		d = frappe.get_doc({
			"doctype":"Event",
			"subject":"test-doc-test-event 2",
			"starts_on": "2014-01-01",
			"event_type": "Public"
		})
		d.insert()
		self.assertTrue(d.name.startswith("EV"))
		self.assertEqual(frappe.db.get_value("Event", d.name, "subject"),
			"test-doc-test-event 2")

	def test_update(self):
		d = self.test_insert()
		d.subject = "subject changed"
		d.save()

		self.assertEqual(frappe.db.get_value(d.doctype, d.name, "subject"), "subject changed")

	def test_value_changed(self):
		d = self.test_insert()
		d.subject = "subject changed again"
		d.save()
		self.assertTrue(d.has_value_changed('subject'))
		self.assertFalse(d.has_value_changed('event_type'))

	def test_mandatory(self):
		# TODO: recheck if it is OK to force delete
		frappe.delete_doc_if_exists("User", "test_mandatory@example.com", 1)

		d = frappe.get_doc({
			"doctype": "User",
			"email": "test_mandatory@example.com",
		})
		self.assertRaises(frappe.MandatoryError, d.insert)

		d.set("first_name", "Test Mandatory")
		d.insert()
		self.assertEqual(frappe.db.get_value("User", d.name), d.name)

	def test_confict_validation(self):
		d1 = self.test_insert()
		d2 = frappe.get_doc(d1.doctype, d1.name)
		d1.save()
		self.assertRaises(frappe.TimestampMismatchError, d2.save)

	def test_confict_validation_single(self):
		d1 = frappe.get_doc("Website Settings", "Website Settings")
		d1.home_page = "test-web-page-1"

		d2 = frappe.get_doc("Website Settings", "Website Settings")
		d2.home_page = "test-web-page-1"

		d1.save()
		self.assertRaises(frappe.TimestampMismatchError, d2.save)

	def test_permission(self):
		frappe.set_user("Guest")
		self.assertRaises(frappe.PermissionError, self.test_insert)
		frappe.set_user("Administrator")

	def test_permission_single(self):
		frappe.set_user("Guest")
		d = frappe.get_doc("Website Settings", "Website Settigns")
		self.assertRaises(frappe.PermissionError, d.save)
		frappe.set_user("Administrator")

	def test_link_validation(self):
		frappe.delete_doc_if_exists("User", "test_link_validation@example.com", 1)

		d = frappe.get_doc({
			"doctype": "User",
			"email": "test_link_validation@example.com",
			"first_name": "Link Validation",
			"roles": [
				{
					"role": "ABC"
				}
			]
		})
		self.assertRaises(frappe.LinkValidationError, d.insert)

		d.roles = []
		d.append("roles", {
			"role": "System Manager"
		})
		d.insert()

		self.assertEqual(frappe.db.get_value("User", d.name), d.name)

	def test_validate(self):
		d = self.test_insert()
		d.starts_on = "2014-01-01"
		d.ends_on = "2013-01-01"
		self.assertRaises(frappe.ValidationError, d.validate)
		self.assertRaises(frappe.ValidationError, d.run_method, "validate")
		self.assertRaises(frappe.ValidationError, d.save)

	def test_update_after_submit(self):
		d = self.test_insert()
		d.starts_on = "2014-09-09"
		self.assertRaises(frappe.UpdateAfterSubmitError, d.validate_update_after_submit)
		d.meta.get_field("starts_on").allow_on_submit = 1
		d.validate_update_after_submit()
		d.meta.get_field("starts_on").allow_on_submit = 0

		# when comparing date(2014, 1, 1) and "2014-01-01"
		d.reload()
		d.starts_on = "2014-01-01"
		d.validate_update_after_submit()

	def test_varchar_length(self):
		d = self.test_insert()
		d.subject = "abcde"*100
		self.assertRaises(frappe.CharacterLengthExceededError, d.save)

	def test_xss_filter(self):
		d = self.test_insert()

		# script
		xss = ''
		escaped_xss = xss.replace('<', '<').replace('>', '>')
		d.subject += xss
		d.save()
		d.reload()

		self.assertTrue(xss not in d.subject)
		self.assertTrue(escaped_xss in d.subject)

		# onload
		xss = '
Test
'
		escaped_xss = '
Test
'
		d.subject += xss
		d.save()
		d.reload()

		self.assertTrue(xss not in d.subject)
		self.assertTrue(escaped_xss in d.subject)

		# css attributes
		xss = '
Test
'
		escaped_xss = '
Test
'
		d.subject += xss
		d.save()
		d.reload()

		self.assertTrue(xss not in d.subject)
		self.assertTrue(escaped_xss in d.subject)

	def test_link_count(self):
		if os.environ.get('CI'):
			# cannot run this test reliably in travis due to its handling
			# of parallelism
			return

		from frappe.model.utils.link_count import update_link_count

		update_link_count()

		doctype, name = 'User', 'test@example.com'

		d = self.test_insert()
		d.append('event_participants', {"reference_doctype": doctype, "reference_docname": name})

		d.save()

		link_count = frappe.cache().get_value('_link_count') or {}
		old_count = link_count.get((doctype, name)) or 0

		frappe.db.commit()

		link_count = frappe.cache().get_value('_link_count') or {}
		new_count = link_count.get((doctype, name)) or 0

		self.assertEqual(old_count + 1, new_count)

		before_update = frappe.db.get_value(doctype, name, 'idx')

		update_link_count()

		after_update = frappe.db.get_value(doctype, name, 'idx')

		self.assertEqual(before_update + new_count, after_update)

	def test_naming_series(self):
		data = ["TEST-", "TEST/17-18/.test_data./.####", "TEST.YYYY.MM.####"]

		for series in data:
			name = make_autoname(series)
			prefix = series

			if ".#" in series:
				prefix = series.rsplit('.',1)[0]

			prefix = parse_naming_series(prefix)
			old_current = frappe.db.get_value('Series', prefix, "current", order_by="name")

			revert_series_if_last(series, name)
			new_current = cint(frappe.db.get_value('Series', prefix, "current", order_by="name"))

			self.assertEqual(cint(old_current) - 1, new_current)

	def test_rename_doc(self):
		from random import choice, sample

		available_documents = []
		doctype = "ToDo"

		# data generation: 4 todo documents
		for num in range(1, 5):
			doc = frappe.get_doc({
				"doctype": doctype,
				"date": add_to_date(now(), days=num),
				"description": "this is todo #{}".format(num)
			}).insert()
			available_documents.append(doc.name)

		# test 1: document renaming
		old_name = choice(available_documents)
		new_name = old_name + '.new'
		self.assertEqual(new_name, frappe.rename_doc(doctype, old_name, new_name, force=True))
		available_documents.remove(old_name)
		available_documents.append(new_name)

		# test 2: merge documents
		first_todo, second_todo = sample(available_documents, 2)

		second_todo_doc = frappe.get_doc(doctype, second_todo)
		second_todo_doc.priority = "High"
		second_todo_doc.save()

		merged_todo = frappe.rename_doc(doctype, first_todo, second_todo, merge=True, force=True)
		merged_todo_doc = frappe.get_doc(doctype, merged_todo)
		available_documents.remove(first_todo)

		with self.assertRaises(DoesNotExistError):
			frappe.get_doc(doctype, first_todo)

		self.assertEqual(merged_todo_doc.priority, second_todo_doc.priority)

		for docname in available_documents:
			frappe.delete_doc(doctype, docname)<commit_msg>test: Added tests for dt controller + db sync<commit_after># Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors
# MIT License. See license.txt
from __future__ import unicode_literals

import os
import unittest

import frappe
from frappe.utils import cint, add_to_date, now
from frappe.model.naming import revert_series_if_last, make_autoname, parse_naming_series
from frappe.exceptions import DoesNotExistError

class TestDocument(unittest.TestCase):
	def test_get_return_empty_list_for_table_field_if_none(self):
		d = frappe.get_doc({"doctype":"User"})
		self.assertEqual(d.get("roles"), [])

	def test_load(self):
		d = frappe.get_doc("DocType", "User")
		self.assertEqual(d.doctype, "DocType")
		self.assertEqual(d.name, "User")
		self.assertEqual(d.allow_rename, 1)
		self.assertTrue(isinstance(d.fields, list))
		self.assertTrue(isinstance(d.permissions, list))
		self.assertTrue(filter(lambda d: d.fieldname=="email", d.fields))

	def test_load_single(self):
		d = frappe.get_doc("Website Settings", "Website Settings")
		self.assertEqual(d.name, "Website Settings")
		self.assertEqual(d.doctype, "Website Settings")
		self.assertTrue(d.disable_signup in (0, 1))

	def test_insert(self):
		d = frappe.get_doc({
			"doctype":"Event",
			"subject":"test-doc-test-event 1",
			"starts_on": "2014-01-01",
			"event_type": "Public"
		})
		d.insert()
		self.assertTrue(d.name.startswith("EV"))
		self.assertEqual(frappe.db.get_value("Event", d.name, "subject"),
			"test-doc-test-event 1")

		# test if default values are added
		self.assertEqual(d.send_reminder, 1)
		return d

	def test_insert_with_child(self):
		d = frappe.get_doc({
			"doctype":"Event",
			"subject":"test-doc-test-event 2",
			"starts_on": "2014-01-01",
			"event_type": "Public"
		})
		d.insert()
		self.assertTrue(d.name.startswith("EV"))
		self.assertEqual(frappe.db.get_value("Event", d.name, "subject"),
			"test-doc-test-event 2")

	def test_update(self):
		d = self.test_insert()
		d.subject = "subject changed"
		d.save()

		self.assertEqual(frappe.db.get_value(d.doctype, d.name, "subject"), "subject changed")

	def test_value_changed(self):
		d = self.test_insert()
		d.subject = "subject changed again"
		d.save()
		self.assertTrue(d.has_value_changed('subject'))
		self.assertFalse(d.has_value_changed('event_type'))

	def test_mandatory(self):
		# TODO: recheck if it is OK to force delete
		frappe.delete_doc_if_exists("User", "test_mandatory@example.com", 1)

		d = frappe.get_doc({
			"doctype": "User",
			"email": "test_mandatory@example.com",
		})
		self.assertRaises(frappe.MandatoryError, d.insert)

		d.set("first_name", "Test Mandatory")
		d.insert()
		self.assertEqual(frappe.db.get_value("User", d.name), d.name)

	def test_confict_validation(self):
		d1 = self.test_insert()
		d2 = frappe.get_doc(d1.doctype, d1.name)
		d1.save()
		self.assertRaises(frappe.TimestampMismatchError, d2.save)

	def test_confict_validation_single(self):
		d1 = frappe.get_doc("Website Settings", "Website Settings")
		d1.home_page = "test-web-page-1"

		d2 = frappe.get_doc("Website Settings", "Website Settings")
		d2.home_page = "test-web-page-1"

		d1.save()
		self.assertRaises(frappe.TimestampMismatchError, d2.save)

	def test_permission(self):
		frappe.set_user("Guest")
		self.assertRaises(frappe.PermissionError, self.test_insert)
		frappe.set_user("Administrator")

	def test_permission_single(self):
		frappe.set_user("Guest")
		d = frappe.get_doc("Website Settings", "Website Settigns")
		self.assertRaises(frappe.PermissionError, d.save)
		frappe.set_user("Administrator")

	def test_link_validation(self):
		frappe.delete_doc_if_exists("User", "test_link_validation@example.com", 1)

		d = frappe.get_doc({
			"doctype": "User",
			"email": "test_link_validation@example.com",
			"first_name": "Link Validation",
			"roles": [
				{
					"role": "ABC"
				}
			]
		})
		self.assertRaises(frappe.LinkValidationError, d.insert)

		d.roles = []
		d.append("roles", {
			"role": "System Manager"
		})
		d.insert()

		self.assertEqual(frappe.db.get_value("User", d.name), d.name)

	def test_validate(self):
		d = self.test_insert()
		d.starts_on = "2014-01-01"
		d.ends_on = "2013-01-01"
		self.assertRaises(frappe.ValidationError, d.validate)
		self.assertRaises(frappe.ValidationError, d.run_method, "validate")
		self.assertRaises(frappe.ValidationError, d.save)

	def test_update_after_submit(self):
		d = self.test_insert()
		d.starts_on = "2014-09-09"
		self.assertRaises(frappe.UpdateAfterSubmitError, d.validate_update_after_submit)
		d.meta.get_field("starts_on").allow_on_submit = 1
		d.validate_update_after_submit()
		d.meta.get_field("starts_on").allow_on_submit = 0

		# when comparing date(2014, 1, 1) and "2014-01-01"
		d.reload()
		d.starts_on = "2014-01-01"
		d.validate_update_after_submit()

	def test_varchar_length(self):
		d = self.test_insert()
		d.subject = "abcde"*100
		self.assertRaises(frappe.CharacterLengthExceededError, d.save)

	def test_xss_filter(self):
		d = self.test_insert()

		# script
		xss = ''
		escaped_xss = xss.replace('<', '<').replace('>', '>')
		d.subject += xss
		d.save()
		d.reload()

		self.assertTrue(xss not in d.subject)
		self.assertTrue(escaped_xss in d.subject)

		# onload
		xss = '
Test
'
		escaped_xss = '
Test
'
		d.subject += xss
		d.save()
		d.reload()

		self.assertTrue(xss not in d.subject)
		self.assertTrue(escaped_xss in d.subject)

		# css attributes
		xss = '
Test
'
		escaped_xss = '
Test
'
		d.subject += xss
		d.save()
		d.reload()

		self.assertTrue(xss not in d.subject)
		self.assertTrue(escaped_xss in d.subject)

	def test_link_count(self):
		if os.environ.get('CI'):
			# cannot run this test reliably in travis due to its handling
			# of parallelism
			return

		from frappe.model.utils.link_count import update_link_count

		update_link_count()

		doctype, name = 'User', 'test@example.com'

		d = self.test_insert()
		d.append('event_participants', {"reference_doctype": doctype, "reference_docname": name})

		d.save()

		link_count = frappe.cache().get_value('_link_count') or {}
		old_count = link_count.get((doctype, name)) or 0

		frappe.db.commit()

		link_count = frappe.cache().get_value('_link_count') or {}
		new_count = link_count.get((doctype, name)) or 0

		self.assertEqual(old_count + 1, new_count)

		before_update = frappe.db.get_value(doctype, name, 'idx')

		update_link_count()

		after_update = frappe.db.get_value(doctype, name, 'idx')

		self.assertEqual(before_update + new_count, after_update)

	def test_naming_series(self):
		data = ["TEST-", "TEST/17-18/.test_data./.####", "TEST.YYYY.MM.####"]

		for series in data:
			name = make_autoname(series)
			prefix = series

			if ".#" in series:
				prefix = series.rsplit('.',1)[0]

			prefix = parse_naming_series(prefix)
			old_current = frappe.db.get_value('Series', prefix, "current", order_by="name")

			revert_series_if_last(series, name)
			new_current = cint(frappe.db.get_value('Series', prefix, "current", order_by="name"))

			self.assertEqual(cint(old_current) - 1, new_current)

	def test_rename_doc(self):
		from random import choice, sample
		from frappe.model.base_document import get_controller

		available_documents = []
		doctype = "ToDo"

		# data generation: 4 todo documents
		for num in range(1, 5):
			doc = frappe.get_doc({
				"doctype": doctype,
				"date": add_to_date(now(), days=num),
				"description": "this is todo #{}".format(num)
			}).insert()
			available_documents.append(doc.name)

		# test 1: document renaming
		old_name = choice(available_documents)
		new_name = old_name + '.new'
		self.assertEqual(new_name, frappe.rename_doc(doctype, old_name, new_name, force=True))
		available_documents.remove(old_name)
		available_documents.append(new_name)

		# test 2: merge documents
		first_todo, second_todo = sample(available_documents, 2)

		second_todo_doc = frappe.get_doc(doctype, second_todo)
		second_todo_doc.priority = "High"
		second_todo_doc.save()

		merged_todo = frappe.rename_doc(doctype, first_todo, second_todo, merge=True, force=True)
		merged_todo_doc = frappe.get_doc(doctype, merged_todo)
		available_documents.remove(first_todo)

		with self.assertRaises(DoesNotExistError):
			frappe.get_doc(doctype, first_todo)

		self.assertEqual(merged_todo_doc.priority, second_todo_doc.priority)

		for docname in available_documents:
			frappe.delete_doc(doctype, docname)

		# test 3: rename doctypes with controller code
		doctype = frappe._dict({
			"old": "Test Rename Document Old",
			"new": "Test Rename Document New",
		})
		doc = frappe.get_doc({
			"doctype": "DocType",
			"module": "Custom",
			"name": doctype.old,
			"custom": 0,
			"fields": [
				{
					"label": "Some Field",
					"fieldname": "some_fieldname",
					"fieldtype": "Data"
				}
			],
			"permissions": [
				{"role": "System Manager", "read": 1}
			],
		})
		doc.save()
		# check if module exists exists;
		# if custom, get_controller will return Document class
		# if not custom, a different class will be returned
		self.assertNotEqual(get_controller(doctype.old), frappe.model.document.Document)

		# rename doc via wrapper API accessible via /desk
		frappe.rename_doc("DocType", doctype.old, doctype.new)

		# check if database and controllers are updated
		self.assertTrue(frappe.db.exists("DocType", doctype.new))
		self.assertFalse(frappe.db.exists("DocType", doctype.old))
		with self.assertRaises(ImportError):
			get_controller(doctype.old)
<|endoftext|>


<commit_before>import requests
import sys

print sys.argv[1:]
lenargv = len(sys.argv[1:])
if lenargv != 3:
	print('3 arguments needed')

userLogin = ''
userKey = ''
message = ''

baseUrl = 'https://smsapi.free-mobile.fr/sendmsg?'

payload = 'user=' + userLogin + '&pass=' + userKey + '&msg=' + message

finalUrl = baseUrl + payload

r = requests.post(finalUrl)

if r.status_code == 200:
	print("Message sent successfully")
elif r.status_code == 400:
	print("One argument is missing")
elif r.status_code == 402:
	print("Too many sms sent, please wait and retry")
elif r.status_code == 403:
	print("The service is not activated on the line holder's account, or incorrect login/userkey")
elif r.status_code == 500:
	print("Server side error. Please try again later")

print("Stopping execution")

<commit_msg>Script works<commit_after>import requests
import sys

print sys.argv[1:]

def main():

    lenargv = len(sys.argv[1:])
    if lenargv != 3:
        print('3 arguments needed: userLogin, userKey, message')
        return -1
    else :
        userLogin = sys.argv[1]
        userKey = sys.argv[2]
        message = sys.argv[3]

        baseUrl = 'https://smsapi.free-mobile.fr/sendmsg?'

        payload = 'user=' + userLogin + '&pass=' + userKey + '&msg=' + message

        finalUrl = baseUrl + payload

        r = requests.post(finalUrl)

        if r.status_code == 200:
                print("Message sent successfully")
                print("Stopping execution")
                return 0
        elif r.status_code == 400:
                print("One argument is missing")
                print("Stopping execution")
                return 400
        elif r.status_code == 402:
                print("Too many sms sent, please wait and retry")
                print("Stopping execution")
                return 402
        elif r.status_code == 403:
                print("The service is not activated on the line holder's account, or incorrect login/userkey")
                print("Stopping execution")
                return 403
        elif r.status_code == 500:
                print("Server side error. Please try again later")
                print("Stopping execution")
                return 500

main()
<|endoftext|>


<commit_before>import os
from setuptools import setup, find_packages
import orb

here = os.path.abspath(os.path.dirname(__file__))
try:
    with open(os.path.join(here, 'README.md')) as f:
        README = f.read()
except IOError:
    README = orb.__doc__

try:
    VERSION = orb.__version__
except AttributeError:
    VERSION = '2.5'

try:
    REQUIREMENTS = orb.__depends__
except AttributeError:
    REQUIREMENTS = []

setup(
    name = 'projex_orb',
    version = VERSION,
    author = 'Projex Software',
    author_email = 'team@projexsoftware.com',
    maintainer = 'Projex Software',
    maintainer_email = 'team@projexsoftware.com',
    description = '''''',
    license = 'LGPL',
    keywords = '',
    url = '',
    include_package_data=True,
    packages = find_packages(),
    install_requires = REQUIREMENTS,
    tests_require = REQUIREMENTS,
    long_description= README,
    classifiers=[],
)<commit_msg>added description to orb<commit_after>import os
from setuptools import setup, find_packages
import orb

here = os.path.abspath(os.path.dirname(__file__))
try:
    with open(os.path.join(here, 'README.md')) as f:
        README = f.read()
except IOError:
    README = orb.__doc__

try:
    VERSION = orb.__version__
except AttributeError:
    VERSION = '2.5'

try:
    REQUIREMENTS = orb.__depends__
except AttributeError:
    REQUIREMENTS = []

setup(
    name = 'projex_orb',
    version = VERSION,
    author = 'Projex Software',
    author_email = 'team@projexsoftware.com',
    maintainer = 'Projex Software',
    maintainer_email = 'team@projexsoftware.com',
    description = 'Database ORM and API builder.',
    license = 'LGPL',
    keywords = '',
    url = '',
    include_package_data=True,
    packages = find_packages(),
    install_requires = REQUIREMENTS,
    tests_require = REQUIREMENTS,
    long_description= README,
    classifiers=[],
)<|endoftext|>


<commit_before>"""Module to initialise and configure the flask app and the db."""
from flask import Flask
from flask_sqlalchemy import SQLAlchemy
from configuration.config import app_config
from flask_restful import Api
from flasgger import Swagger

db = SQLAlchemy()

def create_app(configuration):
    """Initialise and cofigure the app and db."""
    app = Flask(__name__)
    app.config.from_object(app_config[configuration])
    db.init_app(app)
    return app

app = create_app("development")
api = Api(app=app, prefix="/api/v1")
swagger = Swagger(app)
<commit_msg>[ft #148218351] Add swagger configuration<commit_after>"""Module to initialise and configure the flask app and the db."""
from flask import Flask
from flask_sqlalchemy import SQLAlchemy
from configuration.config import app_config
from flask_restful import Api
from flasgger import Swagger

db = SQLAlchemy()

def create_app(configuration):
    """Initialise and cofigure the app and db."""
    app = Flask(__name__)
    app.config.from_object(app_config[configuration])
    db.init_app(app)
    return app

template = {
  "swagger": "2.0",
  "info": {
    "title": "BUCKETLIST API",
    "description": "API for managing your bucketlists",
    "contact": {
      "responsibleDeveloper": "Jackson Onyango",
      "email": "jackson.onyango@andela.com",
    },
    "version": "1.0"
  },
  "schemes": [
    "http",
    "https"
  ],
  "produces": ["application/x-www-form-urlencoded",
               "application/json",
               "application/txt"],
  "operationId": "getmyData",
  "content-type": "text"
}

app = create_app("development")
api = Api(app=app, prefix="/api/v1")
swagger = Swagger(app, template=template)
<|endoftext|>


<commit_before>#!/usr/bin/env python

#To run : python bugWorkloadGen.py -n 3
import os
import re
import sys
import stat
import subprocess
import argparse
import time
import itertools
import json
import pprint
import collections
import threading

from shutil import copyfile
from string import maketrans

#All functions that has options go here
FallocOptions = ['FALLOC_FL_ZERO_RANGE','FALLOC_FL_ZERO_RANGE | FALLOC_FL_KEEP_SIZE','FALLOC_FL_PUNCH_HOLE | FALLOC_FL_KEEP_SIZE', '0',  'FALLOC_FL_KEEP_SIZE']

FsyncOptions = ['fsync','fdatasync']

#This should take care of file name/ dir name
FileOptions = ['foo', 'bar', 'A/foo', 'A/bar']

DirOptions = ['A', 'B']

#this will take care of offset + length combo
#Start = 4-16K , append = 16K-20K, overlap = 8000 - 12096, prepend = 0-4K
WriteOptions = ['start', 'append', 'overlap', 'prepend']

#d_overlap = 8K-12K (has to be aligned)
dWriteOptions = ['start', 'append', 'd_overlap', 'prepend']

OperationSet = ['creat', 'mkdir', 'mknod', 'falloc', 'write', 'dwrite', 'link', 'unlink', 'remove', 'rename', 'symlink', 'fsetxattr', 'removexattr', 'fdatasync']

#We are skipping 041, 336, 342, 343
#The sequences we want to reach to
expected_sequence = []
expected_sync_sequence = []

#----------------------Bug summary-----------------------#

#Length 1 = 1
#Length 2 = 7
#length 3 = 5

# Total encoded = 13

#--------------------------------------------------------#

# 1. btrfs_link_unlink 3
expected_sequence.append([('link', ('foo', 'bar')), ('unlink', ('bar')), ('creat', ('bar'))])
expected_sync_sequence.append([('sync'), ('none'), ('fsync', 'bar')])

# 2. btrfs_rename_special_file 3
expected_sequence.append([('mknod', ('foo')), ('rename', ('foo', 'bar')), ('link', ('bar', 'foo'))])
expected_sync_sequence.append([('fsync', 'bar'), ('none'), ('fsync', 'bar')])

# 3. new_bug1_btrfs 2
expected_sequence.append([('write', ('foo', 'start')), ('falloc', ('foo', 'FALLOC_FL_ZERO_RANGE | FALLOC_FL_KEEP_SIZE', 'append'))])
expected_sync_sequence.append([('fsync', 'foo'), ('fsync', 'foo')])

# 4. new_bug2_f2fs 3
expected_sequence.append([('write', ('foo', 'start')), ('falloc', ('foo', 'FALLOC_FL_ZERO_RANGE | FALLOC_FL_KEEP_SIZE', 'append')), ('fdatasync', ('foo'))])
expected_sync_sequence.append([('sync'), ('none'), ('none')])

# 5. generic_034 2
expected_sequence.append([('creat', ('A/foo')), ('creat', ('A/bar'))])
expected_sync_sequence.append([('sync'), ('fsync', 'A')])

# 6. generic_039 2
expected_sequence.append([('link', ('foo', 'bar')), ('remove', ('bar'))])
expected_sync_sequence.append([('sync'), ('fsync', 'foo')])

# 7. generic_059 2
expected_sequence.append([('write', ('foo', 'start')), ('falloc', ('foo', 'FALLOC_FL_PUNCH_HOLE | FALLOC_FL_KEEP_SIZE', 'overlap'))])
expected_sync_sequence.append([('sync'), ('fsync', 'foo')])

# 8. generic_066 2
expected_sequence.append([('fsetxattr', ('foo')), ('removexattr', ('foo'))])
expected_sync_sequence.append([('sync'), ('fsync', 'foo')])

# 9. generic_341 3
expected_sequence.append([('creat', ('A/foo')), ('rename', ('A', 'B')), ('mkdir', ('A'))])
expected_sync_sequence.append([('sync'), ('none'), ('fsync', 'A')])

# 10. generic_348 1
expected_sequence.append([('symlink', ('foo', 'A/bar'))])
expected_sync_sequence.append([('fsync', 'A')])

# 11. generic_376 2
expected_sequence.append([('rename', ('foo', 'bar')), ('creat', ('foo'))])
expected_sync_sequence.append([('none'), ('fsync', 'bar')])

# 12. generic_468 3
expected_sequence.append([('write', ('foo', 'start')), ('falloc', ('foo', 'FALLOC_FL_KEEP_SIZE', 'append')), ('fdatasync', ('foo'))])
expected_sync_sequence.append([('sync'), ('none'), ('none')])

# 13. ext4_direct_write 2
expected_sequence.append([('write', ('foo', 'start')), ('dwrite', ('foo', 'prepend'))])
expected_sync_sequence.append([('none'), ('none')])

def build_parser():
    parser = argparse.ArgumentParser(description='Bug Workload Generator for XFSMonkey v0.1')

    # global args
    parser.add_argument('--sequence_len', '-l', default='3', help='Number of critical ops in the bugy workload')

    return parser

def print_setup(parsed_args):
    print '\n{: ^50s}'.format('XFSMonkey Bug Workload generatorv0.1\n')
    print '='*20, 'Setup' , '='*20, '\n'
    print '{0:20}  {1}'.format('Sequence length', parsed_args.sequence_len)
    print '\n', '='*48, '\n'

min = 0

def buildTuple(command):
    if command == 'creat':
        d = tuple(FileOptions)
    elif command == 'mkdir':
        d = tuple(DirOptions)
    elif command == 'mknod':
        d = tuple(FileOptions)
    elif command == 'falloc':
        d_tmp = list()
        d_tmp.append(FileOptions)
        d_tmp.append(FallocOptions)
        d_tmp.append(WriteOptions)
        d = list()
        for i in itertools.product(*d_tmp):
            d.append(i)
    elif command == 'write':
        d_tmp = list()
        d_tmp.append(FileOptions)
        d_tmp.append(WriteOptions)
        d = list()
        for i in itertools.product(*d_tmp):
            d.append(i)
    elif command == 'dwrite':
        d_tmp = list()
        d_tmp.append(FileOptions)
        d_tmp.append(dWriteOptions)
        d = list()
        for i in itertools.product(*d_tmp):
            d.append(i)
    elif command == 'link' or command == 'symlink':
        d_tmp = list()
        d_tmp.append(FileOptions)
        d_tmp.append(FileOptions)
        d = list()
        for i in itertools.product(*d_tmp):
            if len(set(i)) == 2:
                d.append(i)
    elif command == 'rename':
        d_tmp = list()
        d_tmp.append(FileOptions)
        d_tmp.append(FileOptions)
        d = list()
        for i in itertools.product(*d_tmp):
            if len(set(i)) == 2:
                d.append(i)
        d_tmp = list()
        d_tmp.append(DirOptions)
        d_tmp.append(DirOptions)
        for i in itertools.product(*d_tmp):
            if len(set(i)) == 2:
                d.append(i)
    elif command == 'remove' or command == 'unlink':
        d = tuple(FileOptions + DirOptions)
    elif command == 'fdatasync' or command == 'fsetxattr' or command == 'removexattr':
        d = tuple(FileOptions)
    elif command == 'fsync':
        d = tuple(FileOptions + DirOptions)
    else:
        d=()
    return d

def isBugWorkload(opList, paramList, syncList):
    for i in xrange(0,len(expected_sequence)):
        if len(opList) != len(expected_sequence[i]):
            continue

        flag = 1

        for j in xrange(0, len(expected_sequence[i])):
            if opList[j] == expected_sequence[i][j][0] and paramList[j] == expected_sequence[i][j][1] and tuple(syncList[j]) == tuple(expected_sync_sequence[i][j]):
                continue
            else:
                flag = 0
                break

        if flag == 1:
            print 'Found match to Bug # ', i+1, ' : '
            print 'Length of seq : ',  len(expected_sequence[i])
            print 'Expected sequence = ' , expected_sequence[i]
            print 'Expected sync sequence = ', expected_sync_sequence[i]
            print 'Auto generator found : '
            print opList
            print paramList
            print syncList
            print '\n\n'
            return True

def main():

    #open log file
    log_file = time.strftime('%Y%m%d_%H%M%S') + '-bugWorkloadGen.log'
    log_file_handle = open(log_file, 'w')

    #Parse input args
    parsed_args = build_parser().parse_args()

    #Print the test setup - just for sanity
    print_setup(parsed_args)

    num_ops = parsed_args.sequence_len

    for i in xrange(0,len(expected_sequence)):
        print 'Bug #', i+1
        print expected_sequence[i]
        print expected_sync_sequence[i]
        print '\n'

    global_count = 0
    parameterList = {}
    for i in OperationSet:
        parameterList[i] = buildTuple(i)
        log = '{0}'.format(parameterList[i]);
        log = `i` + ' : Options = ' + `len(parameterList[i])` + '\n' + log + '\n\n'
        log_file_handle.write(log)

    d = buildTuple('fsync')
    fsync = ('fsync',)
    sync = ('sync')
    none = ('none')
    SyncSet = list()
    for i in xrange(0, len(d)):
        tup = list(fsync)
        tup.append(d[i])
        SyncSet.append(tup)
    SyncSet.append(sync)
    SyncSet.append(none)
    SyncSet = tuple(SyncSet)
#    print SyncSet

    syncPermutations = []
    for i in itertools.product(SyncSet, repeat=int(num_ops)):
        syncPermutations.append(i)
#        print i

#    print 'num fsync combinations = ', len(syncPermutations)

    start_time = time.time()
    count = 0
    permutations = []
#    for i in itertools.permutations(OperationSet, int(num_ops)):
    for i in itertools.product(OperationSet, repeat=int(num_ops)):
        permutations.append(i)
        log = ', '.join(i);
        log = `count` + ' : ' + log + '\n'
        count +=1
        global_count +=1
        log_file_handle.write(log)

        #Now for each of this permutation, find all possible permutation of paramters
        combination = list()
        for length in xrange(0,len(permutations[count-1])):
            combination.append(parameterList[permutations[count-1][length]])
        count_param = 0
        for j in itertools.product(*combination):
            log = '{0}'.format(j);
            log = `count_param` + ' : ' + log + '\n'
            count_param += 1
            global_count +=1
            log_file_handle.write(log)

#            #Let's insert fsync combinations here.
            for insSync in range(0, len(syncPermutations)):
                global_count +=1
                isBugWorkload(permutations[count-1], j, syncPermutations[insSync])

    end_time = time.time()
    print 'Total permutations of input op set = ', count

    print 'Total workloads inspected = ' , global_count

    print 'Time taken to match workloads = ', end_time-start_time, 'seconds\n\n'

    log_file_handle.close()

if __name__ == '__main__':
	main()
<commit_msg>Process in parallel<commit_after>#!/usr/bin/env python

#To run : python bugWorkloadGen.py -n 3
import os
import re
import sys
import stat
import subprocess
import argparse
import time
import itertools
import json
import pprint
import collections
import threading

from shutil import copyfile
from string import maketrans
from multiprocessing import Pool

#All functions that has options go here
FallocOptions = ['FALLOC_FL_ZERO_RANGE','FALLOC_FL_ZERO_RANGE | FALLOC_FL_KEEP_SIZE','FALLOC_FL_PUNCH_HOLE | FALLOC_FL_KEEP_SIZE', '0',  'FALLOC_FL_KEEP_SIZE']

FsyncOptions = ['fsync','fdatasync']

#This should take care of file name/ dir name
FileOptions = ['foo', 'bar', 'A/foo', 'A/bar']

DirOptions = ['A', 'B']

#this will take care of offset + length combo
#Start = 4-16K , append = 16K-20K, overlap = 8000 - 12096, prepend = 0-4K
WriteOptions = ['start', 'append', 'overlap', 'prepend']

#d_overlap = 8K-12K (has to be aligned)
dWriteOptions = ['start', 'append', 'd_overlap', 'prepend']

OperationSet = ['creat', 'mkdir', 'mknod', 'falloc', 'write', 'dwrite', 'link', 'unlink', 'remove', 'rename', 'symlink', 'fsetxattr', 'removexattr', 'fdatasync']

#We are skipping 041, 336, 342, 343
#The sequences we want to reach to
expected_sequence = []
expected_sync_sequence = []

#----------------------Bug summary-----------------------#

#Length 1 = 1
#Length 2 = 7
#length 3 = 5

# Total encoded = 13

#--------------------------------------------------------#

# 1. btrfs_link_unlink 3
expected_sequence.append([('link', ('foo', 'bar')), ('unlink', ('bar')), ('creat', ('bar'))])
expected_sync_sequence.append([('sync'), ('none'), ('fsync', 'bar')])

# 2. btrfs_rename_special_file 3
expected_sequence.append([('mknod', ('foo')), ('rename', ('foo', 'bar')), ('link', ('bar', 'foo'))])
expected_sync_sequence.append([('fsync', 'bar'), ('none'), ('fsync', 'bar')])

# 3. new_bug1_btrfs 2
expected_sequence.append([('write', ('foo', 'start')), ('falloc', ('foo', 'FALLOC_FL_ZERO_RANGE | FALLOC_FL_KEEP_SIZE', 'append'))])
expected_sync_sequence.append([('fsync', 'foo'), ('fsync', 'foo')])

# 4. new_bug2_f2fs 3
expected_sequence.append([('write', ('foo', 'start')), ('falloc', ('foo', 'FALLOC_FL_ZERO_RANGE | FALLOC_FL_KEEP_SIZE', 'append')), ('fdatasync', ('foo'))])
expected_sync_sequence.append([('sync'), ('none'), ('none')])

# 5. generic_034 2
expected_sequence.append([('creat', ('A/foo')), ('creat', ('A/bar'))])
expected_sync_sequence.append([('sync'), ('fsync', 'A')])

# 6. generic_039 2
expected_sequence.append([('link', ('foo', 'bar')), ('remove', ('bar'))])
expected_sync_sequence.append([('sync'), ('fsync', 'foo')])

# 7. generic_059 2
expected_sequence.append([('write', ('foo', 'start')), ('falloc', ('foo', 'FALLOC_FL_PUNCH_HOLE | FALLOC_FL_KEEP_SIZE', 'overlap'))])
expected_sync_sequence.append([('sync'), ('fsync', 'foo')])

# 8. generic_066 2
expected_sequence.append([('fsetxattr', ('foo')), ('removexattr', ('foo'))])
expected_sync_sequence.append([('sync'), ('fsync', 'foo')])

# 9. generic_341 3
expected_sequence.append([('creat', ('A/foo')), ('rename', ('A', 'B')), ('mkdir', ('A'))])
expected_sync_sequence.append([('sync'), ('none'), ('fsync', 'A')])

# 10. generic_348 1
expected_sequence.append([('symlink', ('foo', 'A/bar'))])
expected_sync_sequence.append([('fsync', 'A')])

# 11. generic_376 2
expected_sequence.append([('rename', ('foo', 'bar')), ('creat', ('foo'))])
expected_sync_sequence.append([('none'), ('fsync', 'bar')])

# 12. generic_468 3
expected_sequence.append([('write', ('foo', 'start')), ('falloc', ('foo', 'FALLOC_FL_KEEP_SIZE', 'append')), ('fdatasync', ('foo'))])
expected_sync_sequence.append([('sync'), ('none'), ('none')])

# 13. ext4_direct_write 2
expected_sequence.append([('write', ('foo', 'start')), ('dwrite', ('foo', 'prepend'))])
expected_sync_sequence.append([('none'), ('none')])

def build_parser():
    parser = argparse.ArgumentParser(description='Bug Workload Generator for XFSMonkey v0.1')

    # global args
    parser.add_argument('--sequence_len', '-l', default='3', help='Number of critical ops in the bugy workload')

    return parser

def print_setup(parsed_args):
    print '\n{: ^50s}'.format('XFSMonkey Bug Workload generatorv0.1\n')
    print '='*20, 'Setup' , '='*20, '\n'
    print '{0:20}  {1}'.format('Sequence length', parsed_args.sequence_len)
    print '\n', '='*48, '\n'

min = 0

def buildTuple(command):
    if command == 'creat':
        d = tuple(FileOptions)
    elif command == 'mkdir':
        d = tuple(DirOptions)
    elif command == 'mknod':
        d = tuple(FileOptions)
    elif command == 'falloc':
        d_tmp = list()
        d_tmp.append(FileOptions)
        d_tmp.append(FallocOptions)
        d_tmp.append(WriteOptions)
        d = list()
        for i in itertools.product(*d_tmp):
            d.append(i)
    elif command == 'write':
        d_tmp = list()
        d_tmp.append(FileOptions)
        d_tmp.append(WriteOptions)
        d = list()
        for i in itertools.product(*d_tmp):
            d.append(i)
    elif command == 'dwrite':
        d_tmp = list()
        d_tmp.append(FileOptions)
        d_tmp.append(dWriteOptions)
        d = list()
        for i in itertools.product(*d_tmp):
            d.append(i)
    elif command == 'link' or command == 'symlink':
        d_tmp = list()
        d_tmp.append(FileOptions)
        d_tmp.append(FileOptions)
        d = list()
        for i in itertools.product(*d_tmp):
            if len(set(i)) == 2:
                d.append(i)
    elif command == 'rename':
        d_tmp = list()
        d_tmp.append(FileOptions)
        d_tmp.append(FileOptions)
        d = list()
        for i in itertools.product(*d_tmp):
            if len(set(i)) == 2:
                d.append(i)
        d_tmp = list()
        d_tmp.append(DirOptions)
        d_tmp.append(DirOptions)
        for i in itertools.product(*d_tmp):
            if len(set(i)) == 2:
                d.append(i)
    elif command == 'remove' or command == 'unlink':
        d = tuple(FileOptions + DirOptions)
    elif command == 'fdatasync' or command == 'fsetxattr' or command == 'removexattr':
        d = tuple(FileOptions)
    elif command == 'fsync':
        d = tuple(FileOptions + DirOptions)
    else:
        d=()
    return d

def isBugWorkload(opList, paramList, syncList):
    for i in xrange(0,len(expected_sequence)):
        if len(opList) != len(expected_sequence[i]):
            continue

        flag = 1

        for j in xrange(0, len(expected_sequence[i])):
            if opList[j] == expected_sequence[i][j][0] and paramList[j] == expected_sequence[i][j][1] and tuple(syncList[j]) == tuple(expected_sync_sequence[i][j]):
                continue
            else:
                flag = 0
                break

        if flag == 1:
            print 'Found match to Bug # ', i+1, ' : '
            print 'Length of seq : ',  len(expected_sequence[i])
            print 'Expected sequence = ' , expected_sequence[i]
            print 'Expected sync sequence = ', expected_sync_sequence[i]
            print 'Auto generator found : '
            print opList
            print paramList
            print syncList
            print '\n\n'
            return True

def doPermutation(perm):

    global global_count
    global parameterList
    global num_ops
    global syncPermutations
    global count
    global permutations
    global SyncSet
    global log_file_handle
    global count_param

    permutations.append(perm)
    log = ', '.join(perm);
    log = `count` + ' : ' + log + '\n'
    count +=1
    global_count +=1
    log_file_handle.write(log)

    #Now for each of this permutation, find all possible permutation of paramters
    combination = list()
    for length in xrange(0,len(permutations[count-1])):
        combination.append(parameterList[permutations[count-1][length]])
    count_param = 0
    for j in itertools.product(*combination):
        log = '{0}'.format(j);
        log = `count_param` + ' : ' + log + '\n'
        count_param += 1
        global_count +=1
        log_file_handle.write(log)

        #Let's insert fsync combinations here.
        count_sync = 0
        for insSync in range(0, len(syncPermutations)):
            global_count +=1
            count_sync+=1
            isBugWorkload(permutations[count-1], j, syncPermutations[insSync])

global_count = 0
parameterList = {}
SyncSet = list()
num_ops = 0
syncPermutations = []
count = 0
permutations = []
log_file_handle = 0
count_param = 0

def main():

    global global_count
    global parameterList
    global num_ops
    global syncPermutations
    global count
    global permutations
    global SyncSet
    global log_file_handle
    global count_param

    #open log file
    log_file = time.strftime('%Y%m%d_%H%M%S') + '-bugWorkloadGen.log'
    log_file_handle = open(log_file, 'w')

    #Parse input args
    parsed_args = build_parser().parse_args()

    #Print the test setup - just for sanity
    print_setup(parsed_args)

    num_ops = parsed_args.sequence_len

    for i in xrange(0,len(expected_sequence)):
        print 'Bug #', i+1
        print expected_sequence[i]
        print expected_sync_sequence[i]
        print '\n'

    for i in OperationSet:
        parameterList[i] = buildTuple(i)
        log = '{0}'.format(parameterList[i]);
        log = `i` + ' : Options = ' + `len(parameterList[i])` + '\n' + log + '\n\n'
        log_file_handle.write(log)

    d = buildTuple('fsync')
    fsync = ('fsync',)
    sync = ('sync')
    none = ('none')

    for i in xrange(0, len(d)):
        tup = list(fsync)
        tup.append(d[i])
        SyncSet.append(tup)

    SyncSet.append(sync)
    SyncSet.append(none)
    SyncSet = tuple(SyncSet)
#    print SyncSet

    for i in itertools.product(SyncSet, repeat=int(num_ops)):
        syncPermutations.append(i)
#        print i

#    print 'num fsync combinations = ', len(syncPermutations)

    start_time = time.time()

    pool = Pool(processes = 16)
    pool.map(doPermutation, itertools.product(OperationSet, repeat=int(num_ops)))
    pool.close()

    end_time = time.time()
#    print 'Total permutations of input op set = ', count
#
#    print 'Total parameter combinations = ', count_param
#
##    print 'Total sync combinations = ', count_sync
#
#    print 'Total workloads inspected = ' , global_count

    print 'Time taken to match workloads = ', end_time-start_time, 'seconds\n\n'

    log_file_handle.close()

if __name__ == '__main__':
	main()
<|endoftext|>