<commit_before>#!/usr/bin/python
from optparse import OptionParser
import os
import sqlite3 as sqlite
from misopy.sashimi_plot.plot_utils.plot_gene import readsToWiggle_pysam
import pysam
import json
from scipy.signal import resample

__author__ = 'Hendrik Strobelt'

vials_db_name = 'all_miso_summaries.sqlite'
vials_config_file_name = 'vials_project.json'
samples_meta_file_name = 'samples.json'

def create_index(project_dir, bam_root, sample_bam_file, out_file_name, out_wiggle_name):
    db_file_name = os.path.join(project_dir, vials_db_name)

    con = sqlite.connect(db_file_name)

    with con:

        cur = con.cursor()

        all_events = []
        for event in cur.execute("SELECT DISTINCT event_name, mRNA_starts, mRNA_ends, chrom_orig FROM miso_summaries LIMIT 0,5"):

            all_starts = map(int,event[1].split(','))
            all_ends = map(int, event[2].split(','))
            # print event[0], min(all_starts), max(all_ends)
            all_events.append({
                'event': event[0].encode('ascii'),
                'start': min(all_starts),
                'end': max(all_ends),
                'chrom': event[3].encode('ascii')
            })

    if sample_bam_file and sample_bam_file !='all':
        bam_file = os.path.join(bam_root,sample_bam_file)
        out_file = os.path.join(project_dir, out_file_name)
        wiggle_file_path = os.path.join(project_dir, out_wiggle_name)
        if os.path.isfile(bam_file):
            print ('opening file')
            bamdata = pysam.Samfile(bam_file, 'rb')

            with open(out_file,'wb') as save_file, open(wiggle_file_path,'wb') as wiggle_file:
                save_file.writelines('{\n')
                for index, event in enumerate(all_events):
                    print index, event
                    sample_reads = bamdata.fetch(event['chrom'], event['start'], event['end'])
                    wiggle, sample_jxns = readsToWiggle_pysam(sample_reads, event['start'], event['end'])

                    save_file.writelines(event['event']+':'+json.dumps(sample_jxns)+'\n')
                    wiggle_file.writelines(event['event']+':'+('_'.join(map(str,resample(wiggle, 2000).tolist())))+'\n')

                save_file.writelines('}')

def main():
    parser = OptionParser(usage='usage: %prog [options] <project_directory> <bam_root_dir>')
    parser.add_option("-s", default='all', dest='sample_bam_file', help="define sample file [%default]")
    parser.add_option("-o", default='out.json', dest='output', help="define output file [%default]")
    parser.add_option("-w", default='out.wiggle', dest='wiggle', help="define wiggle file [%default]")
    # parser.add_option("-f", default=False, action='store_true', dest='force', help='forces overwrite if vials project already exists [%default]')
    # parser.add_option("-g", default="hg19", dest='ref_genome', help='identifier for reference genome [%default]')
    # parser.add_option("-m", dest='matching_dir', help='Naive matching of .bam filenames to directory names.. see documentation')

    (options, args) = parser.parse_args()
    if len(args) != 2:
        parser.print_help()
        # parser.error("missing root directory as argument")
    else:
        create_index(args[0], args[1], options.sample_bam_file, options.output, options.wiggle)

if __name__ == '__main__':
    main()
<commit_msg>compatibility<commit_after>#!/usr/bin/python
from optparse import OptionParser
import os
import sqlite3 as sqlite
from misopy.sashimi_plot.plot_utils.plot_gene import readsToWiggle_pysam
import pysam
import json
from scipy.signal import resample

__author__ = 'Hendrik Strobelt'

vials_db_name = 'all_miso_summaries.sqlite'
vials_config_file_name = 'vials_project.json'
samples_meta_file_name = 'samples.json'

def create_index(project_dir, bam_root, sample_bam_file, out_file_name, out_wiggle_name):
    db_file_name = os.path.join(project_dir, vials_db_name)

    con = sqlite.connect(db_file_name)

    with con:

        cur = con.cursor()

        all_events = []
        for event in cur.execute("SELECT DISTINCT event_name, mRNA_starts, mRNA_ends, chrom_orig FROM miso_summaries LIMIT 0,5"):

            all_starts = map(int,event[1].split(','))
            all_ends = map(int, event[2].split(','))
            # print event[0], min(all_starts), max(all_ends)
            all_events.append({
                'event': event[0].encode('ascii'),
                'start': min(all_starts),
                'end': max(all_ends),
                'chrom': event[3].encode('ascii')
            })

    if sample_bam_file and sample_bam_file !='all':
        bam_file = os.path.join(bam_root,sample_bam_file)
        out_file = os.path.join(project_dir, out_file_name)
        wiggle_file_path = os.path.join(project_dir, out_wiggle_name)
        if os.path.isfile(bam_file):
            print ('opening file')
            bamdata = pysam.Samfile(bam_file, 'rb')

            with open(out_file, 'wb') as save_file:
                with open(wiggle_file_path,'wb') as wiggle_file:
                    save_file.writelines('{\n')
                    for index, event in enumerate(all_events):
                        print index, event
                        sample_reads = bamdata.fetch(event['chrom'], event['start'], event['end'])
                        wiggle, sample_jxns = readsToWiggle_pysam(sample_reads, event['start'], event['end'])

                        save_file.writelines(event['event']+':'+json.dumps(sample_jxns)+'\n')
                        wiggle_file.writelines(event['event']+':'+('_'.join(map(str,resample(wiggle, 2000).tolist())))+'\n')

                    save_file.writelines('}')

def main():
    parser = OptionParser(usage='usage: %prog [options] <project_directory> <bam_root_dir>')
    parser.add_option("-s", default='all', dest='sample_bam_file', help="define sample file [%default]")
    parser.add_option("-o", default='out.json', dest='output', help="define output file [%default]")
    parser.add_option("-w", default='out.wiggle', dest='wiggle', help="define wiggle file [%default]")
    # parser.add_option("-f", default=False, action='store_true', dest='force', help='forces overwrite if vials project already exists [%default]')
    # parser.add_option("-g", default="hg19", dest='ref_genome', help='identifier for reference genome [%default]')
    # parser.add_option("-m", dest='matching_dir', help='Naive matching of .bam filenames to directory names.. see documentation')

    (options, args) = parser.parse_args()
    if len(args) != 2:
        parser.print_help()
        # parser.error("missing root directory as argument")
    else:
        create_index(args[0], args[1], options.sample_bam_file, options.output, options.wiggle)

if __name__ == '__main__':
    main()
<|endoftext|>


<commit_before>'''
Created on Mar 29, 2016

Converts Julian date to Gregorian date and vv

@author: mike
'''

GREGORIAN_INCEPTION_AS_GREGORIAN_DATE = (1582, 10, 15)
GREGORIAN_INCEPTION_AS_JULIAN_DATE = (1582, 10, 5)

def get_julian_day(year, month, day, from_julian_date=False):
    assert 1 <= month <= 12
    assert 1 <= day <= 31

    # before Gregorian reform calendars are assumed to be identical
    if not from_julian_date and (year, month, day) < GREGORIAN_INCEPTION_AS_GREGORIAN_DATE:
        if (year, month, day) > GREGORIAN_INCEPTION_AS_JULIAN_DATE:
            year, month, day = GREGORIAN_INCEPTION_AS_JULIAN_DATE
        from_julian_date = True

    a = (14 - month) // 12
    y = year + 4800 - a
    m = month + 12 * a - 3

    if from_julian_date:
        jdn = day + (153 * m + 2) // 5 + 365 * y + y // 4 - 32083
    else:
        jdn = day + (153 * m + 2) // 5 + 365 * y + y // 4 - y // 100 + y / 400 - 32045

    return jdn

def get_calendar(jdn, julian=False):

    if julian:
        f = jdn + 1401
    else:
        f = jdn + 1401 + (((4 * jdn + 274277) // 146097) * 3) // 4 - 38

    e = f * 4 + 3
    g = (e % 1461) // 4
    h = 5 * g + 2

    day = (h % 153) / 5 + 1
    month = ( h // 153 + 2 ) % 12 + 1
    year = e // 1461 - 4716 + (14 - month) // 12

    return year, month, day

def julian_to_gregorian(year, month, day):
    if (year, month, day) < GREGORIAN_INCEPTION_AS_JULIAN_DATE:
        return year, month, day
    jdn = get_julian_day(year, month, day, from_julian_date=True)
    return get_calendar(jdn, julian=False)

def gregorian_to_julian(year, month, day):
    if (year, month, day) < GREGORIAN_INCEPTION_AS_GREGORIAN_DATE:
        if (year, month, day) > GREGORIAN_INCEPTION_AS_JULIAN_DATE:
            return GREGORIAN_INCEPTION_AS_JULIAN_DATE
        return year, month, day
    jdn = get_julian_day(year, month, day, from_julian_date=False)
    return get_calendar(jdn, julian=True)

def day_of_the_week(year, month, day, from_julian_date=False):
    jdn = get_julian_day(year, month, day, from_julian_date=from_julian_date)
    return (jdn + 1) % 7  # 0-Sun, 1-Mon, 2-Tue, 3-Wed, 4-Thu, 5-Fri, 6-Sat

def day_of_the_week_str(year, month, day, from_julian_date=False):
    return ['Sunday',
            'Monday',
            'Tuesday',
            'Wednesday',
            'Thursday',
            'Friday',
            'Saturday'][day_of_the_week(year, month, day, from_julian_date=from_julian_date)]

if __name__ == '__main__':

    jdn = get_julian_day(2016, 3, 30)
    print(jdn)
    print(get_calendar(jdn))
    print(get_calendar(jdn, julian=True))

    print(gregorian_to_julian(2016, 3, 30))
    print(julian_to_gregorian(2016, 3, 17))

    print(julian_to_gregorian(1582, 10, 5))
    print(julian_to_gregorian(1000, 10, 5))
    print(gregorian_to_julian(1000, 10, 5))

    print(gregorian_to_julian(1582, 10, 15))
    print(gregorian_to_julian(1582, 10, 10))
    print(gregorian_to_julian(1582, 10, 6))
    print(gregorian_to_julian(1582, 10, 5))
    print(day_of_the_week_str(1582, 10, 4))
    print(day_of_the_week_str(2016, 3, 30))
    print(day_of_the_week_str(1, 1, 1, from_julian_date=True))
<commit_msg>fixed dicisions - must use integer ones<commit_after>'''
Created on Mar 29, 2016

Converts Julian date to Gregorian date and vv

Code based on algorithms from: https://en.wikipedia.org/wiki/Julian_day
These algorithms were adjusted to assume that before the inception of Gregorian calendar there was
a single calendar (i.e. Gregorian calendar before Oct 5 1582 was identical to Julian.

@author: mike
'''

GREGORIAN_INCEPTION_AS_GREGORIAN_DATE = (1582, 10, 15)
GREGORIAN_INCEPTION_AS_JULIAN_DATE = (1582, 10, 5)

def get_julian_day(year, month, day, from_julian_date=False):
    assert 1 <= month <= 12
    assert 1 <= day <= 31

    # before Gregorian reform calendars are assumed to be identical
    if not from_julian_date and (year, month, day) < GREGORIAN_INCEPTION_AS_GREGORIAN_DATE:
        if (year, month, day) > GREGORIAN_INCEPTION_AS_JULIAN_DATE:
            year, month, day = GREGORIAN_INCEPTION_AS_JULIAN_DATE
        from_julian_date = True

    a = (14 - month) // 12
    y = year + 4800 - a
    m = month + 12 * a - 3

    if from_julian_date:
        jdn = day + (153 * m + 2) // 5 + 365 * y + y // 4 - 32083
    else:
        jdn = day + (153 * m + 2) // 5 + 365 * y + y // 4 - y // 100 + y // 400 - 32045

    return jdn

def get_calendar(jdn, julian=False):

    if julian:
        f = jdn + 1401
    else:
        f = jdn + 1401 + (((4 * jdn + 274277) // 146097) * 3) // 4 - 38

    e = f * 4 + 3
    g = (e % 1461) // 4
    h = 5 * g + 2

    day = (h % 153) // 5 + 1
    month = ( h // 153 + 2 ) % 12 + 1
    year = e // 1461 - 4716 + (14 - month) // 12

    return year, month, day

def julian_to_gregorian(year, month, day):
    if (year, month, day) < GREGORIAN_INCEPTION_AS_JULIAN_DATE:
        return year, month, day
    jdn = get_julian_day(year, month, day, from_julian_date=True)
    return get_calendar(jdn, julian=False)

def gregorian_to_julian(year, month, day):
    if (year, month, day) < GREGORIAN_INCEPTION_AS_GREGORIAN_DATE:
        if (year, month, day) > GREGORIAN_INCEPTION_AS_JULIAN_DATE:
            return GREGORIAN_INCEPTION_AS_JULIAN_DATE
        return year, month, day
    jdn = get_julian_day(year, month, day, from_julian_date=False)
    return get_calendar(jdn, julian=True)

def day_of_the_week(year, month, day, from_julian_date=False):
    jdn = get_julian_day(year, month, day, from_julian_date=from_julian_date)
    return (jdn + 1) % 7  # 0-Sun, 1-Mon, 2-Tue, 3-Wed, 4-Thu, 5-Fri, 6-Sat

def day_of_the_week_str(year, month, day, from_julian_date=False):
    return ['Sunday',
            'Monday',
            'Tuesday',
            'Wednesday',
            'Thursday',
            'Friday',
            'Saturday'][day_of_the_week(year, month, day, from_julian_date=from_julian_date)]

if __name__ == '__main__':

    jdn = get_julian_day(2016, 3, 30)
    print(jdn)
    print(get_calendar(jdn))
    print(get_calendar(jdn, julian=True))

    print(gregorian_to_julian(2016, 3, 30))
    print(julian_to_gregorian(2016, 3, 17))

    print(julian_to_gregorian(1582, 10, 5))
    print(julian_to_gregorian(1000, 10, 5))
    print(gregorian_to_julian(1000, 10, 5))

    print(gregorian_to_julian(1582, 10, 15))
    print(gregorian_to_julian(1582, 10, 10))
    print(gregorian_to_julian(1582, 10, 6))
    print(gregorian_to_julian(1582, 10, 5))
    print(day_of_the_week_str(1582, 10, 4))
    print(day_of_the_week_str(2016, 3, 30))
    print(day_of_the_week_str(1, 1, 1, from_julian_date=True))
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-

'''
  project file generator
  ~~~~~~~~~~~~~~~~~~~~~~

  :copyright: (c) 2016 by Jihoon Kang <kang@ghoon.net>
  :license: Apache 2, see LICENSE for more details
'''

from xml.etree import ElementTree
from xml.dom import minidom
from jinja2 import Environment, FileSystemLoader
import re

def generate_project(configs, output):
    env = Environment(loader=FileSystemLoader('./templates'),
            trim_blocks=True)
    tmpl = env.get_template('project')
    root = ElementTree.fromstring(tmpl.render(configs))
    linkedResources = root.find("linkedResources")

    for entry in configs['links']:
        node = ElementTree.SubElement(linkedResources, 'link')

        name = ElementTree.SubElement(node, 'name')
        name.text = entry['name']

        typeNode = ElementTree.SubElement(node, 'type')
        typeNode.text = '2'

        location = ElementTree.SubElement(node, 'location')
        location.text = "%s/%s" % (configs['rootdir'], entry['location'])

    with open(output, 'w') as f:
        xmlstr = ElementTree.tostring(root)
        # remove all whitespaces before beautifying
        xmlstr = re.sub(r'>\s+', '>', xmlstr)
        xmlstr = minidom.parseString(xmlstr).toprettyxml(indent="\t")
        print xmlstr
        f.write(xmlstr.encode('utf-8'))
<commit_msg>Fix possible error in prgen.py<commit_after># -*- coding: utf-8 -*-

'''
  project file generator
  ~~~~~~~~~~~~~~~~~~~~~~

  :copyright: (c) 2016 by Jihoon Kang <kang@ghoon.net>
  :license: Apache 2, see LICENSE for more details
'''

from xml.etree import ElementTree
from xml.dom import minidom
from jinja2 import Environment, FileSystemLoader
import re

def generate_project(configs, output, tmpl_dir='templates'):
    env = Environment(loader=FileSystemLoader(tmpl_dir),
            trim_blocks=True)
    tmpl = env.get_template('project')
    root = ElementTree.fromstring(tmpl.render(configs))
    linkedResources = root.find("linkedResources")
    links = configs['links'] or []

    for entry in links:
        node = ElementTree.SubElement(linkedResources, 'link')

        name = ElementTree.SubElement(node, 'name')
        name.text = entry['name']

        typeNode = ElementTree.SubElement(node, 'type')
        typeNode.text = '2'

        location = ElementTree.SubElement(node, 'location')
        location.text = "%s/%s" % (configs['rootdir'], entry['location'])

    with open(output, 'w') as f:
        xmlstr = ElementTree.tostring(root)
        # remove all whitespaces before beautifying
        xmlstr = re.sub(r'>\s+', '>', xmlstr)
        xmlstr = minidom.parseString(xmlstr).toprettyxml(indent="\t")
        print xmlstr
        f.write(xmlstr.encode('utf-8'))
<|endoftext|>


<commit_before>#!/usr/bin/env python

__all__ = ['sohu_download']

from ..common import *

import json

def real_url(host, prot, file, new):
    url = 'http://%s/?prot=%s&file=%s&new=%s' % (host, prot, file, new)
    start, _, host, key = get_html(url).split('|')[:4]
    return '%s%s?key=%s' % (start[:-1], new, key)

def sohu_download(url, output_dir = '.', merge = True, info_only = False):
    if re.match(r'http://share.vrs.sohu.com', url):
        vid = r1('id=(\d+)', url)
    else:
        html = get_html(url)
        vid = r1(r'\Wvid\s*[\:=]\s*[\'"]?(\d+)[\'"]?', html)
    assert vid

    if re.match(r'http://tv.sohu.com/', url):
        data = json.loads(get_decoded_html('http://hot.vrs.sohu.com/vrs_flash.action?vid=%s' % vid))
        for qtyp in ["oriVid","superVid","highVid" ,"norVid","relativeId"]:
            hqvid = data['data'][qtyp]
            if hqvid != 0 and hqvid != vid :
                data = json.loads(get_decoded_html('http://hot.vrs.sohu.com/vrs_flash.action?vid=%s' % hqvid))
                break
        host = data['allot']
        prot = data['prot']
        urls = []
        data = data['data']
        title = data['tvName']
        size = sum(data['clipsBytes'])
        assert len(data['clipsURL']) == len(data['clipsBytes']) == len(data['su'])
        for file, new in zip(data['clipsURL'], data['su']):
            urls.append(real_url(host, prot, file, new))
        assert data['clipsURL'][0].endswith('.mp4')

    else:
        data = json.loads(get_decoded_html('http://my.tv.sohu.com/play/videonew.do?vid=%s&referer=http://my.tv.sohu.com' % vid))
        host = data['allot']
        prot = data['prot']
        urls = []
        data = data['data']
        title = data['tvName']
        size = sum([int(clipsBytes) for clipsBytes in data['clipsBytes']])
        assert len(data['clipsURL']) == len(data['clipsBytes']) == len(data['su'])
        for file, new in zip(data['clipsURL'], data['su']):
            urls.append(real_url(host, prot, file, new))
        assert data['clipsURL'][0].endswith('.mp4')

    print_info(site_info, title, 'mp4', size)
    if not info_only:
        download_urls(urls, title, 'mp4', size, output_dir, refer = url, merge = merge)

site_info = "Sohu.com"
download = sohu_download
download_playlist = playlist_not_supported('sohu')
<commit_msg>fix bug #353<commit_after>#!/usr/bin/env python

__all__ = ['sohu_download']

from ..common import *

import json

def real_url(host, prot, file, new):
    url = 'http://%s/?prot=%s&file=%s&new=%s' % (host, prot, file, new)
    start, _, host, key = get_html(url).split('|')[:4]
    return '%s%s?key=%s' % (start[:-1], new, key)

def sohu_download(url, output_dir = '.', merge = True, info_only = False, extractor_proxy=None):
    if re.match(r'http://share.vrs.sohu.com', url):
        vid = r1('id=(\d+)', url)
    else:
        html = get_html(url)
        vid = r1(r'\Wvid\s*[\:=]\s*[\'"]?(\d+)[\'"]?', html)
    assert vid

    if re.match(r'http://tv.sohu.com/', url):
        if extractor_proxy:
            set_proxy(tuple(extractor_proxy.split(":")))
        data = json.loads(get_decoded_html('http://hot.vrs.sohu.com/vrs_flash.action?vid=%s' % vid))
        for qtyp in ["oriVid","superVid","highVid" ,"norVid","relativeId"]:
            hqvid = data['data'][qtyp]
            if hqvid != 0 and hqvid != vid :
                data = json.loads(get_decoded_html('http://hot.vrs.sohu.com/vrs_flash.action?vid=%s' % hqvid))
                break
        if extractor_proxy:
            unset_proxy()
        host = data['allot']
        prot = data['prot']
        urls = []
        data = data['data']
        title = data['tvName']
        size = sum(data['clipsBytes'])
        assert len(data['clipsURL']) == len(data['clipsBytes']) == len(data['su'])
        for file, new in zip(data['clipsURL'], data['su']):
            urls.append(real_url(host, prot, file, new))
        assert data['clipsURL'][0].endswith('.mp4')

    else:
        data = json.loads(get_decoded_html('http://my.tv.sohu.com/play/videonew.do?vid=%s&referer=http://my.tv.sohu.com' % vid))
        host = data['allot']
        prot = data['prot']
        urls = []
        data = data['data']
        title = data['tvName']
        size = sum([int(clipsBytes) for clipsBytes in data['clipsBytes']])
        assert len(data['clipsURL']) == len(data['clipsBytes']) == len(data['su'])
        for file, new in zip(data['clipsURL'], data['su']):
            urls.append(real_url(host, prot, file, new))
        assert data['clipsURL'][0].endswith('.mp4')

    print_info(site_info, title, 'mp4', size)
    if not info_only:
        download_urls(urls, title, 'mp4', size, output_dir, refer = url, merge = merge)

site_info = "Sohu.com"
download = sohu_download
download_playlist = playlist_not_supported('sohu')
<|endoftext|>


<commit_before>import gc
import numpy as np
import xarray as xr
import scipy.ndimage.filters as conv

from . import dc_utilities as utilities
from datetime import datetime

def _tsmi(dataset):
    return (dataset.red.astype('float64') + dataset.green.astype('float64')) * 0.0001 / 2

def tsm(dataset_in, clean_mask=None, no_data=0):
    assert clean_mask is not None, "Please supply a boolean clean mask with the same dimensions as dataset_in"
    assert 'red' in dataset_in and 'green' in dataset_in, "Red and Green bands are required for the TSM analysis."

    tsm = 3983 * _tsmi(dataset_in)**1.6246
    tsm.values[np.invert(clean_mask)] = no_data  # Contains data for clear pixels

    # Create xarray of data
    time = dataset_in.time
    latitude = dataset_in.latitude
    longitude = dataset_in.longitude
    dataset_out = xr.Dataset({'tsm': tsm}, coords={'time': time, 'latitude': latitude, 'longitude': longitude})
    return dataset_out

def mask_water_quality(dataset_in, wofs):
    wofs_criteria = wofs.where(wofs > 0.8)
    wofs_criteria.values[wofs_criteria.values > 0] = 0
    kernel = np.array([[1, 1, 1], [1, 1, 1], [1, 1, 1]])

    mask = conv.convolve(wofs_criteria.values, kernel, mode='constant')
    mask = mask.astype(np.float32)

    dataset_out = dataset_in.copy(deep=True)
    dataset_out.normalized_data.values += mask
    dataset_out.total_clean.values += mask
    utilities.nan_to_num(dataset_out, 0)

    return dataset_out

def watanabe_chlorophyll(dataset_in, clean_mask=None, no_data=0):
    assert clean_mask is not None, "Please supply a boolean clean mask with the same dimensions as dataset_in"
    assert 'red' in dataset_in and 'nir' in dataset_in, "Red and NIR bands are required for the Watanabe Chlorophyll analysis."

    chl_a = 925.001 * (dataset_in.nir.astype('float64') / dataset_in.red.astype('float64')) - 77.16
    chl_a.values[np.invert(clean_mask)] = no_data  # Contains data for clear pixels

    # Create xarray of data
    time = dataset_in.time
    latitude = dataset_in.latitude
    longitude = dataset_in.longitude
    dataset_out = xr.Dataset(
        {
            'watanabe_chlorophyll': chl_a
        }, coords={'time': time,
                   'latitude': latitude,
                   'longitude': longitude})
    return dataset_out

def nazeer_chlorophyll(dataset_in, clean_mask=None, no_data=0):
    assert clean_mask is not None, "Please supply a boolean clean mask with the same dimensions as dataset_in"

    chl_a = (0.57 * (dataset_in.red.astype('float64') * 0.0001) /
             (dataset_in.blue.astype('float64') * 0.0001)**2) - 2.61
    chl_a.values[np.invert(clean_mask)] = no_data  # Contains data for clear pixels

    # Create xarray of data
    time = dataset_in.time
    latitude = dataset_in.latitude
    longitude = dataset_in.longitude
    dataset_out = xr.Dataset(
        {
            'nazeer_chlorophyll': chl_a
        }, coords={'time': time,
                   'latitude': latitude,
                   'longitude': longitude})
    return dataset_out
<commit_msg>Fixed issue with generality in  function.<commit_after>import gc
import numpy as np
import xarray as xr
import scipy.ndimage.filters as conv

from . import dc_utilities as utilities
from datetime import datetime

def _tsmi(dataset):
    return (dataset.red.astype('float64') + dataset.green.astype('float64')) * 0.0001 / 2

def tsm(dataset_in, clean_mask=None, no_data=0):
    assert clean_mask is not None, "Please supply a boolean clean mask with the same dimensions as dataset_in"
    assert 'red' in dataset_in and 'green' in dataset_in, "Red and Green bands are required for the TSM analysis."

    tsm = 3983 * _tsmi(dataset_in)**1.6246
    tsm.values[np.invert(clean_mask)] = no_data  # Contains data for clear pixels

    # Create xarray of data
    _coords = { key:dataset_in[key] for key in dataset_in.dims.keys()}
    dataset_out = xr.Dataset({'tsm': tsm}, coords=_coords)
    return dataset_out

def mask_water_quality(dataset_in, wofs):
    wofs_criteria = wofs.where(wofs > 0.8)
    wofs_criteria.values[wofs_criteria.values > 0] = 0
    kernel = np.array([[1, 1, 1], [1, 1, 1], [1, 1, 1]])

    mask = conv.convolve(wofs_criteria.values, kernel, mode='constant')
    mask = mask.astype(np.float32)

    dataset_out = dataset_in.copy(deep=True)
    dataset_out.normalized_data.values += mask
    dataset_out.total_clean.values += mask
    utilities.nan_to_num(dataset_out, 0)

    return dataset_out

def watanabe_chlorophyll(dataset_in, clean_mask=None, no_data=0):
    assert clean_mask is not None, "Please supply a boolean clean mask with the same dimensions as dataset_in"
    assert 'red' in dataset_in and 'nir' in dataset_in, "Red and NIR bands are required for the Watanabe Chlorophyll analysis."

    chl_a = 925.001 * (dataset_in.nir.astype('float64') / dataset_in.red.astype('float64')) - 77.16
    chl_a.values[np.invert(clean_mask)] = no_data  # Contains data for clear pixels

    # Create xarray of data
    time = dataset_in.time
    latitude = dataset_in.latitude
    longitude = dataset_in.longitude
    dataset_out = xr.Dataset(
        {
            'watanabe_chlorophyll': chl_a
        }, coords={'time': time,
                   'latitude': latitude,
                   'longitude': longitude})
    return dataset_out

def nazeer_chlorophyll(dataset_in, clean_mask=None, no_data=0):
    assert clean_mask is not None, "Please supply a boolean clean mask with the same dimensions as dataset_in"

    chl_a = (0.57 * (dataset_in.red.astype('float64') * 0.0001) /
             (dataset_in.blue.astype('float64') * 0.0001)**2) - 2.61
    chl_a.values[np.invert(clean_mask)] = no_data  # Contains data for clear pixels

    # Create xarray of data
    time = dataset_in.time
    latitude = dataset_in.latitude
    longitude = dataset_in.longitude
    dataset_out = xr.Dataset(
        {
            'nazeer_chlorophyll': chl_a
        }, coords={'time': time,
                   'latitude': latitude,
                   'longitude': longitude})
    return dataset_out
<|endoftext|>


<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-
from ooop import OOOP
import configdb

O = OOOP(**configdb.ooop)

imp_obj = O.GiscedataFacturacioImportacioLinia

imp_del_ids = imp_obj.search([('state','=','erroni'),('info','like','Ja existeix una factura')])
imp_del_ids += imp_obj.search([('state','=','erroni'),('info','like','XML erroni')])
imp_del_ids += imp_obj.search([('state','=','erroni'),('info','like',"XML no es correspon al tipus F1")])
imp_del_ids += imp_obj.search([('state','=','erroni'),('info','like',"Document invàlid")])

total = len(imp_del_ids)
n = 0

for imp_del_id in imp_del_ids:
    try:
        imp_obj.unlink([imp_del_id])
        n +=1
        print "%d/%d" % (n,total)
    except Exception, e:
        print e
<commit_msg>Refactor to new F1 erro's message<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-
from ooop import OOOP
import configdb

O = OOOP(**configdb.ooop)

imp_obj = O.GiscedataFacturacioImportacioLinia

imp_del_ids = imp_obj.search([('state','=','erroni'),('info','like',"Aquest fitxer XML ja s'ha processat en els següents IDs")])
#imp_del_ids += imp_obj.search([('state','=','erroni'),('info','like','XML erroni')])
imp_del_ids += imp_obj.search([('state','=','erroni'),('info','like',"XML no es correspon al tipus F1")])
imp_del_ids += imp_obj.search([('state','=','erroni'),('info','like',"Document invàlid")])

total = len(imp_del_ids)
n = 0

for imp_del_id in imp_del_ids:
    try:
        imp_obj.unlink([imp_del_id])
        n +=1
        print "%d/%d" % (n,total)
    except Exception, e:
        print e
<|endoftext|>


<commit_before>#
# Copyright (c) 2008-2009 rPath, Inc.
#
# This program is distributed under the terms of the Common Public License,
# version 1.0. A copy of this license should have been distributed with this
# source file in a file called LICENSE. If it is not present, the license
# is always available at http://www.rpath.com/permanent/licenses/CPL-1.0.
#
# This program is distributed in the hope that it will be useful, but
# without any warranty; without even the implied warranty of merchantability
# or fitness for a particular purpose. See the Common Public License for
# full details.
#
"""
Checkout command and related utilities.
"""
import os
import tempfile

from conary.lib import util

from rbuild import errors
from rbuild import pluginapi
from rbuild.pluginapi import command

from rbuild_plugins.checkout import derive

#TODO: separate out determining what checkout to get from actually creating
# that checkout to allow for other interfaces

class CheckoutCommand(command.BaseCommand):
    """
    Check out a package, creating a new package if necessary.

    If the product's platform contains a package by the same name,
    then you must specify whether to shadow or derive from that
    upstream version, or to create a new package by the same name
    that is not related to the upstream version.

    A derived package starts with the contents of the binary
    package from the platform, generally used to make modifications
    that do not require rebuilding binaries.

    A shadow allows you to make changes relative to the platform's
    source package, changing how the package is built, and requiring
    rebuilding binaries.
    """
    help = 'Check out packages and groups for editing'
    paramHelp = '[<options>] <packagename>'

    commands = ['checkout']
    docs = {'derive' : "Create derived package (based on upstream binary)",
            'shadow' : "Create shadowed package (based on upstream source)",
            'new' : ("Create a new version of the package even if"
                     " an upstream version exists") }

    def addLocalParameters(self, argDef):
        argDef['derive'] = command.NO_PARAM
        argDef['shadow'] = command.NO_PARAM
        argDef['new']    = command.NO_PARAM

    def runCommand(self, handle, argSet, args):
        packageName, = self.requireParameters(args, ['packageName'])[1:]
        derive = argSet.pop('derive', False)
        new = argSet.pop('new', False)
        shadow = argSet.pop('shadow', False)
        self.runCheckoutCommand(handle, packageName, new=new, shadow=shadow,
                            derive=derive)

    def runCheckoutCommand(self, handle, packageName, new=False, shadow=False,
                       derive=False):
        if [new, shadow, derive].count(True) > 1:
            raise errors.ParseError(
                'Only one of --new, --derive, or --shadow may be specified')
        if new:
            return handle.Checkout.newPackage(packageName)
        elif shadow:
            return handle.Checkout.shadowPackage(packageName)
        elif derive:
            return handle.Checkout.derivePackage(packageName)
        else:
            return handle.Checkout.checkoutPackageDefault(packageName)

class Checkout(pluginapi.Plugin):
    name = 'checkout'

    def registerCommands(self):
        self.handle.Commands.registerCommand(CheckoutCommand)

    def checkoutPackageDefault(self, packageName):
        existingPackage = self._getExistingPackage(packageName)
        if existingPackage:
            rc = self.checkoutPackage(packageName)
            self.handle.ui.info('Checked out existing package %r', packageName)
            return rc
        upstreamLatest = self._getUpstreamPackage(packageName)
        if upstreamLatest:
            raise errors.PluginError('\n'.join((
                    'The upstream source provides a version of this package.',
                    'Please specify:',
                    '  --shadow to shadow this package',
                    '  --derive to derive from it',
                    '  --new to replace it with a new version')))
        self.newPackage(packageName)

    def checkoutPackage(self, packageName):
        currentLabel = self.handle.productStore.getActiveStageLabel()
        self.handle.facade.conary.checkout(packageName, currentLabel)
        return True

    def derivePackage(self, packageName):
        upstreamLatest = self._getUpstreamPackage(packageName)
        if not upstreamLatest:
            raise errors.PluginError(
                        'cannot derive %s: no upstream binary' % packageName)
        derive.derive(self.handle, upstreamLatest)
        self.handle.ui.info('Derived %s from %s=%s[%s].', packageName, *upstreamLatest)
        self.handle.ui.info(
                'Edit recipe to add changes your changes to the binary package')

    def shadowPackage(self, packageName):
        origName, version, flavor = self.handle.facade.conary.parseTroveSpec(packageName)
        package = None

        if version:
            package = self._getRemotePackage(origName, version)
            if package is None:
                raise errors.PluginError(
                        '%s:source does not exist on label %s.' % \
                        (origName, version))
        else:
            package = self._getUpstreamPackage(packageName)
            if package is None:
                raise errors.PluginError(
                        'cannot shadow %s: no upstream binary' % packageName)

        name, version, flavor = package

        currentLabel = self.handle.productStore.getActiveStageLabel()
        self.handle.facade.conary.shadowSourceForBinary(name, version, flavor,
                                                        currentLabel)
        self.checkoutPackage(origName)
        self.handle.ui.info('Shadowed package %r', packageName)

    def newPackage(self, packageName):
        existingPackage = self._getExistingPackage(packageName)
        if existingPackage:
            raise errors.PluginError('\n'.join((
                'This package already exists in the product.',
                'Use "rbuild checkout %s" to checkout the existing package, '
                'or give the new package a different name.' % \
                packageName)))

        upstreamLatest = self._getUpstreamPackage(packageName)
        if upstreamLatest:
            self.handle.ui.warning('Replacing upstream package %s.' % \
                packageName)

        currentLabel = self.handle.productStore.getActiveStageLabel()
        self.handle.facade.conary.createNewPackage(
                                            packageName, currentLabel)
        self.handle.ui.info('Created new package %r', packageName)
        return

    def _getUpstreamPackage(self, packageName):
        product = self.handle.product
        upstreamSources = product.getSearchPaths()
        upstreamSources = [(x.troveName, x.label, None)
                            for x in upstreamSources]
        troveList =  self.handle.facade.conary._findPackageInGroups(
                         upstreamSources,
                         packageName)
        if troveList:
            # FIXME: this could be multiple packages.  We need a good
            # way to select between them
            return troveList[0]
        return None

    def _getExistingPackage(self, packageName):
        currentLabel = self.handle.productStore.getActiveStageLabel()
        return self.handle.facade.conary._findTrove(packageName + ':source',
                                                    currentLabel,
                                                    allowMissing=True)

    def _getRemotePackage(self, packageName, label):
        troveList =  self.handle.facade.conary._findTrove(packageName + ':source',
                                                    label,
                                                    allowMissing=True)
        return troveList
<commit_msg>if troveList is a troveTup, call it a troveTup<commit_after>#
# Copyright (c) 2008-2009 rPath, Inc.
#
# This program is distributed under the terms of the Common Public License,
# version 1.0. A copy of this license should have been distributed with this
# source file in a file called LICENSE. If it is not present, the license
# is always available at http://www.rpath.com/permanent/licenses/CPL-1.0.
#
# This program is distributed in the hope that it will be useful, but
# without any warranty; without even the implied warranty of merchantability
# or fitness for a particular purpose. See the Common Public License for
# full details.
#
"""
Checkout command and related utilities.
"""
import os
import tempfile

from conary.lib import util

from rbuild import errors
from rbuild import pluginapi
from rbuild.pluginapi import command

from rbuild_plugins.checkout import derive

#TODO: separate out determining what checkout to get from actually creating
# that checkout to allow for other interfaces

class CheckoutCommand(command.BaseCommand):
    """
    Check out a package, creating a new package if necessary.

    If the product's platform contains a package by the same name,
    then you must specify whether to shadow or derive from that
    upstream version, or to create a new package by the same name
    that is not related to the upstream version.

    A derived package starts with the contents of the binary
    package from the platform, generally used to make modifications
    that do not require rebuilding binaries.

    A shadow allows you to make changes relative to the platform's
    source package, changing how the package is built, and requiring
    rebuilding binaries.
    """
    help = 'Check out packages and groups for editing'
    paramHelp = '[<options>] <packagename>'

    commands = ['checkout']
    docs = {'derive' : "Create derived package (based on upstream binary)",
            'shadow' : "Create shadowed package (based on upstream source)",
            'new' : ("Create a new version of the package even if"
                     " an upstream version exists") }

    def addLocalParameters(self, argDef):
        argDef['derive'] = command.NO_PARAM
        argDef['shadow'] = command.NO_PARAM
        argDef['new']    = command.NO_PARAM

    def runCommand(self, handle, argSet, args):
        packageName, = self.requireParameters(args, ['packageName'])[1:]
        derive = argSet.pop('derive', False)
        new = argSet.pop('new', False)
        shadow = argSet.pop('shadow', False)
        self.runCheckoutCommand(handle, packageName, new=new, shadow=shadow,
                            derive=derive)

    def runCheckoutCommand(self, handle, packageName, new=False, shadow=False,
                       derive=False):
        if [new, shadow, derive].count(True) > 1:
            raise errors.ParseError(
                'Only one of --new, --derive, or --shadow may be specified')
        if new:
            return handle.Checkout.newPackage(packageName)
        elif shadow:
            return handle.Checkout.shadowPackage(packageName)
        elif derive:
            return handle.Checkout.derivePackage(packageName)
        else:
            return handle.Checkout.checkoutPackageDefault(packageName)

class Checkout(pluginapi.Plugin):
    name = 'checkout'

    def registerCommands(self):
        self.handle.Commands.registerCommand(CheckoutCommand)

    def checkoutPackageDefault(self, packageName):
        existingPackage = self._getExistingPackage(packageName)
        if existingPackage:
            rc = self.checkoutPackage(packageName)
            self.handle.ui.info('Checked out existing package %r', packageName)
            return rc
        upstreamLatest = self._getUpstreamPackage(packageName)
        if upstreamLatest:
            raise errors.PluginError('\n'.join((
                    'The upstream source provides a version of this package.',
                    'Please specify:',
                    '  --shadow to shadow this package',
                    '  --derive to derive from it',
                    '  --new to replace it with a new version')))
        self.newPackage(packageName)

    def checkoutPackage(self, packageName):
        currentLabel = self.handle.productStore.getActiveStageLabel()
        self.handle.facade.conary.checkout(packageName, currentLabel)
        return True

    def derivePackage(self, packageName):
        upstreamLatest = self._getUpstreamPackage(packageName)
        if not upstreamLatest:
            raise errors.PluginError(
                        'cannot derive %s: no upstream binary' % packageName)
        derive.derive(self.handle, upstreamLatest)
        self.handle.ui.info('Derived %s from %s=%s[%s].', packageName, *upstreamLatest)
        self.handle.ui.info(
                'Edit recipe to add changes your changes to the binary package')

    def shadowPackage(self, packageName):
        origName, version, flavor = self.handle.facade.conary.parseTroveSpec(packageName)
        package = None

        if version:
            package = self._getRemotePackage(origName, version)
            if package is None:
                raise errors.PluginError(
                        '%s:source does not exist on label %s.' % \
                        (origName, version))
        else:
            package = self._getUpstreamPackage(packageName)
            if package is None:
                raise errors.PluginError(
                        'cannot shadow %s: no upstream binary' % packageName)

        name, version, flavor = package

        currentLabel = self.handle.productStore.getActiveStageLabel()
        self.handle.facade.conary.shadowSourceForBinary(name, version, flavor,
                                                        currentLabel)
        self.checkoutPackage(origName)
        self.handle.ui.info('Shadowed package %r', packageName)

    def newPackage(self, packageName):
        existingPackage = self._getExistingPackage(packageName)
        if existingPackage:
            raise errors.PluginError('\n'.join((
                'This package already exists in the product.',
                'Use "rbuild checkout %s" to checkout the existing package, '
                'or give the new package a different name.' % \
                packageName)))

        upstreamLatest = self._getUpstreamPackage(packageName)
        if upstreamLatest:
            self.handle.ui.warning('Replacing upstream package %s.' % \
                packageName)

        currentLabel = self.handle.productStore.getActiveStageLabel()
        self.handle.facade.conary.createNewPackage(
                                            packageName, currentLabel)
        self.handle.ui.info('Created new package %r', packageName)
        return

    def _getUpstreamPackage(self, packageName):
        product = self.handle.product
        upstreamSources = product.getSearchPaths()
        upstreamSources = [(x.troveName, x.label, None)
                            for x in upstreamSources]
        troveList =  self.handle.facade.conary._findPackageInGroups(
                         upstreamSources,
                         packageName)
        if troveList:
            # FIXME: this could be multiple packages.  We need a good
            # way to select between them
            return troveList[0]
        return None

    def _getExistingPackage(self, packageName):
        currentLabel = self.handle.productStore.getActiveStageLabel()
        return self.handle.facade.conary._findTrove(packageName + ':source',
                                                    currentLabel,
                                                    allowMissing=True)

    def _getRemotePackage(self, packageName, label):
        troveTup = self.handle.facade.conary._findTrove(packageName + ':source',
                                                        label,
                                                        allowMissing=True)
        return troveTup
<|endoftext|>


<commit_before>try:
    # use gevent if available
    import gevent.socket as socket
    import gevent.ssl as ssl
except ImportError:
    import socket
    import ssl

import struct
from collections import deque
import logging
from .exceptions import EppLoginError, EppConnectionError
from .doc import (EppResponse, EppHello, EppLoginCommand, EppLogoutCommand,
                  EppCreateCommand, EppUpdateCommand, EppRenewCommand, EppTransferCommand, EppDeleteCommand)
from .utils import gen_trid
from backports.ssl_match_hostname import match_hostname, CertificateError

class EppClient(object):
    def __init__(self, host=None, port=700,
                 ssl_enable=True, ssl_keyfile=None, ssl_certfile=None, ssl_cacerts=None,
                 ssl_version=None, ssl_ciphers=None,
                 ssl_validate_hostname=True, socket_timeout=60, socket_connect_timeout=15,
                 ssl_validate_cert=True):
        self.host = host
        self.port = port
        self.ssl_enable = ssl_enable
        # PROTOCOL_SSLv23 gives the best proto version available (including TLSv1 and above)
        # SSLv2 should be disabled by most OpenSSL build
        self.ssl_version = ssl_version or ssl.PROTOCOL_SSLv23
        # `ssl_ciphers`, if given, should be a string (https://www.openssl.org/docs/apps/ciphers.html)
        self.ssl_ciphers = ssl_ciphers  # if not given, use the default in Python version (`ssl._DEFAULT_CIPHERS`)
        self.keyfile = ssl_keyfile
        self.certfile = ssl_certfile
        self.cacerts = ssl_cacerts
        self.socket_timeout = socket_timeout
        self.socket_connect_timeout = socket_connect_timeout
        self.validate_hostname = ssl_validate_hostname
        self.log = logging.getLogger(__name__)
        self.sock = None
        self.greeting = None

        if ssl_validate_cert:
            self.cert_required = ssl.CERT_REQUIRED
        else:
            self.cert_required = ssl.CERT_NONE

    def connect(self, host, port=None):
        self.sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
        self.sock.settimeout(self.socket_connect_timeout)  # connect timeout
        self.sock.connect((host, port or self.port))
        self.sock.settimeout(self.socket_timeout)  # regular timeout
        if self.ssl_enable:
            self.sock = ssl.wrap_socket(self.sock, self.keyfile, self.certfile,
                                        ssl_version=self.ssl_version,
                                        ciphers=self.ssl_ciphers,
                                        server_side=False,
                                        cert_reqs=self.cert_required,
                                        ca_certs=self.cacerts)
            if self.validate_hostname:
                try:
                    match_hostname(self.sock.getpeercert(), host)
                except CertificateError, e:
                    self.log.exception("SSL hostname mismatch")
                    raise EppConnectionError(str(e))

    def remote_info(self):
        return '{}:{}'.format(*self.sock.getpeername())

    def hello(self, log_send_recv=False):
        return self.send(EppHello(), log_send_recv=log_send_recv)

    def login(self, clID, pw, newPW=None, raise_on_fail=True, obj_uris=None, extra_obj_uris=None, extra_ext_uris=None,
              clTRID=None):
        if not self.sock:
            self.connect(self.host, self.port)
            self.greeting = EppResponse.from_xml(self.read())

        cmd = EppLoginCommand(obj_uris=obj_uris, extra_obj_uris=extra_obj_uris, extra_ext_uris=extra_ext_uris)
        cmd.clID = clID
        cmd.pw = pw
        if clTRID:
            cmd['epp']['command']['clTRID'] = clTRID
        if newPW:
            cmd.newPW = newPW
        r = self.send(cmd)
        if not r.success and raise_on_fail:
            raise EppLoginError(r)
        return r

    def logout(self, clTRID=None):
        cmd = EppLogoutCommand()
        if clTRID:
            cmd['epp']['command']['clTRID'] = clTRID
        return self.send(cmd)

    def read(self):
        recvmeth = self.sock.read if self.ssl_enable else self.sock.recv
        siz = ''
        while len(siz) < 4:
            siz += recvmeth(4 - len(siz))
            if not siz:
                # empty string after read means EOF
                self.close()
                raise IOError("No size header read")

        size_remaining = siz = struct.unpack(">I", siz)[0] - 4
        data = ''
        while size_remaining:
            buf = recvmeth(size_remaining)
            if not buf:
                self.close()
                raise IOError("Short / no data read (expected %d bytes, got %d)" % (siz, len(data)))
            size_remaining -= len(buf)
            data += buf

        return data
        #self.log.debug("read total %d bytes:\n%s\n" % (siz+4, data))

    def write(self, data):
        writemeth = self.sock.write if self.ssl_enable else self.sock.sendall
        siz = struct.pack(">I", 4+len(data))
        writemeth(siz + data)

    def write_many(self, docs):
        """
        For testing only.
        Writes multiple documents at once
        """
        writemeth = self.sock.write if self.ssl_enable else self.sock.sendall
        buf = []
        for doc in docs:
            buf.append(struct.pack(">I", 4+len(doc)))
            buf.append(doc)
        writemeth(''.join(buf))

    def send(self, doc, log_send_recv=True, extra_nsmap=None, strip_hints=True):
        self._gen_cltrid(doc)
        buf = doc.to_xml(force_prefix=True).encode('utf-8')
        if log_send_recv:
            self.log.debug("SEND %s: %s", self.remote_info(), buf)
        self.write(buf)
        r = self.read()
        if log_send_recv:
            self.log.debug("RECV %s: %s", self.remote_info(), r)
        resp = EppResponse.from_xml(r, extra_nsmap=extra_nsmap)
        if strip_hints:
            self.strip_hints(resp)
        doc.normalize_response(resp)
        return resp

    def strip_hints(self, data):
        """
        Remove various cruft from the given EppDoc (useful for responses where we don't care about _order etc.
        """
        stack = deque([data])
        while len(stack):
            current = stack.pop()
            for k in list(current.keys()):
                if k in ('@xsi:schemaLocation', '_order'):
                    del current[k]
                else:
                    v = current[k]
                    if isinstance(v, dict):
                        # visit later
                        stack.append(v)
                    elif isinstance(v, list):
                        # visit each dict in the list
                        for elem in v:
                            if isinstance(elem, dict):
                                stack.append(elem)
        return data

    def batchsend(self, docs, readresponse=True, failfast=True, pipeline=False):
        """ Send multiple documents. If ``pipeline`` is True, it will
        send it in a single ``write`` call (which may have the effect
        of having more than one doc packed into a single TCP packet
        if they fits) """
        sent = 0
        recved = 0
        ndocs = len(docs)
        try:
            if pipeline:
                self.write_many(docs)
                sent = ndocs
            else:
                for doc in docs:
                    self.write(str(doc))
                    sent += 1
        except:
            self.log.error("Failed to send all commands (sent %d/%d)" % (sent, ndocs))
            if failfast:
                raise

        if not readresponse:
            return sent

        try:
            out = []
            for _ in xrange(sent):
                r = self.read()
                out.append(EppResponse.from_xml(r))
                recved += 1
        except:
            self.log.error("Failed to receive all responses (recv'ed %d/%d)" % (recved, sent))
            # pad the rest with None
            for _ in xrange(sent-len(out)):
                out.append(None)

        return out

    def write_split(self, data):
        """
        For testing only.
        Writes the size header and first 4 bytes of the payload in one call,
        then the rest of the payload in another call.
        """
        writemeth = self.sock.sendall if self.ssl_enable else self.sock.sendall
        siz = struct.pack(">I", 4+len(data))
        self.log.debug("siz=%d" % (4+len(data)))
        writemeth(siz + data[:4])
        writemeth(data[4:])

    def write_splitsize(self, data):
        """
        For testing only.
        Writes 2 bytes of the header, then another two bytes,
        then the payload in another call.
        """
        writemeth = self.sock.sendall if self.ssl_enable else self.sock.sendall
        siz = struct.pack(">I", 4+len(data))
        self.log.debug("siz=%d" % (4+len(data)))
        writemeth(siz[:2])
        writemeth(siz[2:])
        writemeth(data)

    def write_splitall(self, data):
        """
        For testing only.
        Writes 2 bytes of the header, then another two bytes,
        then 4 bytes of the payload, then the rest of the payload.
        """
        writemeth = self.sock.sendall if self.ssl_enable else self.sock.sendall
        siz = struct.pack(">I", 4+len(data))
        self.log.debug("siz=%d" % (4+len(data)))
        writemeth(siz[:2])
        writemeth(siz[2:])
        writemeth(data[:4])
        writemeth(data[4:])

    def close(self):
        self.sock.close()
        self.sock = None

    def _gen_cltrid(self, doc):
        if isinstance(doc, (EppLoginCommand, EppCreateCommand, EppUpdateCommand, EppDeleteCommand, EppTransferCommand, EppRenewCommand)):
            cmd_node = doc['epp']['command']
            if not cmd_node.get('clTRID'):
                cmd_node['clTRID'] = gen_trid()

    def _get_ssl_protocol_version(self):
        """
        This is a hack to get the negotiated protocol version of an SSL connection.

        WARNING: Do not use this on anything other than Python 2.7
        WARNING: Do not use on non-CPython.
        WARNING: only use it for debugging.
        WARNING: this will probably crash because we may be loading the wrong version of libssl

        From https://github.com/python-git/python/blob/master/Modules/_ssl.c
        the PySSLObject struct looks like this:

        typedef struct {
            PyObject_HEAD
            PySocketSockObject *Socket;	/* Socket on which we're layered */
            SSL_CTX*	ctx;
            SSL*		ssl;
            X509*		peer_cert;
            char		server[X509_NAME_MAXLEN];
            char		issuer[X509_NAME_MAXLEN];

        } PySSLObject;

        and this is stored as self.sock._sslobj so we pry open the mem location
        and call OpenSSL's SSL_get_version C API

        This technique is inspired by http://pyevolve.sourceforge.net/wordpress/?p=2171
        """
        assert self.ssl_enable, "don't use it on non-SSL sockets"
        assert self.sock._sslobj, "don't use it on non-SSL sockets"

        import ctypes
        import ctypes.util

        size_pyobject_head = ctypes.sizeof(ctypes.c_long) + ctypes.sizeof(ctypes.c_voidp)
        real_ssl_offset = size_pyobject_head + ctypes.sizeof(ctypes.c_voidp) * 2 # skip PySocketSockObject* and SSL_CTX*
        ssl_p = ctypes.c_voidp.from_address(id(self.sock._sslobj) + real_ssl_offset)
        # libssl = ctypes.cdll.LoadLibrary('/usr/local/opt/openssl/lib/libssl.1.0.0.dylib')
        libssl = ctypes.cdll.LoadLibrary(ctypes.util.find_library('ssl'))
        if not libssl:
            return None
        libssl.SSL_get_version.restype = ctypes.c_char_p
        libssl.SSL_get_version.argtypes = [ctypes.c_void_p]
        ver = libssl.SSL_get_version(ssl_p)
        return ver
<commit_msg>log connection and TLS version<commit_after>try:
    # use gevent if available
    import gevent.socket as socket
    import gevent.ssl as ssl
except ImportError:
    import socket
    import ssl

import struct
from collections import deque
import logging
from .exceptions import EppLoginError, EppConnectionError
from .doc import (EppResponse, EppHello, EppLoginCommand, EppLogoutCommand,
                  EppCreateCommand, EppUpdateCommand, EppRenewCommand, EppTransferCommand, EppDeleteCommand)
from .utils import gen_trid
from backports.ssl_match_hostname import match_hostname, CertificateError

class EppClient(object):
    def __init__(self, host=None, port=700,
                 ssl_enable=True, ssl_keyfile=None, ssl_certfile=None, ssl_cacerts=None,
                 ssl_version=None, ssl_ciphers=None,
                 ssl_validate_hostname=True, socket_timeout=60, socket_connect_timeout=15,
                 ssl_validate_cert=True):
        self.host = host
        self.port = port
        self.ssl_enable = ssl_enable
        # PROTOCOL_SSLv23 gives the best proto version available (including TLSv1 and above)
        # SSLv2 should be disabled by most OpenSSL build
        self.ssl_version = ssl_version or ssl.PROTOCOL_SSLv23
        # `ssl_ciphers`, if given, should be a string (https://www.openssl.org/docs/apps/ciphers.html)
        self.ssl_ciphers = ssl_ciphers  # if not given, use the default in Python version (`ssl._DEFAULT_CIPHERS`)
        self.keyfile = ssl_keyfile
        self.certfile = ssl_certfile
        self.cacerts = ssl_cacerts
        self.socket_timeout = socket_timeout
        self.socket_connect_timeout = socket_connect_timeout
        self.validate_hostname = ssl_validate_hostname
        self.log = logging.getLogger(__name__)
        self.sock = None
        self.greeting = None

        if ssl_validate_cert:
            self.cert_required = ssl.CERT_REQUIRED
        else:
            self.cert_required = ssl.CERT_NONE

    def connect(self, host, port=None):
        self.sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
        self.sock.settimeout(self.socket_connect_timeout)  # connect timeout
        self.sock.connect((host, port or self.port))
        local_sock_addr = self.sock.getsockname()
        local_addr, local_port = local_sock_addr[:2]
        self.log.debug('connected local=%s:%s remote=%s:%s',
                       local_addr, local_port, self.sock.getpeername()[0], port)
        self.sock.settimeout(self.socket_timeout)  # regular timeout
        if self.ssl_enable:
            self.sock = ssl.wrap_socket(self.sock, self.keyfile, self.certfile,
                                        ssl_version=self.ssl_version,
                                        ciphers=self.ssl_ciphers,
                                        server_side=False,
                                        cert_reqs=self.cert_required,
                                        ca_certs=self.cacerts)
            self.log.debug('%s negotiated with local=%s:%s remote=%s:%s', self.sock.version(),
                           local_addr, local_port, self.sock.getpeername()[0], port)
            if self.validate_hostname:
                try:
                    match_hostname(self.sock.getpeercert(), host)
                except CertificateError as e:
                    self.log.exception("SSL hostname mismatch")
                    raise EppConnectionError(str(e))

    def remote_info(self):
        return '{}:{}'.format(*self.sock.getpeername())

    def hello(self, log_send_recv=False):
        return self.send(EppHello(), log_send_recv=log_send_recv)

    def login(self, clID, pw, newPW=None, raise_on_fail=True, obj_uris=None, extra_obj_uris=None, extra_ext_uris=None,
              clTRID=None):
        if not self.sock:
            self.connect(self.host, self.port)
            self.greeting = EppResponse.from_xml(self.read())

        cmd = EppLoginCommand(obj_uris=obj_uris, extra_obj_uris=extra_obj_uris, extra_ext_uris=extra_ext_uris)
        cmd.clID = clID
        cmd.pw = pw
        if clTRID:
            cmd['epp']['command']['clTRID'] = clTRID
        if newPW:
            cmd.newPW = newPW
        r = self.send(cmd)
        if not r.success and raise_on_fail:
            raise EppLoginError(r)
        return r

    def logout(self, clTRID=None):
        cmd = EppLogoutCommand()
        if clTRID:
            cmd['epp']['command']['clTRID'] = clTRID
        return self.send(cmd)

    def read(self):
        recvmeth = self.sock.read if self.ssl_enable else self.sock.recv
        siz = ''
        while len(siz) < 4:
            siz += recvmeth(4 - len(siz))
            if not siz:
                # empty string after read means EOF
                self.close()
                raise IOError("No size header read")

        size_remaining = siz = struct.unpack(">I", siz)[0] - 4
        data = ''
        while size_remaining:
            buf = recvmeth(size_remaining)
            if not buf:
                self.close()
                raise IOError("Short / no data read (expected %d bytes, got %d)" % (siz, len(data)))
            size_remaining -= len(buf)
            data += buf

        return data
        #self.log.debug("read total %d bytes:\n%s\n" % (siz+4, data))

    def write(self, data):
        writemeth = self.sock.write if self.ssl_enable else self.sock.sendall
        siz = struct.pack(">I", 4+len(data))
        writemeth(siz + data)

    def write_many(self, docs):
        """
        For testing only.
        Writes multiple documents at once
        """
        writemeth = self.sock.write if self.ssl_enable else self.sock.sendall
        buf = []
        for doc in docs:
            buf.append(struct.pack(">I", 4+len(doc)))
            buf.append(doc)
        writemeth(''.join(buf))

    def send(self, doc, log_send_recv=True, extra_nsmap=None, strip_hints=True):
        self._gen_cltrid(doc)
        buf = doc.to_xml(force_prefix=True).encode('utf-8')
        if log_send_recv:
            self.log.debug("SEND %s: %s", self.remote_info(), buf)
        self.write(buf)
        r = self.read()
        if log_send_recv:
            self.log.debug("RECV %s: %s", self.remote_info(), r)
        resp = EppResponse.from_xml(r, extra_nsmap=extra_nsmap)
        if strip_hints:
            self.strip_hints(resp)
        doc.normalize_response(resp)
        return resp

    def strip_hints(self, data):
        """
        Remove various cruft from the given EppDoc (useful for responses where we don't care about _order etc.
        """
        stack = deque([data])
        while len(stack):
            current = stack.pop()
            for k in list(current.keys()):
                if k in ('@xsi:schemaLocation', '_order'):
                    del current[k]
                else:
                    v = current[k]
                    if isinstance(v, dict):
                        # visit later
                        stack.append(v)
                    elif isinstance(v, list):
                        # visit each dict in the list
                        for elem in v:
                            if isinstance(elem, dict):
                                stack.append(elem)
        return data

    def batchsend(self, docs, readresponse=True, failfast=True, pipeline=False):
        """ Send multiple documents. If ``pipeline`` is True, it will
        send it in a single ``write`` call (which may have the effect
        of having more than one doc packed into a single TCP packet
        if they fits) """
        sent = 0
        recved = 0
        ndocs = len(docs)
        try:
            if pipeline:
                self.write_many(docs)
                sent = ndocs
            else:
                for doc in docs:
                    self.write(str(doc))
                    sent += 1
        except:
            self.log.error("Failed to send all commands (sent %d/%d)" % (sent, ndocs))
            if failfast:
                raise

        if not readresponse:
            return sent

        try:
            out = []
            for _ in xrange(sent):
                r = self.read()
                out.append(EppResponse.from_xml(r))
                recved += 1
        except:
            self.log.error("Failed to receive all responses (recv'ed %d/%d)" % (recved, sent))
            # pad the rest with None
            for _ in xrange(sent-len(out)):
                out.append(None)

        return out

    def write_split(self, data):
        """
        For testing only.
        Writes the size header and first 4 bytes of the payload in one call,
        then the rest of the payload in another call.
        """
        writemeth = self.sock.sendall if self.ssl_enable else self.sock.sendall
        siz = struct.pack(">I", 4+len(data))
        self.log.debug("siz=%d" % (4+len(data)))
        writemeth(siz + data[:4])
        writemeth(data[4:])

    def write_splitsize(self, data):
        """
        For testing only.
        Writes 2 bytes of the header, then another two bytes,
        then the payload in another call.
        """
        writemeth = self.sock.sendall if self.ssl_enable else self.sock.sendall
        siz = struct.pack(">I", 4+len(data))
        self.log.debug("siz=%d" % (4+len(data)))
        writemeth(siz[:2])
        writemeth(siz[2:])
        writemeth(data)

    def write_splitall(self, data):
        """
        For testing only.
        Writes 2 bytes of the header, then another two bytes,
        then 4 bytes of the payload, then the rest of the payload.
        """
        writemeth = self.sock.sendall if self.ssl_enable else self.sock.sendall
        siz = struct.pack(">I", 4+len(data))
        self.log.debug("siz=%d" % (4+len(data)))
        writemeth(siz[:2])
        writemeth(siz[2:])
        writemeth(data[:4])
        writemeth(data[4:])

    def close(self):
        self.sock.close()
        self.sock = None

    def _gen_cltrid(self, doc):
        if isinstance(doc, (EppLoginCommand, EppCreateCommand, EppUpdateCommand, EppDeleteCommand, EppTransferCommand, EppRenewCommand)):
            cmd_node = doc['epp']['command']
            if not cmd_node.get('clTRID'):
                cmd_node['clTRID'] = gen_trid()

    def _get_ssl_protocol_version(self):
        """
        This is a hack to get the negotiated protocol version of an SSL connection.

        WARNING: Do not use this on anything other than Python 2.7
        WARNING: Do not use on non-CPython.
        WARNING: only use it for debugging.
        WARNING: this will probably crash because we may be loading the wrong version of libssl

        From https://github.com/python-git/python/blob/master/Modules/_ssl.c
        the PySSLObject struct looks like this:

        typedef struct {
            PyObject_HEAD
            PySocketSockObject *Socket;	/* Socket on which we're layered */
            SSL_CTX*	ctx;
            SSL*		ssl;
            X509*		peer_cert;
            char		server[X509_NAME_MAXLEN];
            char		issuer[X509_NAME_MAXLEN];

        } PySSLObject;

        and this is stored as self.sock._sslobj so we pry open the mem location
        and call OpenSSL's SSL_get_version C API

        This technique is inspired by http://pyevolve.sourceforge.net/wordpress/?p=2171
        """
        assert self.ssl_enable, "don't use it on non-SSL sockets"
        assert self.sock._sslobj, "don't use it on non-SSL sockets"

        import ctypes
        import ctypes.util

        size_pyobject_head = ctypes.sizeof(ctypes.c_long) + ctypes.sizeof(ctypes.c_voidp)
        real_ssl_offset = size_pyobject_head + ctypes.sizeof(ctypes.c_voidp) * 2 # skip PySocketSockObject* and SSL_CTX*
        ssl_p = ctypes.c_voidp.from_address(id(self.sock._sslobj) + real_ssl_offset)
        # libssl = ctypes.cdll.LoadLibrary('/usr/local/opt/openssl/lib/libssl.1.0.0.dylib')
        libssl = ctypes.cdll.LoadLibrary(ctypes.util.find_library('ssl'))
        if not libssl:
            return None
        libssl.SSL_get_version.restype = ctypes.c_char_p
        libssl.SSL_get_version.argtypes = [ctypes.c_void_p]
        ver = libssl.SSL_get_version(ssl_p)
        return ver
<|endoftext|>


<commit_before>import gocept.httpserverlayer.wsgi
import gocept.selenium
import zeit.cms.testing

ZCML_LAYER = zeit.cms.testing.ZCMLLayer('ftesting.zcml', product_config=True)
WSGI_LAYER = zeit.cms.testing.WSGILayer(name='WSGILayer', bases=(ZCML_LAYER,))
HTTP_LAYER = gocept.httpserverlayer.wsgi.Layer(
    name='HTTPLayer', bases=(WSGI_LAYER,))
WD_LAYER = gocept.selenium.WebdriverLayer(
    name='WebdriverLayer', bases=(HTTP_LAYER,))
SELENIUM_LAYER = gocept.selenium.WebdriverSeleneseLayer(
    name='SeleniumLayer', bases=(WD_LAYER,))
<commit_msg>Re ZON-2576: Update test setup<commit_after>import gocept.httpserverlayer.wsgi
import gocept.selenium
import zeit.cms.testing

ZCML_LAYER = zeit.cms.testing.ZCMLLayer(
    'ftesting.zcml', product_config=zeit.cms.testing.cms_product_config)
WSGI_LAYER = zeit.cms.testing.WSGILayer(name='WSGILayer', bases=(ZCML_LAYER,))
HTTP_LAYER = gocept.httpserverlayer.wsgi.Layer(
    name='HTTPLayer', bases=(WSGI_LAYER,))
WD_LAYER = gocept.selenium.WebdriverLayer(
    name='WebdriverLayer', bases=(HTTP_LAYER,))
SELENIUM_LAYER = gocept.selenium.WebdriverSeleneseLayer(
    name='SeleniumLayer', bases=(WD_LAYER,))
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
from ooop import OOOP
import configdb

O = OOOP(**configdb.ooop)

imp_obj = O.GiscedataFacturacioImportacioLinia

imp_del_ids = imp_obj.search([('state','=','erroni'),('info','like','Ja existeix una factura')])
imp_del_ids += imp_obj.search([('state','=','erroni'),('info','like','XML erroni')])
imp_del_ids += imp_obj.search([('state','=','erroni'),('info','like',"XML no es correspon al tipus F1")])
imp_del_ids += imp_obj.search([('state','=','erroni'),('info','like',"Document invàlid")])

total = len(imp_del_ids)
n = 0

for imp_del_id in imp_del_ids:
    try:
        imp_obj.unlink([imp_del_id])
        n +=1
        print "%d/%d" % (n,total)
    except Exception, e:
        print e
<commit_msg>shebang Fact_impF1_eliminar_Ja_existeix.py<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-
from ooop import OOOP
import configdb

O = OOOP(**configdb.ooop)

imp_obj = O.GiscedataFacturacioImportacioLinia

imp_del_ids = imp_obj.search([('state','=','erroni'),('info','like','Ja existeix una factura')])
imp_del_ids += imp_obj.search([('state','=','erroni'),('info','like','XML erroni')])
imp_del_ids += imp_obj.search([('state','=','erroni'),('info','like',"XML no es correspon al tipus F1")])
imp_del_ids += imp_obj.search([('state','=','erroni'),('info','like',"Document invàlid")])

total = len(imp_del_ids)
n = 0

for imp_del_id in imp_del_ids:
    try:
        imp_obj.unlink([imp_del_id])
        n +=1
        print "%d/%d" % (n,total)
    except Exception, e:
        print e
<|endoftext|>


<commit_before>'''
general.py: part of expfactory package

Copyright (c) 2017, Vanessa Sochat
All rights reserved.

Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are met:

* Redistributions of source code must retain the above copyright notice, this
  list of conditions and the following disclaimer.

* Redistributions in binary form must reproduce the above copyright notice,
  this list of conditions and the following disclaimer in the documentation
  and/or other materials provided with the distribution.

* Neither the name of the copyright holder nor the names of its
  contributors may be used to endorse or promote products derived from
  this software without specific prior written permission.

THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

'''

from flask import (
    flash,
    render_template,
    request,
    redirect,
    session
)

from expfactory.utils import get_post_fields

from expfactory.views.utils import perform_checks
from expfactory.server import app
import os
import json

# EXPERIMENT ROUTER ############################################################

def router():

    # Redirects to another template view
    experiment = app.get_next(session)
    if experiment is not None:
        app.logger.info('Next experiment is %s' % experiment)
    return perform_checks('/experiments/%s' % experiment, do_redirect=True)

# Home portal to start experiments
def portal():

    form = ParticipantForm()

    if request.method == "POST":

        # Submit and valid
        if form.validate_on_submit():

            # User name is not required
            username = 'You'
            if form.openid.data not in [None,""]:
                username = form.openid.data

            subid = session.get('subid')
            if subid is None:
                subid = app.generate_subid()
                session['subid'] = subid
                app.logger.info('New session [subid] %s' %subid)

            app.randomize = form.randomize.data
            session['username'] = username
            session['experiments'] = form.exp_ids.data.split(',') # list
            flash('Participant ID: "%s"
 Name %s
 Randomize: "%s"
 Experiments: %s' %
                  (subid, username, app.randomize,
                  str(form.exp_ids.data)))
            return redirect('/start')

        # Submit but not valid
        return render_template('portal/index.html', experiments=app.lookup,
                                                    base=app.base,
                                                    randomize=app.randomize,
                                                    form=form, toggleform=True)

    # Not submit
    return render_template('portal/index.html', experiments=app.lookup,
                                                base=app.base,
                                                form=form)
<commit_msg>missing form import<commit_after>'''
general.py: part of expfactory package

Copyright (c) 2017, Vanessa Sochat
All rights reserved.

Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are met:

* Redistributions of source code must retain the above copyright notice, this
  list of conditions and the following disclaimer.

* Redistributions in binary form must reproduce the above copyright notice,
  this list of conditions and the following disclaimer in the documentation
  and/or other materials provided with the distribution.

* Neither the name of the copyright holder nor the names of its
  contributors may be used to endorse or promote products derived from
  this software without specific prior written permission.

THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

'''

from flask import (
    flash,
    render_template,
    request,
    redirect,
    session
)

from expfactory.utils import get_post_fields

from expfactory.views.utils import perform_checks
from expfactory.server import app
from expfactory.forms import ParticipantForm
import os
import json

# EXPERIMENT ROUTER ############################################################

def router():

    # Redirects to another template view
    experiment = app.get_next(session)
    if experiment is not None:
        app.logger.info('Next experiment is %s' % experiment)
    return perform_checks('/experiments/%s' % experiment, do_redirect=True)

# Home portal to start experiments
def portal():

    form = ParticipantForm()

    if request.method == "POST":

        # Submit and valid
        if form.validate_on_submit():

            # User name is not required
            username = 'You'
            if form.openid.data not in [None,""]:
                username = form.openid.data

            subid = session.get('subid')
            if subid is None:
                subid = app.generate_subid()
                session['subid'] = subid
                app.logger.info('New session [subid] %s' %subid)

            app.randomize = form.randomize.data
            session['username'] = username
            session['experiments'] = form.exp_ids.data.split(',') # list
            flash('Participant ID: "%s"
 Name %s
 Randomize: "%s"
 Experiments: %s' %
                  (subid, username, app.randomize,
                  str(form.exp_ids.data)))
            return redirect('/start')

        # Submit but not valid
        return render_template('portal/index.html', experiments=app.lookup,
                                                    base=app.base,
                                                    randomize=app.randomize,
                                                    form=form, toggleform=True)

    # Not submit
    return render_template('portal/index.html', experiments=app.lookup,
                                                base=app.base,
                                                form=form)
<|endoftext|>