<commit_before>#!/usr/bin/python

"""
.. See the NOTICE file distributed with this work for additional information
   regarding copyright ownership.

   Licensed under the Apache License, Version 2.0 (the "License");
   you may not use this file except in compliance with the License.
   You may obtain a copy of the License at

       http://www.apache.org/licenses/LICENSE-2.0

   Unless required by applicable law or agreed to in writing, software
   distributed under the License is distributed on an "AS IS" BASIS,
   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
   See the License for the specific language governing permissions and
   limitations under the License.

Script to extract FASTQ entry pairs from paired end FASTQ files given a list of
FASTQ IDs.
"""

import argparse
import time

from sets import Set
from fastqreader import fastqreader

def paired_selector(in_file1, in_file2, rows, tag='tmp'):
    """
    Function to divide the FastQ files into separate sub files of 1000000
    sequences so that the aligner can run in parallel.

    Parameters
    ----------
    in_file1 : str
        Location of first paired end FASTQ file
    in_file2 : str
        Location of second paired end FASTQ file
    tag : str
        DEFAULT = tmp
        Tag used to identify the files. Useful if this is getting run
        manually on a single machine multiple times to prevent collisions of
        file names

    Returns
    -------
    Returns: Returns a list of lists of the files that have been generated.
             Each sub list containing the two paired end files for that
             subset.
    paired_files : list
        List of lists of pair end files. Each sub list containing the two
        paired end files for that subset.
    """

    fqr = fastqreader()
    fqr.openFastQ(in_file1, in_file2)
    fqr.createOutputFiles(tag)

    record1 = fqr.next(1)
    record2 = fqr.next(2)

    count_r1 = 0
    count_r2 = 0
    count_r3 = 0
    count_r4 = 0

    file_loc_1 = fqr.fastq1.split("/")
    file_loc_1[-1] = file_loc_1[-1].replace(
        ".fastq",
        "." + str(fqr.output_tag) + "_" + str(fqr.output_file_count) + ".fastq")
    file_loc_1.insert(-1, tag)

    file_loc_2 = fqr.fastq2.split("/")
    file_loc_2[-1] = file_loc_2[-1].replace(
        ".fastq",
        "." + str(fqr.output_tag) + "_" + str(fqr.output_file_count) + ".fastq")
    file_loc_2.insert(-1, tag)
    files_out = [["/".join(file_loc_1), "/".join(file_loc_2)]]

    while fqr.eof(1) is False and fqr.eof(2) is False:
        r1_id = record1["id"].split(" ")
        r2_id = record2["id"].split(" ")

        if r1_id[0] == r2_id[0]:
            if r1_id[0][1:] in rows:
                fqr.writeOutput(record1, 1)
                fqr.writeOutput(record2, 2)
                count_r4 += 1

            record1 = fqr.next(1)
            record2 = fqr.next(2)

            count_r1 += 1
            count_r2 += 1
            count_r3 += 1
        elif r1_id[0] < r2_id[0]:
            record1 = fqr.next(1)
            count_r1 += 1
        else:
            record2 = fqr.next(2)
            count_r2 += 1

        if count_r3 % 1000 == 0:
            print(count_r3, count_r4)

    fqr.closeFastQ()
    fqr.closeOutputFiles()

    return files_out

def single_selector(in_file1, rows, tag='tmp'):
    """
    Function to divide the FastQ files into separate sub files of 1000000
    sequences so that the aligner can run in parallel.

    Parameters
    ----------
    in_file1 : str
        Location of first paired end FASTQ file
    in_file2 : str
        Location of second paired end FASTQ file
    tag : str
        DEFAULT = tmp
        Tag used to identify the files. Useful if this is getting run
        manually on a single machine multiple times to prevent collisions of
        file names

    Returns
    -------
    Returns: Returns a list of lists of the files that have been generated.
             Each sub list containing the two paired end files for that
             subset.
    paired_files : list
        List of lists of pair end files. Each sub list containing the two
        paired end files for that subset.
    """

    fqr = fastqreader()
    fqr.openFastQ(in_file1)
    fqr.createOutputFiles(tag)

    current_milli_time = lambda: int(round(time.time() * 1000))

    time_1 = current_milli_time()

    record1 = fqr.next()

    file_loc_1 = fqr.fastq1.split("/")
    file_loc_1[-1] = file_loc_1[-1].replace(
        ".fastq",
        "." + str(fqr.output_tag) + "_" + str(fqr.output_file_count) + ".fastq")
    file_loc_1.insert(-1, tag)

    files_out = [["/".join(file_loc_1)]]

    counter = 0

    while fqr.eof() is False:
        r1_id = record1["id"].split(" ")

        if r1_id[0][1:] in rows:
            fqr.writeOutput(record1)
            counter += 1

            if counter % 100000 == 0:
                time_2 = current_milli_time()
                print(
                    "Extracted:", str(counter),
                    "reads (avg per 1000: " + str(time_2-time_1) + ") ..."
                )
                time_1 = time_2

        record1 = fqr.next()

    fqr.closeFastQ()
    fqr.closeOutputFiles()

    return files_out

def get_if_list(row_file):
    """
    Get the IDs from the specified file
    """
    fid = open(row_file, 'r')

    id_list = fid.readlines()
    id_list = [i.rstrip() for i in id_list]

    id_set = Set(id_list)

    fid.close()

    return id_set

if __name__ == "__main__":

    # Set up the command line parameters
    PARSER = argparse.ArgumentParser(description="Load adjacency list into HDF5 file")
    PARSER.add_argument("--input_1", help="File 1")
    PARSER.add_argument("--input_2", required=False, default=None, help="File 2")
    PARSER.add_argument("--rows", help="Row File")
    PARSER.add_argument(
        "--output_tag", help="""
            Inserted before the file descriptor and after the file name:
                'matching' would convert file_id-1.fastq to file_id-1.matching.fastq"""
    )

    ARGS = PARSER.parse_args()
    FILE_01 = ARGS.input_1
    FILE_02 = ARGS.input_2
    ROW_FILE = ARGS.rows
    TAG = ARGS.output_tag

    VALID_IDS = get_if_list(ROW_FILE)

    print(FILE_01, FILE_02, ROW_FILE, TAG)

    if FILE_02 is not None:
        paired_selector(FILE_01, FILE_02, VALID_IDS, TAG)
    else:
        single_selector(FILE_01, VALID_IDS, TAG)
<commit_msg>Tidied up the code and added comments to help with understanding<commit_after>#!/usr/bin/python

"""
Script to extract FASTQ entry pairs from paired end FASTQ files given a list of
FASTQ IDs.
"""

import argparse
import time

from sets import Set
from fastqreader import fastqreader

# Extract the rows that matched to chr21
# grep chr21 hic_test/map/SRR1658573_1_full_1-100.map | tr "\t" "~" | cut -d"~" -f1 -f5 | tr ":" "\t" | awk '(NR==1) || (($4>15000000) && ($4<20000000))' | tr "\t" "~" | cut -d "~" -f1 > SRR1658573_1_chr21_1-100.row
# grep chr21 hic_test/map/SRR1658573_2_full_1-100.map | tr "\t" "~" | cut -d"~" -f1 -f5 | tr ":" "\t" | awk '(NR==1) || (($4>15000000) && ($4<20000000))' | tr "\t" "~" | cut -d "~" -f1 > SRR1658573_2_chr21_1-100.row

# grep -Fx -f SRR1658573_1_chr21_1-100.row SRR1658573_2_chr21_1-100.row > SRR1658573_chr21.row

def paired_selector(in_file1, in_file2, rows, tag='tmp'):
    """
    Function to divide the FastQ files into separate sub files of 1000000
    sequences so that the aligner can run in parallel.

    Parameters
    ----------
    in_file1 : str
        Location of first paired end FASTQ file
    in_file2 : str
        Location of second paired end FASTQ file
    tag : str
        DEFAULT = tmp
        Tag used to identify the files. Useful if this is getting run
        manually on a single machine multiple times to prevent collisions of
        file names

    Returns
    -------
    Returns: Returns a list of lists of the files that have been generated.
             Each sub list containing the two paired end files for that
             subset.
    paired_files : list
        List of lists of pair end files. Each sub list containing the two
        paired end files for that subset.
    """

    fqr = fastqreader()
    fqr.openFastQ(in_file1, in_file2)
    fqr.createOutputFiles(tag)

    record1 = fqr.next(1)
    record2 = fqr.next(2)

    count_r1 = 0 # Reads read from FASTQ 1
    count_r2 = 0 # Reads read from FASTQ 2
    count_r3 = 0 # Matching pairs
    count_r4 = 0 # Reads that have been written

    file_loc_1 = fqr.fastq1.split("/")
    file_loc_1[-1] = file_loc_1[-1].replace(
        ".fastq",
        "." + str(fqr.output_tag) + "_" + str(fqr.output_file_count) + ".fastq")
    file_loc_1.insert(-1, tag)

    file_loc_2 = fqr.fastq2.split("/")
    file_loc_2[-1] = file_loc_2[-1].replace(
        ".fastq",
        "." + str(fqr.output_tag) + "_" + str(fqr.output_file_count) + ".fastq")
    file_loc_2.insert(-1, tag)
    files_out = [["/".join(file_loc_1), "/".join(file_loc_2)]]

    current_milli_time = lambda: int(round(time.time() * 1000))
    time_1 = current_milli_time()

    while fqr.eof(1) is False and fqr.eof(2) is False:
        r1_id = record1["id"].split(" ")
        r2_id = record2["id"].split(" ")

        if r1_id[0] == r2_id[0]:
            if r1_id[0][1:] in rows:
                fqr.writeOutput(record1, 1)
                fqr.writeOutput(record2, 2)
                count_r4 += 1

                if count_r4 > 0 and count_r4 % 1000 == 0:
                    time_2 = current_milli_time()
                    print(
                        "Extracted:", str(count_r4),
                        "reads (avg time per 1000: " + str(time_2-time_1) + ") ...")
                    time_1 = time_2

            record1 = fqr.next(1)
            record2 = fqr.next(2)

            count_r1 += 1
            count_r2 += 1
            count_r3 += 1
        elif r1_id[0] < r2_id[0]:
            record1 = fqr.next(1)
            count_r1 += 1
        else:
            record2 = fqr.next(2)
            count_r2 += 1

    fqr.closeFastQ()
    fqr.closeOutputFiles()

    return files_out

def single_selector(in_file1, rows, tag='tmp'):
    """
    Function to divide the FastQ files into separate sub files of 1000000
    sequences so that the aligner can run in parallel.

    Parameters
    ----------
    in_file1 : str
        Location of first paired end FASTQ file
    in_file2 : str
        Location of second paired end FASTQ file
    tag : str
        DEFAULT = tmp
        Tag used to identify the files. Useful if this is getting run
        manually on a single machine multiple times to prevent collisions of
        file names

    Returns
    -------
    Returns: Returns a list of lists of the files that have been generated.
             Each sub list containing the two paired end files for that
             subset.
    paired_files : list
        List of lists of pair end files. Each sub list containing the two
        paired end files for that subset.
    """

    fqr = fastqreader()
    fqr.openFastQ(in_file1)
    fqr.createOutputFiles(tag)

    current_milli_time = lambda: int(round(time.time() * 1000))

    time_1 = current_milli_time()

    record1 = fqr.next()

    file_loc_1 = fqr.fastq1.split("/")
    file_loc_1[-1] = file_loc_1[-1].replace(
        ".fastq",
        "." + str(fqr.output_tag) + "_" + str(fqr.output_file_count) + ".fastq")
    file_loc_1.insert(-1, tag)

    files_out = [["/".join(file_loc_1)]]

    counter = 0

    while fqr.eof() is False:
        r1_id = record1["id"].split(" ")

        if r1_id[0][1:] in rows:
            fqr.writeOutput(record1)
            counter += 1

            if counter % 100000 == 0:
                time_2 = current_milli_time()
                print(
                    "Extracted:", str(counter),
                    "reads (avg per 100000: " + str(time_2-time_1) + ") ...")
                time_1 = time_2

        record1 = fqr.next()

    fqr.closeFastQ()
    fqr.closeOutputFiles()

    return files_out

def get_if_list(row_file):
    """
    Get the IDs from the specified file
    """
    fid = open(row_file, 'r')

    id_list = fid.readlines()
    id_list = [i.rstrip() for i in id_list]

    id_set = Set(id_list)

    fid.close()

    return id_set

if __name__ == "__main__":

    # Set up the command line parameters
    PARSER = argparse.ArgumentParser(description="Load adjacency list into HDF5 file")
    PARSER.add_argument("--input_1", help="File 1")
    PARSER.add_argument("--input_2", required=False, default=None, help="File 2")
    PARSER.add_argument("--rows", help="Row File")
    PARSER.add_argument("--output_tag", help="Inserted before the file descriptor and after the file name: e.g. 'matching' would convert file_id-1.fastq to file_id-1.matching.fastq")

    ARGS = PARSER.parse_args()
    FILE_01 = ARGS.input_1
    FILE_02 = ARGS.input_2
    ROW_FILE = ARGS.rows
    TAG = ARGS.output_tag

    VALID_IDS = get_if_list(ROW_FILE)

    print(FILE_01, FILE_02, ROW_FILE, TAG)

    if FILE_02 is not None:
        paired_selector(FILE_01, FILE_02, VALID_IDS, TAG)
    else:
        single_selector(FILE_01, VALID_IDS, TAG)
<|endoftext|>


<commit_before>from rdflib.graph import ConjunctiveGraph
from rdflib.namespace import RDF, OWL, DC
import argparse
import re

def main():
    parser = argparse.ArgumentParser(description='description',
                                     formatter_class=
                                     argparse.RawTextHelpFormatter)
    parser.add_argument('--input', '-i', type=str, required=True,
                        help='Location of input file')
    parser.add_argument('--output', '-o', type=str, required=True,
                        help='Location of output file')
    parser.add_argument('--format', '-f', type=str, default="turtle",
                        help='format of rdf file (turtle, nt, rdf/xml)')
    args = parser.parse_args()
    property_list = get_properties_from_input(args.input, args.format)
    merged_graph = make_property_graph(property_list)

    # merge graphs
    merged_graph.parse(args.input, format=args.format)

    merged_graph.serialize(args.output, format=args.format)

def get_properties_from_input(file, format):
    input_graph = ConjunctiveGraph()
    input_graph.parse(file, format=format)

    # collapse to single list
    property_set = set()
    for row in input_graph.predicates():
        property_set.add(row)

    return property_set

def make_property_graph(properties):
    graph = ConjunctiveGraph()
    output_graph = ConjunctiveGraph()

    ontologies = [
        'https://raw.githubusercontent.com/monarch-initiative/SEPIO-ontology/master/src/ontology/sepio.owl',
        'https://raw.githubusercontent.com/monarch-initiative/GENO-ontology/develop/src/ontology/geno.owl',
        'https://raw.githubusercontent.com/oborel/obo-relations/master/ro.owl',
        'http://purl.obolibrary.org/obo/iao.owl',
        'http://data.monarchinitiative.org/owl/ero.owl',
        'https://raw.githubusercontent.com/jamesmalone/OBAN/master/ontology/oban_core.ttl',
        'http://purl.obolibrary.org/obo/pco.owl',
        'http://purl.obolibrary.org/obo/xco.owl'
    ]

    for ontology in ontologies:
        print("parsing: " + ontology)
        if re.search(r'\.owl', ontology):
            graph.parse(ontology, format='xml')
        elif re.search(r'\.ttl', ontology):
            graph.parse(ontology, format='turtle')
        else:
            graph.parse(ontology)

    # Get object properties
    output_graph = add_property_to_graph(
        graph.subjects(RDF['type'], OWL['ObjectProperty']),
        output_graph, OWL['ObjectProperty'], properties)

    # Get annotation properties
    output_graph = add_property_to_graph(
        graph.subjects(RDF['type'], OWL['AnnotationProperty']),
        output_graph, OWL['AnnotationProperty'], properties)

    # Get data properties
    output_graph = add_property_to_graph(
        graph.subjects(RDF['type'], OWL['DatatypeProperty']),
        output_graph, OWL['DatatypeProperty'], properties)

    for row in output_graph.predicates(DC['source'], OWL['AnnotationProperty']):
        if row == RDF['type']:
            output_graph.remove((DC['source'], RDF['type'], OWL['AnnotationProperty']))

    output_graph.add((DC['source'], RDF['type'], OWL['ObjectProperty']))

    return output_graph

def add_property_to_graph(results, graph, property_type, property_list):
    for row in results:
        if row in property_list:
            graph.add((row, RDF['type'], property_type))
    return graph

if __name__ == "__main__":
    main()
<commit_msg>add output format option<commit_after>from rdflib.graph import ConjunctiveGraph
from rdflib.namespace import RDF, OWL, DC
import argparse
import re

def main():
    parser = argparse.ArgumentParser(description='description',
                                     formatter_class=
                                     argparse.RawTextHelpFormatter)
    parser.add_argument('--input', '-i', type=str, required=True,
                        help='Location of input file')
    parser.add_argument('--output', '-o', type=str, required=True,
                        help='Location of output file')
    parser.add_argument('--format', '-f', type=str, default="turtle",
                        help='format of rdf file (turtle, nt, rdf/xml)')
    parser.add_argument('--output_format', type=str, default="turtle",
                        help='format of rdf file (turtle, nt, rdf/xml)')
    args = parser.parse_args()
    property_list = get_properties_from_input(args.input, args.format)
    merged_graph = make_property_graph(property_list)

    # merge graphs
    merged_graph.parse(args.input, format=args.format)

    merged_graph.serialize(args.output, format=args.output_format)

def get_properties_from_input(file, format):
    input_graph = ConjunctiveGraph()
    input_graph.parse(file, format=format)

    # collapse to single list
    property_set = set()
    for row in input_graph.predicates():
        property_set.add(row)

    return property_set

def make_property_graph(properties):
    graph = ConjunctiveGraph()
    output_graph = ConjunctiveGraph()

    ontologies = [
        'https://raw.githubusercontent.com/monarch-initiative/SEPIO-ontology/master/src/ontology/sepio.owl',
        'https://raw.githubusercontent.com/monarch-initiative/GENO-ontology/develop/src/ontology/geno.owl',
        'https://raw.githubusercontent.com/oborel/obo-relations/master/ro.owl',
        'http://purl.obolibrary.org/obo/iao.owl',
        'http://data.monarchinitiative.org/owl/ero.owl',
        'https://raw.githubusercontent.com/jamesmalone/OBAN/master/ontology/oban_core.ttl',
        'http://purl.obolibrary.org/obo/pco.owl',
        'http://purl.obolibrary.org/obo/xco.owl'
    ]

    for ontology in ontologies:
        print("parsing: " + ontology)
        if re.search(r'\.owl', ontology):
            graph.parse(ontology, format='xml')
        elif re.search(r'\.ttl', ontology):
            graph.parse(ontology, format='turtle')
        else:
            graph.parse(ontology)

    # Get object properties
    output_graph = add_property_to_graph(
        graph.subjects(RDF['type'], OWL['ObjectProperty']),
        output_graph, OWL['ObjectProperty'], properties)

    # Get annotation properties
    output_graph = add_property_to_graph(
        graph.subjects(RDF['type'], OWL['AnnotationProperty']),
        output_graph, OWL['AnnotationProperty'], properties)

    # Get data properties
    output_graph = add_property_to_graph(
        graph.subjects(RDF['type'], OWL['DatatypeProperty']),
        output_graph, OWL['DatatypeProperty'], properties)

    for row in output_graph.predicates(DC['source'], OWL['AnnotationProperty']):
        if row == RDF['type']:
            output_graph.remove((DC['source'], RDF['type'], OWL['AnnotationProperty']))

    output_graph.add((DC['source'], RDF['type'], OWL['ObjectProperty']))

    return output_graph

def add_property_to_graph(results, graph, property_type, property_list):
    for row in results:
        if row in property_list:
            graph.add((row, RDF['type'], property_type))
    return graph

if __name__ == "__main__":
    main()
<|endoftext|>


<commit_before>import argparse
import csv

import gevent.monkey

gevent.monkey.patch_all()
from closeio_api import Client as CloseIO_API
from gevent.pool import Pool

parser = argparse.ArgumentParser(description='Download a CSV of email sequences and their subscription counts (number of active/paused/finished subscriptions)')

parser.add_argument('--api-key', '-k', required=True, help='API Key')
args = parser.parse_args()

api = CloseIO_API(args.api_key)
org_name = api.get('me')['organizations'][0]['name']

print('Getting email sequences...')

params = {'_fields': 'id'}
has_more = True
offset = 0
sequence_ids = []
while has_more:
    params['_skip'] = offset
    resp = api.get('sequence', params=params)
    for sequence in resp['data']:
        sequence_ids.append(sequence['id'])
    offset += len(resp['data'])
    has_more = resp['has_more']

print(f'Found {len(sequence_ids)} email sequences. Getting their details...')

def fetch_sequence(sequence_id):
    print(f'Getting details for {sequence_id}')
    resp_sequence = api.get(f'sequence/{sequence_id}', params=params)
    active_subscriptions = resp_sequence['subscription_counts_by_status']['active']
    paused_subscriptions = resp_sequence['subscription_counts_by_status']['paused']
    finished_subscriptions = resp_sequence['subscription_counts_by_status']['finished']
    total_subscriptions = active_subscriptions + paused_subscriptions + finished_subscriptions

    sequences.append({
        'id': resp_sequence['id'],
        'name': resp_sequence['name'],
        'is_active': resp_sequence['status'] == 'active',
        'total_subscriptions': total_subscriptions,
        'active_subscriptions': active_subscriptions,
        'paused_subscriptions': paused_subscriptions,
        'finished_subscriptions': finished_subscriptions
    })

sequences = []

pool = Pool(5)
pool.map(fetch_sequence, sequence_ids)

file_name = f'{org_name} Email Sequences.csv'
print(f'Exporting to `{file_name}`')

f = open(file_name, 'wt', encoding='utf-8')
try:
    keys = ['id', 'name', 'is_active', 'total_subscriptions', 'active_subscriptions', 'paused_subscriptions', 'finished_subscriptions']
    writer = csv.DictWriter(f, keys)
    writer.writeheader()
    writer.writerows(sequences)
finally:
    f.close()
<commit_msg>Reformat, avoid slashes in the file name<commit_after>import argparse
import csv

import gevent.monkey

gevent.monkey.patch_all()
from closeio_api import Client as CloseIO_API
from gevent.pool import Pool

parser = argparse.ArgumentParser(
    description='Download a CSV of email sequences and their subscription counts (number of active/paused/finished subscriptions)'
)

parser.add_argument('--api-key', '-k', required=True, help='API Key')
args = parser.parse_args()

api = CloseIO_API(args.api_key)
org_name = api.get('me')['organizations'][0]['name']

print('Getting email sequences...')

params = {'_fields': 'id'}
has_more = True
offset = 0
sequence_ids = []
while has_more:
    params['_skip'] = offset
    resp = api.get('sequence', params=params)
    for sequence in resp['data']:
        sequence_ids.append(sequence['id'])
    offset += len(resp['data'])
    has_more = resp['has_more']

print(f'Found {len(sequence_ids)} email sequences. Getting their details...')

def fetch_sequence(sequence_id):
    resp_sequence = api.get(f'sequence/{sequence_id}', params=params)
    counts_by_status = resp_sequence['subscription_counts_by_status']
    active_subscriptions = counts_by_status['active']
    paused_subscriptions = counts_by_status['paused']
    finished_subscriptions = counts_by_status['finished']
    total_subscriptions = (
        active_subscriptions + paused_subscriptions + finished_subscriptions
    )

    sequences.append(
        {
            'id': resp_sequence['id'],
            'name': resp_sequence['name'],
            'is_active': resp_sequence['status'] == 'active',
            'total_subscriptions': total_subscriptions,
            'active_subscriptions': active_subscriptions,
            'paused_subscriptions': paused_subscriptions,
            'finished_subscriptions': finished_subscriptions,
        }
    )

sequences = []

pool = Pool(5)
pool.map(fetch_sequence, sequence_ids)

file_name = f'{org_name.replace("/", " ")} Email Sequences.csv'
print(f'Exporting to `{file_name}`')

f = open(file_name, 'wt', encoding='utf-8')
try:
    keys = [
        'id',
        'name',
        'is_active',
        'total_subscriptions',
        'active_subscriptions',
        'paused_subscriptions',
        'finished_subscriptions',
    ]
    writer = csv.DictWriter(f, keys)
    writer.writeheader()
    writer.writerows(sequences)
finally:
    f.close()
<|endoftext|>


<commit_before><commit_msg>Added: Plot-1<commit_after><|endoftext|>


<commit_before>"""I produce the hourly analysis used by IEMRE"""
from __future__ import print_function
import sys
import datetime

import netCDF4
import numpy as np
import pytz
import pandas as pd
from pandas.io.sql import read_sql
from metpy.gridding.interpolation import inverse_distance
from pyiem import iemre
from pyiem import meteorology
from pyiem import reference
import pyiem.datatypes as dt
from pyiem.util import get_dbconn

def get_domain():
    """Return where we should be estimating"""
    nc = netCDF4.Dataset('/mesonet/data/iemre/state_weights.nc')
    nc.set_auto_mask(False)
    domain = nc.variables['domain'][:]
    nc.close()
    return domain

def grid_wind(df, domain):
    """
    Grid winds based on u and v components
    @return uwnd, vwnd
    """
    # compute components
    u = []
    v = []
    for _station, row in df.iterrows():
        (_u, _v) = meteorology.uv(dt.speed(row['sknt'], 'KT'),
                                  dt.direction(row['drct'], 'DEG'))
        u.append(_u.value("MPS"))
        v.append(_v.value("MPS"))
    df['u'] = u
    df['v'] = v
    ugrid = generic_gridder(df, 'u', domain)
    vgrid = generic_gridder(df, 'v', domain)
    return ugrid, vgrid

def grid_skyc(df, domain):
    """Hmmmm"""
    v = []
    for _station, row in df.iterrows():
        _v = max(row['max_skyc1'], row['max_skyc2'], row['max_skyc3'])
        v.append(_v)
    df['skyc'] = v
    return generic_gridder(df, 'skyc', domain)

def generic_gridder(df, idx, domain):
    """Generic gridding algorithm for easy variables"""
    df2 = df[pd.notnull(df[idx])]
    xi, yi = np.meshgrid(iemre.XAXIS, iemre.YAXIS)
    res = np.ones(xi.shape) * np.nan
    # set a sentinel of where we won't be estimating
    res = np.where(domain > 0, res, -9999)
    # do our gridding
    grid = inverse_distance(df2['lon'].values, df2['lat'].values,
                            df2[idx].values, xi, yi, 1.5)
    # replace nan values in res with whatever now is in grid
    res = np.where(np.isnan(res), grid, res)
    # Do we still have missing values?
    if np.isnan(res).any():
        # very aggressive with search radius
        grid = inverse_distance(df2['lon'].values, df2['lat'].values,
                                df2[idx].values, xi, yi, 5.5)
        # replace nan values in res with whatever now is in grid
        res = np.where(np.isnan(res), grid, res)
    # replace sentinel back to np.nan
    return np.where(res == -9999, np.nan, res)

def grid_hour(nc, ts):
    """
    I proctor the gridding of data on an hourly basis
    @param ts Timestamp of the analysis, we'll consider a 20 minute window
    """
    domain = get_domain()
    ts0 = ts - datetime.timedelta(minutes=10)
    ts1 = ts + datetime.timedelta(minutes=10)
    offset = iemre.hourly_offset(ts)
    utcnow = datetime.datetime.utcnow()
    utcnow = (utcnow.replace(tzinfo=pytz.timezone("UTC")) -
              datetime.timedelta(hours=36))

    # If we are near realtime, look in IEMAccess instead of ASOS database
    mybuf = 2.
    params = (reference.MW_WEST - mybuf, reference.MW_SOUTH - mybuf,
              reference.MW_WEST - mybuf, reference.MW_NORTH + mybuf,
              reference.MW_EAST + mybuf, reference.MW_NORTH + mybuf,
              reference.MW_EAST + mybuf, reference.MW_SOUTH - mybuf,
              reference.MW_WEST - mybuf, reference.MW_SOUTH - mybuf,
              ts0, ts1)
    if utcnow < ts:
        dbconn = get_dbconn('iem', user='nobody')
        sql = """SELECT t.id as station, ST_x(geom) as lon,
        ST_y(geom) as lat,
 max(case when tmpf > -60 and tmpf < 130 THEN tmpf else null end) as max_tmpf,
 max(case when sknt > 0 and sknt < 100 then sknt else 0 end) as max_sknt,
 max(getskyc(skyc1)) as max_skyc1,
 max(getskyc(skyc2)) as max_skyc2,
 max(getskyc(skyc3)) as max_skyc3,
 max(case when phour > 0 and phour < 1000 then phour else 0 end) as phour,
 max(case when dwpf > -60 and dwpf < 100 THEN dwpf else null end) as max_dwpf,
 max(case when sknt >= 0 then sknt else 0 end) as sknt,
 max(case when sknt >= 0 then drct else 0 end) as drct
 from current_log s JOIN stations t on (s.iemid = t.iemid)
 WHERE ST_Contains(
  ST_GeomFromEWKT('SRID=4326;POLYGON((%s %s, %s  %s, %s %s, %s %s, %s %s))'),
  geom) and valid >= %s and valid < %s GROUP by station, lon, lat
         """
    else:
        dbconn = get_dbconn('asos', user='nobody')
        sql = """SELECT station, ST_x(geom) as lon, st_y(geom) as lat,
 max(case when tmpf > -60 and tmpf < 130 THEN tmpf else null end) as max_tmpf,
 max(case when sknt > 0 and sknt < 100 then sknt else 0 end) as max_sknt,
 max(getskyc(skyc1)) as max_skyc1,
 max(getskyc(skyc2)) as max_skyc2,
 max(getskyc(skyc3)) as max_skyc3,
 max(case when p01i > 0 and p01i < 1000 then p01i else 0 end) as phour,
 max(case when dwpf > -60 and dwpf < 100 THEN dwpf else null end) as max_dwpf,
 max(case when sknt >= 0 then sknt else 0 end) as sknt,
 max(case when sknt >= 0 then drct else 0 end) as drct
 from alldata a JOIN stations t on (a.station = t.id) WHERE
 ST_Contains(
  ST_GeomFromEWKT('SRID=4326;POLYGON((%s %s, %s  %s, %s %s, %s %s, %s %s))'),
  geom) and (t.network ~* 'ASOS' or t.network = 'AWOS') and
 valid >= %s and valid < %s GROUP by station, lon, lat"""

    df = read_sql(sql, dbconn, params=params, index_col='station')
    if df.empty:
        print(("%s has now entries, FAIL"
               ) % (ts.strftime("%Y-%m-%d %H:%M"), ))
        return
    ures, vres = grid_wind(df, domain)
    if ures is None:
        print("iemre.hourly_analysis failure for uwnd at %s" % (ts, ))
    else:
        nc.variables['uwnd'][offset] = ures
        nc.variables['vwnd'][offset] = vres

    tmpk = generic_gridder(df, 'max_tmpf', domain)
    if tmpk is None:
        print("iemre.hourly_analysis failure for tmpk at %s" % (ts, ))
    else:
        dwpk = generic_gridder(df, 'max_dwpf', domain)
        # require that dwpk <= tmpk
        mask = ~np.isnan(dwpk)
        mask[mask] &= dwpk[mask] > tmpk[mask]
        dwpk = np.where(mask, tmpk, dwpk)
        nc.variables['tmpk'][offset] = dt.temperature(tmpk, 'F').value('K')
        nc.variables['dwpk'][offset] = dt.temperature(dwpk, 'F').value('K')

    res = grid_skyc(df, domain)
    if res is None:
        print("iemre.hourly_analysis failure for skyc at %s" % (ts, ))
    else:
        nc.variables['skyc'][offset] = res

def main(argv):
    """Go Main"""
    if len(argv) == 5:
        ts = datetime.datetime(int(argv[1]), int(argv[2]),
                               int(argv[3]), int(argv[4]))
    else:
        ts = datetime.datetime.utcnow()
        ts = ts.replace(second=0, minute=0)
    ts = ts.replace(tzinfo=pytz.utc)
    # Load up our netcdf file!
    nc = netCDF4.Dataset("/mesonet/data/iemre/%s_mw_hourly.nc" % (ts.year,),
                         'a')
    grid_hour(nc, ts)
    nc.close()

if __name__ == "__main__":
    main(sys.argv)
<commit_msg>properly mask these hourly IEMRE variables as well<commit_after>"""I produce the hourly analysis used by IEMRE"""
from __future__ import print_function
import sys
import datetime

import netCDF4
import numpy as np
import pytz
import pandas as pd
from pandas.io.sql import read_sql
from metpy.gridding.interpolation import inverse_distance
from pyiem import iemre
from pyiem import meteorology
from pyiem import reference
import pyiem.datatypes as dt
from pyiem.util import get_dbconn

def get_domain():
    """Return where we should be estimating"""
    nc = netCDF4.Dataset('/mesonet/data/iemre/state_weights.nc')
    nc.set_auto_mask(False)
    domain = nc.variables['domain'][:]
    nc.close()
    return domain

def grid_wind(df, domain):
    """
    Grid winds based on u and v components
    @return uwnd, vwnd
    """
    # compute components
    u = []
    v = []
    for _station, row in df.iterrows():
        (_u, _v) = meteorology.uv(dt.speed(row['sknt'], 'KT'),
                                  dt.direction(row['drct'], 'DEG'))
        u.append(_u.value("MPS"))
        v.append(_v.value("MPS"))
    df['u'] = u
    df['v'] = v
    ugrid = generic_gridder(df, 'u', domain)
    vgrid = generic_gridder(df, 'v', domain)
    return ugrid, vgrid

def grid_skyc(df, domain):
    """Hmmmm"""
    v = []
    for _station, row in df.iterrows():
        _v = max(row['max_skyc1'], row['max_skyc2'], row['max_skyc3'])
        v.append(_v)
    df['skyc'] = v
    return generic_gridder(df, 'skyc', domain)

def generic_gridder(df, idx, domain):
    """Generic gridding algorithm for easy variables"""
    df2 = df[pd.notnull(df[idx])]
    xi, yi = np.meshgrid(iemre.XAXIS, iemre.YAXIS)
    res = np.ones(xi.shape) * np.nan
    # set a sentinel of where we won't be estimating
    res = np.where(domain > 0, res, -9999)
    # do our gridding
    grid = inverse_distance(df2['lon'].values, df2['lat'].values,
                            df2[idx].values, xi, yi, 1.5)
    # replace nan values in res with whatever now is in grid
    res = np.where(np.isnan(res), grid, res)
    # Do we still have missing values?
    if np.isnan(res).any():
        # very aggressive with search radius
        grid = inverse_distance(df2['lon'].values, df2['lat'].values,
                                df2[idx].values, xi, yi, 5.5)
        # replace nan values in res with whatever now is in grid
        res = np.where(np.isnan(res), grid, res)
    # replace sentinel back to np.nan
    # replace sentinel back to np.nan
    res = np.where(res == -9999, np.nan, res)
    return np.ma.array(res, mask=np.isnan(res))

def grid_hour(nc, ts):
    """
    I proctor the gridding of data on an hourly basis
    @param ts Timestamp of the analysis, we'll consider a 20 minute window
    """
    domain = get_domain()
    ts0 = ts - datetime.timedelta(minutes=10)
    ts1 = ts + datetime.timedelta(minutes=10)
    offset = iemre.hourly_offset(ts)
    utcnow = datetime.datetime.utcnow()
    utcnow = (utcnow.replace(tzinfo=pytz.timezone("UTC")) -
              datetime.timedelta(hours=36))

    # If we are near realtime, look in IEMAccess instead of ASOS database
    mybuf = 2.
    params = (reference.MW_WEST - mybuf, reference.MW_SOUTH - mybuf,
              reference.MW_WEST - mybuf, reference.MW_NORTH + mybuf,
              reference.MW_EAST + mybuf, reference.MW_NORTH + mybuf,
              reference.MW_EAST + mybuf, reference.MW_SOUTH - mybuf,
              reference.MW_WEST - mybuf, reference.MW_SOUTH - mybuf,
              ts0, ts1)
    if utcnow < ts:
        dbconn = get_dbconn('iem', user='nobody')
        sql = """SELECT t.id as station, ST_x(geom) as lon,
        ST_y(geom) as lat,
 max(case when tmpf > -60 and tmpf < 130 THEN tmpf else null end) as max_tmpf,
 max(case when sknt > 0 and sknt < 100 then sknt else 0 end) as max_sknt,
 max(getskyc(skyc1)) as max_skyc1,
 max(getskyc(skyc2)) as max_skyc2,
 max(getskyc(skyc3)) as max_skyc3,
 max(case when phour > 0 and phour < 1000 then phour else 0 end) as phour,
 max(case when dwpf > -60 and dwpf < 100 THEN dwpf else null end) as max_dwpf,
 max(case when sknt >= 0 then sknt else 0 end) as sknt,
 max(case when sknt >= 0 then drct else 0 end) as drct
 from current_log s JOIN stations t on (s.iemid = t.iemid)
 WHERE ST_Contains(
  ST_GeomFromEWKT('SRID=4326;POLYGON((%s %s, %s  %s, %s %s, %s %s, %s %s))'),
  geom) and valid >= %s and valid < %s GROUP by station, lon, lat
         """
    else:
        dbconn = get_dbconn('asos', user='nobody')
        sql = """SELECT station, ST_x(geom) as lon, st_y(geom) as lat,
 max(case when tmpf > -60 and tmpf < 130 THEN tmpf else null end) as max_tmpf,
 max(case when sknt > 0 and sknt < 100 then sknt else 0 end) as max_sknt,
 max(getskyc(skyc1)) as max_skyc1,
 max(getskyc(skyc2)) as max_skyc2,
 max(getskyc(skyc3)) as max_skyc3,
 max(case when p01i > 0 and p01i < 1000 then p01i else 0 end) as phour,
 max(case when dwpf > -60 and dwpf < 100 THEN dwpf else null end) as max_dwpf,
 max(case when sknt >= 0 then sknt else 0 end) as sknt,
 max(case when sknt >= 0 then drct else 0 end) as drct
 from alldata a JOIN stations t on (a.station = t.id) WHERE
 ST_Contains(
  ST_GeomFromEWKT('SRID=4326;POLYGON((%s %s, %s  %s, %s %s, %s %s, %s %s))'),
  geom) and (t.network ~* 'ASOS' or t.network = 'AWOS') and
 valid >= %s and valid < %s GROUP by station, lon, lat"""

    df = read_sql(sql, dbconn, params=params, index_col='station')
    if df.empty:
        print(("%s has now entries, FAIL"
               ) % (ts.strftime("%Y-%m-%d %H:%M"), ))
        return
    ures, vres = grid_wind(df, domain)
    if ures is None:
        print("iemre.hourly_analysis failure for uwnd at %s" % (ts, ))
    else:
        nc.variables['uwnd'][offset] = ures
        nc.variables['vwnd'][offset] = vres

    tmpk = generic_gridder(df, 'max_tmpf', domain)
    if tmpk is None:
        print("iemre.hourly_analysis failure for tmpk at %s" % (ts, ))
    else:
        dwpk = generic_gridder(df, 'max_dwpf', domain)
        # require that dwpk <= tmpk
        mask = ~np.isnan(dwpk)
        mask[mask] &= dwpk[mask] > tmpk[mask]
        dwpk = np.where(mask, tmpk, dwpk)
        nc.variables['tmpk'][offset] = dt.temperature(tmpk, 'F').value('K')
        nc.variables['dwpk'][offset] = dt.temperature(dwpk, 'F').value('K')

    res = grid_skyc(df, domain)
    if res is None:
        print("iemre.hourly_analysis failure for skyc at %s" % (ts, ))
    else:
        nc.variables['skyc'][offset] = res

def main(argv):
    """Go Main"""
    if len(argv) == 5:
        ts = datetime.datetime(int(argv[1]), int(argv[2]),
                               int(argv[3]), int(argv[4]))
    else:
        ts = datetime.datetime.utcnow()
        ts = ts.replace(second=0, minute=0)
    ts = ts.replace(tzinfo=pytz.utc)
    # Load up our netcdf file!
    nc = netCDF4.Dataset("/mesonet/data/iemre/%s_mw_hourly.nc" % (ts.year,),
                         'a')
    grid_hour(nc, ts)
    nc.close()

if __name__ == "__main__":
    main(sys.argv)
<|endoftext|>


<commit_before># Need something to ingest the CTRE provided bridge data
# RSAI4
# RLRI4

import mx.DateTime
import sys
# Run every 3 minutes
now = mx.DateTime.now()
if now.minute % 4 != 0:
    sys.exit(0)

import urllib2
import csv
import access
import pg
import secret
import os
accessdb = pg.connect('iem', 'iemdb')

csv = open('/tmp/ctre.txt', 'w')

# Get Saylorville
try:
    req = urllib2.Request("ftp://%s:%s@129.186.224.167/Saylorville_Table3Min_current.dat" % (secret.CTRE_FTPUSER,
                                                        secret.CTRE_FTPPASS))
    data = urllib2.urlopen(req, timeout=30).readlines()
except:
    if now.minute % 15 == 0:
        print 'Download CTRE Bridge Data Failed!!!'
    sys.exit(0)
keys = data[0].strip().replace('"', '').split(',')
vals = data[1].strip().replace('"', '').split(',')
d = {}
for i in range(len(vals)):
    d[ keys[i] ] = vals[i]

ts1 = mx.DateTime.strptime(d['TIMESTAMP'], '%Y-%m-%d %H:%M:%S')

iem = access.Ob( 'RSAI4', "OT")
iem.setObTime( ts1 )
drct = d['WindDir']
iem.data['drct'] = drct
sknt = float(d['WS_mph_S_WVT']) / 1.15
iem.data['sknt'] = sknt
gust = float(d['WS_mph_Max']) / 1.15
iem.data['gust'] = gust
iem.updateDatabase( accessdb )

csv.write("%s,%s,%s,%.1f,%.1f\n" % ('RSAI4',
            ts1.gmtime().strftime("%Y/%m/%d %H:%M:%S"),
      drct, sknt, gust) )

# Get Saylorville
req = urllib2.Request("ftp://%s:%s@129.186.224.167/Red Rock_Table3Min_current.dat" % (secret.CTRE_FTPUSER,
                                                        secret.CTRE_FTPPASS))
#try:
data = urllib2.urlopen(req).readlines()
#except:
#  pass
keys = data[0].strip().replace('"', '').split(',')
vals = data[1].strip().replace('"', '').split(',')
d = {}
for i in range(len(vals)):
    d[ keys[i] ] = vals[i]

ts2 = mx.DateTime.strptime(d['TIMESTAMP'], '%Y-%m-%d %H:%M:%S')

iem = access.Ob( 'RLRI4', "OT")
iem.setObTime( ts2 )
drct = d['WindDir']
iem.data['drct'] = drct
sknt = float(d['WS_mph_S_WVT']) / 1.15
iem.data['sknt'] = sknt
gust = float(d['WS_mph_Max']) / 1.15
iem.data['gust'] = gust
iem.updateDatabase( accessdb )

csv.write("%s,%s,%s,%.1f,%.1f\n" % ('RLRI4',
            ts2.gmtime().strftime("%Y/%m/%d %H:%M:%S"),
      drct, sknt, gust) )

csv.close()

cmd = "/home/ldm/bin/pqinsert -p 'data c 000000000000 csv/ctre.txt bogus txt' /tmp/ctre.txt >& /dev/null"
if ((mx.DateTime.now() - ts1).seconds > 3600. and
   (mx.DateTime.now() - ts2).seconds > 3600.):
    sys.exit()
os.system( cmd )
<commit_msg>cut down on noise somewhat<commit_after>"""
 Need something to ingest the CTRE provided bridge data
  RSAI4
  RLRI4
 Run from RUN_1MIN
"""

import mx.DateTime
import sys
# Run every 3 minutes
now = mx.DateTime.now()
if now.minute % 4 != 0:
    sys.exit(0)

import urllib2
import csv
import access
import pg
import secret
import subprocess
accessdb = pg.connect('iem', 'iemdb')

csv = open('/tmp/ctre.txt', 'w')

# Get Saylorville
try:
    req = urllib2.Request("ftp://%s:%s@129.186.224.167/Saylorville_Table3Min_current.dat" % (secret.CTRE_FTPUSER,
                                                        secret.CTRE_FTPPASS))
    data = urllib2.urlopen(req, timeout=30).readlines()
except:
    if now.minute % 15 == 0:
        print 'Download CTRE Bridge Data Failed!!!'
    sys.exit(0)
if len(data) < 2:
    sys.exit(0)
keys = data[0].strip().replace('"', '').split(',')
vals = data[1].strip().replace('"', '').split(',')
d = {}
for i in range(len(vals)):
    d[ keys[i] ] = vals[i]

ts1 = mx.DateTime.strptime(d['TIMESTAMP'], '%Y-%m-%d %H:%M:%S')

iem = access.Ob( 'RSAI4', "OT")
iem.setObTime( ts1 )
drct = d['WindDir']
iem.data['drct'] = drct
sknt = float(d['WS_mph_S_WVT']) / 1.15
iem.data['sknt'] = sknt
gust = float(d['WS_mph_Max']) / 1.15
iem.data['gust'] = gust
iem.updateDatabase( accessdb )

csv.write("%s,%s,%s,%.1f,%.1f\n" % ('RSAI4',
            ts1.gmtime().strftime("%Y/%m/%d %H:%M:%S"),
      drct, sknt, gust) )

# Get Saylorville
req = urllib2.Request("ftp://%s:%s@129.186.224.167/Red Rock_Table3Min_current.dat" % (secret.CTRE_FTPUSER,
                                                        secret.CTRE_FTPPASS))
#try:
data = urllib2.urlopen(req).readlines()
if len(data) < 2:
    sys.exit(0)
#except:
#  pass
keys = data[0].strip().replace('"', '').split(',')
vals = data[1].strip().replace('"', '').split(',')
d = {}
for i in range(len(vals)):
    d[ keys[i] ] = vals[i]

ts2 = mx.DateTime.strptime(d['TIMESTAMP'], '%Y-%m-%d %H:%M:%S')

iem = access.Ob( 'RLRI4', "OT")
iem.setObTime( ts2 )
drct = d['WindDir']
iem.data['drct'] = drct
sknt = float(d['WS_mph_S_WVT']) / 1.15
iem.data['sknt'] = sknt
gust = float(d['WS_mph_Max']) / 1.15
iem.data['gust'] = gust
iem.updateDatabase( accessdb )

csv.write("%s,%s,%s,%.1f,%.1f\n" % ('RLRI4',
            ts2.gmtime().strftime("%Y/%m/%d %H:%M:%S"),
      drct, sknt, gust) )

csv.close()

cmd = "/home/ldm/bin/pqinsert -p 'data c 000000000000 csv/ctre.txt bogus txt' /tmp/ctre.txt >& /dev/null"
if ((mx.DateTime.now() - ts1).seconds > 3600. and
   (mx.DateTime.now() - ts2).seconds > 3600.):
    sys.exit()
subprocess.call( cmd, shell=True )
<|endoftext|>


<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-
"""
/***************************************************************************
 OSGeo4Mac Python script for generating CMake option string for use in Qt
 Creator with dev builds and installs of QGIS when built off dependencies
 from homebrew-osgeo4mac tap
                              -------------------
        begin    : January 2014
        copyright: (C) 2014 Larry Shaffer
        email    : larrys at dakotacarto dot com
 ***************************************************************************/

/***************************************************************************
 *                                                                         *
 *   This program is free software; you can redistribute it and/or modify  *
 *   it under the terms of the GNU General Public License as published by  *
 *   the Free Software Foundation; either version 2 of the License, or     *
 *   (at your option) any later version.                                   *
 *                                                                         *
 ***************************************************************************/
"""

import os
import sys
from collections import OrderedDict
import argparse

GRASS_VERSION = '6.4.3'
OSG_VERSION = '3.2.0'
HOMEBREW_PREFIX = '/usr/local'  # default for Homebrew

def cmake_opts(qi, hb):
    # ensure libintl.h can be found in gettext for grass
    cxx_flags = "-I{hb}/opt/gettext/include"
    if 'CXXFLAGS' in os.environ:
        cxx_flags += " " + os.environ['CXXFLAGS']

    # search Homebrew's Frameworks directory before those in /Library or /System
    ld_flags = "-F{hb}/Frameworks"
    if 'LDFLAGS' in os.environ:
        ld_flags += " " + os.environ['LDFLAGS']

    # IMPORTANT: mulit-path options need the CMake list semicolon (;) delimiter,
    #            NOT the environment variable path list colon (:) separator

    # set CMAKE_PREFIX_PATH for keg-only installs, and HOMEBREW_PREFIX

    cm_opts = OrderedDict([
        ('CMAKE_INSTALL_PREFIX', qi),
        ('CMAKE_PREFIX_PATH', '"{hb}/opt/libxml2;{hb}/opt/expat;'
                              '{hb}/opt/gettext;{hb}/opt/sqlite;'
                              '{hb}"'),
        ('CMAKE_FRAMEWORK_PATH', '"{hb}/opt/qwt/lib;{hb}/opt/qwtpolar/lib"'),
        ('CMAKE_BUILD_TYPE', 'RelWithDebInfo'),
        ('CMAKE_FIND_FRAMEWORK', 'LAST'),
        ('CMAKE_CXX_FLAGS', '"' + cxx_flags + '"'),
        ('CMAKE_EXE_LINKER_FLAGS', '"' + ld_flags + '"'),
        ('CMAKE_MODULE_LINKER_FLAGS', '"' + ld_flags + '"'),
        ('CMAKE_SHARED_LINKER_FLAGS', '"' + ld_flags + '"'),
        ('CXX_EXTRA_FLAGS', '"-Wno-unused-private-field"'),
        ('BISON_EXECUTABLE', '{hb}/opt/bison/bin/bison'),
        ('QT_QMAKE_EXECUTABLE', '{hb}/bin/qmake'),
        ('GITCOMMAND', '{hb}/bin/git'),
        ('ENABLE_TESTS', 'TRUE'),
        ('WITH_ASTYLE', 'TRUE'),
        ('WITH_INTERNAL_SPATIALITE', 'FALSE'),
        ('WITH_PYSPATIALITE', 'FALSE'),
        ('WITH_INTERNAL_QWTPOLAR', 'FALSE'),
        ('WITH_MAPSERVER', 'TRUE'),
        ('WITH_STAGED_PLUGINS', 'FALSE'),
        ('WITH_PY_COMPILE', 'TRUE'),
        ('WITH_APIDOC', 'FALSE'),
        ('WITH_QSCIAPI', 'FALSE'),
        ('POSTGRES_CONFIG', '{hb}/bin/pg_config'),
        ('WITH_GRASS', 'TRUE'),
        ('GRASS_PREFIX', '{hb}/opt/grass/grass-' + GRASS_VERSION),
        ('WITH_GLOBE', 'TRUE'),
        ('OSG_DIR', '{hb}'),
        ('OSGEARTH_DIR', '{hb}'),
        ('OSG_PLUGINS_PATH', '{hb}/lib/osgPlugins-' + OSG_VERSION),
        ('WITH_ORACLE', 'FALSE'),
        ('QGIS_MACAPP_BUNDLE', '0')
    ])

    # ('CMAKE_FRAMEWORK_PATH', '""')

    # Supplemental
    # ('OCI_INCLUDE_DIR', '{hb}/opt/oracle-client-sdk/sdk/include'),
    # ('OCI_LIBRARY', '{hb}/lib/libclntsh.dylib'),

    # These should be found automatically now...
    # ('SQLITE3_INCLUDE_DIR', '{hb}/opt/sqlite/include'),
    # ('SQLITE3_LIBRARY', '{hb}/opt/sqlite/lib/libsqlite3.dylib'),
    # ('QSCINTILLA_INCLUDE_DIR', '{hb}/opt/qscintilla2/include/Qsci'),
    # ('QSCINTILLA_LIBRARY', '{hb}/opt/qscintilla2/lib/libqscintilla2.dylib'),
    # ('QWT_LIBRARY', '{hb}/opt/qwt/lib/qwt.framework/qwt'),
    # ('QWT_INCLUDE_DIR', '{hb}/opt/qwt/lib/qwt.framework/Headers'),
    # ('QWTPOLAR_LIBRARY',
    #  '{hb}/opt/qwtpolar/lib/qwtpolar.framework/qwt'),
    # ('QWTPOLAR_INCLUDE_DIR',
    #  '{hb}/opt/qwtpolar/lib/qwtpolar.framework/Headers'),

    if os.path.exists(hb + '/Frameworks/Python.framework/Versions/2.7'):
        cm_opts['PYTHON_EXECUTABLE'] = '{hb}/bin/python'
        cm_opts['PYTHON_CUSTOM_FRAMEWORK'] = \
            '{hb}/Frameworks/Python.framework/Versions/2.7'

    return cm_opts

def arg_parser():
    parser = argparse.ArgumentParser(
        description="""\
            Script for generating CMake option string for use in Qt Creator with
            dev builds and installs of QGIS when built off dependencies from
            homebrew-osgeo4mac tap.
            """
    )
    parser.add_argument(
        '-p', '--homebrew-prefix', dest='hb',
        metavar='homebrew_prefix',
        help='homebrew prefix path, or set HOMEBREW_PREFIX (/usr/local default)'
    )
    parser.add_argument(
        'qs', metavar='qgis-src-dir',
        help='QGIS source directory'
    )
    parser.add_argument(
        'qi', metavar='qgis-install-dir',
        help='QGIS install directory'
    )
    return parser

def main():
    # get defined args
    args = arg_parser().parse_args()

    qs = os.path.realpath(args.qs)
    if not os.path.isabs(qs) or not os.path.exists(qs):
        print 'QGIS source directory not resolved to existing absolute path.'
        sys.exit(1)

    qi = os.path.realpath(args.qi)
    if not os.path.isabs(qi) or not os.path.exists(qi):
        print 'QGIS install directory not resolved to existing absolute path.'
        sys.exit(1)

    # generate list of environment variables
    hb_prefix = HOMEBREW_PREFIX
    if 'HOMEBREW_PREFIX' in os.environ:
        hb_prefix = os.environ['HOMEBREW_PREFIX']
    hb = os.path.realpath(args.hb) if args.hb else hb_prefix
    if not os.path.isabs(hb) or not os.path.exists(hb):
        print 'HOMEBREW_PREFIX not resolved to existing absolute path.'
        sys.exit(1)

    # generate cmake options
    cm_opts = cmake_opts(qi, hb)

    cm_opts_s = args.qs
    for k, v in cm_opts.iteritems():
        cm_opts_s += ' -D{0}={1}'.format(k, v.format(hb=hb))

    os.system("echo '{0}' | pbcopy".format(cm_opts_s))
    print "\nThe following has been copied to the clipboard:\n"
    print cm_opts_s

if __name__ == '__main__':
    main()
    sys.exit(0)
<commit_msg>scripts: update qgis cmake options with grass-64<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-
"""
/***************************************************************************
 OSGeo4Mac Python script for generating CMake option string for use in Qt
 Creator with dev builds and installs of QGIS when built off dependencies
 from homebrew-osgeo4mac tap
                              -------------------
        begin    : January 2014
        copyright: (C) 2014 Larry Shaffer
        email    : larrys at dakotacarto dot com
 ***************************************************************************/

/***************************************************************************
 *                                                                         *
 *   This program is free software; you can redistribute it and/or modify  *
 *   it under the terms of the GNU General Public License as published by  *
 *   the Free Software Foundation; either version 2 of the License, or     *
 *   (at your option) any later version.                                   *
 *                                                                         *
 ***************************************************************************/
"""

import os
import sys
from collections import OrderedDict
import argparse

GRASS_VERSION = '6.4.3'
OSG_VERSION = '3.2.0'
HOMEBREW_PREFIX = '/usr/local'  # default for Homebrew

def cmake_opts(qi, hb):
    # ensure libintl.h can be found in gettext for grass
    cxx_flags = "-I{hb}/opt/gettext/include"
    if 'CXXFLAGS' in os.environ:
        cxx_flags += " " + os.environ['CXXFLAGS']

    # search Homebrew's Frameworks directory before those in /Library or /System
    ld_flags = "-F{hb}/Frameworks"
    if 'LDFLAGS' in os.environ:
        ld_flags += " " + os.environ['LDFLAGS']

    # IMPORTANT: mulit-path options need the CMake list semicolon (;) delimiter,
    #            NOT the environment variable path list colon (:) separator

    # set CMAKE_PREFIX_PATH for keg-only installs, and HOMEBREW_PREFIX

    cm_opts = OrderedDict([
        ('CMAKE_INSTALL_PREFIX', qi),
        ('CMAKE_PREFIX_PATH', '"{hb}/opt/libxml2;{hb}/opt/expat;'
                              '{hb}/opt/gettext;{hb}/opt/sqlite;'
                              '{hb}"'),
        ('CMAKE_FRAMEWORK_PATH', '"{hb}/opt/qwt/lib;{hb}/opt/qwtpolar/lib"'),
        ('CMAKE_BUILD_TYPE', 'RelWithDebInfo'),
        ('CMAKE_FIND_FRAMEWORK', 'LAST'),
        ('CMAKE_CXX_FLAGS', '"' + cxx_flags + '"'),
        ('CMAKE_EXE_LINKER_FLAGS', '"' + ld_flags + '"'),
        ('CMAKE_MODULE_LINKER_FLAGS', '"' + ld_flags + '"'),
        ('CMAKE_SHARED_LINKER_FLAGS', '"' + ld_flags + '"'),
        ('CXX_EXTRA_FLAGS', '"-Wno-unused-private-field"'),
        ('BISON_EXECUTABLE', '{hb}/opt/bison/bin/bison'),
        ('QT_QMAKE_EXECUTABLE', '{hb}/bin/qmake'),
        ('GITCOMMAND', '{hb}/bin/git'),
        ('ENABLE_TESTS', 'TRUE'),
        ('WITH_ASTYLE', 'TRUE'),
        ('WITH_INTERNAL_SPATIALITE', 'FALSE'),
        ('WITH_PYSPATIALITE', 'FALSE'),
        ('WITH_INTERNAL_QWTPOLAR', 'FALSE'),
        ('WITH_MAPSERVER', 'TRUE'),
        ('WITH_STAGED_PLUGINS', 'FALSE'),
        ('WITH_PY_COMPILE', 'TRUE'),
        ('WITH_APIDOC', 'FALSE'),
        ('WITH_QSCIAPI', 'FALSE'),
        ('POSTGRES_CONFIG', '{hb}/bin/pg_config'),
        ('WITH_GRASS', 'TRUE'),
        ('GRASS_PREFIX', '{hb}/opt/grass-64/grass-' + GRASS_VERSION),
        ('WITH_GLOBE', 'TRUE'),
        ('OSG_DIR', '{hb}'),
        ('OSGEARTH_DIR', '{hb}'),
        ('OSG_PLUGINS_PATH', '{hb}/lib/osgPlugins-' + OSG_VERSION),
        ('WITH_ORACLE', 'FALSE'),
        ('QGIS_MACAPP_BUNDLE', '0')
    ])

    # ('CMAKE_FRAMEWORK_PATH', '""')

    # Supplemental
    # ('OCI_INCLUDE_DIR', '{hb}/opt/oracle-client-sdk/sdk/include'),
    # ('OCI_LIBRARY', '{hb}/lib/libclntsh.dylib'),

    # These should be found automatically now...
    # ('SQLITE3_INCLUDE_DIR', '{hb}/opt/sqlite/include'),
    # ('SQLITE3_LIBRARY', '{hb}/opt/sqlite/lib/libsqlite3.dylib'),
    # ('QSCINTILLA_INCLUDE_DIR', '{hb}/opt/qscintilla2/include/Qsci'),
    # ('QSCINTILLA_LIBRARY', '{hb}/opt/qscintilla2/lib/libqscintilla2.dylib'),
    # ('QWT_LIBRARY', '{hb}/opt/qwt/lib/qwt.framework/qwt'),
    # ('QWT_INCLUDE_DIR', '{hb}/opt/qwt/lib/qwt.framework/Headers'),
    # ('QWTPOLAR_LIBRARY',
    #  '{hb}/opt/qwtpolar/lib/qwtpolar.framework/qwt'),
    # ('QWTPOLAR_INCLUDE_DIR',
    #  '{hb}/opt/qwtpolar/lib/qwtpolar.framework/Headers'),

    if os.path.exists(hb + '/Frameworks/Python.framework/Versions/2.7'):
        cm_opts['PYTHON_EXECUTABLE'] = '{hb}/bin/python'
        cm_opts['PYTHON_CUSTOM_FRAMEWORK'] = \
            '{hb}/Frameworks/Python.framework/Versions/2.7'

    return cm_opts

def arg_parser():
    parser = argparse.ArgumentParser(
        description="""\
            Script for generating CMake option string for use in Qt Creator with
            dev builds and installs of QGIS when built off dependencies from
            homebrew-osgeo4mac tap.
            """
    )
    parser.add_argument(
        '-p', '--homebrew-prefix', dest='hb',
        metavar='homebrew_prefix',
        help='homebrew prefix path, or set HOMEBREW_PREFIX (/usr/local default)'
    )
    parser.add_argument(
        'qs', metavar='qgis-src-dir',
        help='QGIS source directory'
    )
    parser.add_argument(
        'qi', metavar='qgis-install-dir',
        help='QGIS install directory'
    )
    return parser

def main():
    # get defined args
    args = arg_parser().parse_args()

    qs = os.path.realpath(args.qs)
    if not os.path.isabs(qs) or not os.path.exists(qs):
        print 'QGIS source directory not resolved to existing absolute path.'
        sys.exit(1)

    qi = os.path.realpath(args.qi)
    if not os.path.isabs(qi) or not os.path.exists(qi):
        print 'QGIS install directory not resolved to existing absolute path.'
        sys.exit(1)

    # generate list of environment variables
    hb_prefix = HOMEBREW_PREFIX
    if 'HOMEBREW_PREFIX' in os.environ:
        hb_prefix = os.environ['HOMEBREW_PREFIX']
    hb = os.path.realpath(args.hb) if args.hb else hb_prefix
    if not os.path.isabs(hb) or not os.path.exists(hb):
        print 'HOMEBREW_PREFIX not resolved to existing absolute path.'
        sys.exit(1)

    # generate cmake options
    cm_opts = cmake_opts(qi, hb)

    cm_opts_s = args.qs
    for k, v in cm_opts.iteritems():
        cm_opts_s += ' -D{0}={1}'.format(k, v.format(hb=hb))

    os.system("echo '{0}' | pbcopy".format(cm_opts_s))
    print "\nThe following has been copied to the clipboard:\n"
    print cm_opts_s

if __name__ == '__main__':
    main()
    sys.exit(0)
<|endoftext|>


<commit_before>#!/usr/bin/env python
import argparse
import os
import shutil
import sys
from os.path import abspath, dirname, join

from pytest import main as py_main

CURDIR = dirname(abspath(__file__))
SRC = join(CURDIR, os.pardir, "src")

def remove_output_dir():
    output_dir = os.path.join(CURDIR, "output_dir")
    if os.path.exists(output_dir):
        shutil.rmtree(output_dir)
    os.mkdir(output_dir)

def run_unit_tests(reporter, reporter_args):
    sys.path.insert(0, SRC)
    py_args = [
        "--showlocals",
        "--tb=long",
        f"--rootdir={CURDIR}",
        "-p",
        "no:cacheprovider",
        CURDIR
    ]
    if reporter:
        py_args.insert(0, f"--approvaltests-add-reporter={reporter}")
    if reporter_args:
        py_args.insert(1, f"--approvaltests-add-reporter-args={reporter_args}")
    try:
        result = py_main(py_args)
    except Exception:
        result = 254
    finally:
        sys.path.pop(0)
    return result

if __name__ == "__main__":
    parser = argparse.ArgumentParser(description="SeleniumLibrary Unit test runner.")
    parser.add_argument(
        "-R", "--approvaltests-use-reporter", default="PythonNative", dest="reporter"
    )
    parser.add_argument(
        "-A", "--approvaltests-add-reporter-args", default=None, dest="reporter_args"
    )
    args = parser.parse_args()
    remove_output_dir()
    sys.exit(run_unit_tests(args.reporter, args.reporter_args))
<commit_msg>Improved utest runner<commit_after>#!/usr/bin/env python
import argparse
import os
import shutil
import sys
from os.path import abspath, dirname, join
from pathlib import Path

from pytest import main as py_main

CURDIR = Path(__file__).parent
SRC = join(CURDIR, os.pardir, "src")

def remove_output_dir():
    output_dir = os.path.join(CURDIR, "output_dir")
    if os.path.exists(output_dir):
        shutil.rmtree(output_dir)
    os.mkdir(output_dir)

def run_unit_tests(reporter, reporter_args, suite):
    sys.path.insert(0, SRC)
    suite = CURDIR if not suite else CURDIR / "test" / suite
    py_args = [
        "--showlocals",
        "--tb=long",
        "-v",
        f"--rootdir={CURDIR}",
        "-p",
        "no:cacheprovider",
        str(suite),
    ]
    if reporter:
        py_args.insert(0, f"--approvaltests-add-reporter={reporter}")
    if reporter_args:
        py_args.insert(1, f"--approvaltests-add-reporter-args={reporter_args}")
    try:
        result = py_main(py_args)
    except Exception:
        result = 254
    finally:
        sys.path.pop(0)
    return result

if __name__ == "__main__":
    parser = argparse.ArgumentParser(description="SeleniumLibrary Unit test runner.")
    parser.add_argument(
        "-R", "--approvaltests-use-reporter", default="PythonNative", dest="reporter"
    )
    parser.add_argument(
        "-A", "--approvaltests-add-reporter-args", default=None, dest="reporter_args"
    )
    parser.add_argument(
        "--suite",
        "-S",
        default="",
        help="Select .py file which is only run. Example: locators/test_elementfinder.py or locators/",
    )
    args = parser.parse_args()
    remove_output_dir()
    sys.exit(run_unit_tests(args.reporter, args.reporter_args, args.suite))
<|endoftext|>


<commit_before># Copyright (c) 2012 The Chromium Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

import BaseHTTPServer
from collections import namedtuple
import gzip
import mimetypes
import os
import SimpleHTTPServer
import SocketServer
import StringIO
import sys
import urlparse

from telemetry.core import local_server

ByteRange = namedtuple('ByteRange', ['from_byte', 'to_byte'])
ResourceAndRange = namedtuple('ResourceAndRange', ['resource', 'byte_range'])

class MemoryCacheHTTPRequestHandler(SimpleHTTPServer.SimpleHTTPRequestHandler):

  def do_GET(self):
    """Serve a GET request."""
    resource_range = self.SendHead()

    if not resource_range or not resource_range.resource:
      return
    response = resource_range.resource['response']

    if not resource_range.byte_range:
      self.wfile.write(response)
      return

    start_index = resource_range.byte_range.from_byte
    end_index = resource_range.byte_range.to_byte
    self.wfile.write(response[start_index:end_index + 1])

  def do_HEAD(self):
    """Serve a HEAD request."""
    self.SendHead()

  def log_error(self, fmt, *args):
    pass

  def log_request(self, code='-', size='-'):
    # Dont spam the console unless it is important.
    pass

  def SendHead(self):
    path = os.path.realpath(self.translate_path(self.path))
    if path not in self.server.resource_map:
      self.send_error(404, 'File not found')
      return None

    resource = self.server.resource_map[path]
    total_num_of_bytes = resource['content-length']
    byte_range = self.GetByteRange(total_num_of_bytes)
    if byte_range:
      # request specified a range, so set response code to 206.
      self.send_response(206)
      self.send_header('Content-Range',
                       'bytes %d-%d/%d' % (byte_range.from_byte,
                                           byte_range.to_byte,
                                           total_num_of_bytes))
      total_num_of_bytes = byte_range.to_byte - byte_range.from_byte + 1
    else:
      self.send_response(200)

    self.send_header('Content-Length', str(total_num_of_bytes))
    self.send_header('Content-Type', resource['content-type'])
    self.send_header('Last-Modified',
                     self.date_time_string(resource['last-modified']))
    if resource['zipped']:
      self.send_header('Content-Encoding', 'gzip')
    self.end_headers()
    return ResourceAndRange(resource, byte_range)

  def GetByteRange(self, total_num_of_bytes):
    """Parse the header and get the range values specified.

    Args:
      total_num_of_bytes: Total # of bytes in requested resource,
      used to calculate upper range limit.
    Returns:
      A ByteRange namedtuple object with the requested byte-range values.
      If no Range is explicitly requested or there is a failure parsing,
      return None.
      If range specified is in the format "N-", return N-END. Refer to
      http://www.w3.org/Protocols/rfc2616/rfc2616-sec14.html for details.
      If upper range limit is greater than total # of bytes, return upper index.
    """

    range_header = self.headers.getheader('Range')
    if range_header is None:
      return None
    if not range_header.startswith('bytes='):
      return None

    # The range header is expected to be a string in this format:
    # bytes=0-1
    # Get the upper and lower limits of the specified byte-range.
    # We've already confirmed that range_header starts with 'bytes='.
    byte_range_values = range_header[len('bytes='):].split('-')
    from_byte = 0
    to_byte = 0

    if len(byte_range_values) == 2:
      # If to_range is not defined return all bytes starting from from_byte.
      to_byte = (int(byte_range_values[1]) if  byte_range_values[1]
          else total_num_of_bytes - 1)
      # If from_range is not defined return last 'to_byte' bytes.
      from_byte = (int(byte_range_values[0]) if byte_range_values[0]
          else total_num_of_bytes - to_byte)
    else:
      return None

    # Do some validation.
    if from_byte < 0:
      return None

    # Make to_byte the end byte by default in edge cases.
    if to_byte < from_byte or to_byte >= total_num_of_bytes:
      to_byte = total_num_of_bytes - 1

    return ByteRange(from_byte, to_byte)

class _MemoryCacheHTTPServerImpl(SocketServer.ThreadingMixIn,
                                 BaseHTTPServer.HTTPServer):
  # Increase the request queue size. The default value, 5, is set in
  # SocketServer.TCPServer (the parent of BaseHTTPServer.HTTPServer).
  # Since we're intercepting many domains through this single server,
  # it is quite possible to get more than 5 concurrent requests.
  request_queue_size = 128

  def __init__(self, host_port, handler, paths):
    BaseHTTPServer.HTTPServer.__init__(self, host_port, handler)
    self.resource_map = {}
    for path in paths:
      if os.path.isdir(path):
        self.AddDirectoryToResourceMap(path)
      else:
        self.AddFileToResourceMap(path)

  def AddDirectoryToResourceMap(self, directory_path):
    """Loads all files in directory_path into the in-memory resource map."""
    for root, dirs, files in os.walk(directory_path):
      # Skip hidden files and folders (like .svn and .git).
      files = [f for f in files if f[0] != '.']
      dirs[:] = [d for d in dirs if d[0] != '.']

      for f in files:
        file_path = os.path.join(root, f)
        if not os.path.exists(file_path):  # Allow for '.#' files
          continue
        self.AddFileToResourceMap(file_path)

  def AddFileToResourceMap(self, file_path):
    """Loads file_path into the in-memory resource map."""
    file_path = os.path.realpath(file_path)
    if file_path in self.resource_map:
      return

    with open(file_path, 'rb') as fd:
      response = fd.read()
      fs = os.fstat(fd.fileno())
    content_type = mimetypes.guess_type(file_path)[0]
    zipped = False
    if content_type in ['text/html', 'text/css', 'application/javascript']:
      zipped = True
      sio = StringIO.StringIO()
      gzf = gzip.GzipFile(fileobj=sio, compresslevel=9, mode='wb')
      gzf.write(response)
      gzf.close()
      response = sio.getvalue()
      sio.close()
    self.resource_map[file_path] = {
        'content-type': content_type,
        'content-length': len(response),
        'last-modified': fs.st_mtime,
        'response': response,
        'zipped': zipped
        }

    index = 'index.html'
    if os.path.basename(file_path) == index:
      dir_path = os.path.dirname(file_path)
      self.resource_map[dir_path] = self.resource_map[file_path]

class MemoryCacheHTTPServerBackend(local_server.LocalServerBackend):
  def __init__(self):
    super(MemoryCacheHTTPServerBackend, self).__init__()
    self._httpd = None

  def StartAndGetNamedPorts(self, args):
    base_dir = args['base_dir']
    os.chdir(base_dir)

    paths = args['paths']
    for path in paths:
      if not os.path.realpath(path).startswith(os.path.realpath(os.getcwd())):
        print >> sys.stderr, '"%s" is not under the cwd.' % path
        sys.exit(1)

    server_address = (args['host'], args['port'])
    MemoryCacheHTTPRequestHandler.protocol_version = 'HTTP/1.1'
    self._httpd = _MemoryCacheHTTPServerImpl(
        server_address, MemoryCacheHTTPRequestHandler, paths)
    return [local_server.NamedPort('http', self._httpd.server_address[1])]

  def ServeForever(self):
    return self._httpd.serve_forever()

class MemoryCacheHTTPServer(local_server.LocalServer):
  def __init__(self, paths):
    super(MemoryCacheHTTPServer, self).__init__(
        MemoryCacheHTTPServerBackend)
    self._base_dir = None

    for path in paths:
      assert os.path.exists(path), '%s does not exist.' % path

    paths = list(paths)
    self._paths = paths

    self._paths_as_set = set(map(os.path.realpath, paths))

    common_prefix = os.path.commonprefix(paths)
    if os.path.isdir(common_prefix):
      self._base_dir = common_prefix
    else:
      self._base_dir = os.path.dirname(common_prefix)

  def GetBackendStartupArgs(self):
    return {'base_dir': self._base_dir,
            'paths': self._paths,
            'host': self.host_ip,
            'port': 0}

  @property
  def paths(self):
    return self._paths_as_set

  @property
  def url(self):
    return self.forwarder.url

  def UrlOf(self, path):
    relative_path = os.path.relpath(path, self._base_dir)
    # Preserve trailing slash or backslash.
    # It doesn't matter in a file path, but it does matter in a URL.
    if path.endswith(os.sep) or (os.altsep and path.endswith(os.altsep)):
      relative_path += '/'
    return urlparse.urljoin(self.url, relative_path.replace(os.sep, '/'))
<commit_msg>[Telemetry] Speculative fix for buildbot deadlocks.<commit_after># Copyright (c) 2012 The Chromium Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

import BaseHTTPServer
from collections import namedtuple
import gzip
import mimetypes
import os
import SimpleHTTPServer
import SocketServer
import StringIO
import sys
import urlparse

from telemetry.core import local_server

ByteRange = namedtuple('ByteRange', ['from_byte', 'to_byte'])
ResourceAndRange = namedtuple('ResourceAndRange', ['resource', 'byte_range'])

class MemoryCacheHTTPRequestHandler(SimpleHTTPServer.SimpleHTTPRequestHandler):

  def do_GET(self):
    """Serve a GET request."""
    resource_range = self.SendHead()

    if not resource_range or not resource_range.resource:
      return
    response = resource_range.resource['response']

    if not resource_range.byte_range:
      self.wfile.write(response)
      return

    start_index = resource_range.byte_range.from_byte
    end_index = resource_range.byte_range.to_byte
    self.wfile.write(response[start_index:end_index + 1])

  def do_HEAD(self):
    """Serve a HEAD request."""
    self.SendHead()

  def log_error(self, fmt, *args):
    pass

  def log_request(self, code='-', size='-'):
    # Dont spam the console unless it is important.
    pass

  def SendHead(self):
    path = os.path.realpath(self.translate_path(self.path))
    if path not in self.server.resource_map:
      self.send_error(404, 'File not found')
      return None

    resource = self.server.resource_map[path]
    total_num_of_bytes = resource['content-length']
    byte_range = self.GetByteRange(total_num_of_bytes)
    if byte_range:
      # request specified a range, so set response code to 206.
      self.send_response(206)
      self.send_header('Content-Range',
                       'bytes %d-%d/%d' % (byte_range.from_byte,
                                           byte_range.to_byte,
                                           total_num_of_bytes))
      total_num_of_bytes = byte_range.to_byte - byte_range.from_byte + 1
    else:
      self.send_response(200)

    self.send_header('Content-Length', str(total_num_of_bytes))
    self.send_header('Content-Type', resource['content-type'])
    self.send_header('Last-Modified',
                     self.date_time_string(resource['last-modified']))
    if resource['zipped']:
      self.send_header('Content-Encoding', 'gzip')
    self.end_headers()
    return ResourceAndRange(resource, byte_range)

  def GetByteRange(self, total_num_of_bytes):
    """Parse the header and get the range values specified.

    Args:
      total_num_of_bytes: Total # of bytes in requested resource,
      used to calculate upper range limit.
    Returns:
      A ByteRange namedtuple object with the requested byte-range values.
      If no Range is explicitly requested or there is a failure parsing,
      return None.
      If range specified is in the format "N-", return N-END. Refer to
      http://www.w3.org/Protocols/rfc2616/rfc2616-sec14.html for details.
      If upper range limit is greater than total # of bytes, return upper index.
    """

    range_header = self.headers.getheader('Range')
    if range_header is None:
      return None
    if not range_header.startswith('bytes='):
      return None

    # The range header is expected to be a string in this format:
    # bytes=0-1
    # Get the upper and lower limits of the specified byte-range.
    # We've already confirmed that range_header starts with 'bytes='.
    byte_range_values = range_header[len('bytes='):].split('-')
    from_byte = 0
    to_byte = 0

    if len(byte_range_values) == 2:
      # If to_range is not defined return all bytes starting from from_byte.
      to_byte = (int(byte_range_values[1]) if  byte_range_values[1]
          else total_num_of_bytes - 1)
      # If from_range is not defined return last 'to_byte' bytes.
      from_byte = (int(byte_range_values[0]) if byte_range_values[0]
          else total_num_of_bytes - to_byte)
    else:
      return None

    # Do some validation.
    if from_byte < 0:
      return None

    # Make to_byte the end byte by default in edge cases.
    if to_byte < from_byte or to_byte >= total_num_of_bytes:
      to_byte = total_num_of_bytes - 1

    return ByteRange(from_byte, to_byte)

class _MemoryCacheHTTPServerImpl(SocketServer.ThreadingMixIn,
                                 BaseHTTPServer.HTTPServer):
  # Increase the request queue size. The default value, 5, is set in
  # SocketServer.TCPServer (the parent of BaseHTTPServer.HTTPServer).
  # Since we're intercepting many domains through this single server,
  # it is quite possible to get more than 5 concurrent requests.
  request_queue_size = 128

  # Don't prevent python from exiting when there is thread activity.
  daemon_threads = True

  def __init__(self, host_port, handler, paths):
    BaseHTTPServer.HTTPServer.__init__(self, host_port, handler)
    self.resource_map = {}
    for path in paths:
      if os.path.isdir(path):
        self.AddDirectoryToResourceMap(path)
      else:
        self.AddFileToResourceMap(path)

  def AddDirectoryToResourceMap(self, directory_path):
    """Loads all files in directory_path into the in-memory resource map."""
    for root, dirs, files in os.walk(directory_path):
      # Skip hidden files and folders (like .svn and .git).
      files = [f for f in files if f[0] != '.']
      dirs[:] = [d for d in dirs if d[0] != '.']

      for f in files:
        file_path = os.path.join(root, f)
        if not os.path.exists(file_path):  # Allow for '.#' files
          continue
        self.AddFileToResourceMap(file_path)

  def AddFileToResourceMap(self, file_path):
    """Loads file_path into the in-memory resource map."""
    file_path = os.path.realpath(file_path)
    if file_path in self.resource_map:
      return

    with open(file_path, 'rb') as fd:
      response = fd.read()
      fs = os.fstat(fd.fileno())
    content_type = mimetypes.guess_type(file_path)[0]
    zipped = False
    if content_type in ['text/html', 'text/css', 'application/javascript']:
      zipped = True
      sio = StringIO.StringIO()
      gzf = gzip.GzipFile(fileobj=sio, compresslevel=9, mode='wb')
      gzf.write(response)
      gzf.close()
      response = sio.getvalue()
      sio.close()
    self.resource_map[file_path] = {
        'content-type': content_type,
        'content-length': len(response),
        'last-modified': fs.st_mtime,
        'response': response,
        'zipped': zipped
        }

    index = 'index.html'
    if os.path.basename(file_path) == index:
      dir_path = os.path.dirname(file_path)
      self.resource_map[dir_path] = self.resource_map[file_path]

class MemoryCacheHTTPServerBackend(local_server.LocalServerBackend):
  def __init__(self):
    super(MemoryCacheHTTPServerBackend, self).__init__()
    self._httpd = None

  def StartAndGetNamedPorts(self, args):
    base_dir = args['base_dir']
    os.chdir(base_dir)

    paths = args['paths']
    for path in paths:
      if not os.path.realpath(path).startswith(os.path.realpath(os.getcwd())):
        print >> sys.stderr, '"%s" is not under the cwd.' % path
        sys.exit(1)

    server_address = (args['host'], args['port'])
    MemoryCacheHTTPRequestHandler.protocol_version = 'HTTP/1.1'
    self._httpd = _MemoryCacheHTTPServerImpl(
        server_address, MemoryCacheHTTPRequestHandler, paths)
    return [local_server.NamedPort('http', self._httpd.server_address[1])]

  def ServeForever(self):
    return self._httpd.serve_forever()

class MemoryCacheHTTPServer(local_server.LocalServer):
  def __init__(self, paths):
    super(MemoryCacheHTTPServer, self).__init__(
        MemoryCacheHTTPServerBackend)
    self._base_dir = None

    for path in paths:
      assert os.path.exists(path), '%s does not exist.' % path

    paths = list(paths)
    self._paths = paths

    self._paths_as_set = set(map(os.path.realpath, paths))

    common_prefix = os.path.commonprefix(paths)
    if os.path.isdir(common_prefix):
      self._base_dir = common_prefix
    else:
      self._base_dir = os.path.dirname(common_prefix)

  def GetBackendStartupArgs(self):
    return {'base_dir': self._base_dir,
            'paths': self._paths,
            'host': self.host_ip,
            'port': 0}

  @property
  def paths(self):
    return self._paths_as_set

  @property
  def url(self):
    return self.forwarder.url

  def UrlOf(self, path):
    relative_path = os.path.relpath(path, self._base_dir)
    # Preserve trailing slash or backslash.
    # It doesn't matter in a file path, but it does matter in a URL.
    if path.endswith(os.sep) or (os.altsep and path.endswith(os.altsep)):
      relative_path += '/'
    return urlparse.urljoin(self.url, relative_path.replace(os.sep, '/'))
<|endoftext|>


<commit_before>import datetime
from sqlalchemy import func, distinct, or_, desc
from sqlalchemy.dialects import mysql

from db import session
import model
import util

class TaskStatus:
	LOCKED = 'locked'
	BASE = 'base'
	CORRECTING = 'correcting'
	DONE = 'done'

# Vraci dvojici { task_id : module_cnt } pro vsechny plne odevzdane ulohy
# sum(body) je suma bodu za vsechny moduly v dane uloze
# Plne odevzdane moduly = bez autocorrect, nebo s autocorrect majici plny pocet bodu
# Moduly s maximem 0 bodu jsou bonusove a jsou vzdy fully_submitted (i pokud nebyly odevzdany)
def fully_submitted(user_id, year_id=None):
	if user_id is None:
		return []

	q = session.query(model.Task.id, func.count(distinct(model.Module.id)).label('modules'))
	if year_id is not None:
			q = q.join(model.Wave, model.Task.wave == model.Wave.id).filter(model.Wave.year == year_id)
	q = q.outerjoin(model.Module).group_by(model.Task.id)

	max_modules_count = { task.id: task.modules for task in q.filter(model.Module.bonus == False).all() }

	real_modules_count = { task.id: task.modules for task in q.join(model.Evaluation).filter(model.Evaluation.user == user_id, or_(model.Module.autocorrect != True, model.Module.max_points == model.Evaluation.points)).group_by(model.Task.id).all() }

	return { int(key): int(val) for key, val in real_modules_count.items() if max_modules_count[key] <= val }

# Vraci dvojici { model.Task : sum(body) } pro vsechny jakkoliv odevzdane ulohy
# sum(body) je suma bodu za vsechny moduly v dane uloze
def any_submitted(user_id, year_id):
	# Skore uivatele per modul
	per_module = session.query(model.Evaluation.module.label('module'), func.max(model.Evaluation.points).label('points')).\
		join(model.Module, model.Evaluation.module == model.Module.id).\
		join(model.Task, model.Task.id == model.Module.task).\
		filter(model.Evaluation.user == user_id).\
		group_by(model.Evaluation.module).subquery()

	# Skore per task
	return session.query(model.Task, func.sum(per_module.c.points).label("score")).\
		join(model.Module, model.Module.task == model.Task.id).\
		join(per_module, model.Module.id == per_module.c.module).\
		join(model.Wave, model.Wave.id == model.Task.wave).\
		filter(model.Wave.year == year_id).\
		group_by(model.Task).all()

def after_deadline():
	return { int(task.id) for task in session.query(model.Task).filter(model.Task.time_deadline < datetime.datetime.utcnow() ).all() }

def max_points(task_id):
	points = session.query(func.sum(model.Module.max_points).label('points')).\
		filter(model.Module.task == task_id).first().points

	return int(points) if points else 0

def max_points_dict():
	points_per_task = session.query(model.Module.task.label('id'), func.sum(model.Module.max_points).label('points')).\
		group_by(model.Module.task).all()

	return { task.id: int(task.points) for task in points_per_task }

def points_per_module(task_id, user_id):
	return session.query(model.Module, \
		func.max(model.Evaluation.points).label('points'), model.Evaluation.evaluator.label('evaluator')).\
		join(model.Evaluation, model.Evaluation.module == model.Module.id).\
		filter(model.Module.task == task_id, model.Evaluation.user == user_id).\
		join(model.Task, model.Task.id == model.Module.task).\
		filter(model.Task.evaluation_public).\
		group_by(model.Evaluation.module).all()

def points(task_id, user_id):
	ppm = points_per_module(task_id, user_id)

	if len(ppm) == 0:
		return None

	return sum(module.points for module in ppm if module.points is not None)

# vraci seznam id vsech opravenych uloh daneho uzivatele
# tzn. u uloh, ktere jsou odevzdavany opakovane (automaticky vyhodnocovane)
# vraci ulohu, pokud resitel udelal submit alespon jednoho (teoreticky spatneho) reseni
def corrected(user_id):
	return [ r for (r, ) in session.query(model.Task.id).\
		filter(model.Task.evaluation_public).\
		join(model.Module, model.Module.task == model.Task.id).\
		join(model.Evaluation, model.Evaluation.module == model.Module.id).\
		filter(model.Evaluation.user == user_id).\
		group_by(model.Task).all() ]

def comment_thread(task_id, user_id):
	query = session.query(model.SolutionComment).filter(model.SolutionComment.task == task_id, model.SolutionComment.user == user_id).first()

	return query.thread if query is not None else None

# Vraci seznam automaticky opravovanych uloh, ktere maji plny pocet bodu.
# Pokud uloha nema automaticky opravovane moduly, vrati ji taky.
def autocorrected_full(user_id):
	q = session.query(model.Task.id.label('task_id'), func.count(distinct(model.Module.id)).label('mod_cnt')).\
		join(model.Module, model.Module.task == model.Task.id).\
		filter(model.Module.bonus == False).group_by(model.Task)

	max_modules_count = q.subquery()

	real_modules_count = q.join(model.Evaluation, model.Evaluation.module == model.Module.id).filter(model.Evaluation.user == user_id, or_(model.Module.autocorrect != True, model.Module.max_points == model.Evaluation.points)).subquery()

	return [ r for (r, ) in session.query(model.Task.id).\
		join(max_modules_count, model.Task.id == max_modules_count.c.task_id).\
		join(real_modules_count, model.Task.id == real_modules_count.c.task_id).\
		filter(max_modules_count.c.mod_cnt == real_modules_count.c.mod_cnt).all() ]

# Argumenty None slouzi k tomu, aby se usetrily SQL pozadavky:
#  pri hromadnem ziskavani stavu je mozne je vyplnit a pocet SQL dotazu bude mensi
#  Pokud jsou None, potrebne informace se zjisti z databaze.
def status(task, user, adeadline=None, fsubmitted=None, wave=None, corr=None, acfull=None):
	if not wave:
		wave = session.query(model.Wave).get(task.wave)

	# pokud neni prihlasen zadny uzivatel, otevreme jen ulohu bez prerekvizit
	# = prvvni uloha
	if user is None or user.id is None:
		return TaskStatus.BASE if task.prerequisite is None and wave.public else TaskStatus.LOCKED

	# pokud uloha neni v otevrene vlne, je LOCKED
	# vyjimkou jsou uzivatele s rolemi 'org' a 'admin', tem se zobrazuji vsechny ulohu
	if not wave.public and not user.role in ('org', 'admin'):
		return TaskStatus.LOCKED

	if corr is None: corr = task.id in corrected(user.id)
	if acfull is None: acfull = task.id in autocorrected_full(user.id)

	# Pokud je uloha opravena, je DONE.
	# Uloha neni DONE dokud nemaji vsechny automaticky opravovane moduly plny pocet bodu.
	if corr and acfull:
		return TaskStatus.DONE

	if not fsubmitted:
		fsubmitted = fully_submitted(user.id)

	# Pokud je uloha odevzdana a jeste neopravena, je CORRECTING
	if task.id in fsubmitted:
		return TaskStatus.CORRECTING

	if not adeadline:
		adeadline = after_deadline()

	# pokud je po deadline, zadani ulohy se otevira vsem
	currently_active = adeadline | set(fully_submitted(user.id).keys())
	if task.id in currently_active:
		return TaskStatus.BASE

	# Pokud nenastal ani jeden z vyse zminenych pripadu, otevreme ulohu, pokud
	# jsou splneny prerekvizity
	return TaskStatus.BASE if util.PrerequisitiesEvaluator(task.prerequisite_obj, currently_active).evaluate() or user.role in ('org', 'admin') else TaskStatus.LOCKED

def solution_public(status, task, user):
	return status == TaskStatus.DONE or task.time_deadline < datetime.datetime.utcnow() or user.role in ('org', 'admin')

def time_published(task_id):
	return session.query(model.Wave.time_published).\
		join(model.Task, model.Task.wave == model.Wave.id).\
		filter(model.Task.id == task_id).scalar()

def to_json(task, user=None, adeadline=None, fsubmitted=None, wave=None, corr=None, acfull=None):
	max_points = sum([ module.max_points for module in task.modules ])
	tstatus = status(task, user, adeadline, fsubmitted, wave, corr, acfull)
	pict_base = task.picture_base if task.picture_base is not None else "/taskContent/" + str(task.id) + "/icon/"

	if not wave:
		wave = session.query(model.Wave).get(task.wave)

	return {
		'id': task.id,
		'title': task.title,
		'author': task.author,
		'details': task.id,
		'intro': task.intro,
		'max_score': format(sum([ module.max_points for module in task.modules if not module.bonus ]), '.1f'),
		'time_published': wave.time_published.isoformat(),
		'time_deadline': task.time_deadline.isoformat(),
		'state': tstatus,
		'prerequisities': [] if not task.prerequisite_obj else util.prerequisite.to_json(task.prerequisite_obj),
		'picture_base': pict_base,
		'picture_suffix': '.svg'
	}

def details_to_json(task, user, status, achievements, best_scores, comment_thread=None):
	return {
		'id': task.id,
		'body': task.body,
		'thread': task.thread,
		'modules': [ module.id for module in task.modules ],
		'best_scores': [ best_score.User.id for best_score in best_scores ],
		'comment': comment_thread if task.evaluation_public else None,
		'solution': task.solution if solution_public(status, task, user) else None,
		'achievements': [ achievement.id for achievement in achievements ]
	}

def best_scores(task_id):
	per_modules = session.query(model.User.id.label('user_id'), \
			func.max(model.Evaluation.points).label('points')).\
			join(model.Evaluation, model.Evaluation.user == model.User.id).\
			filter(model.Module.task == task_id, 'points' is not None).\
			join(model.Module, model.Evaluation.module == model.Module.id).\
			join(model.Task, model.Task.id == model.Module.task).\
			filter(model.Task.evaluation_public).\
			group_by(model.Evaluation.module, model.User).subquery()

	return session.query(model.User, func.sum(per_modules.c.points).label('sum')).join(per_modules, per_modules.c.user_id == model.User.id).filter(model.User.role == 'participant').group_by(per_modules.c.user_id).order_by(desc('sum')).slice(0, 5).all()

def best_score_to_json(best_score):
	achievements = session.query(model.UserAchievement).filter(model.UserAchievement.user_id == best_score.User.id).all()

	return {
		'id': best_score.User.id,
		'user': best_score.User.id,
		'achievements': [ achievement.achievement_id for achievement in achievements ],
		'score': format(best_score.sum, '.1f')
	}
<commit_msg>Send wave with task.<commit_after>import datetime
from sqlalchemy import func, distinct, or_, desc
from sqlalchemy.dialects import mysql

from db import session
import model
import util

class TaskStatus:
	LOCKED = 'locked'
	BASE = 'base'
	CORRECTING = 'correcting'
	DONE = 'done'

# Vraci dvojici { task_id : module_cnt } pro vsechny plne odevzdane ulohy
# sum(body) je suma bodu za vsechny moduly v dane uloze
# Plne odevzdane moduly = bez autocorrect, nebo s autocorrect majici plny pocet bodu
# Moduly s maximem 0 bodu jsou bonusove a jsou vzdy fully_submitted (i pokud nebyly odevzdany)
def fully_submitted(user_id, year_id=None):
	if user_id is None:
		return []

	q = session.query(model.Task.id, func.count(distinct(model.Module.id)).label('modules'))
	if year_id is not None:
			q = q.join(model.Wave, model.Task.wave == model.Wave.id).filter(model.Wave.year == year_id)
	q = q.outerjoin(model.Module).group_by(model.Task.id)

	max_modules_count = { task.id: task.modules for task in q.filter(model.Module.bonus == False).all() }

	real_modules_count = { task.id: task.modules for task in q.join(model.Evaluation).filter(model.Evaluation.user == user_id, or_(model.Module.autocorrect != True, model.Module.max_points == model.Evaluation.points)).group_by(model.Task.id).all() }

	return { int(key): int(val) for key, val in real_modules_count.items() if max_modules_count[key] <= val }

# Vraci dvojici { model.Task : sum(body) } pro vsechny jakkoliv odevzdane ulohy
# sum(body) je suma bodu za vsechny moduly v dane uloze
def any_submitted(user_id, year_id):
	# Skore uivatele per modul
	per_module = session.query(model.Evaluation.module.label('module'), func.max(model.Evaluation.points).label('points')).\
		join(model.Module, model.Evaluation.module == model.Module.id).\
		join(model.Task, model.Task.id == model.Module.task).\
		filter(model.Evaluation.user == user_id).\
		group_by(model.Evaluation.module).subquery()

	# Skore per task
	return session.query(model.Task, func.sum(per_module.c.points).label("score")).\
		join(model.Module, model.Module.task == model.Task.id).\
		join(per_module, model.Module.id == per_module.c.module).\
		join(model.Wave, model.Wave.id == model.Task.wave).\
		filter(model.Wave.year == year_id).\
		group_by(model.Task).all()

def after_deadline():
	return { int(task.id) for task in session.query(model.Task).filter(model.Task.time_deadline < datetime.datetime.utcnow() ).all() }

def max_points(task_id):
	points = session.query(func.sum(model.Module.max_points).label('points')).\
		filter(model.Module.task == task_id).first().points

	return int(points) if points else 0

def max_points_dict():
	points_per_task = session.query(model.Module.task.label('id'), func.sum(model.Module.max_points).label('points')).\
		group_by(model.Module.task).all()

	return { task.id: int(task.points) for task in points_per_task }

def points_per_module(task_id, user_id):
	return session.query(model.Module, \
		func.max(model.Evaluation.points).label('points'), model.Evaluation.evaluator.label('evaluator')).\
		join(model.Evaluation, model.Evaluation.module == model.Module.id).\
		filter(model.Module.task == task_id, model.Evaluation.user == user_id).\
		join(model.Task, model.Task.id == model.Module.task).\
		filter(model.Task.evaluation_public).\
		group_by(model.Evaluation.module).all()

def points(task_id, user_id):
	ppm = points_per_module(task_id, user_id)

	if len(ppm) == 0:
		return None

	return sum(module.points for module in ppm if module.points is not None)

# vraci seznam id vsech opravenych uloh daneho uzivatele
# tzn. u uloh, ktere jsou odevzdavany opakovane (automaticky vyhodnocovane)
# vraci ulohu, pokud resitel udelal submit alespon jednoho (teoreticky spatneho) reseni
def corrected(user_id):
	return [ r for (r, ) in session.query(model.Task.id).\
		filter(model.Task.evaluation_public).\
		join(model.Module, model.Module.task == model.Task.id).\
		join(model.Evaluation, model.Evaluation.module == model.Module.id).\
		filter(model.Evaluation.user == user_id).\
		group_by(model.Task).all() ]

def comment_thread(task_id, user_id):
	query = session.query(model.SolutionComment).filter(model.SolutionComment.task == task_id, model.SolutionComment.user == user_id).first()

	return query.thread if query is not None else None

# Vraci seznam automaticky opravovanych uloh, ktere maji plny pocet bodu.
# Pokud uloha nema automaticky opravovane moduly, vrati ji taky.
def autocorrected_full(user_id):
	q = session.query(model.Task.id.label('task_id'), func.count(distinct(model.Module.id)).label('mod_cnt')).\
		join(model.Module, model.Module.task == model.Task.id).\
		filter(model.Module.bonus == False).group_by(model.Task)

	max_modules_count = q.subquery()

	real_modules_count = q.join(model.Evaluation, model.Evaluation.module == model.Module.id).filter(model.Evaluation.user == user_id, or_(model.Module.autocorrect != True, model.Module.max_points == model.Evaluation.points)).subquery()

	return [ r for (r, ) in session.query(model.Task.id).\
		join(max_modules_count, model.Task.id == max_modules_count.c.task_id).\
		join(real_modules_count, model.Task.id == real_modules_count.c.task_id).\
		filter(max_modules_count.c.mod_cnt == real_modules_count.c.mod_cnt).all() ]

# Argumenty None slouzi k tomu, aby se usetrily SQL pozadavky:
#  pri hromadnem ziskavani stavu je mozne je vyplnit a pocet SQL dotazu bude mensi
#  Pokud jsou None, potrebne informace se zjisti z databaze.
def status(task, user, adeadline=None, fsubmitted=None, wave=None, corr=None, acfull=None):
	if not wave:
		wave = session.query(model.Wave).get(task.wave)

	# pokud neni prihlasen zadny uzivatel, otevreme jen ulohu bez prerekvizit
	# = prvvni uloha
	if user is None or user.id is None:
		return TaskStatus.BASE if task.prerequisite is None and wave.public else TaskStatus.LOCKED

	# pokud uloha neni v otevrene vlne, je LOCKED
	# vyjimkou jsou uzivatele s rolemi 'org' a 'admin', tem se zobrazuji vsechny ulohu
	if not wave.public and not user.role in ('org', 'admin'):
		return TaskStatus.LOCKED

	if corr is None: corr = task.id in corrected(user.id)
	if acfull is None: acfull = task.id in autocorrected_full(user.id)

	# Pokud je uloha opravena, je DONE.
	# Uloha neni DONE dokud nemaji vsechny automaticky opravovane moduly plny pocet bodu.
	if corr and acfull:
		return TaskStatus.DONE

	if not fsubmitted:
		fsubmitted = fully_submitted(user.id)

	# Pokud je uloha odevzdana a jeste neopravena, je CORRECTING
	if task.id in fsubmitted:
		return TaskStatus.CORRECTING

	if not adeadline:
		adeadline = after_deadline()

	# pokud je po deadline, zadani ulohy se otevira vsem
	currently_active = adeadline | set(fully_submitted(user.id).keys())
	if task.id in currently_active:
		return TaskStatus.BASE

	# Pokud nenastal ani jeden z vyse zminenych pripadu, otevreme ulohu, pokud
	# jsou splneny prerekvizity
	return TaskStatus.BASE if util.PrerequisitiesEvaluator(task.prerequisite_obj, currently_active).evaluate() or user.role in ('org', 'admin') else TaskStatus.LOCKED

def solution_public(status, task, user):
	return status == TaskStatus.DONE or task.time_deadline < datetime.datetime.utcnow() or user.role in ('org', 'admin')

def time_published(task_id):
	return session.query(model.Wave.time_published).\
		join(model.Task, model.Task.wave == model.Wave.id).\
		filter(model.Task.id == task_id).scalar()

def to_json(task, user=None, adeadline=None, fsubmitted=None, wave=None, corr=None, acfull=None):
	max_points = sum([ module.max_points for module in task.modules ])
	tstatus = status(task, user, adeadline, fsubmitted, wave, corr, acfull)
	pict_base = task.picture_base if task.picture_base is not None else "/taskContent/" + str(task.id) + "/icon/"

	if not wave:
		wave = session.query(model.Wave).get(task.wave)

	return {
		'id': task.id,
		'title': task.title,
		'author': task.author,
		'details': task.id,
		'intro': task.intro,
		'max_score': format(sum([ module.max_points for module in task.modules if not module.bonus ]), '.1f'),
		'time_published': wave.time_published.isoformat(),
		'time_deadline': task.time_deadline.isoformat(),
		'state': tstatus,
		'prerequisities': [] if not task.prerequisite_obj else util.prerequisite.to_json(task.prerequisite_obj),
		'picture_base': pict_base,
		'picture_suffix': '.svg',
		'wave': task.wave
	}

def details_to_json(task, user, status, achievements, best_scores, comment_thread=None):
	return {
		'id': task.id,
		'body': task.body,
		'thread': task.thread,
		'modules': [ module.id for module in task.modules ],
		'best_scores': [ best_score.User.id for best_score in best_scores ],
		'comment': comment_thread if task.evaluation_public else None,
		'solution': task.solution if solution_public(status, task, user) else None,
		'achievements': [ achievement.id for achievement in achievements ]
	}

def best_scores(task_id):
	per_modules = session.query(model.User.id.label('user_id'), \
			func.max(model.Evaluation.points).label('points')).\
			join(model.Evaluation, model.Evaluation.user == model.User.id).\
			filter(model.Module.task == task_id, 'points' is not None).\
			join(model.Module, model.Evaluation.module == model.Module.id).\
			join(model.Task, model.Task.id == model.Module.task).\
			filter(model.Task.evaluation_public).\
			group_by(model.Evaluation.module, model.User).subquery()

	return session.query(model.User, func.sum(per_modules.c.points).label('sum')).join(per_modules, per_modules.c.user_id == model.User.id).filter(model.User.role == 'participant').group_by(per_modules.c.user_id).order_by(desc('sum')).slice(0, 5).all()

def best_score_to_json(best_score):
	achievements = session.query(model.UserAchievement).filter(model.UserAchievement.user_id == best_score.User.id).all()

	return {
		'id': best_score.User.id,
		'user': best_score.User.id,
		'achievements': [ achievement.achievement_id for achievement in achievements ],
		'score': format(best_score.sum, '.1f')
	}
<|endoftext|>