<commit_before># Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements.  See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership.  The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License.  You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied.  See the License for the
# specific language governing permissions and limitations
# under the License.

''''''

import sys
import datetime as dt
import numpy as np

from mpl_toolkits.basemap import shiftgrid
from dateutil.relativedelta import relativedelta

def decode_time_values(dataset, time_var_name):
    ''' Decode NetCDF time values into Python datetime objects.

    :param dataset: The dataset from which time values should be extracted.
    :type dataset: netCDF4.Dataset
    :param time_var_name: The name of the time variable in dataset.
    :type time_var_name: :mod:`string`

    :returns: The list of converted datetime values.

    :raises ValueError: If the time units value couldn't be parsed, if the
        base time value couldn't be parsed, or if the time_var_name could not
        be found in the dataset.
    '''
    time_data = dataset.variables[time_var_name]
    time_format = time_data.units

    time_units = parse_time_units(time_format)
    time_base = parse_time_base(time_format)

    times = []
    arg = {}
    if time_units == 'months':
        # datetime.timedelta doesn't support a 'months' option. To remedy
        # this, a month == 30 days for our purposes.
        for time_val in time_data:
            times.append(time_base + relativedelta(months=int(time_val)))
    else:
        for time_val in time_data:
            arg[time_units] = time_val
            times.append(time_base + dt.timedelta(**arg))

    return times

def parse_time_units(time_format):
    ''' Parse units value from time units string.

    The only units that are supported are: seconds, minutes, hours, days,
        months, or years.

    :param time_format: The time data units string from the dataset
        being processed. The string should be of the format
        '<units> since <base time date>'
    :type time_format: :mod:`string`

    :returns: The unit substring from the time units string

    :raises ValueError: If the units present in the time units string doesn't
        match one of the supported unit value.
    '''
    for unit in ['seconds', 'minutes', 'hours', 'days', 'months', 'years']:
        if unit in time_format:
            return unit
    else:
        cur_frame = sys._getframe().f_code
        err = "{}.{}: Unable to parse valid units from {}".format(
            cur_frame.co_filename,
            cur_frame.co_name,
            time_format
        )
        raise ValueError(err)

def parse_time_base(time_format):
    ''' Parse time base object from the time units string.

    :param time_format: The time data units string from the dataset
        being processed. The string should be of the format
        '<units> since <base time date>'
    :type time_format: :mod:`string`

    :returns: The base time as a datetime object.

    :raises ValueError: When the base time string couldn't be parsed from the
        units time_format string or if the date string didn't match any of the
        expected formats.
    '''
    base_time_string = parse_base_time_string(time_format)

    time_format = time_format.strip()

    possible_time_formats = [
        '%Y:%m:%d %H:%M:%S', '%Y-%m-%d %H-%M-%S', '%Y/%m/%d %H/%M/%S',
        '%Y-%m-%d %H:%M:%S', '%Y/%m/%d %H:%M:%S', '%Y%m%d %H:%M:%S',
        '%Y%m%d%H%M%S', '%Y-%m-%d-%H-%M-%S', '%Y/%m/%d/%H/%M/%S',
        '%Y:%m:%d:%H:%M:%S', '%Y-%m-%d-%H:%M:%S', '%Y-%m-%d %H:%M:%S',
        '%Y/%m/%d%H:%M:%S', '%Y-%m-%d %H:%M', '%Y/%m/%d %H:%M',
        '%Y:%m:%d %H:%M', '%Y%m%d %H:%M', '%Y-%m-%d', '%Y/%m/%d',
        '%Y:%m:%d', '%Y%m%d', '%Y-%m-%d %H:%M:%S.%f', '%Y-%m-%d %H',
    ]

    # Attempt to match the base time string with a possible format parsing string.
    for time_format in possible_time_formats:
        try:
            stripped_time = dt.datetime.strptime(base_time_string, time_format)
            break
        except ValueError:
            # This exception means that the time format attempted was incorrect.
            # No need to report or raise this, simply try the next one!
            pass
    # If we got through the entire loop without a break, we couldn't parse the
    # date string with our known formats.
    else:
        cur_frame = sys._getframe().f_code
        err = "{}.{}: Unable to parse valid date from {}".format(
            cur_frame.co_filename,
            cur_frame.co_name,
            base_time_string
        )

        raise ValueError(err)

    return stripped_time

def parse_base_time_string(time_format):
    ''' Retrieve base time string from time data units information.

    :param time_format: The time data units string from the dataset
        being processed. The string should be of the format
        '<units> since <base time date>'
    :type time_format: :mod:`string`

    :returns: The base time string split out of the time units information.

    :raises ValueError: When the time_format parameter is malformed.
    '''
    if 'since' not in time_format:
        cur_frame = sys._getframe().f_code
        err = "{}.{}: Invalid time_format value {} given".format(
            cur_frame.co_filename,
            cur_frame.co_name,
            time_format
        )

        raise ValueError(err)

    return time_format.split('since')[1].strip()

def normalize_lat_lon_values(lats, lons, values):
    ''' Normalize lat/lon values

    Ensure that lat/lon values are within [-180, 180)/[-90, 90) as well
    as sorted. If the values are off the grid they are shifted into the
    expected range.

    :param lats: A 1D numpy array of sorted lat values.
    :type lats: :class:`numpy.ndarray`
    :param lons: A 1D numpy array of sorted lon values.
    :type lons: :class:`numpy.ndarray`
    :param values: A 3D array of data values.

    :returns: A :func:`tuple` of the form (adjust_lats, adjusted_lons, adjusted_values)

    :raises ValueError: If the lat/lon values are not sorted.
    '''
    # Avoid unnecessary shifting if all lons are higher than 180
    if lons.min() > 180:
        lons -= 360

    # Make sure lats and lons are monotonically increasing
    lats_decreasing = np.diff(lats) < 0
    lons_decreasing = np.diff(lons) < 0

    # If all values are decreasing then they just need to be reversed
    lats_reversed, lons_reversed = lats_decreasing.all(), lons_decreasing.all()

    # If the lat values are unsorted then raise an exception
    if not lats_reversed and lats_decreasing.any():
        raise ValueError('Latitudes must be sorted.')

    # Perform same checks now for lons
    if not lons_reversed and lons_decreasing.any():
        raise ValueError('Longitudes must be sorted.')

    # Also check if lons go from [0, 360), and convert to [-180, 180)
    # if necessary
    lons_shifted = lons.max() > 180
    lats_out, lons_out, data_out = lats[:], lons[:], values[:]
    # Now correct data if latlon grid needs to be shifted
    if lats_reversed:
        lats_out = lats_out[::-1]
        data_out = data_out[..., ::-1, :]

    if lons_reversed:
        lons_out = lons_out[::-1]
        data_out = data_out[..., ::-1]

    if lons_shifted:
        data_out, lons_out = shiftgrid(180, data_out, lons_out, start=False)

    return lats_out, lons_out, data_out

def reshape_monthly_to_annually(dataset):
    ''' Reshape monthly binned dataset to annual bins.

    Reshape a monthly binned dataset's 3D value array with shape
    (num_months, num_lats, num_lons) to a 4D array with shape
    (num_years, 12, num_lats, num_lons). This causes the data to be binned
    annually while retaining its original shape.

    It is assumed that the number of months in the dataset is evenly
    divisible by 12. If it is not you will receive error due to
    an invalid shape.

    Example change of a dataset's shape:
    (24, 90, 180) -> (2, 12, 90, 180)

    :param dataset: Dataset object with full-year format
    :type dataset: :class:`dataset.Dataset`

    :returns: Dataset values array with shape (num_year, 12, num_lat, num_lon)
    :rtype: :class:`numpy.ndarray`
    '''

    values = dataset.values[:]
    data_shape = values.shape
    num_total_month = data_shape[0]
    num_year = num_total_month / 12
    num_month = 12
    year_month_shape = num_year, num_month
    lat_lon_shape = data_shape[1:]
    # Make new shape (num_year, 12, num_lats, num_lons)
    new_shape = tuple(year_month_shape + lat_lon_shape)
    # Reshape data with new shape
    values.shape = new_shape

    return values

def calc_climatology_year(dataset):
    ''' Calculate climatology of dataset's values for each year

    :param dataset: Monthly binned Dataset object with an evenly divisible
        number of months.
    :type dataset: :class:`dataset.Dataset`

    :returns: Mean values for each year (annual_mean) and mean values for all
        years (total_mean)
    :rtype: A :func:`tuple` of two :class:`numpy.ndarray`

    :raise ValueError: If the number of monthly bins is not evenly divisible
        by 12.
    '''

    values_shape = dataset.values.shape
    time_shape = values_shape[0]
    if time_shape % 12:
        raise ValueError('The dataset should be in full-time format.')
    else:
        # Get values reshaped to (num_year, 12, num_lats, num_lons)
        values = reshape_monthly_to_annually(dataset)
        # Calculate mean values over year (num_year, num_lats, num_lons)
        annually_mean = values.mean(axis=1)
        # Calculate mean values over all years (num_lats, num_lons)
        total_mean = annually_mean.mean(axis=0)

    return annually_mean, total_mean

def calc_climatology_season(month_start, month_end, dataset):
    ''' Calculate seasonal mean and time series for given months.

    :param month_start: An integer for beginning month (Jan=1)
    :type month_start: :class:`int`

    :param month_end: An integer for ending month (Jan=1)
    :type month_end: :class:`int`

    :param dataset: Dataset object with full-year format
    :type dataset: :class:`dataset.Dataset`

    :returns:
        t_series - monthly average over the given season
        means - mean over the entire season
    :rtype: A :func:`tuple` of two :class:`numpy.ndarray`
    '''

    if month_start > month_end:
        # Offset the original array so that the the first month
        # becomes month_start, note that this cuts off the first year of data
        offset = slice(month_start - 1, month_start - 13)
        reshape_data = reshape_monthly_to_annually(dataset[offset])
        month_index = slice(0, 13 - month_start + month_end)
    else:
        # Since month_start <= month_end, just take a slice containing those months
        reshape_data = reshape_monthly_to_annually(dataset)
        month_index = slice(month_start - 1, month_end)

    t_series = reshape_data[:, month_index].mean(axis=1)
    means = t_series.mean(axis=0)
    return t_series, means

def calc_climatology_monthly(dataset):
    ''' Calculate monthly mean values for a dataset.

    :param dataset: Monthly binned Dataset object with the number of months
        divisible by 12
    :type dataset: :class:`dataset.Dataset`

    :returns: Mean values for each month of the year
    :rtype: A 3D :class:`numpy.ndarray` of shape (12, num_lats, num_lons)

    :raise ValueError: If the number of monthly bins is not divisible by 12
    '''

    if dataset.values.shape[0] % 12:
        error = (
            "The length of the time axis in the values array should be "
            "divisible by 12."
        )
        raise ValueError(error)
    else:
        return reshape_monthly_to_annually(dataset).mean(axis=0)

<commit_msg>- Update the times array of the DS object when monthly climo is calculated<commit_after># Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements.  See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership.  The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License.  You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied.  See the License for the
# specific language governing permissions and limitations
# under the License.

''''''

import sys
import datetime as dt
import numpy as np
import datetime

from mpl_toolkits.basemap import shiftgrid
from dateutil.relativedelta import relativedelta

def decode_time_values(dataset, time_var_name):
    ''' Decode NetCDF time values into Python datetime objects.

    :param dataset: The dataset from which time values should be extracted.
    :type dataset: netCDF4.Dataset
    :param time_var_name: The name of the time variable in dataset.
    :type time_var_name: :mod:`string`

    :returns: The list of converted datetime values.

    :raises ValueError: If the time units value couldn't be parsed, if the
        base time value couldn't be parsed, or if the time_var_name could not
        be found in the dataset.
    '''
    time_data = dataset.variables[time_var_name]
    time_format = time_data.units

    time_units = parse_time_units(time_format)
    time_base = parse_time_base(time_format)

    times = []
    arg = {}
    if time_units == 'months':
        # datetime.timedelta doesn't support a 'months' option. To remedy
        # this, a month == 30 days for our purposes.
        for time_val in time_data:
            times.append(time_base + relativedelta(months=int(time_val)))
    else:
        for time_val in time_data:
            arg[time_units] = time_val
            times.append(time_base + dt.timedelta(**arg))

    return times

def parse_time_units(time_format):
    ''' Parse units value from time units string.

    The only units that are supported are: seconds, minutes, hours, days,
        months, or years.

    :param time_format: The time data units string from the dataset
        being processed. The string should be of the format
        '<units> since <base time date>'
    :type time_format: :mod:`string`

    :returns: The unit substring from the time units string

    :raises ValueError: If the units present in the time units string doesn't
        match one of the supported unit value.
    '''
    for unit in ['seconds', 'minutes', 'hours', 'days', 'months', 'years']:
        if unit in time_format:
            return unit
    else:
        cur_frame = sys._getframe().f_code
        err = "{}.{}: Unable to parse valid units from {}".format(
            cur_frame.co_filename,
            cur_frame.co_name,
            time_format
        )
        raise ValueError(err)

def parse_time_base(time_format):
    ''' Parse time base object from the time units string.

    :param time_format: The time data units string from the dataset
        being processed. The string should be of the format
        '<units> since <base time date>'
    :type time_format: :mod:`string`

    :returns: The base time as a datetime object.

    :raises ValueError: When the base time string couldn't be parsed from the
        units time_format string or if the date string didn't match any of the
        expected formats.
    '''
    base_time_string = parse_base_time_string(time_format)

    time_format = time_format.strip()

    possible_time_formats = [
        '%Y:%m:%d %H:%M:%S', '%Y-%m-%d %H-%M-%S', '%Y/%m/%d %H/%M/%S',
        '%Y-%m-%d %H:%M:%S', '%Y/%m/%d %H:%M:%S', '%Y%m%d %H:%M:%S',
        '%Y%m%d%H%M%S', '%Y-%m-%d-%H-%M-%S', '%Y/%m/%d/%H/%M/%S',
        '%Y:%m:%d:%H:%M:%S', '%Y-%m-%d-%H:%M:%S', '%Y-%m-%d %H:%M:%S',
        '%Y/%m/%d%H:%M:%S', '%Y-%m-%d %H:%M', '%Y/%m/%d %H:%M',
        '%Y:%m:%d %H:%M', '%Y%m%d %H:%M', '%Y-%m-%d', '%Y/%m/%d',
        '%Y:%m:%d', '%Y%m%d', '%Y-%m-%d %H:%M:%S.%f', '%Y-%m-%d %H',
    ]

    # Attempt to match the base time string with a possible format parsing string.
    for time_format in possible_time_formats:
        try:
            stripped_time = dt.datetime.strptime(base_time_string, time_format)
            break
        except ValueError:
            # This exception means that the time format attempted was incorrect.
            # No need to report or raise this, simply try the next one!
            pass
    # If we got through the entire loop without a break, we couldn't parse the
    # date string with our known formats.
    else:
        cur_frame = sys._getframe().f_code
        err = "{}.{}: Unable to parse valid date from {}".format(
            cur_frame.co_filename,
            cur_frame.co_name,
            base_time_string
        )

        raise ValueError(err)

    return stripped_time

def parse_base_time_string(time_format):
    ''' Retrieve base time string from time data units information.

    :param time_format: The time data units string from the dataset
        being processed. The string should be of the format
        '<units> since <base time date>'
    :type time_format: :mod:`string`

    :returns: The base time string split out of the time units information.

    :raises ValueError: When the time_format parameter is malformed.
    '''
    if 'since' not in time_format:
        cur_frame = sys._getframe().f_code
        err = "{}.{}: Invalid time_format value {} given".format(
            cur_frame.co_filename,
            cur_frame.co_name,
            time_format
        )

        raise ValueError(err)

    return time_format.split('since')[1].strip()

def normalize_lat_lon_values(lats, lons, values):
    ''' Normalize lat/lon values

    Ensure that lat/lon values are within [-180, 180)/[-90, 90) as well
    as sorted. If the values are off the grid they are shifted into the
    expected range.

    :param lats: A 1D numpy array of sorted lat values.
    :type lats: :class:`numpy.ndarray`
    :param lons: A 1D numpy array of sorted lon values.
    :type lons: :class:`numpy.ndarray`
    :param values: A 3D array of data values.

    :returns: A :func:`tuple` of the form (adjust_lats, adjusted_lons, adjusted_values)

    :raises ValueError: If the lat/lon values are not sorted.
    '''
    # Avoid unnecessary shifting if all lons are higher than 180
    if lons.min() > 180:
        lons -= 360

    # Make sure lats and lons are monotonically increasing
    lats_decreasing = np.diff(lats) < 0
    lons_decreasing = np.diff(lons) < 0

    # If all values are decreasing then they just need to be reversed
    lats_reversed, lons_reversed = lats_decreasing.all(), lons_decreasing.all()

    # If the lat values are unsorted then raise an exception
    if not lats_reversed and lats_decreasing.any():
        raise ValueError('Latitudes must be sorted.')

    # Perform same checks now for lons
    if not lons_reversed and lons_decreasing.any():
        raise ValueError('Longitudes must be sorted.')

    # Also check if lons go from [0, 360), and convert to [-180, 180)
    # if necessary
    lons_shifted = lons.max() > 180
    lats_out, lons_out, data_out = lats[:], lons[:], values[:]
    # Now correct data if latlon grid needs to be shifted
    if lats_reversed:
        lats_out = lats_out[::-1]
        data_out = data_out[..., ::-1, :]

    if lons_reversed:
        lons_out = lons_out[::-1]
        data_out = data_out[..., ::-1]

    if lons_shifted:
        data_out, lons_out = shiftgrid(180, data_out, lons_out, start=False)

    return lats_out, lons_out, data_out

def reshape_monthly_to_annually(dataset):
    ''' Reshape monthly binned dataset to annual bins.

    Reshape a monthly binned dataset's 3D value array with shape
    (num_months, num_lats, num_lons) to a 4D array with shape
    (num_years, 12, num_lats, num_lons). This causes the data to be binned
    annually while retaining its original shape.

    It is assumed that the number of months in the dataset is evenly
    divisible by 12. If it is not you will receive error due to
    an invalid shape.

    Example change of a dataset's shape:
    (24, 90, 180) -> (2, 12, 90, 180)

    :param dataset: Dataset object with full-year format
    :type dataset: :class:`dataset.Dataset`

    :returns: Dataset values array with shape (num_year, 12, num_lat, num_lon)
    '''

    values = dataset.values[:]
    data_shape = values.shape
    num_total_month = data_shape[0]
    num_year = num_total_month / 12
    num_month = 12
    year_month_shape = num_year, num_month
    lat_lon_shape = data_shape[1:]
    # Make new shape (num_year, 12, num_lats, num_lons)
    new_shape = tuple(year_month_shape + lat_lon_shape)
    # Reshape data with new shape
    values.shape = new_shape

    return values

def calc_climatology_year(dataset):
    ''' Calculate climatology of dataset's values for each year

    :param dataset: Monthly binned Dataset object with an evenly divisible
        number of months.
    :type dataset: :class:`dataset.Dataset`

    :returns: Mean values for each year (annual_mean) and mean values for all
        years (total_mean)

    :raise ValueError: If the number of monthly bins is not evenly divisible
        by 12.
    '''

    values_shape = dataset.values.shape
    time_shape = values_shape[0]
    if time_shape % 12:
        raise ValueError('The dataset should be in full-time format.')
    else:
        # Get values reshaped to (num_year, 12, num_lats, num_lons)
        values = reshape_monthly_to_annually(dataset)
        # Calculate mean values over year (num_year, num_lats, num_lons)
        annually_mean = values.mean(axis=1)
        # Calculate mean values over all years (num_lats, num_lons)
        total_mean = annually_mean.mean(axis=0)

    return annually_mean, total_mean

def calc_climatology_season(month_start, month_end, dataset):
    ''' Calculate seasonal mean and time series for given months.

    :param month_start: An integer for beginning month (Jan=1)
    :type month_start: :class:`int`

    :param month_end: An integer for ending month (Jan=1)
    :type month_end: :class:`int`

    :param dataset: Dataset object with full-year format
    :type dataset: :class:`dataset.Dataset`

    :returns: t_series - monthly average over the given season
        means - mean over the entire season

    '''

    if month_start > month_end:
        # Offset the original array so that the the first month
        # becomes month_start, note that this cuts off the first year of data
        offset = slice(month_start - 1, month_start - 13)
        reshape_data = reshape_monthly_to_annually(dataset[offset])
        month_index = slice(0, 13 - month_start + month_end)
    else:
        # Since month_start <= month_end, just take a slice containing those months
        reshape_data = reshape_monthly_to_annually(dataset)
        month_index = slice(month_start - 1, month_end)

    t_series = reshape_data[:, month_index].mean(axis=1)
    means = t_series.mean(axis=0)

    return t_series, means

def calc_climatology_monthly(dataset):
    ''' Calculate monthly mean values for a dataset.
    Follow COARDS climo stats calculation, the year can be given as 0
    but the min year allowed in Python is 1
    http://www.cgd.ucar.edu/cms/eaton/netcdf/CF-20010629.htm#climatology

    :param dataset: Monthly binned Dataset object with the number of months
        divisible by 12
    :type dataset: :class:`dataset.Dataset`

    :returns: Mean values for each month of the year of shape (12, num_lats, num_lons)
              and times array of datetime objects of length 12

    :raise ValueError: If the number of monthly bins is not divisible by 12
    '''

    if dataset.values.shape[0] % 12:
        error = (
            "The length of the time axis in the values array should be "
            "divisible by 12."
        )
        raise ValueError(error)
    else:
        values = reshape_monthly_to_annually(dataset).mean(axis=0)

        # A year can commence from any month
        first_month = dataset.times[0].month
        times = np.array([datetime.datetime(1, first_month, 1) + relativedelta(months = x)
                for x in range(12)])
        return values, times

<|endoftext|>


<commit_before>from app.readers import mzidplus as readers
from app.readers import tsv as tsvreader

def merge_mzidtsvs(fns, header):
    for fn in fns:
        if header != tsvreader.get_tsv_header(fn):
            raise RuntimeError('Headers of TSV files to concatenate are '
                               'not identical')
    return readers.generate_tsv_lines_multifile(fns)

def get_percoline(specresult, namespace, line, multipsm, seqdb):
    """Extracts percolator data from specresult and returns a dict"""
    out = line
    out.update({'rank': None})
    try:
        xmlns = '{%s}' % namespace['xmlns']
    except TypeError:
        xmlns = ''
    if multipsm is True:
        pass  # FIXME support later
        # loop through psms in specresult
        # check line sequence (without mods) in seqdb with psm
        # get percodata,
        # percoline = [line-with-correct-rank]
    else:  # only the first element
        perco = readers.get_specidentitem_percolator_data(
            specresult.find('{0}SpectrumIdentificationItem'.format(xmlns)),
            namespace)
    out.update(perco)
    return out

def get_specresult_data(specresults, id_fnlookup):
    specresult = next(specresults)
    scannr = readers.get_specresult_scan_nr(specresult)
    mzmlid = readers.get_specresult_mzml_id(specresult)
    return specresult, {'scan': scannr, 'fn': id_fnlookup[mzmlid]}

def add_percolator_to_mzidtsv(mzidfn, tsvfn, multipsm, header, seqdb=None):
    """Takes a MSGF+ tsv and corresponding mzId, adds percolatordata
    to tsv lines. Generator yields the lines. Multiple PSMs per scan
    can be delivered, in which case rank is also reported.
    """
    namespace = readers.get_mzid_namespace(mzidfn)
    specfnids = readers.get_mzid_specfile_ids(mzidfn, namespace)
    specresults = readers.mzid_spec_result_generator(mzidfn, namespace)
    with open(tsvfn) as mzidfp:
        # skip header
        oldheader = next(mzidfp)
        # multiple lines can belong to one specresult, so we use a nested
        # for/while-true-break construction.
        writelines = []
        specresult, specdata = get_specresult_data(specresults, specfnids)
        for line in mzidfp:
            line = {x: y for x, y in zip(oldheader, line.split('\t'))}
            while True:
                if writelines and not multipsm:
                    # Only keep best ranking psm
                    # FIXME we assume best ranking is first line. Fix this in
                    # future
                    yield writelines
                    writelines = []
                    break
                # FIXME get header names instead of positions!
                if line[oldheader[2]] == specdata['scan'] \
                   and line[oldheader[0]] == specdata['fn']:
                    # add percolator stuff to line
                    outline = get_percoline(specresult, namespace, line,
                                            multipsm, seqdb)
                    writelines.append(outline)
                    break  # goes to next line in tsv
                else:
                    yield writelines
                    writelines = []
                    specresult, specdata = get_specresult_data(specresults,
                                                               specfnids)
        # write last line
        yield writelines

def get_header_with_percolator(fn, multipsm=False):
    header = tsvreader.get_header_from_mzidtsv(fn)
    if multipsm is True:
        # FIXME should this be here???
        header.append('rank')
    header.extend(readers.PERCO_HEADER)
    return header
<commit_msg>Strip lines before parsing them<commit_after>from app.readers import mzidplus as readers
from app.readers import tsv as tsvreader

def merge_mzidtsvs(fns, header):
    for fn in fns:
        if header != tsvreader.get_tsv_header(fn):
            raise RuntimeError('Headers of TSV files to concatenate are '
                               'not identical')
    return readers.generate_tsv_lines_multifile(fns)

def get_percoline(specresult, namespace, line, multipsm, seqdb):
    """Extracts percolator data from specresult and returns a dict"""
    out = line
    out.update({'rank': None})
    try:
        xmlns = '{%s}' % namespace['xmlns']
    except TypeError:
        xmlns = ''
    if multipsm is True:
        pass  # FIXME support later
        # loop through psms in specresult
        # check line sequence (without mods) in seqdb with psm
        # get percodata,
        # percoline = [line-with-correct-rank]
    else:  # only the first element
        perco = readers.get_specidentitem_percolator_data(
            specresult.find('{0}SpectrumIdentificationItem'.format(xmlns)),
            namespace)
    out.update(perco)
    return out

def get_specresult_data(specresults, id_fnlookup):
    specresult = next(specresults)
    scannr = readers.get_specresult_scan_nr(specresult)
    mzmlid = readers.get_specresult_mzml_id(specresult)
    return specresult, {'scan': scannr, 'fn': id_fnlookup[mzmlid]}

def add_percolator_to_mzidtsv(mzidfn, tsvfn, multipsm, header, seqdb=None):
    """Takes a MSGF+ tsv and corresponding mzId, adds percolatordata
    to tsv lines. Generator yields the lines. Multiple PSMs per scan
    can be delivered, in which case rank is also reported.
    """
    namespace = readers.get_mzid_namespace(mzidfn)
    specfnids = readers.get_mzid_specfile_ids(mzidfn, namespace)
    specresults = readers.mzid_spec_result_generator(mzidfn, namespace)
    with open(tsvfn) as mzidfp:
        # skip header
        oldheader = next(mzidfp).strip().split('\t')
        # multiple lines can belong to one specresult, so we use a nested
        # for/while-true-break construction.
        writelines = []
        specresult, specdata = get_specresult_data(specresults, specfnids)
        for line in mzidfp:
            line = {x: y for x, y in zip(oldheader, line.strip().split('\t'))}
            while True:
                if writelines and not multipsm:
                    # Only keep best ranking psm
                    # FIXME we assume best ranking is first line. Fix this in
                    # future
                    yield writelines
                    writelines = []
                    break
                # FIXME get header names instead of positions!
                if line[oldheader[2]] == specdata['scan'] \
                   and line[oldheader[0]] == specdata['fn']:
                    # add percolator stuff to line
                    outline = get_percoline(specresult, namespace, line,
                                            multipsm, seqdb)
                    writelines.append(outline)
                    break  # goes to next line in tsv
                else:
                    yield writelines
                    writelines = []
                    specresult, specdata = get_specresult_data(specresults,
                                                               specfnids)
        # write last line
        yield writelines

def get_header_with_percolator(fn, multipsm=False):
    header = tsvreader.get_header_from_mzidtsv(fn)
    if multipsm is True:
        # FIXME should this be here???
        header.append('rank')
    header.extend(readers.PERCO_HEADER)
    return header
<|endoftext|>


<commit_before><commit_msg>fix call to firm.get_K<commit_after><|endoftext|>


<commit_before># -*- coding: utf-8 -*-
#
# Copyright (c) 2016 NORDUnet A/S
# All rights reserved.
#
#   Redistribution and use in source and binary forms, with or
#   without modification, are permitted provided that the following
#   conditions are met:
#
#     1. Redistributions of source code must retain the above copyright
#        notice, this list of conditions and the following disclaimer.
#     2. Redistributions in binary form must reproduce the above
#        copyright notice, this list of conditions and the following
#        disclaimer in the documentation and/or other materials provided
#        with the distribution.
#     3. Neither the name of the NORDUnet nor the names of its
#        contributors may be used to endorse or promote products derived
#        from this software without specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
# FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
# COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
# INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
# BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
# ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
# POSSIBILITY OF SUCH DAMAGE.
#
from __future__ import absolute_import

import requests
from flask import Blueprint, request, session, current_app, abort

from eduid_common.api.decorators import MarshalWith, UnmarshalWith
from eduid_common.api.schemas.base import FluxStandardAction
from eduid_webapp.signup.schemas import RegisterEmailSchema, AccountCreatedResponse, EmailSchema
from eduid_webapp.signup.verifications import verify_recaptcha, send_verification_mail, verify_email_code
from eduid_webapp.signup.helpers import check_email_status, remove_users_with_mail_address, complete_registration
from eduid_webapp.signup.verifications import CodeDoesNotExist, AlreadyVerifiedException, ProofingLogFailure

signup_views = Blueprint('signup', __name__, url_prefix='', template_folder='templates')

@signup_views.route('/config', methods=['GET'])
@MarshalWith(FluxStandardAction)
def get_config():
    tou_url = current_app.config.get('TOU_URL')
    try:
        r = requests.get(tou_url)
        current_app.logger.debug('Response: {!r} with headers: {!r}'.format(r, r.headers))
        if r.status_code == 302:
            headers = {'Cookie': r.headers.get('Set-Cookie')}
            current_app.logger.debug('Headers: {!r}'.format(headers))
            r = requests.get(tou_url, headers=headers)
            current_app.logger.debug('2nd response: {!r} with headers: {!r}'.format(r, r.headers))
    except requests.exceptions.HTTPError as e:
        current_app.logger.debug('Problem getting config: {!r}'.format(e))
        abort(500)
    if r.status_code != 200:
        current_app.logger.debug('Problem getting config, '
                                 'response status: '
                                 '{!r}'.format(r.status_code))
        abort(500)
    return {
            'csrf_token': session.get_csrf_token(),
            'recaptcha_public_key': current_app.config.get('RECAPTCHA_PUBLIC_KEY'),
            'available_languages': current_app.config.get('AVAILABLE_LANGUAGES'),
            'debug': current_app.config.get('DEBUG'),
            'tous': r.json()['payload'],
            'dashboard_url': current_app.config.get('DASHBOARD_URL'),
            'reset_passwd_url': current_app.config.get('RESET_PASSWD_URL'),
            'students_link': current_app.config.get('STUDENTS_LINK'),
            'technicians_link': current_app.config.get('TECHNICIANS_LINK'),
            'staff_link': current_app.config.get('STAFF_LINK'),
            'faq_link': current_app.config.get('FAQ_LINK'),
            }

@signup_views.route('/trycaptcha', methods=['POST'])
@UnmarshalWith(RegisterEmailSchema)
@MarshalWith(AccountCreatedResponse)
def trycaptcha(email, recaptcha_response, tou_accepted):
    """
    Kantara requires a check for humanness even at level AL1.
    """
    if not tou_accepted:
        return {
                '_status': 'error',
                'message': 'signup.tou-not-accepted'
        }
    config = current_app.config
    remote_ip = request.remote_addr
    recaptcha_public_key = config.get('RECAPTCHA_PUBLIC_KEY', '')

    if recaptcha_public_key:
        recaptcha_private_key = config.get('RECAPTCHA_PRIVATE_KEY', '')
        recaptcha_verified = verify_recaptcha(recaptcha_private_key,
                                              recaptcha_response, remote_ip)
    else:
        # If recaptcha_public_key is not set recaptcha is disabled
        recaptcha_verified = True
        current_app.logger.info('CAPTCHA disabled')

    if recaptcha_verified:
        next = check_email_status(email)
        if next == 'new':
            # Workaround for failed earlier sync of user to userdb: Remove any signup_user with this e-mail address.
            remove_users_with_mail_address(email)
            send_verification_mail(email)
            return {
                'message': 'signup.registering-new',
                'next': next
            }
        elif next == 'resend-code':
            return {
                'next': next
            }
        elif next == 'address-used':
            return {
                '_status': 'error',
                'message': 'signup.registering-address-used',
                'next': next
            }
    return {
            '_status': 'error',
            'message': 'signup.recaptcha-not-verified'
    }

@signup_views.route('/resend-verification', methods=['POST'])
@UnmarshalWith(EmailSchema)
@MarshalWith(FluxStandardAction)
def resend_email_verification(email):
    """
    The user has not yet verified the email address.
    Send a verification message to the address so it can be verified.
    """
    current_app.logger.debug("Resend email confirmation to {!s}".format(email))
    send_verification_mail(email)

    return {'message': 'signup.verification-resent'}

@signup_views.route('/verify-link/<code>', methods=['GET'])
@MarshalWith(FluxStandardAction)
def verify_link(code):
    try:
        user = verify_email_code(code)
    except CodeDoesNotExist:
        return {
                '_status': 'error',
                'status': 'unknown-code',
                'message': 'signup.unknown-code'
                }
    except AlreadyVerifiedException:
        return {
                '_status': 'error',
                'status': 'already-verified',
                'message': 'signup.already-verified'
                }
    except ProofingLogFailure:
        return {
            '_status': 'error',
            'message': 'Temporary technical problems'
        }
    return complete_registration(user)
<commit_msg>log an error as an error<commit_after># -*- coding: utf-8 -*-
#
# Copyright (c) 2016 NORDUnet A/S
# All rights reserved.
#
#   Redistribution and use in source and binary forms, with or
#   without modification, are permitted provided that the following
#   conditions are met:
#
#     1. Redistributions of source code must retain the above copyright
#        notice, this list of conditions and the following disclaimer.
#     2. Redistributions in binary form must reproduce the above
#        copyright notice, this list of conditions and the following
#        disclaimer in the documentation and/or other materials provided
#        with the distribution.
#     3. Neither the name of the NORDUnet nor the names of its
#        contributors may be used to endorse or promote products derived
#        from this software without specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
# FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
# COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
# INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
# BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
# ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
# POSSIBILITY OF SUCH DAMAGE.
#
from __future__ import absolute_import

import requests
from flask import Blueprint, request, session, current_app, abort

from eduid_common.api.decorators import MarshalWith, UnmarshalWith
from eduid_common.api.schemas.base import FluxStandardAction
from eduid_webapp.signup.schemas import RegisterEmailSchema, AccountCreatedResponse, EmailSchema
from eduid_webapp.signup.verifications import verify_recaptcha, send_verification_mail, verify_email_code
from eduid_webapp.signup.helpers import check_email_status, remove_users_with_mail_address, complete_registration
from eduid_webapp.signup.verifications import CodeDoesNotExist, AlreadyVerifiedException, ProofingLogFailure

signup_views = Blueprint('signup', __name__, url_prefix='', template_folder='templates')

@signup_views.route('/config', methods=['GET'])
@MarshalWith(FluxStandardAction)
def get_config():
    tou_url = current_app.config.get('TOU_URL')
    try:
        r = requests.get(tou_url)
        current_app.logger.debug('Response: {!r} with headers: {!r}'.format(r, r.headers))
        if r.status_code == 302:
            headers = {'Cookie': r.headers.get('Set-Cookie')}
            current_app.logger.debug('Headers: {!r}'.format(headers))
            r = requests.get(tou_url, headers=headers)
            current_app.logger.debug('2nd response: {!r} with headers: {!r}'.format(r, r.headers))
    except requests.exceptions.HTTPError as e:
        current_app.logger.error('Problem getting tous from URL {!r}: {!r}'.format(tou_url, e))
        abort(500)
    if r.status_code != 200:
        current_app.logger.debug('Problem getting config, '
                                 'response status: '
                                 '{!r}'.format(r.status_code))
        abort(500)
    return {
            'csrf_token': session.get_csrf_token(),
            'recaptcha_public_key': current_app.config.get('RECAPTCHA_PUBLIC_KEY'),
            'available_languages': current_app.config.get('AVAILABLE_LANGUAGES'),
            'debug': current_app.config.get('DEBUG'),
            'tous': r.json()['payload'],
            'dashboard_url': current_app.config.get('DASHBOARD_URL'),
            'reset_passwd_url': current_app.config.get('RESET_PASSWD_URL'),
            'students_link': current_app.config.get('STUDENTS_LINK'),
            'technicians_link': current_app.config.get('TECHNICIANS_LINK'),
            'staff_link': current_app.config.get('STAFF_LINK'),
            'faq_link': current_app.config.get('FAQ_LINK'),
            }

@signup_views.route('/trycaptcha', methods=['POST'])
@UnmarshalWith(RegisterEmailSchema)
@MarshalWith(AccountCreatedResponse)
def trycaptcha(email, recaptcha_response, tou_accepted):
    """
    Kantara requires a check for humanness even at level AL1.
    """
    if not tou_accepted:
        return {
                '_status': 'error',
                'message': 'signup.tou-not-accepted'
        }
    config = current_app.config
    remote_ip = request.remote_addr
    recaptcha_public_key = config.get('RECAPTCHA_PUBLIC_KEY', '')

    if recaptcha_public_key:
        recaptcha_private_key = config.get('RECAPTCHA_PRIVATE_KEY', '')
        recaptcha_verified = verify_recaptcha(recaptcha_private_key,
                                              recaptcha_response, remote_ip)
    else:
        # If recaptcha_public_key is not set recaptcha is disabled
        recaptcha_verified = True
        current_app.logger.info('CAPTCHA disabled')

    if recaptcha_verified:
        next = check_email_status(email)
        if next == 'new':
            # Workaround for failed earlier sync of user to userdb: Remove any signup_user with this e-mail address.
            remove_users_with_mail_address(email)
            send_verification_mail(email)
            return {
                'message': 'signup.registering-new',
                'next': next
            }
        elif next == 'resend-code':
            return {
                'next': next
            }
        elif next == 'address-used':
            return {
                '_status': 'error',
                'message': 'signup.registering-address-used',
                'next': next
            }
    return {
            '_status': 'error',
            'message': 'signup.recaptcha-not-verified'
    }

@signup_views.route('/resend-verification', methods=['POST'])
@UnmarshalWith(EmailSchema)
@MarshalWith(FluxStandardAction)
def resend_email_verification(email):
    """
    The user has not yet verified the email address.
    Send a verification message to the address so it can be verified.
    """
    current_app.logger.debug("Resend email confirmation to {!s}".format(email))
    send_verification_mail(email)

    return {'message': 'signup.verification-resent'}

@signup_views.route('/verify-link/<code>', methods=['GET'])
@MarshalWith(FluxStandardAction)
def verify_link(code):
    try:
        user = verify_email_code(code)
    except CodeDoesNotExist:
        return {
                '_status': 'error',
                'status': 'unknown-code',
                'message': 'signup.unknown-code'
                }
    except AlreadyVerifiedException:
        return {
                '_status': 'error',
                'status': 'already-verified',
                'message': 'signup.already-verified'
                }
    except ProofingLogFailure:
        return {
            '_status': 'error',
            'message': 'Temporary technical problems'
        }
    return complete_registration(user)
<|endoftext|>


<commit_before>from warnings import warn
import json

from django.core.management.base import LabelCommand, CommandError

from pombola.core.models import Person

# Expected JSON is an array of entries:
#
# [
#     {
#         "name": "John Smith",
#         "bio": "Blah blah"
#     },
#     ....
# ]

class Command(LabelCommand):
    help = 'Set profiles (in summary field)'
    args = '<profile JSON file>'

    def handle_label(self,  input_filename, **options):

        input_entries = json.loads( open(input_filename, 'r').read() )

        for entry in input_entries:

            try:
                person = Person.objects.get(legal_name=entry['name'])
            except Person.DoesNotExist:
                warn("Could not find person matching '%s'" % entry['name'])
                continue

            print "Setting summary for '%s'" % person

            bio = entry['bio']
            bio = bio.replace('\n', '\n\n')
            person.summary = bio
            person.save()
<commit_msg>Added clarification notes.<commit_after>from warnings import warn
import json

from django.core.management.base import LabelCommand, CommandError

from pombola.core.models import Person

# Expected JSON is an array of entries:
#
# [
#     {
#         "name": "John Smith",
#         "bio": "Blah blah"
#     },
#     ....
# ]
#
# ``name`` should match the Person's ``legal_name`` and ``bio`` is stored in their summary field.

class Command(LabelCommand):
    help = 'Set profiles (in summary field)'
    args = '<profile JSON file>'

    def handle_label(self,  input_filename, **options):

        input_entries = json.loads( open(input_filename, 'r').read() )

        for entry in input_entries:

            try:
                person = Person.objects.get(legal_name=entry['name'])
            except Person.DoesNotExist:
                warn("Could not find person matching '%s'" % entry['name'])
                continue

            print "Setting summary for '%s'" % person

            bio = entry['bio']
            bio = bio.replace('\n', '\n\n')
            person.summary = bio
            person.save()
<|endoftext|>


<commit_before>#!/usr/bin/env python
# Copyright (c) 2012 The Chromium Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

"""Uploads files to Google Storage content addressed."""

import hashlib
import optparse
import os
import Queue
import re
import sys
import threading
import time

from download_from_google_storage import check_bucket_permissions
from download_from_google_storage import get_sha1
from download_from_google_storage import Gsutil
from download_from_google_storage import printer_worker

GSUTIL_DEFAULT_PATH = os.path.join(
    os.path.dirname(os.path.abspath(__file__)),
    'third_party', 'gsutil', 'gsutil')

USAGE_STRING = """%prog [options] target [target2 ...].
Target is the file intended to be uploaded to Google Storage.
If target is "-", then a list of files will be taken from standard input

This script will generate a file (original filename).sha1 containing the
sha1 sum of the uploaded file.
It is recommended that the .sha1 file is checked into the repository,
the original file removed from the repository, and a hook added to the
DEPS file to call download_from_google_storage.py.

Example usages
--------------

Scan the current directory and upload all files larger than 1MB:
find . -name .svn -prune -o -size +1000k -type f -print0 | %prog -0 -b bkt -
(Replace "bkt" with the name of a writable bucket.)
"""

def get_md5(filename):
  md5_calculator = hashlib.md5()
  with open(filename, 'rb') as f:
    while True:
      chunk = f.read(1024*1024)
      if not chunk:
        break
      md5_calculator.update(chunk)
  return md5_calculator.hexdigest()

def get_md5_cached(filename):
  """Don't calculate the MD5 if we can find a .md5 file."""
  # See if we can find an existing MD5 sum stored in a file.
  if os.path.exists('%s.md5' % filename):
    with open('%s.md5' % filename, 'rb') as f:
      md5_match = re.search('([a-z0-9]{32})', f.read())
      if md5_match:
        return md5_match.group(1)
  else:
    md5_hash = get_md5(filename)
    with open('%s.md5' % filename, 'wb') as f:
      f.write(md5_hash)
    return md5_hash

def _upload_worker(
    thread_num, upload_queue, base_url, gsutil, md5_lock, force,
    use_md5, stdout_queue, ret_codes):
  while True:
    filename, sha1_sum = upload_queue.get()
    if not filename:
      break
    file_url = '%s/%s' % (base_url, sha1_sum)
    if gsutil.check_call('ls', file_url)[0] == 0 and not force:
      # File exists, check MD5 hash.
      _, out, _ = gsutil.check_call('ls', '-L', file_url)
      etag_match = re.search('ETag:\s+([a-z0-9]{32})', out)
      if etag_match:
        remote_md5 = etag_match.group(1)
        # Calculate the MD5 checksum to match it to Google Storage's ETag.
        with md5_lock:
          if use_md5:
            local_md5 = get_md5_cached(filename)
          else:
            local_md5 = get_md5(filename)
        if local_md5 == remote_md5:
          stdout_queue.put(
              '%d> File %s already exists and MD5 matches, upload skipped' %
              (thread_num, filename))
          continue
    stdout_queue.put('%d> Uploading %s...' % (
        thread_num, filename))
    code, _, err = gsutil.check_call('cp', '-q', filename, file_url)
    if code != 0:
      ret_codes.put(
          (code,
           'Encountered error on uploading %s to %s\n%s' %
           (filename, file_url, err)))
      continue

def get_targets(args, parser, use_null_terminator):
  if not args:
    parser.error('Missing target.')

  if len(args) == 1 and args[0] == '-':
    # Take stdin as a newline or null seperated list of files.
    if use_null_terminator:
      return sys.stdin.read().split('\0')
    else:
      return sys.stdin.read().splitlines()
  else:
    return args

def upload_to_google_storage(
    input_filenames, base_url, gsutil, force,
    use_md5, num_threads, skip_hashing):
  # We only want one MD5 calculation happening at a time to avoid HD thrashing.
  md5_lock = threading.Lock()

  # Start up all the worker threads plus the printer thread.
  all_threads = []
  ret_codes = Queue.Queue()
  ret_codes.put((0, None))
  upload_queue = Queue.Queue()
  upload_timer = time.time()
  stdout_queue = Queue.Queue()
  printer_thread = threading.Thread(target=printer_worker, args=[stdout_queue])
  printer_thread.daemon = True
  printer_thread.start()
  for thread_num in range(num_threads):
    t = threading.Thread(
        target=_upload_worker,
        args=[thread_num, upload_queue, base_url, gsutil, md5_lock,
              force, use_md5, stdout_queue, ret_codes])
    t.daemon = True
    t.start()
    all_threads.append(t)

  # We want to hash everything in a single thread since its faster.
  # The bottleneck is in disk IO, not CPU.
  hashing_start = time.time()
  for filename in input_filenames:
    if not os.path.exists(filename):
      stdout_queue.put('Main> Error: %s not found, skipping.' % filename)
      continue
    if os.path.exists('%s.sha1' % filename) and skip_hashing:
      stdout_queue.put(
          'Main> Found hash for %s, sha1 calculation skipped.' % filename)
      with open(filename + '.sha1', 'rb') as f:
        sha1_file = f.read(1024)
      if not re.match('^([a-z0-9]{40})$', sha1_file):
        print >> sys.stderr, 'Invalid sha1 hash file %s.sha1' % filename
        return 1
      upload_queue.put((filename, sha1_file))
      continue
    stdout_queue.put('Main> Calculating hash for %s...' % filename)
    sha1_sum = get_sha1(filename)
    with open(filename + '.sha1', 'wb') as f:
      f.write(sha1_sum)
    stdout_queue.put('Main> Done calculating hash for %s.' % filename)
    upload_queue.put((filename, sha1_sum))
  hashing_duration = time.time() - hashing_start

  # Wait for everything to finish.
  for _ in all_threads:
    upload_queue.put((None, None))  # To mark the end of the work queue.
  for t in all_threads:
    t.join()
  stdout_queue.put(None)
  printer_thread.join()

  # Print timing information.
  print 'Hashing %s files took %1f seconds' % (
      len(input_filenames), hashing_duration)
  print 'Uploading took %1f seconds' % (time.time() - upload_timer)

  # See if we ran into any errors.
  max_ret_code = 0
  for ret_code, message in ret_codes.queue:
    max_ret_code = max(ret_code, max_ret_code)
    if message:
      print >> sys.stderr, message

  if not max_ret_code:
    print 'Success!'

  return max_ret_code

def main(args):
  parser = optparse.OptionParser(USAGE_STRING)
  parser.add_option('-b', '--bucket',
                    help='Google Storage bucket to upload to.')
  parser.add_option('-e', '--boto', help='Specify a custom boto file.')
  parser.add_option('-f', '--force', action='store_true',
                    help='Force upload even if remote file exists.')
  parser.add_option('-g', '--gsutil_path', default=GSUTIL_DEFAULT_PATH,
                    help='Path to the gsutil script.')
  parser.add_option('-m', '--use_md5', action='store_true',
                    help='Generate MD5 files when scanning, and don\'t check '
                    'the MD5 checksum if a .md5 file is found.')
  parser.add_option('-t', '--num_threads', default=1, type='int',
                    help='Number of uploader threads to run.')
  parser.add_option('-s', '--skip_hashing', action='store_true',
                    help='Skip hashing if .sha1 file exists.')
  parser.add_option('-0', '--use_null_terminator', action='store_true',
                    help='Use \\0 instead of \\n when parsing '
                    'the file list from stdin.  This is useful if the input '
                    'is coming from "find ... -print0".')
  (options, args) = parser.parse_args()

  # Enumerate our inputs.
  input_filenames = get_targets(args, parser, options.use_null_terminator)

  # Make sure we can find a working instance of gsutil.
  if os.path.exists(GSUTIL_DEFAULT_PATH):
    gsutil = Gsutil(GSUTIL_DEFAULT_PATH)
  else:
    gsutil = None
    for path in os.environ["PATH"].split(os.pathsep):
      if os.path.exists(path) and 'gsutil' in os.listdir(path):
        gsutil = Gsutil(os.path.join(path, 'gsutil'))
    if not gsutil:
      parser.error('gsutil not found in %s, bad depot_tools checkout?' %
                   GSUTIL_DEFAULT_PATH)

  # Check we have a valid bucket with valid permissions.
  base_url, code = check_bucket_permissions(options.bucket, gsutil)
  if code:
    return code

  return upload_to_google_storage(
      input_filenames, base_url, gsutil, options.force, options.use_md5,
      options.num_threads, options.skip_hashing)

if __name__ == '__main__':
  sys.exit(main(sys.argv))<commit_msg>pass boto_path to Gsutil<commit_after>#!/usr/bin/env python
# Copyright (c) 2012 The Chromium Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

"""Uploads files to Google Storage content addressed."""

import hashlib
import optparse
import os
import Queue
import re
import sys
import threading
import time

from download_from_google_storage import check_bucket_permissions
from download_from_google_storage import get_sha1
from download_from_google_storage import Gsutil
from download_from_google_storage import printer_worker

GSUTIL_DEFAULT_PATH = os.path.join(
    os.path.dirname(os.path.abspath(__file__)),
    'third_party', 'gsutil', 'gsutil')

USAGE_STRING = """%prog [options] target [target2 ...].
Target is the file intended to be uploaded to Google Storage.
If target is "-", then a list of files will be taken from standard input

This script will generate a file (original filename).sha1 containing the
sha1 sum of the uploaded file.
It is recommended that the .sha1 file is checked into the repository,
the original file removed from the repository, and a hook added to the
DEPS file to call download_from_google_storage.py.

Example usages
--------------

Scan the current directory and upload all files larger than 1MB:
find . -name .svn -prune -o -size +1000k -type f -print0 | %prog -0 -b bkt -
(Replace "bkt" with the name of a writable bucket.)
"""

def get_md5(filename):
  md5_calculator = hashlib.md5()
  with open(filename, 'rb') as f:
    while True:
      chunk = f.read(1024*1024)
      if not chunk:
        break
      md5_calculator.update(chunk)
  return md5_calculator.hexdigest()

def get_md5_cached(filename):
  """Don't calculate the MD5 if we can find a .md5 file."""
  # See if we can find an existing MD5 sum stored in a file.
  if os.path.exists('%s.md5' % filename):
    with open('%s.md5' % filename, 'rb') as f:
      md5_match = re.search('([a-z0-9]{32})', f.read())
      if md5_match:
        return md5_match.group(1)
  else:
    md5_hash = get_md5(filename)
    with open('%s.md5' % filename, 'wb') as f:
      f.write(md5_hash)
    return md5_hash

def _upload_worker(
    thread_num, upload_queue, base_url, gsutil, md5_lock, force,
    use_md5, stdout_queue, ret_codes):
  while True:
    filename, sha1_sum = upload_queue.get()
    if not filename:
      break
    file_url = '%s/%s' % (base_url, sha1_sum)
    if gsutil.check_call('ls', file_url)[0] == 0 and not force:
      # File exists, check MD5 hash.
      _, out, _ = gsutil.check_call('ls', '-L', file_url)
      etag_match = re.search('ETag:\s+([a-z0-9]{32})', out)
      if etag_match:
        remote_md5 = etag_match.group(1)
        # Calculate the MD5 checksum to match it to Google Storage's ETag.
        with md5_lock:
          if use_md5:
            local_md5 = get_md5_cached(filename)
          else:
            local_md5 = get_md5(filename)
        if local_md5 == remote_md5:
          stdout_queue.put(
              '%d> File %s already exists and MD5 matches, upload skipped' %
              (thread_num, filename))
          continue
    stdout_queue.put('%d> Uploading %s...' % (
        thread_num, filename))
    code, _, err = gsutil.check_call('cp', '-q', filename, file_url)
    if code != 0:
      ret_codes.put(
          (code,
           'Encountered error on uploading %s to %s\n%s' %
           (filename, file_url, err)))
      continue

def get_targets(args, parser, use_null_terminator):
  if not args:
    parser.error('Missing target.')

  if len(args) == 1 and args[0] == '-':
    # Take stdin as a newline or null seperated list of files.
    if use_null_terminator:
      return sys.stdin.read().split('\0')
    else:
      return sys.stdin.read().splitlines()
  else:
    return args

def upload_to_google_storage(
    input_filenames, base_url, gsutil, force,
    use_md5, num_threads, skip_hashing):
  # We only want one MD5 calculation happening at a time to avoid HD thrashing.
  md5_lock = threading.Lock()

  # Start up all the worker threads plus the printer thread.
  all_threads = []
  ret_codes = Queue.Queue()
  ret_codes.put((0, None))
  upload_queue = Queue.Queue()
  upload_timer = time.time()
  stdout_queue = Queue.Queue()
  printer_thread = threading.Thread(target=printer_worker, args=[stdout_queue])
  printer_thread.daemon = True
  printer_thread.start()
  for thread_num in range(num_threads):
    t = threading.Thread(
        target=_upload_worker,
        args=[thread_num, upload_queue, base_url, gsutil, md5_lock,
              force, use_md5, stdout_queue, ret_codes])
    t.daemon = True
    t.start()
    all_threads.append(t)

  # We want to hash everything in a single thread since its faster.
  # The bottleneck is in disk IO, not CPU.
  hashing_start = time.time()
  for filename in input_filenames:
    if not os.path.exists(filename):
      stdout_queue.put('Main> Error: %s not found, skipping.' % filename)
      continue
    if os.path.exists('%s.sha1' % filename) and skip_hashing:
      stdout_queue.put(
          'Main> Found hash for %s, sha1 calculation skipped.' % filename)
      with open(filename + '.sha1', 'rb') as f:
        sha1_file = f.read(1024)
      if not re.match('^([a-z0-9]{40})$', sha1_file):
        print >> sys.stderr, 'Invalid sha1 hash file %s.sha1' % filename
        return 1
      upload_queue.put((filename, sha1_file))
      continue
    stdout_queue.put('Main> Calculating hash for %s...' % filename)
    sha1_sum = get_sha1(filename)
    with open(filename + '.sha1', 'wb') as f:
      f.write(sha1_sum)
    stdout_queue.put('Main> Done calculating hash for %s.' % filename)
    upload_queue.put((filename, sha1_sum))
  hashing_duration = time.time() - hashing_start

  # Wait for everything to finish.
  for _ in all_threads:
    upload_queue.put((None, None))  # To mark the end of the work queue.
  for t in all_threads:
    t.join()
  stdout_queue.put(None)
  printer_thread.join()

  # Print timing information.
  print 'Hashing %s files took %1f seconds' % (
      len(input_filenames), hashing_duration)
  print 'Uploading took %1f seconds' % (time.time() - upload_timer)

  # See if we ran into any errors.
  max_ret_code = 0
  for ret_code, message in ret_codes.queue:
    max_ret_code = max(ret_code, max_ret_code)
    if message:
      print >> sys.stderr, message

  if not max_ret_code:
    print 'Success!'

  return max_ret_code

def main(args):
  parser = optparse.OptionParser(USAGE_STRING)
  parser.add_option('-b', '--bucket',
                    help='Google Storage bucket to upload to.')
  parser.add_option('-e', '--boto', help='Specify a custom boto file.')
  parser.add_option('-f', '--force', action='store_true',
                    help='Force upload even if remote file exists.')
  parser.add_option('-g', '--gsutil_path', default=GSUTIL_DEFAULT_PATH,
                    help='Path to the gsutil script.')
  parser.add_option('-m', '--use_md5', action='store_true',
                    help='Generate MD5 files when scanning, and don\'t check '
                    'the MD5 checksum if a .md5 file is found.')
  parser.add_option('-t', '--num_threads', default=1, type='int',
                    help='Number of uploader threads to run.')
  parser.add_option('-s', '--skip_hashing', action='store_true',
                    help='Skip hashing if .sha1 file exists.')
  parser.add_option('-0', '--use_null_terminator', action='store_true',
                    help='Use \\0 instead of \\n when parsing '
                    'the file list from stdin.  This is useful if the input '
                    'is coming from "find ... -print0".')
  (options, args) = parser.parse_args()

  # Enumerate our inputs.
  input_filenames = get_targets(args, parser, options.use_null_terminator)

  # Make sure we can find a working instance of gsutil.
  if os.path.exists(GSUTIL_DEFAULT_PATH):
    gsutil = Gsutil(GSUTIL_DEFAULT_PATH, boto_path=options.boto)
  else:
    gsutil = None
    for path in os.environ["PATH"].split(os.pathsep):
      if os.path.exists(path) and 'gsutil' in os.listdir(path):
        gsutil = Gsutil(os.path.join(path, 'gsutil'), boto_path=options.boto)
    if not gsutil:
      parser.error('gsutil not found in %s, bad depot_tools checkout?' %
                   GSUTIL_DEFAULT_PATH)

  # Check we have a valid bucket with valid permissions.
  base_url, code = check_bucket_permissions(options.bucket, gsutil)
  if code:
    return code

  return upload_to_google_storage(
      input_filenames, base_url, gsutil, options.force, options.use_md5,
      options.num_threads, options.skip_hashing)

if __name__ == '__main__':
  sys.exit(main(sys.argv))
<|endoftext|>


<commit_before>import datetime

# --Definition of person START--
class Person(object):
    def __init__(self, name):
        """create a person called name"""
        self.name = name
        self.birthday = None
        self.lastName = name.split(' ')[-1]

    def getLastName(self):
        """return self's last name"""
        return self.lastName

    def setBirthday(self,month,day,year):
        """sets self's birthday to birthDate"""
        self.birthday = datetime.date(year,month,day)

    def getAge(self):
        """returns self's current age in days"""
        if self.birthday == None:
            raise ValueError
        return (datetime.date.today() - self.birthday).days

    def __lt__(self, other):
        """return True if self's ame is lexicographically
           less than other's name, and False otherwise"""
        if self.lastName == other.lastName:
            return self.name < other.name
        return self.lastName < other.lastName

    def __str__(self):
        """return self's name"""
        return self.name

# me = Person("William Eric Grimson")
# print me
# me.getLastName()
# me.setBirthday(1,2,1927)
# me.getAge()
# her = Person("Cher")
# her.getLastName()
# plist = [me, her]
# for p in plist: print p
# plist.sort()
# for p in plist: print p

class MITPerson(Person):
    nextIdNum = 0 # next ID number to assign

    def __init__(self, name):
        Person.__init__(self, name) # initialize Person attributes
        # new MITPerson attribute: a unique ID number
        self.idNum = MITPerson.nextIdNum
        MITPerson.nextIdNum += 1

    def getIdNum(self):
        return self.idNum

    # sorting MIT people uses their ID number, not name!
    def __lt__(self, other):
        return self.idNum < other.idNum

# p1 = MITPerson('Eric')
# p2 = MITPerson('John')
# p3 = MITPerson('John')
# p4 = Person('John')

# print p1
# p1.getIdNum()
# p2.getIdNum()
# p1 < p2
# p3 < p2
# p4 < p1

# p1 < p4

class Student(MITPerson):
    pass

class UG(Student):
    def __init__(self, name, classYear):
        MITPerson.__init__(self, name)
        self.year = classYear

    def getClass(self):
        return self.year

class Grad(Student):
    pass

class TransferStudent(Student):
    pass

def isStudent(obj):
    return isinstance(obj, Student)

# --Definition of person END--

# --Definition of Gradebook START--
class Grades(object):
    """A mapping from students to a list of grades"""
    def __init__(self):
        """Create empty grade book"""
        self.students = []  # list of Student objects
        self.grades = {}    # maps idNum -> list of grades
        self.isSorted = True # true if self.students is sorted

    def addStudent(self, student):
        """Assumes: student is of type Student
           Add student to the grade book"""
        if student in self.students:
            raise ValueError('Duplicate student')
        self.students.append(student)
        self.grades[student.getIdNum()] = []
        self.isSorted = False

    def addGrade(self, student, grade):
        """Assumes: grade is a float
           Add grade to the list of grades for student"""
        try:
            self.grades[student.getIdNum()].append(grade)
        except KeyError:
            raise ValueError('Student not in grade book')

    def getGrades(self, student):
        """Return a list of grades for student"""
        try:    # return copy of student's grades
            return self.grades[student.getIdNum()][:]
        except KeyError:
            raise ValueError('Student not in grade book')

    def allStudents(self):
        """Return a list of the students in the grade book"""
        if not self.isSorted:
            self.students.sort()
            self.isSorted = True
        return self.students[:] #return copy of list of students

def gradeReport(course):
    """Assumes: course is of type grades"""
    report = []
    for s in course.allStudents():
        # initialize total score to 0
        tot = 0.0
        # numGrades is used in evaluating average grades
        numGrades = 0
        for g in course.getGrades(s):
            tot += g
            numGrades += 1
        try:
            average = tot/numGrades
            report.append(str(s) + '\'s mean grade is '
                          + str(average))
        except ZeroDivisionError:
            report.append(str(s) + ' has no grades')
    return '\n'.join(report)

# ug1 = UG('Jane Doe', 2014)
# ug2 = UG('John Doe', 2015)
# ug3 = UG('David Henry', 2003)
# g1 = Grad('John Henry')
# g2 = Grad('George Steinbrenner')

# six00 = Grades()
# six00.addStudent(g1)
# six00.addStudent(ug2)
# six00.addStudent(ug1)
# six00.addStudent(g2)

# for s in six00.allStudents():
#     six00.addGrade(s, 75)
# six00.addGrade(g1, 100)
# six00.addGrade(g2, 25)

# six00.addStudent(ug3)

# print gradeReport(six00)

# --Definition of Gradebook END--<commit_msg>getGrades and allStudents methods do not need return copies<commit_after>import datetime

# --Definition of person START--
class Person(object):
    def __init__(self, name):
        """create a person called name"""
        self.name = name
        self.birthday = None
        self.lastName = name.split(' ')[-1]

    def getLastName(self):
        """return self's last name"""
        return self.lastName

    def setBirthday(self,month,day,year):
        """sets self's birthday to birthDate"""
        self.birthday = datetime.date(year,month,day)

    def getAge(self):
        """returns self's current age in days"""
        if self.birthday == None:
            raise ValueError
        return (datetime.date.today() - self.birthday).days

    def __lt__(self, other):
        """return True if self's ame is lexicographically
           less than other's name, and False otherwise"""
        if self.lastName == other.lastName:
            return self.name < other.name
        return self.lastName < other.lastName

    def __str__(self):
        """return self's name"""
        return self.name

# me = Person("William Eric Grimson")
# print me
# me.getLastName()
# me.setBirthday(1,2,1927)
# me.getAge()
# her = Person("Cher")
# her.getLastName()
# plist = [me, her]
# for p in plist: print p
# plist.sort()
# for p in plist: print p

class MITPerson(Person):
    nextIdNum = 0 # next ID number to assign

    def __init__(self, name):
        Person.__init__(self, name) # initialize Person attributes
        # new MITPerson attribute: a unique ID number
        self.idNum = MITPerson.nextIdNum
        MITPerson.nextIdNum += 1

    def getIdNum(self):
        return self.idNum

    # sorting MIT people uses their ID number, not name!
    def __lt__(self, other):
        return self.idNum < other.idNum

# p1 = MITPerson('Eric')
# p2 = MITPerson('John')
# p3 = MITPerson('John')
# p4 = Person('John')

# print p1
# p1.getIdNum()
# p2.getIdNum()
# p1 < p2
# p3 < p2
# p4 < p1

# p1 < p4

class Student(MITPerson):
    pass

class UG(Student):
    def __init__(self, name, classYear):
        MITPerson.__init__(self, name)
        self.year = classYear

    def getClass(self):
        return self.year

class Grad(Student):
    pass

class TransferStudent(Student):
    pass

def isStudent(obj):
    return isinstance(obj, Student)

# --Definition of person END--

# --Definition of Gradebook START--
class Grades(object):
    """A mapping from students to a list of grades"""
    def __init__(self):
        """Create empty grade book"""
        self.students = []  # list of Student objects
        self.grades = {}    # maps idNum -> list of grades
        self.isSorted = True # true if self.students is sorted

    def addStudent(self, student):
        """Assumes: student is of type Student
           Add student to the grade book"""
        if student in self.students:
            raise ValueError('Duplicate student')
        self.students.append(student)
        self.grades[student.getIdNum()] = []
        self.isSorted = False

    def addGrade(self, student, grade):
        """Assumes: grade is a float
           Add grade to the list of grades for student"""
        try:
            self.grades[student.getIdNum()].append(grade)
        except KeyError:
            raise ValueError('Student not in grade book')

    def getGrades(self, student):
        """Return a list of grades for student"""
        try:
            return self.grades[student.getIdNum()]
        except KeyError:
            raise ValueError('Student not in grade book')

    def allStudents(self):
        """Return a list of the students in the grade book"""
        if not self.isSorted:
            self.students.sort()
            self.isSorted = True
        return self.students

def gradeReport(course):
    """Assumes: course is of type grades"""
    report = []
    for s in course.allStudents():
        # initialize total score to 0
        tot = 0.0
        # numGrades is used in evaluating average grades
        numGrades = 0
        for g in course.getGrades(s):
            tot += g
            numGrades += 1
        try:
            average = tot/numGrades
            report.append(str(s) + '\'s mean grade is '
                          + str(average))
        except ZeroDivisionError:
            report.append(str(s) + ' has no grades')
    return '\n'.join(report)

# ug1 = UG('Jane Doe', 2014)
# ug2 = UG('John Doe', 2015)
# ug3 = UG('David Henry', 2003)
# g1 = Grad('John Henry')
# g2 = Grad('George Steinbrenner')

# six00 = Grades()
# six00.addStudent(g1)
# six00.addStudent(ug2)
# six00.addStudent(ug1)
# six00.addStudent(g2)

# for s in six00.allStudents():
#     six00.addGrade(s, 75)
# six00.addGrade(g1, 100)
# six00.addGrade(g2, 25)

# six00.addStudent(ug3)

# print gradeReport(six00)

# --Definition of Gradebook END--<|endoftext|>


<commit_before>#
# Copyright (c) 2005 rPath, Inc.
#
# All Rights Reserved
#

import calendar
import time
import rfc822
import urllib2
from elementtree import ElementTree

import database

REFRESH_TIME = 600 # seconds

class NewsCacheInfoTable(database.DatabaseTable):
    name = 'NewsCacheInfo'
    fields = ['age', 'feedLink']

    createSQL = "CREATE TABLE NewsCacheInfo (age INT, feedLink STR);"

    def getAge(self):
        cu = self.db.cursor()
        cu.execute("SELECT age FROM NewsCacheInfo")

        r = cu.fetchone()
        return r and r[0] or 0

    def getLink(self):
        cu = self.db.cursor()
        cu.execute("SELECT feedLink FROM NewsCacheInfo")

        r = cu.fetchone()
        return r and r[0] or ""

    def set(self, t, link):
        cu = self.db.cursor()
        cu.execute("DELETE FROM NewsCacheInfo")
        cu.execute("INSERT INTO NewsCacheInfo VALUES (?, ?)", t, link)
        self.db.commit()
        return True

class NewsCacheTable(database.KeyedTable):
    name = 'NewsCache'
    key = 'itemId'
    createSQL = """CREATE TABLE NewsCache (
                    itemId          INTEGER PRIMARY KEY,
                    title           STR,
                    pubDate         INT,
                    content         STR,
                    link            STR,
                    category        STR
                );
                """
    fields = ['itemId', 'title', 'pubDate', 'content', 'link', 'category']

    def __init__(self, db, cfg):
        database.DatabaseTable.__init__(self, db)
        self.cfg = cfg
        self.ageTable = NewsCacheInfoTable(db)

    def refresh(self, items = 5, purge = True):
        def toUnixTime(t):
            return calendar.timegm(rfc822.parsedate_tz(item.find("pubDate").text))

        if not self.cfg.newsRssFeed:
            return False

        if (time.time() - REFRESH_TIME) < self.ageTable.getAge():
            return False

        cu = self.db.cursor()

        self.db.transaction()
        if purge:
            cu.execute("DELETE FROM NewsCache")

        try:
            url = urllib2.urlopen(self.cfg.newsRssFeed)
            data = url.read()
        except urllib2.URLError:
            self.db.rollback()
            return False

        tree = ElementTree.XML(data)
        feedLink = tree.find("channel/link").text
        for item in tree.findall("channel/item")[:items]:
            link = item.find("link").text
            title = item.find("title").text
            category = item.find("category").text
            content = item.find("{http://purl.org/rss/1.0/modules/content/}encoded").text
            pubDate = toUnixTime(item.find("pubDate").text)

            query = "INSERT INTO NewsCache VALUES (NULL, ?, ?, ?, ?, ?)"
            cu.execute(query, title, pubDate, content, link, category)

        self.ageTable.set(t = time.time(), link = feedLink)
        self.db.commit()
        return True

    def getNews(self):
        cu = self.db.cursor()

        cu.execute("SELECT * FROM NewsCache ORDER BY pubDate DESC")
        data = []

        for r in cu.fetchall():
            item = {}
            for i, key in enumerate(self.fields):
                item[key] = r[i]
            data.append(item)
        return data

    def getNewsLink(self):
        return self.ageTable.getLink()
<commit_msg>try to fetch the news before doing any database operations<commit_after>#
# Copyright (c) 2005 rPath, Inc.
#
# All Rights Reserved
#

import calendar
import time
import rfc822
import urllib2
from elementtree import ElementTree

import database

REFRESH_TIME = 600 # seconds

class NewsCacheInfoTable(database.DatabaseTable):
    name = 'NewsCacheInfo'
    fields = ['age', 'feedLink']

    createSQL = "CREATE TABLE NewsCacheInfo (age INT, feedLink STR);"

    def getAge(self):
        cu = self.db.cursor()
        cu.execute("SELECT age FROM NewsCacheInfo")

        r = cu.fetchone()
        return r and r[0] or 0

    def getLink(self):
        cu = self.db.cursor()
        cu.execute("SELECT feedLink FROM NewsCacheInfo")

        r = cu.fetchone()
        return r and r[0] or ""

    def set(self, t, link):
        cu = self.db.cursor()
        cu.execute("DELETE FROM NewsCacheInfo")
        cu.execute("INSERT INTO NewsCacheInfo VALUES (?, ?)", t, link)
        self.db.commit()
        return True

class NewsCacheTable(database.KeyedTable):
    name = 'NewsCache'
    key = 'itemId'
    createSQL = """CREATE TABLE NewsCache (
                    itemId          INTEGER PRIMARY KEY,
                    title           STR,
                    pubDate         INT,
                    content         STR,
                    link            STR,
                    category        STR
                );
                """
    fields = ['itemId', 'title', 'pubDate', 'content', 'link', 'category']

    def __init__(self, db, cfg):
        database.DatabaseTable.__init__(self, db)
        self.cfg = cfg
        self.ageTable = NewsCacheInfoTable(db)

    def refresh(self, items = 5, purge = True):
        def toUnixTime(t):
            return calendar.timegm(rfc822.parsedate_tz(item.find("pubDate").text))

        if not self.cfg.newsRssFeed:
            return False

        if (time.time() - REFRESH_TIME) < self.ageTable.getAge():
            return False

        try:
            url = urllib2.urlopen(self.cfg.newsRssFeed)
            data = url.read()
        except urllib2.URLError:
            return False

        cu = self.db.cursor()
        if purge:
            cu.execute("DELETE FROM NewsCache")

        tree = ElementTree.XML(data)
        feedLink = tree.find("channel/link").text
        for item in tree.findall("channel/item")[:items]:
            link = item.find("link").text
            title = item.find("title").text
            category = item.find("category").text
            content = item.find("{http://purl.org/rss/1.0/modules/content/}encoded").text
            pubDate = toUnixTime(item.find("pubDate").text)

            query = "INSERT INTO NewsCache VALUES (NULL, ?, ?, ?, ?, ?)"
            cu.execute(query, title, pubDate, content, link, category)

        self.ageTable.set(t = time.time(), link = feedLink)
        return True

    def getNews(self):
        cu = self.db.cursor()

        cu.execute("SELECT * FROM NewsCache ORDER BY pubDate DESC")
        data = []

        for r in cu.fetchall():
            item = {}
            for i, key in enumerate(self.fields):
                item[key] = r[i]
            data.append(item)
        return data

    def getNewsLink(self):
        return self.ageTable.getLink()
<|endoftext|>


<commit_before>import discord
from discord.ext import commands
from .utils import checks
from .utils.dataIO import dataIO
from .utils import chat_formatting as cf
from __main__ import send_cmd_help

import asyncio
import os
import os.path
import sys
import aiohttp
import copy
import glob
from typing import List
from subprocess import PIPE, Popen
from threading import Thread
import shlex

try:
	from Queue import Queue, Empty
except ImportError:
	from queue import Queue, Empty # python 3.x

ON_POSIX = 'posix' in sys.builtin_module_names

class misc:
	"""Misc commands"""

	def __init__(self, bot):
		self.bot = bot
		self.report_base = "data/reports"
		self.evtc_base = "data/reports"

	def enqueue_output(self, out, queue):
		for line in iter(out.readline, b''):
			queue.put(line)
		out.close()

	async def processevtc(self, command):
		"""This function runs a raid_heroes.exe to process a evtc file using a seperate thread so it isn't blocking"""
		process = Popen([command], stdout=PIPE, shell=True, bufsize=1, close_fds=ON_POSIX)
		q = Queue()
		t = Thread(target=self.enqueue_output, args=(process.stdout, q))
		t.daemon = True
		t.start()
		boss = 'error'
		while True:
			try:
				output = q.get_nowait().decode()
			except Empty:
				if t.isAlive() == False and q.empty() == True:
					break
				else:
					pass
			else:
				if output:
					if len(output) > 1:
						try:
							await self.bot.say(output)
						except:
							pass
					if 'Samarog' in output:
						boss = '_sam'
					if 'Sabetha' in output:
						boss = '_sab'
					if 'Overseer' in output:
						boss = '_mo'
					if 'Deimos' in output:
						boss = '_dei'
					if 'Vale' in output:
						boss = '_vg'
					if 'Gorseval' in output:
						boss = '_gor'
					if 'Matthias' in output:
						boss = '_mat'
					if 'Construct' in output:
						boss = '_kc'
					if 'Xera' in output:
						boss = '_xer'
					if 'Cairn' in output:
						boss = '_cai'
		return boss

	@commands.command(hidden=True)
	async def summon(self):
		await self.bot.say("Who dares summon me?")

	@commands.command(pass_context=True)
	@checks.is_owner()
	async def logprocess(self, ctx, link: str=None):
		"""Process an arcdps .evtc file with raid heroes.
		Upload the file as a discord attachment."""
		await self.bot.type()
		server = ctx.message.server
		channel = ctx.message.channel
		attach = ctx.message.attachments
		if len(attach) > 1 or (attach and link):
			await self.bot.say(
				cf.error("Please only add one evtc file at a time."))
			return
		url = ""
		filename = ""
		if attach:
			a = attach[0]
			url = a["url"]
			filename = a["filename"]
		elif link:
			url = "".join(link)
			filename = os.path.basename(
				"_".join(url.split()).replace("%20", "_"))
		else:
			await self.bot.say(
				cf.error("You must provide a Discord attachment."))
			return
		fileextension = os.path.splitext(filename)[1]
		if fileextension != '.evtc':
			await self.bot.say("That's not a evtc file.")
			return
		splitfilename = os.path.splitext(filename)[0]
		splitfilename = splitfilename.replace('-','')
		if splitfilename.isalnum() != True:
			await self.bot.say("That's an invalid file name.")
			return
		filepath = os.path.join(self.evtc_base, filename)
		if os.path.splitext(filename)[0] in self._list_evtc():
			pass
		else:
			async with aiohttp.get(url) as new_evtc:
				f = open(filepath, "wb")
				f.write(await new_evtc.read())
				f.close()
		message = await self.bot.say("Processing...")
		file = '/home/ubuntu/Red-DiscordBot/data/reports/' + filename
		command = '/home/ubuntu/raid_heroes ' + file
		boss = await self.processevtc(command)
		if boss == 'error':
			await self.bot.say("Something went wrong.")
			return
		os.remove(file)
		linktofile = 'redballslair.uk/raidbossreports/reports/' + os.path.splitext(filename)[0] + boss + '.html'
		await self.bot.say(linktofile)
		#await self.bot.send_file(channel, output)

	@commands.command(pass_context=True)
	@checks.is_owner()
	async def logupload(self, ctx, link: str=None):
		"""Adds a new raid heroes log report.
		Upload the file as a discord attachment."""
		await self.bot.type()
		server = ctx.message.server
		attach = ctx.message.attachments
		if len(attach) > 1 or (attach and link):
			await self.bot.say(
				cf.error("Please only add one report at a time."))
			return
		url = ""
		filename = ""
		if attach:
			a = attach[0]
			url = a["url"]
			filename = a["filename"]
		elif link:
			url = "".join(link)
			filename = os.path.basename(
				"_".join(url.split()).replace("%20", "_"))
		else:
			await self.bot.say(
				cf.error("You must provide a Discord attachment."))
			return
		fileextension = os.path.splitext(filename)[1]
		if fileextension != '.html':
			await self.bot.say("That's not a report.")
			return
		filepath = os.path.join(self.report_base, filename)
		if os.path.splitext(filename)[0] in self._list_reports():
			await self.bot.say(
				cf.error("A report with that filename already exists."
						 " Please change the filename and try again."))
			return
		async with aiohttp.get(url) as new_report:
			f = open(filepath, "wb")
			f.write(await new_report.read())
			f.close()
		await self.bot.say(
			cf.info("Report {} added.".format(os.path.splitext(filename)[0])))

	"""async def rename_orun(self, ):
		while self is self.bot.get_cog("misc"):
			server = await self.bot.get_server('294578270064869377')
			user = await self.bot.get_user_info('202429404503212034')
			nickname = "Orun"
			#print(user.nick)
			try:
				if user.nick != nickname:
					await self.bot.change_nickname(user, nickname)
					#print("Renamed Orun")
			except discord.Forbidden:
				print("I cannot do that, I lack the "
								"\"Manage Nicknames\" permission.")
			await asyncio.sleep(30)"""

	def _list_reports(self) -> List[str]:
		return sorted(
			[os.path.splitext(s)[0] for s in os.listdir(os.path.join(
				self.report_base))],
				key=lambda s: s.lower())

	def _list_evtc(self) -> List[str]:
		return sorted(
			[os.path.splitext(s)[0] for s in os.listdir(os.path.join(
				self.evtc_base))],
				key=lambda s: s.lower())

def check_folders():
	folder = "data/reports/toprocess"
	if not os.path.exists(folder):
		print("Creating {} folder...".format(folder))
		os.makedirs(folder)

def setup(bot):
	n = misc(bot)
	check_folders()
	loop = asyncio.get_event_loop()
	#loop.create_task(n.rename_orun())
	bot.add_cog(n)<commit_msg>debug line<commit_after>import discord
from discord.ext import commands
from .utils import checks
from .utils.dataIO import dataIO
from .utils import chat_formatting as cf
from __main__ import send_cmd_help

import asyncio
import os
import os.path
import sys
import aiohttp
import copy
import glob
from typing import List
from subprocess import PIPE, Popen
from threading import Thread
import shlex

try:
	from Queue import Queue, Empty
except ImportError:
	from queue import Queue, Empty # python 3.x

ON_POSIX = 'posix' in sys.builtin_module_names

class misc:
	"""Misc commands"""

	def __init__(self, bot):
		self.bot = bot
		self.report_base = "data/reports"
		self.evtc_base = "data/reports"

	def enqueue_output(self, out, queue):
		for line in iter(out.readline, b''):
			queue.put(line)
		out.close()

	async def processevtc(self, command):
		"""This function runs a raid_heroes.exe to process a evtc file using a seperate thread so it isn't blocking"""
		await self.bot.say(command)
		process = Popen([command], stdout=PIPE, shell=True, bufsize=1, close_fds=ON_POSIX)
		q = Queue()
		t = Thread(target=self.enqueue_output, args=(process.stdout, q))
		t.daemon = True
		t.start()
		boss = 'error'
		while True:
			try:
				output = q.get_nowait().decode()
			except Empty:
				if t.isAlive() == False and q.empty() == True:
					break
				else:
					pass
			else:
				if output:
					if len(output) > 1:
						try:
							await self.bot.say(output)
						except:
							pass
					if 'Samarog' in output:
						boss = '_sam'
					if 'Sabetha' in output:
						boss = '_sab'
					if 'Overseer' in output:
						boss = '_mo'
					if 'Deimos' in output:
						boss = '_dei'
					if 'Vale' in output:
						boss = '_vg'
					if 'Gorseval' in output:
						boss = '_gor'
					if 'Matthias' in output:
						boss = '_mat'
					if 'Construct' in output:
						boss = '_kc'
					if 'Xera' in output:
						boss = '_xer'
					if 'Cairn' in output:
						boss = '_cai'
		return boss

	@commands.command(hidden=True)
	async def summon(self):
		await self.bot.say("Who dares summon me?")

	@commands.command(pass_context=True)
	@checks.is_owner()
	async def logprocess(self, ctx, link: str=None):
		"""Process an arcdps .evtc file with raid heroes.
		Upload the file as a discord attachment."""
		await self.bot.type()
		server = ctx.message.server
		channel = ctx.message.channel
		attach = ctx.message.attachments
		if len(attach) > 1 or (attach and link):
			await self.bot.say(
				cf.error("Please only add one evtc file at a time."))
			return
		url = ""
		filename = ""
		if attach:
			a = attach[0]
			url = a["url"]
			filename = a["filename"]
		elif link:
			url = "".join(link)
			filename = os.path.basename(
				"_".join(url.split()).replace("%20", "_"))
		else:
			await self.bot.say(
				cf.error("You must provide a Discord attachment."))
			return
		fileextension = os.path.splitext(filename)[1]
		if fileextension != '.evtc':
			await self.bot.say("That's not a evtc file.")
			return
		splitfilename = os.path.splitext(filename)[0]
		splitfilename = splitfilename.replace('-','')
		if splitfilename.isalnum() != True:
			await self.bot.say("That's an invalid file name.")
			return
		filepath = os.path.join(self.evtc_base, filename)
		if os.path.splitext(filename)[0] in self._list_evtc():
			pass
		else:
			async with aiohttp.get(url) as new_evtc:
				f = open(filepath, "wb")
				f.write(await new_evtc.read())
				f.close()
		message = await self.bot.say("Processing...")
		file = '/home/ubuntu/Red-DiscordBot/data/reports/' + filename
		command = '/home/ubuntu/raid_heroes ' + file
		boss = await self.processevtc(command)
		if boss == 'error':
			await self.bot.say("Something went wrong.")
			return
		os.remove(file)
		linktofile = 'redballslair.uk/raidbossreports/reports/' + os.path.splitext(filename)[0] + boss + '.html'
		await self.bot.say(linktofile)
		#await self.bot.send_file(channel, output)

	@commands.command(pass_context=True)
	@checks.is_owner()
	async def logupload(self, ctx, link: str=None):
		"""Adds a new raid heroes log report.
		Upload the file as a discord attachment."""
		await self.bot.type()
		server = ctx.message.server
		attach = ctx.message.attachments
		if len(attach) > 1 or (attach and link):
			await self.bot.say(
				cf.error("Please only add one report at a time."))
			return
		url = ""
		filename = ""
		if attach:
			a = attach[0]
			url = a["url"]
			filename = a["filename"]
		elif link:
			url = "".join(link)
			filename = os.path.basename(
				"_".join(url.split()).replace("%20", "_"))
		else:
			await self.bot.say(
				cf.error("You must provide a Discord attachment."))
			return
		fileextension = os.path.splitext(filename)[1]
		if fileextension != '.html':
			await self.bot.say("That's not a report.")
			return
		filepath = os.path.join(self.report_base, filename)
		if os.path.splitext(filename)[0] in self._list_reports():
			await self.bot.say(
				cf.error("A report with that filename already exists."
						 " Please change the filename and try again."))
			return
		async with aiohttp.get(url) as new_report:
			f = open(filepath, "wb")
			f.write(await new_report.read())
			f.close()
		await self.bot.say(
			cf.info("Report {} added.".format(os.path.splitext(filename)[0])))

	"""async def rename_orun(self, ):
		while self is self.bot.get_cog("misc"):
			server = await self.bot.get_server('294578270064869377')
			user = await self.bot.get_user_info('202429404503212034')
			nickname = "Orun"
			#print(user.nick)
			try:
				if user.nick != nickname:
					await self.bot.change_nickname(user, nickname)
					#print("Renamed Orun")
			except discord.Forbidden:
				print("I cannot do that, I lack the "
								"\"Manage Nicknames\" permission.")
			await asyncio.sleep(30)"""

	def _list_reports(self) -> List[str]:
		return sorted(
			[os.path.splitext(s)[0] for s in os.listdir(os.path.join(
				self.report_base))],
				key=lambda s: s.lower())

	def _list_evtc(self) -> List[str]:
		return sorted(
			[os.path.splitext(s)[0] for s in os.listdir(os.path.join(
				self.evtc_base))],
				key=lambda s: s.lower())

def check_folders():
	folder = "data/reports/toprocess"
	if not os.path.exists(folder):
		print("Creating {} folder...".format(folder))
		os.makedirs(folder)

def setup(bot):
	n = misc(bot)
	check_folders()
	loop = asyncio.get_event_loop()
	#loop.create_task(n.rename_orun())
	bot.add_cog(n)<|endoftext|>


<commit_before>import discord
from discord.ext import commands
from .utils import checks
from __main__ import send_cmd_help

import asyncio

class misc:
	"""Misc commands"""

	def __init__(self, bot):
		self.bot = bot

	@commands.command(hidden=True)
	async def summon(self):
		await self.bot.say("Who dares summon me?")

	async def rename_orun(self, ):
		while self is self.bot.get_cog("misc"):
			serverid = "294578270064869377"
			userid = "202429404503212034"
			server = self.bot.get_server(serverid)
			user = server.get_member(userid)
			nickname = "Orun"
			try:
				await self.bot.change_nickname(user, nickname)
				print("Renamed Orun")
			except discord.Forbidden:
				print("I cannot do that, I lack the "
								"\"Manage Nicknames\" permission.")
			await asyncio.sleep(30)

def setup(bot):
	n = misc(bot)
	loop = asyncio.get_event_loop()
	loop.create_task(n.rename_orun())
	bot.add_cog(n)<commit_msg>debug<commit_after>import discord
from discord.ext import commands
from .utils import checks
from __main__ import send_cmd_help

import asyncio

class misc:
	"""Misc commands"""

	def __init__(self, bot):
		self.bot = bot

	@commands.command(hidden=True)
	async def summon(self):
		await self.bot.say("Who dares summon me?")

	async def rename_orun(self, ):
		while self is self.bot.get_cog("misc"):
			serverid = "294578270064869377"
			userid = "202429404503212034"
			server = self.bot.get_server(serverid)
			user = server.get_member(userid)
			print(user.nick)
			nickname = "Orun"
			try:
				await self.bot.change_nickname(user, nickname)
				print("Renamed Orun")
			except discord.Forbidden:
				print("I cannot do that, I lack the "
								"\"Manage Nicknames\" permission.")
			await asyncio.sleep(30)

def setup(bot):
	n = misc(bot)
	loop = asyncio.get_event_loop()
	loop.create_task(n.rename_orun())
	bot.add_cog(n)<|endoftext|>