<commit_before>"""
This script extracts surrounding buildings of the zone geometry from Open street maps
"""
from __future__ import division
from __future__ import print_function

import math
import os

import numpy as np
import osmnx as ox
import shapely.geometry as geometry
from geopandas import GeoDataFrame as Gdf
from geopandas import overlay
from scipy.spatial import Delaunay
from shapely.ops import cascaded_union, polygonize
import pandas as pd

import cea.config
import cea.inputlocator
from cea.datamanagement.zone_helper import parse_building_floors
from cea.demand import constants
from cea.utilities.standardize_coordinates import get_projected_coordinate_system, get_geographic_coordinate_system

__author__ = "Jimeno Fonseca"
__copyright__ = "Copyright 2018, Architecture and Building Systems - ETH Zurich"
__credits__ = ["Jimeno Fonseca"]
__license__ = "MIT"
__version__ = "0.1"
__maintainer__ = "Daren Thomas"
__email__ = "cea@arch.ethz.ch"
__status__ = "Production"

def alpha_shape(points, alpha):
    """
    Compute the alpha shape (concave hull) of a set of points.

    :param points: Iterable container of points.
    :param alpha: alpha value to influence the gooeyness of the border. Smaller
                  numbers don't fall inward as much as larger numbers. Too large,
                  and you lose everything!
    """
    if len(points) < 4:
        # When you have a triangle, there is no sense in computing an alpha
        # shape.
        return geometry.MultiPoint(list(points)).convex_hull

    def add_edge(edges, edge_points, coords, i, j):
        """Add a line between the i-th and j-th points, if not in the list already"""
        if (i, j) in edges or (j, i) in edges:
            # already added
            return
        edges.add((i, j))
        edge_points.append(coords[[i, j]])

    coords = np.array([point.coords[0] for point in points])

    tri = Delaunay(coords)
    edges = set()
    edge_points = []
    # loop over triangles:
    # ia, ib, ic = indices of corner points of the triangle
    for ia, ib, ic in tri.vertices:
        pa = coords[ia]
        pb = coords[ib]
        pc = coords[ic]

        # Lengths of sides of triangle
        a = math.sqrt((pa[0] - pb[0]) ** 2 + (pa[1] - pb[1]) ** 2)
        b = math.sqrt((pb[0] - pc[0]) ** 2 + (pb[1] - pc[1]) ** 2)
        c = math.sqrt((pc[0] - pa[0]) ** 2 + (pc[1] - pa[1]) ** 2)

        # Semiperimeter of triangle
        s = (a + b + c) / 2.0

        # Area of triangle by Heron's formula
        area = math.sqrt(s * (s - a) * (s - b) * (s - c))
        circum_r = a * b * c / (4.0 * area)

        # Here's the radius filter.
        # print circum_r
        if circum_r < 1.0 / alpha:
            add_edge(edges, edge_points, coords, ia, ib)
            add_edge(edges, edge_points, coords, ib, ic)
            add_edge(edges, edge_points, coords, ic, ia)

    m = geometry.MultiLineString(edge_points)
    triangles = list(polygonize(m))
    return cascaded_union(triangles), edge_points

def calc_surrounding_area(zone_gdf, buffer_m):
    geometry_without_holes = zone_gdf.convex_hull
    geometry_without_holes_gdf = Gdf(geometry=geometry_without_holes.values)
    geometry_without_holes_gdf["one_class"] = "buildings"
    geometry_merged = geometry_without_holes_gdf.dissolve(by='one_class', aggfunc='sum')
    geometry_merged_final = Gdf(geometry=geometry_merged.convex_hull)
    new_buffer = Gdf(geometry=geometry_merged_final.buffer(buffer_m))
    area = overlay(geometry_merged_final, new_buffer, how='symmetric_difference')

    # THIS IS ANOTHER METHOD, NOT FUNCTIONAL THOUGH
    # from shapely.ops import Point
    # # new GeoDataFrame with same columns
    # points = []
    # # Extraction of the polygon nodes and attributes values from polys and integration into the new GeoDataFrame
    # for index, row in zone_gdf.iterrows():
    #     for j in list(row['geometry'].exterior.coords):
    #         points.append(Point(j))
    #
    # concave_hull, edge_points = alpha_shape(points, alpha=0.1)
    # simple_polygons = [x for x in concave_hull]
    # geometry_merged_final = Gdf(geometry=simple_polygons)
    # geometry_merged_final.plot()
    # plt.show()
    # new_buffer = Gdf(geometry=geometry_merged_final.buffer(buffer_m))
    # area = overlay(geometry_merged_final, new_buffer, how='symmetric_difference')
    # area.plot()

    return area, geometry_merged_final

def clean_attributes(shapefile, buildings_height, buildings_floors, key):
    # local variables
    no_buildings = shapefile.shape[0]
    list_of_columns = shapefile.columns
    if buildings_height is None and buildings_floors is None:
        print('Warning! you have not indicated a height or number of floors above ground for the buildings, '
              'we are reverting to data stored in Open Street Maps (It might not be accurate at all),'
              'if we do not find data in OSM for a particular building, we get the median in the surroundings, '
              'if we do not get any data we assume 4 floors per building')
        # Check which attributes the OSM has, Sometimes it does not have any and indicate the data source
        if 'building:levels' not in list_of_columns:
            shapefile['building:levels'] = [3] * no_buildings
            shapefile['REFERENCE'] = "CEA - assumption"
        elif pd.isnull(shapefile['building:levels']).all():
            shapefile['building:levels'] = [3] * no_buildings
            shapefile['REFERENCE'] = "CEA - assumption"
        else:
            shapefile['REFERENCE'] = ["OSM - median" if x is np.nan else "OSM - as it is" for x in shapefile['building:levels']]
        if 'roof:levels' not in list_of_columns:
            shapefile['roof:levels'] = [1] * no_buildings

        #get the median from the area:
        data_osm_floors1 = shapefile['building:levels'].fillna(0)
        data_osm_floors2 = shapefile['roof:levels'].fillna(0)
        data_floors_sum = [x + y for x, y in
                           zip([parse_building_floors(x) for x in data_osm_floors1], [parse_building_floors(x) for x in data_osm_floors2])]
        data_floors_sum_with_nan = [np.nan if x < 1.0 else x for x in data_floors_sum]
        data_osm_floors_joined = int(math.ceil(np.nanmedian(data_floors_sum_with_nan)))  # median so we get close to the worse case
        shapefile["floors_ag"] = [int(x) if x is not np.nan else data_osm_floors_joined for x in
                                  data_floors_sum_with_nan]
        shapefile["height_ag"] = shapefile["floors_ag"] * constants.H_F
    else:
        shapefile['REFERENCE'] = "User - assumption"
        if buildings_height is None and buildings_floors is not None:
            shapefile["floors_ag"] = [buildings_floors] * no_buildings
            shapefile["height_ag"] = shapefile["floors_ag"] * constants.H_F
        elif buildings_height is not None and buildings_floors is None:
            shapefile["height_ag"] = [buildings_height] * no_buildings
            shapefile["floors_ag"] = [int(math.floor(x)) for x in shapefile["height_ag"] / constants.H_F]
        else:  # both are not none
            shapefile["height_ag"] = [buildings_height] * no_buildings
            shapefile["floors_ag"] = [buildings_floors] * no_buildings

    #add description
    if "description" in list_of_columns:
        shapefile["description"] = shapefile['description']
    elif 'addr:housename' in list_of_columns:
        shapefile["description"] = shapefile['addr:housename']
    elif 'amenity' in list_of_columns:
        shapefile["description"] = shapefile['amenity']
    else:
        shapefile["description"] = [np.nan]*no_buildings

    shapefile["category"] = shapefile['building']
    shapefile["Name"] = [key + str(x + 1000) for x in
                         range(no_buildings)]  # start in a big number to avoid potential confusion\
    result = shapefile[
        ["Name", "height_ag", "floors_ag", "description", "category", "geometry",  "REFERENCE"]]

    result.reset_index(inplace=True, drop=True)

    return result

def erase_no_surrounding_areas(all_surroundings, area_buffer):
    polygon = area_buffer.geometry[0]
    all_surroundings.within(polygon)
    subset = all_surroundings[all_surroundings.within(polygon)]
    return subset

def geometry_extractor_osm(locator, config):
    """this is where the action happens if it is more than a few lines in ``main``.
    NOTE: ADD YOUR SCRIPT'S DOCUMENATION HERE (how)
    NOTE: RENAME THIS FUNCTION (SHOULD PROBABLY BE THE SAME NAME AS THE MODULE)
    """

    # local variables:
    buffer_m = config.surroundings_helper.buffer
    buildings_height = config.surroundings_helper.height_ag
    buildings_floors = config.surroundings_helper.floors_ag
    shapefile_out_path = locator.get_surroundings_geometry()
    zone = Gdf.from_file(locator.get_zone_geometry())

    # trnasform zone file to geographic coordinates
    zone = zone.to_crs(get_geographic_coordinate_system())
    lon = zone.geometry[0].centroid.coords.xy[0][0]
    lat = zone.geometry[0].centroid.coords.xy[1][0]
    zone = zone.to_crs(get_projected_coordinate_system(float(lat), float(lon)))

    # get a polygon of the surrounding area, and one polygon representative of the zone area
    area_with_buffer, _ = calc_surrounding_area(zone, buffer_m)
    area_with_buffer.crs = get_projected_coordinate_system(float(lat), float(lon))
    area_with_buffer = area_with_buffer.to_crs(get_geographic_coordinate_system())

    # get footprints of all the surroundings
    all_surroundings = ox.footprints.create_footprints_gdf(polygon=area_with_buffer['geometry'].values[0])

    # erase overlapping area
    surroundings = erase_no_surrounding_areas(all_surroundings.copy(), area_with_buffer)

    assert surroundings.shape[0] > 0, 'No buildings were found within range based on buffer parameter.'

    # clean attributes of height, name and number of floors
    result = clean_attributes(surroundings, buildings_height, buildings_floors, key="CEA")
    result = result.to_crs(get_projected_coordinate_system(float(lat), float(lon)))

    # save to shapefile
    result.to_file(shapefile_out_path)

def main(config):
    """
    Create the surroundings.shp file

    :param config:
    :type config: cea.config.Configuration
    :return:
    """
    assert os.path.exists(config.scenario), 'Scenario not found: %s' % config.scenario
    locator = cea.inputlocator.InputLocator(config.scenario)

    geometry_extractor_osm(locator, config)

if __name__ == '__main__':
    main(cea.config.Configuration())
<commit_msg>Reformat code<commit_after>"""
This script extracts surrounding buildings of the zone geometry from Open street maps
"""
from __future__ import division
from __future__ import print_function

import math
import os

import numpy as np
import osmnx as ox
import shapely.geometry as geometry
from geopandas import GeoDataFrame as Gdf
from geopandas import overlay
from scipy.spatial import Delaunay
from shapely.ops import cascaded_union, polygonize
import pandas as pd

import cea.config
import cea.inputlocator
from cea.datamanagement.zone_helper import parse_building_floors
from cea.demand import constants
from cea.utilities.standardize_coordinates import get_projected_coordinate_system, get_geographic_coordinate_system

__author__ = "Jimeno Fonseca"
__copyright__ = "Copyright 2018, Architecture and Building Systems - ETH Zurich"
__credits__ = ["Jimeno Fonseca"]
__license__ = "MIT"
__version__ = "0.1"
__maintainer__ = "Daren Thomas"
__email__ = "cea@arch.ethz.ch"
__status__ = "Production"

def alpha_shape(points, alpha):
    """
    Compute the alpha shape (concave hull) of a set of points.

    :param points: Iterable container of points.
    :param alpha: alpha value to influence the gooeyness of the border. Smaller
                  numbers don't fall inward as much as larger numbers. Too large,
                  and you lose everything!
    """
    if len(points) < 4:
        # When you have a triangle, there is no sense in computing an alpha
        # shape.
        return geometry.MultiPoint(list(points)).convex_hull

    def add_edge(edges, edge_points, coords, i, j):
        """Add a line between the i-th and j-th points, if not in the list already"""
        if (i, j) in edges or (j, i) in edges:
            # already added
            return
        edges.add((i, j))
        edge_points.append(coords[[i, j]])

    coords = np.array([point.coords[0] for point in points])

    tri = Delaunay(coords)
    edges = set()
    edge_points = []
    # loop over triangles:
    # ia, ib, ic = indices of corner points of the triangle
    for ia, ib, ic in tri.vertices:
        pa = coords[ia]
        pb = coords[ib]
        pc = coords[ic]

        # Lengths of sides of triangle
        a = math.sqrt((pa[0] - pb[0]) ** 2 + (pa[1] - pb[1]) ** 2)
        b = math.sqrt((pb[0] - pc[0]) ** 2 + (pb[1] - pc[1]) ** 2)
        c = math.sqrt((pc[0] - pa[0]) ** 2 + (pc[1] - pa[1]) ** 2)

        # Semiperimeter of triangle
        s = (a + b + c) / 2.0

        # Area of triangle by Heron's formula
        area = math.sqrt(s * (s - a) * (s - b) * (s - c))
        circum_r = a * b * c / (4.0 * area)

        # Here's the radius filter.
        # print circum_r
        if circum_r < 1.0 / alpha:
            add_edge(edges, edge_points, coords, ia, ib)
            add_edge(edges, edge_points, coords, ib, ic)
            add_edge(edges, edge_points, coords, ic, ia)

    m = geometry.MultiLineString(edge_points)
    triangles = list(polygonize(m))
    return cascaded_union(triangles), edge_points

def calc_surrounding_area(zone_gdf, buffer_m):
    geometry_without_holes = zone_gdf.convex_hull
    geometry_without_holes_gdf = Gdf(geometry=geometry_without_holes.values)
    geometry_without_holes_gdf["one_class"] = "buildings"
    geometry_merged = geometry_without_holes_gdf.dissolve(by='one_class', aggfunc='sum')
    geometry_merged_final = Gdf(geometry=geometry_merged.convex_hull)
    new_buffer = Gdf(geometry=geometry_merged_final.buffer(buffer_m))
    area = overlay(geometry_merged_final, new_buffer, how='symmetric_difference')

    # THIS IS ANOTHER METHOD, NOT FUNCTIONAL THOUGH
    # from shapely.ops import Point
    # # new GeoDataFrame with same columns
    # points = []
    # # Extraction of the polygon nodes and attributes values from polys and integration into the new GeoDataFrame
    # for index, row in zone_gdf.iterrows():
    #     for j in list(row['geometry'].exterior.coords):
    #         points.append(Point(j))
    #
    # concave_hull, edge_points = alpha_shape(points, alpha=0.1)
    # simple_polygons = [x for x in concave_hull]
    # geometry_merged_final = Gdf(geometry=simple_polygons)
    # geometry_merged_final.plot()
    # plt.show()
    # new_buffer = Gdf(geometry=geometry_merged_final.buffer(buffer_m))
    # area = overlay(geometry_merged_final, new_buffer, how='symmetric_difference')
    # area.plot()

    return area, geometry_merged_final

def clean_attributes(shapefile, buildings_height, buildings_floors, key):
    # local variables
    no_buildings = shapefile.shape[0]
    list_of_columns = shapefile.columns
    if buildings_height is None and buildings_floors is None:
        print('Warning! you have not indicated a height or number of floors above ground for the buildings, '
              'we are reverting to data stored in Open Street Maps (It might not be accurate at all),'
              'if we do not find data in OSM for a particular building, we get the median in the surroundings, '
              'if we do not get any data we assume 4 floors per building')
        # Check which attributes the OSM has, Sometimes it does not have any and indicate the data source
        if 'building:levels' not in list_of_columns:
            shapefile['building:levels'] = [3] * no_buildings
            shapefile['REFERENCE'] = "CEA - assumption"
        elif pd.isnull(shapefile['building:levels']).all():
            shapefile['building:levels'] = [3] * no_buildings
            shapefile['REFERENCE'] = "CEA - assumption"
        else:
            shapefile['REFERENCE'] = ["OSM - median" if x is np.nan else "OSM - as it is" for x in
                                      shapefile['building:levels']]
        if 'roof:levels' not in list_of_columns:
            shapefile['roof:levels'] = [1] * no_buildings

        # get the median from the area:
        data_osm_floors1 = shapefile['building:levels'].fillna(0)
        data_osm_floors2 = shapefile['roof:levels'].fillna(0)
        data_floors_sum = [x + y for x, y in
                           zip([parse_building_floors(x) for x in data_osm_floors1],
                               [parse_building_floors(x) for x in data_osm_floors2])]
        data_floors_sum_with_nan = [np.nan if x < 1.0 else x for x in data_floors_sum]
        data_osm_floors_joined = int(
            math.ceil(np.nanmedian(data_floors_sum_with_nan)))  # median so we get close to the worse case
        shapefile["floors_ag"] = [int(x) if x is not np.nan else data_osm_floors_joined for x in
                                  data_floors_sum_with_nan]
        shapefile["height_ag"] = shapefile["floors_ag"] * constants.H_F
    else:
        shapefile['REFERENCE'] = "User - assumption"
        if buildings_height is None and buildings_floors is not None:
            shapefile["floors_ag"] = [buildings_floors] * no_buildings
            shapefile["height_ag"] = shapefile["floors_ag"] * constants.H_F
        elif buildings_height is not None and buildings_floors is None:
            shapefile["height_ag"] = [buildings_height] * no_buildings
            shapefile["floors_ag"] = [int(math.floor(x)) for x in shapefile["height_ag"] / constants.H_F]
        else:  # both are not none
            shapefile["height_ag"] = [buildings_height] * no_buildings
            shapefile["floors_ag"] = [buildings_floors] * no_buildings

    # add description
    if "description" in list_of_columns:
        shapefile["description"] = shapefile['description']
    elif 'addr:housename' in list_of_columns:
        shapefile["description"] = shapefile['addr:housename']
    elif 'amenity' in list_of_columns:
        shapefile["description"] = shapefile['amenity']
    else:
        shapefile["description"] = [np.nan] * no_buildings

    shapefile["category"] = shapefile['building']
    shapefile["Name"] = [key + str(x + 1000) for x in
                         range(no_buildings)]  # start in a big number to avoid potential confusion\
    result = shapefile[
        ["Name", "height_ag", "floors_ag", "description", "category", "geometry", "REFERENCE"]]

    result.reset_index(inplace=True, drop=True)

    return result

def erase_no_surrounding_areas(all_surroundings, area_buffer):
    polygon = area_buffer.geometry[0]
    all_surroundings.within(polygon)
    subset = all_surroundings[all_surroundings.within(polygon)]
    return subset

def geometry_extractor_osm(locator, config):
    """this is where the action happens if it is more than a few lines in ``main``.
    NOTE: ADD YOUR SCRIPT'S DOCUMENATION HERE (how)
    NOTE: RENAME THIS FUNCTION (SHOULD PROBABLY BE THE SAME NAME AS THE MODULE)
    """

    # local variables:
    buffer_m = config.surroundings_helper.buffer
    buildings_height = config.surroundings_helper.height_ag
    buildings_floors = config.surroundings_helper.floors_ag
    shapefile_out_path = locator.get_surroundings_geometry()
    zone = Gdf.from_file(locator.get_zone_geometry())

    # trnasform zone file to geographic coordinates
    zone = zone.to_crs(get_geographic_coordinate_system())
    lon = zone.geometry[0].centroid.coords.xy[0][0]
    lat = zone.geometry[0].centroid.coords.xy[1][0]
    zone = zone.to_crs(get_projected_coordinate_system(float(lat), float(lon)))

    # get a polygon of the surrounding area, and one polygon representative of the zone area
    area_with_buffer, _ = calc_surrounding_area(zone, buffer_m)
    area_with_buffer.crs = get_projected_coordinate_system(float(lat), float(lon))
    area_with_buffer = area_with_buffer.to_crs(get_geographic_coordinate_system())

    # get footprints of all the surroundings
    all_surroundings = ox.footprints.create_footprints_gdf(polygon=area_with_buffer['geometry'].values[0])

    # erase overlapping area
    surroundings = erase_no_surrounding_areas(all_surroundings.copy(), area_with_buffer)

    assert surroundings.shape[0] > 0, 'No buildings were found within range based on buffer parameter.'

    # clean attributes of height, name and number of floors
    result = clean_attributes(surroundings, buildings_height, buildings_floors, key="CEA")
    result = result.to_crs(get_projected_coordinate_system(float(lat), float(lon)))

    # save to shapefile
    result.to_file(shapefile_out_path)

def main(config):
    """
    Create the surroundings.shp file

    :param config:
    :type config: cea.config.Configuration
    :return:
    """
    assert os.path.exists(config.scenario), 'Scenario not found: %s' % config.scenario
    locator = cea.inputlocator.InputLocator(config.scenario)

    geometry_extractor_osm(locator, config)

if __name__ == '__main__':
    main(cea.config.Configuration())
<|endoftext|>


<commit_before>from pprint import pformat
import datetime
from dateutil.parser import parse
from dateutil.tz import tzutc

from marquise import Marquise

import payload as p

keys_to_delete = [
    "timestamp",
    "volume",
    "created_at",
    "updated_at",
    "id",
    "size",
]

def process_sample(sample):
    sample = sample.as_dict()
    processed = []
    if ("event_type" in sample["resource_metadata"]):
        processed.append(process_consolidated(sample))
    processed.append(process_raw(sample))
    return processed

def _remove_extraneous(sourcedict):
    for k in keys_to_delete:
        try:
            del[sourcedict[k]]
        except KeyError:
            pass

#Potentially destructive on sample
def process_raw(sample):
    # Generate the unique identifer for the sample
    identifier = sample["resource_id"] + sample["project_id"] + \
                 sample["name"] + sample["type"] + sample["unit"]
    address = Marquise.hash_identifier(identifier)

    # Sanitize timestamp (will parse timestamp to nanoseconds since epoch)
    timestamp = sanitize_timestamp(sample["timestamp"])

    # Our payload is the volume (later parsed to
    # "counter_volume" in ceilometer)
    payload = sanitize(sample["volume"])

    # Rebuild the sample as a source dict
    sourcedict = dict(sample)

    # Vaultaire cares about the datatype of the payload
    if type(payload) == float:
        sourcedict["_float"] = 1

    # Cast unit as a special metadata type
    sourcedict["_unit"] = sanitize(sourcedict.pop("unit"))

    # If it's a cumulative value, we need to tell vaultaire
    if sourcedict["type"] == "cumulative":
        sourcedict["_counter"] = 1

    # Cast Identifier sections with unique names, in case of
    # metadata overlap

    # XXX: why do we call these counter_* even when they're
    # not counters?
    sourcedict["counter_name"] = sanitize(sourcedict.pop("name"))
    sourcedict["counter_type"] = sanitize(sourcedict.pop("type"))

    _remove_extraneous(sourcedict)

    # Remove the original resource_metadata and substitute
    # our own flattened version
    sourcedict.update(flatten(sourcedict.pop("resource_metadata")))

    for k, v in sourcedict.iteritems():
        sourcedict[k] = sanitize(str(v))
    return (address, sourcedict, timestamp, payload)

def process_consolidated(sample):
    # Pull out and clean fields which are always present

    name         = sample["name"]
    project_id   = sample["project_id"]
    resource_id  = sample["resource_id"]
    metadata     = sample["resource_metadata"]
    ## Cast unit as a special metadata type
    counter_unit = sanitize(sample["unit"])
    counter_type = sample["type"]
    ## Sanitize timestamp (will parse timestamp to nanoseconds since epoch)
    timestamp    = sanitize_timestamp(sample["timestamp"])
    ## Our payload is the volume (later parsed to "counter_volume" in ceilometer)
    payload      = sanitize(sample["volume"])
    ## Modify the payload for instance events
    if name.startswith("instance"):
        payload = p.constructPayload(metadata["event_type"], metadata["message"], p.instanceToRawPayload(metadata["instance_type"]))

    # Build the source dict

    sourcedict = {}
    sourcedict["_event"]        = 1
    sourcedict["project_id"]    = project_id
    sourcedict["resource_id"]   = resource_id
    sourcedict["counter_name"]  = name
    sourcedict["counter_unit"]  = counter_unit
    sourcedict["counter_type"]  = counter_type
    sourcedict["display_name"]  = sample.get("display_name", "")
    ## Vaultaire cares about the datatype of the payload
    if type(payload) == float:
        sourcedict["_float"] = 1

    ## If it's a cumulative value, we need to tell vaultaire
    if counter_type == "cumulative":
        sourcedict["_counter"] = 1

    ## Add specifics for CPU
    cpu_number = metadata.get("cpu_number", "")
    if name == "cpu":
        sourcedict["cpu_number"] = cpu_number

    # Generate the unique identifer for the sample

    ## Instance-specific

    ## If the flavor object is present, the flavor type is the "name" field of the flavor object
    ## Otherwise it is "instance_type"
    flavor_type = ""
    if "flavor" in sample:
        flavor_type = sample["flavor"].get("name", "")
    elif "instance_type" in sample:
        flavor_type = sample["instance_type"]

    ## Common = r_id + p_id + counter_(name, type, unit)
    ## When present we also care about resource-specifics.
    ## Currently flavor type and cpu_number
    identifier = resource_id + project_id + name + counter_type + counter_unit + \
                 flavor_type + cpu_number + "_event"
    address = Marquise.hash_identifier(identifier)

    return (address, sourcedict, timestamp, payload)

def sanitize(v):
    """Sanitize a value into something that Marquise can use. This weeds
    out None keys/values, and ensures that timestamps are consistently
    formatted.
    """
    if v is None:
        return ""
    if v in (True,False):
        return 1 if v is True else 0
    if type(v) is unicode:
        v = str(v)
    if type(v) is str: # XXX: will be incorrect/ambiguous in Python 3.
        v = v.replace(":","-")
        v = v.replace(",","-")
    return v

def flatten(n, prefix=""):
    """Take a (potentially) nested dictionary and flatten it into a single
    level. Also remove any keys/values that Marquise/Vaultaire can't handle.
    """
    flattened_dict = {}
    for k,v in n.items():
        k = sanitize(k)

        # Vaultaire doesn't care about generated URLs for Ceilometer
        # API references.
        if str(k) == "links":
            continue
        if str(k).endswith('_url'):
            continue

        # Vaultaire doesn't want values if they have no contents.
        if v is None:
            continue

        # If key has a parent, concatenate it into the new keyname.
        if prefix:
            k = "{}-{}".format(prefix,k)

        # This was previously a check for __iter__, but strings have those now,
        # so let's just check for dict-ness instead. No good on lists anyway.
        if type(v) is not dict:
            v = sanitize(v)
            flattened_dict[k] = v
        else:
            flattened_dict.update(flatten(v, k))
    return flattened_dict

def sanitize_timestamp(v):
    """Convert a timestamp value into a standard form. Does no exception
    handling, as we really want to know if this fails."""
    if type(v) is unicode:
        v = str(v)
    # Try and take a value and use dateutil to parse it.  If there's no TZ
    # spec in the string, assume it's UTC because that's what Ceilometer
    # uses.
    # Eg. 2014-08-10T12:14:13Z      # timezone-aware
    # Eg. 2014-08-10 12:14:13       # timezone-naive
    # Eg. 2014-08-10 12:14:13+1000  # timezone-aware
    NANOSECONDS_PER_SECOND = 10**9
    if type(v) is datetime.datetime:
        timestamp = v
    else: # We have a string.
        timestamp = parse(v)
        if timestamp.tzinfo is None:
            timestamp = timestamp.replace(tzinfo=tzutc())
    # If we get here, we've successfully grabbed a datetime.
    epoch = datetime.datetime(1970, 1, 1, tzinfo=tzutc())
    time_since_epoch = (timestamp - epoch).total_seconds() # total_seconds() is in Py2.7 and later.
    return int(time_since_epoch * NANOSECONDS_PER_SECOND)
<commit_msg>Sourcedict + address identifier fixes<commit_after>from pprint import pformat
import datetime
from dateutil.parser import parse
from dateutil.tz import tzutc

from marquise import Marquise

import payload as p

keys_to_delete = [
    "timestamp",
    "volume",
    "created_at",
    "updated_at",
    "id",
    "size",
]

def process_sample(sample):
    sample = sample.as_dict()
    processed = []
    if ("event_type" in sample["resource_metadata"]):
        processed.append(process_consolidated(sample))
    processed.append(process_raw(sample))
    return processed

def _remove_extraneous(sourcedict):
    for k in keys_to_delete:
        try:
            del[sourcedict[k]]
        except KeyError:
            pass

#Potentially destructive on sample
def process_raw(sample):
    cpu_number = sample["resource_metadata"].get("cpu_number", "")
    event_type = sample["resource_metadata"].get("event_type", "")
    # Generate the unique identifer for the sample
    identifier = sample["resource_id"] + sample["project_id"] + \
                 sample["name"] + sample["type"] + sample["unit"] + \
                 event_type + cpu_number
    address = Marquise.hash_identifier(identifier)

    # Sanitize timestamp (will parse timestamp to nanoseconds since epoch)
    timestamp = sanitize_timestamp(sample["timestamp"])

    # Our payload is the volume (later parsed to
    # "counter_volume" in ceilometer)
    payload = sanitize(sample["volume"])

    # Rebuild the sample as a source dict
    sourcedict = dict(sample)

    # Vaultaire cares about the datatype of the payload
    if type(payload) == float:
        sourcedict["_float"] = 1

    # Cast unit as a special metadata type
    sourcedict["_unit"] = sanitize(sourcedict.pop("unit"))

    # If it's a cumulative value, we need to tell vaultaire
    if sourcedict["type"] == "cumulative":
        sourcedict["_counter"] = 1

    # Cast Identifier sections with unique names, in case of
    # metadata overlap

    # XXX: why do we call these counter_* even when they're
    # not counters?
    sourcedict["counter_name"] = sanitize(sourcedict.pop("name"))
    sourcedict["counter_type"] = sanitize(sourcedict.pop("type"))

    _remove_extraneous(sourcedict)

    # Remove the original resource_metadata and substitute
    # our own flattened version
    sourcedict.update(flatten(sourcedict.pop("resource_metadata")))

    for k, v in sourcedict.iteritems():
        sourcedict[k] = sanitize(str(v))
    return (address, sourcedict, timestamp, payload)

def process_consolidated(sample):
    # Pull out and clean fields which are always present

    name         = sample["name"]
    project_id   = sample["project_id"]
    resource_id  = sample["resource_id"]
    metadata     = sample["resource_metadata"]
    ## Cast unit as a special metadata type
    counter_unit = sanitize(sample["unit"])
    counter_type = sample["type"]
    ## Sanitize timestamp (will parse timestamp to nanoseconds since epoch)
    timestamp    = sanitize_timestamp(sample["timestamp"])
    ## Our payload is the volume (later parsed to "counter_volume" in ceilometer)
    payload      = sanitize(sample["volume"])
    ## Modify the payload for instance events
    if name.startswith("instance"):
        payload = p.constructPayload(metadata["event_type"], metadata["message"], p.instanceToRawPayload(metadata["instance_type"]))

    # Build the source dict

    sourcedict = {}
    sourcedict["_event"]        = 1
    sourcedict["project_id"]    = project_id
    sourcedict["resource_id"]   = resource_id
    sourcedict["counter_name"]  = name
    sourcedict["counter_unit"]  = counter_unit
    sourcedict["counter_type"]  = counter_type
    sourcedict["display_name"]  = sample.get("display_name", "")
    ## Vaultaire cares about the datatype of the payload
    if type(payload) == float:
        sourcedict["_float"] = 1

    ## If it's a cumulative value, we need to tell vaultaire
    if counter_type == "cumulative":
        sourcedict["_counter"] = 1

    ## Add specifics for CPU
    cpu_number = metadata.get("cpu_number", "")
    if cpu_number != "":
        sourcedict["cpu_number"] = cpu_number

    # Generate the unique identifer for the sample

    ## Instance-specific

    ## If the flavor object is present, the flavor type is the "name" field of the flavor object
    ## Otherwise it is "instance_type"
    flavor_type = ""
    if "flavor" in sample:
        flavor_type = sample["flavor"].get("name", "")
    elif "instance_type" in sample:
        flavor_type = sample["instance_type"]

    ## Common = r_id + p_id + counter_(name, type, unit)
    ## When present we also care about resource-specifics.
    ## Currently flavor type and cpu_number
    identifier = resource_id + project_id + name + counter_type + counter_unit + \
                 flavor_type + cpu_number + "_event"
    address = Marquise.hash_identifier(identifier)

    return (address, sourcedict, timestamp, payload)

def sanitize(v):
    """Sanitize a value into something that Marquise can use. This weeds
    out None keys/values, and ensures that timestamps are consistently
    formatted.
    """
    if v is None:
        return ""
    if v in (True,False):
        return 1 if v is True else 0
    if type(v) is unicode:
        v = str(v)
    if type(v) is str: # XXX: will be incorrect/ambiguous in Python 3.
        v = v.replace(":","-")
        v = v.replace(",","-")
    return v

def flatten(n, prefix=""):
    """Take a (potentially) nested dictionary and flatten it into a single
    level. Also remove any keys/values that Marquise/Vaultaire can't handle.
    """
    flattened_dict = {}
    for k,v in n.items():
        k = sanitize(k)

        # Vaultaire doesn't care about generated URLs for Ceilometer
        # API references.
        if str(k) == "links":
            continue
        if str(k).endswith('_url'):
            continue

        # Vaultaire doesn't want values if they have no contents.
        if v is None:
            continue

        # If key has a parent, concatenate it into the new keyname.
        if prefix:
            k = "{}-{}".format(prefix,k)

        # This was previously a check for __iter__, but strings have those now,
        # so let's just check for dict-ness instead. No good on lists anyway.
        if type(v) is not dict:
            v = sanitize(v)
            flattened_dict[k] = v
        else:
            flattened_dict.update(flatten(v, k))
    return flattened_dict

def sanitize_timestamp(v):
    """Convert a timestamp value into a standard form. Does no exception
    handling, as we really want to know if this fails."""
    if type(v) is unicode:
        v = str(v)
    # Try and take a value and use dateutil to parse it.  If there's no TZ
    # spec in the string, assume it's UTC because that's what Ceilometer
    # uses.
    # Eg. 2014-08-10T12:14:13Z      # timezone-aware
    # Eg. 2014-08-10 12:14:13       # timezone-naive
    # Eg. 2014-08-10 12:14:13+1000  # timezone-aware
    NANOSECONDS_PER_SECOND = 10**9
    if type(v) is datetime.datetime:
        timestamp = v
    else: # We have a string.
        timestamp = parse(v)
        if timestamp.tzinfo is None:
            timestamp = timestamp.replace(tzinfo=tzutc())
    # If we get here, we've successfully grabbed a datetime.
    epoch = datetime.datetime(1970, 1, 1, tzinfo=tzutc())
    time_since_epoch = (timestamp - epoch).total_seconds() # total_seconds() is in Py2.7 and later.
    return int(time_since_epoch * NANOSECONDS_PER_SECOND)
<|endoftext|>


<commit_before>import re
import urllib2
import string
import sqlite3

class Google_Blacklist(object):
    """
    Google Blacklist class that is used to fetch and prepare hashes to be
    stored in the database.
    """
    def __init__(self,key,url,dbname="",badware_type="malware"):
        """
        The constructor initializes the module.
        """
        badware_dict = {"malware": "M","black": "B"}
        try:
            self.key = key
            self.dbname = dbname
            self.url = url
            self.badware_type = badware_type
            try:
                self.badware_code = badware_dict[badware_type]
            except KeyError:
                raise KeyError("Invalid Badware Type")
            assert self.key
            assert self.dbname
            assert self.url
            assert self.badware_type
        except AssertionError:
            raise AssertionError("Key/Dbname/URL is/are missing.")

    def fetch_data(self):
            try:
                conn = sqlite3.connect(self.dbname)
                cur = conn.cursor()
                cur.execute("select * from %s_version;" %(self.badware_type))
                row = cur.fetchall()
                s = string.Template(self.url)
                try:
                    assert row
                    self.version_number = row[0][0]
                    self.final_url = s.safe_substitute(key = self.key,\
                                                       badware_type = self.badware_type,\
                                                       version = self.version_number)
                    self.fetch_url_pointer = urllib2.urlopen(self.final_url)
                    self.url_hashes_data = self.fetch_url_pointer.readlines()
                    if self.url_hashes_data == []:
                        return 0
                    for url_hash in self.url_hashes_data[1:-1]:
                        if re.match("^-\w+", url_hash):
                            cur.execute("delete from url_hashes_table where badware_type='%s' and url_hash='%s';" %(self.badware_code, url_hash[1:].strip()))
                            del self.url_hashes_data[self.url_hashes_data.index(url_hash)]
                    new_version_number = ":".join(re.compile("\d\.\d+").search(self.url_hashes_data[0]).group().split("."))
                    cur.execute("update %s_version set version_number='%s' where version_number='%s';" %(self.badware_type, new_version_number, self.version_number))
                    self.version_number = new_version_number
                    for url_hash in self.url_hashes_data[1:]:
                        if not url_hash == '\n':
                            cur.execute("insert into url_hashes_table (badware_type,url_hash) values ('%s','%s');" %(self.badware_code, url_hash[1:].strip()))
                except AssertionError:
                    # Start from Version 1:-1
                    self.version_number = "1:-1"
                    self.final_url = s.safe_substitute(key = self.key, badware_type = self.badware_type, version = self.version_number)
                    self.fetch_url_pointer = urllib2.urlopen(self.final_url)
                    self.url_hashes_data = self.fetch_url_pointer.readlines()
                    if self.url_hashes_data == []:
                        # No data, so no point checking version
                        # number. This case might be because of
                        # throttling or no updates available.
                        return 0
                    new_version_number = ":".join(re.compile("\d\.\d+").search(self.url_hashes_data[0]).group().split("."))
                    self.version_number = new_version_number
                    cur.execute("insert into %s_version (version_number) values ('%s');" %(self.badware_type, self.version_number))
                    for url_hash in self.url_hashes_data[1:]:
                        if not url_hash == '\n':
                            cur.execute("insert into url_hashes_table (badware_type,url_hash) values ('%s','%s');" %(self.badware_code, url_hash[1:].strip()))
                cur.close()
                conn.commit()
                conn.close()
                return 0
            except sqlite3.DatabaseError:
                raise sqlite3.DatabaseError("Error in a Database Specific Operation.")

<commit_msg>Added python2.4 support in hash_fetch<commit_after>import re
import urllib2
import string
try:
    import sqlite3 as sqlite
except ImportError:
    from pysqlite2 import dbapi2 as sqlite

class Google_Blacklist(object):
    """
    Google Blacklist class that is used to fetch and prepare hashes to be
    stored in the database.
    """
    def __init__(self,key,url,dbname="",badware_type="malware"):
        """
        The constructor initializes the module.
        """
        badware_dict = {"malware": "M","black": "B"}
        try:
            self.key = key
            self.dbname = dbname
            self.url = url
            self.badware_type = badware_type
            try:
                self.badware_code = badware_dict[badware_type]
            except KeyError:
                raise KeyError("Invalid Badware Type")
            assert self.key
            assert self.dbname
            assert self.url
            assert self.badware_type
        except AssertionError:
            raise AssertionError("Key/Dbname/URL is/are missing.")

    def fetch_data(self):
            try:
                conn = sqlite.connect(self.dbname)
                cur = conn.cursor()
                cur.execute("select * from %s_version;" %(self.badware_type))
                row = cur.fetchall()
                s = string.Template(self.url)
                try:
                    assert row
                    self.version_number = row[0][0]
                    self.final_url = s.safe_substitute(key = self.key,\
                                                       badware_type = self.badware_type,\
                                                       version = self.version_number)
                    self.fetch_url_pointer = urllib2.urlopen(self.final_url)
                    self.url_hashes_data = self.fetch_url_pointer.readlines()
                    if self.url_hashes_data == []:
                        return 0
                    for url_hash in self.url_hashes_data[1:-1]:
                        if re.match("^-\w+", url_hash):
                            cur.execute("delete from url_hashes_table where badware_type='%s' and url_hash='%s';" %(self.badware_code, url_hash[1:].strip()))
                            del self.url_hashes_data[self.url_hashes_data.index(url_hash)]
                    new_version_number = ":".join(re.compile("\d\.\d+").search(self.url_hashes_data[0]).group().split("."))
                    cur.execute("update %s_version set version_number='%s' where version_number='%s';" %(self.badware_type, new_version_number, self.version_number))
                    self.version_number = new_version_number
                    for url_hash in self.url_hashes_data[1:]:
                        if not url_hash == '\n':
                            cur.execute("insert into url_hashes_table (badware_type,url_hash) values ('%s','%s');" %(self.badware_code, url_hash[1:].strip()))
                except AssertionError:
                    # Start from Version 1:-1
                    self.version_number = "1:-1"
                    self.final_url = s.safe_substitute(key = self.key, badware_type = self.badware_type, version = self.version_number)
                    self.fetch_url_pointer = urllib2.urlopen(self.final_url)
                    self.url_hashes_data = self.fetch_url_pointer.readlines()
                    if self.url_hashes_data == []:
                        # No data, so no point checking version
                        # number. This case might be because of
                        # throttling or no updates available.
                        return 0
                    new_version_number = ":".join(re.compile("\d\.\d+").search(self.url_hashes_data[0]).group().split("."))
                    self.version_number = new_version_number
                    cur.execute("insert into %s_version (version_number) values ('%s');" %(self.badware_type, self.version_number))
                    for url_hash in self.url_hashes_data[1:]:
                        if not url_hash == '\n':
                            cur.execute("insert into url_hashes_table (badware_type,url_hash) values ('%s','%s');" %(self.badware_code, url_hash[1:].strip()))
                cur.close()
                conn.commit()
                conn.close()
                return 0
            except sqlite.DatabaseError:
                raise sqlite.DatabaseError("Error in a Database Specific Operation.")

<|endoftext|>


<commit_before>import logging
import os
import warnings

import pandas as pd

import csv
import itertools

from cellpy import filefinder, prms
from cellpy.exceptions import ExportFailed, NullData, WrongFileVersion
import cellpy.parameters.internal_settings

# logger = logging.getLogger(__name__)

def look_up_and_get(cellpy_file_name, table_name, root=None):
    """Extracts table from cellpy hdf5-file."""

    # infoname = '/CellpyData/info'
    # dataname = '/CellpyData/dfdata'
    # summaryname = '/CellpyData/dfsummary'
    # fidname = '/CellpyData/fidtable'
    # stepname = '/CellpyData/step_table'

    if root is None:
        root = "/CellpyData"
    table_path = "/".join([root, table_name])

    logging.debug(f"look_up_and_get({cellpy_file_name}, {table_name}")
    store = pd.HDFStore(cellpy_file_name)
    try:
        table = store.select(table_path)
    except KeyError as e:
        logging.warning("Could not read the table")
        store.close()
        raise WrongFileVersion(e)
    return table

def create_folder_structure(project_name, batch_name):
    """This function creates a folder structure for the batch project.

    The folder structure consists of main working folder ``project_name`
    located in the ``outdatadir`` (as defined in the cellpy configuration file)
    with a sub-folder named ``batch_name``. It also creates a folder
    inside the ``batch_name`` folder for storing the raw data.
    If the folders does not exist, they will be made. The function also returns
    the name of the info-df.

    Args:
        project_name: name of the project
        batch_name: name of the batch

    Returns: (info_file, (project_dir, batch_dir, raw_dir))

    """
    out_data_dir = prms.Paths["outdatadir"]
    project_dir = os.path.join(out_data_dir, project_name)
    batch_dir = os.path.join(project_dir, batch_name)
    raw_dir = os.path.join(batch_dir, "raw_data")

    # create folders
    if not os.path.isdir(project_dir):
        os.mkdir(project_dir)
    if not os.path.isdir(batch_dir):
        os.mkdir(batch_dir)
    if not os.path.isdir(raw_dir):
        os.mkdir(raw_dir)

    # create file-name for the info_df (json)
    info_file = "cellpy_batch_%s.json" % batch_name
    info_file = os.path.join(project_dir, info_file)
    return info_file, (project_dir, batch_dir, raw_dir)

def find_files(info_dict, filename_cache=None):
    # searches for the raw data files and the cellpyfile-name
    for run_name in info_dict["filenames"]:
        logging.debug(f"checking for {run_name}")
        if prms._use_filename_cache:
            raw_files, cellpyfile, filename_cache = filefinder.search_for_files(
                run_name, cache=filename_cache
            )
        else:
            raw_files, cellpyfile = filefinder.search_for_files(run_name)
        if not raw_files:
            raw_files = None
        info_dict["raw_file_names"].append(raw_files)
        info_dict["cellpy_file_names"].append(cellpyfile)

    return info_dict

def fix_groups(groups):
    """Takes care of strange group numbers."""
    _groups = []
    for g in groups:
        try:
            if not float(g) > 0:
                _groups.append(1000)
            else:
                _groups.append(int(g))
        except TypeError as e:
            logging.info("Error in reading group number (check your db)")
            logging.debug(g)
            logging.debug(e)
            _groups.append(1000)
    return _groups

def save_multi(data, file_name, sep=";"):
    """Convenience function for storing data column-wise in a csv-file."""
    logging.debug("saving multi")
    with open(file_name, "w", newline="") as f:
        logging.debug(f"{file_name} opened")
        writer = csv.writer(f, delimiter=sep)
        try:
            writer.writerows(itertools.zip_longest(*data))
            logging.info(f"{file_name} OK")
        except Exception as e:
            logging.info(f"Exception encountered in batch._save_multi: {e}")
            raise ExportFailed
        logging.debug("wrote rows using itertools in _save_multi")

def make_unique_groups(info_df):
    """This function cleans up the group numbers a bit."""
    # fixes group numbering
    unique_g = info_df.groups.unique()
    unique_g = sorted(unique_g)
    new_unique_g = list(range(len(unique_g)))
    info_df["sub_groups"] = info_df["groups"] * 0
    for i, j in zip(unique_g, new_unique_g):
        counter = 1
        for indx, row in info_df.loc[info_df.groups == i].iterrows():
            info_df.at[indx, "sub_groups"] = counter
            # info_df.set_value(indx, "sub_groups", counter)
            counter += 1
        info_df.loc[info_df.groups == i, "groups"] = j + 1
    return info_df

def _remove_date(label):
    _ = label.split("_")
    return _[1] + "_" + _[2]

def create_labels(label, *args):
    """Returns a re-formatted label (currently it only removes the dates
    from the run-name)"""
    return _remove_date(label)

def create_selected_summaries_dict(summaries_list):
    """Creates a dictionary with summary column headers.

    Examples:
        >>> summaries_to_output = ["discharge_capacity", "charge_capacity"]
        >>> summaries_to_output_dict = create_selected_summaries_dict(
        >>>    summaries_to_output
        >>> )
        >>> print(summaries_to_output_dict)
        {'discharge_capacity': "Discharge_Capacity(mAh/g)",
               'charge_capacity': "Charge_Capacity(mAh/g)}

    Args:
        summaries_list: list containing cellpy summary column id names

    Returns: dictionary of the form {cellpy id name: cellpy summary
        header name,}

    """
    headers_summary = cellpy.parameters.internal_settings.get_headers_summary()
    selected_summaries = dict()
    for h in summaries_list:
        selected_summaries[h] = headers_summary[h]
    return selected_summaries

def pick_summary_data(key, summary_df, selected_summaries):
    """picks the selected pandas.DataFrame"""

    selected_summaries_dict = create_selected_summaries_dict(selected_summaries)
    value = selected_summaries_dict[key]
    return summary_df.iloc[:, summary_df.columns.get_level_values(1) == value]

def join_summaries(summary_frames, selected_summaries, keep_old_header=False):
    """parse the summaries and combine based on column (selected_summaries)"""
    if not summary_frames:
        raise NullData("No summaries available to join")
    selected_summaries_dict = create_selected_summaries_dict(selected_summaries)
    frames = []
    keys = []  # test-name
    for key in summary_frames:
        keys.append(key)
        if summary_frames[key].empty:
            logging.debug("Empty summary_frame encountered")
        frames.append(summary_frames[key])

    out = []

    summary_df = pd.concat(frames, keys=keys, axis=1)
    for key, value in selected_summaries_dict.items():
        _summary_df = summary_df.iloc[
            :, summary_df.columns.get_level_values(1) == value
        ]
        _summary_df.name = key

        if not keep_old_header:
            try:
                _summary_df.columns = _summary_df.columns.droplevel(-1)
            except AttributeError as e:
                logging.debug("could not drop level from frame")
                logging.debug(e)

        out.append(_summary_df)
    logging.debug("finished joining summaries")

    return out

def generate_folder_names(name, project):
    """Creates sensible folder names."""

    out_data_dir = prms.Paths.outdatadir
    project_dir = os.path.join(out_data_dir, project)
    batch_dir = os.path.join(project_dir, name)
    raw_dir = os.path.join(batch_dir, "raw_data")
    return out_data_dir, project_dir, batch_dir, raw_dir

def _extract_dqdv(cell_data, extract_func, last_cycle):
    """Simple wrapper around the cellpy.utils.ica.dqdv function."""

    from cellpy.utils.ica import dqdv

    list_of_cycles = cell_data.get_cycle_numbers()
    if last_cycle is not None:
        list_of_cycles = [c for c in list_of_cycles if c <= int(last_cycle)]
        logging.debug(f"only processing up to cycle {last_cycle}")
        logging.debug(f"you have {len(list_of_cycles)} cycles to process")
    out_data = []
    for cycle in list_of_cycles:
        try:
            c, v = extract_func(cycle)
            v, dq = dqdv(v, c)
            v = v.tolist()
            dq = dq.tolist()
        except NullData as e:
            v = list()
            dq = list()
            logging.info(" Ups! Could not process this (cycle %i)" % cycle)
            logging.info(" %s" % e)

        header_x = "dQ cycle_no %i" % cycle
        header_y = "voltage cycle_no %i" % cycle
        dq.insert(0, header_x)
        v.insert(0, header_y)

        out_data.append(v)
        out_data.append(dq)
    return out_data

def export_dqdv(cell_data, savedir, sep, last_cycle=None):
    """Exports dQ/dV data from a CellpyData instance.

    Args:
        cell_data: CellpyData instance
        savedir: path to the folder where the files should be saved
        sep: separator for the .csv-files.
        last_cycle: only export up to this cycle (if not None)
    """
    logging.debug("exporting dqdv")
    filename = cell_data.cell.loaded_from
    no_merged_sets = ""
    firstname, extension = os.path.splitext(filename)
    firstname += no_merged_sets
    if savedir:
        firstname = os.path.join(savedir, os.path.basename(firstname))
        logging.debug(f"savedir is true: {firstname}")

    outname_charge = firstname + "_dqdv_charge.csv"
    outname_discharge = firstname + "_dqdv_discharge.csv"

    list_of_cycles = cell_data.get_cycle_numbers()
    number_of_cycles = len(list_of_cycles)
    logging.debug("%s: you have %i cycles" % (filename, number_of_cycles))

    # extracting charge
    out_data = _extract_dqdv(cell_data, cell_data.get_ccap, last_cycle)
    logging.debug("extracted ica for charge")
    try:
        save_multi(data=out_data, file_name=outname_charge, sep=sep)
    except ExportFailed as e:
        logging.info("could not export ica for charge")
        warnings.warn(f"ExportFailed exception raised: {e}")
    else:
        logging.debug("saved ica for charge")

    # extracting discharge
    out_data = _extract_dqdv(cell_data, cell_data.get_dcap, last_cycle)
    logging.debug("extracxted ica for discharge")
    try:
        save_multi(data=out_data, file_name=outname_discharge, sep=sep)
    except ExportFailed as e:
        logging.info("could not export ica for discharge")
        warnings.warn(f"ExportFailed exception raised: {e}")
    else:
        logging.debug("saved ica for discharge")
<commit_msg>better label creation in batch<commit_after>import logging
import os
import warnings

import pandas as pd

import csv
import itertools

from cellpy import filefinder, prms
from cellpy.exceptions import ExportFailed, NullData, WrongFileVersion
import cellpy.parameters.internal_settings

# logger = logging.getLogger(__name__)

def look_up_and_get(cellpy_file_name, table_name, root=None):
    """Extracts table from cellpy hdf5-file."""

    # infoname = '/CellpyData/info'
    # dataname = '/CellpyData/dfdata'
    # summaryname = '/CellpyData/dfsummary'
    # fidname = '/CellpyData/fidtable'
    # stepname = '/CellpyData/step_table'

    if root is None:
        root = "/CellpyData"
    table_path = "/".join([root, table_name])

    logging.debug(f"look_up_and_get({cellpy_file_name}, {table_name}")
    store = pd.HDFStore(cellpy_file_name)
    try:
        table = store.select(table_path)
    except KeyError as e:
        logging.warning("Could not read the table")
        store.close()
        raise WrongFileVersion(e)
    return table

def create_folder_structure(project_name, batch_name):
    """This function creates a folder structure for the batch project.

    The folder structure consists of main working folder ``project_name`
    located in the ``outdatadir`` (as defined in the cellpy configuration file)
    with a sub-folder named ``batch_name``. It also creates a folder
    inside the ``batch_name`` folder for storing the raw data.
    If the folders does not exist, they will be made. The function also returns
    the name of the info-df.

    Args:
        project_name: name of the project
        batch_name: name of the batch

    Returns: (info_file, (project_dir, batch_dir, raw_dir))

    """
    out_data_dir = prms.Paths["outdatadir"]
    project_dir = os.path.join(out_data_dir, project_name)
    batch_dir = os.path.join(project_dir, batch_name)
    raw_dir = os.path.join(batch_dir, "raw_data")

    # create folders
    if not os.path.isdir(project_dir):
        os.mkdir(project_dir)
    if not os.path.isdir(batch_dir):
        os.mkdir(batch_dir)
    if not os.path.isdir(raw_dir):
        os.mkdir(raw_dir)

    # create file-name for the info_df (json)
    info_file = "cellpy_batch_%s.json" % batch_name
    info_file = os.path.join(project_dir, info_file)
    return info_file, (project_dir, batch_dir, raw_dir)

def find_files(info_dict, filename_cache=None):
    # searches for the raw data files and the cellpyfile-name
    for run_name in info_dict["filenames"]:
        logging.debug(f"checking for {run_name}")
        if prms._use_filename_cache:
            raw_files, cellpyfile, filename_cache = filefinder.search_for_files(
                run_name, cache=filename_cache
            )
        else:
            raw_files, cellpyfile = filefinder.search_for_files(run_name)
        if not raw_files:
            raw_files = None
        info_dict["raw_file_names"].append(raw_files)
        info_dict["cellpy_file_names"].append(cellpyfile)

    return info_dict

def fix_groups(groups):
    """Takes care of strange group numbers."""
    _groups = []
    for g in groups:
        try:
            if not float(g) > 0:
                _groups.append(1000)
            else:
                _groups.append(int(g))
        except TypeError as e:
            logging.info("Error in reading group number (check your db)")
            logging.debug(g)
            logging.debug(e)
            _groups.append(1000)
    return _groups

def save_multi(data, file_name, sep=";"):
    """Convenience function for storing data column-wise in a csv-file."""
    logging.debug("saving multi")
    with open(file_name, "w", newline="") as f:
        logging.debug(f"{file_name} opened")
        writer = csv.writer(f, delimiter=sep)
        try:
            writer.writerows(itertools.zip_longest(*data))
            logging.info(f"{file_name} OK")
        except Exception as e:
            logging.info(f"Exception encountered in batch._save_multi: {e}")
            raise ExportFailed
        logging.debug("wrote rows using itertools in _save_multi")

def make_unique_groups(info_df):
    """This function cleans up the group numbers a bit."""
    # fixes group numbering
    unique_g = info_df.groups.unique()
    unique_g = sorted(unique_g)
    new_unique_g = list(range(len(unique_g)))
    info_df["sub_groups"] = info_df["groups"] * 0
    for i, j in zip(unique_g, new_unique_g):
        counter = 1
        for indx, row in info_df.loc[info_df.groups == i].iterrows():
            info_df.at[indx, "sub_groups"] = counter
            # info_df.set_value(indx, "sub_groups", counter)
            counter += 1
        info_df.loc[info_df.groups == i, "groups"] = j + 1
    return info_df

def _remove_date_and_celltype(label,):
    parts = label.split("_")
    parts.pop(0)
    if parts[-1] in ["cc", "ec", "eth"]:
        parts.pop(-1)
    return "_".join(parts)

def create_labels(label, *args):
    """Returns a re-formatted label (currently it only removes the dates
    from the run-name)"""
    return _remove_date_and_celltype(label)

def create_selected_summaries_dict(summaries_list):
    """Creates a dictionary with summary column headers.

    Examples:
        >>> summaries_to_output = ["discharge_capacity", "charge_capacity"]
        >>> summaries_to_output_dict = create_selected_summaries_dict(
        >>>    summaries_to_output
        >>> )
        >>> print(summaries_to_output_dict)
        {'discharge_capacity': "Discharge_Capacity(mAh/g)",
               'charge_capacity': "Charge_Capacity(mAh/g)}

    Args:
        summaries_list: list containing cellpy summary column id names

    Returns: dictionary of the form {cellpy id name: cellpy summary
        header name,}

    """
    headers_summary = cellpy.parameters.internal_settings.get_headers_summary()
    selected_summaries = dict()
    for h in summaries_list:
        selected_summaries[h] = headers_summary[h]
    return selected_summaries

def pick_summary_data(key, summary_df, selected_summaries):
    """picks the selected pandas.DataFrame"""

    selected_summaries_dict = create_selected_summaries_dict(selected_summaries)
    value = selected_summaries_dict[key]
    return summary_df.iloc[:, summary_df.columns.get_level_values(1) == value]

def join_summaries(summary_frames, selected_summaries, keep_old_header=False):
    """parse the summaries and combine based on column (selected_summaries)"""
    if not summary_frames:
        raise NullData("No summaries available to join")
    selected_summaries_dict = create_selected_summaries_dict(selected_summaries)
    frames = []
    keys = []  # test-name
    for key in summary_frames:
        keys.append(key)
        if summary_frames[key].empty:
            logging.debug("Empty summary_frame encountered")
        frames.append(summary_frames[key])

    out = []

    summary_df = pd.concat(frames, keys=keys, axis=1)
    for key, value in selected_summaries_dict.items():
        _summary_df = summary_df.iloc[
            :, summary_df.columns.get_level_values(1) == value
        ]
        _summary_df.name = key

        if not keep_old_header:
            try:
                _summary_df.columns = _summary_df.columns.droplevel(-1)
            except AttributeError as e:
                logging.debug("could not drop level from frame")
                logging.debug(e)

        out.append(_summary_df)
    logging.debug("finished joining summaries")

    return out

def generate_folder_names(name, project):
    """Creates sensible folder names."""

    out_data_dir = prms.Paths.outdatadir
    project_dir = os.path.join(out_data_dir, project)
    batch_dir = os.path.join(project_dir, name)
    raw_dir = os.path.join(batch_dir, "raw_data")
    return out_data_dir, project_dir, batch_dir, raw_dir

def _extract_dqdv(cell_data, extract_func, last_cycle):
    """Simple wrapper around the cellpy.utils.ica.dqdv function."""

    from cellpy.utils.ica import dqdv

    list_of_cycles = cell_data.get_cycle_numbers()
    if last_cycle is not None:
        list_of_cycles = [c for c in list_of_cycles if c <= int(last_cycle)]
        logging.debug(f"only processing up to cycle {last_cycle}")
        logging.debug(f"you have {len(list_of_cycles)} cycles to process")
    out_data = []
    for cycle in list_of_cycles:
        try:
            c, v = extract_func(cycle)
            v, dq = dqdv(v, c)
            v = v.tolist()
            dq = dq.tolist()
        except NullData as e:
            v = list()
            dq = list()
            logging.info(" Ups! Could not process this (cycle %i)" % cycle)
            logging.info(" %s" % e)

        header_x = "dQ cycle_no %i" % cycle
        header_y = "voltage cycle_no %i" % cycle
        dq.insert(0, header_x)
        v.insert(0, header_y)

        out_data.append(v)
        out_data.append(dq)
    return out_data

def export_dqdv(cell_data, savedir, sep, last_cycle=None):
    """Exports dQ/dV data from a CellpyData instance.

    Args:
        cell_data: CellpyData instance
        savedir: path to the folder where the files should be saved
        sep: separator for the .csv-files.
        last_cycle: only export up to this cycle (if not None)
    """
    logging.debug("exporting dqdv")
    filename = cell_data.cell.loaded_from
    no_merged_sets = ""
    firstname, extension = os.path.splitext(filename)
    firstname += no_merged_sets
    if savedir:
        firstname = os.path.join(savedir, os.path.basename(firstname))
        logging.debug(f"savedir is true: {firstname}")

    outname_charge = firstname + "_dqdv_charge.csv"
    outname_discharge = firstname + "_dqdv_discharge.csv"

    list_of_cycles = cell_data.get_cycle_numbers()
    number_of_cycles = len(list_of_cycles)
    logging.debug("%s: you have %i cycles" % (filename, number_of_cycles))

    # extracting charge
    out_data = _extract_dqdv(cell_data, cell_data.get_ccap, last_cycle)
    logging.debug("extracted ica for charge")
    try:
        save_multi(data=out_data, file_name=outname_charge, sep=sep)
    except ExportFailed as e:
        logging.info("could not export ica for charge")
        warnings.warn(f"ExportFailed exception raised: {e}")
    else:
        logging.debug("saved ica for charge")

    # extracting discharge
    out_data = _extract_dqdv(cell_data, cell_data.get_dcap, last_cycle)
    logging.debug("extracxted ica for discharge")
    try:
        save_multi(data=out_data, file_name=outname_discharge, sep=sep)
    except ExportFailed as e:
        logging.info("could not export ica for discharge")
        warnings.warn(f"ExportFailed exception raised: {e}")
    else:
        logging.debug("saved ica for discharge")
<|endoftext|>


<commit_before>from django.core.paginator import Paginator, InvalidPage
from django.utils.encoding import force_unicode
from django.utils.translation import ugettext_lazy as _
from django.http import Http404
from django.db import models
from django.utils.functional import update_wrapper
from django.shortcuts import render_to_response
from django.template import RequestContext
from django.contrib import admin
from django.contrib.admin.views.main import PAGE_VAR, ALL_VAR, SEARCH_VAR
from django.conf import settings
from django.core.management.commands.diffsettings import module_to_dict
from haystack.query import SearchQuerySet
from haystackbrowser.models import HaystackResults, SearchResultWrapper
from haystackbrowser.forms import PreSelectedModelSearchForm

try:
    from haystack.constants import DJANGO_CT, DJANGO_ID
except ImportError:
    DJANGO_CT = 'django_ct'
    DJANGO_ID = 'django_id'

class HaystackResultsAdmin(object):
    fields = None
    fieldsets = None
    exclude = None
    date_hierarchy = None
    ordering = None
    list_select_related = False
    save_as = False
    save_on_top = False

    def __init__(self, model, admin_site):
        self.model = model
        self.opts = model._meta
        self.admin_site = admin_site

    def get_model_perms(self, request):
        return {
            'add': self.has_add_permission(request),
            'change': self.has_change_permission(request),
            'delete': self.has_delete_permission(request)
        }

    def has_add_permission(self, request):
        return False

    def has_change_permission(self, request, obj=None):
        return request.user.is_superuser

    def has_delete_permission(self, request, obj=None):
        return False

    def urls(self):
        from django.conf.urls.defaults import patterns, url
        def wrap(view):
            def wrapper(*args, **kwargs):
                return self.admin_site.admin_view(view)(*args, **kwargs)
            return update_wrapper(wrapper, view)
        return patterns('',
            url(regex=r'^(?P<content_type>.+)/(?P<pk>.+)/$',
                view=self.view,
                name='%s_%s_change' % (self.model._meta.app_label,
                    self.model._meta.module_name)
            ),
            url(regex=r'^$',
                view=self.index,
                name='%s_%s_changelist' % (self.model._meta.app_label,
                    self.model._meta.module_name)
            ),
        )
    urls = property(urls)

    def get_results_per_page(self, request):
        return getattr(settings, 'HAYSTACK_SEARCH_RESULTS_PER_PAGE', 20)

    def get_paginator_var(self, request):
        return PAGE_VAR

    def get_search_var(self, request):
        return SEARCH_VAR

    def get_all_results_var(self, request):
        return ALL_VAR

    def get_searchresult_wrapper(self):
        return SearchResultWrapper

    def get_wrapped_search_results(self, object_list):
        klass = self.get_searchresult_wrapper()
        return [klass(x, self.admin_site.name) for x in object_list]

    def get_settings(self):
        filtered_settings = {}
        searching_for = u'HAYSTACK_'
        all_settings = module_to_dict(settings._wrapped)
        for setting_name, setting_value in all_settings.items():

            if setting_name.startswith(searching_for):
                setting_name = setting_name.replace(searching_for, '').replace('_', ' ')
                filtered_settings[setting_name] = setting_value
        return filtered_settings

    def index(self, request):
        page_var = self.get_paginator_var(request)
        form = PreSelectedModelSearchForm(request.GET or None, searchqueryset=SearchQuerySet(), load_all=False)
        sqs = form.search()
        try:
            paginator = Paginator(sqs, self.get_results_per_page(request))
            page_qs = request.GET.get(page_var, 1)
            page = paginator.page(int(page_qs))
        except (InvalidPage, ValueError):
            # paginator.page may raise InvalidPage if we've gone too far
            # meanwhile, casting the querystring parameter may raise ValueError
            # if it's None, or '', or other silly input.
            raise Http404
        context = {
            'results': self.get_wrapped_search_results(page.object_list),
            'pagination_required': page.has_other_pages(),
            'page_range': paginator.page_range,
            'page_num': page.number,
            'result_count': paginator.count,
            'opts': self.model._meta,
            'title': self.model._meta.verbose_name_plural,
            'root_path': self.admin_site.root_path,
            'app_label': self.model._meta.app_label,
            'filtered': True,
            'form': form,
            'params': dict(request.GET.items()),
            'search_var': self.get_search_var(request),
            'page_var': page_var,
            'module_name': force_unicode(self.model._meta.verbose_name_plural),
        }
        return render_to_response('admin/haystackbrowser/result_list.html', context,
            context_instance=RequestContext(request))

    def view(self, request, content_type, pk):
        query = {DJANGO_ID: pk, DJANGO_CT: content_type}
        try:
            sqs = self.get_wrapped_search_results(SearchQuerySet().filter(**query)[:1])[0]
        except IndexError:
            raise Http404
        context = {
            'original': sqs,
            'title': _('View stored data for this %s') % force_unicode(sqs.verbose_name),
            'app_label': self.model._meta.app_label,
            'module_name': force_unicode(self.model._meta.verbose_name_plural),
            'haystack_settings': self.get_settings(),
            'has_change_permission': self.has_change_permission(request, sqs)
        }
        return render_to_response('admin/haystackbrowser/view.html', context,
            context_instance=RequestContext(request))
admin.site.register(HaystackResults, HaystackResultsAdmin)
<commit_msg>Drop the explicit use of SearchQuerySet in the form<commit_after>from django.core.paginator import Paginator, InvalidPage
from django.utils.encoding import force_unicode
from django.utils.translation import ugettext_lazy as _
from django.http import Http404
from django.db import models
from django.utils.functional import update_wrapper
from django.shortcuts import render_to_response
from django.template import RequestContext
from django.contrib import admin
from django.contrib.admin.views.main import PAGE_VAR, ALL_VAR, SEARCH_VAR
from django.conf import settings
from django.core.management.commands.diffsettings import module_to_dict
from haystack.query import SearchQuerySet
from haystackbrowser.models import HaystackResults, SearchResultWrapper
from haystackbrowser.forms import PreSelectedModelSearchForm

try:
    from haystack.constants import DJANGO_CT, DJANGO_ID
except ImportError:
    DJANGO_CT = 'django_ct'
    DJANGO_ID = 'django_id'

class HaystackResultsAdmin(object):
    fields = None
    fieldsets = None
    exclude = None
    date_hierarchy = None
    ordering = None
    list_select_related = False
    save_as = False
    save_on_top = False

    def __init__(self, model, admin_site):
        self.model = model
        self.opts = model._meta
        self.admin_site = admin_site

    def get_model_perms(self, request):
        return {
            'add': self.has_add_permission(request),
            'change': self.has_change_permission(request),
            'delete': self.has_delete_permission(request)
        }

    def has_add_permission(self, request):
        return False

    def has_change_permission(self, request, obj=None):
        return request.user.is_superuser

    def has_delete_permission(self, request, obj=None):
        return False

    def urls(self):
        from django.conf.urls.defaults import patterns, url
        def wrap(view):
            def wrapper(*args, **kwargs):
                return self.admin_site.admin_view(view)(*args, **kwargs)
            return update_wrapper(wrapper, view)
        return patterns('',
            url(regex=r'^(?P<content_type>.+)/(?P<pk>.+)/$',
                view=self.view,
                name='%s_%s_change' % (self.model._meta.app_label,
                    self.model._meta.module_name)
            ),
            url(regex=r'^$',
                view=self.index,
                name='%s_%s_changelist' % (self.model._meta.app_label,
                    self.model._meta.module_name)
            ),
        )
    urls = property(urls)

    def get_results_per_page(self, request):
        return getattr(settings, 'HAYSTACK_SEARCH_RESULTS_PER_PAGE', 20)

    def get_paginator_var(self, request):
        return PAGE_VAR

    def get_search_var(self, request):
        return SEARCH_VAR

    def get_all_results_var(self, request):
        return ALL_VAR

    def get_searchresult_wrapper(self):
        return SearchResultWrapper

    def get_wrapped_search_results(self, object_list):
        klass = self.get_searchresult_wrapper()
        return [klass(x, self.admin_site.name) for x in object_list]

    def get_settings(self):
        filtered_settings = {}
        searching_for = u'HAYSTACK_'
        all_settings = module_to_dict(settings._wrapped)
        for setting_name, setting_value in all_settings.items():

            if setting_name.startswith(searching_for):
                setting_name = setting_name.replace(searching_for, '').replace('_', ' ')
                filtered_settings[setting_name] = setting_value
        return filtered_settings

    def index(self, request):
        page_var = self.get_paginator_var(request)
        form = PreSelectedModelSearchForm(request.GET or None, load_all=False)
        sqs = form.search()
        try:
            paginator = Paginator(sqs, self.get_results_per_page(request))
            page_qs = request.GET.get(page_var, 1)
            page = paginator.page(int(page_qs))
        except (InvalidPage, ValueError):
            # paginator.page may raise InvalidPage if we've gone too far
            # meanwhile, casting the querystring parameter may raise ValueError
            # if it's None, or '', or other silly input.
            raise Http404
        context = {
            'results': self.get_wrapped_search_results(page.object_list),
            'pagination_required': page.has_other_pages(),
            'page_range': paginator.page_range,
            'page_num': page.number,
            'result_count': paginator.count,
            'opts': self.model._meta,
            'title': self.model._meta.verbose_name_plural,
            'root_path': self.admin_site.root_path,
            'app_label': self.model._meta.app_label,
            'filtered': True,
            'form': form,
            'params': dict(request.GET.items()),
            'search_var': self.get_search_var(request),
            'page_var': page_var,
            'module_name': force_unicode(self.model._meta.verbose_name_plural),
        }
        return render_to_response('admin/haystackbrowser/result_list.html', context,
            context_instance=RequestContext(request))

    def view(self, request, content_type, pk):
        query = {DJANGO_ID: pk, DJANGO_CT: content_type}
        try:
            sqs = self.get_wrapped_search_results(SearchQuerySet().filter(**query)[:1])[0]
        except IndexError:
            raise Http404
        context = {
            'original': sqs,
            'title': _('View stored data for this %s') % force_unicode(sqs.verbose_name),
            'app_label': self.model._meta.app_label,
            'module_name': force_unicode(self.model._meta.verbose_name_plural),
            'haystack_settings': self.get_settings(),
            'has_change_permission': self.has_change_permission(request, sqs)
        }
        return render_to_response('admin/haystackbrowser/view.html', context,
            context_instance=RequestContext(request))
admin.site.register(HaystackResults, HaystackResultsAdmin)
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
from __future__ import unicode_literals

from django.db import migrations
from django.db import transaction

@transaction.atomic
def create_forms_and_submissions(apps, schema_editor):
    Call = apps.get_model("cfp", "Call")
    Form = apps.get_model("formbuilder", "Form")

    for call in Call.objects.filter(state='approved'):
        conference = call.conference
        form = Form(name="{} CFP Form".format(conference.name))
        form.save()

        form.field_set.create(
            name='title',
            required=True,
            order=1,
        )

        form.field_set.create(
            name='abstract',
            required=True,
            widget=1,
            order=2,
        )

        if call.needs_audience:
            audience = form.field_set.create(
                name='audience',
                kind=3,
                required=True,
                order=5,
            )
            audience.option_set.create(value='Beginner')
            audience.option_set.create(value='Intermidiate')
            audience.option_set.create(value='Advancded')

        if conference.track_set.count():
            field = form.field_set.create(
                name='track',
                kind=3,
                required=True,
                order=4,
            )
            for track in conference.track_set.all():
                field.option_set.create(value=track.name)

        if conference.format_set.count():
            field = form.field_set.create(
                name='format',
                kind=3,
                required=True,
                order=3,
            )
            for fmt in conference.format_set.all():
                field.option_set.create(value=fmt.name)

        call.form = form
        call.save()

    Talk = apps.get_model("cfp", "Talk")
    Submission = apps.get_model("formbuilder", "Submission")

    for talk in Talk.objects.all():
        print(talk.title)
        sub = Submission(form=talk.call.form)
        sub.save()

        for field in talk.call.form.field_set.all():
            if field.name == 'title':
                sub.entry_set.create(
                    field=field,
                    value=talk.title,
                )

            if field.name == 'abstract':
                sub.entry_set.create(
                    field=field,
                    value=talk.abstract
                )

            if field.name == 'track':
                sub.entry_set.create(
                    field=field,
                    value=talk.track.name,
                )

            if field.name == 'audience':
                levels = dict([
                    (1, 'Beginner'),
                    (2, 'Intermidiate'),
                    (3, 'Advancded'),
                ])

                sub.entry_set.create(
                    field=field,
                    value=levels.get(talk.audience, 'Beginner'),
                )
            if field.name == 'format':
                sub.entry_set.create(
                    field=field,
                    value=talk.format.name,
                )

        talk.submission = sub
        talk.save()

@transaction.atomic
def delete_forms_and_submissions(apps, schema_editor):
    Talk = apps.get_model("cfp", "Talk")
    for talk in Talk.objects.filter(submission__isnull=False):
        talk.submission = None
        talk.save()

    Call = apps.get_model("cfp", "Call")
    for call in Call.objects.filter(form__isnull=False):
        call.form = None
        call.save()

    Submission = apps.get_model("formbuilder", "Submission")
    Form = apps.get_model("formbuilder", "Form")

    Submission.objects.all().delete()
    Form.objects.all().delete()

class Migration(migrations.Migration):

    dependencies = [
        ('cfp', '0025_auto_20150227_0848'),
    ]

    operations = [
        migrations.RunPython(create_forms_and_submissions,
                             delete_forms_and_submissions),
    ]
<commit_msg>Only add field if value is set<commit_after># -*- coding: utf-8 -*-
from __future__ import unicode_literals

from django.db import migrations
from django.db import transaction

@transaction.atomic
def create_forms_and_submissions(apps, schema_editor):
    Call = apps.get_model("cfp", "Call")
    Form = apps.get_model("formbuilder", "Form")

    for call in Call.objects.filter(state='approved'):
        conference = call.conference
        form = Form(name="{} CFP Form".format(conference.name))
        form.save()

        form.field_set.create(
            name='title',
            required=True,
            order=1,
        )

        form.field_set.create(
            name='abstract',
            required=True,
            widget=1,
            order=2,
        )

        if call.needs_audience:
            audience = form.field_set.create(
                name='audience',
                kind=3,
                required=True,
                order=5,
            )
            audience.option_set.create(value='Beginner')
            audience.option_set.create(value='Intermidiate')
            audience.option_set.create(value='Advancded')

        if conference.track_set.count():
            field = form.field_set.create(
                name='track',
                kind=3,
                required=True,
                order=4,
            )
            for track in conference.track_set.all():
                field.option_set.create(value=track.name)

        if conference.format_set.count():
            field = form.field_set.create(
                name='format',
                kind=3,
                required=True,
                order=3,
            )
            for fmt in conference.format_set.all():
                field.option_set.create(value=fmt.name)

        call.form = form
        call.save()

    Talk = apps.get_model("cfp", "Talk")
    Submission = apps.get_model("formbuilder", "Submission")

    for talk in Talk.objects.all():
        print(talk.title)
        sub = Submission(form=talk.call.form)
        sub.save()

        for field in talk.call.form.field_set.all():
            if field.name == 'title':
                sub.entry_set.create(
                    field=field,
                    value=talk.title,
                )

            if field.name == 'abstract':
                sub.entry_set.create(
                    field=field,
                    value=talk.abstract
                )

            if field.name == 'track' and talk.track:
                sub.entry_set.create(
                    field=field,
                    value=talk.track.name,
                )

            if field.name == 'audience' and talk.audience:
                levels = dict([
                    (1, 'Beginner'),
                    (2, 'Intermidiate'),
                    (3, 'Advancded'),
                ])

                sub.entry_set.create(
                    field=field,
                    value=levels.get(talk.audience, 'Beginner'),
                )
            if field.name == 'format' and talk.format:
                sub.entry_set.create(
                    field=field,
                    value=talk.format.name,
                )

        talk.submission = sub
        talk.save()

@transaction.atomic
def delete_forms_and_submissions(apps, schema_editor):
    Talk = apps.get_model("cfp", "Talk")
    for talk in Talk.objects.filter(submission__isnull=False):
        talk.submission = None
        talk.save()

    Call = apps.get_model("cfp", "Call")
    for call in Call.objects.filter(form__isnull=False):
        call.form = None
        call.save()

    Submission = apps.get_model("formbuilder", "Submission")
    Form = apps.get_model("formbuilder", "Form")

    Submission.objects.all().delete()
    Form.objects.all().delete()

class Migration(migrations.Migration):

    dependencies = [
        ('cfp', '0025_auto_20150227_0848'),
    ]

    operations = [
        migrations.RunPython(create_forms_and_submissions,
                             delete_forms_and_submissions),
    ]
<|endoftext|>


<commit_before>import dj_database_url
import os

from .base import *

DEBUG = False
TEMPLATE_DEBUG = DEBUG

ADMINS = [('znotdead', 'zhirafchik@gmail.com')]

DATABASES['default'] = dj_database_url.config()

STATICFILES_STORAGE = 'whitenoise.django.GzipManifestStaticFilesStorage'
STATIC_ROOT = os.path.join(BASE_DIR, 'staticfiles')
STATIC_URL = '/static/'

ALLOWED_HOSTS = [".herokuapp.com", ]

REDIS_POOL = redis.ConnectionPool.from_url(os.environ.get('REDISCLOUD_URL'))
REDIS_CONN = redis.Redis(connection_pool=REDIS_POOL)

if not DEBUG:
    SECURE_SSL_REDIRECT = True

JWT_AUTH = {
    'JWT_EXPIRATION_DELTA': datetime.timedelta(days=1),
}
<commit_msg>add missing import<commit_after>import dj_database_url
import os
import datetime

from .base import *

DEBUG = False
TEMPLATE_DEBUG = DEBUG

ADMINS = [('znotdead', 'zhirafchik@gmail.com')]

DATABASES['default'] = dj_database_url.config()

STATICFILES_STORAGE = 'whitenoise.django.GzipManifestStaticFilesStorage'
STATIC_ROOT = os.path.join(BASE_DIR, 'staticfiles')
STATIC_URL = '/static/'

ALLOWED_HOSTS = [".herokuapp.com", ]

REDIS_POOL = redis.ConnectionPool.from_url(os.environ.get('REDISCLOUD_URL'))
REDIS_CONN = redis.Redis(connection_pool=REDIS_POOL)

if not DEBUG:
    SECURE_SSL_REDIRECT = True

JWT_AUTH = {
    'JWT_EXPIRATION_DELTA': datetime.timedelta(days=1),
}
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
from __future__ import division, print_function

import h5py
import matplotlib as mpl

from hdf5plotter import silent_del, u, update_attrs
from hdf5plotter._util import (replace_unit_label, replace_latex_label,
                               get_unit_attr, is_container)

class PlotFromManyFiles(object):
    """A prototype of a plottting class that plots one dataset from
    many files."""
    def __init__(self):
        self.groups = []
        self.files = set()
        self.rcParams = {}

    def add(self, filename, group='/'):
        """Adds the group from filename, or a list of filenames,
        to the list of groups."""
        fh = h5py.File(filename)
        self.files.add(fh)
        self.groups.append(fh[group])

    def close(self):
        """Close open h5py files on delete."""
        for group in self.groups:
            group.file.close()

    def rescale(self, old_dset, new_dset, new_unit):
        """Rescale data in old_dset to the units new_unit, and put the result
        in a dataset named new_dset."""
        if new_dset == old_dset:
            raise ValueError("'new_dset' and 'old_dset' must be different.")
        for group in self.groups:
            # Make sure the new dataset doesn't exist, so we can define it
            silent_del(group, new_dset)
            group.copy(old_dset, new_dset)
            old_unit = u(group[old_dset].attrs['unit'])
            scaled_dset = (group[old_dset][:] * old_unit).to(new_unit).magnitude
            group[new_dset][:] = scaled_dset

            new_attrs = group[new_dset].attrs
            # Update the unit in the new_attrs
            new_attrs['unit'] = get_unit_attr(new_unit)
            new_attrs['label'] = replace_unit_label(
                new_attrs['label'], new_unit)
            new_attrs['label_latex'] = replace_latex_label(
                new_attrs['label_latex'], new_unit)

    def map(self, function, new_dset, new_attrs):
        """Transforms data using function, and puts the result in new_dset, and
        attachs the attributes new_attrs."""
        for group in self.groups:
            # Make sure the new dataset doesn't exist, so we can define it
            silent_del(group, new_dset)
            group[new_dset] = function(group)
            update_attrs(group[new_dset].attrs, new_attrs)

    def plot(self, x='x', y='y', scale='linear', shape='-', xlim=None, ylim=None,
             rcParams=None, filename=None, save_fig_kwargs={}):
        if rcParams is not None:
            self.rcParams = rcParams
        for key, val in self.rcParams.items():
            mpl.rcParams[key] = val

        import matplotlib.pyplot as plt

        fig, ax = plt.subplots()

        plotting_functions = OrderedDict((
        ('linear', ax.plot),
        ('semilogy', ax.semilogy),
        ('semilogx', ax.semilogx),
        ('loglog', ax.loglog)))

        plot = plotting_functions[scale]

        for group in self.groups:
            line = plot(group[x].value, group[y].value, shape)

        ax.set_xlabel(group[x].attrs['label'])
        ax.set_ylabel(group[y].attrs['label'])

        if xlim is not None:
            ax.set_xlim(xlim)
        if ylim is not None:
            ax.set_ylim(ylim)

        fig.canvas.draw()

        if filename is not None:
            fig.savefig(filename, **save_fig_kwargs)

        self.fig = fig
        self.ax = ax
        return fig, ax
<commit_msg>Switched to a normal dictionary in plot<commit_after># -*- coding: utf-8 -*-
from __future__ import division, print_function

import h5py
import matplotlib as mpl

from hdf5plotter import silent_del, u, update_attrs
from hdf5plotter._util import (replace_unit_label, replace_latex_label,
                               get_unit_attr, is_container)

class PlotFromManyFiles(object):
    """A prototype of a plottting class that plots one dataset from
    many files."""
    def __init__(self):
        self.groups = []
        self.files = set()
        self.rcParams = {}

    def add(self, filename, group='/'):
        """Adds the group from filename, or a list of filenames,
        to the list of groups."""
        fh = h5py.File(filename)
        self.files.add(fh)
        self.groups.append(fh[group])

    def close(self):
        """Close open h5py files on delete."""
        for group in self.groups:
            group.file.close()

    def rescale(self, old_dset, new_dset, new_unit):
        """Rescale data in old_dset to the units new_unit, and put the result
        in a dataset named new_dset."""
        if new_dset == old_dset:
            raise ValueError("'new_dset' and 'old_dset' must be different.")
        for group in self.groups:
            # Make sure the new dataset doesn't exist, so we can define it
            silent_del(group, new_dset)
            group.copy(old_dset, new_dset)
            old_unit = u(group[old_dset].attrs['unit'])
            scaled_dset = (group[old_dset][:] * old_unit).to(new_unit).magnitude
            group[new_dset][:] = scaled_dset

            new_attrs = group[new_dset].attrs
            # Update the unit in the new_attrs
            new_attrs['unit'] = get_unit_attr(new_unit)
            new_attrs['label'] = replace_unit_label(
                new_attrs['label'], new_unit)
            new_attrs['label_latex'] = replace_latex_label(
                new_attrs['label_latex'], new_unit)

    def map(self, function, new_dset, new_attrs):
        """Transforms data using function, and puts the result in new_dset, and
        attachs the attributes new_attrs."""
        for group in self.groups:
            # Make sure the new dataset doesn't exist, so we can define it
            silent_del(group, new_dset)
            group[new_dset] = function(group)
            update_attrs(group[new_dset].attrs, new_attrs)

    def plot(self, x='x', y='y', scale='linear', shape='-', xlim=None, ylim=None,
            filename=None, save_fig_kwargs={}):

        for key, val in self.rcParams.items():
            mpl.rcParams[key] = val

        import matplotlib.pyplot as plt

        fig, ax = plt.subplots()

        plotting_functions = {
            'linear': ax.plot,
            'semilogy': ax.semilogy,
            'semilogx': ax.semilogx,
            'loglog': ax.loglog}

        plot = plotting_functions[scale]

        for group in self.groups:
            line = plot(group[x].value, group[y].value, shape)

        ax.set_xlabel(group[x].attrs['label'])
        ax.set_ylabel(group[y].attrs['label'])

        if xlim is not None:
            ax.set_xlim(xlim)
        if ylim is not None:
            ax.set_ylim(ylim)

        fig.canvas.draw()

        if filename is not None:
            fig.savefig(filename, **save_fig_kwargs)

        self.fig = fig
        self.ax = ax
        return fig, ax
<|endoftext|>