<commit_before>import logging
import re

from bottle import request
from bottle import route
from bottle import run
from spyonweb import spyonweb
from TRX import TRX

ValidHostnameRegex = "^(([a-zA-Z0-9]|[a-zA-Z0-9][a-zA-Z0-9\-]*[a-zA-Z0-9])\.)*([A-Za-z0-9]|[A-Za-z0-9][A-Za-z0-9\-]*[A-Za-z0-9])$"
ValidIpAddressRegex = "^(([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])\.){3}([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])$"
ValidAdsenseRegex = "^pub-[0-9]+$"
ValidAnalyticsRegex = "^UA-[0-9]+$"

# dispatcher

@route('/summary', method='POST')
def summary():
    if request.body.len > 0:
        incoming = TRX.MaltegoMsg(request.body.getvalue())
        if (incoming.Type == "Domain" or incoming.Type == "DNSName") and re.match(ValidHostnameRegex, incoming.Value):
            s = spyonweb.Spyonweb(incoming.TransformSettings['api'])
            data = s.summary(incoming.Value)
            return process_summary(data)
        else:
            xform = TRX.MaltegoTransform()
            xform.addException("Must submit a valid host name or domain name")
            return xform.throwExceptions()

@route('/domain', method='POST')
def domain():
    if request.body.len > 0:
        incoming = TRX.MaltegoMsg(request.body.getvalue())
        if (incoming.Type == "Domain" or incoming.Type == "DNSName") and re.match(ValidHostnameRegex, incoming.Value):
            s = spyonweb.Spyonweb(incoming.TransformSettings['api'])
            data = s.domain(incoming.Value)
            return process_domain(data)
        else:
            xform = TRX.MaltegoTransform()
            xform.addException("Must submit a valid host name or domain name")
            return xform.throwExceptions()

@route('/adsense', method='POST')
def adsense():
    if request.body.len > 0:
        incoming = TRX.MaltegoMsg(request.body.getvalue())
        xform = TRX.MaltegoTransform()
        if incoming.Type == "Phrase" and re.match(ValidAdsenseRegex, incoming.Value):
            s = spyonweb.Spyonweb(incoming.TransformSettings['api'])
            data = s.adsense(incoming.Value, limit=incoming.Slider)
            for name in data:
                ent = xform.addEntity("maltego.Domain", name)
                ent.setLinkLabel(data[name])  # date ID was associated with domain
        else:
            xform.addException("Must submit a valid Adsense publisher ID")
            return xform.throwExceptions()

@route('/analytics', method='POST')
def analytics():
    if request.body.len > 0:
        incoming = TRX.MaltegoMsg(request.body.getvalue())
        xform = TRX.MaltegoTransform()
        if incoming.Type == "Phrase" and re.match(ValidAnalyticsRegex, incoming.Value):
            s = spyonweb.Spyonweb(incoming.TransformSettings['api'])
            data = s.analytics(incoming.Value, limit=incoming.Slider)
            for name in data:
                ent = xform.addEntity("maltego.Domain", name)
                ent.setLinkLabel(data[name])  # date ID was associated with domain
        else:
            xform.addException("Must submit a valid Analytics tracking ID")
            return xform.throwExceptions()

@route('/ip', method='POST')
def ip():
    if request.body.len > 0:
        incoming = TRX.MaltegoMsg(request.body.getvalue())
        xform = TRX.MaltegoTransform()
        if incoming.Type == "IPv4Address" and re.match(ValidIpAddressRegex, incoming.Value):
            s = spyonweb.Spyonweb(incoming.TransformSettings['api'])
            data = s.ipaddress(incoming.Value, limit=incoming.Slider)
            for name in data:
                ent = xform.addEntity("maltego.Domain", name)
                ent.setLinkLabel(data[name])  # date IP address was associated with domain
        else:
            xform.addException("Must submit a valid IPv4 address")
            return xform.throwExceptions()

@route('/dns_domain', method='POST')
def dns_domain():
    if request.body.len > 0:
        incoming = TRX.MaltegoMsg(request.body.getvalue())
        xform = TRX.MaltegoTransform()
        if (incoming.Type == "Domain" or incoming.Type == "DNSName" or incoming.Type == "NSRecord") and re.match(ValidHostnameRegex, incoming.Value):
            s = spyonweb.Spyonweb(incoming.TransformSettings['api'])
            data = s.dns_domain(incoming.Value, limit=incoming.Slider)
            for name in data:
                ent = xform.addEntity("maltego.Domain", name)
                ent.setLinkLabel(data[name])  # date domain name was associated with server
        else:
            xform.addException("Must submit a valid host name")
            return xform.throwExceptions()

@route('/ip_dns', method='POST')
def ip_dns():
    if request.body.len > 0:
        incoming = TRX.MaltegoMsg(request.body.getvalue())
        xform = TRX.MaltegoTransform()
        if incoming.Type == "IPv4Address" and re.match(ValidIpAddressRegex, incoming.Value):
            s = spyonweb.Spyonweb(incoming.TransformSettings['api'])
            data = s.ip_dns(incoming.Value, limit=incoming.Slider)
            for name in data:
                ent = xform.addEntity("maltego.NSRecord", name)
                ent.setLinkLabel(data[name])  # date domain name was associated with server
        else:
            xform.addException("Must submit a valid host name")
            return xform.throwExceptions()

# process results into Maltego messages

def process_summary(data):
    xform = TRX.MaltegoTransform()
    if data['status'] != 'found':
        xform.addUIMessage("No results found", TRX.UIM_FATAL)
        return xform.returnOutput()
    name = data['result']['summary'].keys()[0]
    for code in data['result']['summary'][name]['items'].get('adsense', ''):
        ent = xform.addEntity("maltego.Phrase", code)
        ent.setWeight(data['result']['summary'][name]['items']['adsense'][code])
    for code in data['result']['summary'][name]['items'].get('analytics', ''):
        ent = xform.addEntity("maltego.Phrase", code)
        ent.setWeight(data['result']['summary'][name]['items']['analytics'][code])
    for server in data['result']['summary'][name]['items'].get('dns_servers', ''):
        ent = xform.addEntity("maltego.NSRecord", server)
        ent.setWeight(data['result']['summary'][name]['items']['dns_servers'][server])
    for ip in data['result']['summary'][name]['items'].get('ip', ''):
        ent = xform.addEntity("maltego.IPv4Address", ip)
        ent.setWeight(data['result']['summary'][name]['items']['ip'][ip])
    return xform.returnOutput()

def process_domain(data):
    xform = TRX.MaltegoTransform()
    if data['status'] != 'found':
        xform.addUIMessage("No results found", TRX.UIM_FATAL)
        return xform.returnOutput()
    name = data['result']['domain'].keys()[0]
    for code in data['result']['domain'][name]['items'].get('adsense', ''):
        ent = xform.addEntity("maltego.Phrase", code)
        ent.setWeight(data['result']['domain'][name]['items']['adsense'][code])
    for code in data['result']['domain'][name]['items'].get('analytics', ''):
        ent = xform.addEntity("maltego.Phrase", code)
        ent.setWeight(data['result']['domain'][name]['items']['analytics'][code])
    for server in data['result']['domain'][name]['items'].get('dns_servers', ''):
        ent = xform.addEntity("maltego.NSRecord", server)
        # TODO: how to represent IP addresses returned in this API call?
    for ip in data['result']['domain'][name]['items'].get('ip', ''):
        ent = xform.addEntity("maltego.IPv4Address", ip)
        ent.setWeight(data['result']['domain'][name]['items']['ip'][ip])
    return xform.returnOutput()

# instantiate our class
# Start server
logging.basicConfig(filename='spyonweb-maltego.log', level=logging.DEBUG)
run(host='0.0.0.0', port=8192, debug=True, reloader=True)
<commit_msg>Remove non-standard port spec<commit_after>import logging
import re

from bottle import request
from bottle import route
from bottle import run
from spyonweb import spyonweb
from TRX import TRX

ValidHostnameRegex = "^(([a-zA-Z0-9]|[a-zA-Z0-9][a-zA-Z0-9\-]*[a-zA-Z0-9])\.)*([A-Za-z0-9]|[A-Za-z0-9][A-Za-z0-9\-]*[A-Za-z0-9])$"
ValidIpAddressRegex = "^(([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])\.){3}([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])$"
ValidAdsenseRegex = "^pub-[0-9]+$"
ValidAnalyticsRegex = "^UA-[0-9]+$"

# dispatcher

@route('/summary', method='POST')
def summary():
    if request.body.len > 0:
        incoming = TRX.MaltegoMsg(request.body.getvalue())
        if (incoming.Type == "Domain" or incoming.Type == "DNSName") and re.match(ValidHostnameRegex, incoming.Value):
            s = spyonweb.Spyonweb(incoming.TransformSettings['api'])
            data = s.summary(incoming.Value)
            return process_summary(data)
        else:
            xform = TRX.MaltegoTransform()
            xform.addException("Must submit a valid host name or domain name")
            return xform.throwExceptions()

@route('/domain', method='POST')
def domain():
    if request.body.len > 0:
        incoming = TRX.MaltegoMsg(request.body.getvalue())
        if (incoming.Type == "Domain" or incoming.Type == "DNSName") and re.match(ValidHostnameRegex, incoming.Value):
            s = spyonweb.Spyonweb(incoming.TransformSettings['api'])
            data = s.domain(incoming.Value)
            return process_domain(data)
        else:
            xform = TRX.MaltegoTransform()
            xform.addException("Must submit a valid host name or domain name")
            return xform.throwExceptions()

@route('/adsense', method='POST')
def adsense():
    if request.body.len > 0:
        incoming = TRX.MaltegoMsg(request.body.getvalue())
        xform = TRX.MaltegoTransform()
        if incoming.Type == "Phrase" and re.match(ValidAdsenseRegex, incoming.Value):
            s = spyonweb.Spyonweb(incoming.TransformSettings['api'])
            data = s.adsense(incoming.Value, limit=incoming.Slider)
            for name in data:
                ent = xform.addEntity("maltego.Domain", name)
                ent.setLinkLabel(data[name])  # date ID was associated with domain
        else:
            xform.addException("Must submit a valid Adsense publisher ID")
            return xform.throwExceptions()

@route('/analytics', method='POST')
def analytics():
    if request.body.len > 0:
        incoming = TRX.MaltegoMsg(request.body.getvalue())
        xform = TRX.MaltegoTransform()
        if incoming.Type == "Phrase" and re.match(ValidAnalyticsRegex, incoming.Value):
            s = spyonweb.Spyonweb(incoming.TransformSettings['api'])
            data = s.analytics(incoming.Value, limit=incoming.Slider)
            for name in data:
                ent = xform.addEntity("maltego.Domain", name)
                ent.setLinkLabel(data[name])  # date ID was associated with domain
        else:
            xform.addException("Must submit a valid Analytics tracking ID")
            return xform.throwExceptions()

@route('/ip', method='POST')
def ip():
    if request.body.len > 0:
        incoming = TRX.MaltegoMsg(request.body.getvalue())
        xform = TRX.MaltegoTransform()
        if incoming.Type == "IPv4Address" and re.match(ValidIpAddressRegex, incoming.Value):
            s = spyonweb.Spyonweb(incoming.TransformSettings['api'])
            data = s.ipaddress(incoming.Value, limit=incoming.Slider)
            for name in data:
                ent = xform.addEntity("maltego.Domain", name)
                ent.setLinkLabel(data[name])  # date IP address was associated with domain
        else:
            xform.addException("Must submit a valid IPv4 address")
            return xform.throwExceptions()

@route('/dns_domain', method='POST')
def dns_domain():
    if request.body.len > 0:
        incoming = TRX.MaltegoMsg(request.body.getvalue())
        xform = TRX.MaltegoTransform()
        if (incoming.Type == "Domain" or incoming.Type == "DNSName" or incoming.Type == "NSRecord") and re.match(ValidHostnameRegex, incoming.Value):
            s = spyonweb.Spyonweb(incoming.TransformSettings['api'])
            data = s.dns_domain(incoming.Value, limit=incoming.Slider)
            for name in data:
                ent = xform.addEntity("maltego.Domain", name)
                ent.setLinkLabel(data[name])  # date domain name was associated with server
        else:
            xform.addException("Must submit a valid host name")
            return xform.throwExceptions()

@route('/ip_dns', method='POST')
def ip_dns():
    if request.body.len > 0:
        incoming = TRX.MaltegoMsg(request.body.getvalue())
        xform = TRX.MaltegoTransform()
        if incoming.Type == "IPv4Address" and re.match(ValidIpAddressRegex, incoming.Value):
            s = spyonweb.Spyonweb(incoming.TransformSettings['api'])
            data = s.ip_dns(incoming.Value, limit=incoming.Slider)
            for name in data:
                ent = xform.addEntity("maltego.NSRecord", name)
                ent.setLinkLabel(data[name])  # date domain name was associated with server
        else:
            xform.addException("Must submit a valid host name")
            return xform.throwExceptions()

# process results into Maltego messages

def process_summary(data):
    xform = TRX.MaltegoTransform()
    if data['status'] != 'found':
        xform.addUIMessage("No results found", TRX.UIM_FATAL)
        return xform.returnOutput()
    name = data['result']['summary'].keys()[0]
    for code in data['result']['summary'][name]['items'].get('adsense', ''):
        ent = xform.addEntity("maltego.Phrase", code)
        ent.setWeight(data['result']['summary'][name]['items']['adsense'][code])
    for code in data['result']['summary'][name]['items'].get('analytics', ''):
        ent = xform.addEntity("maltego.Phrase", code)
        ent.setWeight(data['result']['summary'][name]['items']['analytics'][code])
    for server in data['result']['summary'][name]['items'].get('dns_servers', ''):
        ent = xform.addEntity("maltego.NSRecord", server)
        ent.setWeight(data['result']['summary'][name]['items']['dns_servers'][server])
    for ip in data['result']['summary'][name]['items'].get('ip', ''):
        ent = xform.addEntity("maltego.IPv4Address", ip)
        ent.setWeight(data['result']['summary'][name]['items']['ip'][ip])
    return xform.returnOutput()

def process_domain(data):
    xform = TRX.MaltegoTransform()
    if data['status'] != 'found':
        xform.addUIMessage("No results found", TRX.UIM_FATAL)
        return xform.returnOutput()
    name = data['result']['domain'].keys()[0]
    for code in data['result']['domain'][name]['items'].get('adsense', ''):
        ent = xform.addEntity("maltego.Phrase", code)
        ent.setWeight(data['result']['domain'][name]['items']['adsense'][code])
    for code in data['result']['domain'][name]['items'].get('analytics', ''):
        ent = xform.addEntity("maltego.Phrase", code)
        ent.setWeight(data['result']['domain'][name]['items']['analytics'][code])
    for server in data['result']['domain'][name]['items'].get('dns_servers', ''):
        ent = xform.addEntity("maltego.NSRecord", server)
        # TODO: how to represent IP addresses returned in this API call?
    for ip in data['result']['domain'][name]['items'].get('ip', ''):
        ent = xform.addEntity("maltego.IPv4Address", ip)
        ent.setWeight(data['result']['domain'][name]['items']['ip'][ip])
    return xform.returnOutput()

# instantiate our class
# Start server
logging.basicConfig(filename='spyonweb-maltego.log', level=logging.DEBUG)
run(host='0.0.0.0', debug=True, reloader=True)
<|endoftext|>


<commit_before><commit_msg>[FIX] purchase: use date everywhere in report<commit_after><|endoftext|>


<commit_before># Copyright 2016 United States Government as represented by the Administrator
# of the National Aeronautics and Space Administration. All Rights Reserved.
#
# Portion of this code is Copyright Geoscience Australia, Licensed under the
# Apache License, Version 2.0 (the "License"); you may not use this file
# except in compliance with the License. You may obtain a copy of the License
# at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# The CEOS 2 platform is licensed under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
# http://www.apache.org/licenses/LICENSE-2.0.
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.

# datacube imports.
import datacube
from datacube.api import GridWorkflow

# Author: AHDS
# Creation date: 2016-06-23
# Modified by:
# Last modified date: 2016-08-05

class DataAccessApi:
    """
    Class that provides wrapper functionality for the DataCube.
    """

    # defaults for all the required fields.
    product_default = 'ls7_ledaps'
    platform_default = 'LANDSAT_7'

    def __init__(self, config='/home/localuser/Datacube/data_cube_ui/config/.datacube.conf'):
        self.dc = datacube.Datacube(config=config)

    """
    query params are defined in datacube.api.query
    """

    def get_dataset_by_extent(self,
                              product,
                              product_type=None,
                              platform=None,
                              time=None,
                              longitude=None,
                              latitude=None,
                              measurements=None,
                              output_crs=None,
                              resolution=None,
                              crs=None,
                              dask_chunks=None):
        """
        Gets and returns data based on lat/long bounding box inputs.
        All params are optional. Leaving one out will just query the dc without it, (eg leaving out
        lat/lng but giving product returns dataset containing entire product.)

        Args:
            product (string): The name of the product associated with the desired dataset.
            product_type (string): The type of product associated with the desired dataset.
            platform (string): The platform associated with the desired dataset.
            time (tuple): A tuple consisting of the start time and end time for the dataset.
            longitude (tuple): A tuple of floats specifying the min,max longitude bounds.
            latitude (tuple): A tuple of floats specifying the min,max latitutde bounds.
            crs (string): CRS lat/lon bounds are specified in, defaults to WGS84.
            measurements (list): A list of strings that represents all measurements.
            output_crs (string): Determines reprojection of the data before its returned
            resolution (tuple): A tuple of min,max ints to determine the resolution of the data.
            dask_chunks (dict): Lazy loaded array block sizes, not lazy loaded by default.

        Returns:
            data (xarray): dataset with the desired data.
        """

        # there is probably a better way to do this but I'm not aware of it.
        query = {}
        if product_type is not None:
            query['product_type'] = product_type
        if platform is not None:
            query['platform'] = platform
        if time is not None:
            query['time'] = time
        if longitude is not None and latitude is not None:
            query['longitude'] = longitude
            query['latitude'] = latitude
        if crs is not None:
            query['crs'] = crs

        data = self.dc.load(
            product=product,
            measurements=measurements,
            output_crs=output_crs,
            resolution=resolution,
            dask_chunks=dask_chunks,
            **query)
        return data

    def get_stacked_datasets_by_extent(self,
                                       products,
                                       product_type=None,
                                       platforms=None,
                                       time=None,
                                       longitude=None,
                                       latitude=None,
                                       measurements=None,
                                       output_crs=None,
                                       resolution=None,
                                       dask_chunks=None):
        """
          Gets and returns data based on lat/long bounding box inputs.
          All params are optional. Leaving one out will just query the dc without it, (eg leaving out
          lat/lng but giving product returns dataset containing entire product.)

          Args:
              products (array of strings): The names of the product associated with the desired dataset.
              product_type (string): The type of product associated with the desired dataset.
              platforms (array of strings): The platforms associated with the desired dataset.
              time (tuple): A tuple consisting of the start time and end time for the dataset.
              longitude (tuple): A tuple of floats specifying the min,max longitude bounds.
              latitude (tuple): A tuple of floats specifying the min,max latitutde bounds.
              measurements (list): A list of strings that represents all measurements.
              output_crs (string): Determines reprojection of the data before its returned
              resolution (tuple): A tuple of min,max ints to determine the resolution of the data.

          Returns:
              data (xarray): dataset with the desired data.
          """

        data_array = []

        for index, product in enumerate(products):
            product_data = self.get_dataset_by_extent(
                product,
                product_type=product_type,
                platform=platforms[index],
                time=time,
                longitude=longitude,
                latitude=latitude,
                measurements=measurements,
                output_crs=output_crs,
                resolution=resolution,
                dask_chunks=dask_chunks)
            if 'time' in product_data:
                product_data['satellite'] = xr.DataArray(
                    np.full(product_data.cf_mask.values.shape, index, dtype="int16"),
                    dims=('time', 'latitude', 'longitude'))
                data_array.append(product_data.copy(deep=True))

        data = None
        if len(data_array) > 0:
            combined_data = xr.concat(data_array, 'time')
            data = combined_data.reindex({'time': sorted(combined_data.time.values)})

        return data

    def get_dataset_tiles(self,
                          product,
                          product_type=None,
                          platform=None,
                          time=None,
                          longitude=None,
                          latitude=None,
                          measurements=None,
                          output_crs=None,
                          resolution=None):
        """
        Gets and returns data based on lat/long bounding box inputs.
        All params are optional. Leaving one out will just query the dc without it, (eg leaving out
        lat/lng but giving product returns dataset containing entire product.)

        Args:
            product (string): The name of the product associated with the desired dataset.
            product_type (string): The type of product associated with the desired dataset.
            platform (string): The platform associated with the desired dataset.
            time (tuple): A tuple consisting of the start time and end time for the dataset.
            longitude (tuple): A tuple of floats specifying the min,max longitude bounds.
            latitude (tuple): A tuple of floats specifying the min,max latitutde bounds.
            measurements (list): A list of strings that represents all measurements.
            output_crs (string): Determines reprojection of the data before its returned
            resolution (tuple): A tuple of min,max ints to determine the resolution of the data.

        Returns:
            data (xarray): dataset with the desired data in tiled sections.
        """

        # there is probably a better way to do this but I'm not aware of it.
        query = {}
        if product_type is not None:
            query['product_type'] = product_type
        if platform is not None:
            query['platform'] = platform
        if time is not None:
            query['time'] = time
        if longitude is not None and latitude is not None:
            query['longitude'] = longitude
            query['latitude'] = latitude

        # set up the grid workflow
        gw = GridWorkflow(self.dc.index, product=product)

        # dict of tiles.
        request_tiles = gw.list_cells(
            product=product, measurements=measurements, output_crs=output_crs, resolution=resolution, **query)
        """
        tile_def = defaultdict(dict)
        for cell, tiles in request_tiles.items():
            for time, tile in tiles.items():
                tile_def[cell, time]['request'] = tile

        keys = list(tile_def)

        data_tiles = {}
        for key in keys:
            tile = tile_def[key]['request']
            data_tiles[key[0]] = gw.load(key[0], tile)
        """
        # cells now return stacked xarrays of data.
        data_tiles = {}
        for tile_key in request_tiles:
            tile = request_tiles[tile_key]
            data_tiles[tile_key] = gw.load(tile, measurements=measurements)

        return data_tiles

    def get_scene_metadata(self, platform, product, longitude=None, latitude=None, crs=None, time=None):
        """
        Gets a descriptor based on a request.

        Args:
            platform (string): Platform for which data is requested
            product (string): The name of the product associated with the desired dataset.
            longitude (tuple): Tuple of min,max floats for longitude
            latitude (tuple): Tuple of min,max floats for latitutde
            crs (string): Describes the coordinate system of params lat and long
            time (tuple): Tuple of start and end datetimes for requested data

        Returns:
            scene_metadata (dict): Dictionary containing a variety of data that can later be
                                   accessed.
        """

        dataset = self.get_dataset_by_extent(
            platform=platform,
            product=product,
            longitude=longitude,
            latitude=latitude,
            crs=crs,
            time=time,
            dask_chunks={})

        if not dataset:
            return {
                'lat_extents': (0, 0),
                'lon_extents': (0, 0),
                'time_extents': (0, 0),
                'scene_count': 0,
                'pixel_count': 0,
                'tile_count': 0,
                'storage_units': {}
            }

        lon_min, lat_min, lon_max, lat_max = dataset.geobox.extent.envelope
        return {
            'lat_extents': (lat_min, lat_max),
            'lon_extents': (lon_min, lon_max),
            'time_extents': (dataset.time[0].values.astype('M8[ms]').tolist(),
                             dataset.time[-1].values.astype('M8[ms]').tolist()),
            'scene_count':
            dataset.time.size,
            'pixel_count':
            dataset.geobox.shape[0] * dataset.geobox.shape[1],
            # TODO: is 'tile_count' needed?
            # TODO: is 'storage_units' it needed?
        }

    def list_acquisition_dates(self, platform, product, longitude=None, latitude=None, crs=None, time=None):
        """
        Get a list of all acquisition dates for a query.

        Args:
            platform (string): Platform for which data is requested
            product (string): The name of the product associated with the desired dataset.
            longitude (tuple): Tuple of min,max floats for longitude
            latitude (tuple): Tuple of min,max floats for latitutde
            crs (string): Describes the coordinate system of params lat and long
            time (tuple): Tuple of start and end datetimes for requested data

        Returns:
            times (list): Python list of dates that can be used to query the dc for single time
                          sliced data.
        """
        dataset = self.get_dataset_by_extent(
            platform=platform,
            product=product,
            longitude=longitude,
            latitude=latitude,
            crs=crs,
            time=time,
            dask_chunks={})

        if not dataset:
            return []

        return dataset.time.values.astype('M8[ms]').tolist()

    def get_datacube_metadata(self, platform, product):
        """
        Gets some details on the cube and its contents.

        Args:
            platform (string): Desired platform for requested data.
            product (string): Desired product for requested data.

        Returns:
            datacube_metadata (dict): a dict with multiple keys containing relevant metadata.
        """

        return self.get_scene_metadata(platform, product)
<commit_msg>fixed stacked datasets missing dependencies removed in pr<commit_after># Copyright 2016 United States Government as represented by the Administrator
# of the National Aeronautics and Space Administration. All Rights Reserved.
#
# Portion of this code is Copyright Geoscience Australia, Licensed under the
# Apache License, Version 2.0 (the "License"); you may not use this file
# except in compliance with the License. You may obtain a copy of the License
# at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# The CEOS 2 platform is licensed under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
# http://www.apache.org/licenses/LICENSE-2.0.
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.

# datacube imports.
import datacube
from datacube.api import GridWorkflow
import xarray as xr
import numpy as np

# Author: AHDS
# Creation date: 2016-06-23
# Modified by:
# Last modified date: 2016-08-05

class DataAccessApi:
    """
    Class that provides wrapper functionality for the DataCube.
    """

    # defaults for all the required fields.
    product_default = 'ls7_ledaps'
    platform_default = 'LANDSAT_7'

    def __init__(self, config='/home/localuser/Datacube/data_cube_ui/config/.datacube.conf'):
        self.dc = datacube.Datacube(config=config)

    """
    query params are defined in datacube.api.query
    """

    def get_dataset_by_extent(self,
                              product,
                              product_type=None,
                              platform=None,
                              time=None,
                              longitude=None,
                              latitude=None,
                              measurements=None,
                              output_crs=None,
                              resolution=None,
                              crs=None,
                              dask_chunks=None):
        """
        Gets and returns data based on lat/long bounding box inputs.
        All params are optional. Leaving one out will just query the dc without it, (eg leaving out
        lat/lng but giving product returns dataset containing entire product.)

        Args:
            product (string): The name of the product associated with the desired dataset.
            product_type (string): The type of product associated with the desired dataset.
            platform (string): The platform associated with the desired dataset.
            time (tuple): A tuple consisting of the start time and end time for the dataset.
            longitude (tuple): A tuple of floats specifying the min,max longitude bounds.
            latitude (tuple): A tuple of floats specifying the min,max latitutde bounds.
            crs (string): CRS lat/lon bounds are specified in, defaults to WGS84.
            measurements (list): A list of strings that represents all measurements.
            output_crs (string): Determines reprojection of the data before its returned
            resolution (tuple): A tuple of min,max ints to determine the resolution of the data.
            dask_chunks (dict): Lazy loaded array block sizes, not lazy loaded by default.

        Returns:
            data (xarray): dataset with the desired data.
        """

        # there is probably a better way to do this but I'm not aware of it.
        query = {}
        if product_type is not None:
            query['product_type'] = product_type
        if platform is not None:
            query['platform'] = platform
        if time is not None:
            query['time'] = time
        if longitude is not None and latitude is not None:
            query['longitude'] = longitude
            query['latitude'] = latitude
        if crs is not None:
            query['crs'] = crs

        data = self.dc.load(
            product=product,
            measurements=measurements,
            output_crs=output_crs,
            resolution=resolution,
            dask_chunks=dask_chunks,
            **query)
        return data

    def get_stacked_datasets_by_extent(self,
                                       products,
                                       product_type=None,
                                       platforms=None,
                                       time=None,
                                       longitude=None,
                                       latitude=None,
                                       measurements=None,
                                       output_crs=None,
                                       resolution=None,
                                       dask_chunks=None):
        """
          Gets and returns data based on lat/long bounding box inputs.
          All params are optional. Leaving one out will just query the dc without it, (eg leaving out
          lat/lng but giving product returns dataset containing entire product.)

          Args:
              products (array of strings): The names of the product associated with the desired dataset.
              product_type (string): The type of product associated with the desired dataset.
              platforms (array of strings): The platforms associated with the desired dataset.
              time (tuple): A tuple consisting of the start time and end time for the dataset.
              longitude (tuple): A tuple of floats specifying the min,max longitude bounds.
              latitude (tuple): A tuple of floats specifying the min,max latitutde bounds.
              measurements (list): A list of strings that represents all measurements.
              output_crs (string): Determines reprojection of the data before its returned
              resolution (tuple): A tuple of min,max ints to determine the resolution of the data.

          Returns:
              data (xarray): dataset with the desired data.
          """

        data_array = []

        for index, product in enumerate(products):
            product_data = self.get_dataset_by_extent(
                product,
                product_type=product_type,
                platform=platforms[index],
                time=time,
                longitude=longitude,
                latitude=latitude,
                measurements=measurements,
                output_crs=output_crs,
                resolution=resolution,
                dask_chunks=dask_chunks)
            if 'time' in product_data:
                product_data['satellite'] = xr.DataArray(
                    np.full(product_data[list(product_data.data_vars)[0]].values.shape, index, dtype="int16"),
                    dims=('time', 'latitude', 'longitude'))
                data_array.append(product_data.copy(deep=True))

        data = None
        if len(data_array) > 0:
            combined_data = xr.concat(data_array, 'time')
            data = combined_data.reindex({'time': sorted(combined_data.time.values)})

        return data

    def get_dataset_tiles(self,
                          product,
                          product_type=None,
                          platform=None,
                          time=None,
                          longitude=None,
                          latitude=None,
                          measurements=None,
                          output_crs=None,
                          resolution=None):
        """
        Gets and returns data based on lat/long bounding box inputs.
        All params are optional. Leaving one out will just query the dc without it, (eg leaving out
        lat/lng but giving product returns dataset containing entire product.)

        Args:
            product (string): The name of the product associated with the desired dataset.
            product_type (string): The type of product associated with the desired dataset.
            platform (string): The platform associated with the desired dataset.
            time (tuple): A tuple consisting of the start time and end time for the dataset.
            longitude (tuple): A tuple of floats specifying the min,max longitude bounds.
            latitude (tuple): A tuple of floats specifying the min,max latitutde bounds.
            measurements (list): A list of strings that represents all measurements.
            output_crs (string): Determines reprojection of the data before its returned
            resolution (tuple): A tuple of min,max ints to determine the resolution of the data.

        Returns:
            data (xarray): dataset with the desired data in tiled sections.
        """

        # there is probably a better way to do this but I'm not aware of it.
        query = {}
        if product_type is not None:
            query['product_type'] = product_type
        if platform is not None:
            query['platform'] = platform
        if time is not None:
            query['time'] = time
        if longitude is not None and latitude is not None:
            query['longitude'] = longitude
            query['latitude'] = latitude

        # set up the grid workflow
        gw = GridWorkflow(self.dc.index, product=product)

        # dict of tiles.
        request_tiles = gw.list_cells(
            product=product, measurements=measurements, output_crs=output_crs, resolution=resolution, **query)
        """
        tile_def = defaultdict(dict)
        for cell, tiles in request_tiles.items():
            for time, tile in tiles.items():
                tile_def[cell, time]['request'] = tile

        keys = list(tile_def)

        data_tiles = {}
        for key in keys:
            tile = tile_def[key]['request']
            data_tiles[key[0]] = gw.load(key[0], tile)
        """
        # cells now return stacked xarrays of data.
        data_tiles = {}
        for tile_key in request_tiles:
            tile = request_tiles[tile_key]
            data_tiles[tile_key] = gw.load(tile, measurements=measurements)

        return data_tiles

    def get_scene_metadata(self, platform, product, longitude=None, latitude=None, crs=None, time=None):
        """
        Gets a descriptor based on a request.

        Args:
            platform (string): Platform for which data is requested
            product (string): The name of the product associated with the desired dataset.
            longitude (tuple): Tuple of min,max floats for longitude
            latitude (tuple): Tuple of min,max floats for latitutde
            crs (string): Describes the coordinate system of params lat and long
            time (tuple): Tuple of start and end datetimes for requested data

        Returns:
            scene_metadata (dict): Dictionary containing a variety of data that can later be
                                   accessed.
        """

        dataset = self.get_dataset_by_extent(
            platform=platform,
            product=product,
            longitude=longitude,
            latitude=latitude,
            crs=crs,
            time=time,
            dask_chunks={})

        if not dataset:
            return {
                'lat_extents': (0, 0),
                'lon_extents': (0, 0),
                'time_extents': (0, 0),
                'scene_count': 0,
                'pixel_count': 0,
                'tile_count': 0,
                'storage_units': {}
            }

        lon_min, lat_min, lon_max, lat_max = dataset.geobox.extent.envelope
        return {
            'lat_extents': (lat_min, lat_max),
            'lon_extents': (lon_min, lon_max),
            'time_extents': (dataset.time[0].values.astype('M8[ms]').tolist(),
                             dataset.time[-1].values.astype('M8[ms]').tolist()),
            'scene_count':
            dataset.time.size,
            'pixel_count':
            dataset.geobox.shape[0] * dataset.geobox.shape[1],
        }

    def list_acquisition_dates(self, platform, product, longitude=None, latitude=None, crs=None, time=None):
        """
        Get a list of all acquisition dates for a query.

        Args:
            platform (string): Platform for which data is requested
            product (string): The name of the product associated with the desired dataset.
            longitude (tuple): Tuple of min,max floats for longitude
            latitude (tuple): Tuple of min,max floats for latitutde
            crs (string): Describes the coordinate system of params lat and long
            time (tuple): Tuple of start and end datetimes for requested data

        Returns:
            times (list): Python list of dates that can be used to query the dc for single time
                          sliced data.
        """
        dataset = self.get_dataset_by_extent(
            platform=platform,
            product=product,
            longitude=longitude,
            latitude=latitude,
            crs=crs,
            time=time,
            dask_chunks={})

        if not dataset:
            return []

        return dataset.time.values.astype('M8[ms]').tolist()

    def get_datacube_metadata(self, platform, product):
        """
        Gets some details on the cube and its contents.

        Args:
            platform (string): Desired platform for requested data.
            product (string): Desired product for requested data.

        Returns:
            datacube_metadata (dict): a dict with multiple keys containing relevant metadata.
        """

        return self.get_scene_metadata(platform, product)
<|endoftext|>


<commit_before>import os
import numpy as np
from scipy import ndimage
from scipy.misc import imresize
import csv
import pickle

from settings import image_size, pixel_depth, color_channels,\
    number_of_train_images, number_of_test_images

def get_training(folder):
    images = np.ndarray(shape=(number_of_train_images, image_size,
                               image_size, color_channels),
                        dtype=np.float32)
    labels = []
    num_images = 0
    image_categories_dir = os.listdir(folder)
    for directory in image_categories_dir:
        directory_path = os.path.join(folder, directory)
        image_files = os.listdir(directory_path)
        for image_filename in image_files:
            image_path = os.path.join(directory_path, image_filename)
            try:
                image_data = (ndimage.imread(image_path).astype(float) -
                              (pixel_depth / 2)) / pixel_depth
                if image_data.shape != (image_size, image_size):
                    image = imresize(image_data, size=(image_size, image_size))
                else:
                    image = image_data
                images[num_images, :, :, :] = image
                num_images = num_images + 1
                label = int(directory)
                labels.append(label)
            except OSError as e:
                pass
    images = images[0:num_images, :, :, :]
    return {'train_data': images, 'train_labels': labels}

def get_test(folder):
    images = np.ndarray(shape=(number_of_test_images, image_size,
                               image_size, color_channels),
                        dtype=np.float32)
    labels = []
    num_images = 0
    image_files = os.listdir(folder)
    images_info = {}
    with open(os.path.join(folder, 'images_info.csv'), 'rt') as cf:
        reader = csv.reader(cf)
        next(reader, None)
        images_info = {row[0].split(';')[0]: row[0].split(';')[-1]
                       for row in reader}
    for image_filename in image_files:
        image_path = os.path.join(folder, image_filename)
        try:
            image_data = (ndimage.imread(image_path).astype(float) -
                          (pixel_depth / 2)) / pixel_depth
            if image_data.shape != (image_size, image_size):
                image = imresize(image_data, size=(image_size, image_size))
            else:
                image = image_data
            images[num_images, :, :, :] = image
            num_images = num_images + 1
            label = int(images_info[image_filename])
            labels.append(label)
        except OSError as e:
            pass
    images = images[0:num_images, :, :, :]
    return {'test_data': images, 'test_labels': labels}

def pickle_data(train_folder, test_folder, pickled_name):
    dataset = {}
    train = get_training(train_folder)
    test = get_test(test_folder)
    dataset.update(train)
    dataset.update(test)
    with open(pickled_name, 'wb') as f:
        pickle.dump(dataset, f, pickle.HIGHEST_PROTOCOL)

if __name__ == '__main__':
    pickle_data('./dataset/train_images', './dataset/test_images',
                './dataset/dataset.pickle')
<commit_msg>Update type of labels<commit_after>import os
import numpy as np
from scipy import ndimage
from scipy.misc import imresize
import csv
import pickle

from settings import image_size, pixel_depth, color_channels,\
    number_of_train_images, number_of_test_images

def get_training(folder):
    images = np.ndarray(shape=(number_of_train_images, image_size,
                               image_size, color_channels),
                        dtype=np.float32)
    labels = np.ndarray(number_of_train_images, dtype=np.int32)
    num_images = 0
    image_categories_dir = os.listdir(folder)
    for directory in image_categories_dir:
        directory_path = os.path.join(folder, directory)
        image_files = os.listdir(directory_path)
        for image_filename in image_files:
            image_path = os.path.join(directory_path, image_filename)
            try:
                image_data = (ndimage.imread(image_path).astype(float) -
                              (pixel_depth / 2)) / pixel_depth
                if image_data.shape != (image_size, image_size):
                    image = imresize(image_data, size=(image_size, image_size))
                else:
                    image = image_data
                images[num_images, :, :, :] = image
                labels[num_images] = int(directory)
                num_images = num_images + 1
            except OSError as e:
                pass
    images = images[0:num_images, :, :, :]
    labels = labels[0:num_images]
    return {'train_data': images, 'train_labels': labels}

def get_test(folder):
    images = np.ndarray(shape=(number_of_test_images, image_size,
                               image_size, color_channels),
                        dtype=np.float32)
    labels = np.ndarray(number_of_train_images, dtype=np.int32)
    num_images = 0
    image_files = os.listdir(folder)
    images_info = {}
    with open(os.path.join(folder, 'images_info.csv'), 'rt') as cf:
        reader = csv.reader(cf)
        next(reader, None)
        images_info = {row[0].split(';')[0]: row[0].split(';')[-1]
                       for row in reader}
    for image_filename in image_files:
        image_path = os.path.join(folder, image_filename)
        try:
            image_data = (ndimage.imread(image_path).astype(float) -
                          (pixel_depth / 2)) / pixel_depth
            if image_data.shape != (image_size, image_size):
                image = imresize(image_data, size=(image_size, image_size))
            else:
                image = image_data
            images[num_images, :, :, :] = image
            labels[num_images] = int(images_info[image_filename])
            num_images = num_images + 1
        except OSError as e:
            pass
    images = images[0:num_images, :, :, :]
    labels = labels[0:num_images]
    return {'test_data': images, 'test_labels': labels}

def pickle_data(train_folder, test_folder, pickled_name):
    dataset = {}
    train = get_training(train_folder)
    test = get_test(test_folder)
    dataset.update(train)
    dataset.update(test)
    with open(pickled_name, 'wb') as f:
        pickle.dump(dataset, f, pickle.HIGHEST_PROTOCOL)

if __name__ == '__main__':
    pickle_data('./dataset/train_images', './dataset/test_images',
                './dataset/dataset.pickle')
<|endoftext|>


<commit_before>#-*- coding: utf-8 -*-

from datetime import datetime, date
from json import load
from random import choice
from uuid import uuid4
from smtplib import SMTP
from flask import url_for
from sqlalchemy import UniqueConstraint, Column, Integer, String, Date, DateTime, ForeignKey
from sqlalchemy.orm import relationship
from config import MAIL_FROM, MAIL_DEBUG
from config.main import DEBUG
from config.mail import *
from . import Base

def sendmail(addr_from, addr_to, msg):
    with SMTP("localhost") as s:
        s.sendmail(addr_from.encode("utf-8"),
                   addr_to.encode("utf-8"),
                   msg.encode("utf-8"),
                   mail_options=["SMTPUTF8"])

class Reminder(Base):
    __tablename__ = "reminders"

    id = Column(Integer, primary_key=True)
    phone_number = Column(String(20), unique=True, nullable=False)
    time = Column(Integer)
    date_added = Column(Date, nullable=False)
    last_called = Column(DateTime)
    times_called = Column(Integer, nullable=False)
    times_forwarded = Column(Integer, nullable=False)
    time_connected = Column(Integer, nullable=False)

    def __init__(self, phone_number, time=None):
        self.phone_number = phone_number
        self.time = time
        self.date_added = date.today()
        self.last_called = None
        self.times_called = 0
        self.times_forwarded = 0
        self.time_connected = 0

    def __repr__(self):
        return "<Reminder for {}>".format(self.phone_number)

class Mail(Base):
    __tablename__ = "mails"

    id = Column(Integer, primary_key=True)
    sender_id = Column(Integer, ForeignKey("senders.id"))
    recipient = Column(String(5), nullable=False)
    date_sent = Column(DateTime)

    __table_args__ = tuple(
            UniqueConstraint(sender_id, recipient)
            )

    def __init__(self, sender, recipient):
        self.sender = sender
        self.recipient = recipient

    def send(self):
        self.date_sent = datetime.now()

        rep = reps.get_representative_by_id(self.recipient)

        addr_from = MAIL_FROM + " <" + MAIL_FROM + ">"
        if DEBUG:
            addr_to = MAIL_DEBUG + " <" + MAIL_DEBUG + ">"
        else:
            addr_to = str(rep) + " <" + rep.contact.mail + ">"
        salutation = "Sehr geehrter Herr" if rep.sex == "male" else "Sehr geehrte Frau"
        msg = MAIL_DISCLAIMER.format(name_user=self.sender.name, mail_user=self.sender.email_address) + "\n" * 2
        msg = msg + MAIL_REPRESENTATIVE.format(name_rep=str(rep), name_user=self.sender.name, salutation=salutation)
        sendmail(addr_from, addr_to, msg)

class Sender(Base):
    __tablename__ = "senders"

    id = Column(Integer, primary_key=True)
    name = Column(String(256), nullable=False)
    email_address = Column(String(254), unique=True, nullable=False)
    mails = relationship("Mail", order_by="Mail.id", backref="sender")
    hash = Column(String(64), unique=True, nullable=False)
    date_validated = Column(DateTime)
    date_requested = Column(DateTime, nullable=False)

    def __init__(self, name, email_address):
        self.name = name
        self.email_address = email_address
        self.request_validation()

    def validate(self):
        self.date_validated = datetime.now()

        addr_from = MAIL_FROM + " <" + MAIL_FROM + ">"
        addr_to = self.name + " <" + self.email_address + ">"
        msg = MAIL_WELCOME.format(name_user=self.name)
        sendmail(addr_from, addr_to, msg)

    def request_validation(self):
        self.hash = uuid4().hex
        self.date_requested = datetime.now()

        addr_from = MAIL_FROM + " <" + MAIL_FROM + ">"
        addr_to = self.name + " <" + self.email_address + ">"
        url = url_for("validate", hash=self.hash, _external=True)
        msg = MAIL_VALIDATE.format(name_user=self.name, url=url)
        sendmail(addr_from, addr_to, msg)

class Representatives():
    def __init__(self):
        self.parties = load_parties()
        self.teams = load_teams()
        self.representatives = load_representatives("representatives.json", self.parties, self.teams)
        self.government = load_representatives("government.json", self.parties, self.teams)

    def get_representative_by_id(self, id):
        representatives = self.representatives + self.government
        if id == "00000":
            return choice([rep for rep in representatives if rep.team.prettyname == "spy"])

        try:
            rep = [rep for rep in representatives if rep.id == id][0]
        except IndexError:
            rep = None
        return rep

    def get_representative_by_name(self, prettyname):
        representatives = self.representatives + self.government
        try:
            rep = [rep for rep in representatives if rep.name.prettyname == prettyname][0]
        except IndexError:
            rep = None
        return rep

    def get_party(self, shortname):
        return self.parties[shortname]

class Contact():
    def __init__(self, mail, phone, facebook, twitter):
        self.mail = mail
        self.phone = phone
        self.facebook = facebook
        self.twitter = twitter

class Party():
    def __init__(self, name, shortname, prettyname, color, contact):
        self.name = name
        self.shortname = shortname
        self.prettyname = prettyname
        self.color = color
        self.contact = contact

class Name():
    def __init__(self, firstname, lastname, prettyname, prefix, suffix):
        self.firstname = firstname
        self.lastname = lastname
        self.prettyname = prettyname
        self.prefix = prefix
        self.suffix = suffix

class Image():
    def __init__(self, url, copyright):
        self.url = url
        self.copyright = copyright

class Team():
    def __init__(self, name, prettyname):
        self.name = name
        self.prettyname = prettyname

    def __repr__(self):
        return self.name

class Representative():
    def __init__(self, id, name, contact, image, party, team, sex, state):
        self.id = id
        self.name = name
        self.contact = contact
        self.image = image
        self.party = party
        self.team = team
        self.sex = sex
        self.state = state

        if not self.contact.mail:
            self.contact.mail = party.contact.mail
        if not self.contact.phone:
            self.contact.phone = party.contact.phone
        if not self.contact.facebook:
            self.contact.facebook = party.contact.facebook
        if not self.contact.twitter:
            self.contact.twitter = party.contact.twitter

    def __repr__(self):
        return self.name.firstname + " " + self.name.lastname

    def fullname(self):
        return (self.name.prefix + " " if self.name.prefix else "") + self.name.firstname + " " + self.name.lastname + (" " + self.name.suffix if self.name.suffix else "")

def load_parties():
    parties = {}

    with open("ueberwachungspaket/data/parties.json", "r") as f:
        lparties = load(f)

    for prettyname in lparties:
        lparty = lparties[prettyname]
        lcontact = lparty["contact"]
        contact = Contact(lcontact["mail"], lcontact["phone"], lcontact["facebook"], lcontact["twitter"])
        party = Party(lparty["name"], lparty["shortname"], prettyname, lparty["color"], contact)
        parties[prettyname] = party

    return parties

def load_teams():
    teams = {}

    with open("ueberwachungspaket/data/teams.json", "r") as f:
        lteams = load(f)

    for prettyname in lteams:
        lteam = lteams[prettyname]
        team = Team(lteam["name"], prettyname)
        teams[prettyname] = team

    return teams

def load_representatives(filename, parties, teams):
    representatives = []

    with open("ueberwachungspaket/data/" + filename, "r") as f:
        lrepresentatives = load(f)

    for lrep in lrepresentatives:
        lname = lrep["name"]
        name = Name(lname["firstname"], lname["lastname"], lname["prettyname"], lname["prefix"], lname["suffix"])
        lcontact = lrep["contact"]
        contact = Contact(lcontact["mail"], lcontact["phone"], lcontact["facebook"], lcontact["twitter"])
        image = Image(lrep["image"]["url"], lrep["image"]["copyright"])
        party = parties[lrep["party"]]
        team = teams[lrep["team"]]
        representative = Representative(lrep["id"], name, contact, image, party, team, lrep["sex"], lrep["state"])
        representatives.append(representative)

    return representatives

reps = Representatives()
<commit_msg>Remove SMTPUTF8 mail option.<commit_after>#-*- coding: utf-8 -*-

from datetime import datetime, date
from json import load
from random import choice
from uuid import uuid4
from smtplib import SMTP
from flask import url_for
from sqlalchemy import UniqueConstraint, Column, Integer, String, Date, DateTime, ForeignKey
from sqlalchemy.orm import relationship
from config import MAIL_FROM, MAIL_DEBUG
from config.main import DEBUG
from config.mail import *
from . import Base

def sendmail(addr_from, addr_to, msg):
    with SMTP("localhost") as s:
        s.sendmail(addr_from,
                   addr_to,
                   msg.encode("utf-8"))

class Reminder(Base):
    __tablename__ = "reminders"

    id = Column(Integer, primary_key=True)
    phone_number = Column(String(20), unique=True, nullable=False)
    time = Column(Integer)
    date_added = Column(Date, nullable=False)
    last_called = Column(DateTime)
    times_called = Column(Integer, nullable=False)
    times_forwarded = Column(Integer, nullable=False)
    time_connected = Column(Integer, nullable=False)

    def __init__(self, phone_number, time=None):
        self.phone_number = phone_number
        self.time = time
        self.date_added = date.today()
        self.last_called = None
        self.times_called = 0
        self.times_forwarded = 0
        self.time_connected = 0

    def __repr__(self):
        return "<Reminder for {}>".format(self.phone_number)

class Mail(Base):
    __tablename__ = "mails"

    id = Column(Integer, primary_key=True)
    sender_id = Column(Integer, ForeignKey("senders.id"))
    recipient = Column(String(5), nullable=False)
    date_sent = Column(DateTime)

    __table_args__ = tuple(
            UniqueConstraint(sender_id, recipient)
            )

    def __init__(self, sender, recipient):
        self.sender = sender
        self.recipient = recipient

    def send(self):
        self.date_sent = datetime.now()

        rep = reps.get_representative_by_id(self.recipient)

        addr_from = MAIL_FROM + " <" + MAIL_FROM + ">"
        if DEBUG:
            addr_to = MAIL_DEBUG + " <" + MAIL_DEBUG + ">"
        else:
            addr_to = str(rep) + " <" + rep.contact.mail + ">"
        salutation = "Sehr geehrter Herr" if rep.sex == "male" else "Sehr geehrte Frau"
        msg = MAIL_DISCLAIMER.format(name_user=self.sender.name, mail_user=self.sender.email_address) + "\n" * 2
        msg = msg + MAIL_REPRESENTATIVE.format(name_rep=str(rep), name_user=self.sender.name, salutation=salutation)
        sendmail(addr_from, addr_to, msg)

class Sender(Base):
    __tablename__ = "senders"

    id = Column(Integer, primary_key=True)
    name = Column(String(256), nullable=False)
    email_address = Column(String(254), unique=True, nullable=False)
    mails = relationship("Mail", order_by="Mail.id", backref="sender")
    hash = Column(String(64), unique=True, nullable=False)
    date_validated = Column(DateTime)
    date_requested = Column(DateTime, nullable=False)

    def __init__(self, name, email_address):
        self.name = name
        self.email_address = email_address
        self.request_validation()

    def validate(self):
        self.date_validated = datetime.now()

        addr_from = MAIL_FROM + " <" + MAIL_FROM + ">"
        addr_to = self.name + " <" + self.email_address + ">"
        msg = MAIL_WELCOME.format(name_user=self.name)
        sendmail(addr_from, addr_to, msg)

    def request_validation(self):
        self.hash = uuid4().hex
        self.date_requested = datetime.now()

        addr_from = MAIL_FROM + " <" + MAIL_FROM + ">"
        addr_to = self.name + " <" + self.email_address + ">"
        url = url_for("validate", hash=self.hash, _external=True)
        msg = MAIL_VALIDATE.format(name_user=self.name, url=url)
        sendmail(addr_from, addr_to, msg)

class Representatives():
    def __init__(self):
        self.parties = load_parties()
        self.teams = load_teams()
        self.representatives = load_representatives("representatives.json", self.parties, self.teams)
        self.government = load_representatives("government.json", self.parties, self.teams)

    def get_representative_by_id(self, id):
        representatives = self.representatives + self.government
        if id == "00000":
            return choice([rep for rep in representatives if rep.team.prettyname == "spy"])

        try:
            rep = [rep for rep in representatives if rep.id == id][0]
        except IndexError:
            rep = None
        return rep

    def get_representative_by_name(self, prettyname):
        representatives = self.representatives + self.government
        try:
            rep = [rep for rep in representatives if rep.name.prettyname == prettyname][0]
        except IndexError:
            rep = None
        return rep

    def get_party(self, shortname):
        return self.parties[shortname]

class Contact():
    def __init__(self, mail, phone, facebook, twitter):
        self.mail = mail
        self.phone = phone
        self.facebook = facebook
        self.twitter = twitter

class Party():
    def __init__(self, name, shortname, prettyname, color, contact):
        self.name = name
        self.shortname = shortname
        self.prettyname = prettyname
        self.color = color
        self.contact = contact

class Name():
    def __init__(self, firstname, lastname, prettyname, prefix, suffix):
        self.firstname = firstname
        self.lastname = lastname
        self.prettyname = prettyname
        self.prefix = prefix
        self.suffix = suffix

class Image():
    def __init__(self, url, copyright):
        self.url = url
        self.copyright = copyright

class Team():
    def __init__(self, name, prettyname):
        self.name = name
        self.prettyname = prettyname

    def __repr__(self):
        return self.name

class Representative():
    def __init__(self, id, name, contact, image, party, team, sex, state):
        self.id = id
        self.name = name
        self.contact = contact
        self.image = image
        self.party = party
        self.team = team
        self.sex = sex
        self.state = state

        if not self.contact.mail:
            self.contact.mail = party.contact.mail
        if not self.contact.phone:
            self.contact.phone = party.contact.phone
        if not self.contact.facebook:
            self.contact.facebook = party.contact.facebook
        if not self.contact.twitter:
            self.contact.twitter = party.contact.twitter

    def __repr__(self):
        return self.name.firstname + " " + self.name.lastname

    def fullname(self):
        return (self.name.prefix + " " if self.name.prefix else "") + self.name.firstname + " " + self.name.lastname + (" " + self.name.suffix if self.name.suffix else "")

def load_parties():
    parties = {}

    with open("ueberwachungspaket/data/parties.json", "r") as f:
        lparties = load(f)

    for prettyname in lparties:
        lparty = lparties[prettyname]
        lcontact = lparty["contact"]
        contact = Contact(lcontact["mail"], lcontact["phone"], lcontact["facebook"], lcontact["twitter"])
        party = Party(lparty["name"], lparty["shortname"], prettyname, lparty["color"], contact)
        parties[prettyname] = party

    return parties

def load_teams():
    teams = {}

    with open("ueberwachungspaket/data/teams.json", "r") as f:
        lteams = load(f)

    for prettyname in lteams:
        lteam = lteams[prettyname]
        team = Team(lteam["name"], prettyname)
        teams[prettyname] = team

    return teams

def load_representatives(filename, parties, teams):
    representatives = []

    with open("ueberwachungspaket/data/" + filename, "r") as f:
        lrepresentatives = load(f)

    for lrep in lrepresentatives:
        lname = lrep["name"]
        name = Name(lname["firstname"], lname["lastname"], lname["prettyname"], lname["prefix"], lname["suffix"])
        lcontact = lrep["contact"]
        contact = Contact(lcontact["mail"], lcontact["phone"], lcontact["facebook"], lcontact["twitter"])
        image = Image(lrep["image"]["url"], lrep["image"]["copyright"])
        party = parties[lrep["party"]]
        team = teams[lrep["team"]]
        representative = Representative(lrep["id"], name, contact, image, party, team, lrep["sex"], lrep["state"])
        representatives.append(representative)

    return representatives

reps = Representatives()
<|endoftext|>


<commit_before>from functools import partial
from pathlib import Path
import warnings
import pandas
import itertools
import re
import numpy as np
import uuid
import numbers
from . import blob, hash
from .errors import DataJointError
from .settings import config
from .utils import safe_write

class key:
    """
    object that allows requesting the primary key as an argument in expression.fetch()
    The string "KEY" can be used instead of the class key
    """

    pass

def is_key(attr):
    return attr is key or attr == "KEY"

def to_dicts(recarray):
    """convert record array to a dictionaries"""
    for rec in recarray:
        yield dict(zip(recarray.dtype.names, rec.tolist()))

def _get(connection, attr, data, squeeze, download_path, disable_checksum):
    """
    This function is called for every attribute

    :param connection: a dj.Connection object
    :param attr: attribute name from the table's heading
    :param data: literal value fetched from the table
    :param squeeze: if True squeeze blobs
    :param download_path: for fetches that download data, e.g. attachments
    :return: unpacked data
    """
    if data is None:
        return

    extern = (
        connection.schemas[attr.database].external[attr.store]
        if attr.is_external
        else None
    )

    # apply attribute adapter if present
    adapt = attr.adapter.get if attr.adapter else lambda x: x

    if attr.is_filepath:
        return adapt(extern.download_filepath(uuid.UUID(bytes=data))[0])

    if attr.is_attachment:
        # Steps:
        # 1. get the attachment filename
        # 2. check if the file already exists at download_path, verify checksum
        # 3. if exists and checksum passes then return the local filepath
        # 4. Otherwise, download the remote file and return the new filepath
        _uuid = uuid.UUID(bytes=data) if attr.is_external else None
        attachment_name = (
            extern.get_attachment_name(_uuid)
            if attr.is_external
            else data.split(b"\0", 1)[0].decode()
        )
        local_filepath = Path(download_path) / attachment_name
        if local_filepath.is_file() and not disable_checksum:
            attachment_checksum = (
                _uuid if attr.is_external else hash.uuid_from_buffer(data)
            )
            if attachment_checksum == hash.uuid_from_file(
                local_filepath, init_string=attachment_name + "\0"
            ):
                return adapt(
                    str(local_filepath)
                )  # checksum passed, no need to download again
            # generate the next available alias filename
            for n in itertools.count():
                f = local_filepath.parent / (
                    local_filepath.stem + "_%04x" % n + local_filepath.suffix
                )
                if not f.is_file():
                    local_filepath = f
                    break
                if attachment_checksum == hash.uuid_from_file(
                    f, init_string=attachment_name + "\0"
                ):
                    return adapt(str(f))  # checksum passed, no need to download again
        # Save attachment
        if attr.is_external:
            extern.download_attachment(_uuid, attachment_name, local_filepath)
        else:
            # write from buffer
            safe_write(local_filepath, data.split(b"\0", 1)[1])
        return adapt(str(local_filepath))  # download file from remote store

    return adapt(
        uuid.UUID(bytes=data)
        if attr.uuid
        else (
            blob.unpack(
                extern.get(uuid.UUID(bytes=data)) if attr.is_external else data,
                squeeze=squeeze,
            )
            if attr.is_blob
            else data
        )
    )

def _flatten_attribute_list(primary_key, attrs):
    """
    :param primary_key: list of attributes in primary key
    :param attrs: list of attribute names, which may include "KEY", "KEY DESC" or "KEY ASC"
    :return: generator of attributes where "KEY" is replaces with its component attributes
    """
    for a in attrs:
        if re.match(r"^\s*KEY(\s+[aA][Ss][Cc])?\s*$", a):
            yield from primary_key
        elif re.match(r"^\s*KEY\s+[Dd][Ee][Ss][Cc]\s*$", a):
            yield from (q + " DESC" for q in primary_key)
        else:
            yield a

class Fetch:
    """
    A fetch object that handles retrieving elements from the table expression.

    :param expression: the QueryExpression object to fetch from.
    """

    def __init__(self, expression):
        self._expression = expression

    def __call__(
        self,
        *attrs,
        offset=None,
        limit=None,
        order_by=None,
        format=None,
        as_dict=None,
        squeeze=False,
        download_path=".",
        disable_checksum=False
    ):
        """
        Fetches the expression results from the database into an np.array or list of dictionaries and
        unpacks blob attributes.

        :param attrs: zero or more attributes to fetch. If not provided, the call will return all attributes of this
                        relation. If provided, returns tuples with an entry for each attribute.
        :param offset: the number of tuples to skip in the returned result
        :param limit: the maximum number of tuples to return
        :param order_by: a single attribute or the list of attributes to order the results. No ordering should be assumed
                        if order_by=None. To reverse the order, add DESC to the attribute name or names: e.g. ("age DESC",
                        "frequency") To order by primary key, use "KEY" or "KEY DESC"
        :param format: Effective when as_dict=None and when attrs is empty None: default from config['fetch_format'] or
                        'array' if not configured "array": use numpy.key_array "frame": output pandas.DataFrame. .
        :param as_dict: returns a list of dictionaries instead of a record array. Defaults to False for .fetch() and to
                        True for .fetch('KEY')
        :param squeeze:  if True, remove extra dimensions from arrays
        :param download_path: for fetches that download data, e.g. attachments
        :return: the contents of the relation in the form of a structured numpy.array or a dict list
        """
        if disable_checksum and config["safemode"]:
            warn_str = (
                b"Are you sure that you want to disable checksums?\n"
                + b"There is no guarantee that any files you retrieve have been unmodified.\n"
                + "(Y/N):"
            )
            choice = input(warn_str)
            if choice == "N":
                return "Nothing fetched (fetch aborted)"
            elif choice != "Y":
                return "Nothing fetched (invalid prompt input)"

        if order_by is not None:
            # if 'order_by' passed in a string, make into list
            if isinstance(order_by, str):
                order_by = [order_by]
            # expand "KEY" or "KEY DESC"
            order_by = list(
                _flatten_attribute_list(self._expression.primary_key, order_by)
            )

        attrs_as_dict = as_dict and attrs
        if attrs_as_dict:
            # absorb KEY into attrs and prepare to return attributes as dict (issue #595)
            if any(is_key(k) for k in attrs):
                attrs = list(self._expression.primary_key) + [
                    a for a in attrs if a not in self._expression.primary_key
                ]
        if as_dict is None:
            as_dict = bool(attrs)  # default to True for "KEY" and False otherwise
        # format should not be specified with attrs or is_dict=True
        if format is not None and (as_dict or attrs):
            raise DataJointError(
                "Cannot specify output format when as_dict=True or "
                "when attributes are selected to be fetched separately."
            )
        if format not in {None, "array", "frame"}:
            raise DataJointError(
                "Fetch output format must be in "
                '{{"array", "frame"}} but "{}" was given'.format(format)
            )

        if not (attrs or as_dict) and format is None:
            format = config["fetch_format"]  # default to array
            if format not in {"array", "frame"}:
                raise DataJointError(
                    'Invalid entry "{}" in datajoint.config["fetch_format"]: '
                    'use "array" or "frame"'.format(format)
                )

        if limit is None and offset is not None:
            warnings.warn(
                "Offset set, but no limit. Setting limit to a large number. "
                "Consider setting a limit explicitly."
            )
            limit = 8000000000  # just a very large number to effect no limit

        get = partial(
            _get,
            self._expression.connection,
            squeeze=squeeze,
            download_path=download_path,
            disable_checksum=disable_checksum,
        )
        if attrs:  # a list of attributes provided
            attributes = [a for a in attrs if not is_key(a)]
            ret = self._expression.proj(*attributes)
            ret = ret.fetch(
                offset=offset,
                limit=limit,
                order_by=order_by,
                as_dict=False,
                squeeze=squeeze,
                download_path=download_path,
                format="array",
            )
            if attrs_as_dict:
                ret = [
                    {k: v for k, v in zip(ret.dtype.names, x) if k in attrs}
                    for x in ret
                ]
            else:
                return_values = [
                    list(
                        (to_dicts if as_dict else lambda x: x)(
                            ret[self._expression.primary_key]
                        )
                    )
                    if is_key(attribute)
                    else ret[attribute]
                    for attribute in attrs
                ]
                ret = return_values[0] if len(attrs) == 1 else return_values
        else:  # fetch all attributes as a numpy.record_array or pandas.DataFrame
            cur = self._expression.cursor(
                as_dict=as_dict, limit=limit, offset=offset, order_by=order_by
            )
            heading = self._expression.heading
            if as_dict:
                ret = [
                    dict((name, get(heading[name], d[name])) for name in heading.names)
                    for d in cur
                ]
            else:
                ret = list(cur.fetchall())
                record_type = (
                    heading.as_dtype
                    if not ret
                    else np.dtype(
                        [
                            (
                                name,
                                type(value),
                            )  # use the first element to determine blob type
                            if heading[name].is_blob
                            and isinstance(value, numbers.Number)
                            else (name, heading.as_dtype[name])
                            for value, name in zip(ret[0], heading.as_dtype.names)
                        ]
                    )
                )
                try:
                    ret = np.array(ret, dtype=record_type)
                except Exception as e:
                    raise e
                for name in heading:
                    # unpack blobs and externals
                    ret[name] = list(map(partial(get, heading[name]), ret[name]))
                if format == "frame":
                    ret = pandas.DataFrame(ret).set_index(heading.primary_key)
        return ret

class Fetch1:
    """
    Fetch object for fetching the result of a query yielding one row.

    :param expression: a query expression to fetch from.
    """

    def __init__(self, expression):
        self._expression = expression

    def __call__(
        self, *attrs, squeeze=False, download_path=".", disable_checksum=False
    ):
        """
        Fetches the result of a query expression that yields one entry.

        If no attributes are specified, returns the result as a dict.
        If attributes are specified returns the corresponding results as a tuple.

        Examples:
        d = rel.fetch1()   # as a dictionary
        a, b = rel.fetch1('a', 'b')   # as a tuple

        :params *attrs: attributes to return when expanding into a tuple.
                 If attrs is empty, the return result is a dict
        :param squeeze:  When true, remove extra dimensions from arrays in attributes
        :param download_path: for fetches that download data, e.g. attachments
        :return: the one tuple in the relation in the form of a dict
        """
        heading = self._expression.heading

        if disable_checksum and config["safemode"]:
            warn_str = (
                b"Are you sure that you want to disable checksums?\n"
                + b"There is no guarantee that any files you retrieve have been unmodified.\n"
                + "(Y/N):"
            )
            choice = input(warn_str)
            if choice == "N":
                return "Nothing fetched (fetch aborted)"
            elif choice != "Y":
                return "Nothing fetched (invalid prompt input)"

        if not attrs:  # fetch all attributes, return as ordered dict
            cur = self._expression.cursor(as_dict=True)
            ret = cur.fetchone()
            if not ret or cur.fetchone():
                raise DataJointError(
                    "fetch1 requires exactly one tuple in the input set."
                )
            ret = dict(
                (
                    name,
                    _get(
                        self._expression.connection,
                        heading[name],
                        ret[name],
                        squeeze=squeeze,
                        download_path=download_path,
                        disable_checksum=disable_checksum,
                    ),
                )
                for name in heading.names
            )
        else:  # fetch some attributes, return as tuple
            attributes = [a for a in attrs if not is_key(a)]
            result = self._expression.proj(*attributes).fetch(
                squeeze=squeeze, download_path=download_path, format="array"
            )
            if len(result) != 1:
                raise DataJointError(
                    "fetch1 should only return one tuple. %d tuples found" % len(result)
                )
            return_values = tuple(
                next(to_dicts(result[self._expression.primary_key]))
                if is_key(attribute)
                else result[attribute][0]
                for attribute in attrs
            )
            ret = return_values[0] if len(attrs) == 1 else return_values
        return ret
<commit_msg>fix minor bug.<commit_after>from functools import partial
from pathlib import Path
import warnings
import pandas
import itertools
import re
import numpy as np
import uuid
import numbers
from . import blob, hash
from .errors import DataJointError
from .settings import config
from .utils import safe_write

class key:
    """
    object that allows requesting the primary key as an argument in expression.fetch()
    The string "KEY" can be used instead of the class key
    """

    pass

def is_key(attr):
    return attr is key or attr == "KEY"

def to_dicts(recarray):
    """convert record array to a dictionaries"""
    for rec in recarray:
        yield dict(zip(recarray.dtype.names, rec.tolist()))

def _get(connection, attr, data, squeeze, download_path, disable_checksum):
    """
    This function is called for every attribute

    :param connection: a dj.Connection object
    :param attr: attribute name from the table's heading
    :param data: literal value fetched from the table
    :param squeeze: if True squeeze blobs
    :param download_path: for fetches that download data, e.g. attachments
    :return: unpacked data
    """
    if data is None:
        return

    extern = (
        connection.schemas[attr.database].external[attr.store]
        if attr.is_external
        else None
    )

    # apply attribute adapter if present
    adapt = attr.adapter.get if attr.adapter else lambda x: x

    if attr.is_filepath:
        return adapt(extern.download_filepath(uuid.UUID(bytes=data))[0])

    if attr.is_attachment:
        # Steps:
        # 1. get the attachment filename
        # 2. check if the file already exists at download_path, verify checksum
        # 3. if exists and checksum passes then return the local filepath
        # 4. Otherwise, download the remote file and return the new filepath
        _uuid = uuid.UUID(bytes=data) if attr.is_external else None
        attachment_name = (
            extern.get_attachment_name(_uuid)
            if attr.is_external
            else data.split(b"\0", 1)[0].decode()
        )
        local_filepath = Path(download_path) / attachment_name
        if local_filepath.is_file() and not disable_checksum:
            attachment_checksum = (
                _uuid if attr.is_external else hash.uuid_from_buffer(data)
            )
            if attachment_checksum == hash.uuid_from_file(
                local_filepath, init_string=attachment_name + "\0"
            ):
                return adapt(
                    str(local_filepath)
                )  # checksum passed, no need to download again
            # generate the next available alias filename
            for n in itertools.count():
                f = local_filepath.parent / (
                    local_filepath.stem + "_%04x" % n + local_filepath.suffix
                )
                if not f.is_file():
                    local_filepath = f
                    break
                if attachment_checksum == hash.uuid_from_file(
                    f, init_string=attachment_name + "\0"
                ):
                    return adapt(str(f))  # checksum passed, no need to download again
        # Save attachment
        if attr.is_external:
            extern.download_attachment(_uuid, attachment_name, local_filepath)
        else:
            # write from buffer
            safe_write(local_filepath, data.split(b"\0", 1)[1])
        return adapt(str(local_filepath))  # download file from remote store

    return adapt(
        uuid.UUID(bytes=data)
        if attr.uuid
        else (
            blob.unpack(
                extern.get(uuid.UUID(bytes=data)) if attr.is_external else data,
                squeeze=squeeze,
            )
            if attr.is_blob
            else data
        )
    )

def _flatten_attribute_list(primary_key, attrs):
    """
    :param primary_key: list of attributes in primary key
    :param attrs: list of attribute names, which may include "KEY", "KEY DESC" or "KEY ASC"
    :return: generator of attributes where "KEY" is replaces with its component attributes
    """
    for a in attrs:
        if re.match(r"^\s*KEY(\s+[aA][Ss][Cc])?\s*$", a):
            yield from primary_key
        elif re.match(r"^\s*KEY\s+[Dd][Ee][Ss][Cc]\s*$", a):
            yield from (q + " DESC" for q in primary_key)
        else:
            yield a

class Fetch:
    """
    A fetch object that handles retrieving elements from the table expression.

    :param expression: the QueryExpression object to fetch from.
    """

    def __init__(self, expression):
        self._expression = expression

    def __call__(
        self,
        *attrs,
        offset=None,
        limit=None,
        order_by=None,
        format=None,
        as_dict=None,
        squeeze=False,
        download_path=".",
        disable_checksum=False
    ):
        """
        Fetches the expression results from the database into an np.array or list of dictionaries and
        unpacks blob attributes.

        :param attrs: zero or more attributes to fetch. If not provided, the call will return all attributes of this
                        relation. If provided, returns tuples with an entry for each attribute.
        :param offset: the number of tuples to skip in the returned result
        :param limit: the maximum number of tuples to return
        :param order_by: a single attribute or the list of attributes to order the results. No ordering should be assumed
                        if order_by=None. To reverse the order, add DESC to the attribute name or names: e.g. ("age DESC",
                        "frequency") To order by primary key, use "KEY" or "KEY DESC"
        :param format: Effective when as_dict=None and when attrs is empty None: default from config['fetch_format'] or
                        'array' if not configured "array": use numpy.key_array "frame": output pandas.DataFrame. .
        :param as_dict: returns a list of dictionaries instead of a record array. Defaults to False for .fetch() and to
                        True for .fetch('KEY')
        :param squeeze:  if True, remove extra dimensions from arrays
        :param download_path: for fetches that download data, e.g. attachments
        :return: the contents of the relation in the form of a structured numpy.array or a dict list
        """
        if disable_checksum and config["safemode"]:
            warn_str = (
                "Are you sure that you want to disable checksums?\n"
                + "There is no guarantee that any files you retrieve have been unmodified.\n"
                + "(Y/N):"
            )
            choice = input(warn_str)
            if choice == "N":
                return "Nothing fetched (fetch aborted)"
            elif choice != "Y":
                return "Nothing fetched (invalid prompt input)"

        if order_by is not None:
            # if 'order_by' passed in a string, make into list
            if isinstance(order_by, str):
                order_by = [order_by]
            # expand "KEY" or "KEY DESC"
            order_by = list(
                _flatten_attribute_list(self._expression.primary_key, order_by)
            )

        attrs_as_dict = as_dict and attrs
        if attrs_as_dict:
            # absorb KEY into attrs and prepare to return attributes as dict (issue #595)
            if any(is_key(k) for k in attrs):
                attrs = list(self._expression.primary_key) + [
                    a for a in attrs if a not in self._expression.primary_key
                ]
        if as_dict is None:
            as_dict = bool(attrs)  # default to True for "KEY" and False otherwise
        # format should not be specified with attrs or is_dict=True
        if format is not None and (as_dict or attrs):
            raise DataJointError(
                "Cannot specify output format when as_dict=True or "
                "when attributes are selected to be fetched separately."
            )
        if format not in {None, "array", "frame"}:
            raise DataJointError(
                "Fetch output format must be in "
                '{{"array", "frame"}} but "{}" was given'.format(format)
            )

        if not (attrs or as_dict) and format is None:
            format = config["fetch_format"]  # default to array
            if format not in {"array", "frame"}:
                raise DataJointError(
                    'Invalid entry "{}" in datajoint.config["fetch_format"]: '
                    'use "array" or "frame"'.format(format)
                )

        if limit is None and offset is not None:
            warnings.warn(
                "Offset set, but no limit. Setting limit to a large number. "
                "Consider setting a limit explicitly."
            )
            limit = 8000000000  # just a very large number to effect no limit

        get = partial(
            _get,
            self._expression.connection,
            squeeze=squeeze,
            download_path=download_path,
            disable_checksum=disable_checksum,
        )
        if attrs:  # a list of attributes provided
            attributes = [a for a in attrs if not is_key(a)]
            ret = self._expression.proj(*attributes)
            ret = ret.fetch(
                offset=offset,
                limit=limit,
                order_by=order_by,
                as_dict=False,
                squeeze=squeeze,
                download_path=download_path,
                format="array",
            )
            if attrs_as_dict:
                ret = [
                    {k: v for k, v in zip(ret.dtype.names, x) if k in attrs}
                    for x in ret
                ]
            else:
                return_values = [
                    list(
                        (to_dicts if as_dict else lambda x: x)(
                            ret[self._expression.primary_key]
                        )
                    )
                    if is_key(attribute)
                    else ret[attribute]
                    for attribute in attrs
                ]
                ret = return_values[0] if len(attrs) == 1 else return_values
        else:  # fetch all attributes as a numpy.record_array or pandas.DataFrame
            cur = self._expression.cursor(
                as_dict=as_dict, limit=limit, offset=offset, order_by=order_by
            )
            heading = self._expression.heading
            if as_dict:
                ret = [
                    dict((name, get(heading[name], d[name])) for name in heading.names)
                    for d in cur
                ]
            else:
                ret = list(cur.fetchall())
                record_type = (
                    heading.as_dtype
                    if not ret
                    else np.dtype(
                        [
                            (
                                name,
                                type(value),
                            )  # use the first element to determine blob type
                            if heading[name].is_blob
                            and isinstance(value, numbers.Number)
                            else (name, heading.as_dtype[name])
                            for value, name in zip(ret[0], heading.as_dtype.names)
                        ]
                    )
                )
                try:
                    ret = np.array(ret, dtype=record_type)
                except Exception as e:
                    raise e
                for name in heading:
                    # unpack blobs and externals
                    ret[name] = list(map(partial(get, heading[name]), ret[name]))
                if format == "frame":
                    ret = pandas.DataFrame(ret).set_index(heading.primary_key)
        return ret

class Fetch1:
    """
    Fetch object for fetching the result of a query yielding one row.

    :param expression: a query expression to fetch from.
    """

    def __init__(self, expression):
        self._expression = expression

    def __call__(
        self, *attrs, squeeze=False, download_path=".", disable_checksum=False
    ):
        """
        Fetches the result of a query expression that yields one entry.

        If no attributes are specified, returns the result as a dict.
        If attributes are specified returns the corresponding results as a tuple.

        Examples:
        d = rel.fetch1()   # as a dictionary
        a, b = rel.fetch1('a', 'b')   # as a tuple

        :params *attrs: attributes to return when expanding into a tuple.
                 If attrs is empty, the return result is a dict
        :param squeeze:  When true, remove extra dimensions from arrays in attributes
        :param download_path: for fetches that download data, e.g. attachments
        :return: the one tuple in the relation in the form of a dict
        """
        heading = self._expression.heading

        if disable_checksum and config["safemode"]:
            warn_str = (
                "Are you sure that you want to disable checksums?\n"
                + "There is no guarantee that any files you retrieve have been unmodified.\n"
                + "(Y/N):"
            )
            choice = input(warn_str)
            if choice == "N":
                return "Nothing fetched (fetch aborted)"
            elif choice != "Y":
                return "Nothing fetched (invalid prompt input)"

        if not attrs:  # fetch all attributes, return as ordered dict
            cur = self._expression.cursor(as_dict=True)
            ret = cur.fetchone()
            if not ret or cur.fetchone():
                raise DataJointError(
                    "fetch1 requires exactly one tuple in the input set."
                )
            ret = dict(
                (
                    name,
                    _get(
                        self._expression.connection,
                        heading[name],
                        ret[name],
                        squeeze=squeeze,
                        download_path=download_path,
                        disable_checksum=disable_checksum,
                    ),
                )
                for name in heading.names
            )
        else:  # fetch some attributes, return as tuple
            attributes = [a for a in attrs if not is_key(a)]
            result = self._expression.proj(*attributes).fetch(
                squeeze=squeeze, download_path=download_path, format="array"
            )
            if len(result) != 1:
                raise DataJointError(
                    "fetch1 should only return one tuple. %d tuples found" % len(result)
                )
            return_values = tuple(
                next(to_dicts(result[self._expression.primary_key]))
                if is_key(attribute)
                else result[attribute][0]
                for attribute in attrs
            )
            ret = return_values[0] if len(attrs) == 1 else return_values
        return ret
<|endoftext|>