<commit_before>#!/usr/bin/env python3
import pygame
import subprocess
import os
import configparser
import sys
import time
import calendar
import Image

config = {
    "photostorage": "Pictures",
    "chdkptp": "chdkptp",
    "fullscreen": "-w" not in sys.argv,  # -w for windowed
    "bgcolor": (0, 0, 0),
    "textcolor": (255, 255, 255),
    "displayPhotoFor": 5,
    "photosPerSet": 4,
    "countdown": 4,
    "countdownSpeed": 1
}

def configure():
    cf = configparser.SafeConfigParser()
    cf.read(['photobooth.cfg', os.path.expanduser('~/.photobooth.cfg')])
    if cf.has_section('photobooth'):
        for key in config:
            if cf.has_option('photobooth', key):
                if type(config[key]) == int:
                    config[key] = cf.getint('photobooth', key)
                else:
                    config[key] = cf.get('photobooth', key)
                    if key in ["bgcolor", "textcolor"]:
                        config[key] = config[key].split(",")
                        for color in range(len(config[key])):
                            config[key][color] = int(config[key][color].strip())
                        config[key] = tuple(config[key])
    print(config)

def waitForInput(stream, waitFor=">"):
    char = stream.read(1).decode('utf-8')
    while char != waitFor:
        print(char, end="")
        sys.stdout.flush()
        char = stream.read(1).decode('utf-8')
    print(char)

def takePhotoSet(chdkptp, game):
    stripnumber = calendar.timegm(time.gmtime())
    filename = "%s/%s" % (config['photostorage'], stripnumber)
    # Enter rsint mode
    chdkptp.stdin.write(("rsint %s-last\n" % filename).encode())
    waitForInput(chdkptp.stdout)
    photonumber = 0
    while photonumber < config['photosPerSet']:
        doCountdown(game)
        takePhoto(chdkptp)
        os.rename("%s-last.jpg" % filename, "%s-%s.jpg" % (filename, photonumber))
        displayPhoto("%s-%s.jpg" % (filename, photonumber), game)
        photonumber += 1
    chdkptp.stdin.write(b"q\n")
    waitForInput(chdkptp.stdout)

def renderText(textstr, game, fontSize=1000):
    font = pygame.font.Font(None, fontSize)
    text = font.render(textstr, 1, config['textcolor'])
    textpos = text.get_rect()
    textpos.centerx = game['background'].get_rect().centerx
    game['background'].blit(text, textpos)
    game['screen'].blit(game['background'], (0, 0))
    pygame.display.flip()
    game['clock'].tick()

def doCountdown(game):
    countdown = config['countdown']
    while countdown >= 0:
        game['background'].fill(config['bgcolor'])
        if countdown > 0:
            renderText(str(countdown), game, fontSize=1500)
        elif countdown == 0:
            renderText("smile", game)
        countdown -= 1
        time.sleep(config['countdownSpeed'])

def takePhoto(chdkptp):
    chdkptp.stdin.write(b"s\n")
    waitForInput(chdkptp.stdout)

def displayPhoto(filename, game):
    img = pygame.image.load(filename)
    imgposition = img.get_rect()
    imgposition.centerx = game['background'].get_rect().centerx
    imgposition.centery = game['background'].get_rect().centery
    game['background'].blit(img, imgposition)
    game['screen'].blit(game['background'], (0, 0))
    pygame.display.flip()
    game['clock'].tick()
    time.sleep(config['displayPhotoFor'])

def waitForTrigger(game):
    triggered = False
    game['background'].fill(config['bgcolor'])
    renderText("Press Space to begin", game, fontSize=200)
    while not triggered:
        for event in pygame.event.get():
            if event.type == pygame.QUIT:
                sys.exit()
            if event.type == pygame.KEYDOWN:
                if event.key == 32:  # spacebar
                    triggered = True
                elif event.key in [113, 27]:  # q, esc
                    sys.exit()
        time.sleep(0.05)
    print("triggered!!")

def main():
    configure()

    pygame.init()

    displayinfo = pygame.display.Info()  # Find some info out about the display
    if config['fullscreen']:
        size = displayinfo.current_w, displayinfo.current_h
    else:
        size = 1800, 800

    screen = pygame.display.set_mode(size)

    if config['fullscreen']:
        pygame.display.toggle_fullscreen()
    c = pygame.time.Clock()

    background = pygame.Surface(screen.get_size())

    chdkptp = subprocess.Popen(config['chdkptp'],
                               stdin=subprocess.PIPE,
                               stdout=subprocess.PIPE,
                               stderr=subprocess.PIPE,
                               bufsize=0)
    waitForInput(chdkptp.stdout)
    chdkptp.stdin.write(b"c\n")
    print("Connecting to camera....")
    waitForInput(chdkptp.stdout)
    chdkptp.stdin.write(b"rec\n")
    print("Entering shooting mode...")
    waitForInput(chdkptp.stdout)
    print("Annnd into the loop we go!")

    game = {"screen": screen, "background": background, "clock": c}
    while True:
        waitForTrigger(game)
        takePhotoSet(chdkptp, game)

if __name__ == "__main__":
    main()
<commit_msg>Removed unused dependency<commit_after>#!/usr/bin/env python3
import pygame
import subprocess
import os
import configparser
import sys
import time
import calendar

config = {
    "photostorage": "Pictures",
    "chdkptp": "chdkptp",
    "fullscreen": "-w" not in sys.argv,  # -w for windowed
    "bgcolor": (0, 0, 0),
    "textcolor": (255, 255, 255),
    "displayPhotoFor": 5,
    "photosPerSet": 4,
    "countdown": 4,
    "countdownSpeed": 1
}

def configure():
    cf = configparser.SafeConfigParser()
    cf.read(['photobooth.cfg', os.path.expanduser('~/.photobooth.cfg')])
    if cf.has_section('photobooth'):
        for key in config:
            if cf.has_option('photobooth', key):
                if type(config[key]) == int:
                    config[key] = cf.getint('photobooth', key)
                else:
                    config[key] = cf.get('photobooth', key)
                    if key in ["bgcolor", "textcolor"]:
                        config[key] = config[key].split(",")
                        for color in range(len(config[key])):
                            config[key][color] = int(config[key][color].strip())
                        config[key] = tuple(config[key])
    print(config)

def waitForInput(stream, waitFor=">"):
    char = stream.read(1).decode('utf-8')
    while char != waitFor:
        print(char, end="")
        sys.stdout.flush()
        char = stream.read(1).decode('utf-8')
    print(char)

def takePhotoSet(chdkptp, game):
    stripnumber = calendar.timegm(time.gmtime())
    filename = "%s/%s" % (config['photostorage'], stripnumber)
    # Enter rsint mode
    chdkptp.stdin.write(("rsint %s-last\n" % filename).encode())
    waitForInput(chdkptp.stdout)
    photonumber = 0
    while photonumber < config['photosPerSet']:
        doCountdown(game)
        takePhoto(chdkptp)
        os.rename("%s-last.jpg" % filename, "%s-%s.jpg" % (filename, photonumber))
        displayPhoto("%s-%s.jpg" % (filename, photonumber), game)
        photonumber += 1
    chdkptp.stdin.write(b"q\n")
    waitForInput(chdkptp.stdout)

def renderText(textstr, game, fontSize=1000):
    font = pygame.font.Font(None, fontSize)
    text = font.render(textstr, 1, config['textcolor'])
    textpos = text.get_rect()
    textpos.centerx = game['background'].get_rect().centerx
    game['background'].blit(text, textpos)
    game['screen'].blit(game['background'], (0, 0))
    pygame.display.flip()
    game['clock'].tick()

def doCountdown(game):
    countdown = config['countdown']
    while countdown >= 0:
        game['background'].fill(config['bgcolor'])
        if countdown > 0:
            renderText(str(countdown), game, fontSize=1500)
        elif countdown == 0:
            renderText("smile", game)
        countdown -= 1
        time.sleep(config['countdownSpeed'])

def takePhoto(chdkptp):
    chdkptp.stdin.write(b"s\n")
    waitForInput(chdkptp.stdout)

def displayPhoto(filename, game):
    img = pygame.image.load(filename)
    imgposition = img.get_rect()
    imgposition.centerx = game['background'].get_rect().centerx
    imgposition.centery = game['background'].get_rect().centery
    game['background'].blit(img, imgposition)
    game['screen'].blit(game['background'], (0, 0))
    pygame.display.flip()
    game['clock'].tick()
    time.sleep(config['displayPhotoFor'])

def waitForTrigger(game):
    triggered = False
    game['background'].fill(config['bgcolor'])
    renderText("Press Space to begin", game, fontSize=200)
    while not triggered:
        for event in pygame.event.get():
            if event.type == pygame.QUIT:
                sys.exit()
            if event.type == pygame.KEYDOWN:
                if event.key == 32:  # spacebar
                    triggered = True
                elif event.key in [113, 27]:  # q, esc
                    sys.exit()
        time.sleep(0.05)
    print("triggered!!")

def main():
    configure()

    pygame.init()

    displayinfo = pygame.display.Info()  # Find some info out about the display
    if config['fullscreen']:
        size = displayinfo.current_w, displayinfo.current_h
    else:
        size = 1800, 800

    screen = pygame.display.set_mode(size)

    if config['fullscreen']:
        pygame.display.toggle_fullscreen()
    c = pygame.time.Clock()

    background = pygame.Surface(screen.get_size())

    chdkptp = subprocess.Popen(config['chdkptp'],
                               stdin=subprocess.PIPE,
                               stdout=subprocess.PIPE,
                               stderr=subprocess.PIPE,
                               bufsize=0)
    waitForInput(chdkptp.stdout)
    chdkptp.stdin.write(b"c\n")
    print("Connecting to camera....")
    waitForInput(chdkptp.stdout)
    chdkptp.stdin.write(b"rec\n")
    print("Entering shooting mode...")
    waitForInput(chdkptp.stdout)
    print("Annnd into the loop we go!")

    game = {
        "screen": screen,
        "background": background,
        "clock": c,
        "size": size
    }

    while True:
        waitForTrigger(game)
        takePhotoSet(chdkptp, game)

if __name__ == "__main__":
    main()
<|endoftext|>


<commit_before>from pylons import config
import ckan.logic as logic
import ckan.lib.base as base
import ckan.model as model
import ckan.plugins as p
from ckan.common import _, c
from ckan.lib.helpers import link_to
import logging
import json
from ckanext.nhm.lib.helpers import resource_view_get_view
from ckanext.nhm.views import DarwinCoreView

log = logging.getLogger(__name__)

render = base.render
abort = base.abort
redirect = base.redirect

NotFound = logic.NotFound
NotAuthorized = logic.NotAuthorized
ValidationError = logic.ValidationError
get_action = logic.get_action

TILED_MAP_TYPE = 'tiledmap'  # The view type for the tiledmap

class RecordController(base.BaseController):
    """
    Controller for displaying an individual record

    Loads all the data and then defers render function to view objects
    """

    def _load_data(self, package_name, resource_id, record_id):
        """
        Load the data for dataset, resource and record (into C var)
        @param package_name:
        @param resource_id:
        @param record_id:
        @return:
        """
        self.context = {'model': model, 'session': model.Session, 'user': c.user or c.author}

        # Try & get the resource
        try:
            c.resource = get_action('resource_show')(self.context, {'id': resource_id})
            c.package = get_action('package_show')(self.context, {'id': package_name})
            # required for nav menu
            c.pkg = self.context['package']
            c.pkg_dict = c.package
            record = get_action('record_get')(self.context, {'resource_id': resource_id, 'record_id': record_id})
            c.record_dict = record['data']
            record_field_types = {f['id']: f['type'] for f in record['fields']}

        except NotFound:
            abort(404, _('Resource not found'))
        except NotAuthorized:
            abort(401, _('Unauthorized to read resource %s') % package_name)

        field_names = {
            'image': c.resource.get('_image_field', None),
            'title': c.resource.get('_title_field', None),
            'latitude': None,
            'longitude': None
        }

        # Get lat/long fields
        # Loop through all the views - if we have a tiled map view with lat/lon fields
        # We'll use those fields to add the map
        views = p.toolkit.get_action('resource_view_list')(self.context, {'id': resource_id})
        for view in views:
            if view['view_type'] == TILED_MAP_TYPE:
                field_names['latitude'] = view[u'latitude_field']
                field_names['longitude'] = view[u'longitude_field']
                break

        # If this is a DwC dataset, add some default for image and lat/lon fields
        if c.resource['format'].lower() == 'dwc':
            for field_name, dwc_field in [('latitude', 'decimalLatitude'), ('longitude', 'decimalLongitude')]:
                if dwc_field in c.record_dict:
                    field_names[field_name] = dwc_field

        # Assign title based on the title field
        c.record_title = c.record_dict.get(field_names['title'], 'Record %s' % c.record_dict.get('_id'))

        # Sanity check: image field hasn't been set to _id
        if field_names['image'] and field_names['image'] != '_id':

            image_field_type = record_field_types[field_names['image']]

            default_copyright = '© The Trustees of the Natural History Museum, London'
            licence_id = c.resource.get('_image_licence') or 'cc-by'
            licence = model.Package.get_license_register()[licence_id]
            default_licence = 'Licence: %s' % link_to(licence.title, licence.url, target='_blank')

            image_field_value = c.record_dict.pop(field_names['image'])

            if image_field_value:

                if image_field_type == 'json':

                    c.images = []

                    try:
                        images = json.loads(image_field_value)
                    except ValueError:
                        pass
                    else:

                        for image in images:
                            url = image.get('identifier', None)
                            if url:

                                license = link_to(image.get('license'), image.get('license')) if image.get('license', None) else None

                                c.images.append({
                                    'modal_title': image.get('title', None) or c.record_title,
                                    'url': url,
                                    'description': '%s
%s' % (license or default_licence, image.get('rightsHolder', None) or default_copyright)
                                })

                else:
                    try:
                        # Pop the image field so it won't be output as part of the record_dict / field_data dict (see self.view())
                        c.images = [{'modal_title': c.record_title, 'url': image.strip(), 'description': '%s
%s' % (default_licence, default_copyright)} for image in image_field_value.split(';') if image.strip()]
                    except (KeyError, AttributeError):
                        # Skip errors - there are no images
                        pass

        if field_names['latitude'] and field_names['longitude']:
            latitude, longitude = c.record_dict.get(field_names['latitude']), c.record_dict.get(field_names['longitude'])

            if latitude and longitude:
                c.record_map = json.dumps({
                    'type': 'Point',
                    'coordinates': [longitude, latitude]
                })

    def view(self, package_name, resource_id, record_id):

        """
        View an individual record
        :param id:
        :param resource_id:
        :param record_id:
        :return: html
        """

        self._load_data(package_name, resource_id, record_id)

        view_cls = resource_view_get_view(c.resource)

        return view_cls.render_record(c)

    def dwc(self, package_name, resource_id, record_id):
        """
        Explicit DwC view
        @param package_name:
        @param resource_id:
        @param record_id:
        @return:
        """

        self._load_data(package_name, resource_id, record_id)

        # Is this a DwC view of an additional dataset?
        # In which case, provide links back to the original record view
        c.additional_view = True

        view = DarwinCoreView()
        return view.render_record(c)<commit_msg>Fixed default licence option<commit_after>from pylons import config
import ckan.logic as logic
import ckan.lib.base as base
import ckan.model as model
import ckan.plugins as p
from ckan.common import _, c
from ckan.lib.helpers import link_to
import logging
import json
from ckanext.nhm.lib.helpers import resource_view_get_view
from ckanext.nhm.views import DarwinCoreView

log = logging.getLogger(__name__)

render = base.render
abort = base.abort
redirect = base.redirect

NotFound = logic.NotFound
NotAuthorized = logic.NotAuthorized
ValidationError = logic.ValidationError
get_action = logic.get_action

TILED_MAP_TYPE = 'tiledmap'  # The view type for the tiledmap

class RecordController(base.BaseController):
    """
    Controller for displaying an individual record

    Loads all the data and then defers render function to view objects
    """

    def _load_data(self, package_name, resource_id, record_id):
        """
        Load the data for dataset, resource and record (into C var)
        @param package_name:
        @param resource_id:
        @param record_id:
        @return:
        """
        self.context = {'model': model, 'session': model.Session, 'user': c.user or c.author}

        # Try & get the resource
        try:
            c.resource = get_action('resource_show')(self.context, {'id': resource_id})
            c.package = get_action('package_show')(self.context, {'id': package_name})
            # required for nav menu
            c.pkg = self.context['package']
            c.pkg_dict = c.package
            record = get_action('record_get')(self.context, {'resource_id': resource_id, 'record_id': record_id})
            c.record_dict = record['data']
            record_field_types = {f['id']: f['type'] for f in record['fields']}

        except NotFound:
            abort(404, _('Resource not found'))
        except NotAuthorized:
            abort(401, _('Unauthorized to read resource %s') % package_name)

        field_names = {
            'image': c.resource.get('_image_field', None),
            'title': c.resource.get('_title_field', None),
            'latitude': None,
            'longitude': None
        }
        # Get lat/long fields
        # Loop through all the views - if we have a tiled map view with lat/lon fields
        # We'll use those fields to add the map
        views = p.toolkit.get_action('resource_view_list')(self.context, {'id': resource_id})
        for view in views:
            if view['view_type'] == TILED_MAP_TYPE:
                field_names['latitude'] = view[u'latitude_field']
                field_names['longitude'] = view[u'longitude_field']
                break

        # If this is a DwC dataset, add some default for image and lat/lon fields
        if c.resource['format'].lower() == 'dwc':
            for field_name, dwc_field in [('latitude', 'decimalLatitude'), ('longitude', 'decimalLongitude')]:
                if dwc_field in c.record_dict:
                    field_names[field_name] = dwc_field

        # Assign title based on the title field
        c.record_title = c.record_dict.get(field_names['title'], 'Record %s' % c.record_dict.get('_id'))

        # Sanity check: image field hasn't been set to _id
        if field_names['image'] and field_names['image'] != '_id':

            image_field_type = record_field_types[field_names['image']]

            default_copyright = '© The Trustees of the Natural History Museum, London'
            licence_id = c.resource.get('_image_licence') or 'ODC-BY-1.0'
            licence = model.Package.get_license_register()[licence_id]
            default_licence = 'Licence: %s' % link_to(licence.title, licence.url, target='_blank')

            image_field_value = c.record_dict.pop(field_names['image'])

            if image_field_value:

                if image_field_type == 'json':

                    c.images = []

                    try:
                        images = json.loads(image_field_value)
                    except ValueError:
                        pass
                    else:

                        for image in images:
                            url = image.get('identifier', None)
                            if url:

                                license = link_to(image.get('license'), image.get('license')) if image.get('license', None) else None

                                c.images.append({
                                    'modal_title': image.get('title', None) or c.record_title,
                                    'url': url,
                                    'description': '%s
%s' % (license or default_licence, image.get('rightsHolder', None) or default_copyright)
                                })

                else:
                    try:
                        # Pop the image field so it won't be output as part of the record_dict / field_data dict (see self.view())
                        c.images = [{'modal_title': c.record_title, 'url': image.strip(), 'description': '%s
%s' % (default_licence, default_copyright)} for image in image_field_value.split(';') if image.strip()]
                    except (KeyError, AttributeError):
                        # Skip errors - there are no images
                        pass

        if field_names['latitude'] and field_names['longitude']:
            latitude, longitude = c.record_dict.get(field_names['latitude']), c.record_dict.get(field_names['longitude'])

            if latitude and longitude:
                c.record_map = json.dumps({
                    'type': 'Point',
                    'coordinates': [longitude, latitude]
                })

    def view(self, package_name, resource_id, record_id):

        """
        View an individual record
        :param id:
        :param resource_id:
        :param record_id:
        :return: html
        """

        self._load_data(package_name, resource_id, record_id)

        view_cls = resource_view_get_view(c.resource)

        return view_cls.render_record(c)

    def dwc(self, package_name, resource_id, record_id):
        """
        Explicit DwC view
        @param package_name:
        @param resource_id:
        @param record_id:
        @return:
        """

        self._load_data(package_name, resource_id, record_id)

        # Is this a DwC view of an additional dataset?
        # In which case, provide links back to the original record view
        c.additional_view = True

        view = DarwinCoreView()
        return view.render_record(c)<|endoftext|>


<commit_before>################################################################################
# Copyright (c) 2017-2018, National Research Foundation (Square Kilometre Array)
#
# Licensed under the BSD 3-Clause License (the "License"); you may not use
# this file except in compliance with the License. You may obtain a copy
# of the License at
#
#   https://opensource.org/licenses/BSD-3-Clause
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
################################################################################

"""Various sources of correlator data and metadata."""

import urlparse
import os
import logging

import katsdptelstate
import redis
import numpy as np

from .sensordata import TelstateSensorData
from .chunkstore_s3 import S3ChunkStore
from .chunkstore_npy import NpyFileChunkStore
from .chunkstore import StoreUnavailable

logger = logging.getLogger(__name__)

class DataSourceNotFound(Exception):
    """File associated with DataSource not found or server not responding."""

class AttrsSensors(object):
    """Metadata in the form of attributes and sensors.

    Parameters
    ----------
    attrs : mapping from string to object
        Metadata attributes
    sensors : mapping from string to :class:`SensorData` objects
        Metadata sensor cache mapping sensor names to raw sensor data
    name : string, optional
        Identifier that describes the origin of the metadata (backend-specific)

    """
    def __init__(self, attrs, sensors, name='custom'):
        self.attrs = attrs
        self.sensors = sensors
        self.name = name

class VisFlagsWeights(object):
    """Correlator data in the form of visibilities, flags and weights.

    Parameters
    ----------
    vis : array-like of complex64, shape (*T*, *F*, *B*)
        Complex visibility data as a function of time, frequency and baseline
    flags : array-like of uint8, shape (*T*, *F*, *B*)
        Flags as a function of time, frequency and baseline
    weights : array-like of float32, shape (*T*, *F*, *B*)
        Visibility weights as a function of time, frequency and baseline
    name : string, optional
        Identifier that describes the origin of the data (backend-specific)

    """
    def __init__(self, vis, flags, weights, name='custom'):
        if not (vis.shape == flags.shape == weights.shape):
            raise ValueError("Shapes of vis %s, flags %s and weights %s differ"
                             % (vis.shape, flags.shape, weights.shape))
        self.vis = vis
        self.flags = flags
        self.weights = weights
        self.name = name

    @property
    def shape(self):
        return self.vis.shape

class ChunkStoreVisFlagsWeights(VisFlagsWeights):
    """Correlator data stored in a chunk store.

    Parameters
    ----------
    store : :class:`ChunkStore` object
        Chunk store
    base_name : string
        Name of dataset in store, as array name prefix (akin to a filename)
    chunk_info : dict mapping array name to info dict
        Dict specifying dtype, shape and chunks per array
    """
    def __init__(self, store, base_name, chunk_info):
        self.store = store
        da = {}
        for array, info in chunk_info.iteritems():
            array_name = store.join(base_name, array)
            da[array] = store.get_dask_array(array_name, info['chunks'],
                                             info['dtype'])
        vis = da['correlator_data']
        flags = da['flags']
        # Combine low-resolution weights and high-resolution weights_channel
        weights = da['weights'] * da['weights_channel'][..., np.newaxis]
        VisFlagsWeights.__init__(self, vis, flags, weights, base_name)

class DataSource(object):
    """A generic data source presenting both correlator data and metadata.

    Parameters
    ----------
    metadata : :class:`AttrsSensors` object
        Metadata attributes and sensors
    timestamps : array-like of float, length *T*
        Timestamps at centroids of visibilities in UTC seconds since Unix epoch
    data : :class:`VisFlagsWeights` object, optional
        Correlator data (visibilities, flags and weights)

    """
    def __init__(self, metadata, timestamps, data=None):
        self.metadata = metadata
        self.timestamps = timestamps
        self.data = data

    @property
    def name(self):
        name = self.metadata.name
        if self.data and self.data.name != name:
            name += ' | ' + self.data.name
        return name

def view_capture_stream(telstate, capture_block_id=None, stream_name=None):
    """Create telstate view based on capture block ID and stream name.

    This figures out the appropriate capture block ID and L0 stream name from
    a capture-stream specific telstate, or uses the provided ones. It then
    constructs a view on `telstate` with at least the prefixes

      - <capture_block_id>_<stream_name>
      - <capture_block_id>
      - <stream_name>

    Parameters
    ----------
    telstate : :class:`katsdptelstate.TelescopeState` object
        Original telescope state
    capture_block_id : string, optional
        Specify capture block ID explicitly (detected otherwise)
    stream_name : string, optional
        Specify L0 stream name explicitly (detected otherwise)

    Returns
    -------
    telstate : :class:`katsdptelstate.TelescopeState` object
        Telstate with a view that incorporates capture block, stream and combo

    Raises
    ------
    ValueError
        If no capture block or L0 stream could be detected (with no override)
    """
    # Detect the capture block
    if not capture_block_id:
        try:
            capture_block_id = str(telstate['capture_block_id'])
        except KeyError:
            raise ValueError('No capture block ID found in telstate - '
                             'please specify it manually')
    # Detect the captured stream
    if not stream_name:
        try:
            stream_name = str(telstate['stream_name'])
        except KeyError:
            raise ValueError('No captured stream found in telstate - '
                             'please specify the stream manually')
    # Check the stream type
    telstate = telstate.view(stream_name)
    stream_type = telstate.get('stream_type', 'unknown')
    expected_type = 'sdp.vis'
    if stream_type != expected_type:
        raise ValueError("Found stream {!r} but it has the wrong type {!r},"
                         " expected {!r}".format(stream_name, stream_type,
                                                 expected_type))
    logger.info('Using capture block %r and stream %r',
                capture_block_id, stream_name)
    telstate = telstate.view(capture_block_id)
    capture_stream = telstate.SEPARATOR.join((capture_block_id, stream_name))
    telstate = telstate.view(capture_stream)
    return telstate

def _shorten_key(telstate, key):
    """Shorten telstate key by subtracting the first prefix that fits.

    Parameters
    ----------
    telstate : :class:`katsdptelstate.TelescopeState` object
        Telescope state
    key : string
        Telescope state key

    Returns
    -------
    short_key : string
        Suffix of `key` after subtracting first matching prefix, or empty
        string if `key` does not start with any of the prefixes (or exactly
        matches a prefix, which is also considered pathological)

    """
    for prefix in telstate.prefixes:
        if key.startswith(prefix):
            return key[len(prefix):]
    return ''

class TelstateDataSource(DataSource):
    """A data source based on :class:`katsdptelstate.TelescopeState`.

    It is assumed that the provided `telstate` already has the appropriate
    views to find observation, stream and chunk store information. It typically
    needs the following prefixes:

      - <capture block ID>_<L0 stream>
      - <capture block ID>
      - <L0 stream>

    Parameters
    ----------
    telstate : :class:`katsdptelstate.TelescopeState` object
        Telescope state with appropriate views
    chunk_store : :class:`katdal.ChunkStore` object, optional
        Chunk store for visibility data (the default is no data - metadata only)
    source_name : string, optional
        Name of telstate source (used for metadata name)
    """
    def __init__(self, telstate, chunk_store=None, source_name='telstate'):
        self.telstate = telstate
        # Collect sensors
        sensors = {}
        for key in telstate.keys():
            if not telstate.is_immutable(key):
                sensor_name = _shorten_key(telstate, key)
                if sensor_name:
                    sensors[sensor_name] = TelstateSensorData(telstate, key)
        metadata = AttrsSensors(telstate, sensors, name=source_name)
        try:
            t0 = telstate['sync_time'] + telstate['first_timestamp']
            int_time = telstate['int_time']
            chunk_name = telstate['chunk_name']
            chunk_info = telstate['chunk_info']
        except KeyError:
            # Metadata without data or timestamps
            DataSource.__init__(self, metadata, None)
        else:
            # Extract timestamps from telstate
            n_dumps = chunk_info['correlator_data']['shape'][0]
            timestamps = t0 + np.arange(n_dumps) * int_time
            data = ChunkStoreVisFlagsWeights(
                chunk_store, chunk_name, chunk_info) if chunk_store else None
            # Metadata and timestamps with or without data
            DataSource.__init__(self, metadata, timestamps, data)

    @classmethod
    def from_url(cls, url, chunk_store='auto'):
        """Construct TelstateDataSource from URL (RDB file / REDIS server)."""
        url_parts = urlparse.urlparse(url, scheme='file')
        kwargs = dict(urlparse.parse_qsl(url_parts.query))
        # Extract Redis database number if provided
        db = int(kwargs.pop('db', '0'))
        source_name = url_parts.geturl()
        if url_parts.scheme == 'file':
            # RDB dump file
            telstate = katsdptelstate.TelescopeState()
            try:
                telstate.load_from_file(url_parts.path)
            except OSError as err:
                raise DataSourceNotFound(str(err))
            telstate = view_capture_stream(telstate, **kwargs)
            # Look for adjacent data directory (presumably containing NPY files)
            if chunk_store == 'auto':
                capture_block_path = os.path.dirname(url_parts.path)
                store_path = os.path.join(capture_block_path, '..')
                try:
                    data_path = os.path.join(store_path, telstate['chunk_name'])
                except KeyError:
                    chunk_store = None
            if chunk_store == 'auto' and os.path.isdir(data_path):
                chunk_store = NpyFileChunkStore(os.path.normpath(store_path))
            else:
                chunk_store = S3ChunkStore(telstate['s3_endpoint_url'])
            return cls(telstate, chunk_store, source_name)
        elif url_parts.scheme == 'redis':
            # Redis server
            try:
                telstate = katsdptelstate.TelescopeState(url_parts.netloc, db)
            except (redis.ConnectionError, redis.TimeoutError) as e:
                raise DataSourceNotFound(str(e))
            telstate = view_capture_stream(telstate, **kwargs)
            if chunk_store == 'auto':
                chunk_store = S3ChunkStore(telstate['s3_endpoint_url'])
            return cls(telstate, chunk_store, source_name)

def open_data_source(url, *args, **kwargs):
    """Construct the data source described by the given URL."""
    try:
        return TelstateDataSource.from_url(url, *args, **kwargs)
    except DataSourceNotFound as err:
        # Amend the error message for the case of an IP address without scheme
        url_parts = urlparse.urlparse(url, scheme='file')
        if url_parts.scheme == 'file' and not os.path.isfile(url_parts.path):
            raise DataSourceNotFound(
                '{} (add a URL scheme if {!r} is not meant to be a file)'
                .format(err, url_parts.path))
<commit_msg>Fix S3 chunk store construction<commit_after>################################################################################
# Copyright (c) 2017-2018, National Research Foundation (Square Kilometre Array)
#
# Licensed under the BSD 3-Clause License (the "License"); you may not use
# this file except in compliance with the License. You may obtain a copy
# of the License at
#
#   https://opensource.org/licenses/BSD-3-Clause
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
################################################################################

"""Various sources of correlator data and metadata."""

import urlparse
import os
import logging

import katsdptelstate
import redis
import numpy as np

from .sensordata import TelstateSensorData
from .chunkstore_s3 import S3ChunkStore
from .chunkstore_npy import NpyFileChunkStore
from .chunkstore import StoreUnavailable

logger = logging.getLogger(__name__)

class DataSourceNotFound(Exception):
    """File associated with DataSource not found or server not responding."""

class AttrsSensors(object):
    """Metadata in the form of attributes and sensors.

    Parameters
    ----------
    attrs : mapping from string to object
        Metadata attributes
    sensors : mapping from string to :class:`SensorData` objects
        Metadata sensor cache mapping sensor names to raw sensor data
    name : string, optional
        Identifier that describes the origin of the metadata (backend-specific)

    """
    def __init__(self, attrs, sensors, name='custom'):
        self.attrs = attrs
        self.sensors = sensors
        self.name = name

class VisFlagsWeights(object):
    """Correlator data in the form of visibilities, flags and weights.

    Parameters
    ----------
    vis : array-like of complex64, shape (*T*, *F*, *B*)
        Complex visibility data as a function of time, frequency and baseline
    flags : array-like of uint8, shape (*T*, *F*, *B*)
        Flags as a function of time, frequency and baseline
    weights : array-like of float32, shape (*T*, *F*, *B*)
        Visibility weights as a function of time, frequency and baseline
    name : string, optional
        Identifier that describes the origin of the data (backend-specific)

    """
    def __init__(self, vis, flags, weights, name='custom'):
        if not (vis.shape == flags.shape == weights.shape):
            raise ValueError("Shapes of vis %s, flags %s and weights %s differ"
                             % (vis.shape, flags.shape, weights.shape))
        self.vis = vis
        self.flags = flags
        self.weights = weights
        self.name = name

    @property
    def shape(self):
        return self.vis.shape

class ChunkStoreVisFlagsWeights(VisFlagsWeights):
    """Correlator data stored in a chunk store.

    Parameters
    ----------
    store : :class:`ChunkStore` object
        Chunk store
    base_name : string
        Name of dataset in store, as array name prefix (akin to a filename)
    chunk_info : dict mapping array name to info dict
        Dict specifying dtype, shape and chunks per array
    """
    def __init__(self, store, base_name, chunk_info):
        self.store = store
        da = {}
        for array, info in chunk_info.iteritems():
            array_name = store.join(base_name, array)
            da[array] = store.get_dask_array(array_name, info['chunks'],
                                             info['dtype'])
        vis = da['correlator_data']
        flags = da['flags']
        # Combine low-resolution weights and high-resolution weights_channel
        weights = da['weights'] * da['weights_channel'][..., np.newaxis]
        VisFlagsWeights.__init__(self, vis, flags, weights, base_name)

class DataSource(object):
    """A generic data source presenting both correlator data and metadata.

    Parameters
    ----------
    metadata : :class:`AttrsSensors` object
        Metadata attributes and sensors
    timestamps : array-like of float, length *T*
        Timestamps at centroids of visibilities in UTC seconds since Unix epoch
    data : :class:`VisFlagsWeights` object, optional
        Correlator data (visibilities, flags and weights)

    """
    def __init__(self, metadata, timestamps, data=None):
        self.metadata = metadata
        self.timestamps = timestamps
        self.data = data

    @property
    def name(self):
        name = self.metadata.name
        if self.data and self.data.name != name:
            name += ' | ' + self.data.name
        return name

def view_capture_stream(telstate, capture_block_id=None, stream_name=None):
    """Create telstate view based on capture block ID and stream name.

    This figures out the appropriate capture block ID and L0 stream name from
    a capture-stream specific telstate, or uses the provided ones. It then
    constructs a view on `telstate` with at least the prefixes

      - <capture_block_id>_<stream_name>
      - <capture_block_id>
      - <stream_name>

    Parameters
    ----------
    telstate : :class:`katsdptelstate.TelescopeState` object
        Original telescope state
    capture_block_id : string, optional
        Specify capture block ID explicitly (detected otherwise)
    stream_name : string, optional
        Specify L0 stream name explicitly (detected otherwise)

    Returns
    -------
    telstate : :class:`katsdptelstate.TelescopeState` object
        Telstate with a view that incorporates capture block, stream and combo

    Raises
    ------
    ValueError
        If no capture block or L0 stream could be detected (with no override)
    """
    # Detect the capture block
    if not capture_block_id:
        try:
            capture_block_id = str(telstate['capture_block_id'])
        except KeyError:
            raise ValueError('No capture block ID found in telstate - '
                             'please specify it manually')
    # Detect the captured stream
    if not stream_name:
        try:
            stream_name = str(telstate['stream_name'])
        except KeyError:
            raise ValueError('No captured stream found in telstate - '
                             'please specify the stream manually')
    # Check the stream type
    telstate = telstate.view(stream_name)
    stream_type = telstate.get('stream_type', 'unknown')
    expected_type = 'sdp.vis'
    if stream_type != expected_type:
        raise ValueError("Found stream {!r} but it has the wrong type {!r},"
                         " expected {!r}".format(stream_name, stream_type,
                                                 expected_type))
    logger.info('Using capture block %r and stream %r',
                capture_block_id, stream_name)
    telstate = telstate.view(capture_block_id)
    capture_stream = telstate.SEPARATOR.join((capture_block_id, stream_name))
    telstate = telstate.view(capture_stream)
    return telstate

def _shorten_key(telstate, key):
    """Shorten telstate key by subtracting the first prefix that fits.

    Parameters
    ----------
    telstate : :class:`katsdptelstate.TelescopeState` object
        Telescope state
    key : string
        Telescope state key

    Returns
    -------
    short_key : string
        Suffix of `key` after subtracting first matching prefix, or empty
        string if `key` does not start with any of the prefixes (or exactly
        matches a prefix, which is also considered pathological)

    """
    for prefix in telstate.prefixes:
        if key.startswith(prefix):
            return key[len(prefix):]
    return ''

class TelstateDataSource(DataSource):
    """A data source based on :class:`katsdptelstate.TelescopeState`.

    It is assumed that the provided `telstate` already has the appropriate
    views to find observation, stream and chunk store information. It typically
    needs the following prefixes:

      - <capture block ID>_<L0 stream>
      - <capture block ID>
      - <L0 stream>

    Parameters
    ----------
    telstate : :class:`katsdptelstate.TelescopeState` object
        Telescope state with appropriate views
    chunk_store : :class:`katdal.ChunkStore` object, optional
        Chunk store for visibility data (the default is no data - metadata only)
    source_name : string, optional
        Name of telstate source (used for metadata name)
    """
    def __init__(self, telstate, chunk_store=None, source_name='telstate'):
        self.telstate = telstate
        # Collect sensors
        sensors = {}
        for key in telstate.keys():
            if not telstate.is_immutable(key):
                sensor_name = _shorten_key(telstate, key)
                if sensor_name:
                    sensors[sensor_name] = TelstateSensorData(telstate, key)
        metadata = AttrsSensors(telstate, sensors, name=source_name)
        try:
            t0 = telstate['sync_time'] + telstate['first_timestamp']
            int_time = telstate['int_time']
            chunk_name = telstate['chunk_name']
            chunk_info = telstate['chunk_info']
        except KeyError:
            # Metadata without data or timestamps
            DataSource.__init__(self, metadata, None)
        else:
            # Extract timestamps from telstate
            n_dumps = chunk_info['correlator_data']['shape'][0]
            timestamps = t0 + np.arange(n_dumps) * int_time
            data = ChunkStoreVisFlagsWeights(
                chunk_store, chunk_name, chunk_info) if chunk_store else None
            # Metadata and timestamps with or without data
            DataSource.__init__(self, metadata, timestamps, data)

    @classmethod
    def from_url(cls, url, chunk_store='auto'):
        """Construct TelstateDataSource from URL (RDB file / REDIS server)."""
        url_parts = urlparse.urlparse(url, scheme='file')
        kwargs = dict(urlparse.parse_qsl(url_parts.query))
        # Extract Redis database number if provided
        db = int(kwargs.pop('db', '0'))
        source_name = url_parts.geturl()
        if url_parts.scheme == 'file':
            # RDB dump file
            telstate = katsdptelstate.TelescopeState()
            try:
                telstate.load_from_file(url_parts.path)
            except OSError as err:
                raise DataSourceNotFound(str(err))
            telstate = view_capture_stream(telstate, **kwargs)
            # Look for adjacent data directory (presumably containing NPY files)
            if chunk_store == 'auto':
                capture_block_path = os.path.dirname(url_parts.path)
                store_path = os.path.join(capture_block_path, '..')
                try:
                    data_path = os.path.join(store_path, telstate['chunk_name'])
                except KeyError:
                    chunk_store = None
            if chunk_store == 'auto' and os.path.isdir(data_path):
                chunk_store = NpyFileChunkStore(os.path.normpath(store_path))
            else:
                chunk_store = S3ChunkStore.from_url(telstate['s3_endpoint_url'])
            return cls(telstate, chunk_store, source_name)
        elif url_parts.scheme == 'redis':
            # Redis server
            try:
                telstate = katsdptelstate.TelescopeState(url_parts.netloc, db)
            except (redis.ConnectionError, redis.TimeoutError) as e:
                raise DataSourceNotFound(str(e))
            telstate = view_capture_stream(telstate, **kwargs)
            if chunk_store == 'auto':
                chunk_store = S3ChunkStore.from_url(telstate['s3_endpoint_url'])
            return cls(telstate, chunk_store, source_name)

def open_data_source(url, *args, **kwargs):
    """Construct the data source described by the given URL."""
    try:
        return TelstateDataSource.from_url(url, *args, **kwargs)
    except DataSourceNotFound as err:
        # Amend the error message for the case of an IP address without scheme
        url_parts = urlparse.urlparse(url, scheme='file')
        if url_parts.scheme == 'file' and not os.path.isfile(url_parts.path):
            raise DataSourceNotFound(
                '{} (add a URL scheme if {!r} is not meant to be a file)'
                .format(err, url_parts.path))
<|endoftext|>


<commit_before>import contextlib
import boto
import os
import json
import shutil
import tempfile
import time
from contextlib import closing
from django.contrib.auth.models import User
import csvkit as csv

from celery import Task
from django.core.exceptions import ImproperlyConfigured
from django.db import InternalError
from django.utils.six import text_type
from dateutil.relativedelta import relativedelta
from django.conf import settings

from .utils import OBIEEExporter
from .models import Export
from .constants import EXPORT_STATUS

@contextlib.contextmanager
def csv_writer(csv_file):
    yield csv.writer(csv_file)

def import_form(class_name):
    mod = __import__('reports.forms', fromlist=[class_name])
    return getattr(mod, class_name)

class ExportTaskBase(Task):
    def __init__(self):
        self.filepath = ''
        self.message = ''
        self.export = None
        self.form = None
        self.user = None

    def _create_export(self):
        self.export = Export.objects.create(
            user=self.user,
            task_id=self.request.id,
            status=EXPORT_STATUS.started,
        )

    def _set_up_form(self, form_class_name, post_data):
        form_class = import_form(form_class_name)
        self.form = form_class()
        self.form.data = json.loads(post_data)
        self.form.is_bound = True
        if not self.form.is_valid():
            self.message = u'The form submitted was not valid'
            raise Exception(self.message)

    def _filepath(self, filename):
        file_name, file_ext = os.path.splitext(filename)
        user_datetime = '%s-%s' % (self.user.pk, time.strftime("%Y-%m-%d-%H%M%S"))
        filename = '%s-%s%s' % (file_name, user_datetime, file_ext)
        return os.path.join(settings.TEMP_DIR, filename)

    def on_success(self, retval, task_id, args, kwargs):
        self.export.status = EXPORT_STATUS.created
        self.export.path = self.filepath
        self.export.message = self.message
        self.export.save()

    def on_failure(self, exc, task_id, args, kwargs, einfo):
        self.export.status = EXPORT_STATUS.failed
        self.export.message = self.message
        self.export.save()

    def send_to_s3(self):
        conn = boto.connect_s3(
            settings.AWS_ACCESS_KEY_ID,
            settings.AWS_SECRET_ACCESS_KEY)
        bucket = conn.lookup(settings.AWS_STORAGE_BUCKET_NAME)
        k = bucket.new_key(settings.EXPORT_DIR + os.path.basename(self.filepath))
        export = open(self.filepath)
        k.set_contents_from_file(export)
        k.set_acl('private')
        try:
            os.remove(self.filepath)
        except Exception:
            pass

class ExportTask(ExportTaskBase):

    def run(self, user_id, filename, form_class_name, post_data, *args, **kwargs):
        self.user = User.objects.get(pk=user_id)
        self._create_export()
        self._set_up_form(form_class_name, post_data)

        self.filepath = self._filepath(filename)
        try:
            csv_data = list(self.form)
            with csv_writer(open(self.filepath, 'w')) as writer:
                map(writer.writerow, csv_data)
            self.send_to_s3()
        except InternalError as error:
            error_message = text_type(error).strip()
            if 'wrong key' in error_message.lower() or 'corrupt data' in \
                    error_message.lower():
                # e.g. if pgcrypto key is incorrect
                self.message = u'Check passphrase and try again'
            else:
                self.message = u'An error occurred:\n%s' % error_message

class OBIEEExportTask(ExportTaskBase):
    def run(self, user_id, filename, form_class_name, post_data, *args, **kwargs):
        """
        Export a full dump of the db for OBIEE export and make it available
        for downloads
        """
        self.user = User.objects.get(pk=user_id)
        self._create_export()
        self._set_up_form(form_class_name, post_data)

        diversity_keyphrase = self.form.cleaned_data['passphrase']
        start = self.form.month
        end = self.form.month + relativedelta(months=1)
        if not settings.OBIEE_ZIP_PASSWORD:
            raise ImproperlyConfigured('OBIEE Zip password must be set.')

        self.filepath = self._filepath(filename)
        with closing(OBIEEExporter(settings.TEMP_DIR, diversity_keyphrase,
                start, end, filename=os.path.basename(self._filepath(filename)))) as exporter:
            try:
                self.filepath = exporter.export()
                self.send_to_s3()
            except Exception:
                self.message = u'An error occurred creating the zip file'
                raise
            finally:
                pass
                # shutil.rmtree(self.filepath, ignore_errors=True)

<commit_msg>close csv export file object before sending to s3<commit_after>import contextlib
import boto
import os
import json
import shutil
import tempfile
import time
from contextlib import closing
from django.contrib.auth.models import User
import csvkit as csv

from celery import Task
from django.core.exceptions import ImproperlyConfigured
from django.db import InternalError
from django.utils.six import text_type
from dateutil.relativedelta import relativedelta
from django.conf import settings

from .utils import OBIEEExporter
from .models import Export
from .constants import EXPORT_STATUS

@contextlib.contextmanager
def csv_writer(csv_file):
    yield csv.writer(csv_file)

def import_form(class_name):
    mod = __import__('reports.forms', fromlist=[class_name])
    return getattr(mod, class_name)

class ExportTaskBase(Task):
    def __init__(self):
        self.filepath = ''
        self.message = ''
        self.export = None
        self.form = None
        self.user = None

    def _create_export(self):
        self.export = Export.objects.create(
            user=self.user,
            task_id=self.request.id,
            status=EXPORT_STATUS.started,
        )

    def _set_up_form(self, form_class_name, post_data):
        form_class = import_form(form_class_name)
        self.form = form_class()
        self.form.data = json.loads(post_data)
        self.form.is_bound = True
        if not self.form.is_valid():
            self.message = u'The form submitted was not valid'
            raise Exception(self.message)

    def _filepath(self, filename):
        file_name, file_ext = os.path.splitext(filename)
        user_datetime = '%s-%s' % (self.user.pk, time.strftime("%Y-%m-%d-%H%M%S"))
        filename = '%s-%s%s' % (file_name, user_datetime, file_ext)
        return os.path.join(settings.TEMP_DIR, filename)

    def on_success(self, retval, task_id, args, kwargs):
        self.export.status = EXPORT_STATUS.created
        self.export.path = self.filepath
        self.export.message = self.message
        self.export.save()

    def on_failure(self, exc, task_id, args, kwargs, einfo):
        self.export.status = EXPORT_STATUS.failed
        self.export.message = self.message
        self.export.save()

    def send_to_s3(self):
        conn = boto.connect_s3(
            settings.AWS_ACCESS_KEY_ID,
            settings.AWS_SECRET_ACCESS_KEY)
        bucket = conn.lookup(settings.AWS_STORAGE_BUCKET_NAME)
        k = bucket.new_key(settings.EXPORT_DIR + os.path.basename(self.filepath))
        export = open(self.filepath)
        k.set_contents_from_file(export)
        k.set_acl('private')
        try:
            os.remove(self.filepath)
        except Exception:
            pass

class ExportTask(ExportTaskBase):

    def run(self, user_id, filename, form_class_name, post_data, *args, **kwargs):
        self.user = User.objects.get(pk=user_id)
        self._create_export()
        self._set_up_form(form_class_name, post_data)

        self.filepath = self._filepath(filename)
        try:
            csv_data = list(self.form)
            csv_file = open(self.filepath, 'w')
            with csv_writer(csv_file) as writer:
                map(writer.writerow, csv_data)
            csv_file.close()
            self.send_to_s3()
        except InternalError as error:
            error_message = text_type(error).strip()
            if 'wrong key' in error_message.lower() or 'corrupt data' in \
                    error_message.lower():
                # e.g. if pgcrypto key is incorrect
                self.message = u'Check passphrase and try again'
            else:
                self.message = u'An error occurred:\n%s' % error_message

class OBIEEExportTask(ExportTaskBase):
    def run(self, user_id, filename, form_class_name, post_data, *args, **kwargs):
        """
        Export a full dump of the db for OBIEE export and make it available
        for downloads
        """
        self.user = User.objects.get(pk=user_id)
        self._create_export()
        self._set_up_form(form_class_name, post_data)

        diversity_keyphrase = self.form.cleaned_data['passphrase']
        start = self.form.month
        end = self.form.month + relativedelta(months=1)
        if not settings.OBIEE_ZIP_PASSWORD:
            raise ImproperlyConfigured('OBIEE Zip password must be set.')

        self.filepath = self._filepath(filename)
        with closing(OBIEEExporter(settings.TEMP_DIR, diversity_keyphrase,
                start, end, filename=os.path.basename(self._filepath(filename)))) as exporter:
            try:
                self.filepath = exporter.export()
                self.send_to_s3()
            except Exception:
                self.message = u'An error occurred creating the zip file'
                raise
            finally:
                pass
                # shutil.rmtree(self.filepath, ignore_errors=True)

<|endoftext|>


<commit_before>from qgis.core import QgsRasterLayer, QgsProject

import ee
ee.Initialize()

def get_image_url(image):
    map_id = ee.data.getMapId({'image': image})
    url = map_id['tile_fetcher'].url_format
    return url

def add_ee_xyz_tile_layer(url, name):
    layer = QgsRasterLayer("type=xyz&url=" + url, name, "wms")
    layer.setCustomProperty('ee-layer', True)
    QgsProject.instance().addMapLayer(layer)

def add_ee_image_layer(image, name):
    url = get_image_url(image)
    add_ee_xyz_tile_layer(url, name)

def update_ee_image_layer(image, layer):
    url = get_image_url(image)
    layer.setDataUrl("type=xyz&url=" + url)
    print('Updated url: ' + url)

def get_layer_by_name(name):
    layers = QgsProject.instance().mapLayers().values()

    for l in layers:
        if(l.name() == name):
            return l

    return None

def add_or_update_ee_image_layer(image, name):
    layer = get_layer_by_name(name)

    if layer:
        if not layer.customProperty('ee-layer'):
            raise Exception('Layer is not an EE layer: ' + name)

        update_ee_image_layer(image, layer)
    else:
        add_ee_image_layer(image, name)

image = ee.Image('USGS/SRTMGL1_003').unitScale(0, 5000)
# add_or_update_ee_image_layer(image, 'ee-layer1')
add_ee_image_layer(image, 'dem')
<commit_msg>added working example to add or update EE XYZ tile layer<commit_after>from qgis.core import QgsRasterLayer, QgsProject

import ee
ee.Initialize()

def get_image_url(image):
    map_id = ee.data.getMapId({'image': image})
    url = map_id['tile_fetcher'].url_format
    return url

def add_ee_xyz_tile_layer(url, name):
    layer = QgsRasterLayer("type=xyz&url=" + url, name, "wms")
    layer.setCustomProperty('ee-layer', True)
    QgsProject.instance().addMapLayer(layer)

def add_ee_image_layer(image, name):
    url = get_image_url(image)
    add_ee_xyz_tile_layer(url, name)

def update_ee_image_layer(image, layer):
    url = "type=xyz&url=" + get_image_url(image)

    layer.dataProvider().setDataSourceUri(url)
    layer.dataProvider().reloadData()
    layer.triggerRepaint()
    layer.reload()

    iface.mapCanvas().refresh()

def get_layer_by_name(name):
    layers = QgsProject.instance().mapLayers().values()

    for l in layers:
        if(l.name() == name):
            return l

    return None

def add_or_update_ee_image_layer(image, name):
    layer = get_layer_by_name(name)

    if layer:
        if not layer.customProperty('ee-layer'):
            raise Exception('Layer is not an EE layer: ' + name)

        update_ee_image_layer(image, layer)
    else:
        add_ee_image_layer(image, name)

image = ee.Image('USGS/SRTMGL1_003').unitScale(0, 5000) \
    .visualize(**{ 'palette': ['blue', 'red']})

add_or_update_ee_image_layer(image, 'dem')

<|endoftext|>


<commit_before>#!/usr/bin/python
# -*- coding: utf-8 -*-

"""
=========================================================
Ordinary Least Squares and Ridge Regression Variance
=========================================================
Due to the few points in each dimension and the straight
line that linear regression uses to follow these points
as well as it can, noise on the observations will cause
great variance as shown in the first plot. Every line's slope
can vary quite a bit for each prediction due to the noise
induced in the observations.

Ridge regression is basically minimizing a penalised version
of the least-squared function. The penalising `shrinks` the
value of the regression coefficients.
Despite the few data points in each dimension, the slope
of the prediction is much more stable and the variance
in the line itself is greatly reduced, in comparison to that
of the standard linear regression
"""
print(__doc__)

# Code source: Gaël Varoquaux
# Modified for documentation by Jaques Grobler
# License: BSD 3 clause

import numpy as np
import pylab as pl

from sklearn import linear_model

X_train = np.c_[.5, 1].T
y_train = [.5, 1]
X_test = np.c_[0, 2].T

np.random.seed(0)

classifiers = dict(ols=linear_model.LinearRegression(),
                   ridge=linear_model.Ridge(alpha=.1))

fignum = 1
for name, clf in classifiers.iteritems():
    fig = pl.figure(fignum, figsize=(4, 3))
    pl.clf()
    ax = pl.axes([.12, .12, .8, .8])

    for _ in range(6):
        this_X = .1 * np.random.normal(size=(2, 1)) + X_train
        clf.fit(this_X, y_train)

        ax.plot(X_test, clf.predict(X_test), color='.5')
        ax.scatter(this_X, y_train, s=3, c='.5', marker='o', zorder=10)

    clf.fit(X_train, y_train)
    ax.plot(X_test, clf.predict(X_test), linewidth=2, color='blue')
    ax.scatter(X_train, y_train, s=30, c='r', marker='+', zorder=10)

    ax.set_xticks(())
    ax.set_yticks(())
    ax.set_ylim((0, 1.6))
    ax.set_xlabel('X')
    ax.set_ylabel('y')
    ax.set_xlim(0, 2)
    fignum += 1

pl.show()
<commit_msg>Add plot titles and newstyle plt import in OLS vs Ridge example<commit_after>#!/usr/bin/python
# -*- coding: utf-8 -*-

"""
=========================================================
Ordinary Least Squares and Ridge Regression Variance
=========================================================
Due to the few points in each dimension and the straight
line that linear regression uses to follow these points
as well as it can, noise on the observations will cause
great variance as shown in the first plot. Every line's slope
can vary quite a bit for each prediction due to the noise
induced in the observations.

Ridge regression is basically minimizing a penalised version
of the least-squared function. The penalising `shrinks` the
value of the regression coefficients.
Despite the few data points in each dimension, the slope
of the prediction is much more stable and the variance
in the line itself is greatly reduced, in comparison to that
of the standard linear regression
"""
print(__doc__)

# Code source: Gaël Varoquaux
# Modified for documentation by Jaques Grobler
# License: BSD 3 clause

import numpy as np
import matplotlib.pyplot as plt

from sklearn import linear_model

X_train = np.c_[.5, 1].T
y_train = [.5, 1]
X_test = np.c_[0, 2].T

np.random.seed(0)

classifiers = dict(ols=linear_model.LinearRegression(),
                   ridge=linear_model.Ridge(alpha=.1))

fignum = 1
for name, clf in classifiers.iteritems():
    fig = plt.figure(fignum, figsize=(4, 3))
    plt.clf()
    plt.title(name)
    ax = plt.axes([.12, .12, .8, .8])

    for _ in range(6):
        this_X = .1 * np.random.normal(size=(2, 1)) + X_train
        clf.fit(this_X, y_train)

        ax.plot(X_test, clf.predict(X_test), color='.5')
        ax.scatter(this_X, y_train, s=3, c='.5', marker='o', zorder=10)

    clf.fit(X_train, y_train)
    ax.plot(X_test, clf.predict(X_test), linewidth=2, color='blue')
    ax.scatter(X_train, y_train, s=30, c='r', marker='+', zorder=10)

    ax.set_xticks(())
    ax.set_yticks(())
    ax.set_ylim((0, 1.6))
    ax.set_xlabel('X')
    ax.set_ylabel('y')
    ax.set_xlim(0, 2)
    fignum += 1

plt.show()
<|endoftext|>


<commit_before># This file is part of ELiDE, frontend to LiSE, a framework for life simulation games.
# Copyright (c) Zachary Spector, public@zacharyspector.com
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program.  If not, see <https://www.gnu.org/licenses/>.
"""Widget to display the contents of a :class:`kivy.atlas.Atlas` in
one :class:`kivy.uix.togglebutton.ToggleButton` apiece, arranged in a
:class:`kivy.uix.stacklayout.StackLayout`. The user selects graphics
from the :class:`Pallet`, and the :class:`Pallet` updates its
``selection`` list to show what the user selected."""
from kivy.clock import Clock
from kivy.properties import (
    DictProperty,
    NumericProperty,
    ObjectProperty,
    OptionProperty,
    ListProperty,
    ReferenceListProperty,
    StringProperty
)
from kivy.resources import resource_find
from kivy.atlas import Atlas
from kivy.lang import Builder
from kivy.logger import Logger
from kivy.uix.boxlayout import BoxLayout
from kivy.uix.togglebutton import ToggleButton
from kivy.uix.stacklayout import StackLayout

class SwatchButton(ToggleButton):
    """Toggle button containing a texture and its name, which, when
    toggled, will report the fact to the :class:`Pallet` it's in.

    """
    name = StringProperty()
    """A name, either from a filename or a key into an atlas"""
    tex = ObjectProperty()
    """Texture to display here"""

    def on_state(self, *args):
        if self.state == 'down':
            assert(self not in self.parent.selection)
            if self.parent.selection_mode == 'single':
                for wid in self.parent.selection:
                    if wid is not self:
                        wid.state = 'normal'
                self.parent.selection = [self]
            else:
                self.parent.selection.append(self)
        else:
            if self in self.parent.selection:
                self.parent.selection.remove(self)

kv = """
<SwatchButton>:
    Image:
        id: theimg
        center: root.center
        texture: root.tex
        size: self.texture_size if root.tex else (1, 1)
        size_hint: (None, None)
        pos_hint: {'x': None, 'y': None}
    Label:
        text: root.name
        size: self.texture_size
        pos_hint: {'x': None, 'y': None}
        x: root.x + 5
        y: theimg.y - self.height
"""
Builder.load_string(kv)

class Pallet(StackLayout):
    """Many :class:`SwatchButton`, gathered from an :class:`kivy.atlas.Atlas`."""
    atlas = ObjectProperty()
    """:class:`kivy.atlas.Atlas` object I'll make :class:`SwatchButton` from."""
    filename = StringProperty()
    """Path to an atlas; will construct :class:`kivy.atlas.Atlas` when set"""
    swatches = DictProperty({})
    """:class:`SwatchButton` widgets here, keyed by name of their graphic"""
    swatch_width = NumericProperty(100)
    """Width of each and every :class:`SwatchButton` here"""
    swatch_height = NumericProperty(75)
    """Height of each and every :class:`SwatchButton` here"""
    swatch_size = ReferenceListProperty(swatch_width, swatch_height)
    """Size of each and every :class:`SwatchButton` here"""
    selection = ListProperty([])
    """List of :class:`SwatchButton`s that are selected"""
    selection_mode = OptionProperty('single', options=['single', 'multiple'])
    """Whether to allow only a 'single' selected :class:`SwatchButton` (default), or 'multiple'"""

    def on_selection(self, *args):
        Logger.debug(
            'Pallet: {} got selection {}'.format(
                self.filename, self.selection
            )
        )

    def on_filename(self, *args):
        if not self.filename:
            return
        resource = resource_find(self.filename)
        if not resource:
            raise ValueError("Couldn't find atlas: {}".format(self.filename))
        self.atlas = Atlas(resource)

    def on_atlas(self, *args):
        if self.atlas is None:
            return
        self.upd_textures()
        self.atlas.bind(textures=self.upd_textures)

    def upd_textures(self, *args):
        """Create one :class:`SwatchButton` for each texture"""
        if self.canvas is None:
            Clock.schedule_once(self.upd_textures, 0)
            return
        for name in list(self.swatches.keys()):
            if name not in self.atlas.textures:
                self.remove_widget(self.swatches[name])
                del self.swatches[name]
        for (name, tex) in self.atlas.textures.items():
            if name in self.swatches and self.swatches[name] != tex:
                self.remove_widget(self.swatches[name])
            if name not in self.swatches or self.swatches[name] != tex:
                self.swatches[name] = SwatchButton(
                    name=name,
                    tex=tex,
                    size_hint=(None, None),
                    size=self.swatch_size
                )
                self.add_widget(self.swatches[name])

kv = """
<Pallet>:
    orientation: 'lr-tb'
    padding_y: 100
    size_hint: (None, None)
    height: self.minimum_height
"""
Builder.load_string(kv)

class PalletBox(BoxLayout):
    pallets = ListProperty()
<commit_msg>Optimize Pallet.upd_textures<commit_after># This file is part of ELiDE, frontend to LiSE, a framework for life simulation games.
# Copyright (c) Zachary Spector, public@zacharyspector.com
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program.  If not, see <https://www.gnu.org/licenses/>.
"""Widget to display the contents of a :class:`kivy.atlas.Atlas` in
one :class:`kivy.uix.togglebutton.ToggleButton` apiece, arranged in a
:class:`kivy.uix.stacklayout.StackLayout`. The user selects graphics
from the :class:`Pallet`, and the :class:`Pallet` updates its
``selection`` list to show what the user selected."""
from kivy.clock import Clock
from kivy.properties import (
    DictProperty,
    NumericProperty,
    ObjectProperty,
    OptionProperty,
    ListProperty,
    ReferenceListProperty,
    StringProperty
)
from kivy.resources import resource_find
from kivy.atlas import Atlas
from kivy.lang import Builder
from kivy.logger import Logger
from kivy.uix.boxlayout import BoxLayout
from kivy.uix.togglebutton import ToggleButton
from kivy.uix.stacklayout import StackLayout

class SwatchButton(ToggleButton):
    """Toggle button containing a texture and its name, which, when
    toggled, will report the fact to the :class:`Pallet` it's in.

    """
    name = StringProperty()
    """A name, either from a filename or a key into an atlas"""
    tex = ObjectProperty()
    """Texture to display here"""

    def on_state(self, *args):
        if self.state == 'down':
            assert(self not in self.parent.selection)
            if self.parent.selection_mode == 'single':
                for wid in self.parent.selection:
                    if wid is not self:
                        wid.state = 'normal'
                self.parent.selection = [self]
            else:
                self.parent.selection.append(self)
        else:
            if self in self.parent.selection:
                self.parent.selection.remove(self)

kv = """
<SwatchButton>:
    Image:
        id: theimg
        center: root.center
        texture: root.tex
        size: self.texture_size if root.tex else (1, 1)
        size_hint: (None, None)
        pos_hint: {'x': None, 'y': None}
    Label:
        text: root.name
        size: self.texture_size
        pos_hint: {'x': None, 'y': None}
        x: root.x + 5
        y: theimg.y - self.height
"""
Builder.load_string(kv)

class Pallet(StackLayout):
    """Many :class:`SwatchButton`, gathered from an :class:`kivy.atlas.Atlas`."""
    atlas = ObjectProperty()
    """:class:`kivy.atlas.Atlas` object I'll make :class:`SwatchButton` from."""
    filename = StringProperty()
    """Path to an atlas; will construct :class:`kivy.atlas.Atlas` when set"""
    swatches = DictProperty({})
    """:class:`SwatchButton` widgets here, keyed by name of their graphic"""
    swatch_width = NumericProperty(100)
    """Width of each and every :class:`SwatchButton` here"""
    swatch_height = NumericProperty(75)
    """Height of each and every :class:`SwatchButton` here"""
    swatch_size = ReferenceListProperty(swatch_width, swatch_height)
    """Size of each and every :class:`SwatchButton` here"""
    selection = ListProperty([])
    """List of :class:`SwatchButton`s that are selected"""
    selection_mode = OptionProperty('single', options=['single', 'multiple'])
    """Whether to allow only a 'single' selected :class:`SwatchButton` (default), or 'multiple'"""

    def on_selection(self, *args):
        Logger.debug(
            'Pallet: {} got selection {}'.format(
                self.filename, self.selection
            )
        )

    def on_filename(self, *args):
        if not self.filename:
            return
        resource = resource_find(self.filename)
        if not resource:
            raise ValueError("Couldn't find atlas: {}".format(self.filename))
        self.atlas = Atlas(resource)

    def on_atlas(self, *args):
        if self.atlas is None:
            return
        self.upd_textures()
        self.atlas.bind(textures=self.upd_textures)

    def upd_textures(self, *args):
        """Create one :class:`SwatchButton` for each texture"""
        if self.canvas is None:
            Clock.schedule_once(self.upd_textures, 0)
            return
        swatches = self.swatches
        atlas_textures = self.atlas.textures
        remove_widget = self.remove_widget
        add_widget = self.add_widget
        swatch_size = self.swatch_size
        for name, swatch in list(swatches.items()):
            if name not in atlas_textures:
                remove_widget(swatch)
                del swatches[name]
        for (name, tex) in atlas_textures.items():
            if name in swatches and swatches[name] != tex:
                remove_widget(swatches[name])
            if name not in swatches or swatches[name] != tex:
                swatches[name] = SwatchButton(
                    name=name,
                    tex=tex,
                    size_hint=(None, None),
                    size=swatch_size
                )
                add_widget(swatches[name])

kv = """
<Pallet>:
    orientation: 'lr-tb'
    padding_y: 100
    size_hint: (None, None)
    height: self.minimum_height
"""
Builder.load_string(kv)

class PalletBox(BoxLayout):
    pallets = ListProperty()
<|endoftext|>


<commit_before>import re
from .parse_string import parse_string
from .parse_eval import parse_eval
from .ParseError import *
from .ParseFile import ParseFile

def parse_url(current, to_parse):
    if to_parse.match('\s*url\('):
        s = parse_string(to_parse)

        if s:
            if to_parse.match('\s*\)\s*;'):
                current['value'] = s
                current['value_type'] = 'url'
                return

        else:
            if to_parse.match('([^\)]*)\);'):
                current['value'] = to_parse.match_group(1)
                current['value_type'] = 'url'
                return
    else:
        if to_parse.match('\s*([^;]*);'):
            current['value'] = to_parse.match_group(1)
            current['value_type'] = 'url'
            return to_parse

def parse_value(current, to_parse):
    # strip prepending whitespace
    while True:
        if not to_parse.match('\s+'):
            break;

    # "..." or '...'
    s = parse_string(to_parse)
    if s is not None:
        if to_parse.match('\s*;'):
            current['value'] = s
            current['value_type'] = 'string'
            return

    # eval( ... )
    elif to_parse.match('eval\s*\(\s*'):
        fp = to_parse.pos()
        r = parse_string(to_parse)
        value = None
        if r:
            r = ParseFile(content=r)
            try:
                value = parse_eval(r)
            except ParseError as e:
                to_parse.seek(fp)
                pass

        if value == None:
            value = parse_eval(to_parse)

        if not to_parse.match('\s*\)\s*;'):
            raise ParseError(to_parse, 'Error parsing eval statement')

        current['value'] = value
        current['value_type'] = 'eval'
        return

    # url( ... )
    elif to_parse.match('url\s*\(', wind=None):
        return parse_url(current, to_parse)

    # it's a "value" (letters, digits, -, _, #)
    elif to_parse.match('\s*([A-Za-z0-9\-_#]*)\s*;'):
        current['value_type'] = 'value';
        current['value'] = to_parse.match_group(1).strip()

        if current['value'] == '':
            current['value'] = None

    # function call (handle like eval( ... ) but without eval() ;) )
    else:
        value = parse_eval(to_parse)

        if not to_parse.match('\s*;'):
            raise ParseError(to_parse, 'Error parsing eval statement')

        current['value'] = value
        current['value_type'] = 'eval'
        return

    return
<commit_msg>Parser::parse_value(): accept ',' and '.' as chars for normal values<commit_after>import re
from .parse_string import parse_string
from .parse_eval import parse_eval
from .ParseError import *
from .ParseFile import ParseFile

def parse_url(current, to_parse):
    if to_parse.match('\s*url\('):
        s = parse_string(to_parse)

        if s:
            if to_parse.match('\s*\)\s*;'):
                current['value'] = s
                current['value_type'] = 'url'
                return

        else:
            if to_parse.match('([^\)]*)\);'):
                current['value'] = to_parse.match_group(1)
                current['value_type'] = 'url'
                return
    else:
        if to_parse.match('\s*([^;]*);'):
            current['value'] = to_parse.match_group(1)
            current['value_type'] = 'url'
            return to_parse

def parse_value(current, to_parse):
    # strip prepending whitespace
    while True:
        if not to_parse.match('\s+'):
            break;

    # "..." or '...'
    s = parse_string(to_parse)
    if s is not None:
        if to_parse.match('\s*;'):
            current['value'] = s
            current['value_type'] = 'string'
            return

    # eval( ... )
    elif to_parse.match('eval\s*\(\s*'):
        fp = to_parse.pos()
        r = parse_string(to_parse)
        value = None
        if r:
            r = ParseFile(content=r)
            try:
                value = parse_eval(r)
            except ParseError as e:
                to_parse.seek(fp)
                pass

        if value == None:
            value = parse_eval(to_parse)

        if not to_parse.match('\s*\)\s*;'):
            raise ParseError(to_parse, 'Error parsing eval statement')

        current['value'] = value
        current['value_type'] = 'eval'
        return

    # url( ... )
    elif to_parse.match('url\s*\(', wind=None):
        return parse_url(current, to_parse)

    # it's a "value" (letters, digits, -, _, #)
    elif to_parse.match('\s*([A-Za-z0-9\-_#,\.]*)\s*;'):
        current['value_type'] = 'value';
        current['value'] = to_parse.match_group(1).strip()

        if current['value'] == '':
            current['value'] = None

    # function call (handle like eval( ... ) but without eval() ;) )
    else:
        value = parse_eval(to_parse)

        if not to_parse.match('\s*;'):
            raise ParseError(to_parse, 'Error parsing eval statement')

        current['value'] = value
        current['value_type'] = 'eval'
        return

    return
<|endoftext|>


<commit_before>#!/usr/bin/python3
"""

Various functions related to date, time,  and timestamps

 Kim Brugger (07 Jan 2019), contact: kim@brugger.dk
"""

import sys
import pprint
pp = pprint.PrettyPrinter(indent=4)

import time
import datetime

def now():
    """ retuns current time as seconds inc microseconds """

    return time.time()

    return  datetime.datetime.now()

def to_string(ts):
    """ Convert a time to string, if an int will transform it to a timestamp before returning the string"""

    if isinstance(ts, str):
        return ts
    elif isinstance(ts, datetime.datetime):
        return str(ts.timestamp())
    else:
        return str(datetime.datetime.fromtimestamp( ts ))

    raise ValueError("Cannot convert {} to a string".format( ts))

def to_sec( ts ):

    if isinstance(ts, int) or isinstance(ts, float):
        return int(ts)
    elif isinstance( ts, datetime.datetime ):
        return  int( ts.timestamp() )
    elif isinstance( ts, str):
        return datetime.time.fromisoformat( ts )
#        return str(datetime.datetime.fromisoformat( ts ))
#        return int( ts )
    else:
        raise ValueError("Cannot convert {} to an int".format( ts))

<commit_msg>renamed and added some new functions<commit_after>#!/usr/bin/python3
"""

Various functions related to date, time,  and timestamps

 Kim Brugger (07 Jan 2019), contact: kim@brugger.dk
"""

import sys
import pprint
pp = pprint.PrettyPrinter(indent=4)

import time
import datetime

def now() -> float:
    """ retuns current time as seconds inc microseconds """

    return time.time()

def to_string(ts):
    """ Convert a time to string, if an int will transform it to a timestamp before returning the string"""

    if isinstance(ts, str):
        return ts
    elif isinstance(ts, datetime.datetime):
        return str(ts.timestamp())
    else:
        return str(datetime.datetime.fromtimestamp( ts ))

    raise ValueError("Cannot convert {} to a string".format( ts))

def to_sec_since_epoch( ts ):

    if isinstance(ts, int) or isinstance(ts, float):
        return int(ts)
    elif isinstance( ts, datetime.datetime ):
        return  int( ts.timestamp() )
    elif isinstance( ts, str):
        return datetime.time.fromisoformat( ts )
    else:
        raise ValueError("Cannot convert {} to an int".format( ts))

def to_days_since_epoch(ts:int) -> int:
    timestamp = datetime.datetime.fromtimestamp(ts)
    stime = "{}-{}-{} 00:00:00".format( timestamp.year, timestamp.month, timestamp.day )

    return datetime.datetime.strptime(stime, "%Y-%m-%d %H:%M:%S").timestamp()

def to_months_since_epoch(ts:int) -> int:
    timestamp = datetime.datetime.fromtimestamp(ts)
    stime = "{}-{}-{} 00:00:00".format( timestamp.year, timestamp.month, timestamp.day )

    return datetime.datetime.strptime(stime, "%Y-%m-%d %H:%M:%S").timestamp()

def datestr_to_ts(datetime_str:str) -> any:

    for time_string in ["%Y-%m-%dT%H:%M:%S.%f%z",
                        "%Y-%m-%dT%H:%M:%S.%f",
                        "%Y-%m-%dT%H:%M:%S%z",
                        "%Y-%m-%dT%H:%M:%S",
                        "%Y-%m-%d %H:%M:%S.%f%z",
                        "%Y-%m-%d %H:%M:%S.%f",
                        "%Y-%m-%d %H:%M:%S%z",
                        "%Y-%m-%d %H:%M:%S"]:
        try:
            ts = datetime.datetime.strptime(datetime_str, time_string)
            return ts
        except:
            pass

    raise RuntimeError(f"cannot convert datetime string '{datetime}' to timestamp ")

def epoch_to_timestr(time:int):
    ts = datetime.datetime.fromtimestamp(time)
    return ts

def weekday(timestamp:str) -> []:

    try:
        ts = datetime.datetime.strptime(timestamp, "%Y-%m-%d %H:%M:%S.%f%z")
    except:
        try:
            ts = datetime.datetime.strptime(timestamp, "%Y-%m-%d %H:%M:%S%z")
        except:
            print(f"Cannot convert timestamp {timestamp} to weekday")

    return ts.weekday(), ts.hour, ts.minute

#def time_delta(ts:int)
#    tdelta = datetime.strptime(s2, FMT) - datetime.strptime(s1, FMT)<|endoftext|>


<commit_before>'''
Minimal clang-rename integration with Vim.

Before installing make sure one of the following is satisfied:

* clang-rename is in your PATH
* `g:clang_rename_path` in ~/.vimrc points to valid clang-rename executable
* `binary` in clang-rename.py points to valid to clang-rename executable

To install, simply put this into your ~/.vimrc

    noremap <leader>cr :pyf <path-to>/clang-rename.py<cr>

IMPORTANT NOTE: Before running the tool, make sure you saved the file.

All you have to do now is to place a cursor on a variable/function/class which
you would like to rename and press '<leader>cr'. You will be prompted for a new
name if the cursor points to a valid symbol.
'''

import vim
import subprocess
import sys

def main():
    binary = 'clang-rename'
    if vim.eval('exists("g:clang_rename_path")') == "1":
        binary = vim.eval('g:clang_rename_path')

    # Get arguments for clang-rename binary.
    offset = int(vim.eval('line2byte(line("."))+col(".")')) - 2
    if offset < 0:
        print >> sys.stderr, '''Couldn\'t determine cursor position.
                                Is your file empty?'''
        return
    filename = vim.current.buffer.name

    new_name_request_message = 'type new name:'
    new_name = vim.eval("input('{}\n')".format(new_name_request_message))

    # Call clang-rename.
    command = [binary,
               filename,
               '-i',
               '-offset', str(offset),
               '-new-name', str(new_name)]
    # FIXME: make it possible to run the tool on unsaved file.
    p = subprocess.Popen(command,
                         stdout=subprocess.PIPE,
                         stderr=subprocess.PIPE)
    stdout, stderr = p.communicate()

    if stderr:
        print stderr

    # Reload all buffers in Vim.
    vim.command("bufdo edit")

if __name__ == '__main__':
    main()
<commit_msg>[clang-rename] Use checktime when reloading vim buffer after applying clang-rename<commit_after>'''
Minimal clang-rename integration with Vim.

Before installing make sure one of the following is satisfied:

* clang-rename is in your PATH
* `g:clang_rename_path` in ~/.vimrc points to valid clang-rename executable
* `binary` in clang-rename.py points to valid to clang-rename executable

To install, simply put this into your ~/.vimrc

    noremap <leader>cr :pyf <path-to>/clang-rename.py<cr>

IMPORTANT NOTE: Before running the tool, make sure you saved the file.

All you have to do now is to place a cursor on a variable/function/class which
you would like to rename and press '<leader>cr'. You will be prompted for a new
name if the cursor points to a valid symbol.
'''

import vim
import subprocess
import sys

def main():
    binary = 'clang-rename'
    if vim.eval('exists("g:clang_rename_path")') == "1":
        binary = vim.eval('g:clang_rename_path')

    # Get arguments for clang-rename binary.
    offset = int(vim.eval('line2byte(line("."))+col(".")')) - 2
    if offset < 0:
        print >> sys.stderr, '''Couldn\'t determine cursor position.
                                Is your file empty?'''
        return
    filename = vim.current.buffer.name

    new_name_request_message = 'type new name:'
    new_name = vim.eval("input('{}\n')".format(new_name_request_message))

    # Call clang-rename.
    command = [binary,
               filename,
               '-i',
               '-offset', str(offset),
               '-new-name', str(new_name)]
    # FIXME: make it possible to run the tool on unsaved file.
    p = subprocess.Popen(command,
                         stdout=subprocess.PIPE,
                         stderr=subprocess.PIPE)
    stdout, stderr = p.communicate()

    if stderr:
        print stderr

    # Reload all buffers in Vim.
    vim.command("checktime")

if __name__ == '__main__':
    main()
<|endoftext|>