<commit_before>#!/usr/bin/env python
#
# Copyright (c) 2021, Djaodjin Inc.
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are met:
#
# 1. Redistributions of source code must retain the above copyright notice,
#    this list of conditions and the following disclaimer.
# 2. Redistributions in binary form must reproduce the above copyright notice,
#    this list of conditions and the following disclaimer in the documentation
#    and/or other materials provided with the distribution.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
# THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR
# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
# OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
# WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR
# OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
# ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

import datetime, decimal, json, logging, os, re, sys

import boto3, requests, six
from pytz import utc

LOGGER = logging.getLogger(__name__)

class JSONEncoder(json.JSONEncoder):

    def default(self, obj): #pylint: disable=method-hidden
        # parameter is called `o` in json.JSONEncoder.
        if hasattr(obj, 'isoformat'):
            return obj.isoformat()
        if isinstance(obj, decimal.Decimal):
            return float(obj)
        return super(JSONEncoder, self).default(obj)

class LastRunCache(object):
    """
    Cache for last run on a log file.
    """

    def __init__(self, filename):
        self.filename = filename
        self.last_run_logs = {}
        self.load()

    def load(self):
        if os.path.exists(self.filename):
            with open(self.filename) as last_run:
                self.last_run_logs = json.load(
                    last_run, object_hook=datetime_hook)

    def save(self):
        if not os.path.isdir(os.path.dirname(self.filename)):
            os.makedirs(os.path.dirname(self.filename))
        with open(self.filename, 'w') as last_run:
            json.dump(self.last_run_logs, last_run, cls=JSONEncoder, indent=2)

    def more_recent(self, logname, last_modified, update=False):
        result = (not logname in self.last_run_logs
            or self.last_run_logs[logname] < last_modified)
        if result and update:
            self.last_run_logs[logname] = last_modified
        return result

def as_keyname(filename, logsuffix=None, prefix=None, ext='.log'):
    """
    The keyname returned is in a format as expected by AWS S3
    (i.e. no leading '/') whether `filename` is an absolute path or
    a subdirectory of the current path.
    """
    filename = filename.lstrip('/')
    result = filename
    if ext.startswith('.'):
        ext = ext[1:]
    if logsuffix:
        look = re.match(r'^(\S+\.%s)(\S*)$' % ext, filename)
        if look:
            result = look.group(1) + logsuffix + look.group(2)
    if prefix:
        result = "%s/%s" % (prefix.strip('/'), result)
    return result

def as_filename(key_name, logsuffix=None, prefix=None, ext='.log'):
    result = key_name
    if ext.startswith('.'):
        ext = ext[1:]
    if logsuffix:
        look = re.match(r'^(\S+\.%s)%s(\S*)$' % (ext, logsuffix), key_name)
        if look:
            result = look.group(1) + look.group(2)
    if prefix is not None:
        if result.startswith(prefix):
            result = result[len(prefix):]
        result = result.lstrip('/')
    return result

def as_logname(key_name, logsuffix=None, prefix=None, ext='.log'):
    if ext.startswith('.'):
        ext = ext[1:]
    result = as_filename(key_name, logsuffix=logsuffix, prefix=prefix)
    look = re.match(r'(\S+\.%s)((-\S+)\.gz)' % ext, result)
    if look:
        result = look.group(1)
    return result

def datetime_hook(json_dict):
    for key, value in list(six.iteritems(json_dict)):
        for fmt in ("%Y-%m-%dT%H:%M:%S.%f+00:00", "%Y-%m-%dT%H:%M:%S+00:00"):
            try:
                json_dict[key] = datetime.datetime.strptime(value, fmt)
                if json_dict[key].tzinfo is None:
                    json_dict[key] = json_dict[key].replace(tzinfo=utc)
                break
            except ValueError:
                pass
        if not isinstance(json_dict[key], datetime.datetime):
            LOGGER.warning("%s: cannot convert '%s' to a datetime object.",
                key, value)
    return json_dict

def get_last_modified(item):
    return item['LastModified']

def list_local(lognames, prefix=None, list_all=False):
    """
    Returns a list of rotated log files with their timestamp.

    Example:
    [{ "Key": "/var/log/nginx/www.example.com.log-20160106.gz",
       "LastModified": "Mon, 06 Jan 2016 00:00:00 UTC"},
     { "Key": "/var/log/nginx/www.example.com.log-20160105.gz",
       "LastModified": "Mon, 05 Jan 2016 00:00:00 UTC"},
    ]
    """
    results = []
    for logname in lognames:
        dirname = os.path.dirname(logname)
        _, ext = os.path.splitext(logname)
        if prefix:
            prefixed_dirname = prefix + dirname
        else:
            prefixed_dirname = dirname
        if os.path.isdir(prefixed_dirname):
            for filename in os.listdir(prefixed_dirname):
                fullpath = os.path.join(dirname, filename)
                prefixed_fullpath = os.path.join(prefixed_dirname, filename)
                if (as_logname(fullpath, ext=ext) == logname
                    and (list_all or not fullpath == logname)):
                    mtime = datetime.datetime.fromtimestamp(
                        os.path.getmtime(prefixed_fullpath), tz=utc)
                    results += [{"Key": fullpath, "LastModified": mtime}]
    return results

def list_s3(bucket, lognames, prefix=None, time_from_logsuffix=False):
    """
    Returns a list of rotated log files present in a bucket
    with their timestamp.

    Example:
    [{ "Key": "var/log/nginx/www.example.com.log-0ce5c29636da94d4c-20160106.gz",
       "LastModified": "Mon, 06 Jan 2016 00:00:00 UTC"},
     { "Key": "var/log/nginx/www.example.com.log-0ce5c29636da94d4c-20160105.gz",
       "LastModified": "Mon, 05 Jan 2016 00:00:00 UTC"},
    ]
    """
    results = []
    s3_resource = boto3.resource('s3')
    for logname in lognames:
        logprefix = os.path.splitext(logname)[0].lstrip('/')
        if prefix:
            logprefix = "%s/%s" % (prefix.strip('/'), logprefix)
        for s3_key in s3_resource.Bucket(bucket).objects.filter(
                Prefix=logprefix):
            logkey = as_logname(s3_key.key, prefix=prefix)
            if logname.startswith('/'):
                logkey = '/' + logkey
            if logkey == logname:
                look = re.match(r'\S+-(\d\d\d\d\d\d\d\d)\.gz', s3_key.key)
                if time_from_logsuffix and look:
                    last_modified = datetime.datetime.strptime(
                        look.group(1), "%Y%m%d")
                else:
                    last_modified = s3_key.last_modified
                if last_modified.tzinfo is None:
                    last_modified = last_modified.replace(tzinfo=utc)
                results += [{"Key": s3_key.key, "LastModified": last_modified}]
    return results

def list_updates(local_items, s3_items, logsuffix=None, prefix=None):
    """
    Returns two lists of updated files. The first list is all the files
    in the list *s3_items* which are more recent that files in the list
    *local_items*.
    The second returned list is all the files in the list *local_items*
    which are more recent that files in the list *s3_items*.

    Example:
    [{ "Key": "abc.txt",
       "LastModified": "Mon, 05 Jan 2015 12:00:00 UTC"},
     { "Key": "def.txt",
       "LastModified": "Mon, 05 Jan 2015 12:00:001 UTC"},
    ]
    """
    local_results = []
    local_index = {}
    for local_val in local_items:
        local_index[as_keyname(local_val['Key'],
            logsuffix=logsuffix, prefix=prefix)] = local_val
    for s3_val in s3_items:
        s3_key = s3_val['Key']
        local_val = local_index.get(s3_key, None)
        if local_val:
            local_datetime = local_val['LastModified']
            s3_datetime = s3_val['LastModified']
            if s3_datetime > local_datetime:
                local_results += [s3_val]
        else:
            local_results += [s3_val]

    s3_results = []
    s3_index = {}
    for s3_val in s3_items:
        s3_index[as_filename(s3_val['Key'],
            logsuffix=logsuffix, prefix=prefix)] = s3_val
    for local_val in local_items:
        local_key = local_val['Key'].lstrip('/')
        s3_val = s3_index.get(local_key, None)
        if s3_val:
            s3_datetime = s3_val['LastModified']
            local_datetime = local_val['LastModified']
            if local_datetime > s3_datetime:
                s3_results += [local_val]
        else:
            s3_results += [local_val]

    return local_results, s3_results

def download_updated_logs(lognames,
                          local_prefix=None, logsuffix=None,
                          bucket=None, s3_prefix=None,
                          last_run=None, list_all=False,
                          time_from_logsuffix=False):
    """
    Fetches log files which are on S3 and more recent that specified
    in last_run and returns a list of filenames.
    """
    #pylint:disable=too-many-arguments,too-many-locals
    local_update, _ = list_updates(
        list_local(lognames, prefix=local_prefix, list_all=list_all),
        list_s3(bucket, lognames, prefix=s3_prefix,
            time_from_logsuffix=time_from_logsuffix),
        logsuffix=logsuffix, prefix=s3_prefix)

    downloaded = []
    s3_resource = boto3.resource('s3')
    for item in sorted(local_update, key=get_last_modified):
        keyname = item['Key']
        filename = as_filename(keyname, prefix=s3_prefix)
        if filename.startswith('/'):
            filename = '.' + filename
        logname = as_logname(filename)
        if not last_run or last_run.more_recent(
                logname, item['LastModified'], update=True):
            s3_key = s3_resource.Object(bucket, keyname)
            if not s3_key.storage_class or s3_key.storage_class == 'STANDARD':
                LOGGER.info("download %s to %s\n" % (
                    keyname, os.path.abspath(filename)))
                if not os.path.isdir(os.path.dirname(filename)):
                    os.makedirs(os.path.dirname(filename))
                s3_key.download_file(filename)
                downloaded += [filename]
            else:
                LOGGER.info("skip %s (on %s storage)\n" % (
                    keyname, s3_key.storage_class))

    # It is possible some files were already downloaded as part of a previous
    # run so we construct the list of recent files here.
    downloaded = []
    for item in sorted(list_local(lognames,
                prefix=local_prefix, list_all=False), key=get_last_modified):
        keyname = item['Key']
        filename = as_filename(keyname, prefix=s3_prefix)
        if filename.startswith('/'):
            filename = '.' + filename
        logname = as_logname(filename)
        if not last_run or last_run.more_recent(
                logname, item['LastModified'], update=True):
            downloaded += [filename]
    return downloaded

def upload_log(s3_location, filename, logsuffix=None):
    """
    Upload a local log file to an S3 bucket. If logsuffix is ``None``,
    the instance-id will be automatically added as a suffix in the log filename.
    """
    headers = {'ContentType': 'text/plain'}
    if filename.endswith('.gz'):
        headers.update({'ContentEncoding': 'gzip'})
    parts = s3_location[5:].split('/')
    s3_bucket = parts[0]
    s3_prefix = '/'.join(parts[1:])
    if not logsuffix:
        # https://github.com/boto/boto3/issues/313
        resp = requests.get('http://instance-data/latest/meta-data/instance-id')
        logsuffix = resp.text
        if logsuffix.startswith('i-'):
            logsuffix = logsuffix[1:]
    keyname = as_keyname(
        filename, logsuffix=logsuffix, prefix=s3_prefix)
    LOGGER.info("Upload %s ... to s3://%s/%s\n"
        % (filename, s3_bucket, keyname))
    s3_client = boto3.client('s3')
    s3_client.upload_file(filename, s3_bucket, keyname, ExtraArgs=headers)
<commit_msg>takes local_prefix into account when downloading logs<commit_after>#!/usr/bin/env python
#
# Copyright (c) 2021, Djaodjin Inc.
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are met:
#
# 1. Redistributions of source code must retain the above copyright notice,
#    this list of conditions and the following disclaimer.
# 2. Redistributions in binary form must reproduce the above copyright notice,
#    this list of conditions and the following disclaimer in the documentation
#    and/or other materials provided with the distribution.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
# THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR
# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
# OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
# WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR
# OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
# ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

import datetime, decimal, json, logging, os, re, sys

import boto3, requests, six
from pytz import utc

LOGGER = logging.getLogger(__name__)

class JSONEncoder(json.JSONEncoder):

    def default(self, obj): #pylint: disable=method-hidden
        # parameter is called `o` in json.JSONEncoder.
        if hasattr(obj, 'isoformat'):
            return obj.isoformat()
        if isinstance(obj, decimal.Decimal):
            return float(obj)
        return super(JSONEncoder, self).default(obj)

class LastRunCache(object):
    """
    Cache for last run on a log file.
    """

    def __init__(self, filename):
        self.filename = filename
        self.last_run_logs = {}
        self.load()

    def load(self):
        if os.path.exists(self.filename):
            with open(self.filename) as last_run:
                self.last_run_logs = json.load(
                    last_run, object_hook=datetime_hook)

    def save(self):
        if not os.path.isdir(os.path.dirname(self.filename)):
            os.makedirs(os.path.dirname(self.filename))
        with open(self.filename, 'w') as last_run:
            json.dump(self.last_run_logs, last_run, cls=JSONEncoder, indent=2)

    def more_recent(self, logname, last_modified, update=False):
        result = (not logname in self.last_run_logs
            or self.last_run_logs[logname] < last_modified)
        if result and update:
            self.last_run_logs[logname] = last_modified
        return result

def as_keyname(filename, logsuffix=None, prefix=None, ext='.log'):
    """
    The keyname returned is in a format as expected by AWS S3
    (i.e. no leading '/') whether `filename` is an absolute path or
    a subdirectory of the current path.
    """
    filename = filename.lstrip(os.sep)
    result = filename
    if ext.startswith('.'):
        ext = ext[1:]
    if logsuffix:
        look = re.match(r'^(\S+\.%s)(\S*)$' % ext, filename)
        if look:
            result = look.group(1) + logsuffix + look.group(2)
    if prefix:
        result = "%s/%s" % (prefix.strip('/'), result)
    return result

def as_filename(key_name, logsuffix=None, prefix=None, ext='.log'):
    result = key_name
    if ext.startswith('.'):
        ext = ext[1:]
    if logsuffix:
        look = re.match(r'^(\S+\.%s)%s(\S*)$' % (ext, logsuffix), key_name)
        if look:
            result = look.group(1) + look.group(2)
    if prefix is not None:
        if result.startswith(prefix):
            result = result[len(prefix):]
        result = result.lstrip('/')
    return result

def as_logname(key_name, logsuffix=None, prefix=None, ext='.log'):
    if ext.startswith('.'):
        ext = ext[1:]
    result = as_filename(key_name, logsuffix=logsuffix, prefix=prefix)
    look = re.match(r'(\S+\.%s)((-\S+)\.gz)' % ext, result)
    if look:
        result = look.group(1)
    return result

def datetime_hook(json_dict):
    for key, value in list(six.iteritems(json_dict)):
        for fmt in ("%Y-%m-%dT%H:%M:%S.%f+00:00", "%Y-%m-%dT%H:%M:%S+00:00"):
            try:
                json_dict[key] = datetime.datetime.strptime(value, fmt)
                if json_dict[key].tzinfo is None:
                    json_dict[key] = json_dict[key].replace(tzinfo=utc)
                break
            except ValueError:
                pass
        if not isinstance(json_dict[key], datetime.datetime):
            LOGGER.warning("%s: cannot convert '%s' to a datetime object.",
                key, value)
    return json_dict

def get_last_modified(item):
    return item['LastModified']

def list_local(lognames, prefix=None, list_all=False):
    """
    Returns a list of rotated log files with their timestamp.

    Example:
    [{ "Key": "/var/log/nginx/www.example.com.log-20160106.gz",
       "LastModified": "Mon, 06 Jan 2016 00:00:00 UTC"},
     { "Key": "/var/log/nginx/www.example.com.log-20160105.gz",
       "LastModified": "Mon, 05 Jan 2016 00:00:00 UTC"},
    ]
    """
    results = []
    for logname in lognames:
        dirname = os.path.dirname(logname)
        _, ext = os.path.splitext(logname)
        if prefix:
            prefixed_dirname = prefix + dirname
        else:
            prefixed_dirname = dirname
        if os.path.isdir(prefixed_dirname):
            for filename in os.listdir(prefixed_dirname):
                fullpath = os.path.join(dirname, filename)
                prefixed_fullpath = os.path.join(prefixed_dirname, filename)
                if (as_logname(fullpath, ext=ext) == logname
                    and (list_all or not fullpath == logname)):
                    mtime = datetime.datetime.fromtimestamp(
                        os.path.getmtime(prefixed_fullpath), tz=utc)
                    results += [{"Key": fullpath, "LastModified": mtime}]
    return results

def list_s3(bucket, lognames, prefix=None, time_from_logsuffix=False):
    """
    Returns a list of rotated log files present in a bucket
    with their timestamp.

    Example:
    [{ "Key": "var/log/nginx/www.example.com.log-0ce5c29636da94d4c-20160106.gz",
       "LastModified": "Mon, 06 Jan 2016 00:00:00 UTC"},
     { "Key": "var/log/nginx/www.example.com.log-0ce5c29636da94d4c-20160105.gz",
       "LastModified": "Mon, 05 Jan 2016 00:00:00 UTC"},
    ]
    """
    results = []
    s3_resource = boto3.resource('s3')
    for logname in lognames:
        logprefix = os.path.splitext(logname)[0].lstrip('/')
        if prefix:
            logprefix = "%s/%s" % (prefix.strip('/'), logprefix)
        for s3_key in s3_resource.Bucket(bucket).objects.filter(
                Prefix=logprefix):
            logkey = as_logname(s3_key.key, prefix=prefix)
            if logname.startswith('/'):
                logkey = '/' + logkey
            if logkey == logname:
                look = re.match(r'\S+-(\d\d\d\d\d\d\d\d)\.gz', s3_key.key)
                if time_from_logsuffix and look:
                    last_modified = datetime.datetime.strptime(
                        look.group(1), "%Y%m%d")
                else:
                    last_modified = s3_key.last_modified
                if last_modified.tzinfo is None:
                    last_modified = last_modified.replace(tzinfo=utc)
                results += [{"Key": s3_key.key, "LastModified": last_modified}]
    return results

def list_updates(local_items, s3_items, logsuffix=None, prefix=None):
    """
    Returns two lists of updated files. The first list is all the files
    in the list *s3_items* which are more recent that files in the list
    *local_items*.
    The second returned list is all the files in the list *local_items*
    which are more recent that files in the list *s3_items*.

    Example:
    [{ "Key": "abc.txt",
       "LastModified": "Mon, 05 Jan 2015 12:00:00 UTC"},
     { "Key": "def.txt",
       "LastModified": "Mon, 05 Jan 2015 12:00:001 UTC"},
    ]
    """
    local_results = []
    local_index = {}
    for local_val in local_items:
        local_index[as_keyname(local_val['Key'],
            logsuffix=logsuffix, prefix=prefix)] = local_val
    for s3_val in s3_items:
        s3_key = s3_val['Key']
        local_val = local_index.get(s3_key, None)
        if local_val:
            local_datetime = local_val['LastModified']
            s3_datetime = s3_val['LastModified']
            if s3_datetime > local_datetime:
                local_results += [s3_val]
        else:
            local_results += [s3_val]

    s3_results = []
    s3_index = {}
    for s3_val in s3_items:
        s3_index[as_filename(s3_val['Key'],
            logsuffix=logsuffix, prefix=prefix)] = s3_val
    for local_val in local_items:
        local_key = local_val['Key'].lstrip('/')
        s3_val = s3_index.get(local_key, None)
        if s3_val:
            s3_datetime = s3_val['LastModified']
            local_datetime = local_val['LastModified']
            if local_datetime > s3_datetime:
                s3_results += [local_val]
        else:
            s3_results += [local_val]

    return local_results, s3_results

def download_updated_logs(lognames,
                          local_prefix=None, logsuffix=None,
                          bucket=None, s3_prefix=None,
                          last_run=None, list_all=False,
                          time_from_logsuffix=False):
    """
    Fetches log files which are on S3 and more recent that specified
    in last_run and returns a list of filenames.
    """
    #pylint:disable=too-many-arguments,too-many-locals
    local_update, _ = list_updates(
        list_local(lognames, prefix=local_prefix, list_all=list_all),
        list_s3(bucket, lognames, prefix=s3_prefix,
            time_from_logsuffix=time_from_logsuffix),
        logsuffix=logsuffix, prefix=s3_prefix)

    downloaded = []
    s3_resource = boto3.resource('s3')
    for item in sorted(local_update, key=get_last_modified):
        keyname = item['Key']
        logname = as_logname(keyname)
        filename = as_filename(keyname, prefix=s3_prefix)
        filename = filename.lstrip(os.sep)
        if local_prefix:
            filename = os.path.join('.', local_prefix, filename)
        else:
            filename = os.path.join('.', filename)
        if not last_run or last_run.more_recent(
                logname, item['LastModified'], update=True):
            s3_key = s3_resource.Object(bucket, keyname)
            if not s3_key.storage_class or s3_key.storage_class == 'STANDARD':
                LOGGER.info("download %s to %s\n" % (
                    keyname, os.path.abspath(filename)))
                if not os.path.isdir(os.path.dirname(filename)):
                    os.makedirs(os.path.dirname(filename))
                s3_key.download_file(filename)
                downloaded += [filename]
            else:
                LOGGER.info("skip %s (on %s storage)\n" % (
                    keyname, s3_key.storage_class))

    # It is possible some files were already downloaded as part of a previous
    # run so we construct the list of recent files here.
    downloaded = []
    for item in sorted(list_local(lognames,
                prefix=local_prefix, list_all=False), key=get_last_modified):
        keyname = item['Key']
        filename = as_filename(keyname, prefix=s3_prefix)
        if filename.startswith('/'):
            filename = '.' + filename
        logname = as_logname(filename)
        if not last_run or last_run.more_recent(
                logname, item['LastModified'], update=True):
            downloaded += [filename]
    return downloaded

def upload_log(s3_location, filename, logsuffix=None):
    """
    Upload a local log file to an S3 bucket. If logsuffix is ``None``,
    the instance-id will be automatically added as a suffix in the log filename.
    """
    headers = {'ContentType': 'text/plain'}
    if filename.endswith('.gz'):
        headers.update({'ContentEncoding': 'gzip'})
    parts = s3_location[5:].split('/')
    s3_bucket = parts[0]
    s3_prefix = '/'.join(parts[1:])
    if not logsuffix:
        # https://github.com/boto/boto3/issues/313
        resp = requests.get('http://instance-data/latest/meta-data/instance-id')
        logsuffix = resp.text
        if logsuffix.startswith('i-'):
            logsuffix = logsuffix[1:]
    keyname = as_keyname(
        filename, logsuffix=logsuffix, prefix=s3_prefix)
    LOGGER.info("Upload %s ... to s3://%s/%s\n"
        % (filename, s3_bucket, keyname))
    s3_client = boto3.client('s3')
    s3_client.upload_file(filename, s3_bucket, keyname, ExtraArgs=headers)
<|endoftext|>


<commit_before># Copyright Least Authority Enterprises.
# See LICENSE for details.

"""
An in-memory implementation of the Kubernetes client interface.
"""

from functools import partial

from json import dumps, loads

import attr

from pyrsistent import pset

from zope.interface import implementer

from twisted.python.url import URL

from twisted.web.resource import Resource, NoResource
from twisted.web.http import CREATED

from klein import Klein

from werkzeug.exceptions import NotFound

from eliot import Message

from treq.testing import RequestTraversalAgent

from . import (
    IKubernetes, network_kubernetes,
    ObjectCollection, Namespace, ConfigMap,
)

def memory_kubernetes():
    """
    Create an in-memory Kubernetes-alike service.

    This serves as a places to hold state for stateful Kubernetes interactions
    allowed by ``IKubernetesClient``.  Only clients created against the same
    instance will all share state.

    :return IKubernetes: The new Kubernetes-alike service.
    """
    return _MemoryKubernetes()

@implementer(IKubernetes)
class _MemoryKubernetes(object):
    """
    ``_MemoryKubernetes`` maintains state in-memory which approximates
    the state of a real Kubernetes deployment sufficiently to expose a
    subset of the external Kubernetes API.
    """
    def __init__(self):
        base_url = URL.fromText(u"https://kubernetes.example.invalid./")
        self._state = _KubernetesState()
        self._resource = _kubernetes_resource(self._state)
        self._kubernetes = network_kubernetes(
            base_url=base_url,
            credentials=None,
            agent=RequestTraversalAgent(self._resource),
        )

    def client(self, *args, **kwargs):
        """
        :return IKubernetesClient: A new client which interacts with this
            object rather than a real Kubernetes deployment.
        """
        return self._kubernetes.client(*args, **kwargs)

def _kubernetes_resource(state):
    return _Kubernetes(state).app.resource()

@attr.s
class _KubernetesState(object):
    namespaces = attr.ib(default=ObjectCollection())
    configmaps = attr.ib(default=ObjectCollection())

def terminate(obj):
    return obj.transform([u"status", u"phase"], u"Terminating")

@attr.s(frozen=True)
class _Kubernetes(object):
    """
    A place to stick a bunch of Klein definitions.

    :ivar _KubernetesState state: The Kubernetes state with which the API will
        be interacting.
    """
    state = attr.ib()

    def _list(self, request, namespace, collection):
        if namespace is not None:
            # Unfortunately pset does not support transform. :( Use this more
            # verbose .set() operation.
            collection = collection.set(
                u"items",
                pset(obj for obj in collection.items if obj.metadata.namespace == namespace),
            )
        request.responseHeaders.setRawHeaders(u"content-type", [u"application/json"])
        return dumps(collection.to_raw())

    def _get(self, request, collection, name):
        obj = collection.item_by_name(name)
        request.responseHeaders.setRawHeaders(u"content-type", [u"application/json"])
        return dumps(obj.to_raw())

    def _create(self, request, type, collection, collection_name):
        obj = type.from_raw(loads(request.content.read()))
        setattr(self.state, collection_name, collection.add(obj))
        request.responseHeaders.setRawHeaders(u"content-type", [u"application/json"])
        request.setResponseCode(CREATED)
        return dumps(obj.to_raw())

    def _delete(self, request, collection, collection_name, name):
        obj = collection.item_by_name(name)
        setattr(self.state, collection_name, collection.replace(obj, terminate(obj)))
        request.responseHeaders.setRawHeaders(u"content-type", [u"application/json"])
        return dumps(obj.to_raw())

    app = Klein()
    @app.handle_errors(NotFound)
    def not_found(self, request, name):
        request.responseHeaders.setRawHeader(u"content-type", [u"application/json"])
        return dumps({u"message": u"boo"})

    with app.subroute(u"/api/v1") as app:
        @app.route(u"/namespaces", methods=[u"GET"])
        def list_namespaces(self, request):
            """
            Get all existing Namespaces.
            """
            return self._list(request, None, self.state.namespaces)

        @app.route(u"/namespaces/<namespace>", methods=[u"GET"])
        def get_namespace(self, request, namespace):
            """
            Get one Namespace by name.
            """
            return self._get(request, self.state.namespaces, namespace)

        @app.route(u"/namespaces/<namespace>", methods=[u"DELETE"])
        def delete_namespace(self, request, namespace):
            """
            Delete one Namespace by name.
            """
            return self._delete(
                request, self.state.namespaces, "namespaces", namespace,
            )

        @app.route(u"/namespaces", methods=[u"POST"])
        def create_namespace(self, request):
            """
            Create a new Namespace.
            """
            return self._create(request, Namespace, self.state.namespaces, "namespaces")

        @app.route(u"/configmaps", methods=[u"GET"])
        def list_configmaps(self, request, namespace=None):
            """
            Get all existing ConfigMaps.
            """
            return self._list(request, namespace, self.state.configmaps)

        @app.route(u"/configmaps/<configmap>", methods=[u"GET"])
        def get_configmap(self, request, configmap):
            """
            Get one ConfigMap by name.
            """
            return self._get(request, self.state.configmaps, configmap)

        @app.route(u"/namespaces/<namespace>/configmaps", methods=[u"POST"])
        def create_configmap(self, request, namespace):
            """
            Create a new ConfigMap.
            """
            return self._create(request, ConfigMap, self.state.configmaps, "configmaps")
<commit_msg>There is another piece of state associated with the Terminating phase.<commit_after># Copyright Least Authority Enterprises.
# See LICENSE for details.

"""
An in-memory implementation of the Kubernetes client interface.
"""

from functools import partial

from json import dumps, loads

import attr

from pyrsistent import pset

from zope.interface import implementer

from twisted.python.url import URL

from twisted.web.resource import Resource, NoResource
from twisted.web.http import CREATED

from klein import Klein

from werkzeug.exceptions import NotFound

from eliot import Message

from treq.testing import RequestTraversalAgent

from . import (
    IKubernetes, network_kubernetes,
    ObjectCollection, Namespace, ConfigMap,
)

def memory_kubernetes():
    """
    Create an in-memory Kubernetes-alike service.

    This serves as a places to hold state for stateful Kubernetes interactions
    allowed by ``IKubernetesClient``.  Only clients created against the same
    instance will all share state.

    :return IKubernetes: The new Kubernetes-alike service.
    """
    return _MemoryKubernetes()

@implementer(IKubernetes)
class _MemoryKubernetes(object):
    """
    ``_MemoryKubernetes`` maintains state in-memory which approximates
    the state of a real Kubernetes deployment sufficiently to expose a
    subset of the external Kubernetes API.
    """
    def __init__(self):
        base_url = URL.fromText(u"https://kubernetes.example.invalid./")
        self._state = _KubernetesState()
        self._resource = _kubernetes_resource(self._state)
        self._kubernetes = network_kubernetes(
            base_url=base_url,
            credentials=None,
            agent=RequestTraversalAgent(self._resource),
        )

    def client(self, *args, **kwargs):
        """
        :return IKubernetesClient: A new client which interacts with this
            object rather than a real Kubernetes deployment.
        """
        return self._kubernetes.client(*args, **kwargs)

def _kubernetes_resource(state):
    return _Kubernetes(state).app.resource()

@attr.s
class _KubernetesState(object):
    namespaces = attr.ib(default=ObjectCollection())
    configmaps = attr.ib(default=ObjectCollection())

def terminate(obj):
    # TODO: Add deletionTimestamp?  See #24
    return obj.transform([u"status", u"phase"], u"Terminating")

@attr.s(frozen=True)
class _Kubernetes(object):
    """
    A place to stick a bunch of Klein definitions.

    :ivar _KubernetesState state: The Kubernetes state with which the API will
        be interacting.
    """
    state = attr.ib()

    def _list(self, request, namespace, collection):
        if namespace is not None:
            # Unfortunately pset does not support transform. :( Use this more
            # verbose .set() operation.
            collection = collection.set(
                u"items",
                pset(obj for obj in collection.items if obj.metadata.namespace == namespace),
            )
        request.responseHeaders.setRawHeaders(u"content-type", [u"application/json"])
        return dumps(collection.to_raw())

    def _get(self, request, collection, name):
        obj = collection.item_by_name(name)
        request.responseHeaders.setRawHeaders(u"content-type", [u"application/json"])
        return dumps(obj.to_raw())

    def _create(self, request, type, collection, collection_name):
        obj = type.from_raw(loads(request.content.read()))
        setattr(self.state, collection_name, collection.add(obj))
        request.responseHeaders.setRawHeaders(u"content-type", [u"application/json"])
        request.setResponseCode(CREATED)
        return dumps(obj.to_raw())

    def _delete(self, request, collection, collection_name, name):
        obj = collection.item_by_name(name)
        setattr(self.state, collection_name, collection.replace(obj, terminate(obj)))
        request.responseHeaders.setRawHeaders(u"content-type", [u"application/json"])
        return dumps(obj.to_raw())

    app = Klein()
    @app.handle_errors(NotFound)
    def not_found(self, request, name):
        request.responseHeaders.setRawHeader(u"content-type", [u"application/json"])
        return dumps({u"message": u"boo"})

    with app.subroute(u"/api/v1") as app:
        @app.route(u"/namespaces", methods=[u"GET"])
        def list_namespaces(self, request):
            """
            Get all existing Namespaces.
            """
            return self._list(request, None, self.state.namespaces)

        @app.route(u"/namespaces/<namespace>", methods=[u"GET"])
        def get_namespace(self, request, namespace):
            """
            Get one Namespace by name.
            """
            return self._get(request, self.state.namespaces, namespace)

        @app.route(u"/namespaces/<namespace>", methods=[u"DELETE"])
        def delete_namespace(self, request, namespace):
            """
            Delete one Namespace by name.
            """
            return self._delete(
                request, self.state.namespaces, "namespaces", namespace,
            )

        @app.route(u"/namespaces", methods=[u"POST"])
        def create_namespace(self, request):
            """
            Create a new Namespace.
            """
            return self._create(request, Namespace, self.state.namespaces, "namespaces")

        @app.route(u"/configmaps", methods=[u"GET"])
        def list_configmaps(self, request, namespace=None):
            """
            Get all existing ConfigMaps.
            """
            return self._list(request, namespace, self.state.configmaps)

        @app.route(u"/configmaps/<configmap>", methods=[u"GET"])
        def get_configmap(self, request, configmap):
            """
            Get one ConfigMap by name.
            """
            return self._get(request, self.state.configmaps, configmap)

        @app.route(u"/namespaces/<namespace>/configmaps", methods=[u"POST"])
        def create_configmap(self, request, namespace):
            """
            Create a new ConfigMap.
            """
            return self._create(request, ConfigMap, self.state.configmaps, "configmaps")
<|endoftext|>


<commit_before># Built-in modules
from os import path, pardir
import sys
import logging

# not used in this stub but often useful for finding various files
PROJECT_ROOT_DIRPATH = path.join(path.dirname(__file__), pardir, pardir)
sys.path.append(PROJECT_ROOT_DIRPATH)

# third party modules
import click
from dotenv import find_dotenv, load_dotenv
# Hand-made modules
from src.data.unzip import Unzipper
from src.data.amedas import AmedasHandler
from src.data.surface import SurfaceHandler
from src.data.forecast import ForecastHandler
from src.data.sola import SolarPhotovoltaicHandler
from src.data.concatenation import DatasetCollector

AMD_MASTER_FILENAME = "amd_master.tsv"
SFC_MASTER_FILENAME = "sfc_master.tsv"
TRAIN_KWH_FILENAME = "train_kwh.tsv"
OBJECTIVE_COLUMN_NAME = "kwh"

LATLNGALT_UKISHIMA = (35.517558, 139.786920, 4.7)
LATLNGALT_OUGISHIMA = (35.488680, 139.727451, 4.8)
LATLNGALT_YONEKURAYAMA = (35.583302, 138.573118, 366.9)

LOCATIONS = (
    "ukishima",
    "ougishima",
    "yonekurayama",
)
LATLNG_BASE_COORDINATES = (
    LATLNGALT_UKISHIMA,
    LATLNGALT_OUGISHIMA,
    LATLNGALT_YONEKURAYAMA,
)

@click.command()
@click.argument("input_dirpath", type=click.Path(exists=True))
@click.option("--amd_half_mashgrid_size", "-a", type=float, default=0.2)
@click.option("--sfc_half_mashgrid_size", "-s", type=float, default=0.4)
@click.option("--is_unzip", "-z", type=bool, default=True)
def main(input_dirpath,
         amd_half_mashgrid_size,
         sfc_half_mashgrid_size,
         is_unzip):
    """ Runs data processing scripts to turn raw data from (../raw) into
        cleaned data ready to be analyzed (saved in ../processed).
    """
    logger = logging.getLogger(__name__)
    logger.info('#0: making final data set from raw data')

    #
    # Unzipping
    #
    if is_unzip:
        unzipper = Unzipper()
        for unzip_filepath in unzipper.gen_unzip_filepath_list(input_dirpath):
            unzipper.unzip(unzip_filepath, unzipper.INTERIM_DATA_BASEPATH)
            logger.info('#1: unzipped {f} !'.format(f=path.basename(unzip_filepath)))

        logger.info('#1: end unzip !')
        del unzipper
    else:
        logger.info('#1: skipped unzipping !')

    #
    # amedas information
    #
    amd = AmedasHandler(path.join(input_dirpath, AMD_MASTER_FILENAME))

    for location, latlng_base_coordinate in zip(LOCATIONS, LATLNG_BASE_COORDINATES):
        retrieve_amd_point = amd.get_near_observation_points(
            latlng_base_coordinate[0], latlng_base_coordinate[1], amd_half_mashgrid_size
        )

        logger.info('#2: get amedas observation points in {l} !'.format(l=location))

        amd_filepath = amd.gen_filepath_list(retrieve_amd_point.index)
        df_amd = amd.retrieve_data(amd_filepath, retrieve_amd_point["name"].as_matrix())
        amd.to_tsv(df_amd, path.join(amd.INTERIM_DATA_BASEPATH, "amd_data_near.{l}.tsv".format(l=location)))

        logger.info('#2: gather amedas data and save as a middle file in {l} !'.format(l=location))
        del (df_amd, retrieve_amd_point)

    logger.info('#2: end amedas data processing !')
    del amd

    #
    # surface weather information
    #
    sfc = SurfaceHandler(path.join(input_dirpath, SFC_MASTER_FILENAME))

    for location, latlng_base_coordinate in zip(LOCATIONS, LATLNG_BASE_COORDINATES):
        retrieve_sfc_point = sfc.get_near_observation_points(
            latlng_base_coordinate[0], latlng_base_coordinate[1], sfc_half_mashgrid_size
        )

        logger.info('#3: get surface weather observation points in {l} !'.format(l=location))

        sfc_filepath = sfc.gen_filepath_list(retrieve_sfc_point.index)
        df_sfc = sfc.retrieve_data(sfc_filepath, retrieve_sfc_point["name"].as_matrix())
        sfc.to_tsv(df_sfc, path.join(sfc.INTERIM_DATA_BASEPATH, "sfc_data_near.{l}.tsv".format(l=location)))

        logger.info('#3: gather surface weather data and save as a middle file in {l} !'.format(l=location))
        del (df_sfc, retrieve_sfc_point)

    logger.info('#3: end surface weather data processing !')
    del sfc

    #
    # forecast information
    #
    forecast = ForecastHandler()

    for location in LOCATIONS:
        forecast_filepath = forecast.gen_filepath(location)
        df_forecast = forecast.read_tsv(forecast_filepath)
        df_forecast_expanded = forecast.add_datetime_ticks(df_forecast)

        whole_day_data_name_list = \
            forecast.get_whole_day_data_columns(df_forecast.columns)

        for whole_day_data_name in whole_day_data_name_list:
            sr_expand_whole_day_data = \
                forecast.expand_whole_day_data(df_forecast[whole_day_data_name])
            df_forecast_expanded.loc[
                sr_expand_whole_day_data.index, whole_day_data_name
            ] = sr_expand_whole_day_data

            logger.info('#4: expand "{n}" in forecast'.format(n=whole_day_data_name))
            del sr_expand_whole_day_data

        time_ranged_data_name_list = \
            forecast.get_time_ranged_data_columns(df_forecast.columns)

        for time_ranged_data_name in time_ranged_data_name_list:
            sr_expand_time_ranged_data = \
                forecast.expand_time_ranged_data(df_forecast[time_ranged_data_name])
            df_forecast_expanded.loc[
                sr_expand_time_ranged_data.index,
                forecast.extract_attribute_from_time_ranged_column_name(time_ranged_data_name)
            ] = sr_expand_time_ranged_data

            logger.info('#4: expand "{n}" in {l}'.format(n=time_ranged_data_name, l=location))
            del sr_expand_time_ranged_data

        df_forecast_expanded.drop(time_ranged_data_name_list, axis=1, inplace=True)
        forecast.to_tsv(
            df_forecast_expanded,
            path.join(forecast.INTERIM_DATA_BASEPATH, "forecast_data.{l}.tsv".format(l=location))
        )

        logger.info('#4: gather forecast data and save as a middle file in {l} !'.format(l=location))
        del (
            df_forecast,
            df_forecast_expanded,
            whole_day_data_name_list,
            time_ranged_data_name_list
        )

    logger.info('#4: end forecast data processing !')
    del forecast

    #
    # train_kwh information
    #
    sola = SolarPhotovoltaicHandler()
    df_train_kwh = sola.read_tsv(path.join(input_dirpath, TRAIN_KWH_FILENAME))

    for location in LOCATIONS:
        col_label = sola.SOLA_LOCATION_LABEL_NAMES[location]
        df_sola = df_train_kwh.loc[:, col_label].to_frame(name=OBJECTIVE_COLUMN_NAME)
        sola.to_tsv(df_sola, path.join(sola.INTERIM_DATA_BASEPATH, "sola_data.{l}.tsv".format(l=location)))

        logger.info('#5: get solar data and save as a middle file in {l}!'.format(l=location))
        del df_sola

    logger.info('#5: end solar data processing !')
    del (sola, df_train_kwh)

    #
    # gether data
    #
    collector = DatasetCollector()

    for location in LOCATIONS:
        location_filepath = collector.gen_filepath_list(location)
        df_train_for_each_location = collector.retrieve_data(location_filepath)
        collector.to_tsv(
            df_train_for_each_location,
            path.join(
                collector.PROCESSED_DATA_BASEPATH,
                "dataset.amd_sfc_forecast_kwh.{l}.tsv".format(l=location)
            )
        )

        logger.info('#6: generate and save the dataset as a processed file in {l} !'.format(l=location))
        del df_train_for_each_location

if __name__ == '__main__':
    log_fmt = '%(asctime)s - %(name)s - %(levelname)s - %(message)s'
    logging.basicConfig(level=logging.INFO, format=log_fmt)

    # find .env automagically by walking up directories until it's found, then
    # load up the .env entries as environment variables
    load_dotenv(find_dotenv())

    main()
<commit_msg>Log intermediate processing files by bloscpack<commit_after># Built-in modules
from os import path, pardir
import sys
import logging

# not used in this stub but often useful for finding various files
PROJECT_ROOT_DIRPATH = path.join(path.dirname(__file__), pardir, pardir)
sys.path.append(PROJECT_ROOT_DIRPATH)

# third party modules
import click
from dotenv import find_dotenv, load_dotenv
# Hand-made modules
from src.data.unzip import Unzipper
from src.data.amedas import AmedasHandler
from src.data.surface import SurfaceHandler
from src.data.forecast import ForecastHandler
from src.data.sola import SolarPhotovoltaicHandler
from src.data.concatenation import DatasetCollector

AMD_MASTER_FILENAME = "amd_master.tsv"
SFC_MASTER_FILENAME = "sfc_master.tsv"
TRAIN_KWH_FILENAME = "train_kwh.tsv"
OBJECTIVE_COLUMN_NAME = "kwh"

LATLNGALT_UKISHIMA = (35.517558, 139.786920, 4.7)
LATLNGALT_OUGISHIMA = (35.488680, 139.727451, 4.8)
LATLNGALT_YONEKURAYAMA = (35.583302, 138.573118, 366.9)

LOCATIONS = (
    "ukishima",
    "ougishima",
    "yonekurayama",
)
LATLNG_BASE_COORDINATES = (
    LATLNGALT_UKISHIMA,
    LATLNGALT_OUGISHIMA,
    LATLNGALT_YONEKURAYAMA,
)

@click.command()
@click.argument("input_dirpath", type=click.Path(exists=True))
@click.option("--amd_half_mashgrid_size", "-a", type=float, default=0.2)
@click.option("--sfc_half_mashgrid_size", "-s", type=float, default=0.4)
@click.option("--is_unzip", "-z", type=bool, default=True)
def main(input_dirpath,
         amd_half_mashgrid_size,
         sfc_half_mashgrid_size,
         is_unzip):
    """ Runs data processing scripts to turn raw data from (../raw) into
        cleaned data ready to be analyzed (saved in ../processed).
    """
    logger = logging.getLogger(__name__)
    logger.info('#0: making final data set from raw data')

    #
    # Unzipping
    #
    if is_unzip:
        unzipper = Unzipper()
        for unzip_filepath in unzipper.gen_unzip_filepath_list(input_dirpath):
            unzipper.unzip(unzip_filepath, unzipper.INTERIM_DATA_BASEPATH)
            logger.info('#1: unzipped {f} !'.format(f=path.basename(unzip_filepath)))

        logger.info('#1: end unzip !')
        del unzipper
    else:
        logger.info('#1: skipped unzipping !')

    #
    # amedas information
    #
    amd = AmedasHandler(path.join(input_dirpath, AMD_MASTER_FILENAME))

    for location, latlng_base_coordinate in zip(LOCATIONS, LATLNG_BASE_COORDINATES):
        retrieve_amd_point = amd.get_near_observation_points(
            latlng_base_coordinate[0], latlng_base_coordinate[1], amd_half_mashgrid_size
        )

        logger.info('#2: get amedas observation points in {l} !'.format(l=location))

        amd_filepath = amd.gen_filepath_list(retrieve_amd_point.index)
        df_amd = amd.retrieve_data(amd_filepath, retrieve_amd_point["name"].as_matrix())
        amd.to_blp_via_df(
            df_amd,
            path.join(amd.INTERIM_DATA_BASEPATH, "amd_data"),
            "{l}.data#2".format(l=location)
        )

        logger.info('#2: gather amedas data and save as a middle file in {l} !'.format(l=location))
        del (df_amd, retrieve_amd_point)

    logger.info('#2: end amedas data processing !')
    del amd

    #
    # surface weather information
    #
    sfc = SurfaceHandler(path.join(input_dirpath, SFC_MASTER_FILENAME))

    for location, latlng_base_coordinate in zip(LOCATIONS, LATLNG_BASE_COORDINATES):
        retrieve_sfc_point = sfc.get_near_observation_points(
            latlng_base_coordinate[0], latlng_base_coordinate[1], sfc_half_mashgrid_size
        )

        logger.info('#3: get surface weather observation points in {l} !'.format(l=location))

        sfc_filepath = sfc.gen_filepath_list(retrieve_sfc_point.index)
        df_sfc = sfc.retrieve_data(sfc_filepath, retrieve_sfc_point["name"].as_matrix())
        sfc.to_blp_via_df(
            df_sfc,
            path.join(sfc.INTERIM_DATA_BASEPATH, "sfc_data"),
            "{l}.data#3".format(l=location)
        )

        logger.info('#3: gather surface weather data and save as a middle file in {l} !'.format(l=location))
        del (df_sfc, retrieve_sfc_point)

    logger.info('#3: end surface weather data processing !')
    del sfc

    #
    # forecast information
    #
    forecast = ForecastHandler()

    for location in LOCATIONS:
        forecast_filepath = forecast.gen_filepath(location)
        df_forecast = forecast.read_tsv(forecast_filepath)
        df_forecast_expanded = forecast.add_datetime_ticks(df_forecast)

        whole_day_data_name_list = \
            forecast.get_whole_day_data_columns(df_forecast.columns)

        for whole_day_data_name in whole_day_data_name_list:
            sr_expand_whole_day_data = \
                forecast.expand_whole_day_data(df_forecast[whole_day_data_name])
            df_forecast_expanded.loc[
                sr_expand_whole_day_data.index, whole_day_data_name
            ] = sr_expand_whole_day_data

            logger.info('#4: expand "{n}" in forecast'.format(n=whole_day_data_name))
            del sr_expand_whole_day_data

        time_ranged_data_name_list = \
            forecast.get_time_ranged_data_columns(df_forecast.columns)

        for time_ranged_data_name in time_ranged_data_name_list:
            sr_expand_time_ranged_data = \
                forecast.expand_time_ranged_data(df_forecast[time_ranged_data_name])
            df_forecast_expanded.loc[
                sr_expand_time_ranged_data.index,
                forecast.extract_attribute_from_time_ranged_column_name(time_ranged_data_name)
            ] = sr_expand_time_ranged_data

            logger.info('#4: expand "{n}" in {l}'.format(n=time_ranged_data_name, l=location))
            del sr_expand_time_ranged_data

        df_forecast_expanded.drop(time_ranged_data_name_list, axis=1, inplace=True)
        forecast.to_blp_via_df(
            df_forecast_expanded,
            path.join(forecast.INTERIM_DATA_BASEPATH, "forecast_data"),
            "{l}.data#4".format(l=location)
        )

        logger.info('#4: gather forecast data and save as a middle file in {l} !'.format(l=location))
        del (
            df_forecast,
            df_forecast_expanded,
            whole_day_data_name_list,
            time_ranged_data_name_list
        )

    logger.info('#4: end forecast data processing !')
    del forecast

    #
    # train_kwh information
    #
    sola = SolarPhotovoltaicHandler()
    df_train_kwh = sola.read_tsv(path.join(input_dirpath, TRAIN_KWH_FILENAME))

    for location in LOCATIONS:
        col_label = sola.SOLA_LOCATION_LABEL_NAMES[location]
        df_sola = df_train_kwh.loc[:, col_label].to_frame(name=OBJECTIVE_COLUMN_NAME)
        sola.to_blp_via_df(
            df_sola,
            path.join(sola.INTERIM_DATA_BASEPATH, "sola_data"),
            "{l}.data#5".format(l=location)
        )

        logger.info('#5: get solar data and save as a middle file in {l}!'.format(l=location))
        del df_sola

    logger.info('#5: end solar data processing !')
    del (sola, df_train_kwh)

    #
    # gether data
    #
    collector = DatasetCollector()

    for location in LOCATIONS:
        filepath_prefix_suffix_nested_list = \
            collector.gen_filepath_prefix_suffix_nested_list(location)
        df_train_for_each_location = \
            collector.retrieve_data(filepath_prefix_suffix_nested_list)
        collector.to_blp_via_df(
            df_train_for_each_location,
            path.join(collector.PROCESSED_DATA_BASEPATH, "dataset.amd_sfc_forecast_kwh"),
            "{l}.blp".format(l=location)
        )

        logger.info('#6: generate and save the dataset as a processed file in {l} !'.format(l=location))
        del df_train_for_each_location

if __name__ == '__main__':
    log_fmt = '%(asctime)s - %(name)s - %(levelname)s - %(message)s'
    logging.basicConfig(level=logging.INFO, format=log_fmt)

    # find .env automagically by walking up directories until it's found, then
    # load up the .env entries as environment variables
    load_dotenv(find_dotenv())

    main()
<|endoftext|>


<commit_before>from __future__ import unicode_literals

import argparse
import os
import sys
import config

from prompt_toolkit import prompt
from pygments.token import Token
from prompt_toolkit.application import Application
from prompt_toolkit.interface import CommandLineInterface

#default config path
CONF = os.path.abspath('/data/config/')+'config.yml'

#defapplication = Application(
#    layout=layout,
#    key_bindings_registry=registry,

    # Let's add mouse support as well.
 #   mouse_support=True,

    # For fullscreen:
 #   use_alternate_screen=True)

def main():
    """
    This is the main function used for cli functionability.
    """

    # configuration = config.load_configuration()

    parser = argparse.ArgumentParser()

    # transform()

    # b) define a config file based transformation
    # default case w/o parameters
<commit_msg>cli script updated<commit_after>#!/usr/bin/python
import os
import codecs
import argparse
import yaml

parser = argparse.ArgumentParser()
parser.add_argument('resources', help='a file containing paths of a resource (corpus, event, speaker) to transform')

def main():
    """
    Main Method for dgd to cmdi transformation
    :return:
    """
    args = parser.parse_args()
    with codecs.open(args.resources, mode='r', encoding='utf-8') as resfile:
        resources = yaml.safe_load(resfile)
    xslt = resources.get('xslt')
    print "Stylesheet paths:\n"
    print "Corpus: {}".format(xslt['corpus'])
    print "Event: {}".format(xslt['event'])
    print "Speaker: {}".format(xslt['speaker'])

if __name__ == '__main__':
    main()

<|endoftext|>


<commit_before># -*- coding: iso-8859-1 -*-
# -----------------------------------------------------------------------------
# core.py - distribution core functions for kaa packages
# -----------------------------------------------------------------------------
# $Id$
#
# -----------------------------------------------------------------------------
# Copyright (C) 2006 Dirk Meyer, Jason Tackaberry
#
# First Edition: Dirk Meyer <dmeyer@tzi.de>
# Maintainer:    Dirk Meyer <dmeyer@tzi.de>
#
# Please see the file AUTHORS for a complete list of authors.
#
# This library is free software; you can redistribute it and/or modify
# it under the terms of the GNU Lesser General Public License version
# 2.1 as published by the Free Software Foundation.
#
# This library is distributed in the hope that it will be useful, but
# WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
# Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public
# License along with this library; if not, write to the Free Software
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
# 02110-1301 USA
#
# -----------------------------------------------------------------------------

# python imports
import os
import sys
import math
import stat
import re
import tempfile
import distutils.core

# internal imports
from version import Version
from build_py import build_py

__all__ = ['compile', 'check_library', 'get_library', 'setup', 'ConfigFile',
           'Extension', 'Library']

_libraries = {}

def compile(includes, code, args=''):
    fd, outfile = tempfile.mkstemp()
    os.close(fd)
    f = os.popen("cc -x c - -o %s %s 2>/dev/null >/dev/null" % (outfile, args), "w")
    if not f:
        return False

    for i in includes:
        f.write('#include %s\n' % i)
    f.write('int main() { ' + code + '\nreturn 0;\n};\n')
    result = f.close()

    if os.path.exists(outfile):
        os.unlink(outfile)

    if result == None:
        return True
    return False

def check_library(name, *args):
    lib = Library(name)
    if len(args) < 2:
        minver = args[0]
        print 'checking for', name, '>=', minver, '...',
        sys.__stdout__.flush()
        found, version = lib.check(minver)
        if found:
            print version
        elif version:
            print 'no (%s)' % version
        else:
            print 'no'
    else:
        print 'checking for', name, '...',
        sys.__stdout__.flush()
        if lib.compile(*args):
            print 'ok'
        else:
            print 'no'
    _libraries[name] = lib
    return lib

def get_library(name):
    lib = _libraries.get(name)
    if lib and lib.valid:
        return lib
    return None

class Library(object):
    def __init__(self, name):
        self.name = name
        self.include_dirs = []
        self.library_dirs = []
        self.libraries = []
        self.version = None
        self.valid = False

    def compare_versions(self, a, b):
        # Get maximum component length in both version.
        maxlen =  max([ len(x) for x in (a + '.' + b).split('.') ])
        # Pad each component of A and B to maxlen
        a = [ x.zfill(maxlen) for x in a.split('.') ]
        b = [ x.zfill(maxlen) for x in b.split('.') ]
        # TODO: special handling of rc and beta (others?) suffixes, so
        # that 1.0.0 > 1.0.0rc1
        return cmp(a, b)

    def get_numeric_version(self, version = None):
        """
        Returns an integer version suitable for numeric comparison.  This
        returns the version of the library gotten after calling check().  If
        check() was not called this method will raise an exception.

        This algorithm is pretty naive and won't work at all if the version
        contains any non-numeric characters.
        """
        if version is None:
            version = self.version

        if version is None:
            raise ValueError, "Version is not known; call check() first"

        if not version.replace(".", "").isdigit():
            raise ValueError, "Version cannot have non-numeric characters."

        # Performs: 1.2.3.4 => 4<<0 + 3<<9 + 2<<18 + 1<<27
        return sum([ int(v) << 9*s for s,v in enumerate(reversed(version.split('.'))) ])

    def check(self, minver):
        """
        Check dependencies add add the flags to include_dirs, library_dirs and
        libraries. The basic logic is taken from pygame.
        """
        if os.system("%s-config --version >/dev/null 2>/dev/null" % self.name) == 0:
            # Use foo-config if it exists.
            command = "%s-config %%s 2>/dev/null" % self.name
            version_arg = "--version"
        elif os.system("pkg-config %s --exists >/dev/null 2>/dev/null" % self.name) == 0:
            # Otherwise try pkg-config foo.
            command = "pkg-config %s %%s 2>/dev/null" % self.name
            version_arg = "--modversion"
        else:
            return False, None

        version = os.popen(command % version_arg).read().strip()
        if len(version) == 0:
            return False, None
        if minver and self.compare_versions(minver, version) > 0:
            return False, version

        for inc in os.popen(command % "--cflags").read().strip().split(' '):
            if inc[2:] and not inc[2:] in self.include_dirs:
                self.include_dirs.append(inc[2:])

        for flag in os.popen(command % "--libs").read().strip().split(' '):
            if flag[:2] == '-L' and not flag[2:] in self.library_dirs:
                self.library_dirs.append(flag[2:])
            if flag[:2] == '-l' and not flag[2:] in self.libraries:
                self.libraries.append(flag[2:])

        self.version = version
        self.valid = True
        return True, version

    def compile(self, includes, code, args=''):
        if compile(includes, code, args):
            self.valid = True
            return True

        return False

class ConfigFile(object):
    """
    Config file for the build process.
    """
    def __init__(self, filename):
        self.file = os.path.abspath(filename)
        # create config file
        open(self.file, 'w').close()

    def append(self, line):
        """
        Append something to the config file.
        """
        f = open(self.file, 'a')
        f.write(line + '\n')
        f.close()

    def define(self, variable, value=None):
        """
        Set a #define.
        """
        if value == None:
            self.append('#define %s' % variable)
        else:
            self.append('#define %s %s' % (variable, value))

    def unlink(self):
        """
        Delete config file.
        """
        os.unlink(self.file)

class Extension(object):
    """
    Extension wrapper with additional functions to find libraries and
    support for config files.
    """
    def __init__(self, output, files, include_dirs=[], library_dirs=[],
                 libraries=[], extra_compile_args = [], config=None):
        """
        Init the Extension object.
        """
        self.output = output
        self.files = files
        self.include_dirs = include_dirs[:]
        self.library_dirs = library_dirs[:]
        self.libraries = libraries[:]
        self.library_objects = []
        self.extra_compile_args = ["-Wall"] + extra_compile_args
        if config:
            self.configfile = ConfigFile(config)
        else:
            self.configfile = None

    def config(self, line):
        """
        Write a line to the config file.
        """
        if not self.configfile:
            raise AttributeError('No config file defined')
        self.configfile.append(line)

    def add_library(self, name):
        """
        """
        lib = get_library(name)
        if lib:
            self.library_objects.append(lib)
        return False

    def get_library(self, name):
        for lib in self.library_objects:
            if lib.name == name:
                return lib

    def check_library(self, name, minver):
        """
        Check dependencies add add the flags to include_dirs, library_dirs and
        libraries. The basic logic is taken from pygame.
        """
        try:
            lib = Library(name)
            found, version = lib.check(minver)
            if not found:
                if not version:
                    version = 'none'
                raise ValueError, 'requires %s version %s (none found)' % \
                                  (name, minver, version)
            self.library_objects.append(lib)
            return True
        except Exception, e:
            self.error = e

        return False

    def check_cc(self, includes, code, args=''):
        """
        Check the given code with the linker. The optional parameter args
        can contain additional command line options like -l.
        """
        return compile(includes, code, args)

    def convert(self):
        """
        Convert Extension into a distutils.core.Extension.
        """
        library_dirs = self.library_dirs[:]
        include_dirs = self.include_dirs[:]
        libraries = self.libraries[:]
        for lib in self.library_objects:
            library_dirs.extend(lib.library_dirs)
            include_dirs.extend(lib.include_dirs)
            libraries.extend(lib.libraries)

        return distutils.core.Extension(self.output, self.files,
                                        library_dirs=library_dirs,
                                        include_dirs=include_dirs,
                                        libraries=libraries,
                                        extra_compile_args=self.extra_compile_args)

    def __del__(self):
        """
        Delete the config file.
        """
        if self.configfile:
            self.configfile.unlink()

class EmptyExtensionsList(list):
    """
    A list that is non-zero even when empty.  Used for the ext_modules
    kwarg in setup() for modules with no ext_modules.

    This is a kludge to solve a peculiar problem.  On architectures like
    x86_64, distutils will install "pure" modules (i.e. no C extensions)
    under /usr/lib, and platform-specific modules (with extensions) under
    /usr/lib64.  This is a problem for kaa, because with kaa we have a single
    namespace (kaa/) that have several independent modules: that is, each
    module in Kaa can be installed separately, but they coexist under the
    kaa/ directory hierarchy.

    On x86_64, this results in some kaa modules being installed in /usr/lib
    and others installed in /usr/lib64.  The first problem is that
    kaa/__init__.py is provided by kaa.base, so if kaa.base is installed
    in /usr/lib/ then /usr/lib64/python*/site-packages/kaa/__init__.py does
    not exist and therefore we can't import any modules from that.  We could
    drop a dummy __init__.py there, but then python will have two valid kaa
    modules and only ever see one of them, the other will be ignored.

    So this kludge makes distutils always think the module has extensions,
    so it will install in the platform-specific libdir.  As a result, on
    x86_64, all kaa modules will be installed in /usr/lib64, which is what
    we want.
    """
    def __nonzero__(self):
        return True

class GentooEbuild (distutils.core.Command):

    description = "create gentoo ebuild"

    user_options = [
        ('prefix=', None,
         "portage prefix"),
        ]

    def initialize_options (self):
        self.prefix = None

    def finalize_options (self):
        pass

    def run (self):
        if not self.prefix:
            print 'Please provide overlay portage prefix'
            return 0
        name = self.distribution.metadata.name
        version = self.distribution.metadata.version
        if not os.path.isfile('%s.ebuild' % name):
            print 'No ebuild template provided'
            return 0
        fd = open('%s.ebuild' % name)
        edata = fd.readlines()
        fd.close()
        ebuild = '%s/dev-python/%s/%s-%s.ebuild' % (self.prefix, name, name, version)
        if not os.path.isdir(os.path.dirname(ebuild)):
            os.makedirs(os.path.dirname(ebuild))
        fd = open(ebuild, 'w')
        for line in edata:
            fd.write(line)
        fd.close()
        os.system('ebuild %s digest' % ebuild)

def setup(**kwargs):
    """
    A setup script wrapper for kaa modules.
    """
    def _find_packages((kwargs, prefix), dirname, files):
        """
        Helper function to create 'packages' and 'package_dir'.
        """
        if not '__init__.py' in files:
            return
        python_dirname = prefix + dirname[3:].replace('/', '.')
        # Anything under module/src/extensions/foo gets translated to
        # kaa.module.foo.
        python_dirname = python_dirname.replace(".extensions.", ".")
        kwargs['package_dir'][python_dirname] = dirname
        kwargs['packages'].append(python_dirname)

    if not kwargs.has_key('module'):
        raise AttributeError('\'module\' not defined')

    # create name
    kwargs['name'] = 'kaa-' + kwargs['module']

    # search for source files and add it package_dir and packages
    kwargs['package_dir'] = {}
    kwargs['packages']    = []
    if kwargs['module'] == 'base':
        os.path.walk('src', _find_packages, (kwargs, 'kaa'))
    else:
        os.path.walk('src', _find_packages, (kwargs, 'kaa.' + kwargs['module']))

    # convert Extensions
    if kwargs.get('ext_modules'):
        kaa_ext_modules = kwargs['ext_modules']
        ext_modules = []
        for ext in kaa_ext_modules:
            ext_modules.append(ext.convert())
        kwargs['ext_modules'] = ext_modules
    else:
        # No extensions, but trick distutils into thinking we do have, so
        # the module gets installed in the platform-specific libdir.
        kwargs['ext_modules'] = EmptyExtensionsList()

    # check version.py information
    write_version = False
    if kwargs.has_key('version') and not kwargs['module'] == 'base':
        write_version = True
        # check if a version.py is there
        if os.path.isfile('src/version.py'):
            # read the file to find the old version information
            f = open('src/version.py')
            for line in f.readlines():
                if line.startswith('VERSION'):
                    if eval(line[10:-1]) == kwargs['version']:
                        write_version = False
                    break
            f.close()

    # delete 'module' information, not used by distutils.setup
    del kwargs['module']

    if write_version:
        # Write a version.py and add it to the list of files to
        # be installed.
        f = open('src/version.py', 'w')
        f.write('# version information for %s\n' % kwargs['name'])
        f.write('# autogenerated by kaa.distribution\n\n')
        f.write('from kaa.version import Version\n')
        f.write('VERSION = Version(\'%s\')\n' % kwargs['version'])
        f.close()

    # add some missing keywords
    if not kwargs.has_key('author'):
        kwargs['author'] = 'Freevo Development Team'
    if not kwargs.has_key('author_email'):
        kwargs['author_email'] = 'freevo-devel@lists.sourceforge.net'
    if not kwargs.has_key('url'):
        kwargs['url'] = 'http://freevo.sourceforge.net/kaa'

    # add extra commands
    if not 'cmdclass' in kwargs:
        kwargs['cmdclass'] = {}
    kwargs['cmdclass']['build_py'] = build_py
    kwargs['cmdclass']['ebuild'] = GentooEbuild

    # run the distutils.setup function
    return distutils.core.setup(**kwargs)
<commit_msg>Use $CC, $CFLAGS, and $LDFLAGS.<commit_after># -*- coding: iso-8859-1 -*-
# -----------------------------------------------------------------------------
# core.py - distribution core functions for kaa packages
# -----------------------------------------------------------------------------
# $Id$
#
# -----------------------------------------------------------------------------
# Copyright (C) 2006 Dirk Meyer, Jason Tackaberry
#
# First Edition: Dirk Meyer <dmeyer@tzi.de>
# Maintainer:    Dirk Meyer <dmeyer@tzi.de>
#
# Please see the file AUTHORS for a complete list of authors.
#
# This library is free software; you can redistribute it and/or modify
# it under the terms of the GNU Lesser General Public License version
# 2.1 as published by the Free Software Foundation.
#
# This library is distributed in the hope that it will be useful, but
# WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
# Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public
# License along with this library; if not, write to the Free Software
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
# 02110-1301 USA
#
# -----------------------------------------------------------------------------

# python imports
import os
import sys
import math
import stat
import re
import tempfile
import distutils.core

# internal imports
from version import Version
from build_py import build_py

__all__ = ['compile', 'check_library', 'get_library', 'setup', 'ConfigFile',
           'Extension', 'Library']

_libraries = {}

def compile(includes, code, args=''):
    fd, outfile = tempfile.mkstemp()
    os.close(fd)
    args += ' %s %s' % (os.getenv('CFLAGS', ''), os.getenv('LDFLAGS', ''))
    cc = os.getenv('CC', 'cc')
    f = os.popen("%s -x c - -o %s %s 2>/dev/null >/dev/null" % (cc, outfile, args), "w")
    if not f:
        return False

    for i in includes:
        f.write('#include %s\n' % i)
    f.write('int main() { ' + code + '\nreturn 0;\n};\n')
    result = f.close()

    if os.path.exists(outfile):
        os.unlink(outfile)

    if result == None:
        return True
    return False

def check_library(name, *args):
    lib = Library(name)
    if len(args) < 2:
        minver = args[0]
        print 'checking for', name, '>=', minver, '...',
        sys.__stdout__.flush()
        found, version = lib.check(minver)
        if found:
            print version
        elif version:
            print 'no (%s)' % version
        else:
            print 'no'
    else:
        print 'checking for', name, '...',
        sys.__stdout__.flush()
        if lib.compile(*args):
            print 'ok'
        else:
            print 'no'
    _libraries[name] = lib
    return lib

def get_library(name):
    lib = _libraries.get(name)
    if lib and lib.valid:
        return lib
    return None

class Library(object):
    def __init__(self, name):
        self.name = name
        self.include_dirs = []
        self.library_dirs = []
        self.libraries = []
        self.version = None
        self.valid = False

    def compare_versions(self, a, b):
        # Get maximum component length in both version.
        maxlen =  max([ len(x) for x in (a + '.' + b).split('.') ])
        # Pad each component of A and B to maxlen
        a = [ x.zfill(maxlen) for x in a.split('.') ]
        b = [ x.zfill(maxlen) for x in b.split('.') ]
        # TODO: special handling of rc and beta (others?) suffixes, so
        # that 1.0.0 > 1.0.0rc1
        return cmp(a, b)

    def get_numeric_version(self, version = None):
        """
        Returns an integer version suitable for numeric comparison.  This
        returns the version of the library gotten after calling check().  If
        check() was not called this method will raise an exception.

        This algorithm is pretty naive and won't work at all if the version
        contains any non-numeric characters.
        """
        if version is None:
            version = self.version

        if version is None:
            raise ValueError, "Version is not known; call check() first"

        if not version.replace(".", "").isdigit():
            raise ValueError, "Version cannot have non-numeric characters."

        # Performs: 1.2.3.4 => 4<<0 + 3<<9 + 2<<18 + 1<<27
        return sum([ int(v) << 9*s for s,v in enumerate(reversed(version.split('.'))) ])

    def check(self, minver):
        """
        Check dependencies add add the flags to include_dirs, library_dirs and
        libraries. The basic logic is taken from pygame.
        """
        if os.system("%s-config --version >/dev/null 2>/dev/null" % self.name) == 0:
            # Use foo-config if it exists.
            command = "%s-config %%s 2>/dev/null" % self.name
            version_arg = "--version"
        elif os.system("pkg-config %s --exists >/dev/null 2>/dev/null" % self.name) == 0:
            # Otherwise try pkg-config foo.
            command = "pkg-config %s %%s 2>/dev/null" % self.name
            version_arg = "--modversion"
        else:
            return False, None

        version = os.popen(command % version_arg).read().strip()
        if len(version) == 0:
            return False, None
        if minver and self.compare_versions(minver, version) > 0:
            return False, version

        for inc in os.popen(command % "--cflags").read().strip().split(' '):
            if inc[2:] and not inc[2:] in self.include_dirs:
                self.include_dirs.append(inc[2:])

        for flag in os.popen(command % "--libs").read().strip().split(' '):
            if flag[:2] == '-L' and not flag[2:] in self.library_dirs:
                self.library_dirs.append(flag[2:])
            if flag[:2] == '-l' and not flag[2:] in self.libraries:
                self.libraries.append(flag[2:])

        self.version = version
        self.valid = True
        return True, version

    def compile(self, includes, code, args=''):
        if compile(includes, code, args):
            self.valid = True
            return True

        return False

class ConfigFile(object):
    """
    Config file for the build process.
    """
    def __init__(self, filename):
        self.file = os.path.abspath(filename)
        # create config file
        open(self.file, 'w').close()

    def append(self, line):
        """
        Append something to the config file.
        """
        f = open(self.file, 'a')
        f.write(line + '\n')
        f.close()

    def define(self, variable, value=None):
        """
        Set a #define.
        """
        if value == None:
            self.append('#define %s' % variable)
        else:
            self.append('#define %s %s' % (variable, value))

    def unlink(self):
        """
        Delete config file.
        """
        os.unlink(self.file)

class Extension(object):
    """
    Extension wrapper with additional functions to find libraries and
    support for config files.
    """
    def __init__(self, output, files, include_dirs=[], library_dirs=[],
                 libraries=[], extra_compile_args = [], config=None):
        """
        Init the Extension object.
        """
        self.output = output
        self.files = files
        self.include_dirs = include_dirs[:]
        self.library_dirs = library_dirs[:]
        self.libraries = libraries[:]
        self.library_objects = []
        self.extra_compile_args = ["-Wall"] + extra_compile_args
        if config:
            self.configfile = ConfigFile(config)
        else:
            self.configfile = None

    def config(self, line):
        """
        Write a line to the config file.
        """
        if not self.configfile:
            raise AttributeError('No config file defined')
        self.configfile.append(line)

    def add_library(self, name):
        """
        """
        lib = get_library(name)
        if lib:
            self.library_objects.append(lib)
        return False

    def get_library(self, name):
        for lib in self.library_objects:
            if lib.name == name:
                return lib

    def check_library(self, name, minver):
        """
        Check dependencies add add the flags to include_dirs, library_dirs and
        libraries. The basic logic is taken from pygame.
        """
        try:
            lib = Library(name)
            found, version = lib.check(minver)
            if not found:
                if not version:
                    version = 'none'
                raise ValueError, 'requires %s version %s (none found)' % \
                                  (name, minver, version)
            self.library_objects.append(lib)
            return True
        except Exception, e:
            self.error = e

        return False

    def check_cc(self, includes, code, args=''):
        """
        Check the given code with the linker. The optional parameter args
        can contain additional command line options like -l.
        """
        return compile(includes, code, args)

    def convert(self):
        """
        Convert Extension into a distutils.core.Extension.
        """
        library_dirs = self.library_dirs[:]
        include_dirs = self.include_dirs[:]
        libraries = self.libraries[:]
        for lib in self.library_objects:
            library_dirs.extend(lib.library_dirs)
            include_dirs.extend(lib.include_dirs)
            libraries.extend(lib.libraries)

        return distutils.core.Extension(self.output, self.files,
                                        library_dirs=library_dirs,
                                        include_dirs=include_dirs,
                                        libraries=libraries,
                                        extra_compile_args=self.extra_compile_args)

    def __del__(self):
        """
        Delete the config file.
        """
        if self.configfile:
            self.configfile.unlink()

class EmptyExtensionsList(list):
    """
    A list that is non-zero even when empty.  Used for the ext_modules
    kwarg in setup() for modules with no ext_modules.

    This is a kludge to solve a peculiar problem.  On architectures like
    x86_64, distutils will install "pure" modules (i.e. no C extensions)
    under /usr/lib, and platform-specific modules (with extensions) under
    /usr/lib64.  This is a problem for kaa, because with kaa we have a single
    namespace (kaa/) that have several independent modules: that is, each
    module in Kaa can be installed separately, but they coexist under the
    kaa/ directory hierarchy.

    On x86_64, this results in some kaa modules being installed in /usr/lib
    and others installed in /usr/lib64.  The first problem is that
    kaa/__init__.py is provided by kaa.base, so if kaa.base is installed
    in /usr/lib/ then /usr/lib64/python*/site-packages/kaa/__init__.py does
    not exist and therefore we can't import any modules from that.  We could
    drop a dummy __init__.py there, but then python will have two valid kaa
    modules and only ever see one of them, the other will be ignored.

    So this kludge makes distutils always think the module has extensions,
    so it will install in the platform-specific libdir.  As a result, on
    x86_64, all kaa modules will be installed in /usr/lib64, which is what
    we want.
    """
    def __nonzero__(self):
        return True

class GentooEbuild (distutils.core.Command):

    description = "create gentoo ebuild"

    user_options = [
        ('prefix=', None,
         "portage prefix"),
        ]

    def initialize_options (self):
        self.prefix = None

    def finalize_options (self):
        pass

    def run (self):
        if not self.prefix:
            print 'Please provide overlay portage prefix'
            return 0
        name = self.distribution.metadata.name
        version = self.distribution.metadata.version
        if not os.path.isfile('%s.ebuild' % name):
            print 'No ebuild template provided'
            return 0
        fd = open('%s.ebuild' % name)
        edata = fd.readlines()
        fd.close()
        ebuild = '%s/dev-python/%s/%s-%s.ebuild' % (self.prefix, name, name, version)
        if not os.path.isdir(os.path.dirname(ebuild)):
            os.makedirs(os.path.dirname(ebuild))
        fd = open(ebuild, 'w')
        for line in edata:
            fd.write(line)
        fd.close()
        os.system('ebuild %s digest' % ebuild)

def setup(**kwargs):
    """
    A setup script wrapper for kaa modules.
    """
    def _find_packages((kwargs, prefix), dirname, files):
        """
        Helper function to create 'packages' and 'package_dir'.
        """
        if not '__init__.py' in files:
            return
        python_dirname = prefix + dirname[3:].replace('/', '.')
        # Anything under module/src/extensions/foo gets translated to
        # kaa.module.foo.
        python_dirname = python_dirname.replace(".extensions.", ".")
        kwargs['package_dir'][python_dirname] = dirname
        kwargs['packages'].append(python_dirname)

    if not kwargs.has_key('module'):
        raise AttributeError('\'module\' not defined')

    # create name
    kwargs['name'] = 'kaa-' + kwargs['module']

    # search for source files and add it package_dir and packages
    kwargs['package_dir'] = {}
    kwargs['packages']    = []
    if kwargs['module'] == 'base':
        os.path.walk('src', _find_packages, (kwargs, 'kaa'))
    else:
        os.path.walk('src', _find_packages, (kwargs, 'kaa.' + kwargs['module']))

    # convert Extensions
    if kwargs.get('ext_modules'):
        kaa_ext_modules = kwargs['ext_modules']
        ext_modules = []
        for ext in kaa_ext_modules:
            ext_modules.append(ext.convert())
        kwargs['ext_modules'] = ext_modules
    else:
        # No extensions, but trick distutils into thinking we do have, so
        # the module gets installed in the platform-specific libdir.
        kwargs['ext_modules'] = EmptyExtensionsList()

    # check version.py information
    write_version = False
    if kwargs.has_key('version') and not kwargs['module'] == 'base':
        write_version = True
        # check if a version.py is there
        if os.path.isfile('src/version.py'):
            # read the file to find the old version information
            f = open('src/version.py')
            for line in f.readlines():
                if line.startswith('VERSION'):
                    if eval(line[10:-1]) == kwargs['version']:
                        write_version = False
                    break
            f.close()

    # delete 'module' information, not used by distutils.setup
    del kwargs['module']

    if write_version:
        # Write a version.py and add it to the list of files to
        # be installed.
        f = open('src/version.py', 'w')
        f.write('# version information for %s\n' % kwargs['name'])
        f.write('# autogenerated by kaa.distribution\n\n')
        f.write('from kaa.version import Version\n')
        f.write('VERSION = Version(\'%s\')\n' % kwargs['version'])
        f.close()

    # add some missing keywords
    if not kwargs.has_key('author'):
        kwargs['author'] = 'Freevo Development Team'
    if not kwargs.has_key('author_email'):
        kwargs['author_email'] = 'freevo-devel@lists.sourceforge.net'
    if not kwargs.has_key('url'):
        kwargs['url'] = 'http://freevo.sourceforge.net/kaa'

    # add extra commands
    if not 'cmdclass' in kwargs:
        kwargs['cmdclass'] = {}
    kwargs['cmdclass']['build_py'] = build_py
    kwargs['cmdclass']['ebuild'] = GentooEbuild

    # run the distutils.setup function
    return distutils.core.setup(**kwargs)
<|endoftext|>