<commit_before>#!/usr/bin/python
# -*- coding: utf-8 -*-
import os
import logging
import tempfile
import gzip
import shutil
import time
import csv
import copy
import sys
from mtgraphite import MTGraphiteClient
import json
import multiprocessing
import Queue
from crawler_exceptions import (EmitterUnsupportedFormat,
                                EmitterUnsupportedProtocol,
                                EmitterBadURL,
                                EmitterEmitTimeout)
# External dependencies that must be pip install'ed separately

import kafka as kafka_python
import pykafka
import requests

from misc import NullHandler

logger = logging.getLogger('crawlutils')

def kafka_send(kurl, temp_fpath, format, topic, queue=None):
    try:
        kafka_python_client = kafka_python.KafkaClient(kurl)
        kafka_python_client.ensure_topic_exists(topic)
        kafka = pykafka.KafkaClient(hosts=kurl)

        publish_topic_object = kafka.topics[topic]
        # the default partitioner is random_partitioner
        producer = publish_topic_object.get_producer()

        if format == 'csv':
            with open(temp_fpath, 'r') as fp:
                text = fp.read()
                producer.produce([text])
        elif format == 'graphite':
            with open(temp_fpath, 'r') as fp:
                for line in fp.readlines():
                    producer.produce([line])
        else:
            raise EmitterUnsupportedFormat('Unsupported format: %s' % format)

        queue and queue.put((True, None))
    except Exception as e:
        if queue:
            queue.put((False, e))
        else:
            raise
    finally:
        queue and queue.close()

class Emitter:

    """Class that abstracts the outputs supported by the crawler, like
    stdout, or kafka.

    An object of this class is created for every frame emitted. A frame is
    emitted for every container and at every crawling interval.
    """

    # We want to use a global to store the MTGraphite client class so it
    # persists across metric intervals.

    mtgclient = None

    kafka_timeout_secs = 30

    # Debugging TIP: use url='file://<local-file>' to emit the frame data into
    # a local file

    def __init__(
        self,
        urls,
        emitter_args={},
        format='csv',
        max_emit_retries=9,
        kafka_timeout_secs=30
    ):

        self.urls = urls
        self.emitter_args = emitter_args
        self.compress = emitter_args.get('compress', False)
        self.format = format
        self.max_emit_retries = max_emit_retries
        self.mtgclient = None
        self.kafka_timeout_secs = kafka_timeout_secs

    def __enter__(self):
        (self.temp_fd, self.temp_fpath) = \
            tempfile.mkstemp(prefix='emit.')
        os.close(self.temp_fd)  # close temporary file descriptor

        # as we open immediately
        # need to find a better fix later

        if self.compress:
            self.emitfile = gzip.open(self.temp_fpath, 'wb')
        else:
            self.emitfile = open(self.temp_fpath, 'wb')
        self.csv_writer = csv.writer(self.emitfile, delimiter='\t',
                                     quotechar="'")
        self.begin_time = time.time()
        self.num_features = 0
        return self

    def emit_dict_as_graphite(
        self,
        sysname,
        group,
        suffix,
        data,
        timestamp=None,
    ):
        timestamp = int(timestamp or time.time())
        items = data.items()

        # this is for issue #343

        sysname = sysname.replace('/', '.')

        for (metric, value) in items:
            try:
                value = float(value)
            except Exception:

                # value was not a float or anything that looks like a float

                continue

            metric = metric.replace('(', '_').replace(')', '')
            metric = metric.replace(' ', '_').replace('-', '_')
            metric = metric.replace('/', '_').replace('\\', '_')

            suffix = suffix.replace('_', '-')
            if 'cpu' in suffix or 'memory' in suffix:
                metric = metric.replace('_', '-')
            if 'if' in metric:
                metric = metric.replace('_tx', '.tx')
                metric = metric.replace('_rx', '.rx')
            if suffix == 'load':
                suffix = 'load.load'
            suffix = suffix.replace('/', '$')

            tmp_message = '%s.%s.%s %f %d\r\n' % (sysname, suffix,
                                                  metric, value, timestamp)
            self.emitfile.write(tmp_message)

    # Added optional feature_type so that we can bypass feature type discovery
    # for FILE crawlmode

    def emit(
        self,
        feature_key,
        feature_val,
        feature_type=None,
    ):

        # Add metadata as first feature

        if self.num_features == 0:
            metadata = copy.deepcopy(self.emitter_args)

            # Update timestamp to the actual emit time

            metadata['timestamp'] = \
                time.strftime('%Y-%m-%dT%H:%M:%S%z')
            if 'extra' in metadata:
                del metadata['extra']
                if self.emitter_args['extra']:
                    metadata.update(json.loads(self.emitter_args['extra'
                                                                 ]))
            if 'extra_all_features' in metadata:
                del metadata['extra_all_features']
            if self.format == 'csv' or self.format == 'json':
                self.csv_writer.writerow(
                    ['metadata',
                     json.dumps('metadata'),
                     json.dumps(metadata,
                                separators=(',', ':'))])
                self.num_features += 1

        if isinstance(feature_val, dict):
            feature_val_as_dict = feature_val
        else:
            feature_val_as_dict = feature_val._asdict()
        if 'extra' in self.emitter_args and self.emitter_args['extra'] \
                and 'extra_all_features' in self.emitter_args \
                and self.emitter_args['extra_all_features']:
            feature_val_as_dict.update(json.loads(self.emitter_args['extra'
                                                                    ]))
        if self.format == 'csv' or self.format == 'json':
            self.csv_writer.writerow(
                [feature_type,
                 json.dumps(feature_key),
                 json.dumps(feature_val_as_dict,
                            separators=(',', ':'))])
        elif self.format == 'graphite':
            if 'namespace' in self.emitter_args:
                namespace = self.emitter_args['namespace']
            else:
                namespace = 'undefined'
            self.emit_dict_as_graphite(
                namespace,
                feature_type,
                feature_key,
                feature_val_as_dict)
        else:
            raise EmitterUnsupportedFormat(
                'Unsupported format: %s' % self.format)
        self.num_features += 1

    def _close_file(self):

        # close the output file

        self.emitfile.close()

    def _publish_to_stdout(self):
        if self.format == 'json':
            raise NotImplementedError('json format is not supported')
        with open(self.temp_fpath, 'r') as fd:
            if self.compress:
                print '%s' % fd.read()
            else:
                for line in fd.readlines():
                    print line.strip()
                    sys.stdout.flush()

    def __make_http_post(self, url, headers, payload, max_emit_retries):
        for attempt in range(max_emit_retries):
            try:
                response = requests.post(url, headers=headers,
                                         params=self.emitter_args,
                                         data=payload)
            except requests.exceptions.ChunkedEncodingError as e:
                logger.exception(e)
                logger.error(
                    "POST to %s resulted in exception (attempt %d of %d), "
                    "will not re-try" % (url, attempt + 1, max_emit_retries))
                break
            except requests.exceptions.RequestException as e:
                logger.exception(e)
                logger.error(
                    "POST to %s resulted in exception (attempt %d of %d)" %
                    (url, attempt + 1, max_emit_retries))
                time.sleep(2.0 ** attempt * 0.1)
                continue
            if response.status_code != requests.codes.ok:
                logger.error("POST to %s resulted in status code %s: %s "
                             "(attempt %d of %d)" %
                             (url, str(response.status_code),
                              response.text, attempt + 1, max_emit_retries))
                time.sleep(2.0 ** attempt * 0.1)
            else:
                break

    def _publish_to_http(self, url, max_emit_retries=5):
        namespace = None
        headers = {}
        if self.compress:
            headers['content-encoding'] = 'gzip'
        with open(self.temp_fpath, 'rb') as framefp:
            if self.format == 'json':
                headers = {'content-type': 'application/json'}
                for feature in framefp:
                    feature_parts = feature.split()
                    if len(feature_parts) < 3:
                        logger.error(
                            "Invalid feature data found. %s %d" %
                            (feature_parts, len(feature_parts)))
                        continue

                    feature_name = feature_parts[0]
                    feature_value = feature_parts[1].strip("\"")

                    feature_data = json.loads("".join(feature_parts[2:]))

                    if feature_name == "metadata":
                        namespace = feature_data.get('namespace', None)
                    else:
                        feature_data['namespace'] = namespace
                    feature_data[feature_name] = feature_value
                    payload = json.dumps(feature_data)
                    self.__make_http_post(
                        url, headers, payload, max_emit_retries)

            elif self.format == 'csv' or self.format == 'graphite':
                headers = {'content-type': 'application/csv'}
                self.__make_http_post(url, headers, framefp, max_emit_retries)
            else:
                raise EmitterUnsupportedFormat(
                    'Unsupported format: %s' % self.format)

    def _publish_to_kafka_no_retries(self, url):

        list = url[len('kafka://'):].split('/')

        if len(list) == 2:
            kurl, topic = list
        else:
            raise EmitterBadURL(
                'The kafka url provided does not seem to be valid: %s. '
                'It should be something like this: '
                'kafka://[ip|hostname]:[port]/[kafka_topic]. '
                'For example: kafka://1.1.1.1:1234/metrics' % url)

        # Kafka logs too much
        h = NullHandler()
        logging.getLogger('kafka').addHandler(h)

        queue = multiprocessing.Queue()
        try:
            child_process = multiprocessing.Process(
                name='kafka-emitter', target=kafka_send, args=(
                    kurl, self.temp_fpath, self.format, topic, queue))
            child_process.start()
        except OSError:
            queue.close()
            raise

        try:
            (result, child_exception) = queue.get(
                timeout=self.kafka_timeout_secs)
        except Queue.Empty:
            child_exception = EmitterEmitTimeout()

        child_process.join(self.kafka_timeout_secs)

        if child_process.is_alive():
            errmsg = ('Timed out waiting for process %d to exit.' %
                      child_process.pid)
            queue.close()
            os.kill(child_process.pid, 9)
            logger.error(errmsg)
            raise EmitterEmitTimeout(errmsg)

        if child_exception:
            raise child_exception

    def _publish_to_kafka(self, url, max_emit_retries=8):
        if self.format == 'json':
            raise NotImplementedError('json format is not supported')

        broker_alive = False
        retries = 0
        while not broker_alive and retries <= max_emit_retries:
            try:
                retries += 1
                self._publish_to_kafka_no_retries(url)
                broker_alive = True
            except Exception as e:
                logger.debug(
                    '_publish_to_kafka_no_retries {0}: {1}'.format(
                        url, e))
                if retries <= max_emit_retries:

                    # Wait for (2^retries * 100) milliseconds

                    wait_time = 2.0 ** retries * 0.1
                    logger.error(
                        'Could not connect to the kafka server at %s. Retry '
                        'in %f seconds.' % (url, wait_time))
                    time.sleep(wait_time)
                else:
                    raise e

    def _publish_to_mtgraphite(self, url):
        if self.format == 'json':
            raise NotImplementedError('json format is not supported')
        if not Emitter.mtgclient:
            Emitter.mtgclient = MTGraphiteClient(url)
        with open(self.temp_fpath, 'r') as fp:
            num_pushed_to_queue = \
                Emitter.mtgclient.send_messages(fp.readlines())
            logger.debug('Pushed %d messages to mtgraphite queue'
                         % num_pushed_to_queue)

    def _write_to_file(self, url):
        if self.format == 'json':
            raise NotImplementedError('json format is not supported')
        output_path = url[len('file://'):]
        if self.compress:
            output_path += '.gz'
        shutil.move(self.temp_fpath, output_path)

    def __exit__(
        self,
        typ,
        exc,
        trc,
    ):
        if exc:
            self._close_file()
            if os.path.exists(self.temp_fpath):
                os.remove(self.temp_fpath)
            raise exc
        try:
            self._close_file()
            for url in self.urls:
                logger.debug('Emitting frame to {0}'.format(url))
                if url.startswith('stdout://'):
                    self._publish_to_stdout()
                elif url.startswith('http://'):
                    self._publish_to_http(url, self.max_emit_retries)
                elif url.startswith('file://'):
                    self._write_to_file(url)
                elif url.startswith('kafka://'):
                    self._publish_to_kafka(url, self.max_emit_retries)
                elif url.startswith('mtgraphite://'):
                    self._publish_to_mtgraphite(url)
                else:
                    if os.path.exists(self.temp_fpath):
                        os.remove(self.temp_fpath)
                    raise EmitterUnsupportedProtocol(
                        'Unsupported URL protocol {0}'.format(url))
        finally:
            if os.path.exists(self.temp_fpath):
                os.remove(self.temp_fpath)
        self.end_time = time.time()
        elapsed_time = self.end_time - self.begin_time
        logger.info(
            'Emitted {0} features in {1} seconds'.format(
                self.num_features,
                elapsed_time))
<commit_msg>emitter cleanup<commit_after>#!/usr/bin/python
# -*- coding: utf-8 -*-
import os
import logging
import tempfile
import gzip
import shutil
import time
import csv
import copy
import sys
from mtgraphite import MTGraphiteClient
import json
import multiprocessing
import Queue
from crawler_exceptions import (EmitterUnsupportedFormat,
                                EmitterUnsupportedProtocol,
                                EmitterBadURL,
                                EmitterEmitTimeout)
# External dependencies that must be pip install'ed separately

import kafka as kafka_python
import pykafka
import requests

from misc import NullHandler

logger = logging.getLogger('crawlutils')

def kafka_send(kurl, temp_fpath, format, topic, queue=None):
    try:
        kafka_python_client = kafka_python.KafkaClient(kurl)
        kafka_python_client.ensure_topic_exists(topic)
        kafka = pykafka.KafkaClient(hosts=kurl)

        publish_topic_object = kafka.topics[topic]
        # the default partitioner is random_partitioner
        producer = publish_topic_object.get_producer()

        if format == 'csv':
            with open(temp_fpath, 'r') as fp:
                text = fp.read()
                producer.produce([text])
        elif format == 'graphite':
            with open(temp_fpath, 'r') as fp:
                for line in fp.readlines():
                    producer.produce([line])
        else:
            raise EmitterUnsupportedFormat('Unsupported format: %s' % format)

        queue and queue.put((True, None))
    except Exception as e:
        if queue:
            queue.put((False, e))
        else:
            raise
    finally:
        queue and queue.close()

class Emitter:

    """Class that abstracts the outputs supported by the crawler, like
    stdout, or kafka.

    An object of this class is created for every frame emitted. A frame is
    emitted for every container and at every crawling interval.
    """

    # We want to use a global to store the MTGraphite client class so it
    # persists across metric intervals.

    mtgclient = None

    kafka_timeout_secs = 30

    # Debugging TIP: use url='file://<local-file>' to emit the frame data into
    # a local file

    def __init__(
        self,
        urls,
        emitter_args={},
        format='csv',
        max_emit_retries=9,
        kafka_timeout_secs=30
    ):

        self.urls = urls
        self.emitter_args = emitter_args
        self.compress = emitter_args.get('compress', False)
        self.format = format
        self.max_emit_retries = max_emit_retries
        self.mtgclient = None
        self.kafka_timeout_secs = kafka_timeout_secs

    def __enter__(self):
        (self.temp_fd, self.temp_fpath) = \
            tempfile.mkstemp(prefix='emit.')
        os.close(self.temp_fd)  # close temporary file descriptor

        # as we open immediately
        # need to find a better fix later

        if self.compress:
            self.emitfile = gzip.open(self.temp_fpath, 'wb')
        else:
            self.emitfile = open(self.temp_fpath, 'wb')
        self.csv_writer = csv.writer(self.emitfile, delimiter='\t',
                                     quotechar="'")
        self.begin_time = time.time()
        self.num_features = 0
        return self

    def emit_dict_as_graphite(
        self,
        sysname,
        group,
        suffix,
        data,
        timestamp=None,
    ):
        timestamp = int(timestamp or time.time())
        items = data.items()

        # this is for issue #343

        sysname = sysname.replace('/', '.')

        for (metric, value) in items:
            try:
                value = float(value)
            except Exception:

                # value was not a float or anything that looks like a float

                continue

            metric = metric.replace('(', '_').replace(')', '')
            metric = metric.replace(' ', '_').replace('-', '_')
            metric = metric.replace('/', '_').replace('\\', '_')

            suffix = suffix.replace('_', '-')
            if 'cpu' in suffix or 'memory' in suffix:
                metric = metric.replace('_', '-')
            if 'if' in metric:
                metric = metric.replace('_tx', '.tx')
                metric = metric.replace('_rx', '.rx')
            if suffix == 'load':
                suffix = 'load.load'
            suffix = suffix.replace('/', '$')

            tmp_message = '%s.%s.%s %f %d\r\n' % (sysname, suffix,
                                                  metric, value, timestamp)
            self.emitfile.write(tmp_message)

    def _emit_metadata_feature(self):
        metadata = copy.deepcopy(self.emitter_args)

        # Update timestamp to the actual emit time

        metadata['timestamp'] = \
            time.strftime('%Y-%m-%dT%H:%M:%S%z')
        if 'extra' in metadata:
            del metadata['extra']
            if self.emitter_args['extra']:
                metadata.update(json.loads(self.emitter_args['extra'
                                                             ]))
        if 'extra_all_features' in metadata:
            del metadata['extra_all_features']
        if self.format == 'csv' or self.format == 'json':
            self.csv_writer.writerow(
                ['metadata',
                 json.dumps('metadata'),
                 json.dumps(metadata,
                            separators=(',', ':'))])
            self.num_features += 1

    def emit(
        self,
        feature_key,
        feature_val,
        feature_type=None,
    ):

        if self.num_features == 0:
            self._emit_metadata_feature()

        if isinstance(feature_val, dict):
            feature_val_as_dict = feature_val
        else:
            feature_val_as_dict = feature_val._asdict()

        if 'extra' in self.emitter_args and self.emitter_args['extra'] \
                and 'extra_all_features' in self.emitter_args \
                and self.emitter_args['extra_all_features']:
            feature_val_as_dict.update(json.loads(self.emitter_args['extra'
                                                                    ]))
        if self.format == 'csv' or self.format == 'json':
            self.csv_writer.writerow(
                [feature_type,
                 json.dumps(feature_key),
                 json.dumps(feature_val_as_dict,
                            separators=(',', ':'))])
        elif self.format == 'graphite':
            if 'namespace' in self.emitter_args:
                namespace = self.emitter_args['namespace']
            else:
                namespace = 'undefined'
            self.emit_dict_as_graphite(
                namespace,
                feature_type,
                feature_key,
                feature_val_as_dict)
        else:
            raise EmitterUnsupportedFormat(
                'Unsupported format: %s' % self.format)
        self.num_features += 1

    def _close_file(self):

        # close the output file

        self.emitfile.close()

    def _publish_to_stdout(self):
        if self.format == 'json':
            raise NotImplementedError('json format is not supported')
        with open(self.temp_fpath, 'r') as fd:
            if self.compress:
                print '%s' % fd.read()
            else:
                for line in fd.readlines():
                    print line.strip()
                    sys.stdout.flush()

    def __make_http_post(self, url, headers, payload, max_emit_retries):
        for attempt in range(max_emit_retries):
            try:
                response = requests.post(url, headers=headers,
                                         params=self.emitter_args,
                                         data=payload)
            except requests.exceptions.ChunkedEncodingError as e:
                logger.exception(e)
                logger.error(
                    "POST to %s resulted in exception (attempt %d of %d), "
                    "will not re-try" % (url, attempt + 1, max_emit_retries))
                break
            except requests.exceptions.RequestException as e:
                logger.exception(e)
                logger.error(
                    "POST to %s resulted in exception (attempt %d of %d)" %
                    (url, attempt + 1, max_emit_retries))
                time.sleep(2.0 ** attempt * 0.1)
                continue
            if response.status_code != requests.codes.ok:
                logger.error("POST to %s resulted in status code %s: %s "
                             "(attempt %d of %d)" %
                             (url, str(response.status_code),
                              response.text, attempt + 1, max_emit_retries))
                time.sleep(2.0 ** attempt * 0.1)
            else:
                break

    def _publish_to_http(self, url, max_emit_retries=5):
        namespace = None
        headers = {}
        if self.compress:
            headers['content-encoding'] = 'gzip'
        with open(self.temp_fpath, 'rb') as framefp:
            if self.format == 'json':
                headers = {'content-type': 'application/json'}
                for feature in framefp:
                    feature_parts = feature.split()
                    if len(feature_parts) < 3:
                        logger.error(
                            "Invalid feature data found. %s %d" %
                            (feature_parts, len(feature_parts)))
                        continue

                    feature_name = feature_parts[0]
                    feature_value = feature_parts[1].strip("\"")

                    feature_data = json.loads("".join(feature_parts[2:]))

                    if feature_name == "metadata":
                        namespace = feature_data.get('namespace', None)
                    else:
                        feature_data['namespace'] = namespace
                    feature_data[feature_name] = feature_value
                    payload = json.dumps(feature_data)
                    self.__make_http_post(
                        url, headers, payload, max_emit_retries)

            elif self.format == 'csv' or self.format == 'graphite':
                headers = {'content-type': 'application/csv'}
                self.__make_http_post(url, headers, framefp, max_emit_retries)
            else:
                raise EmitterUnsupportedFormat(
                    'Unsupported format: %s' % self.format)

    def _publish_to_kafka_no_retries(self, url):

        list = url[len('kafka://'):].split('/')

        if len(list) == 2:
            kurl, topic = list
        else:
            raise EmitterBadURL(
                'The kafka url provided does not seem to be valid: %s. '
                'It should be something like this: '
                'kafka://[ip|hostname]:[port]/[kafka_topic]. '
                'For example: kafka://1.1.1.1:1234/metrics' % url)

        # Kafka logs too much
        h = NullHandler()
        logging.getLogger('kafka').addHandler(h)

        queue = multiprocessing.Queue()
        try:
            child_process = multiprocessing.Process(
                name='kafka-emitter', target=kafka_send, args=(
                    kurl, self.temp_fpath, self.format, topic, queue))
            child_process.start()
        except OSError:
            queue.close()
            raise

        try:
            (result, child_exception) = queue.get(
                timeout=self.kafka_timeout_secs)
        except Queue.Empty:
            child_exception = EmitterEmitTimeout()

        child_process.join(self.kafka_timeout_secs)

        if child_process.is_alive():
            errmsg = ('Timed out waiting for process %d to exit.' %
                      child_process.pid)
            queue.close()
            os.kill(child_process.pid, 9)
            logger.error(errmsg)
            raise EmitterEmitTimeout(errmsg)

        if child_exception:
            raise child_exception

    def _publish_to_kafka(self, url, max_emit_retries=8):
        if self.format == 'json':
            raise NotImplementedError('json format is not supported')

        broker_alive = False
        retries = 0
        while not broker_alive and retries <= max_emit_retries:
            try:
                retries += 1
                self._publish_to_kafka_no_retries(url)
                broker_alive = True
            except Exception as e:
                logger.debug(
                    '_publish_to_kafka_no_retries {0}: {1}'.format(
                        url, e))
                if retries <= max_emit_retries:

                    # Wait for (2^retries * 100) milliseconds

                    wait_time = 2.0 ** retries * 0.1
                    logger.error(
                        'Could not connect to the kafka server at %s. Retry '
                        'in %f seconds.' % (url, wait_time))
                    time.sleep(wait_time)
                else:
                    raise e

    def _publish_to_mtgraphite(self, url):
        if self.format == 'json':
            raise NotImplementedError('json format is not supported')
        if not Emitter.mtgclient:
            Emitter.mtgclient = MTGraphiteClient(url)
        with open(self.temp_fpath, 'r') as fp:
            num_pushed_to_queue = \
                Emitter.mtgclient.send_messages(fp.readlines())
            logger.debug('Pushed %d messages to mtgraphite queue'
                         % num_pushed_to_queue)

    def _write_to_file(self, url):
        if self.format == 'json':
            raise NotImplementedError('json format is not supported')
        output_path = url[len('file://'):]
        if self.compress:
            output_path += '.gz'
        shutil.move(self.temp_fpath, output_path)

    def _publish(self, url):
        logger.debug('Emitting frame to {0}'.format(url))
        if url.startswith('stdout://'):
            self._publish_to_stdout()
        elif url.startswith('http://'):
            self._publish_to_http(url, self.max_emit_retries)
        elif url.startswith('file://'):
            self._write_to_file(url)
        elif url.startswith('kafka://'):
            self._publish_to_kafka(url, self.max_emit_retries)
        elif url.startswith('mtgraphite://'):
            self._publish_to_mtgraphite(url)
        else:
            if os.path.exists(self.temp_fpath):
                os.remove(self.temp_fpath)
            raise EmitterUnsupportedProtocol(
                'Unsupported URL protocol {0}'.format(url))

    def __exit__(
        self,
        typ,
        exc,
        trc,
    ):
        if exc:
            self._close_file()
            if os.path.exists(self.temp_fpath):
                os.remove(self.temp_fpath)
            raise exc

        try:
            self._close_file()
            for url in self.urls:
                self._publish(url)
        finally:
            if os.path.exists(self.temp_fpath):
                os.remove(self.temp_fpath)

        self.end_time = time.time()
        elapsed_time = self.end_time - self.begin_time
        logger.info(
            'Emitted {0} features in {1} seconds'.format(
                self.num_features,
                elapsed_time))
<|endoftext|>


<commit_before>import numpy as np
import os
import os.path
import shutil

from opensfm import config
from opensfm import io
from opensfm.dataset import DataSet

class MetaDataSet():
    def __init__(self, data_path):
        '''
        Create meta dataset instance for large scale reconstruction.

        :param data_path: Path to directory containing meta dataset
        '''
        self.data_path = os.path.abspath(data_path)

        config_file = os.path.join(self.data_path, 'config.yaml')
        self.config = config.load_config(config_file)

        self._image_list_file_name = 'image_list_with_gps.tsv'
        self._clusters_file_name = 'clusters.npz'
        self._clusters_with_neighbors_file_name = 'clusters_with_neighbors.npz'
        self._clusters_with_neighbors_geojson_file_name = 'clusters_with_neighbors.geojson'
        self._clusters_geojson_file_name = 'clusters.geojson'

        io.mkdir_p(self._submodels_path())

    def _submodels_path(self):
        return os.path.join(self.data_path, self.config['submodels_relpath'])

    def _submodel_path(self, i):
        """Path to submodel i folder."""
        template = self.config['submodel_relpath_template']
        return os.path.join(self.data_path, template % i)

    def _submodel_images_path(self, i):
        """Path to submodel i images folder."""
        template = self.config['submodel_images_relpath_template']
        return os.path.join(self.data_path, template % i)

    def _image_groups_path(self):
        return os.path.join(self.data_path, 'image_groups.txt')

    def _image_list_path(self):
        return os.path.join(self._submodels_path(), self._image_list_file_name)

    def _clusters_path(self):
        return os.path.join(self._submodels_path(), self._clusters_file_name)

    def _clusters_with_neighbors_path(self):
        return os.path.join(self._submodels_path(), self._clusters_with_neighbors_file_name)

    def _clusters_with_neighbors_geojson_path(self):
        return os.path.join(self._submodels_path(), self._clusters_with_neighbors_geojson_file_name)

    def _clusters_geojson_path(self):
        return os.path.join(self._submodels_path(), self._clusters_geojson_file_name)

    def _create_symlink(self, base_path, dir_name):
        link_path = os.path.join(base_path, dir_name)

        if os.path.islink(link_path):
            os.unlink(link_path)

        os.symlink(
            os.path.relpath(os.path.join(self.data_path, dir_name), base_path),
            os.path.join(link_path))

    def image_groups_exists(self):
        return os.path.isfile(self._image_groups_path())

    def load_image_groups(self):
        with open(self._image_groups_path()) as fin:
            for line in fin:
                yield line.split()

    def image_list_exists(self):
        return os.path.isfile(self._image_list_path())

    def create_image_list(self, ills):
        with io.open_wt(self._image_list_path()) as csvfile:
            for image, lat, lon in ills:
                csvfile.write(u'{}\t{}\t{}\n'.format(image, lat, lon))

    def images_with_gps(self):
        with io.open_rt(self._image_list_path()) as csvfile:
            for line in csvfile:
                image, lat, lon = line.split(u'\t')
                yield image, float(lat), float(lon)

    def save_clusters(self, images, positions, labels, centers):
        filepath = self._clusters_path()
        np.savez_compressed(
            filepath,
            images=images,
            positions=positions,
            labels=labels,
            centers=centers)

    def load_clusters(self):
        c = np.load(self._clusters_path())
        return c['images'], c['positions'], c['labels'], c['centers']

    def save_clusters_with_neighbors(self, clusters):
        filepath = self._clusters_with_neighbors_path()
        np.savez_compressed(
            filepath,
            clusters=clusters)

    def load_clusters_with_neighbors(self):
        c = np.load(self._clusters_with_neighbors_path())
        return c['clusters']

    def save_cluster_with_neighbors_geojson(self, geojson):
        filepath = self._clusters_with_neighbors_geojson_path()
        with io.open_wt(filepath) as fout:
            io.json_dump(geojson, fout)

    def save_clusters_geojson(self, geojson):
        filepath = self._clusters_geojson_path()
        with io.open_wt(filepath) as fout:
            io.json_dump(geojson, fout)

    def remove_submodels(self):
        sm = self._submodels_path()
        paths = [os.path.join(sm, o) for o in os.listdir(sm) if os.path.isdir(os.path.join(sm, o))]
        for path in paths:
            shutil.rmtree(path)

    def create_submodels(self, clusters):
        data = DataSet(self.data_path)
        for i, cluster in enumerate(clusters):
            # create sub model dirs
            submodel_path = self._submodel_path(i)
            submodel_images_path = self._submodel_images_path(i)
            io.mkdir_p(submodel_path)
            io.mkdir_p(submodel_images_path)

            # create image list file
            image_list_path = os.path.join(submodel_path, 'image_list.txt')
            with io.open_wt(image_list_path) as txtfile:
                for image in cluster:
                    src = data.image_files[image]
                    dst = os.path.join(submodel_images_path, image)
                    if not os.path.isfile(dst):
                        os.symlink(src, dst)
                    dst_relpath = os.path.relpath(dst, submodel_path)
                    txtfile.write(dst_relpath + "\n")

            # copy config.yaml if exists
            config_file_path = os.path.join(self.data_path, 'config.yaml')
            if os.path.exists(config_file_path):
                shutil.copyfile(config_file_path, os.path.join(submodel_path, 'config.yaml'))

            # create symlinks to additional files
            filenames = ['camera_models.json', 'reference_lla.json', 'exif',
                         'features', 'matches', 'masks', 'mask_list.txt',
                         'segmentations']
            for filename in filenames:
                self._create_symlink(submodel_path, filename)

    def get_submodel_paths(self):
        submodel_paths = []
        for i in range(999999):
            submodel_path = self._submodel_path(i)
            if os.path.isdir(submodel_path):
                submodel_paths.append(submodel_path)
            else:
                break
        return submodel_paths
<commit_msg>feat(split/merge): symlink only existing files<commit_after>import numpy as np
import os
import os.path
import shutil

from opensfm import config
from opensfm import io
from opensfm.dataset import DataSet

class MetaDataSet():
    def __init__(self, data_path):
        '''
        Create meta dataset instance for large scale reconstruction.

        :param data_path: Path to directory containing meta dataset
        '''
        self.data_path = os.path.abspath(data_path)

        config_file = os.path.join(self.data_path, 'config.yaml')
        self.config = config.load_config(config_file)

        self._image_list_file_name = 'image_list_with_gps.tsv'
        self._clusters_file_name = 'clusters.npz'
        self._clusters_with_neighbors_file_name = 'clusters_with_neighbors.npz'
        self._clusters_with_neighbors_geojson_file_name = 'clusters_with_neighbors.geojson'
        self._clusters_geojson_file_name = 'clusters.geojson'

        io.mkdir_p(self._submodels_path())

    def _submodels_path(self):
        return os.path.join(self.data_path, self.config['submodels_relpath'])

    def _submodel_path(self, i):
        """Path to submodel i folder."""
        template = self.config['submodel_relpath_template']
        return os.path.join(self.data_path, template % i)

    def _submodel_images_path(self, i):
        """Path to submodel i images folder."""
        template = self.config['submodel_images_relpath_template']
        return os.path.join(self.data_path, template % i)

    def _image_groups_path(self):
        return os.path.join(self.data_path, 'image_groups.txt')

    def _image_list_path(self):
        return os.path.join(self._submodels_path(), self._image_list_file_name)

    def _clusters_path(self):
        return os.path.join(self._submodels_path(), self._clusters_file_name)

    def _clusters_with_neighbors_path(self):
        return os.path.join(self._submodels_path(), self._clusters_with_neighbors_file_name)

    def _clusters_with_neighbors_geojson_path(self):
        return os.path.join(self._submodels_path(), self._clusters_with_neighbors_geojson_file_name)

    def _clusters_geojson_path(self):
        return os.path.join(self._submodels_path(), self._clusters_geojson_file_name)

    def _create_symlink(self, base_path, filename):
        src = os.path.join(self.data_path, filename)
        dst = os.path.join(base_path, filename)

        if not os.path.exists(src):
            return

        if os.path.islink(dst):
            os.unlink(dst)

        os.symlink(os.path.relpath(src, base_path), dst)

    def image_groups_exists(self):
        return os.path.isfile(self._image_groups_path())

    def load_image_groups(self):
        with open(self._image_groups_path()) as fin:
            for line in fin:
                yield line.split()

    def image_list_exists(self):
        return os.path.isfile(self._image_list_path())

    def create_image_list(self, ills):
        with io.open_wt(self._image_list_path()) as csvfile:
            for image, lat, lon in ills:
                csvfile.write(u'{}\t{}\t{}\n'.format(image, lat, lon))

    def images_with_gps(self):
        with io.open_rt(self._image_list_path()) as csvfile:
            for line in csvfile:
                image, lat, lon = line.split(u'\t')
                yield image, float(lat), float(lon)

    def save_clusters(self, images, positions, labels, centers):
        filepath = self._clusters_path()
        np.savez_compressed(
            filepath,
            images=images,
            positions=positions,
            labels=labels,
            centers=centers)

    def load_clusters(self):
        c = np.load(self._clusters_path())
        return c['images'], c['positions'], c['labels'], c['centers']

    def save_clusters_with_neighbors(self, clusters):
        filepath = self._clusters_with_neighbors_path()
        np.savez_compressed(
            filepath,
            clusters=clusters)

    def load_clusters_with_neighbors(self):
        c = np.load(self._clusters_with_neighbors_path())
        return c['clusters']

    def save_cluster_with_neighbors_geojson(self, geojson):
        filepath = self._clusters_with_neighbors_geojson_path()
        with io.open_wt(filepath) as fout:
            io.json_dump(geojson, fout)

    def save_clusters_geojson(self, geojson):
        filepath = self._clusters_geojson_path()
        with io.open_wt(filepath) as fout:
            io.json_dump(geojson, fout)

    def remove_submodels(self):
        sm = self._submodels_path()
        paths = [os.path.join(sm, o) for o in os.listdir(sm) if os.path.isdir(os.path.join(sm, o))]
        for path in paths:
            shutil.rmtree(path)

    def create_submodels(self, clusters):
        data = DataSet(self.data_path)
        for i, cluster in enumerate(clusters):
            # create sub model dirs
            submodel_path = self._submodel_path(i)
            submodel_images_path = self._submodel_images_path(i)
            io.mkdir_p(submodel_path)
            io.mkdir_p(submodel_images_path)

            # create image list file
            image_list_path = os.path.join(submodel_path, 'image_list.txt')
            with io.open_wt(image_list_path) as txtfile:
                for image in cluster:
                    src = data.image_files[image]
                    dst = os.path.join(submodel_images_path, image)
                    if not os.path.isfile(dst):
                        os.symlink(src, dst)
                    dst_relpath = os.path.relpath(dst, submodel_path)
                    txtfile.write(dst_relpath + "\n")

            # copy config.yaml if exists
            config_file_path = os.path.join(self.data_path, 'config.yaml')
            if os.path.exists(config_file_path):
                shutil.copyfile(config_file_path, os.path.join(submodel_path, 'config.yaml'))

            # create symlinks to additional files
            filenames = ['camera_models.json', 'reference_lla.json', 'exif',
                         'features', 'matches', 'masks', 'mask_list.txt',
                         'segmentations']
            for filename in filenames:
                self._create_symlink(submodel_path, filename)

    def get_submodel_paths(self):
        submodel_paths = []
        for i in range(999999):
            submodel_path = self._submodel_path(i)
            if os.path.isdir(submodel_path):
                submodel_paths.append(submodel_path)
            else:
                break
        return submodel_paths
<|endoftext|>


<commit_before>"""
sentry.services.http
~~~~~~~~~~~~~~~~~~~~

:copyright: (c) 2010-2016 by the Sentry Team, see AUTHORS for more details.
:license: BSD, see LICENSE for more details.
"""
from __future__ import absolute_import, print_function

import os
import sys
from sentry.services.base import Service

def convert_options_to_env(options):
    for k, v in options.iteritems():
        if v is None:
            continue
        key = 'UWSGI_' + k.upper().replace('-', '_')
        if isinstance(v, basestring):
            value = v
        elif v is True:
            value = 'true'
        elif v is False:
            value = 'false'
        elif isinstance(v, (int, long)):
            value = str(v)
        else:
            raise TypeError('Unknown option type: %r (%s)' % (k, type(v)))
        yield key, value

class SentryHTTPServer(Service):
    name = 'http'

    def __init__(self, host=None, port=None, debug=False, workers=None,
                 validate=True, extra_options=None):
        from django.conf import settings

        if validate:
            self.validate_settings()

        host = host or settings.SENTRY_WEB_HOST
        port = port or settings.SENTRY_WEB_PORT

        options = (settings.SENTRY_WEB_OPTIONS or {}).copy()
        if extra_options is not None:
            for k, v in extra_options.iteritems():
                options[k] = v
        options.setdefault('module', 'sentry.wsgi:application')
        options.setdefault('protocol', 'http')
        options.setdefault('auto-procname', True)
        options.setdefault('procname-prefix-spaced', '[Sentry]')
        options.setdefault('workers', 3)
        options.setdefault('threads', 4)
        options.setdefault('http-timeout', 30)
        options.setdefault('vacuum', True)
        options.setdefault('thunder-lock', True)
        options.setdefault('log-x-forwarded-for', False)
        options.setdefault('buffer-size', 32768)
        options.setdefault('post-buffering', 65536)
        options.setdefault('limit-post', 20971520)
        options.setdefault('need-app', True)
        options.setdefault('disable-logging', False)
        options.setdefault('memory-report', True)
        options.setdefault('reload-on-rss', 600)
        options.setdefault('ignore-sigpipe', True)
        options.setdefault('ignore-write-errors', True)
        options.setdefault('disable-write-exception', True)
        options.setdefault('virtualenv', sys.prefix)
        options.setdefault('die-on-term', True)
        options.setdefault('log-format', '%(addr) - %(user) [%(ltime)] "%(method) %(uri) %(proto)" %(status) %(size) "%(referer)" "%(uagent)"')

        options.setdefault('%s-socket' % options['protocol'], '%s:%s' % (host, port))

        # We only need to set uid/gid when stepping down from root, but if
        # we are trying to run as root, then ignore it entirely.
        uid = os.getuid()
        if uid > 0:
            options.setdefault('uid', uid)
        gid = os.getgid()
        if gid > 0:
            options.setdefault('gid', gid)

        # Required arguments that should not be overridden
        options['master'] = True
        options['enable-threads'] = True
        options['lazy-apps'] = True
        options['single-interpreter'] = True

        if workers:
            options['workers'] = workers

        # Old options from gunicorn
        if 'bind' in options:
            options['%s-socket' % options['protocol']] = options.pop('bind')
        if 'accesslog' in options:
            if options['accesslog'] != '-':
                options['logto'] = options['accesslog']
            del options['accesslog']
        if 'errorlog' in options:
            if options['errorlog'] != '-':
                options['logto2'] = options['errorlog']
            del options['errorlog']
        if 'timeout' in options:
            options['http-timeout'] = options.pop('timeout')
        if 'proc_name' in options:
            options['procname-prefix-spaced'] = options.pop('proc_name')
        if 'secure_scheme_headers' in options:
            del options['secure_scheme_headers']
        if 'loglevel' in options:
            del options['loglevel']

        self.options = options

    def validate_settings(self):
        from django.conf import settings as django_settings
        from sentry.utils.settings import validate_settings

        validate_settings(django_settings)

    def prepare_environment(self, env=None):
        if env is None:
            env = os.environ

        # Move all of the options into UWSGI_ env vars
        for k, v in convert_options_to_env(self.options):
            env.setdefault(k, v)

        # This has already been validated inside __init__
        env['SENTRY_SKIP_BACKEND_VALIDATION'] = '1'

        # Look up the bin directory where `sentry` exists, which should be
        # sys.argv[0], then inject that to the front of our PATH so we can reliably
        # find the `uwsgi` that's installed when inside virtualenv.
        # This is so the virtualenv doesn't need to be sourced in, which effectively
        # does exactly this.
        virtualenv_path = os.path.dirname(os.path.abspath(sys.argv[0]))
        current_path = env.get('PATH', '')
        if virtualenv_path not in current_path:
            env['PATH'] = '%s:%s' % (virtualenv_path, current_path)

    def run(self):
        self.prepare_environment()
        os.execvp('uwsgi', ('uwsgi',))
<commit_msg>Set env var from http service to signal that it's running from uwsgi<commit_after>"""
sentry.services.http
~~~~~~~~~~~~~~~~~~~~

:copyright: (c) 2010-2016 by the Sentry Team, see AUTHORS for more details.
:license: BSD, see LICENSE for more details.
"""
from __future__ import absolute_import, print_function

import os
import sys
from sentry.services.base import Service

def convert_options_to_env(options):
    for k, v in options.iteritems():
        if v is None:
            continue
        key = 'UWSGI_' + k.upper().replace('-', '_')
        if isinstance(v, basestring):
            value = v
        elif v is True:
            value = 'true'
        elif v is False:
            value = 'false'
        elif isinstance(v, (int, long)):
            value = str(v)
        else:
            raise TypeError('Unknown option type: %r (%s)' % (k, type(v)))
        yield key, value

class SentryHTTPServer(Service):
    name = 'http'

    def __init__(self, host=None, port=None, debug=False, workers=None,
                 validate=True, extra_options=None):
        from django.conf import settings

        if validate:
            self.validate_settings()

        host = host or settings.SENTRY_WEB_HOST
        port = port or settings.SENTRY_WEB_PORT

        options = (settings.SENTRY_WEB_OPTIONS or {}).copy()
        if extra_options is not None:
            for k, v in extra_options.iteritems():
                options[k] = v
        options.setdefault('module', 'sentry.wsgi:application')
        options.setdefault('protocol', 'http')
        options.setdefault('auto-procname', True)
        options.setdefault('procname-prefix-spaced', '[Sentry]')
        options.setdefault('workers', 3)
        options.setdefault('threads', 4)
        options.setdefault('http-timeout', 30)
        options.setdefault('vacuum', True)
        options.setdefault('thunder-lock', True)
        options.setdefault('log-x-forwarded-for', False)
        options.setdefault('buffer-size', 32768)
        options.setdefault('post-buffering', 65536)
        options.setdefault('limit-post', 20971520)
        options.setdefault('need-app', True)
        options.setdefault('disable-logging', False)
        options.setdefault('memory-report', True)
        options.setdefault('reload-on-rss', 600)
        options.setdefault('ignore-sigpipe', True)
        options.setdefault('ignore-write-errors', True)
        options.setdefault('disable-write-exception', True)
        options.setdefault('virtualenv', sys.prefix)
        options.setdefault('die-on-term', True)
        options.setdefault('log-format', '%(addr) - %(user) [%(ltime)] "%(method) %(uri) %(proto)" %(status) %(size) "%(referer)" "%(uagent)"')

        options.setdefault('%s-socket' % options['protocol'], '%s:%s' % (host, port))

        # We only need to set uid/gid when stepping down from root, but if
        # we are trying to run as root, then ignore it entirely.
        uid = os.getuid()
        if uid > 0:
            options.setdefault('uid', uid)
        gid = os.getgid()
        if gid > 0:
            options.setdefault('gid', gid)

        # Required arguments that should not be overridden
        options['master'] = True
        options['enable-threads'] = True
        options['lazy-apps'] = True
        options['single-interpreter'] = True

        if workers:
            options['workers'] = workers

        # Old options from gunicorn
        if 'bind' in options:
            options['%s-socket' % options['protocol']] = options.pop('bind')
        if 'accesslog' in options:
            if options['accesslog'] != '-':
                options['logto'] = options['accesslog']
            del options['accesslog']
        if 'errorlog' in options:
            if options['errorlog'] != '-':
                options['logto2'] = options['errorlog']
            del options['errorlog']
        if 'timeout' in options:
            options['http-timeout'] = options.pop('timeout')
        if 'proc_name' in options:
            options['procname-prefix-spaced'] = options.pop('proc_name')
        if 'secure_scheme_headers' in options:
            del options['secure_scheme_headers']
        if 'loglevel' in options:
            del options['loglevel']

        self.options = options

    def validate_settings(self):
        from django.conf import settings as django_settings
        from sentry.utils.settings import validate_settings

        validate_settings(django_settings)

    def prepare_environment(self, env=None):
        if env is None:
            env = os.environ

        # Move all of the options into UWSGI_ env vars
        for k, v in convert_options_to_env(self.options):
            env.setdefault(k, v)

        # Signal that we're running within uwsgi
        env['SENTRY_RUNNING_UWSGI'] = '1'

        # This has already been validated inside __init__
        env['SENTRY_SKIP_BACKEND_VALIDATION'] = '1'

        # Look up the bin directory where `sentry` exists, which should be
        # sys.argv[0], then inject that to the front of our PATH so we can reliably
        # find the `uwsgi` that's installed when inside virtualenv.
        # This is so the virtualenv doesn't need to be sourced in, which effectively
        # does exactly this.
        virtualenv_path = os.path.dirname(os.path.abspath(sys.argv[0]))
        current_path = env.get('PATH', '')
        if virtualenv_path not in current_path:
            env['PATH'] = '%s:%s' % (virtualenv_path, current_path)

    def run(self):
        self.prepare_environment()
        os.execvp('uwsgi', ('uwsgi',))
<|endoftext|>


<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-
import os, sys
from datetime import datetime as dt

def send_mail(user, db):
	from packages.format_email import createhtmlmail
	from packages.private import username, passw
	import smtplib

	fromaddrs = "crawlex@cortext.net"
	toaddrs  = user
	html = db.mail_report()
	txt = db.show_stats()
	subject = "Crawlex on Duty: report of %s, breaking news from the front!" %str(db.db_name)
	msg = createhtmlmail(html, txt, subject, fromaddrs)
	# The actual mail send
	server = smtplib.SMTP('smtp.gmail.com:587')
	server.starttls()
	server.login(username,passw)
	server.sendmail(fromaddrs, toaddrs, msg)
	server.quit()
	return True

def generate_report(task, db):
	date = dt.now()
	date = date.strftime('%d-%m-%Y_%H-%M')
	directory = os.path.join(task['directory'], 'reports')
	if not os.path.exists(directory):
		os.makedirs(directory)
	filename = "%s/%s.txt" %(directory, date)

	with open(filename, 'w') as f:
		f.write("\n======PROJECT PARAMS======\n")
		for k, v in task.items():
			if k not in ["action", "status","msg", "date", "creation_date", "_id"]:
				if k == "creation_date":
					v = task[k].strftime('%d-%m-%Y %H-%M-%S')
					f.write(str(k)+": "+str(v)+"\n")
				try:
					f.write(str(k)+": "+str(v)+"\n")
				except Exception:
					pass

		f.write(db.show_stats())

		f.write("\n\n======HISTORY OF THE PROJECT======\n")

		date_list = [n.strftime('%d-%m-%Y %H-%M-%S') for n in task["date"]]
		status_list = list(zip(task["action"], task["status"],task["msg"], date_list))
		for msg in status_list:
			f.write("\n-"+str(msg))
	print "Your report is ready!\nCheck here: %s" %(filename)
	return True<commit_msg>Modifying entries for report<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-
import os, sys
from datetime import datetime as dt

def send_mail(user, db):
	from packages.format_email import createhtmlmail
	from packages.private import username, passw
	import smtplib

	fromaddrs = "crawlex@cortext.net"
	toaddrs  = user
	html = db.mail_report()
	txt = db.show_stats()
	subject = "Crawlex on Duty: report of %s, breaking news from the front!" %str(db.db_name)
	msg = createhtmlmail(html, txt, subject, fromaddrs)
	# The actual mail send
	server = smtplib.SMTP('smtp.gmail.com:587')
	server.starttls()
	server.login(username,passw)
	server.sendmail(fromaddrs, toaddrs, msg)
	server.quit()
	return True

def generate_report(task, db, directory):
	date = dt.now()
	date = date.strftime('%d-%m-%Y_%H-%M')
	directory = os.path.join(directory, 'reports')
	if not os.path.exists(directory):
		os.makedirs(directory)
	filename = "%s/%s.txt" %(directory, date)

	with open(filename, 'w') as f:
		f.write("\n======PROJECT PARAMS======\n")
		for k, v in task.items():
			if k not in ["action", "status","msg", "date", "creation_date", "_id"]:
				if k == "creation_date":
					v = task[k].strftime('%d-%m-%Y %H-%M-%S')
					f.write(str(k)+": "+str(v)+"\n")
				try:
					f.write(str(k)+": "+str(v)+"\n")
				except Exception:
					pass

		f.write(db.show_stats())

		f.write("\n\n======HISTORY OF THE PROJECT======\n")

		date_list = [n.strftime('%d-%m-%Y %H-%M-%S') for n in task["date"]]
		status_list = list(zip(task["action"], task["status"],task["msg"], date_list))
		for msg in status_list:
			f.write("\n-"+str(msg))
	print "Your report is ready!\nCheck here: %s" %(filename)
	return True
<|endoftext|>


<commit_before>#!/usr/bin/env python
#
# EC2 synthesis is designed for synthesis approach for EC2 Cloudl Application
# Basic purpose of this script is preparing kexec by making customized VM
# at block device.
#
# Copyright (C) 2011-2012 Carnegie Mellon University
#
# This program is free software; you can redistribute it and/or modify it
# under the terms of version 2 of the GNU General Public License as published
# by the Free Software Foundation.  A copy of the GNU General Public License
# should have been distributed along with this program in the file
# LICENSE.GPL.
#
# This program is distributed in the hope that it will be useful, but
# WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
# or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
# for more details.
#

import os
import urllib2
from multiprocessing import Process, Queue, Pipe, JoinableQueue
from optparse import OptionParser
from datetime import datetime
import subprocess
import sys
import tempfile
from synthesis import network_worker, decomp_worker, delta_worker

CHUNK_SIZE = 1024*16

def piping_synthesis(overlay_url, base_path):
    prev = datetime.now()
    delta_processes = []
    tmp_dir = tempfile.mkdtemp()
    time_transfer = Queue()
    time_decomp = Queue()
    time_delta = Queue()

    download_queue = JoinableQueue()
    decomp_queue = JoinableQueue()
    (download_pipe_in, download_pipe_out) = Pipe()
    (decomp_pipe_in, decomp_pipe_out) = Pipe()
    recover_file = os.path.join(tmp_dir, overlay_url.split("/")[-1] + ".recover")

    # synthesis
    print "[INFO] Start Downloading at %s" % (overlay_url)
    url = urllib2.urlopen(overlay_url)
    download_process = Process(target=network_worker, args=(url, download_queue, time_transfer, CHUNK_SIZE))
    decomp_process = Process(target=decomp_worker, args=(download_queue, decomp_queue, time_decomp))
    delta_process = Process(target=delta_worker, args=(decomp_queue, time_delta, base_path, recover_file))
    delta_processes.append(delta_process)
    download_process.start()
    decomp_process.start()
    delta_process.start()

    # wait until end
    delta_process.join()
    print "\n[Time] Total Time for synthesis(including download) : " + str(datetime.now()-prev)
    return recover_file

def mount_launchVM(launch_disk_path):
    mount_dir = tempfile.mkdtemp()
    raw_vm = launch_disk_path + ".raw"

    # qemu-img convert
    start_time = datetime.now()
    cmd_convert = "qemu-img convert -f qcow2 %s -O raw %s" % (launch_disk_path, raw_vm)
    proc = subprocess.Popen(cmd_convert, shell=True, stdin=sys.stdin, stdout=sys.stdout)
    proc.wait()
    if proc.returncode != 0 or os.path.exists(raw_vm) == False:
        print >> sys.stderr, "Error, Failed to QEMU-IMG Converting"
        print >> sys.stderr, "CMD: %s" % (cmd_convert)
        sys.exit(2)
    convert_time = datetime.now()-start_time

    # mount
    start_time = datetime.now()
    cmd_mapping = "sudo kpartx -av %s" % (raw_vm)
    proc = subprocess.Popen(cmd_mapping, shell=True, stdin=sys.stdin, stdout=subprocess.PIPE)
    proc.wait()
    if proc.returncode != 0:
        print >> sys.stderr, "Error, Failed to kpartx"
        sys.exit(2)
    output = proc.stdout.readline()
    output = output[output.find("loop"):]
    mapper_dev = output.split(" ")[0].strip()
    cmd_mount = "sudo mount /dev/mapper/%s %s" % (mapper_dev, mount_dir)
    proc = subprocess.Popen(cmd_mount, shell=True, stdin=sys.stdin, stdout=sys.stdout)
    proc.wait()
    if proc.returncode != 0:
        print >> sys.stderr, "Error, Failed to mount, ret code : " + str(proc.returncode)
        sys.exit(2)
    mount_time = datetime.now()-start_time

    print "[TIME] QCOW2 to Raw converting time : %s" % (str(convert_time))
    print "[TIME] Raw Mouting time : %s" % (str(mount_time))
    return mount_dir, raw_vm

def rsync_overlayVM(vm_dir, instance_dir):
    instance_dir = os.path.join(instance_dir, ".")

    # restart init process because it might inidicate original init process
    print "[INFO] Restart Init processs"
    subprocess.Popen("sudo telinit u", shell=True, stdin=sys.stdin, stdout=sys.stdout).wait()

    # possible mount residue at origianl block
    new_mount_dir = os.path.join(instance_dir, "mnt")
    print "[INFO] Remove possible mountings at : " + new_mount_dir
    subprocess.Popen("sudo umount %s" % (new_mount_dir), shell=True, stdin=sys.stdin, stdout=sys.stdout).wait()

    # erase instance dir
    if os.path.exists(instance_dir):
        message = "Are you sure to delete all files at %s?(y/N) " % (instance_dir)
        ret = raw_input(message)
        if str(ret) != 'y':
            sys.exit(1)
    else:
        print >> sys.strerr, "Instance directory does not exists, " + str(instance_dir)
        sys.exit(1)

    cmd_erase = "sudo rm -rf %s" % os.path.join(instance_dir, "*")
    print "[INFO] Erase instance dir: %s" % (cmd_erase)
    subprocess.Popen(cmd_erase, shell=True, stdin=sys.stdin, stdout=sys.stdout).wait()

    # rsync
    print "[INFO] rsync from %s to %s" % (vm_dir, instance_dir)
    start_time = datetime.now()
    cmd_rsync = "sudo rsync -aHx %s/ %s/" % (vm_dir, instance_dir)
    subprocess.Popen(cmd_rsync, shell=True, stdin=sys.stdin, stdout=sys.stdout).wait()
    subprocess.Popen("sudo sync", shell=True, stdin=sys.stdin, stdout=sys.stdout).wait()
    rsync_time = datetime.now()-start_time
    print "[TIME] rsync time : %s" % (str(rsync_time))

    # umount
    print "[INFO] umount instance dir, %s" % (instance_dir)
    subprocess.Popen("sudo umount %s" % (instance_dir), shell=True, stdin=sys.stdin, stdout=sys.stdout).wait()
    print "[INFO] umount VM dir, %s" % (vm_dir)
    subprocess.Popen("sudo umount %s" % (vm_dir), shell=True, stdin=sys.stdin, stdout=sys.stdout).wait()
    subprocess.Popen("sudo sync", shell=True, stdin=sys.stdin, stdout=sys.stdout).wait()

def clean_up(raw_vm, launch_vm):
    print "[INFO] clean up files : %s, %s" % (os.path.abspath(raw_vm), os.path.abspath(launch_vm))
    os.remove(raw_vm)
    os.remove(launch_vm)

def process_command_line(argv):
    help_message = "\nEC2 synthesis is designed for rapid launching of customizing instance at Amazon EC2"

    parser = OptionParser(usage="usage: %prog -o [Overlay Download URL] -b [Base VM Path] -m [Instance Mount Path]" + help_message,
            version="EC2 Synthesys v0.1.1")
    parser.add_option(
            '-o', '--overlay', action='store', type='string', dest='overlay_download_url',
            help='Set overlay disk download URL.')
    parser.add_option(
            '-b', '--base', action='store', type='string', dest='base_path',
            help='Set Base disk path.')
    parser.add_option(
            '-m', '--mount', action='store', type='string', dest='output_mount',
            help='Set output Mount point. This Mount point is EC2 inital disk.')
    settings, args = parser.parse_args(argv)
    if not len(args) == 0:
        parser.error('program takes no command-line arguments; "%s" ignored.' % (args,))
    if settings.overlay_download_url == None or settings.base_path == None or settings.output_mount == None:
        parser.error('Read usage')

    if not os.path.exists(settings.base_path):
        print >> sys.stderr, "[Error] Base VM does not exist at %s" % (settings.base_path)
        sys.exit(2)

    if not os.path.exists(settings.output_mount):
        print >> sys.stderr, "[Error] Mount directory does not exist at %s" % ( settings.output_mount)
        sys.exit(2)

    if not os.path.ismount(settings.output_mount):
        print >> sys.stderr, "[Error] It is not valid mount point at %s" % (settings.output_mount)
        sys.exit(2)

    return settings, args

def main(argv=None):
    settings, args = process_command_line(sys.argv[1:])

    # Synthesis overlay
    qcow_launch_image = piping_synthesis(settings.overlay_download_url, settings.base_path)

    # Mount VM File system
    launchVM_dir, VM_path =  mount_launchVM(qcow_launch_image)

    # rsync VM to origianl disk
    rsync_overlayVM(launchVM_dir, settings.output_mount)

    # clean up
    clean_up(VM_path, qcow_launch_image)

    return 0

if __name__ == "__main__":
    status = main()
    sys.exit(status)
<commit_msg>run synthesis without asking deletion<commit_after>#!/usr/bin/env python
#
# EC2 synthesis is designed for synthesis approach for EC2 Cloudl Application
# Basic purpose of this script is preparing kexec by making customized VM
# at block device.
#
# Copyright (C) 2011-2012 Carnegie Mellon University
#
# This program is free software; you can redistribute it and/or modify it
# under the terms of version 2 of the GNU General Public License as published
# by the Free Software Foundation.  A copy of the GNU General Public License
# should have been distributed along with this program in the file
# LICENSE.GPL.
#
# This program is distributed in the hope that it will be useful, but
# WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
# or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
# for more details.
#

import os
import urllib2
from multiprocessing import Process, Queue, Pipe, JoinableQueue
from optparse import OptionParser
from datetime import datetime
import subprocess
import sys
import tempfile
from synthesis import network_worker, decomp_worker, delta_worker

CHUNK_SIZE = 1024*16

def piping_synthesis(overlay_url, base_path):
    prev = datetime.now()
    delta_processes = []
    tmp_dir = tempfile.mkdtemp()
    time_transfer = Queue()
    time_decomp = Queue()
    time_delta = Queue()

    download_queue = JoinableQueue()
    decomp_queue = JoinableQueue()
    (download_pipe_in, download_pipe_out) = Pipe()
    (decomp_pipe_in, decomp_pipe_out) = Pipe()
    recover_file = os.path.join(tmp_dir, overlay_url.split("/")[-1] + ".recover")

    # synthesis
    print "[INFO] Start Downloading at %s" % (overlay_url)
    url = urllib2.urlopen(overlay_url)
    download_process = Process(target=network_worker, args=(url, download_queue, time_transfer, CHUNK_SIZE))
    decomp_process = Process(target=decomp_worker, args=(download_queue, decomp_queue, time_decomp))
    delta_process = Process(target=delta_worker, args=(decomp_queue, time_delta, base_path, recover_file))
    delta_processes.append(delta_process)
    download_process.start()
    decomp_process.start()
    delta_process.start()

    # wait until end
    delta_process.join()
    print "\n[Time] Total Time for synthesis(including download) : " + str(datetime.now()-prev)
    return recover_file

def mount_launchVM(launch_disk_path):
    mount_dir = tempfile.mkdtemp()
    raw_vm = launch_disk_path + ".raw"

    # qemu-img convert
    start_time = datetime.now()
    cmd_convert = "qemu-img convert -f qcow2 %s -O raw %s" % (launch_disk_path, raw_vm)
    proc = subprocess.Popen(cmd_convert, shell=True, stdin=sys.stdin, stdout=sys.stdout)
    proc.wait()
    if proc.returncode != 0 or os.path.exists(raw_vm) == False:
        print >> sys.stderr, "Error, Failed to QEMU-IMG Converting"
        print >> sys.stderr, "CMD: %s" % (cmd_convert)
        sys.exit(2)
    convert_time = datetime.now()-start_time

    # mount
    start_time = datetime.now()
    cmd_mapping = "sudo kpartx -av %s" % (raw_vm)
    proc = subprocess.Popen(cmd_mapping, shell=True, stdin=sys.stdin, stdout=subprocess.PIPE)
    proc.wait()
    if proc.returncode != 0:
        print >> sys.stderr, "Error, Failed to kpartx"
        sys.exit(2)
    output = proc.stdout.readline()
    output = output[output.find("loop"):]
    mapper_dev = output.split(" ")[0].strip()
    cmd_mount = "sudo mount /dev/mapper/%s %s" % (mapper_dev, mount_dir)
    proc = subprocess.Popen(cmd_mount, shell=True, stdin=sys.stdin, stdout=sys.stdout)
    proc.wait()
    if proc.returncode != 0:
        print >> sys.stderr, "Error, Failed to mount, ret code : " + str(proc.returncode)
        sys.exit(2)
    mount_time = datetime.now()-start_time

    print "[TIME] QCOW2 to Raw converting time : %s" % (str(convert_time))
    print "[TIME] Raw Mouting time : %s" % (str(mount_time))
    return mount_dir, raw_vm

def rsync_overlayVM(vm_dir, instance_dir):
    instance_dir = os.path.join(instance_dir, ".")

    # restart init process because it might inidicate original init process
    print "[INFO] Restart Init processs"
    subprocess.Popen("sudo telinit u", shell=True, stdin=sys.stdin, stdout=sys.stdout).wait()

    # possible mount residue at origianl block
    new_mount_dir = os.path.join(instance_dir, "mnt")
    print "[INFO] Remove possible mountings at : " + new_mount_dir
    subprocess.Popen("sudo umount %s" % (new_mount_dir), shell=True, stdin=sys.stdin, stdout=sys.stdout).wait()

    # erase instance dir
    '''
    if os.path.exists(instance_dir):
        message = "Are you sure to delete all files at %s?(y/N) " % (instance_dir)
        ret = raw_input(message)
        if str(ret) != 'y':
            sys.exit(1)
    else:
        print >> sys.strerr, "Instance directory does not exists, " + str(instance_dir)
        sys.exit(1)
    '''

    cmd_erase = "sudo rm -rf %s" % os.path.join(instance_dir, "*")
    print "[INFO] Erase instance dir: %s" % (cmd_erase)
    subprocess.Popen(cmd_erase, shell=True, stdin=sys.stdin, stdout=sys.stdout).wait()

    # rsync
    print "[INFO] rsync from %s to %s" % (vm_dir, instance_dir)
    start_time = datetime.now()
    cmd_rsync = "sudo rsync -aHx %s/ %s/" % (vm_dir, instance_dir)
    subprocess.Popen(cmd_rsync, shell=True, stdin=sys.stdin, stdout=sys.stdout).wait()
    subprocess.Popen("sudo sync", shell=True, stdin=sys.stdin, stdout=sys.stdout).wait()
    rsync_time = datetime.now()-start_time
    print "[TIME] rsync time : %s" % (str(rsync_time))

    # umount
    print "[INFO] umount instance dir, %s" % (instance_dir)
    subprocess.Popen("sudo umount %s" % (instance_dir), shell=True, stdin=sys.stdin, stdout=sys.stdout).wait()
    print "[INFO] umount VM dir, %s" % (vm_dir)
    subprocess.Popen("sudo umount %s" % (vm_dir), shell=True, stdin=sys.stdin, stdout=sys.stdout).wait()
    subprocess.Popen("sudo sync", shell=True, stdin=sys.stdin, stdout=sys.stdout).wait()

def clean_up(raw_vm, launch_vm):
    print "[INFO] clean up files : %s, %s" % (os.path.abspath(raw_vm), os.path.abspath(launch_vm))
    os.remove(raw_vm)
    os.remove(launch_vm)

def process_command_line(argv):
    help_message = "\nEC2 synthesis is designed for rapid launching of customizing instance at Amazon EC2"

    parser = OptionParser(usage="usage: %prog -o [Overlay Download URL] -b [Base VM Path] -m [Instance Mount Path]" + help_message,
            version="EC2 Synthesys v0.1.1")
    parser.add_option(
            '-o', '--overlay', action='store', type='string', dest='overlay_download_url',
            help='Set overlay disk download URL.')
    parser.add_option(
            '-b', '--base', action='store', type='string', dest='base_path',
            help='Set Base disk path.')
    parser.add_option(
            '-m', '--mount', action='store', type='string', dest='output_mount',
            help='Set output Mount point. This Mount point is EC2 inital disk.')
    settings, args = parser.parse_args(argv)
    if not len(args) == 0:
        parser.error('program takes no command-line arguments; "%s" ignored.' % (args,))
    if settings.overlay_download_url == None or settings.base_path == None or settings.output_mount == None:
        parser.error('Read usage')

    if not os.path.exists(settings.base_path):
        print >> sys.stderr, "[Error] Base VM does not exist at %s" % (settings.base_path)
        sys.exit(2)

    if not os.path.exists(settings.output_mount):
        print >> sys.stderr, "[Error] Mount directory does not exist at %s" % ( settings.output_mount)
        sys.exit(2)

    if not os.path.ismount(settings.output_mount):
        print >> sys.stderr, "[Error] It is not valid mount point at %s" % (settings.output_mount)
        sys.exit(2)

    return settings, args

def main(argv=None):
    settings, args = process_command_line(sys.argv[1:])

    # Synthesis overlay
    qcow_launch_image = piping_synthesis(settings.overlay_download_url, settings.base_path)

    # Mount VM File system
    launchVM_dir, VM_path =  mount_launchVM(qcow_launch_image)

    # rsync VM to origianl disk
    rsync_overlayVM(launchVM_dir, settings.output_mount)

    # clean up
    clean_up(VM_path, qcow_launch_image)

    return 0

if __name__ == "__main__":
    status = main()
    sys.exit(status)
<|endoftext|>


<commit_before>#!/usr/bin/env python3

#
# Copyright (c) 2017 Erik Nordstrøm <erik@nordstroem.no>
#
# Permission to use, copy, modify, and/or distribute this software for any
# purpose with or without fee is hereby granted, provided that the above
# copyright notice and this permission notice appear in all copies.
#
# THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
# WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
# MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
# ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
# WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
# ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
# OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
#

import sys, os

debug = False

class FileRef:

    def __init__ (self, fname, direct_deps = [], directly_associated = []):

        self.fname = fname
        self.direct_deps = direct_deps
        self.directly_associated = directly_associated

    def __str__ (self):

        return self.fname

    def __repr__ (self):

        return type(self).__name__ + "('" + self.fname + "')"

    def __lt__ (self, other):

        return self.fname < str(other)

    def __eq__ (self, other):

        return self.fname == str(other)

    def get_all_deps (self, cur_depth = 0):

        deps = []

        for fref in self.direct_deps + self.directly_associated:

            deps.append(str(fref))

            if isinstance(fref, Source):

                subdeps = fref.get_all_deps(cur_depth + 1)

                for subdep in subdeps:

                    if not(subdep in deps):

                        deps.append(subdep)

        return deps

class Source (FileRef):

    def __init__ (self, fname, search_assoc = None):

        super(Source, self).__init__(fname)

        self.search_assoc = search_assoc

        self.directly_associated_fnames = []

        if not(search_assoc is None):

            self.directly_associated_fnames = \
                self.get_directly_associated_fnames()

    def refresh_directly_associated (self):

        directly_associated = []

        for fname in self.directly_associated_fnames:

            directly_associated.append( \
                self.search_assoc[self.search_assoc.index(fname)])

        self.directly_associated = directly_associated

def c_style_local_include_fnames (self):

    local_inc_fnames = []

    with open(str(self), 'r') as f:

        for line in f:

            if line.startswith('#include "'):

                fname = os.path.join('src', line[10:-2])

                local_inc_fnames.append(fname)

    local_inc_fnames.sort()

    return local_inc_fnames

class Cfile (Source):

    get_directly_associated_fnames = c_style_local_include_fnames

class Hfile (Source):

    get_directly_associated_fnames = c_style_local_include_fnames

class Buildable (FileRef):

    def __init__ (self, fname, direct_deps = []):

        super(Buildable, self).__init__(fname, direct_deps)

class Ofile (Buildable):

    def mfstr (self):

        mf_ofile = str(self)
        mf_deps = ' '.join([str(dep) for dep in self.get_all_deps()])
        mf_ddeps = ' '.join([str(ddep) for ddep in self.direct_deps])

        return \
"""
$(OBJDIR)/{}: {}
	mkdir -p $(OBJDIR)
	$(CC) $(CFLAGS) -c {} -o $@
""".format(mf_ofile, mf_deps, mf_ddeps)

class Gennedfile (Buildable):

    def __init__ (self, fname, creator):

        super(Gennedfile, self).__init__(fname)

        self.creator = creator

    def mfstr (self):

        return \
"""
$(SHAREDIR)/{}: {}
	mkdir -p $(SHAREDIR)/{}
	./{} $@
""".format(str(self), creator, os.path.dirname(str(self)), creator)

def genfile (creator, subpath, stem, tgtext):

    return Gennedfile(os.path.join(subpath, stem + tgtext), creator)

def invalid_set_of_args ():

    sys.stderr.write('Usage: ' + sys.argv[0] + ' -d | <Makefile>\n')
    sys.exit(1)

if len(sys.argv) != 2:

    invalid_set_of_args()

makefile = None

if sys.argv[1] == '-':

    makefile = sys.stdout

elif sys.argv[1][0] != '-':

    makefile = open(sys.argv[1], 'w')

else:

    if sys.argv[1] == '-d':

        debug = True

    else:

        invalid_set_of_args()

project = os.path.basename(os.path.dirname(os.path.realpath(__file__)))

cfiles = []
hfiles = []

dir = 'src'
for file in os.listdir(dir):

    f = os.path.join(dir, file)

    if file.endswith('.c'):

        cfiles.append(Cfile(f, hfiles))

    elif file.endswith('.h'):

        hfiles.append(Hfile(f, hfiles))

cfiles.sort()
hfiles.sort()

for fref in cfiles + hfiles:

    fref.refresh_directly_associated()

ofiles = []

for cfile in cfiles:

    ofile = Ofile(os.path.splitext(os.path.basename(str(cfile)))[0] + '.o',
        [cfile])

    ofiles.append(ofile)

mf_ofiles = ' '.join([os.path.join('$(OBJDIR)',str(ofile))
    for ofile in ofiles])

gennedfiles = []

subpath = 'img'
srcpath = os.path.join('assets', 'gen', subpath)
for file in os.listdir(srcpath):

    creator = os.path.join(srcpath, file)
    stem, srcext = os.path.splitext(file)
    gennedfiles.append(genfile(creator, subpath, stem, '.png'))

gennedfiles.sort()

mf_gennedfiles = ' '.join([str(gennedfile) for gennedfile in gennedfiles])

if debug:

    sys.stderr.write('No current debug info to show.\n')

    sys.exit(0)

makefile.write('#\n')

with open('LICENSE', 'r') as flic:

    for line in flic:

        makefile.write('#' + (' ' if (len(line) > 1) else '') + line)

makefile.write('#\n')

makefile.write(
"""
PROJECT = {}

MAKE != sh -c '[ ! -z "$(MAKE)" ] && echo "$(MAKE)" || echo "make"'
CC != sh -c '[ ! -z "$(CC)" ] && echo "$(CC)" || echo "cc"'

CFLAGS = -Wall -Werror -I/usr/local/include
LDFLAGS = -L/usr/local/lib
LDLIBS = -lSDL2 -lSDL2_image -lm

HOST_TRIPLET != sh -c '$(CC) -dumpmachine || echo $$(uname -m)-unknown-$$(uname -s)'

TARGET_TRIPLET != sh -c '[ ! -z "$(TARGET_TRIPLET)" ] && echo "$(TARGET_TRIPLET)" || echo "$(HOST_TRIPLET)"'

BUILDTYPE != sh -c '[ ! -z "$(BUILDTYPE)" ] && echo "$(BUILDTYPE)" || echo "debug"'
DEBUG != sh -c '[ "$(BUILDTYPE)" = "debug" ] && echo "-g -O0 -DDEBUG" || echo "-O2"'
CFLAGS += $(DEBUG)

CONFIGID = $(HOST_TRIPLET)/$(MAKE)+$(CC)/$(BUILDTYPE)/$(TARGET_TRIPLET)

OUT != sh -c '[ ! -z "$(OUT)" ] && echo "$(OUT)" || echo "out"'
OUTDIR = $(OUT)/$(CONFIGID)
BINDIR = $(OUTDIR)/bin

BUILDS != sh -c '[ ! -z "$(BUILDS)" ] && echo "$(BUILDS)" || echo "build"'
OBJDIR = $(BUILDS)/$(CONFIGID)
OBJS = {}

MAIN_EXECUTABLE = $(BINDIR)/$(PROJECT)

SHAREPDIR = $(OUTDIR)/share
SHAREDIR = $(SHAREPDIR)/jumper
DYNASTS = LICENSE {}
DYNASTS != sh -c "echo '$(DYNASTS)' | sed 's@[^ ]\{{1,\}}@$(SHAREDIR)/&@g'"

all: $(MAIN_EXECUTABLE) $(DYNASTS)

$(SHAREDIR)/LICENSE: LICENSE
	mkdir -p $(SHAREDIR)
	cp $> $^ $@
""".format(project, mf_ofiles, mf_gennedfiles))

for tgt in gennedfiles + ofiles:

    makefile.write(tgt.mfstr())

makefile.write(
"""
$(MAIN_EXECUTABLE): $(OBJS)
	mkdir -p $(BINDIR)
	$(CC) $(LDFLAGS) $> $^ -o $@ $(LDLIBS)

.PHONY: clean
clean:
	for obj in $(OBJS) ; do test -f $$obj && rm $$obj || true ; done
	test -d $(OBJDIR) && rmdir $(OBJDIR) || true

.PHONY: distclean_triplet
distclean_triplet: clean
	test -f $(MAIN_EXECUTABLE) && rm $(MAIN_EXECUTABLE) || true
	for dynast in $(DYNASTS) ; do \
	  test -f $$dynast && rm $$dynast || true ; \
	done
	test -d $(SHAREDIR)/img && rmdir $(SHAREDIR)/img || true
	test -d $(SHAREDIR) && rmdir $(SHAREDIR) || true
	test -d $(SHAREPDIR) && rmdir $(SHAREPDIR) || true
	test -d $(BINDIR) && rmdir $(BINDIR) || true
	test -d $(OUTDIR) && rmdir $(OUTDIR) || true

.PHONY: distclean
distclean:
	rm -rf $(BUILDS)
	rm -rf $(OUT)

.PHONY: run
run: $(MAIN_EXECUTABLE) $(DYNASTS)
	cd $(SHAREDIR) && $(MAIN_EXECUTABLE)

.PHONY: targets
""")

makefile.close()
<commit_msg>Parse arguments at start of script.<commit_after>#!/usr/bin/env python3

#
# Copyright (c) 2017 Erik Nordstrøm <erik@nordstroem.no>
#
# Permission to use, copy, modify, and/or distribute this software for any
# purpose with or without fee is hereby granted, provided that the above
# copyright notice and this permission notice appear in all copies.
#
# THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
# WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
# MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
# ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
# WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
# ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
# OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
#

import sys, os

debug = False

def invalid_set_of_args ():

    sys.stderr.write('Usage: ' + sys.argv[0] + ' -d | <Makefile>\n')
    sys.exit(1)

if len(sys.argv) != 2:

    invalid_set_of_args()

makefile = None

if sys.argv[1] == '-':

    makefile = sys.stdout

elif sys.argv[1][0] != '-':

    makefile = open(sys.argv[1], 'w')

else:

    if sys.argv[1] == '-d':

        debug = True

    else:

        invalid_set_of_args()

class FileRef:

    def __init__ (self, fname, direct_deps = [], directly_associated = []):

        self.fname = fname
        self.direct_deps = direct_deps
        self.directly_associated = directly_associated

    def __str__ (self):

        return self.fname

    def __repr__ (self):

        return type(self).__name__ + "('" + self.fname + "')"

    def __lt__ (self, other):

        return self.fname < str(other)

    def __eq__ (self, other):

        return self.fname == str(other)

    def get_all_deps (self, cur_depth = 0):

        deps = []

        for fref in self.direct_deps + self.directly_associated:

            deps.append(str(fref))

            if isinstance(fref, Source):

                subdeps = fref.get_all_deps(cur_depth + 1)

                for subdep in subdeps:

                    if not(subdep in deps):

                        deps.append(subdep)

        return deps

class Source (FileRef):

    def __init__ (self, fname, search_assoc = None):

        super(Source, self).__init__(fname)

        self.search_assoc = search_assoc

        self.directly_associated_fnames = []

        if not(search_assoc is None):

            self.directly_associated_fnames = \
                self.get_directly_associated_fnames()

    def refresh_directly_associated (self):

        directly_associated = []

        for fname in self.directly_associated_fnames:

            directly_associated.append( \
                self.search_assoc[self.search_assoc.index(fname)])

        self.directly_associated = directly_associated

def c_style_local_include_fnames (self):

    local_inc_fnames = []

    with open(str(self), 'r') as f:

        for line in f:

            if line.startswith('#include "'):

                fname = os.path.join('src', line[10:-2])

                local_inc_fnames.append(fname)

    local_inc_fnames.sort()

    return local_inc_fnames

class Cfile (Source):

    get_directly_associated_fnames = c_style_local_include_fnames

class Hfile (Source):

    get_directly_associated_fnames = c_style_local_include_fnames

class Buildable (FileRef):

    def __init__ (self, fname, direct_deps = []):

        super(Buildable, self).__init__(fname, direct_deps)

class Ofile (Buildable):

    def mfstr (self):

        mf_ofile = str(self)
        mf_deps = ' '.join([str(dep) for dep in self.get_all_deps()])
        mf_ddeps = ' '.join([str(ddep) for ddep in self.direct_deps])

        return \
"""
$(OBJDIR)/{}: {}
	mkdir -p $(OBJDIR)
	$(CC) $(CFLAGS) -c {} -o $@
""".format(mf_ofile, mf_deps, mf_ddeps)

class Gennedfile (Buildable):

    def __init__ (self, fname, creator):

        super(Gennedfile, self).__init__(fname)

        self.creator = creator

    def mfstr (self):

        return \
"""
$(SHAREDIR)/{}: {}
	mkdir -p $(SHAREDIR)/{}
	./{} $@
""".format(str(self), creator, os.path.dirname(str(self)), creator)

def genfile (creator, subpath, stem, tgtext):

    return Gennedfile(os.path.join(subpath, stem + tgtext), creator)

project = os.path.basename(os.path.dirname(os.path.realpath(__file__)))

cfiles = []
hfiles = []

dir = 'src'
for file in os.listdir(dir):

    f = os.path.join(dir, file)

    if file.endswith('.c'):

        cfiles.append(Cfile(f, hfiles))

    elif file.endswith('.h'):

        hfiles.append(Hfile(f, hfiles))

cfiles.sort()
hfiles.sort()

for fref in cfiles + hfiles:

    fref.refresh_directly_associated()

ofiles = []

for cfile in cfiles:

    ofile = Ofile(os.path.splitext(os.path.basename(str(cfile)))[0] + '.o',
        [cfile])

    ofiles.append(ofile)

mf_ofiles = ' '.join([os.path.join('$(OBJDIR)',str(ofile))
    for ofile in ofiles])

gennedfiles = []

subpath = 'img'
srcpath = os.path.join('assets', 'gen', subpath)
for file in os.listdir(srcpath):

    creator = os.path.join(srcpath, file)
    stem, srcext = os.path.splitext(file)
    gennedfiles.append(genfile(creator, subpath, stem, '.png'))

gennedfiles.sort()

mf_gennedfiles = ' '.join([str(gennedfile) for gennedfile in gennedfiles])

if debug:

    sys.stderr.write('No current debug info to show.\n')

    sys.exit(0)

makefile.write('#\n')

with open('LICENSE', 'r') as flic:

    for line in flic:

        makefile.write('#' + (' ' if (len(line) > 1) else '') + line)

makefile.write('#\n')

makefile.write(
"""
PROJECT = {}

MAKE != sh -c '[ ! -z "$(MAKE)" ] && echo "$(MAKE)" || echo "make"'
CC != sh -c '[ ! -z "$(CC)" ] && echo "$(CC)" || echo "cc"'

CFLAGS = -Wall -Werror -I/usr/local/include
LDFLAGS = -L/usr/local/lib
LDLIBS = -lSDL2 -lSDL2_image -lm

HOST_TRIPLET != sh -c '$(CC) -dumpmachine || echo $$(uname -m)-unknown-$$(uname -s)'

TARGET_TRIPLET != sh -c '[ ! -z "$(TARGET_TRIPLET)" ] && echo "$(TARGET_TRIPLET)" || echo "$(HOST_TRIPLET)"'

BUILDTYPE != sh -c '[ ! -z "$(BUILDTYPE)" ] && echo "$(BUILDTYPE)" || echo "debug"'
DEBUG != sh -c '[ "$(BUILDTYPE)" = "debug" ] && echo "-g -O0 -DDEBUG" || echo "-O2"'
CFLAGS += $(DEBUG)

CONFIGID = $(HOST_TRIPLET)/$(MAKE)+$(CC)/$(BUILDTYPE)/$(TARGET_TRIPLET)

OUT != sh -c '[ ! -z "$(OUT)" ] && echo "$(OUT)" || echo "out"'
OUTDIR = $(OUT)/$(CONFIGID)
BINDIR = $(OUTDIR)/bin

BUILDS != sh -c '[ ! -z "$(BUILDS)" ] && echo "$(BUILDS)" || echo "build"'
OBJDIR = $(BUILDS)/$(CONFIGID)
OBJS = {}

MAIN_EXECUTABLE = $(BINDIR)/$(PROJECT)

SHAREPDIR = $(OUTDIR)/share
SHAREDIR = $(SHAREPDIR)/jumper
DYNASTS = LICENSE {}
DYNASTS != sh -c "echo '$(DYNASTS)' | sed 's@[^ ]\{{1,\}}@$(SHAREDIR)/&@g'"

all: $(MAIN_EXECUTABLE) $(DYNASTS)

$(SHAREDIR)/LICENSE: LICENSE
	mkdir -p $(SHAREDIR)
	cp $> $^ $@
""".format(project, mf_ofiles, mf_gennedfiles))

for tgt in gennedfiles + ofiles:

    makefile.write(tgt.mfstr())

makefile.write(
"""
$(MAIN_EXECUTABLE): $(OBJS)
	mkdir -p $(BINDIR)
	$(CC) $(LDFLAGS) $> $^ -o $@ $(LDLIBS)

.PHONY: clean
clean:
	for obj in $(OBJS) ; do test -f $$obj && rm $$obj || true ; done
	test -d $(OBJDIR) && rmdir $(OBJDIR) || true

.PHONY: distclean_triplet
distclean_triplet: clean
	test -f $(MAIN_EXECUTABLE) && rm $(MAIN_EXECUTABLE) || true
	for dynast in $(DYNASTS) ; do \
	  test -f $$dynast && rm $$dynast || true ; \
	done
	test -d $(SHAREDIR)/img && rmdir $(SHAREDIR)/img || true
	test -d $(SHAREDIR) && rmdir $(SHAREDIR) || true
	test -d $(SHAREPDIR) && rmdir $(SHAREPDIR) || true
	test -d $(BINDIR) && rmdir $(BINDIR) || true
	test -d $(OUTDIR) && rmdir $(OUTDIR) || true

.PHONY: distclean
distclean:
	rm -rf $(BUILDS)
	rm -rf $(OUT)

.PHONY: run
run: $(MAIN_EXECUTABLE) $(DYNASTS)
	cd $(SHAREDIR) && $(MAIN_EXECUTABLE)

.PHONY: targets
""")

makefile.close()
<|endoftext|>