<commit_before># Copyright 2019 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
"""Contains global variables related to mixed precision.

This is not part of mixed_precision.py to avoid a circular dependency.
mixed_precision.py depends on Session, and Session depends on this file.
"""

from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

from tensorflow.python.util.tf_export import tf_export

# Whether the mixed precision graph rewrite has been enabled or not with
# `enable_mixed_precision_graph_rewrite`. Used to turn on auto_mixed_precision
# in ConfigProtos passed to Sessions.
# TODO(scottzhu): change to private attribute once the usage in OSS keras is
# removed.
mixed_precision_graph_rewrite_is_enabled = False

# True if a Session has been created without the mixed precision graph rewrite
# being enabled. Used to give a warning if mixed precision is enabled after a
# Session has already been created.
_non_mixed_precision_session_created = False

# Whether the global tf.keras.mixed_precision.Policy uses mixed precision. Used
# to raise an error message if both a mixed Policy and the graph rewrite are
# used at the same time.
# TODO(scottzhu): change to private attribute once the usage in OSS keras is
# removed.
using_mixed_precision_policy = False

@tf_export('__internal__.train.is_mixed_precision_graph_rewrite_enabled', v1=[])
def is_mixed_precision_graph_rewrite_enabled():
  return mixed_precision_graph_rewrite_is_enabled

def set_mixed_precision_graph_rewrite_enabled(enabled):
  global mixed_precision_graph_rewrite_is_enabled
  mixed_precision_graph_rewrite_is_enabled = enabled

def non_mixed_precision_session_created():
  return _non_mixed_precision_session_created

def set_non_mixed_precision_session_created(created):
  global _non_mixed_precision_session_created
  _non_mixed_precision_session_created = created

def is_using_mixed_precision_policy():
  return using_mixed_precision_policy

@tf_export('__internal__.train.set_using_mixed_precision_policy', v1=[])
def set_using_mixed_precision_policy(is_using):
  global using_mixed_precision_policy
  using_mixed_precision_policy = is_using
<commit_msg>Cleanup the TODO item after the usage has been removed in OSS keras.<commit_after># Copyright 2019 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
"""Contains global variables related to mixed precision.

This is not part of mixed_precision.py to avoid a circular dependency.
mixed_precision.py depends on Session, and Session depends on this file.
"""

from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

from tensorflow.python.util.tf_export import tf_export

# Whether the mixed precision graph rewrite has been enabled or not with
# `enable_mixed_precision_graph_rewrite`. Used to turn on auto_mixed_precision
# in ConfigProtos passed to Sessions.
_mixed_precision_graph_rewrite_is_enabled = False

# True if a Session has been created without the mixed precision graph rewrite
# being enabled. Used to give a warning if mixed precision is enabled after a
# Session has already been created.
_non_mixed_precision_session_created = False

# Whether the global tf.keras.mixed_precision.Policy uses mixed precision. Used
# to raise an error message if both a mixed Policy and the graph rewrite are
# used at the same time.
_using_mixed_precision_policy = False

@tf_export('__internal__.train.is_mixed_precision_graph_rewrite_enabled', v1=[])
def is_mixed_precision_graph_rewrite_enabled():
  return _mixed_precision_graph_rewrite_is_enabled

def set_mixed_precision_graph_rewrite_enabled(enabled):
  global _mixed_precision_graph_rewrite_is_enabled
  _mixed_precision_graph_rewrite_is_enabled = enabled

def non_mixed_precision_session_created():
  return _non_mixed_precision_session_created

def set_non_mixed_precision_session_created(created):
  global _non_mixed_precision_session_created
  _non_mixed_precision_session_created = created

def is_using_mixed_precision_policy():
  return _using_mixed_precision_policy

@tf_export('__internal__.train.set_using_mixed_precision_policy', v1=[])
def set_using_mixed_precision_policy(is_using):
  global _using_mixed_precision_policy
  _using_mixed_precision_policy = is_using
<|endoftext|>


<commit_before>import json
import re

from django.utils.datetime_safe import datetime
import datetime as dtime
import pytz
from twitter import Twitter, OAuth, TwitterHTTPError
from bot.libraries.launchlibrarysdk import LaunchLibrarySDK
from bot.libraries.onesignalsdk import OneSignalSdk
from bot.utils.config import keys
from bot.models import Notification
from bot.utils.deserializer import launch_json_to_model
from bot.utils.util import seconds_to_time
import logging

AUTH_TOKEN_HERE = keys['AUTH_TOKEN_HERE']
APP_ID = keys['APP_ID']
DAEMON_SLEEP = 600
TAG = 'Notification Server'

# Get an instance of a logger
logger = logging.getLogger('bot.notifications')

def json_default(value):
    if isinstance(value, dtime.date):
        return dict(year=value.year, month=value.month, day=value.day)
    else:
        return value.__dict__

class NotificationServer:
    def __init__(self, debug=None, version=None):
        self.one_signal = OneSignalSdk(AUTH_TOKEN_HERE, APP_ID)
        if version is None:
            version = '1.2.1'
        self.launchLibrary = LaunchLibrarySDK(version=version)
        if debug is None:
            self.DEBUG = False
        else:
            self.DEBUG = debug
        response = self.one_signal.get_app(APP_ID)
        assert response.status_code == 200
        self.app = response.json()
        assert isinstance(self.app, dict)
        assert self.app['id'] and self.app['name'] and self.app['updated_at'] and self.app['created_at']
        self.app_auth_key = self.app['basic_auth_key']
        self.twitter = Twitter(
            auth=OAuth(keys['TOKEN_KEY'], keys['TOKEN_SECRET'], keys['CONSUMER_KEY'], keys['CONSUMER_SECRET'])
        )

    def send_to_twitter(self, message, notification):
        # Need to add actual twitter post here.

        if message.endswith(' (1/1)'):
            message = message[:-6]
        if len(message) > 120:
            end = message[-5:]
            if re.search("([1-9]*/[1-9])", end):
                message = (message[:111] + '... ' + end)
            else:
                message = (message[:117] + '...')
        logger.info('Sending to Twitter | %s | %s | DEBUG %s' % (message, str(len(message)), self.DEBUG))
        if not self.DEBUG:
            try:
                self.twitter.statuses.update(status=message)
            except TwitterHTTPError as e:
                logger.error("%s %s" % (str(e), message))

        notification.last_twitter_post = datetime.now()
        notification.last_net_stamp = notification.launch.netstamp
        notification.last_net_stamp_timestamp = datetime.now()
        logger.info('Updating Notification %s to timestamp %s' % (notification.launch.id,
                                                                  notification.last_twitter_post
                                                                  .strftime("%A %d. %B %Y")))

        notification.save()

    def get_next_launches(self):
        logger.info("Getting next launches...")
        response = self.launchLibrary.get_next_launches()
        if response.status_code is 200:
            response_json = response.json()
            launch_data = response_json['launches']
            launches = []
            for launch in launch_data:
                launch = launch_json_to_model(launch)
                if len(launch.location_name) > 20:
                    launch.location_name = launch.location_name.split(", ")[0]
                else:
                    launch.location_name = launch.location_name
                launch.save()
                launches.append(launch)
            return launches
        else:
            logger.error(response.status_code + ' ' + response)

    def check_next_launch(self):
        for launch in self.get_next_launches():
            if launch.netstamp > 0:
                current_time = datetime.utcnow()
                launch_time = datetime.utcfromtimestamp(int(launch.netstamp))
                if current_time <= launch_time:
                    diff = int((launch_time - current_time).total_seconds())
                    logger.info('%s in %s hours' % (launch.name, (diff / 60) / 60))
                    self.check_launch_window(diff, launch)

    def netstamp_changed(self, launch, notification, diff):
        logger.info('Netstamp change detected for %s' % launch.name)
        date = datetime.fromtimestamp(launch.netstamp).replace(tzinfo=pytz.UTC)
        message = 'SCHEDULE UPDATE: %s now launching in %s at %s.' % (launch.name,
                                                                      seconds_to_time(diff),
                                                                      date.strftime("%H:%M %Z (%d/%m)"))
        launch.save()
        self.send_to_twitter(message, notification)

        # If launch is within 24 hours...
        if 86400 >= diff > 3600:
            logger.info('Launch is within 24 hours, resetting notifications.')
            notification.wasNotifiedTwentyFourHour = True
            notification.wasNotifiedOneHour = False
            notification.wasNotifiedTenMinutes = False
        elif 3600 >= diff > 600:
            logger.info('Launch is within one hour, resetting Ten minute notifications.')
            notification.wasNotifiedOneHour = True
            notification.wasNotifiedTwentyFourHour = True
            notification.wasNotifiedTenMinutes = False
        elif diff <= 600:
            logger.info('Launch is within ten minutes hour.')
            notification.wasNotifiedOneHour = True
            notification.wasNotifiedTwentyFourHour = True
            notification.wasNotifiedTenMinutes = True
        notification.save()

    def check_twitter(self, diff, launch, notification):
        logger.info('Diff - %d for %s' % (diff, launch.name,))
        logger.debug('LAUNCH DATA: %s', json.dumps(launch, default=lambda o: o.__dict__))
        logger.debug('NOTIFICAITON DATA: %s', json.dumps(notification, default=json_default))
        if (notification.last_net_stamp is not None or 0)\
                and abs(notification.last_net_stamp - launch.netstamp) > 600\
                and diff <= 259200:
            self.netstamp_changed(launch, notification, diff)
        elif notification.last_twitter_post is not None:
            time_since_twitter = (datetime.now() - notification.last_twitter_post).total_seconds()
            logger.info('Seconds since last update on Twitter %d for %s' % (time_since_twitter,
                                                                            launch.name))
            if diff <= 86400 and notification.wasNotifiedTwentyFourHour is False:
                message = '%s | %s launching from %s in %s.' % (launch.name, launch.mission_name, launch.location_name,
                                                                seconds_to_time(diff))
                logger.info(message)
                self.send_to_twitter(message, notification)
            elif 3600 >= diff > 600 and time_since_twitter >= 43200 and notification.wasNotifiedOneHour is False:
                message = '%s | %s launching from %s in %s.' % (launch.name, launch.mission_name, launch.location_name,
                                                                seconds_to_time(diff))
                logger.info(message)
                self.send_to_twitter(message, notification)
            elif diff <= 600 and (time_since_twitter >= 600) and notification.wasNotifiedOneHour is False:
                message = '%s | %s launching from %s in %s.' % (launch.name, launch.mission_name, launch.location_name,
                                                                seconds_to_time(diff))
                logger.info(message)
                self.send_to_twitter(message, notification)
        elif notification.last_twitter_post is None:
            if diff <= 86400:
                message = '%s | %s launching from %s in %s.' % (launch.name, launch.mission_name, launch.location_name,
                                                                seconds_to_time(diff))
                logger.info(message)
                self.send_to_twitter(message, notification)
            elif 3600 >= diff > 600:
                message = '%s | %s launching from %s in %s.' % (launch.name, launch.mission_name, launch.location_name,
                                                                seconds_to_time(diff))
                logger.info(message)
                self.send_to_twitter(message, notification)
            elif diff <= 600:
                message = '%s | %s launching from %s in %s.' % (launch.name, launch.mission_name, launch.location_name,
                                                                seconds_to_time(diff))
                logger.info(message)
                self.send_to_twitter(message, notification)

def check_launch_window(self, diff, launch):
    notification = Notification.objects.get(launch=launch)
    self.check_twitter(diff, launch, notification)
    logger.info('Checking launch window for %s' % notification.launch.name)

    # If launch is within 24 hours...
    if 86400 >= diff > 3600 and not notification.wasNotifiedTwentyFourHour:
        logger.info('Launch is within 24 hours, sending notifications.')
        self.send_notification(launch)
        notification.wasNotifiedTwentyFourHour = True
    elif 3600 >= diff > 600 and not notification.wasNotifiedOneHour:
        logger.info('Launch is within one hour, sending notifications.')
        self.send_notification(launch)
        notification.wasNotifiedOneHour = True
    elif diff <= 600 and not notification.wasNotifiedTenMinutes:
        logger.info('Launch is within ten minutes, sending notifications.')
        self.send_notification(launch)
        notification.wasNotifiedTenMinutes = True
    else:
        logger.info('%s does not meet notification criteria.' % notification.launch.name)
    notification.save()

def send_notification(self, launch):
    self.one_signal.user_auth_key = self.app_auth_key
    self.one_signal.app_id = APP_ID
    logger.info('Creating notification for %s' % launch.name)

    # Create a notification
    contents = '%s launching from %s' % (launch.name, launch.location_name)
    kwargs = dict(
        content_available=True,
        included_segments=['Debug'],
        isAndroid=True,
        data={"silent": True,
              "background": True}
    )
    url = 'https://launchlibrary.net'
    heading = 'Space Launch Now'
    if not self.DEBUG:
        logger.debug('Sending notification - %s' % contents)
        response = self.one_signal.create_notification(contents, heading, url, **kwargs)
        if response.status_code == 200:
            logger.info('Response received %s %s' % (response.status_code, response.json()))
        else:
            logger.error(response.text)

        notification_data = response.json()
        notification_id = notification_data['id']
        assert notification_data['id'] and notification_data['recipients']

        # Get the notification
        response = self.one_signal.get_notification(APP_ID, notification_id, self.app_auth_key)
        notification_data = response.json()
        assert notification_data['id'] == notification_id
        assert notification_data['contents']['en'] == contents
<commit_msg>Fix notifications<commit_after>import json
import re

from django.utils.datetime_safe import datetime
import datetime as dtime
import pytz
from twitter import Twitter, OAuth, TwitterHTTPError
from bot.libraries.launchlibrarysdk import LaunchLibrarySDK
from bot.libraries.onesignalsdk import OneSignalSdk
from bot.utils.config import keys
from bot.models import Notification
from bot.utils.deserializer import launch_json_to_model
from bot.utils.util import seconds_to_time
import logging

AUTH_TOKEN_HERE = keys['AUTH_TOKEN_HERE']
APP_ID = keys['APP_ID']
DAEMON_SLEEP = 600
TAG = 'Notification Server'

# Get an instance of a logger
logger = logging.getLogger('bot.notifications')

def json_default(value):
    if isinstance(value, dtime.date):
        return dict(year=value.year, month=value.month, day=value.day)
    else:
        return value.__dict__

class NotificationServer:
    def __init__(self, debug=None, version=None):
        self.one_signal = OneSignalSdk(AUTH_TOKEN_HERE, APP_ID)
        if version is None:
            version = '1.2.1'
        self.launchLibrary = LaunchLibrarySDK(version=version)
        if debug is None:
            self.DEBUG = False
        else:
            self.DEBUG = debug
        response = self.one_signal.get_app(APP_ID)
        assert response.status_code == 200
        self.app = response.json()
        assert isinstance(self.app, dict)
        assert self.app['id'] and self.app['name'] and self.app['updated_at'] and self.app['created_at']
        self.app_auth_key = self.app['basic_auth_key']
        self.twitter = Twitter(
            auth=OAuth(keys['TOKEN_KEY'], keys['TOKEN_SECRET'], keys['CONSUMER_KEY'], keys['CONSUMER_SECRET'])
        )

    def send_to_twitter(self, message, notification):
        # Need to add actual twitter post here.

        if message.endswith(' (1/1)'):
            message = message[:-6]
        if len(message) > 120:
            end = message[-5:]
            if re.search("([1-9]*/[1-9])", end):
                message = (message[:111] + '... ' + end)
            else:
                message = (message[:117] + '...')
        logger.info('Sending to Twitter | %s | %s | DEBUG %s' % (message, str(len(message)), self.DEBUG))
        if not self.DEBUG:
            try:
                self.twitter.statuses.update(status=message)
            except TwitterHTTPError as e:
                logger.error("%s %s" % (str(e), message))

        notification.last_twitter_post = datetime.now()
        notification.last_net_stamp = notification.launch.netstamp
        notification.last_net_stamp_timestamp = datetime.now()
        logger.info('Updating Notification %s to timestamp %s' % (notification.launch.id,
                                                                  notification.last_twitter_post
                                                                  .strftime("%A %d. %B %Y")))

        notification.save()

    def get_next_launches(self):
        logger.info("Getting next launches...")
        response = self.launchLibrary.get_next_launches()
        if response.status_code is 200:
            response_json = response.json()
            launch_data = response_json['launches']
            launches = []
            for launch in launch_data:
                launch = launch_json_to_model(launch)
                if len(launch.location_name) > 20:
                    launch.location_name = launch.location_name.split(", ")[0]
                else:
                    launch.location_name = launch.location_name
                launch.save()
                launches.append(launch)
            return launches
        else:
            logger.error(response.status_code + ' ' + response)

    def check_next_launch(self):
        for launch in self.get_next_launches():
            if launch.netstamp > 0:
                current_time = datetime.utcnow()
                launch_time = datetime.utcfromtimestamp(int(launch.netstamp))
                if current_time <= launch_time:
                    diff = int((launch_time - current_time).total_seconds())
                    logger.info('%s in %s hours' % (launch.name, (diff / 60) / 60))
                    self.check_launch_window(diff, launch)

    def netstamp_changed(self, launch, notification, diff):
        logger.info('Netstamp change detected for %s' % launch.name)
        date = datetime.fromtimestamp(launch.netstamp).replace(tzinfo=pytz.UTC)
        message = 'SCHEDULE UPDATE: %s now launching in %s at %s.' % (launch.name,
                                                                      seconds_to_time(diff),
                                                                      date.strftime("%H:%M %Z (%d/%m)"))
        launch.save()
        self.send_to_twitter(message, notification)

        # If launch is within 24 hours...
        if 86400 >= diff > 3600:
            logger.info('Launch is within 24 hours, resetting notifications.')
            notification.wasNotifiedTwentyFourHour = True
            notification.wasNotifiedOneHour = False
            notification.wasNotifiedTenMinutes = False
        elif 3600 >= diff > 600:
            logger.info('Launch is within one hour, resetting Ten minute notifications.')
            notification.wasNotifiedOneHour = True
            notification.wasNotifiedTwentyFourHour = True
            notification.wasNotifiedTenMinutes = False
        elif diff <= 600:
            logger.info('Launch is within ten minutes hour.')
            notification.wasNotifiedOneHour = True
            notification.wasNotifiedTwentyFourHour = True
            notification.wasNotifiedTenMinutes = True
        notification.save()

    def check_twitter(self, diff, launch, notification):
        logger.info('Diff - %d for %s' % (diff, launch.name,))
        logger.debug('LAUNCH DATA: %s', json.dumps(launch, default=lambda o: o.__dict__))
        logger.debug('NOTIFICAITON DATA: %s', json.dumps(notification, default=json_default))
        if (notification.last_net_stamp is not None or 0)\
                and abs(notification.last_net_stamp - launch.netstamp) > 600\
                and diff <= 259200:
            self.netstamp_changed(launch, notification, diff)
        elif notification.last_twitter_post is not None:
            time_since_twitter = (datetime.now() - notification.last_twitter_post).total_seconds()
            logger.info('Seconds since last update on Twitter %d for %s' % (time_since_twitter,
                                                                            launch.name))
            if diff <= 86400 and notification.wasNotifiedTwentyFourHour is False:
                message = '%s | %s launching from %s in %s.' % (launch.name, launch.mission_name, launch.location_name,
                                                                seconds_to_time(diff))
                logger.info(message)
                self.send_to_twitter(message, notification)
            elif 3600 >= diff > 600 and time_since_twitter >= 43200 and notification.wasNotifiedOneHour is False:
                message = '%s | %s launching from %s in %s.' % (launch.name, launch.mission_name, launch.location_name,
                                                                seconds_to_time(diff))
                logger.info(message)
                self.send_to_twitter(message, notification)
            elif diff <= 600 and (time_since_twitter >= 600) and notification.wasNotifiedOneHour is False:
                message = '%s | %s launching from %s in %s.' % (launch.name, launch.mission_name, launch.location_name,
                                                                seconds_to_time(diff))
                logger.info(message)
                self.send_to_twitter(message, notification)
        elif notification.last_twitter_post is None:
            if diff <= 86400:
                message = '%s | %s launching from %s in %s.' % (launch.name, launch.mission_name, launch.location_name,
                                                                seconds_to_time(diff))
                logger.info(message)
                self.send_to_twitter(message, notification)
            elif 3600 >= diff > 600:
                message = '%s | %s launching from %s in %s.' % (launch.name, launch.mission_name, launch.location_name,
                                                                seconds_to_time(diff))
                logger.info(message)
                self.send_to_twitter(message, notification)
            elif diff <= 600:
                message = '%s | %s launching from %s in %s.' % (launch.name, launch.mission_name, launch.location_name,
                                                                seconds_to_time(diff))
                logger.info(message)
                self.send_to_twitter(message, notification)

    def check_launch_window(self, diff, launch):
        notification = Notification.objects.get(launch=launch)
        self.check_twitter(diff, launch, notification)
        logger.info('Checking launch window for %s' % notification.launch.name)

        # If launch is within 24 hours...
        if 86400 >= diff > 3600 and not notification.wasNotifiedTwentyFourHour:
            logger.info('Launch is within 24 hours, sending notifications.')
            self.send_notification(launch)
            notification.wasNotifiedTwentyFourHour = True
        elif 3600 >= diff > 600 and not notification.wasNotifiedOneHour:
            logger.info('Launch is within one hour, sending notifications.')
            self.send_notification(launch)
            notification.wasNotifiedOneHour = True
        elif diff <= 600 and not notification.wasNotifiedTenMinutes:
            logger.info('Launch is within ten minutes, sending notifications.')
            self.send_notification(launch)
            notification.wasNotifiedTenMinutes = True
        else:
            logger.info('%s does not meet notification criteria.' % notification.launch.name)
        notification.save()

    def send_notification(self, launch):
        self.one_signal.user_auth_key = self.app_auth_key
        self.one_signal.app_id = APP_ID
        logger.info('Creating notification for %s' % launch.name)

        # Create a notification
        contents = '%s launching from %s' % (launch.name, launch.location_name)
        kwargs = dict(
            content_available=True,
            included_segments=['Debug'],
            isAndroid=True,
            data={"silent": True,
                  "background": True}
        )
        url = 'https://launchlibrary.net'
        heading = 'Space Launch Now'
        if not self.DEBUG:
            logger.debug('Sending notification - %s' % contents)
            response = self.one_signal.create_notification(contents, heading, url, **kwargs)
            if response.status_code == 200:
                logger.info('Response received %s %s' % (response.status_code, response.json()))
            else:
                logger.error(response.text)

            notification_data = response.json()
            notification_id = notification_data['id']
            assert notification_data['id'] and notification_data['recipients']

            # Get the notification
            response = self.one_signal.get_notification(APP_ID, notification_id, self.app_auth_key)
            notification_data = response.json()
            assert notification_data['id'] == notification_id
            assert notification_data['contents']['en'] == contents
<|endoftext|>


<commit_before>from keras.models import Sequential, Model
from keras.layers import Convolution2D, Flatten, Dense
from keras.layers import Input, MaxPooling2D, UpSampling2D, merge
from keras.callbacks import EarlyStopping
from iro.data import Data

class Generator:
    def __init__(self):
        inputs = Input((128, 128, 3))
        conv1 = Convolution2D(32, 3, 3, activation='relu', border_mode='same')(inputs)
        conv1 = Convolution2D(32, 3, 3, activation='relu', border_mode='same')(conv1)
        pool1 = MaxPooling2D(pool_size=(2, 2))(conv1)

        conv2 = Convolution2D(64, 3, 3, activation='relu', border_mode='same')(pool1)
        conv2 = Convolution2D(64, 3, 3, activation='relu', border_mode='same')(conv2)
        pool2 = MaxPooling2D(pool_size=(2, 2))(conv2)

        conv3 = Convolution2D(128, 3, 3, activation='relu', border_mode='same')(pool2)
        conv3 = Convolution2D(128, 3, 3, activation='relu', border_mode='same')(conv3)
        pool3 = MaxPooling2D(pool_size=(2, 2))(conv3)

        conv4 = Convolution2D(256, 3, 3, activation='relu', border_mode='same')(pool3)
        conv4 = Convolution2D(256, 3, 3, activation='relu', border_mode='same')(conv4)

        up5 = merge([UpSampling2D(size=(2, 2))(conv4), conv3], mode='concat', concat_axis=3)
        conv5 = Convolution2D(128, 3, 3, activation='relu', border_mode='same')(up5)
        conv5 = Convolution2D(128, 3, 3, activation='relu', border_mode='same')(conv5)

        up6 = merge([UpSampling2D(size=(2, 2))(conv5), conv2], mode='concat', concat_axis=3)
        conv6 = Convolution2D(64, 3, 3, activation='relu', border_mode='same')(up6)
        conv6 = Convolution2D(64, 3, 3, activation='relu', border_mode='same')(conv6)

        up7 = merge([UpSampling2D(size=(2, 2))(conv6), conv1], mode='concat', concat_axis=3)
        conv7 = Convolution2D(32, 3, 3, activation='relu', border_mode='same')(up7)
        conv7 = Convolution2D(32, 3, 3, activation='relu', border_mode='same')(conv7)

        conv8 = Convolution2D(3, 1, 1, activation='sigmoid')(conv7)
        self.network = Model(input=inputs, output=conv8)
        self.network.compile(
            optimizer='RMSprop',
            loss='mse',
        )

class Discriminator:
    def __init__(self):
        self.network = Sequential([
            Convolution2D(128, 2, 2, activation='relu', border_mode='same', input_shape=(128, 128, 3)),
            Convolution2D(128, 3, 3, activation='relu', border_mode='same'),
            MaxPooling2D(pool_size=(2,2)),
            Convolution2D(64, 2, 2, activation='relu', border_mode='same'),
            Convolution2D(64, 3, 3, activation='relu', border_mode='same'),
            MaxPooling2D(pool_size=(2, 2)),
            Convolution2D(32, 2, 2, activation='relu', border_mode='same'),
            Convolution2D(32, 3, 3, activation='relu', border_mode='same'),
            MaxPooling2D(pool_size=(2, 2)),
            Flatten(),
            Dense(64, activation='relu'),
            Dense(1),
        ])
        self.network.compile(
            optimizer='RMSprop',
            loss='mse',
        )

class GAN:
    def __init__(self):
        self.data = None
        self.generator_network = Generator().network
        self.discriminator_network = Discriminator().network

        self.gan_network = Sequential([
            self.generator_network,
            self.discriminator_network,
        ], name='GAN')

        self.gan_network.compile(
            optimizer='RMSprop',
            loss='mse',
        )

    def load_data(self):
        self.data = Data()

    def load_weights(self,
                     generator_weights='./checkpoint/generator.new.hdf5',
                     discriminator_weights='./checkpoint/discriminator.new.hdf5',
                     gan_weights='./checkpoint/gan.new.hdf5'):
        self.generator_network.load_weights(generator_weights)
        self.discriminator_network.load_weights(discriminator_weights)
        self.gan_network.load_weights(gan_weights)

    def train(self, batch_size=8, nb_epoch=10, samples=128, nb_worker=1):
        while True:
            print('Training Discriminator')
            self.discriminator_network.trainable = True

            self.discriminator_network.fit_generator(
                self.data.discriminator_next(self.generator_network, batch_size=batch_size),
                samples_per_epoch=samples,
                nb_epoch=nb_epoch,
                nb_worker=nb_worker,
                callbacks=[EarlyStopping(monitor='loss', patience=2, verbose=0, mode='auto')]
            )

            print('Training Generator')
            self.discriminator_network.trainable = False
            self.gan_network.fit_generator(
                self.data.gan_next(batch_size=batch_size),
                samples_per_epoch=samples,
                nb_epoch=nb_epoch,
                nb_worker=nb_worker,
                callbacks=[EarlyStopping(monitor='loss', patience=2, verbose=0, mode='auto')]
            )

            self.discriminator_network.save_weights('./checkpoint/discriminator.new.hdf5')
            self.generator_network.save_weights('./checkpoint/generator.new.hdf5')
            self.gan_network.save_weights('./checkpoint/gan.new.hdf5')
<commit_msg>Freeze network every time trainable changed<commit_after>from keras.models import Sequential, Model
from keras.layers import Convolution2D, Flatten, Dense
from keras.layers import Input, MaxPooling2D, UpSampling2D, merge
from keras.callbacks import EarlyStopping
from iro.data import Data

class Generator:
    def __init__(self):
        inputs = Input((128, 128, 3))
        conv1 = Convolution2D(32, 3, 3, activation='relu', border_mode='same')(inputs)
        conv1 = Convolution2D(32, 3, 3, activation='relu', border_mode='same')(conv1)
        pool1 = MaxPooling2D(pool_size=(2, 2))(conv1)

        conv2 = Convolution2D(64, 3, 3, activation='relu', border_mode='same')(pool1)
        conv2 = Convolution2D(64, 3, 3, activation='relu', border_mode='same')(conv2)
        pool2 = MaxPooling2D(pool_size=(2, 2))(conv2)

        conv3 = Convolution2D(128, 3, 3, activation='relu', border_mode='same')(pool2)
        conv3 = Convolution2D(128, 3, 3, activation='relu', border_mode='same')(conv3)
        pool3 = MaxPooling2D(pool_size=(2, 2))(conv3)

        conv4 = Convolution2D(256, 3, 3, activation='relu', border_mode='same')(pool3)
        conv4 = Convolution2D(256, 3, 3, activation='relu', border_mode='same')(conv4)

        up5 = merge([UpSampling2D(size=(2, 2))(conv4), conv3], mode='concat', concat_axis=3)
        conv5 = Convolution2D(128, 3, 3, activation='relu', border_mode='same')(up5)
        conv5 = Convolution2D(128, 3, 3, activation='relu', border_mode='same')(conv5)

        up6 = merge([UpSampling2D(size=(2, 2))(conv5), conv2], mode='concat', concat_axis=3)
        conv6 = Convolution2D(64, 3, 3, activation='relu', border_mode='same')(up6)
        conv6 = Convolution2D(64, 3, 3, activation='relu', border_mode='same')(conv6)

        up7 = merge([UpSampling2D(size=(2, 2))(conv6), conv1], mode='concat', concat_axis=3)
        conv7 = Convolution2D(32, 3, 3, activation='relu', border_mode='same')(up7)
        conv7 = Convolution2D(32, 3, 3, activation='relu', border_mode='same')(conv7)

        conv8 = Convolution2D(3, 1, 1, activation='sigmoid')(conv7)
        self.network = Model(input=inputs, output=conv8)
        self.network.compile(
            optimizer='RMSprop',
            loss='mse',
        )

class Discriminator:
    def __init__(self):
        self.network = Sequential([
            Convolution2D(128, 2, 2, activation='relu', border_mode='same', input_shape=(128, 128, 3)),
            Convolution2D(128, 3, 3, activation='relu', border_mode='same'),
            MaxPooling2D(pool_size=(2,2)),
            Convolution2D(64, 2, 2, activation='relu', border_mode='same'),
            Convolution2D(64, 3, 3, activation='relu', border_mode='same'),
            MaxPooling2D(pool_size=(2, 2)),
            Convolution2D(32, 2, 2, activation='relu', border_mode='same'),
            Convolution2D(32, 3, 3, activation='relu', border_mode='same'),
            MaxPooling2D(pool_size=(2, 2)),
            Flatten(),
            Dense(64, activation='relu'),
            Dense(1),
        ])
        self.network.compile(
            optimizer='RMSprop',
            loss='mse',
        )

class GAN:
    def __init__(self):
        self.data = None
        self.generator_network = Generator().network
        self.discriminator_network = Discriminator().network

        self.gan_network = Sequential([
            self.generator_network,
            self.discriminator_network,
        ], name='GAN')

        self.gan_network.compile(
            optimizer='RMSprop',
            loss='mse',
        )

    def load_data(self):
        self.data = Data()

    def load_weights(self,
                     generator_weights='./checkpoint/generator.new.hdf5',
                     discriminator_weights='./checkpoint/discriminator.new.hdf5',
                     gan_weights='./checkpoint/gan.new.hdf5'):
        self.generator_network.load_weights(generator_weights)
        self.discriminator_network.load_weights(discriminator_weights)
        self.gan_network.load_weights(gan_weights)

    def train(self, batch_size=8, nb_epoch=1, samples=128, nb_worker=1):
        while True:
            print('Training Discriminator')
            self.discriminator_network.trainable = True
            self.discriminator_network.compile(
                optimizer='RMSprop',
                loss='mse',
            )

            self.discriminator_network.fit_generator(
                self.data.discriminator_next(self.generator_network, batch_size=batch_size),
                samples_per_epoch=samples,
                nb_epoch=nb_epoch,
                nb_worker=nb_worker)

            print('Training Generator')
            self.discriminator_network.trainable = False
            self.discriminator_network.compile(
                optimizer='RMSprop',
                loss='mse',
            )

            self.gan_network.fit_generator(
                self.data.gan_next(batch_size=batch_size),
                samples_per_epoch=samples,
                nb_epoch=nb_epoch,
                nb_worker=nb_worker)

            self.discriminator_network.save_weights('./checkpoint/discriminator.new.hdf5')
            self.generator_network.save_weights('./checkpoint/generator.new.hdf5')
            self.gan_network.save_weights('./checkpoint/gan.new.hdf5')
<|endoftext|>


<commit_before>import os
import tempfile
from glob import glob
import re
import logging
import shutil
import numpy as np
import pyfits
import argparse
from configobj import ConfigObj

import Ska.arc5gl
import Ska.DBI
from Chandra.Time import DateTime
import Ska.File

# archive_dir set by main()
archive_dir = None
arc5 = Ska.arc5gl.Arc5gl()
aca_db = Ska.DBI.DBI(dbi='sybase', server='sybase', user='aca_read')
apstat = Ska.DBI.DBI(dbi='sybase', server='sqlocc', database='axafapstat')

logger = logging.getLogger('asp1 fetch')
logger.setLevel(logging.INFO)
logger.addHandler(logging.StreamHandler())

# borrowed from eng_archive
archfiles_hdr_cols = ('tstart', 'tstop', 'caldbver', 'content',
                      'ascdsver', 'revision', 'date')

def get_arch_info(i, f, archfiles, db):

    """Read filename ``f`` with index ``i`` (position within list of
    filenames).  The file has type ``filetype`` and will be added to
    MSID file at row index ``row``.  ``colnames`` is the list of
    column names for the content type (not used here).
    """

    filename = os.path.basename(f)

    # Read FITS archive file and accumulate data into dats list and
    # header into headers dict
    logger.debug('Reading (%d / %d) %s' % (i, len(archfiles), filename))
    hdus = pyfits.open(f)
    hdu = hdus[1]

    # Accumlate relevant info about archfile that will be ingested into
    # MSID h5 files.  Commit info before h5 ingest so if there is a failure
    # the needed info will be available to do the repair.
    archfiles_row = dict((x, hdu.header.get(x.upper()))
                         for x in archfiles_hdr_cols)
    archfiles_row['checksum'] = hdu._checksum
    archfiles_row['filename'] = filename
    archfiles_row['filetime'] = int(
        re.search(r'(\d+)', archfiles_row['filename']).group(1))
    filedate = DateTime(archfiles_row['filetime']).date
    year, doy = (int(x)
                 for x in re.search(r'(\d\d\d\d):(\d\d\d)', filedate).groups())
    archfiles_row['year'] = year
    archfiles_row['doy'] = doy
    hdus.close()
    return archfiles_row

def get_options():
#    from optparse import OptionParser
#    parser = OptionParser()
    parser = argparse.ArgumentParser()
    parser.add_argument("-c", "--conf-file",
                        help="Config file",
                        metavar="FILE",
                        default="archive.conf")
    # if we've got any of the below-defined options
    # they go into remaining_argv instead of throwing an error
    args, remaining_argv = parser.parse_known_args()
    defaults = dict(version='last')
    if args.conf_file:
        config = ConfigParser.SafeConfigParser()
        config.read([args.conf_file])
        defaults = dict(config.items("asp_l1_defaults"))
    parser.set_defaults(**defaults)
    parser.add_argument("--obsid",
                        type=int)
    parser.add_argument("--version",
                        default='last')
    parser.add_argument("--firstrun",
                        action='store_true',
                        help="for archive init., ignore rev in aspect_1 table")
    parser.add_argument("--data-root")
    parser.add_argument("--temp-root")

    opt = parser.parse_args(remaining_argv)
    return opt

def get_file_ver(tempdir, fileglob="*fidpr1*fits*"):
    files = glob(os.path.join(tempdir, fileglob))
    if not files:
        return None
    versions = {}
    for f in files:
        fmatch = re.search('pcadf\d+N(\d{3})_', f)
        if fmatch:
            versions[int(fmatch.group(1))] = 1
        if len(versions) > 1:
            raise ValueError("Different version files in %s" % tempdir)
    # update version to number
    version = versions.keys()[0]
    return version

def get_ver_num(obsid, version='default'):
    # this is obi agnostic, all obis should be same
    # version anyway...
    tempdir = tempfile.mkdtemp()
    arc5.sendline("reset")
    arc5.sendline("cd %s" % tempdir)
    arc5.sendline("obsid=%d" % obsid)
    if version != 'default':
        arc5.sendline("version=%s" % version)
    # just fetch the aspect solution to start
    arc5.sendline("get asp1{fidprops}")
    version = get_file_ver(tempdir)
    for afile in glob(os.path.join(tempdir, "*")):
        os.remove(afile)
    os.removedirs(tempdir)
    return version

def get_asp(obsid, version='last'):
    n_version = get_ver_num(obsid, version=version)
    if n_version is None:
        raise ValueError("No ASP1 for ver %s" % version)
    # use numeric version instead of 'last' or 'default'
    version = n_version

    # set up directory for data
    strobs = "%05d_v%02d" % (obsid, version)
    chunk_dir = strobs[0:2]
    obs_dir = os.path.join(archive_dir, chunk_dir, strobs)
    if not os.path.exists(obs_dir):
        logger.info("making directory %s" % obs_dir)
        os.makedirs(obs_dir)
    else:
        logger.info("obsid dir %s already exists" % obs_dir)
        return

    # get data
    tempdir = Ska.File.TempDir().name
    arc5.sendline("reset")
    arc5.sendline("cd %s" % tempdir)
    logger.info("retrieving data for %d in %s" % (obsid, tempdir))
    arc5.sendline("obsid=%d" % obsid)
    # if multi-obi, we'll need to be specific
    obis = apstat.fetchall(
        "select distinct obi from obidet_0_5 where obsid = %d" % obsid)
    if len(obis) > 1:
        minobi = np.min(obis['obi'])
        logger.info("limiting arc5gl to obi %d" % minobi)
        arc5.sendline("obi=%d" % minobi)
    arc5.sendline("version=%s" % version)
    arc5.sendline("get asp1")
    logger.info("copying asp1 from %s to %s" % (tempdir, obs_dir))
    archfiles = glob(os.path.join(tempdir, "*"))
    if archfiles:
        db = Ska.DBI.DBI(dbi='sqlite', server='archfiles.db3',
                         autocommit=False)
        existing = db.fetchall(
            "select * from archfiles where obsid = %d and revision = '%s'"
            % (obsid, version))
        for i, f in enumerate(archfiles):
            if len(existing) and f in existing['filename']:
                raise ValueError()
            arch_info = get_arch_info(i, f, archfiles, db)
            arch_info['obsid'] = obsid
            db.insert(arch_info, 'archfiles')
            shutil.copy(f, obs_dir)
            #os.remove(f)
    #os.removedirs(tempdir)
        db.commit()
    return obs_dir, chunk_dir

# this needs help
def update_link(obsid):
    # for the obsid get the default and last version numbers
    default_ver = get_ver_num(obsid, 'default')
    last_ver = get_ver_num(obsid, 'last')

    # set up directory for data
    chunk_dir = ("%05d" % obsid)[0:2]
    chunk_dir_path = os.path.join(archive_dir, chunk_dir)
    obs_ln = os.path.join(archive_dir, chunk_dir, "%05d" % obsid)
    obs_ln_last = os.path.join(archive_dir, chunk_dir,
                               "%05d_last" % obsid)

    if default_ver is not None:
        def_ver_dir = os.path.join(archive_dir, chunk_dir,
                                   '%05d_v%02d' % (obsid, default_ver))
        # link the default version
        logger.info("linking %s -> %s" % (
                os.path.relpath(def_ver_dir, chunk_dir_path),
                obs_ln))
        # don't use os.path.exists here because if we have a broken
        # link we want to remove it too
        if os.path.islink(obs_ln):
            os.unlink(obs_ln)
        os.symlink(
                os.path.relpath(def_ver_dir, chunk_dir_path),
                obs_ln)

    # nothing to do if last_ver undefined
    if last_ver is None:
        return

    last_ver_dir = os.path.join(archive_dir, chunk_dir,
                                '%05d_v%02d' % (obsid, last_ver))
    # if last and default are different, and we have last data,
    # make a link to it
    if last_ver != default_ver:
        if os.path.exists(last_ver_dir):
            obs_ln_last = os.path.join(archive_dir, chunk_dir,
                                       "%05d_last" % obsid)
            logger.info("linking %s -> %s" % (
                    os.path.relpath(last_ver_dir, chunk_dir_path),
                    obs_ln_last))
            if os.path.islink(obs_ln_last):
                os.unlink(obs_ln_last)
            os.symlink(
                    os.path.relpath(last_ver_dir, chunk_dir_path),
                    obs_ln_last)
    else:
        # if default and last are the same and we have a last link,
        # delete it
        if os.path.exists(obs_ln_last):
            logger.info("removing outdated link %s"
                        % obs_ln_last)
            os.remove(obs_ln_last)

def get_todo_from_links():
    chunk_dirs = glob(os.path.join(archive_dir, "??"))
    todo_obs = []
    for cdir in chunk_dirs:
        last_links = glob(os.path.join(cdir, "?????_last"))
        obs_links = glob(os.path.join(cdir, "?????"))
        for link in last_links:
            lmatch = re.search('(\d{5})(_last)?$', link)
            if lmatch:
                obs = dict(obsid=int(lmatch.group(1)),
                           revision='default')
                todo_obs.append(obs)
        for link in obs_links:
            if not os.path.exists(link):
                lmatch = re.search('(\d{5})$', link)
                if lmatch:
                    obs = dict(obsid=int(lmatch.group(1)),
                               revision='default')
                    todo_obs.append(obs)

    return todo_obs

def main(opt):

    global archive_dir
    archive_dir = opt.data_root
    last_id_file = os.path.join(archive_dir, 'last_asp1_id.txt')

    # if an obsid is requested, just do that
    if opt.obsid:
        get_asp(opt.obsid, version=opt.version)
        update_link(opt.obsid)
        return

    # look for all previous "provisional" aspect solutions
    # and broken links and and update if possible
    prov_data = get_todo_from_links()
    for obs in prov_data:
        logger.info("running get_asp for obsid %d ver %s, "
                    % (obs['obsid'], obs['revision']))
        try:
            get_asp(obs['obsid'], obs['revision'])
        except ValueError as ve:
            logger.info("skipping %d, default ver not available"
                        % obs['obsid'])
            logger.debug(ve)
        update_link(obs['obsid'])

    # if no obsid specified, try to retrieve all asp_1 runs
    # since tool last run
    # use an aspect_1_id in a file
    last_id = 0
    if os.path.exists(last_id_file):
        last_id_fh = open(last_id_file)
        last_id = int(last_id_fh.read().rstrip())
        last_id_fh.close()
    todo = apstat.fetchall("""select * from aspect_1
                              where aspect_1_id > %d
                              order by aspect_1_id"""
                           % last_id)
    for obs in todo:
        logger.info("running get_asp for obsid %d aspect run on %s"
                    % (obs['obsid'], obs['ap_date']))
        if opt.firstrun:
            # ignore aspect_1 table versions and just try for default
            # also ignore errors in this case
            try:
                get_asp(obs['obsid'], version='default')
            except ValueError as ve:
                logger.info("skipping %d, default ver not available"
                            % obs['obsid'])
                logger.debug(ve)
        else:
            get_asp(obs['obsid'], version=obs['revision'])
        update_link(obs['obsid'])
        last_id_fh = open(last_id_file, 'w')
        last_id_fh.write("%d" % obs['aspect_1_id'])
        last_id_fh.close()
    if not len(todo):
        logger.info("No new aspect_1 data")

if __name__ == '__main__':
    opt = get_options()
    main(opt)
<commit_msg>Set to skip rest of loops (link building) on error<commit_after>import os
import tempfile
from glob import glob
import re
import logging
import shutil
import numpy as np
import pyfits
import argparse
from configobj import ConfigObj

import Ska.arc5gl
import Ska.DBI
from Chandra.Time import DateTime
import Ska.File

# archive_dir set by main()
archive_dir = None
arc5 = Ska.arc5gl.Arc5gl()
aca_db = Ska.DBI.DBI(dbi='sybase', server='sybase', user='aca_read')
apstat = Ska.DBI.DBI(dbi='sybase', server='sqlocc', database='axafapstat')

logger = logging.getLogger('asp1 fetch')
logger.setLevel(logging.INFO)
logger.addHandler(logging.StreamHandler())

# borrowed from eng_archive
archfiles_hdr_cols = ('tstart', 'tstop', 'caldbver', 'content',
                      'ascdsver', 'revision', 'date')

def get_arch_info(i, f, archfiles, db):

    """Read filename ``f`` with index ``i`` (position within list of
    filenames).  The file has type ``filetype`` and will be added to
    MSID file at row index ``row``.  ``colnames`` is the list of
    column names for the content type (not used here).
    """

    filename = os.path.basename(f)

    # Read FITS archive file and accumulate data into dats list and
    # header into headers dict
    logger.debug('Reading (%d / %d) %s' % (i, len(archfiles), filename))
    hdus = pyfits.open(f)
    hdu = hdus[1]

    # Accumlate relevant info about archfile that will be ingested into
    # MSID h5 files.  Commit info before h5 ingest so if there is a failure
    # the needed info will be available to do the repair.
    archfiles_row = dict((x, hdu.header.get(x.upper()))
                         for x in archfiles_hdr_cols)
    archfiles_row['checksum'] = hdu._checksum
    archfiles_row['filename'] = filename
    archfiles_row['filetime'] = int(
        re.search(r'(\d+)', archfiles_row['filename']).group(1))
    filedate = DateTime(archfiles_row['filetime']).date
    year, doy = (int(x)
                 for x in re.search(r'(\d\d\d\d):(\d\d\d)', filedate).groups())
    archfiles_row['year'] = year
    archfiles_row['doy'] = doy
    hdus.close()
    return archfiles_row

def get_options():
#    from optparse import OptionParser
#    parser = OptionParser()
    parser = argparse.ArgumentParser()
    parser.add_argument("-c", "--conf-file",
                        help="Config file",
                        metavar="FILE",
                        default="archive.conf")
    # if we've got any of the below-defined options
    # they go into remaining_argv instead of throwing an error
    args, remaining_argv = parser.parse_known_args()
    defaults = dict(version='last')
    if args.conf_file:
        config = ConfigParser.SafeConfigParser()
        config.read([args.conf_file])
        defaults = dict(config.items("asp_l1_defaults"))
    parser.set_defaults(**defaults)
    parser.add_argument("--obsid",
                        type=int)
    parser.add_argument("--version",
                        default='last')
    parser.add_argument("--firstrun",
                        action='store_true',
                        help="for archive init., ignore rev in aspect_1 table")
    parser.add_argument("--data-root")
    parser.add_argument("--temp-root")

    opt = parser.parse_args(remaining_argv)
    return opt

def get_file_ver(tempdir, fileglob="*fidpr1*fits*"):
    files = glob(os.path.join(tempdir, fileglob))
    if not files:
        return None
    versions = {}
    for f in files:
        fmatch = re.search('pcadf\d+N(\d{3})_', f)
        if fmatch:
            versions[int(fmatch.group(1))] = 1
        if len(versions) > 1:
            raise ValueError("Different version files in %s" % tempdir)
    # update version to number
    version = versions.keys()[0]
    return version

def get_ver_num(obsid, version='default'):
    # this is obi agnostic, all obis should be same
    # version anyway...
    tempdir = tempfile.mkdtemp()
    arc5.sendline("reset")
    arc5.sendline("cd %s" % tempdir)
    arc5.sendline("obsid=%d" % obsid)
    if version != 'default':
        arc5.sendline("version=%s" % version)
    # just fetch the aspect solution to start
    arc5.sendline("get asp1{fidprops}")
    version = get_file_ver(tempdir)
    for afile in glob(os.path.join(tempdir, "*")):
        os.remove(afile)
    os.removedirs(tempdir)
    return version

def get_asp(obsid, version='last'):
    n_version = get_ver_num(obsid, version=version)
    if n_version is None:
        raise ValueError("No ASP1 for ver %s" % version)
    # use numeric version instead of 'last' or 'default'
    version = n_version

    # set up directory for data
    strobs = "%05d_v%02d" % (obsid, version)
    chunk_dir = strobs[0:2]
    obs_dir = os.path.join(archive_dir, chunk_dir, strobs)
    if not os.path.exists(obs_dir):
        logger.info("making directory %s" % obs_dir)
        os.makedirs(obs_dir)
    else:
        logger.info("obsid dir %s already exists" % obs_dir)
        return

    # get data
    tempdir = Ska.File.TempDir().name
    arc5.sendline("reset")
    arc5.sendline("cd %s" % tempdir)
    logger.info("retrieving data for %d in %s" % (obsid, tempdir))
    arc5.sendline("obsid=%d" % obsid)
    # if multi-obi, we'll need to be specific
    obis = apstat.fetchall(
        "select distinct obi from obidet_0_5 where obsid = %d" % obsid)
    if len(obis) > 1:
        minobi = np.min(obis['obi'])
        logger.info("limiting arc5gl to obi %d" % minobi)
        arc5.sendline("obi=%d" % minobi)
    arc5.sendline("version=%s" % version)
    arc5.sendline("get asp1")
    logger.info("copying asp1 from %s to %s" % (tempdir, obs_dir))
    archfiles = glob(os.path.join(tempdir, "*"))
    if archfiles:
        db = Ska.DBI.DBI(dbi='sqlite', server='archfiles.db3',
                         autocommit=False)
        existing = db.fetchall(
            "select * from archfiles where obsid = %d and revision = '%s'"
            % (obsid, version))
        for i, f in enumerate(archfiles):
            if len(existing) and f in existing['filename']:
                raise ValueError()
            arch_info = get_arch_info(i, f, archfiles, db)
            arch_info['obsid'] = obsid
            db.insert(arch_info, 'archfiles')
            shutil.copy(f, obs_dir)
            #os.remove(f)
    #os.removedirs(tempdir)
        db.commit()
    return obs_dir, chunk_dir

# this needs help
def update_link(obsid):
    # for the obsid get the default and last version numbers
    default_ver = get_ver_num(obsid, 'default')
    last_ver = get_ver_num(obsid, 'last')

    # set up directory for data
    chunk_dir = ("%05d" % obsid)[0:2]
    chunk_dir_path = os.path.join(archive_dir, chunk_dir)
    obs_ln = os.path.join(archive_dir, chunk_dir, "%05d" % obsid)
    obs_ln_last = os.path.join(archive_dir, chunk_dir,
                               "%05d_last" % obsid)

    if default_ver is not None:
        def_ver_dir = os.path.join(archive_dir, chunk_dir,
                                   '%05d_v%02d' % (obsid, default_ver))
        # link the default version
        logger.info("linking %s -> %s" % (
                os.path.relpath(def_ver_dir, chunk_dir_path),
                obs_ln))
        # don't use os.path.exists here because if we have a broken
        # link we want to remove it too
        if os.path.islink(obs_ln):
            os.unlink(obs_ln)
        os.symlink(
                os.path.relpath(def_ver_dir, chunk_dir_path),
                obs_ln)

    # nothing to do if last_ver undefined
    if last_ver is None:
        return

    last_ver_dir = os.path.join(archive_dir, chunk_dir,
                                '%05d_v%02d' % (obsid, last_ver))
    # if last and default are different, and we have last data,
    # make a link to it
    if last_ver != default_ver:
        if os.path.exists(last_ver_dir):
            obs_ln_last = os.path.join(archive_dir, chunk_dir,
                                       "%05d_last" % obsid)
            logger.info("linking %s -> %s" % (
                    os.path.relpath(last_ver_dir, chunk_dir_path),
                    obs_ln_last))
            if os.path.islink(obs_ln_last):
                os.unlink(obs_ln_last)
            os.symlink(
                    os.path.relpath(last_ver_dir, chunk_dir_path),
                    obs_ln_last)
    else:
        # if default and last are the same and we have a last link,
        # delete it
        if os.path.exists(obs_ln_last):
            logger.info("removing outdated link %s"
                        % obs_ln_last)
            os.remove(obs_ln_last)

def get_todo_from_links():
    chunk_dirs = glob(os.path.join(archive_dir, "??"))
    todo_obs = []
    for cdir in chunk_dirs:
        last_links = glob(os.path.join(cdir, "?????_last"))
        obs_links = glob(os.path.join(cdir, "?????"))
        for link in last_links:
            lmatch = re.search('(\d{5})(_last)?$', link)
            if lmatch:
                obs = dict(obsid=int(lmatch.group(1)),
                           revision='default')
                todo_obs.append(obs)
        for link in obs_links:
            if not os.path.exists(link):
                lmatch = re.search('(\d{5})$', link)
                if lmatch:
                    obs = dict(obsid=int(lmatch.group(1)),
                               revision='default')
                    todo_obs.append(obs)

    return todo_obs

def main(opt):

    global archive_dir
    archive_dir = opt.data_root
    last_id_file = os.path.join(archive_dir, 'last_asp1_id.txt')

    # if an obsid is requested, just do that
    if opt.obsid:
        get_asp(opt.obsid, version=opt.version)
        update_link(opt.obsid)
        return

    # look for all previous "provisional" aspect solutions
    # and broken links and and update if possible
    prov_data = get_todo_from_links()
    for obs in prov_data:
        logger.info("running get_asp for obsid %d ver %s, "
                    % (obs['obsid'], obs['revision']))
        try:
            get_asp(obs['obsid'], obs['revision'])
        except ValueError as ve:
            logger.info("skipping %d, default ver not available"
                        % obs['obsid'])
            logger.debug(ve)
            continue
        update_link(obs['obsid'])

    # if no obsid specified, try to retrieve all asp_1 runs
    # since tool last run
    # use an aspect_1_id in a file
    last_id = 0
    if os.path.exists(last_id_file):
        last_id_fh = open(last_id_file)
        last_id = int(last_id_fh.read().rstrip())
        last_id_fh.close()
    todo = apstat.fetchall("""select * from aspect_1
                              where aspect_1_id > %d
                              order by aspect_1_id"""
                           % last_id)
    for obs in todo:
        logger.info("running get_asp for obsid %d aspect run on %s"
                    % (obs['obsid'], obs['ap_date']))
        if opt.firstrun:
            # ignore aspect_1 table versions and just try for default
            # also ignore errors in this case
            try:
                get_asp(obs['obsid'], version='default')
            except ValueError as ve:
                logger.info("skipping %d, default ver not available"
                            % obs['obsid'])
                logger.debug(ve)
                continue
        else:
            get_asp(obs['obsid'], version=obs['revision'])
        update_link(obs['obsid'])
        last_id_fh = open(last_id_file, 'w')
        last_id_fh.write("%d" % obs['aspect_1_id'])
        last_id_fh.close()
    if not len(todo):
        logger.info("No new aspect_1 data")

if __name__ == '__main__':
    opt = get_options()
    main(opt)
<|endoftext|>


<commit_before># coding: utf-8

import os

import netaddr

import lglass.dns
import lglass.nic

def _format_as_block_key(obj):
    return "{}_{}".format(obj.start, obj.end)

def fix_object(obj):
    if obj.object_class in {"route", "route6"}:
        for origin in obj.get("origin"):
            nobj = lglass.nic.RouteObject(obj)
            nobj.remove("origin")
            nobj.insert(1, "origin", origin)
            yield nobj
    elif obj.object_class == "as-block":
        nobj = lglass.nic.ASBlockObject(obj)
        nobj.object_key = nobj.primary_key
        yield nobj
    elif obj.object_class in {"inetnum", "inet6num"}:
        nobj = InetnumObject(obj)
        if obj.object_class == "inet6num":
            nobj.object_key = nobj.primary_key
        nobj.remove("nserver")
        nobj.remove("ds-rrdata")
        yield nobj
    else:
        yield obj

class InetnumObject(lglass.nic.InetnumObject):
    def to_domain_objects(self):
        for _, rdns_domain in self.rdns_domains():
            obj = DomainObject([("domain", rdns_domain)])
            for k, v in self.items():
                if k in {"zone-c", "admin-c", "tech-c", "mnt-by", "nserver",
                        "ds-rrdata", "created", "last-modified", "source"}:
                    obj.add(k, v)
            yield obj

class DomainObject(lglass.nic.NicObject):
    pass

class DN42Database(lglass.nic.FileDatabase):
    version = 0

    def __init__(self, path, force_version=None):
        lglass.nic.FileDatabase.__init__(self, path)
        self.object_class_types = dict(self.object_class_types)
        self.object_class_types.update({
            "domain": DomainObject,
            "inetnum": InetnumObject,
            "inet6num": InetnumObject
        })
        self._domain_cache = {}
        try:
            with open(os.path.join(self._path, "DatabaseVersion")) as fh:
                self.version = int(fh.read())
        except FileNotFoundError:
            if "dn42-version" in self.manifest:
                self.version = int(self.manifest["dn42-version"])
            else:
                self.version = 0
        except ValueError:
            self.version = 1
        if force_version is not None:
            self.version = force_version
        if self.version == 0:
            self.object_classes = set(self.object_classes)
            self.object_classes.remove("domain")
            self.object_classes.add("dns")
            self.primary_key_rules = dict(self.primary_key_rules)
            self.primary_key_rules["route"] = ["route"]
            self.primary_key_rules["route6"] = ["route6"]
            self.primary_key_rules["as-block"] = _format_as_block_key

    def flush_cache(self):
        self._domain_cache = {}

    def preload_domains(self):
        existing_domains = {domain for _, domain in super().lookup(types="dns")}
        inetnums = list(self.find(types={"inetnum", "inet6num"}))
        inetnums = sorted(inetnums,
                key=lambda d: d.ip_network.prefixlen,
                reverse=True)
        for inetnum in inetnums:
            for domain in inetnum.to_domain_objects():
                if domain in existing_domains: continue
                if domain.object_key in self._domain_cache: continue
                self._domain_cache[domain.object_key] = domain

    def fetch(self, object_class, object_key):
        object_class = self.primary_class(object_class)
        if self.version == 0 and object_class == "dns":
            return self._fetch_dns(object_key)
        return super().fetch(object_class, object_key)

    def _fetch_dns(self, object_key):
        try:
            return super().fetch("dns", object_key)
        except KeyError:
            pass
        if object_key in self._domain_cache:
            return self._domain_cache[object_key]
        network = lglass.dns.rdns_network(object_key)
        if network is None:
            raise KeyError(repr(("dns", object_key)))
        object_class = "inetnum" if network.version == 4 else "inet6num"
        for net in [network] + network.supernet()[::-1]:
            try:
                obj = self.fetch(object_class, str(net))
            except KeyError:
                continue
            for domain_object in obj.to_domain_objects():
                if object_key == domain_object.object_key:
                    self._domain_cache[object_key] = domain_object
                    return domain_object
        raise KeyError(repr(("dns", object_key)))

    def _lookup_class(self, object_class, keys):
        object_class = self.primary_class(object_class)
        if object_class == "dns" and self.version == 0:
            yield from self._lookup_dns(keys)
            return
        yield from super()._lookup_class(object_class, keys)

    def _lookup_dns(self, keys):
        if isinstance(keys, str) and not keys.endswith(("in-addr.arpa", "ip6.arpa")):
            yield from super()._lookup_class("dns", keys)
            return
        if isinstance(keys, set):
            keys = set(keys)
            for key in frozenset(keys):
                if not key.endswith(("in-addr.arpa", "ip6.arpa")):
                    keys.remove(key)
                    yield from super()._lookup_class("dns", {key})
        if not keys:
            return
        found = set()
        for dns in super()._lookup_class("dns", keys):
            found.add(dns)
            yield dns
        for inetnum in self.lookup(types={"inetnum", "inet6num"}):
            try:
                inetnum = self.fetch(*inetnum)
                if inetnum.ip_network.prefixlen > 24 and \
                        inetnum.ip_network.version == 4:
                    continue
                for domain in inetnum.to_domain_objects():
                    if domain in found: continue
                    self._domain_cache[domain.object_key] = domain
                    if lglass.database.perform_key_match(keys, domain.object_key):
                        yield ("dns", domain.object_key)
            except netaddr.core.AddrFormatError:
                pass

    def save(self, obj, **options):
        if self.version == 1:
            for nobj in fix_object(obj):
                super().save(nobj, **options)
            return
        super().save(nobj, **options)

<commit_msg>Only create domain objects if inetnum has nserver attribute<commit_after># coding: utf-8

import os

import netaddr

import lglass.dns
import lglass.nic

def _format_as_block_key(obj):
    return "{}_{}".format(obj.start, obj.end)

def fix_object(obj):
    if obj.object_class in {"route", "route6"}:
        for origin in obj.get("origin"):
            nobj = lglass.nic.RouteObject(obj)
            nobj.remove("origin")
            nobj.insert(1, "origin", origin)
            yield nobj
    elif obj.object_class == "as-block":
        nobj = lglass.nic.ASBlockObject(obj)
        nobj.object_key = nobj.primary_key
        yield nobj
    elif obj.object_class in {"inetnum", "inet6num"}:
        nobj = InetnumObject(obj)
        if obj.object_class == "inet6num":
            nobj.object_key = nobj.primary_key
        nobj.remove("nserver")
        nobj.remove("ds-rrdata")
        yield nobj
    else:
        yield obj

class InetnumObject(lglass.nic.InetnumObject):
    def to_domain_objects(self):
        if "nserver" not in self:
            return
        for _, rdns_domain in self.rdns_domains():
            obj = DomainObject([("domain", rdns_domain)])
            for k, v in self.items():
                if k in {"zone-c", "admin-c", "tech-c", "mnt-by", "nserver",
                        "ds-rrdata", "created", "last-modified", "source"}:
                    obj.add(k, v)
            yield obj

class DomainObject(lglass.nic.NicObject):
    pass

class DN42Database(lglass.nic.FileDatabase):
    version = 0

    def __init__(self, path, force_version=None):
        lglass.nic.FileDatabase.__init__(self, path)
        self.object_class_types = dict(self.object_class_types)
        self.object_class_types.update({
            "domain": DomainObject,
            "inetnum": InetnumObject,
            "inet6num": InetnumObject
        })
        self._domain_cache = {}
        try:
            with open(os.path.join(self._path, "DatabaseVersion")) as fh:
                self.version = int(fh.read())
        except FileNotFoundError:
            if "dn42-version" in self.manifest:
                self.version = int(self.manifest["dn42-version"])
            else:
                self.version = 0
        except ValueError:
            self.version = 1
        if force_version is not None:
            self.version = force_version
        if self.version == 0:
            self.object_classes = set(self.object_classes)
            self.object_classes.remove("domain")
            self.object_classes.add("dns")
            self.primary_key_rules = dict(self.primary_key_rules)
            self.primary_key_rules["route"] = ["route"]
            self.primary_key_rules["route6"] = ["route6"]
            self.primary_key_rules["as-block"] = _format_as_block_key

    def flush_cache(self):
        self._domain_cache = {}

    def preload_domains(self):
        existing_domains = {domain for _, domain in super().lookup(types="dns")}
        inetnums = list(self.find(types={"inetnum", "inet6num"}))
        inetnums = sorted(inetnums,
                key=lambda d: d.ip_network.prefixlen,
                reverse=True)
        for inetnum in inetnums:
            for domain in inetnum.to_domain_objects():
                if domain in existing_domains: continue
                if domain.object_key in self._domain_cache: continue
                self._domain_cache[domain.object_key] = domain

    def fetch(self, object_class, object_key):
        object_class = self.primary_class(object_class)
        if self.version == 0 and object_class == "dns":
            return self._fetch_dns(object_key)
        return super().fetch(object_class, object_key)

    def _fetch_dns(self, object_key):
        try:
            return super().fetch("dns", object_key)
        except KeyError:
            pass
        if object_key in self._domain_cache:
            return self._domain_cache[object_key]
        network = lglass.dns.rdns_network(object_key)
        if network is None:
            raise KeyError(repr(("dns", object_key)))
        object_class = "inetnum" if network.version == 4 else "inet6num"
        for net in [network] + network.supernet()[::-1]:
            try:
                obj = self.fetch(object_class, str(net))
            except KeyError:
                continue
            for domain_object in obj.to_domain_objects():
                if object_key == domain_object.object_key:
                    self._domain_cache[object_key] = domain_object
                    return domain_object
        raise KeyError(repr(("dns", object_key)))

    def _lookup_class(self, object_class, keys):
        object_class = self.primary_class(object_class)
        if object_class == "dns" and self.version == 0:
            yield from self._lookup_dns(keys)
            return
        yield from super()._lookup_class(object_class, keys)

    def _lookup_dns(self, keys):
        if isinstance(keys, str) and not keys.endswith(("in-addr.arpa", "ip6.arpa")):
            yield from super()._lookup_class("dns", keys)
            return
        if isinstance(keys, set):
            keys = set(keys)
            for key in frozenset(keys):
                if not key.endswith(("in-addr.arpa", "ip6.arpa")):
                    keys.remove(key)
                    yield from super()._lookup_class("dns", {key})
        if keys is not None and not keys:
            return
        found = set()
        for dns in super()._lookup_class("dns", keys):
            found.add(dns)
            yield dns
        for inetnum in self.lookup(types={"inetnum", "inet6num"}):
            try:
                inetnum = self.fetch(*inetnum)
                if inetnum.ip_network.prefixlen > 24 and \
                        inetnum.ip_network.version == 4:
                    continue
                for domain in inetnum.to_domain_objects():
                    if domain in found: continue
                    self._domain_cache[domain.object_key] = domain
                    if lglass.database.perform_key_match(keys, domain.object_key):
                        yield ("dns", domain.object_key)
            except netaddr.core.AddrFormatError:
                pass

    def save(self, obj, **options):
        if self.version == 1:
            for nobj in fix_object(obj):
                super().save(nobj, **options)
            return
        super().save(nobj, **options)

<|endoftext|>


<commit_before>from math import cos, sin, tan, sqrt, pi, radians, degrees, asin, atan2
import logging

class MapzenIsolines:

    NUMBER_OF_ANGLES = 24
    MAX_ITERS = 5
    TOLERANCE = 0.1

    EARTH_RADIUS_METERS = 6367444

    def __init__(self, matrix_client):
        self._matrix_client = matrix_client

    """Get an isochrone using mapzen API.

    The implementation tries to sick close to the SQL API:
    cdb_isochrone(source geometry, mode text, range integer[], [options text[]]) -> SETOF isoline

    But this calculates just one isoline.

    Args:
        origin dict containing {lat: y, lon: x}
        transport_mode string, for the moment just "car" or "walk"
        isorange int range of the isoline in seconds

    Returns:
        Array of {lon: x, lat: y} as a representation of the isoline
    """
    def calculate_isochrone(self, origin, transport_mode, isorange):

        # NOTE: not for production
        #logging.basicConfig(level=logging.DEBUG, filename='/tmp/isolines.log')
        #logging.basicConfig(level=logging.DEBUG)
        logging.debug('origin = %s' % origin)
        logging.debug('transport_mode = %s' % transport_mode)
        logging.debug('isorange = %d' % isorange)

        if transport_mode != 'walk':
            # TODO move this restriction to the appropriate place
            raise NotImplementedError('walk is the only supported mode for the moment')

        # Formally, a solution is an array of {angle, radius, lat, lon, cost} with cardinality NUMBER_OF_ANGLES
        # we're looking for a solution in which abs(cost - isorange) / isorange <= TOLERANCE

        # Initial setup
        angles = self._get_angles(self.NUMBER_OF_ANGLES) # array of angles
        upper_rmax = 3.3333333 * isorange # an upper bound for the radius, assuming 12km/h walking speed
        rmax = [upper_rmax] * self.NUMBER_OF_ANGLES
        rmin = [0.0] * self.NUMBER_OF_ANGLES
        location_estimates = [self._calculate_dest_location(origin, a, upper_rmax / 2.0) for a in angles]

        # Iterate to refine the first solution
        for i in xrange(0, self.MAX_ITERS):
            # Calculate the "actual" cost for each location estimate.
            # NOTE: sometimes it cannot calculate the cost and returns None.
            #   Just assume isorange and stop the calculations there
            response = self._matrix_client.one_to_many([origin] + location_estimates,  'pedestrian')
            costs = [(c['time'] or isorange) for c in response['one_to_many'][0][1:]]
            logging.debug('i = %d, costs = %s' % (i, costs))

            errors = [(cost - isorange) / float(isorange) for cost in costs]
            max_abs_error = max([abs(e) for e in errors])
            if max_abs_error <= self.TOLERANCE:
                # good enough, stop there
                break

            # let's refine the solution, binary search
            for j in xrange(0, self.NUMBER_OF_ANGLES):

                if abs(errors[j]) > self.TOLERANCE:
                    if errors[j] > 0:
                        rmax[j] = (rmax[j] + rmin[j]) / 2.0
                    else:
                        rmin[j] = (rmax[j] + rmin[j]) / 2.0

                    location_estimates[j] = self._calculate_dest_location(origin, angles[j], (rmax[j]+rmin[j])/2.0)

        # delete points that got None
        location_estimates_filtered = []
        for i, c in enumerate(costs):
            if c <> isorange:
                location_estimates_filtered.append(location_estimates[i])

        return location_estimates_filtered

    # NOTE: all angles in calculations are in radians
    def _get_angles(self, number_of_angles):
        step = (2.0 * pi) / number_of_angles
        return [(x * step) for x in xrange(0, number_of_angles)]

    def _calculate_dest_location(self, origin, angle, radius):
        origin_lat_radians = radians(origin['lat'])
        origin_long_radians = radians(origin['lon'])
        dest_lat_radians = asin(sin(origin_lat_radians) * cos(radius / self.EARTH_RADIUS_METERS) + cos(origin_lat_radians) * sin(radius / self.EARTH_RADIUS_METERS) * cos(angle))
        dest_lng_radians = origin_long_radians + atan2(sin(angle) * sin(radius / self.EARTH_RADIUS_METERS) * cos(origin_lat_radians), cos(radius / self.EARTH_RADIUS_METERS) - sin(origin_lat_radians) * sin(dest_lat_radians))

        return {
            'lon': degrees(dest_lng_radians),
            'lat': degrees(dest_lat_radians)
        }
<commit_msg>Add transport mode car<commit_after>from math import cos, sin, tan, sqrt, pi, radians, degrees, asin, atan2
import logging

class MapzenIsolines:

    NUMBER_OF_ANGLES = 24
    MAX_ITERS = 5
    TOLERANCE = 0.1

    EARTH_RADIUS_METERS = 6367444

    def __init__(self, matrix_client):
        self._matrix_client = matrix_client

    """Get an isochrone using mapzen API.

    The implementation tries to sick close to the SQL API:
    cdb_isochrone(source geometry, mode text, range integer[], [options text[]]) -> SETOF isoline

    But this calculates just one isoline.

    Args:
        origin dict containing {lat: y, lon: x}
        transport_mode string, for the moment just "car" or "walk"
        isorange int range of the isoline in seconds

    Returns:
        Array of {lon: x, lat: y} as a representation of the isoline
    """
    def calculate_isochrone(self, origin, transport_mode, isorange):

        # NOTE: not for production
        #logging.basicConfig(level=logging.DEBUG, filename='/tmp/isolines.log')
        #logging.basicConfig(level=logging.DEBUG)
        logging.debug('origin = %s' % origin)
        logging.debug('transport_mode = %s' % transport_mode)
        logging.debug('isorange = %d' % isorange)

        if transport_mode == 'walk':
            upper_rmax = 3.3333333 * isorange # an upper bound for the radius, assuming 12km/h walking speed
            costing_model = 'pedestrian'
        elif transport_mode == 'car':
            upper_rmax = 41.67 * isorange # assuming 140km/h max speed
            costing_model = 'auto'
        else:
            raise NotImplementedError('car and walk are the only supported modes for the moment')

        # Formally, a solution is an array of {angle, radius, lat, lon, cost} with cardinality NUMBER_OF_ANGLES
        # we're looking for a solution in which abs(cost - isorange) / isorange <= TOLERANCE

        # Initial setup
        angles = self._get_angles(self.NUMBER_OF_ANGLES) # array of angles
        rmax = [upper_rmax] * self.NUMBER_OF_ANGLES
        rmin = [0.0] * self.NUMBER_OF_ANGLES
        location_estimates = [self._calculate_dest_location(origin, a, upper_rmax / 2.0) for a in angles]

        # Iterate to refine the first solution
        for i in xrange(0, self.MAX_ITERS):
            # Calculate the "actual" cost for each location estimate.
            # NOTE: sometimes it cannot calculate the cost and returns None.
            #   Just assume isorange and stop the calculations there

            response = self._matrix_client.one_to_many([origin] + location_estimates,  costing_model)
            costs = [(c['time'] or isorange) for c in response['one_to_many'][0][1:]]

            logging.debug('i = %d, costs = %s' % (i, costs))

            errors = [(cost - isorange) / float(isorange) for cost in costs]
            max_abs_error = max([abs(e) for e in errors])
            if max_abs_error <= self.TOLERANCE:
                # good enough, stop there
                break

            # let's refine the solution, binary search
            for j in xrange(0, self.NUMBER_OF_ANGLES):

                if abs(errors[j]) > self.TOLERANCE:
                    if errors[j] > 0:
                        rmax[j] = (rmax[j] + rmin[j]) / 2.0
                    else:
                        rmin[j] = (rmax[j] + rmin[j]) / 2.0

                    location_estimates[j] = self._calculate_dest_location(origin, angles[j], (rmax[j]+rmin[j])/2.0)

        # delete points that got None
        location_estimates_filtered = []
        for i, c in enumerate(costs):
            if c <> isorange:
                location_estimates_filtered.append(location_estimates[i])

        return location_estimates_filtered

    # NOTE: all angles in calculations are in radians
    def _get_angles(self, number_of_angles):
        step = (2.0 * pi) / number_of_angles
        return [(x * step) for x in xrange(0, number_of_angles)]

    def _calculate_dest_location(self, origin, angle, radius):
        origin_lat_radians = radians(origin['lat'])
        origin_long_radians = radians(origin['lon'])
        dest_lat_radians = asin(sin(origin_lat_radians) * cos(radius / self.EARTH_RADIUS_METERS) + cos(origin_lat_radians) * sin(radius / self.EARTH_RADIUS_METERS) * cos(angle))
        dest_lng_radians = origin_long_radians + atan2(sin(angle) * sin(radius / self.EARTH_RADIUS_METERS) * cos(origin_lat_radians), cos(radius / self.EARTH_RADIUS_METERS) - sin(origin_lat_radians) * sin(dest_lat_radians))

        return {
            'lon': degrees(dest_lng_radians),
            'lat': degrees(dest_lat_radians)
        }
<|endoftext|>


<commit_before>"""
Functions that return external URLs, for example for the Vesper documentation.
"""

import vesper.version as vesper_version

_USE_LATEST_DOCUMENTATION_VERSION = False
"""Set this `True` during development, `False` for release."""

def _create_documentation_url():

    if _USE_LATEST_DOCUMENTATION_VERSION:
        doc_version = 'latest'
    else:
        doc_version = vesper_version.full_version

    return 'https://vesper.readthedocs.io/en/' + doc_version + '/'

def _create_tutorial_url():
    return _create_documentation_url() + 'tutorial.html'

documentation_url = _create_documentation_url()

tutorial_url = _create_tutorial_url()

source_code_url = 'https://github.com/HaroldMills/Vesper'
<commit_msg>Edit external URLs module for development.<commit_after>"""
Functions that return external URLs, for example for the Vesper documentation.
"""

import vesper.version as vesper_version

_USE_LATEST_DOCUMENTATION_VERSION = True
"""Set this `True` during development, `False` for release."""

def _create_documentation_url():

    if _USE_LATEST_DOCUMENTATION_VERSION:
        doc_version = 'latest'
    else:
        doc_version = vesper_version.full_version

    return 'https://vesper.readthedocs.io/en/' + doc_version + '/'

def _create_tutorial_url():
    return _create_documentation_url() + 'tutorial.html'

documentation_url = _create_documentation_url()

tutorial_url = _create_tutorial_url()

source_code_url = 'https://github.com/HaroldMills/Vesper'
<|endoftext|>


<commit_before># Plugin mais feio do mundo, mas é só pra um chat específico.
# Aqui temos comandos que são pequenos demais pra terem seu próprio módulo.

# O stats fora inicialmente implementado aqui, depois fora transferido [Tadeu, 23/Ago]

# A maioria é um port bem rápido de https://github.com/lucasberti/telegrao/blob/master/plugins/taup.lua

# Este plugin é, de certa maneira, uma aberração, porque precisa ser processado
# por último e geralmente dá problema quando alguém desativa/ativa comando pelo
# !plugin [cycrano]... Idealmente, eu me livraria deste plugin, mas ele tem lá
# seus usos, então talvez eu tenha de dar um jeito de o multipurpose ser invo-
# cado diretamente pelo reborn ao invés de ser listado no config... [Tadeu, 23/Ago]

from api import send_message, send_audio_id, send_sticker
from random import randint
import requests
import re
import plugins.stats as stats
import plugins.ed as ed

def on_msg_received(msg, matches):
    chat = msg["chat"]["id"]
    text = msg["text"]

    stats.do_statistics(msg)
    #ed.run_ed(msg) -> A API foi para o saco. TODO: reparar

    # Precisamos manter log de todas as mensagens pro /xet e /wordcloud
    with open("data/log.txt", "a", encoding='utf-8') as f:
        f.write(text + "\n")

    # /ip
    pattern = re.compile("^[!/]ip(?:@PintaoBot)?$")
    match = pattern.search(text)

    if match:
        # A versão original retornava um IP hardcodded
        # Esta se conecta a um serviço de checkagem de IP externo
        ipregex = re.compile("(\d{1,3}.\d{1,3}.\d{1,3}.\d{1,3})")
        url = "http://checkip.dyndns.com/"
        saida = ""

        try:
            req = requests.get(url)
            body = str(req.content)
            ipmatch = ipregex.search(body)

            if ipmatch:
                saida = ipmatch[0]
            else:
                saida = "Sei lá, deu algum outro pau aqui... "

        except Exception as e:
            saida = str(e)

        send_message(chat, saida)

    # /mps
    pattern = re.compile("^[!/]mps(?:@PintaoBot)?$")
    match = pattern.search(text)

    if match:
        send_message(chat, "ok to calculando aki q esistem " + str(randint(500, 10000)) + "/s por segundo de SUPER MAEMES NESNTE CHAT1")

    # @todos
    pattern = re.compile("(?:@todos|@todomundo)")
    match = pattern.search(text)

    if match:
        if chat = "-1001650218495":
            send_message(chat, "@nariama, @Tadeu, @xisteaga, @Criativo, @CriaLuly, @mariitagli, @ThaCillo, @Dinoko, @Thameus")
        else:
            send_message(chat, "@berti @beaea @getulhao @rauzao @xisteaga @axasdas @Garzarella")

    # TODO: fazer esta listagem de modo dinâmico e, talvez, por plugin

    # calma
    pattern = re.compile("^calma$")
    match = pattern.search(text)

    if match:
        send_message(chat, "ok esto mais calmo obrigada")

    # vc esta ai
    pattern = re.compile("^Ping$")
    match = pattern.search(text)

    if match:
        send_message(chat, "Pong")

    # rau
    pattern = re.compile("^rau$")
    match = pattern.search(text)

    if match:
        send_message(chat, "meu pau no seu cu")

    # Contribuição de Humberto
    pattern = re.compile("^!+$")
    match = pattern.search(text)

    if match:
        send_audio_id(chat, "CQADAQADFQAD4CAoRdcd4TJB2ecNAg")

    # /on
    pattern = re.compile("^[!/]on(?:@MercuriumBot)?$")
    match = pattern.search(text)

    if match:
        send_sticker(chat, "CAADAQADYQEAAsptnwnj5ix5ioNoNwI")

    # /off
    pattern = re.compile("^[!/]off(?:@MercuriumBot)?$")
    match = pattern.search(text)

    if match:
        send_sticker(chat, "CAADAQADYgEAAsptnwlEQdht940XBgI")

    # /on
    pattern = re.compile("^[Ee]stou on$")
    match = pattern.search(text)

    if match:
        send_sticker(chat, "CAADAQADYQEAAsptnwnj5ix5ioNoNwI")
<commit_msg>I'm dumb, you know<commit_after># Plugin mais feio do mundo, mas é só pra um chat específico.
# Aqui temos comandos que são pequenos demais pra terem seu próprio módulo.

# O stats fora inicialmente implementado aqui, depois fora transferido [Tadeu, 23/Ago]

# A maioria é um port bem rápido de https://github.com/lucasberti/telegrao/blob/master/plugins/taup.lua

# Este plugin é, de certa maneira, uma aberração, porque precisa ser processado
# por último e geralmente dá problema quando alguém desativa/ativa comando pelo
# !plugin [cycrano]... Idealmente, eu me livraria deste plugin, mas ele tem lá
# seus usos, então talvez eu tenha de dar um jeito de o multipurpose ser invo-
# cado diretamente pelo reborn ao invés de ser listado no config... [Tadeu, 23/Ago]

from api import send_message, send_audio_id, send_sticker
from random import randint
import requests
import re
import plugins.stats as stats
import plugins.ed as ed

def on_msg_received(msg, matches):
    chat = msg["chat"]["id"]
    text = msg["text"]

    stats.do_statistics(msg)
    #ed.run_ed(msg) -> A API foi para o saco. TODO: reparar

    # Precisamos manter log de todas as mensagens pro /xet e /wordcloud
    with open("data/log.txt", "a", encoding='utf-8') as f:
        f.write(text + "\n")

    # /ip
    pattern = re.compile("^[!/]ip(?:@PintaoBot)?$")
    match = pattern.search(text)

    if match:
        # A versão original retornava um IP hardcodded
        # Esta se conecta a um serviço de checkagem de IP externo
        ipregex = re.compile("(\d{1,3}.\d{1,3}.\d{1,3}.\d{1,3})")
        url = "http://checkip.dyndns.com/"
        saida = ""

        try:
            req = requests.get(url)
            body = str(req.content)
            ipmatch = ipregex.search(body)

            if ipmatch:
                saida = ipmatch[0]
            else:
                saida = "Sei lá, deu algum outro pau aqui... "

        except Exception as e:
            saida = str(e)

        send_message(chat, saida)

    # /mps
    pattern = re.compile("^[!/]mps(?:@PintaoBot)?$")
    match = pattern.search(text)

    if match:
        send_message(chat, "ok to calculando aki q esistem " + str(randint(500, 10000)) + "/s por segundo de SUPER MAEMES NESNTE CHAT1")

    # @todos
    pattern = re.compile("(?:@todos|@todomundo)")
    match = pattern.search(text)

    if match:
        if chat == "-1001650218495":
            send_message(chat, "@nariama, @Tadeu, @xisteaga, @Criativo, @CriaLuly, @mariitagli, @ThaCillo, @Dinoko, @Thameus")
        else:
            send_message(chat, "@berti @beaea @getulhao @rauzao @xisteaga @axasdas @Garzarella")

    # TODO: fazer esta listagem de modo dinâmico e, talvez, por plugin

    # calma
    pattern = re.compile("^calma$")
    match = pattern.search(text)

    if match:
        send_message(chat, "ok esto mais calmo obrigada")

    # vc esta ai
    pattern = re.compile("^Ping$")
    match = pattern.search(text)

    if match:
        send_message(chat, "Pong")

    # rau
    pattern = re.compile("^rau$")
    match = pattern.search(text)

    if match:
        send_message(chat, "meu pau no seu cu")

    # Contribuição de Humberto
    pattern = re.compile("^!+$")
    match = pattern.search(text)

    if match:
        send_audio_id(chat, "CQADAQADFQAD4CAoRdcd4TJB2ecNAg")

    # /on
    pattern = re.compile("^[!/]on(?:@MercuriumBot)?$")
    match = pattern.search(text)

    if match:
        send_sticker(chat, "CAADAQADYQEAAsptnwnj5ix5ioNoNwI")

    # /off
    pattern = re.compile("^[!/]off(?:@MercuriumBot)?$")
    match = pattern.search(text)

    if match:
        send_sticker(chat, "CAADAQADYgEAAsptnwlEQdht940XBgI")

    # /on
    pattern = re.compile("^[Ee]stou on$")
    match = pattern.search(text)

    if match:
        send_sticker(chat, "CAADAQADYQEAAsptnwnj5ix5ioNoNwI")
<|endoftext|>