<commit_before># -*- coding: utf-8 -*-
'''
Listener worker process
'''
from __future__ import absolute_import

# Import pythond stdlib
import os
import signal
import logging
import threading

# Import third party libs
import zmq
import umsgpack
import nacl.utils
import nacl.secret
from prometheus_client import Counter

# Import napalm-logs pkgs
import napalm_logs.utils
from napalm_logs.config import PUB_IPC_URL
from napalm_logs.config import SERIALIZER
from napalm_logs.proc import NapalmLogsProc
from napalm_logs.transport import get_transport
from napalm_logs.serializer import get_serializer
# exceptions
from napalm_logs.exceptions import NapalmLogsExit

log = logging.getLogger(__name__)

class NapalmLogsPublisherProc(NapalmLogsProc):
    '''
    publisher sub-process class.
    '''
    def __init__(self,
                 opts,
                 address,
                 port,
                 transport_type,
                 serializer,
                 private_key,
                 signing_key,
                 publisher_opts,
                 disable_security=False,
                 pub_id=None):
        self.__up = False
        self.opts = opts
        self.pub_id = pub_id
        self.address = publisher_opts.pop('address', None) or address
        self.port = publisher_opts.pop('port', None) or port
        log.debug('Publishing to %s:%d', self.address, self.port)
        self.serializer = publisher_opts.get('serializer') or serializer
        self.default_serializer = self.serializer == SERIALIZER
        self.disable_security = publisher_opts.get('disable_security', disable_security)
        self._transport_type = transport_type
        self.publisher_opts = publisher_opts
        self.error_whitelist = publisher_opts.get('error_whitelist', [])
        self.error_blacklist = publisher_opts.get('error_blacklist', [])
        if not disable_security:
            self.__safe = nacl.secret.SecretBox(private_key)
            self.__signing_key = signing_key
        self._setup_transport()

    def _exit_gracefully(self, signum, _):
        log.debug('Caught signal in publisher process')
        self.stop()

    def _setup_ipc(self):
        '''
        Subscribe to the pub IPC
        and publish the messages
        on the right transport.
        '''
        self.ctx = zmq.Context()
        log.debug('Setting up the %s publisher subscriber #%d', self._transport_type, self.pub_id)
        self.sub = self.ctx.socket(zmq.SUB)
        self.sub.connect(PUB_IPC_URL)
        self.sub.setsockopt(zmq.SUBSCRIBE, b'')
        try:
            self.sub.setsockopt(zmq.HWM, self.opts['hwm'])
            # zmq 2
        except AttributeError:
            # zmq 3
            self.sub.setsockopt(zmq.RCVHWM, self.opts['hwm'])

    def _setup_transport(self):
        '''
        Setup the transport.
        '''
        if 'RAW' in self.error_whitelist:
            log.info('%s %d will publish partially parsed messages', self._transport_type, self.pub_id)
        if 'UNKNOWN' in self.error_whitelist:
            log.info('%s %d will publish unknown messages', self._transport_type, self.pub_id)
        transport_class = get_transport(self._transport_type)
        log.debug('Serializing the object for %s using %s',
                  self._transport_type,
                  self.serializer)
        self.serializer_fun = get_serializer(self.serializer)
        self.transport = transport_class(self.address,
                                         self.port,
                                         **self.publisher_opts)
        self.__transport_encrypt = True
        if hasattr(self.transport, 'NO_ENCRYPT') and\
           getattr(self.transport, 'NO_ENCRYPT') is True:
            self.__transport_encrypt = False

    def _prepare(self, serialized_obj):
        '''
        Prepare the object to be sent over the untrusted channel.
        '''
        # generating a nonce
        nonce = nacl.utils.random(nacl.secret.SecretBox.NONCE_SIZE)
        # encrypting using the nonce
        encrypted = self.__safe.encrypt(serialized_obj, nonce)
        # sign the message
        signed = self.__signing_key.sign(encrypted)
        return signed

    def _serialize(self, obj, bin_obj):
        if self.default_serializer:
            # It's just easier to poll a boolean, rather than
            # re-serializing the message.
            # The reasoning why this is needed is because the structured
            # messages are passed anyway serialized between processes,
            # but the user may request a different serializer.
            return bin_obj
        return self.serializer_fun(obj)

    def start(self):
        '''
        Listen to messages and publish them.
        '''
        # metrics
        napalm_logs_publisher_received_messages = Counter(
            'napalm_logs_publisher_received_messages',
            "Count of messages received by the publisher",
            ['publisher_type', 'address', 'port']
        )
        napalm_logs_publisher_whitelist_blacklist_check_fail = Counter(
            'napalm_logs_publisher_whitelist_blacklist_check_fail',
            "Count of messages which fail the whitelist/blacklist check",
            ['publisher_type', 'address', 'port']
        )
        napalm_logs_publisher_messages_published = Counter(
            'napalm_logs_publisher_messages_published',
            "Count of published messages",
            ['publisher_type', 'address', 'port']
        )
        self._setup_ipc()
        # Start suicide polling thread
        thread = threading.Thread(target=self._suicide_when_without_parent, args=(os.getppid(),))
        thread.start()
        signal.signal(signal.SIGTERM, self._exit_gracefully)
        self.transport.start()
        self.__up = True
        while self.__up:
            try:
                bin_obj = self.sub.recv()
            except zmq.ZMQError as error:
                if self.__up is False:
                    log.info('Exiting on process shutdown')
                    return
                else:
                    log.error(error, exc_info=True)
                    raise NapalmLogsExit(error)
            obj = umsgpack.unpackb(bin_obj)
            napalm_logs_publisher_received_messages.labels(
                publisher_type=self._transport_type,
                address=self.address,
                port=self.port
            ).inc()
            if not napalm_logs.ext.check_whitelist_blacklist(obj['error'],
                                                             whitelist=self.error_whitelist,
                                                             blacklist=self.error_blacklist):
                # Apply the whitelist / blacklist logic
                # If it doesn't match, jump over.
                log.debug('This error type is %s. Skipping for %s #%d',
                          obj['error'],
                          self._transport_type,
                          self.pub_id)
                napalm_logs_publisher_whitelist_blacklist_check_fail.labels(
                    publisher_type=self._transport_type,
                    address=self.address,
                    port=self.port
                ).inc()
                continue
            serialized_obj = self._serialize(obj, bin_obj)
            log.debug('Publishing the OC object')
            if not self.disable_security and self.__transport_encrypt:
                # Encrypt only when needed.
                serialized_obj = self._prepare(serialized_obj)
            self.transport.publish(serialized_obj)
            napalm_logs_publisher_messages_published.labels(
                publisher_type=self._transport_type,
                address=self.address,
                port=self.port
            ).inc()

    def stop(self):
        log.info('Stopping publisher process %s (publisher #%d)', self._transport_type, self.pub_id)
        self.__up = False
        self.sub.close()
        self.ctx.term()
<commit_msg>Add strip_message_details option<commit_after># -*- coding: utf-8 -*-
'''
Listener worker process
'''
from __future__ import absolute_import

# Import pythond stdlib
import os
import signal
import logging
import threading

# Import third party libs
import zmq
import umsgpack
import nacl.utils
import nacl.secret
from prometheus_client import Counter

# Import napalm-logs pkgs
import napalm_logs.utils
from napalm_logs.config import PUB_IPC_URL
from napalm_logs.config import SERIALIZER
from napalm_logs.proc import NapalmLogsProc
from napalm_logs.transport import get_transport
from napalm_logs.serializer import get_serializer
# exceptions
from napalm_logs.exceptions import NapalmLogsExit

log = logging.getLogger(__name__)

class NapalmLogsPublisherProc(NapalmLogsProc):
    '''
    publisher sub-process class.
    '''
    def __init__(self,
                 opts,
                 address,
                 port,
                 transport_type,
                 serializer,
                 private_key,
                 signing_key,
                 publisher_opts,
                 disable_security=False,
                 pub_id=None):
        self.__up = False
        self.opts = opts
        self.pub_id = pub_id
        self.address = publisher_opts.pop('address', None) or address
        self.port = publisher_opts.pop('port', None) or port
        log.debug('Publishing to %s:%d', self.address, self.port)
        self.serializer = publisher_opts.get('serializer') or serializer
        self.default_serializer = self.serializer == SERIALIZER
        self.disable_security = publisher_opts.get('disable_security', disable_security)
        self._transport_type = transport_type
        self.publisher_opts = publisher_opts
        self.error_whitelist = publisher_opts.get('error_whitelist', [])
        self.error_blacklist = publisher_opts.get('error_blacklist', [])
        if not disable_security:
            self.__safe = nacl.secret.SecretBox(private_key)
            self.__signing_key = signing_key
        self._strip_message_details = publisher_opts.pop('strip_message_details', False)
        self._setup_transport()

    def _exit_gracefully(self, signum, _):
        log.debug('Caught signal in publisher process')
        self.stop()

    def _setup_ipc(self):
        '''
        Subscribe to the pub IPC
        and publish the messages
        on the right transport.
        '''
        self.ctx = zmq.Context()
        log.debug('Setting up the %s publisher subscriber #%d', self._transport_type, self.pub_id)
        self.sub = self.ctx.socket(zmq.SUB)
        self.sub.connect(PUB_IPC_URL)
        self.sub.setsockopt(zmq.SUBSCRIBE, b'')
        try:
            self.sub.setsockopt(zmq.HWM, self.opts['hwm'])
            # zmq 2
        except AttributeError:
            # zmq 3
            self.sub.setsockopt(zmq.RCVHWM, self.opts['hwm'])

    def _setup_transport(self):
        '''
        Setup the transport.
        '''
        if 'RAW' in self.error_whitelist:
            log.info('%s %d will publish partially parsed messages', self._transport_type, self.pub_id)
        if 'UNKNOWN' in self.error_whitelist:
            log.info('%s %d will publish unknown messages', self._transport_type, self.pub_id)
        transport_class = get_transport(self._transport_type)
        log.debug('Serializing the object for %s using %s',
                  self._transport_type,
                  self.serializer)
        self.serializer_fun = get_serializer(self.serializer)
        self.transport = transport_class(self.address,
                                         self.port,
                                         **self.publisher_opts)
        self.__transport_encrypt = True
        if hasattr(self.transport, 'NO_ENCRYPT') and\
           getattr(self.transport, 'NO_ENCRYPT') is True:
            self.__transport_encrypt = False

    def _prepare(self, serialized_obj):
        '''
        Prepare the object to be sent over the untrusted channel.
        '''
        # generating a nonce
        nonce = nacl.utils.random(nacl.secret.SecretBox.NONCE_SIZE)
        # encrypting using the nonce
        encrypted = self.__safe.encrypt(serialized_obj, nonce)
        # sign the message
        signed = self.__signing_key.sign(encrypted)
        return signed

    def _serialize(self, obj, bin_obj):
        if self.default_serializer:
            # It's just easier to poll a boolean, rather than
            # re-serializing the message.
            # The reasoning why this is needed is because the structured
            # messages are passed anyway serialized between processes,
            # but the user may request a different serializer.
            return bin_obj
        return self.serializer_fun(obj)

    def start(self):
        '''
        Listen to messages and publish them.
        '''
        # metrics
        napalm_logs_publisher_received_messages = Counter(
            'napalm_logs_publisher_received_messages',
            "Count of messages received by the publisher",
            ['publisher_type', 'address', 'port']
        )
        napalm_logs_publisher_whitelist_blacklist_check_fail = Counter(
            'napalm_logs_publisher_whitelist_blacklist_check_fail',
            "Count of messages which fail the whitelist/blacklist check",
            ['publisher_type', 'address', 'port']
        )
        napalm_logs_publisher_messages_published = Counter(
            'napalm_logs_publisher_messages_published',
            "Count of published messages",
            ['publisher_type', 'address', 'port']
        )
        self._setup_ipc()
        # Start suicide polling thread
        thread = threading.Thread(target=self._suicide_when_without_parent, args=(os.getppid(),))
        thread.start()
        signal.signal(signal.SIGTERM, self._exit_gracefully)
        self.transport.start()
        self.__up = True
        while self.__up:
            try:
                bin_obj = self.sub.recv()
            except zmq.ZMQError as error:
                if self.__up is False:
                    log.info('Exiting on process shutdown')
                    return
                else:
                    log.error(error, exc_info=True)
                    raise NapalmLogsExit(error)
            obj = umsgpack.unpackb(bin_obj)
            if self._strip_message_details:
                obj.pop('message_details', None)
                bin_obj = self.serializer_fun(obj)
            napalm_logs_publisher_received_messages.labels(
                publisher_type=self._transport_type,
                address=self.address,
                port=self.port
            ).inc()
            if not napalm_logs.ext.check_whitelist_blacklist(obj['error'],
                                                             whitelist=self.error_whitelist,
                                                             blacklist=self.error_blacklist):
                # Apply the whitelist / blacklist logic
                # If it doesn't match, jump over.
                log.debug('This error type is %s. Skipping for %s #%d',
                          obj['error'],
                          self._transport_type,
                          self.pub_id)
                napalm_logs_publisher_whitelist_blacklist_check_fail.labels(
                    publisher_type=self._transport_type,
                    address=self.address,
                    port=self.port
                ).inc()
                continue
            serialized_obj = self._serialize(obj, bin_obj)
            log.debug('Publishing the OC object')
            if not self.disable_security and self.__transport_encrypt:
                # Encrypt only when needed.
                serialized_obj = self._prepare(serialized_obj)
            self.transport.publish(serialized_obj)
            napalm_logs_publisher_messages_published.labels(
                publisher_type=self._transport_type,
                address=self.address,
                port=self.port
            ).inc()

    def stop(self):
        log.info('Stopping publisher process %s (publisher #%d)', self._transport_type, self.pub_id)
        self.__up = False
        self.sub.close()
        self.ctx.term()
<|endoftext|>


<commit_before>from datetime import datetime, timedelta
import re
import logging

from bs4 import BeautifulSoup
from core.observables import Ip, Hostname, Hash, Url
from core.feed import Feed
from core.errors import ObservableValidationError
import requests

class FeodoTracker(Feed):
    descriptions = {
        'A':
            "Hosted on compromised webservers running an nginx proxy on port 8080 TCP forwarding all botnet traffic to a tier 2 proxy node. Botnet traffic usually directly hits these hosts on port 8080 TCP without using a domain name.",
        'B':
            "Hosted on servers rented and operated by cybercriminals for the exclusive purpose of hosting a Feodo botnet controller. Usually taking advantage of a domain name within ccTLD .ru. Botnet traffic usually hits these domain names using port 80 TCP.",
        'C':
            "Successor of Feodo, completely different code. Hosted on the same botnet infrastructure as Version A (compromised webservers, nginx on port 8080 TCP or port 7779 TCP, no domain names) but using a different URL structure. This Version is also known as Geodo.",
        'D':
            "Successor of Cridex. This version is also known as Dridex",
        'E':
            "Successor of Geodo / Emotet (Version C) called Heodo. First appeared in March 2017."
    }

    variants = {
        'A': "Feodo",
        'B': "Feodo",
        'C': "Geodo",
        'D': "Dridex",
        'E': "Heodo"
    }

    default_values = {
        "frequency":
            timedelta(hours=1),
        "name":
            "FeodoTracker",
        "source":
            "https://feodotracker.abuse.ch/feodotracker.rss",
        "description":
            "Feodo Tracker RSS Feed. This feed shows the latest twenty Feodo C2 servers which Feodo Tracker has identified.",
    }

    def update(self):
        for item in self.update_xml(
                'item', ["title", "link", "description", "guid"]):
            self.analyze(item)

    def analyze(self, item):
        context = item
        date_string = re.search(
            r"\((?P<datetime>[\d\- :]+)\)", context['title']).group('datetime')
        try:
            context['date_added'] = datetime.strptime(
                date_string, "%Y-%m-%d %H:%M:%S")
        except ValueError:
            pass

        g = re.match(
            r'^Host: (?P<host>.+), Version: (?P<version>\w)',
            context['description'])
        g = g.groupdict()
        context['version'] = g['version']
        context['description'] = FeodoTracker.descriptions[g['version']]
        context['subfamily'] = FeodoTracker.variants[g['version']]
        context['source'] = self.name
        del context['title']
        new = None
        variant_tag = FeodoTracker.variants[g['version']].lower()
        try:
            if re.search(r"[0-9]{1,3}.[0-9]{1,3}.[0-9]{1,3}.[0-9]{1,3}",
                         g['host']):
                new = Ip.get_or_create(value=g['host'])
            else:
                new = Hostname.get_or_create(value=g['host'])
            new.add_context(context)
            new.add_source("feed")
            new.tag([variant_tag, 'malware', 'crimeware', 'banker', 'c2'])

        except ObservableValidationError as e:
            logging.error(e)

        try:
            url_fedeo = context['guid']
            r = requests.get(url_fedeo)
            if r.status_code == 200:

                html_source = r.text

                soup = BeautifulSoup(html_source, 'html.parser')
                tab = soup.find('table', attrs='sortable')
                results = []
                if tab:
                    all_tr = tab[0].find_all('tr')

                    for tr in all_tr:
                        all_td = tr.find_all('td')
                        if all_td and len(all_td) == 7:
                            results.append({
                                'timestamp': all_td[0].text,
                                'md5_hash': all_td[1].text,
                                'filesize': all_td[2].text,
                                'VT': all_td[3].text,
                                'Host': all_td[4].text,
                                'Port': all_td[5].text,
                                'SSL Certif or method': all_td[6].text
                            })

                for r in results:
                    new_hash = Hash.get_or_create(value=r['md5_hash'])
                    new_hash.add_context(context)
                    new_hash.add_source('feed')
                    new_hash.tag([
                        variant_tag, 'malware', 'crimeware', 'banker', 'payload'
                    ])
                    new_hash.active_link_to(
                        new, 'c2', self.name, clean_old=False)
                    host = Url.get_or_create(
                        value='https://%s:%s' % (g['host'], r['Port']))
                    host.add_source('feed')
                    host.add_context(context)
                    host.tag(
                        [variant_tag, 'malware', 'crimeware', 'banker', 'c2'])
                    new_hash.active_link_to(
                        host, 'c2', self.name, clean_old=False)

        except ObservableValidationError as e:
            logging.error(e)
<commit_msg>change access object<commit_after>from datetime import datetime, timedelta
import re
import logging

from bs4 import BeautifulSoup
from core.observables import Ip, Hostname, Hash, Url
from core.feed import Feed
from core.errors import ObservableValidationError
import requests

class FeodoTracker(Feed):
    descriptions = {
        'A':
            "Hosted on compromised webservers running an nginx proxy on port 8080 TCP forwarding all botnet traffic to a tier 2 proxy node. Botnet traffic usually directly hits these hosts on port 8080 TCP without using a domain name.",
        'B':
            "Hosted on servers rented and operated by cybercriminals for the exclusive purpose of hosting a Feodo botnet controller. Usually taking advantage of a domain name within ccTLD .ru. Botnet traffic usually hits these domain names using port 80 TCP.",
        'C':
            "Successor of Feodo, completely different code. Hosted on the same botnet infrastructure as Version A (compromised webservers, nginx on port 8080 TCP or port 7779 TCP, no domain names) but using a different URL structure. This Version is also known as Geodo.",
        'D':
            "Successor of Cridex. This version is also known as Dridex",
        'E':
            "Successor of Geodo / Emotet (Version C) called Heodo. First appeared in March 2017."
    }

    variants = {
        'A': "Feodo",
        'B': "Feodo",
        'C': "Geodo",
        'D': "Dridex",
        'E': "Heodo"
    }

    default_values = {
        "frequency":
            timedelta(hours=1),
        "name":
            "FeodoTracker",
        "source":
            "https://feodotracker.abuse.ch/feodotracker.rss",
        "description":
            "Feodo Tracker RSS Feed. This feed shows the latest twenty Feodo C2 servers which Feodo Tracker has identified.",
    }

    def update(self):
        for item in self.update_xml(
                'item', ["title", "link", "description", "guid"]):
            self.analyze(item)

    def analyze(self, item):
        context = item
        date_string = re.search(
            r"\((?P<datetime>[\d\- :]+)\)", context['title']).group('datetime')
        try:
            context['date_added'] = datetime.strptime(
                date_string, "%Y-%m-%d %H:%M:%S")
        except ValueError:
            pass

        g = re.match(
            r'^Host: (?P<host>.+), Version: (?P<version>\w)',
            context['description'])
        g = g.groupdict()
        context['version'] = g['version']
        context['description'] = FeodoTracker.descriptions[g['version']]
        context['subfamily'] = FeodoTracker.variants[g['version']]
        context['source'] = self.name
        del context['title']
        new = None
        variant_tag = FeodoTracker.variants[g['version']].lower()
        try:
            if re.search(r"[0-9]{1,3}.[0-9]{1,3}.[0-9]{1,3}.[0-9]{1,3}",
                         g['host']):
                new = Ip.get_or_create(value=g['host'])
            else:
                new = Hostname.get_or_create(value=g['host'])
            new.add_context(context)
            new.add_source("feed")
            new.tag([variant_tag, 'malware', 'crimeware', 'banker', 'c2'])

        except ObservableValidationError as e:
            logging.error(e)

        try:
            url_fedeo = context['guid']
            r = requests.get(url_fedeo)
            if r.status_code == 200:

                html_source = r.text

                soup = BeautifulSoup(html_source, 'html.parser')
                tab = soup.find('table', attrs='sortable')
                results = []

                if tab:
                    all_tr = tab.find_all('tr')

                    for tr in all_tr:
                        all_td = tr.find_all('td')
                        if all_td and len(all_td) == 7:
                            results.append({
                                'timestamp': all_td[0].text,
                                'md5_hash': all_td[1].text,
                                'filesize': all_td[2].text,
                                'VT': all_td[3].text,
                                'Host': all_td[4].text,
                                'Port': all_td[5].text,
                                'SSL Certif or method': all_td[6].text
                            })

                for r in results:
                    new_hash = Hash.get_or_create(value=r['md5_hash'])
                    new_hash.add_context(context)
                    new_hash.add_source('feed')
                    new_hash.tag([
                        variant_tag, 'malware', 'crimeware', 'banker', 'payload'
                    ])
                    new_hash.active_link_to(
                        new, 'c2', self.name, clean_old=False)
                    host = Url.get_or_create(
                        value='https://%s:%s' % (g['host'], r['Port']))
                    host.add_source('feed')
                    host.add_context(context)
                    host.tag(
                        [variant_tag, 'malware', 'crimeware', 'banker', 'c2'])
                    new_hash.active_link_to(
                        host, 'c2', self.name, clean_old=False)

        except ObservableValidationError as e:
            logging.error(e)
<|endoftext|>


<commit_before>import threading
import time
import math
from flask import jsonify
import datetime
import json

from data import speed_calc
from data import gear_calc
from data import torque_calc
from data import engine_speed_calc
from data import fuel_consumed_calc
from data import odometer_calc
from data import fuel_level_calc
from data import heading_calc
from data import lat_calc
from data import lon_calc

class DynamicsModel(object):
    def __init__(self, update_callback):
        self._initialize_data(update_callback)

        t = threading.Thread(target=self.physics_loop, name="Thread-Physics")
        t.setDaemon(True)
        t.start()

        print("Dynamics Model initialized")

    def _initialize_data(self, update_callback):
        self.calculations = []
        self.calculations.append(speed_calc.SpeedCalc())
        self.calculations.append(gear_calc.GearCalc(update_callback))
        self.calculations.append(torque_calc.TorqueCalc())
        self.calculations.append(engine_speed_calc.EngineSpeedCalc())
        self.calculations.append(fuel_consumed_calc.FuelConsumedCalc())
        self.calculations.append(odometer_calc.OdometerCalc())
        self.calculations.append(fuel_level_calc.FuelLevelCalc())
        self.calculations.append(heading_calc.HeadingCalc())
        self.calculations.append(lat_calc.LatCalc())
        self.calculations.append(lon_calc.LonCalc())

        self.snapshot = {}
        for data in self.calculations:
            self.snapshot[data.name] = data.get()

        self.delay_100Hz = datetime.timedelta(0,0,10000)
        self.next_iterate = datetime.datetime.now() + self.delay_100Hz
        self.zero_timedelta = datetime.timedelta(0,0,0)

        self.accelerator = 0.0
        self.brake = 0.0
        self.steering_wheel_angle = 0.0
        self.parking_brake_status = False
        self.engine_running = True
        self.ignition_data = 'run'

        print self.snapshot

        self.stopped = False

    def physics_loop(self):
        while True:
            if not self.stopped:
                time_til_calc = self.next_iterate - datetime.datetime.now()
                if time_til_calc > self.zero_timedelta:
                    time.sleep(time_til_calc.microseconds / 1000000.0)
                    #Assuming less than a second.
                self.next_iterate = self.next_iterate + self.delay_100Hz

                # Store the latest user input...
                self.snapshot['accelerator'] = self.accelerator
                self.snapshot['brake'] = self.brake
                self.snapshot['steering_wheel_angle'] = self.steering_wheel_angle
                self.snapshot['parking_brake_status'] = self.parking_brake_status
                self.snapshot['engine_running'] = self.engine_running
                self.snapshot['ignition_data'] = self.ignition_data

                new_snapshot = {}
                for data in self.calculations:
                    data.iterate(self.snapshot)
                    new_snapshot[data.name] = data.get()

                self.snapshot = new_snapshot
# Properties  ---------------------

    @property
    def torque(self):
        return self.snapshot['torque_at_transmission']

    @property
    def engine_speed(self):
        return self.snapshot['engine_speed']

    @property
    def vehicle_speed(self):
        return math.fabs(self.snapshot['vehicle_speed'])

    @property
    def brake_pedal_status(self):
        return self.brake > 0.0

    @property
    def fuel_consumed(self):
        return self.snapshot['fuel_consumed_since_restart']

    @property
    def odometer(self):
        return self.snapshot['odometer']

    @property
    def fuel_level(self):
        return self.snapshot['fuel_level']

    @property
    def lat(self):
        return self.snapshot['latitude']

    @property
    def lon(self):
        return self.snapshot['longitude']

    @property
    def data(self):
        return json.dumps(self.snapshot)
#        return jsonify(vehicle_speed=self.vehicle_speed,
#                       transmission_gear_posiotion=self.gear_data.get(),
#                       torque_at_transmission=self.torque,
#                       engine_speed=self.engine_speed,
#                       fuel_consumed_since_restart=self.fuel_consumed,
#                       odometer=self.odometer,
#                       fuel_level=self.fuel_level,
#                       latitude=self.lat,
#                       longitude=self.lon)

    @property
    def ignition_status(self):
        return self.ignition_data

    @ignition_status.setter
    def ignition_status(self, value):
        self.ignition_data = value
        if value == 'start':
            self.engine_running = True
        elif value == 'off':
            self.engine_running = False
        elif value == 'accessory':
            self.engine_running = False
<commit_msg>Remove debug output from startup.<commit_after>import threading
import time
import math
from flask import jsonify
import datetime
import json

from data import speed_calc
from data import gear_calc
from data import torque_calc
from data import engine_speed_calc
from data import fuel_consumed_calc
from data import odometer_calc
from data import fuel_level_calc
from data import heading_calc
from data import lat_calc
from data import lon_calc

class DynamicsModel(object):
    def __init__(self, update_callback):
        self._initialize_data(update_callback)

        t = threading.Thread(target=self.physics_loop, name="Thread-Physics")
        t.setDaemon(True)
        t.start()

        print("Dynamics Model initialized")

    def _initialize_data(self, update_callback):
        self.calculations = []
        self.calculations.append(speed_calc.SpeedCalc())
        self.calculations.append(gear_calc.GearCalc(update_callback))
        self.calculations.append(torque_calc.TorqueCalc())
        self.calculations.append(engine_speed_calc.EngineSpeedCalc())
        self.calculations.append(fuel_consumed_calc.FuelConsumedCalc())
        self.calculations.append(odometer_calc.OdometerCalc())
        self.calculations.append(fuel_level_calc.FuelLevelCalc())
        self.calculations.append(heading_calc.HeadingCalc())
        self.calculations.append(lat_calc.LatCalc())
        self.calculations.append(lon_calc.LonCalc())

        self.snapshot = {}
        for data in self.calculations:
            self.snapshot[data.name] = data.get()

        self.delay_100Hz = datetime.timedelta(0,0,10000)
        self.next_iterate = datetime.datetime.now() + self.delay_100Hz
        self.zero_timedelta = datetime.timedelta(0,0,0)

        self.accelerator = 0.0
        self.brake = 0.0
        self.steering_wheel_angle = 0.0
        self.parking_brake_status = False
        self.engine_running = True
        self.ignition_data = 'run'

        self.stopped = False

    def physics_loop(self):
        while True:
            if not self.stopped:
                time_til_calc = self.next_iterate - datetime.datetime.now()
                if time_til_calc > self.zero_timedelta:
                    time.sleep(time_til_calc.microseconds / 1000000.0)
                    #Assuming less than a second.
                self.next_iterate = self.next_iterate + self.delay_100Hz

                # Store the latest user input...
                self.snapshot['accelerator'] = self.accelerator
                self.snapshot['brake'] = self.brake
                self.snapshot['steering_wheel_angle'] = self.steering_wheel_angle
                self.snapshot['parking_brake_status'] = self.parking_brake_status
                self.snapshot['engine_running'] = self.engine_running
                self.snapshot['ignition_data'] = self.ignition_data

                new_snapshot = {}
                for data in self.calculations:
                    data.iterate(self.snapshot)
                    new_snapshot[data.name] = data.get()

                self.snapshot = new_snapshot
# Properties  ---------------------

    @property
    def torque(self):
        return self.snapshot['torque_at_transmission']

    @property
    def engine_speed(self):
        return self.snapshot['engine_speed']

    @property
    def vehicle_speed(self):
        return math.fabs(self.snapshot['vehicle_speed'])

    @property
    def brake_pedal_status(self):
        return self.brake > 0.0

    @property
    def fuel_consumed(self):
        return self.snapshot['fuel_consumed_since_restart']

    @property
    def odometer(self):
        return self.snapshot['odometer']

    @property
    def fuel_level(self):
        return self.snapshot['fuel_level']

    @property
    def lat(self):
        return self.snapshot['latitude']

    @property
    def lon(self):
        return self.snapshot['longitude']

    @property
    def data(self):
        return json.dumps(self.snapshot)
#        return jsonify(vehicle_speed=self.vehicle_speed,
#                       transmission_gear_posiotion=self.gear_data.get(),
#                       torque_at_transmission=self.torque,
#                       engine_speed=self.engine_speed,
#                       fuel_consumed_since_restart=self.fuel_consumed,
#                       odometer=self.odometer,
#                       fuel_level=self.fuel_level,
#                       latitude=self.lat,
#                       longitude=self.lon)

    @property
    def ignition_status(self):
        return self.ignition_data

    @ignition_status.setter
    def ignition_status(self, value):
        self.ignition_data = value
        if value == 'start':
            self.engine_running = True
        elif value == 'off':
            self.engine_running = False
        elif value == 'accessory':
            self.engine_running = False
<|endoftext|>


<commit_before># 10cbazbt3 - a menu to interact with the 10Centuries.org social network.
# (c) Barrie Turner, 2016-03-04 onwards.
# If you want a version number, you can have 2016-03-10(Late) or 0.13.

# Routines based on the curl examples at https://docs.10centuries.org

# Prerequisites:
# Python 3 (this simply will not work in Python 2; not even the menu!)

# Important:
# Created & tested only on a Raspberry Pi 2 B running Raspbian Linux,
# Created using Python 3 (IDLE).
# All this application's data files are currently stored in '/home/pi/10cv4/' on my machine,
# To make *your* application work please check each subroutine and replace the folder location as necessary!
# THIS APPLICATION DOESN'T YET PARSE (JSON), SO API RESPONSES ARE *ALMOST* HUMAN-READABLE!

# SETUP:

# Load an http library:
import requests
# Load system-specific library:
import sys
# Load time-related library stuff:
from time import strftime

# DEFINE SUBROUTINES:

# Define the 'Blurb' (social post) subroutine:
def blurb():
    # Input some text:
    posttext = input("Write some text: ")
    # Saves the input text to 'posttext.txt':
    file = open("/home/pi/10cv4/posttext.txt", "w")
    file.write(posttext)
    file.close()
    # Uses the global header & creates the data to be passed to the url:
    url = 'https://api.10centuries.org/content'
    data = {'content': posttext}
    response = requests.post(url, headers=headers, data=data)
    # Saves the server's response to 'serverresponse.txt':
    file = open("/home/pi/10cv4/serverresponse.txt", "w")
    file.write(response.text)
    file.close()
    # Displays the server's response:
    responsestatus = response.status_code
    if responsestatus == 200:
        print("Ok.")
    elif responsestatus != 200:
        print("Something went wrong. Are you logged in?")
    print("")

# Define the 'Post' (blog post) subroutine:
def post():
    # Input blog post data:
    posttitle = input("Write a blog post title: ")
    print("Write a blog post:")
    print("(Press [ctrl-d] to 'save' when you finish writing.)")
    posttext = sys.stdin.read()
    # Adds a post date & time, currently set as 'now':
    postdatetime = strftime("%Y-%m-%d %H:%M:%S")
    # (I decided to not save blog post text to a file for blog posts.)
    # Uses the global header & creates the data to be passed to the url:
    url = 'https://api.10centuries.org/content'
    # IMPORTANT: @bazbt3's channel id = 6, site id = 8.  SUBSTITUTE WITH *YOUR* CHANNEL_ID!
    data = {'title': posttitle, 'content': posttext, 'channel_id': '6', 'send_blurb': 'Y', 'pubdts': postdatetime}
    response = requests.post(url, headers=headers, data=data)
    # Saves the server's response to 'serverresponse.txt':
    file = open("/home/pi/10cv4/serverresponse.txt", "w")
    file.write(response.text)
    file.close()
    # Displays the server's response:
    responsestatus = response.status_code
    if responsestatus == 200:
        print("Ok.")
    elif responsestatus != 200:
        print("Something went wrong. Are you logged in?")
    print("")

# Define the 'Mentions' subroutine:
def mentions():
    # How many mentions posts to retrieve? ***(Doesn't currently work!)***
    mentionscount = input("How many mentions: ")
    mentionscount = str(mentionscount)
    # Uses the global header & creates the data to be passed to the url:
    url = 'https://api.10centuries.org/content/blurbs/mentions'
    data = {'count': mentionscount}
    response = requests.get(url, headers=headers, data=data)
    # Saves the server's response to 'serverresponse.txt':
    file = open("/home/pi/10cv4/servermentionsresponse.txt", "w")
    file.write(response.text)
    file.close()
    # Displays the server's response - *all* of it. Page after page!
    # Will be made better when I can extract data from the server responses.
    print(response.text)
    print("")
    print("Done - see servermentionsresponse.txt.")
    print("")

# Define the 'Reply' subroutine:
def reply():
    # Input a reply-to post number:
    replytoid = input("Post number to reply to: ")
    # Input some text:
    posttext = input("Write some text: ")
    # Saves the input text to 'posttext.txt':
    file = open("/home/pi/10cv4/posttext.txt", "w")
    file.write(posttext)
    file.close()
    # Uses the global header & creates the data to be passed to the url:
    url = 'https://api.10centuries.org/content'
    data = {'reply_to': replytoid, 'content': posttext}
    response = requests.post(url, headers=headers, data=data)
    # Saves the server's response to 'serverresponse.txt':
    file = open("/home/pi/10cv4/serverresponse.txt", "w")
    file.write(response.text)
    file.close()
    # Displays the server's response:
    responsestatus = response.status_code
    if responsestatus == 200:
        print("Ok.")
    elif responsestatus != 200:
        print("Something went wrong. Are you logged in?")
    print("")

# Admin subroutines start here:

# Define 'Authorise' subroutine:
def authorise():
    # Define 'headertokenonly' and 'headers' as global variables, with actual values defined below:
    global headertokenonly
    global headers
    # Define 'headertokenonly' global authentication variable - read from /home/pi/10cv4/authtoken.txt,
    # ***The text file MUST exist and contain one line, ONLY the text of the auth token,***
    # It's returned from the API at the end of the 'Login' subroutine,
    # This header contains only the token - used when only an auth header is required,
    authtokenonlyfile = open("/home/pi/10cv4/authtoken.txt", "r")
    authtokenonly = authtokenonlyfile.read()
    headertokenonly = {'Authorization': authtokenonly}
    # Define 'headers' global variable - uses 'authtokenonly' from above,
    # This header is used throughout the application:
    headers = {'Authorization': authtokenonly, 'Content-Type': 'application/x-www-form-urlencoded'}

# Define the 'Login' subroutine:
def login():
    # Input account name:
    my_acctname = input("Username (email): ")
    # Input account password:
    my_acctpass = input("Password: ")
    # The login URL:
    url = 'https://api.10centuries.org/auth/login'
    loginheaders = {'Content-Type': 'application/x-www-form-urlencoded'}
    data = {'client_guid': my_client_guid, 'acctname': my_acctname, 'acctpass': my_acctpass}
    response = requests.post(url, headers=loginheaders, data=data)
    # Saves the server's response to 'loginresponse.txt':
    file = open("/home/pi/10cv4/loginresponse.txt", "w")
    file.write(response.text)
    file.close()
    # Displays the server's response onscreen - *all* of it:
    # Will be made better when I can extract data from the server responses.
    print(response.text)
    print("")
    # Have the *user* parse the Authentication Token and manually enter it, here:
    print("Please copy the Authentication Token text line - between {} - from above and paste it into the line below:")
    temptoken = input("Paste it here: ")
    file = open("/home/pi/10cv4/authtoken.txt", "w")
    file.write(temptoken)
    file.close()
    print("")
    # Re-authorise now:
    authorise()
    print("Re-authorised (but please check!)")
    print("")

# Define the 'Logout' subroutine:
def logout():
    # The logout URL:
    url = 'https://api.10centuries.org/auth/logout'
    response = requests.post(url, headers=headertokenonly)
    # Saves the server's response to 'logoutresponse.txt':
    file = open("/home/pi/10cv4/logoutresponse.txt", "w")
    file.write(response.text)
    file.close()
    # Displays the server's response onscreen - *all* of it:
    # Will be made better when I can extract data from the server responses.
    print(response.text)
    print("")
    print("Done - see logoutresponse.txt.")
    print("")

# Define the 'Sites' query subroutine:
def sites():
    # Uses the global header & creates the data to be passed to the url:
    url = 'https://api.10centuries.org/users/sites'
    response = requests.get(url, headers=headertokenonly)
    # Saves the server's response to 'serverresponse.txt':
    file = open("/home/pi/10cv4/serverresponse.txt", "w")
    file.write(response.text)
    file.close()
    # Displays the server's response onscreen - *all* of it:
    # Will be made better when I can extract data from the server responses.
    print(response.text)
    print("")
    print("Done - see serverresponse.txt.")
    print("")

# MAIN ROUTINE STARTS:

# AUTHENTICATION:

# Define 'my_client_guid' GUID variable - read from /home/pi/10cv4/10cv4guid.txt,
# ***The text file MUST exist and contain one line - ONLY the text of the Client Key,***
# It's obtained from your Admin page's https://admin.10centuries.org/apps/
# And must be added by hand!
myclientguidfile = open("/home/pi/10cv4/10cv4guid.txt", "r")
my_client_guid = myclientguidfile.read()
# Use the Authentication Token (now handled by 'authorise' subroutine):
authorise()

# MENU:

# The menu has no input validation outside valid options:
print("10cbazbt3 menu:")
print("  b = Blurb (social post)")
print("  p = Post (blog post)")
print("  m = Mentions")
print("  r = Reply")
print("")
print("  exit = Exit")
print("")
print("Admin:")
print("  sites =  Sites owned by user")
print("  Login =  Login (deletes current auth token!)")
print("  Logout = Logout (deletes current auth token!)")
print("")

choice = "Little Bobby Tables"
while choice != 'exit':
    choice = input("Choice? ")
    print("")
    if choice == 'b':
        blurb()
    elif choice == 'p':
        post()
    elif choice == 'm':
        mentions()
    elif choice == 'r':
        reply()
    elif choice == 'sites':
        sites()
    elif choice == 'Login':
        login()
    elif choice == 'Logout':
        logout()

# The menu exits here:
print("You chose 'exit': Goodbye!")
<commit_msg>10cbazbt3.py version # updated (post code commit)<commit_after># 10cbazbt3 - a menu to interact with the 10Centuries.org social network.
# (c) Barrie Turner, 2016-03-04 onwards.
# If you want a version number, you can have 2016-03-12(Reasonable hour) or 0.14.

# Routines based on the curl examples at https://docs.10centuries.org

# Prerequisites:
# Python 3 (this simply will not work in Python 2; not even the menu!)

# Important:
# Created & tested only on a Raspberry Pi 2 B running Raspbian Linux,
# Created using Python 3 (IDLE).
# All this application's data files are currently stored in '/home/pi/10cv4/' on my machine,
# To make *your* application work please check each subroutine and replace the folder location as necessary!
# THIS APPLICATION DOESN'T YET PARSE (JSON), SO API RESPONSES ARE *ALMOST* HUMAN-READABLE!

# SETUP:

# Load an http library:
import requests
# Load system-specific library:
import sys
# Load time-related library stuff:
from time import strftime

# DEFINE SUBROUTINES:

# Define the 'Blurb' (social post) subroutine:
def blurb():
    # Input some text:
    posttext = input("Write some text: ")
    # Saves the input text to 'posttext.txt':
    file = open("/home/pi/10cv4/posttext.txt", "w")
    file.write(posttext)
    file.close()
    # Uses the global header & creates the data to be passed to the url:
    url = 'https://api.10centuries.org/content'
    data = {'content': posttext}
    response = requests.post(url, headers=headers, data=data)
    # Saves the server's response to 'serverresponse.txt':
    file = open("/home/pi/10cv4/serverresponse.txt", "w")
    file.write(response.text)
    file.close()
    # Displays the server's response:
    responsestatus = response.status_code
    if responsestatus == 200:
        print("Ok.")
    elif responsestatus != 200:
        print("Something went wrong. Are you logged in?")
    print("")

# Define the 'Post' (blog post) subroutine:
def post():
    # Input blog post data:
    posttitle = input("Write a blog post title: ")
    print("Write a blog post:")
    print("(Press [ctrl-d] to 'save' when you finish writing.)")
    posttext = sys.stdin.read()
    # Adds a post date & time, currently set as 'now':
    postdatetime = strftime("%Y-%m-%d %H:%M:%S")
    # (I decided to not save blog post text to a file for blog posts.)
    # Uses the global header & creates the data to be passed to the url:
    url = 'https://api.10centuries.org/content'
    # IMPORTANT: @bazbt3's channel id = 6, site id = 8.  SUBSTITUTE WITH *YOUR* CHANNEL_ID!
    data = {'title': posttitle, 'content': posttext, 'channel_id': '6', 'send_blurb': 'Y', 'pubdts': postdatetime}
    response = requests.post(url, headers=headers, data=data)
    # Saves the server's response to 'serverresponse.txt':
    file = open("/home/pi/10cv4/serverresponse.txt", "w")
    file.write(response.text)
    file.close()
    # Displays the server's response:
    responsestatus = response.status_code
    if responsestatus == 200:
        print("Ok.")
    elif responsestatus != 200:
        print("Something went wrong. Are you logged in?")
    print("")

# Define the 'Mentions' subroutine:
def mentions():
    # How many mentions posts to retrieve? ***(Doesn't currently work!)***
    mentionscount = input("How many mentions: ")
    mentionscount = str(mentionscount)
    # Uses the global header & creates the data to be passed to the url:
    url = 'https://api.10centuries.org/content/blurbs/mentions'
    data = {'count': mentionscount}
    response = requests.get(url, headers=headers, data=data)
    # Saves the server's response to 'serverresponse.txt':
    file = open("/home/pi/10cv4/servermentionsresponse.txt", "w")
    file.write(response.text)
    file.close()
    # Displays the server's response - *all* of it. Page after page!
    # Will be made better when I can extract data from the server responses.
    print(response.text)
    print("")
    print("Done - see servermentionsresponse.txt.")
    print("")

# Define the 'Reply' subroutine:
def reply():
    # Input a reply-to post number:
    replytoid = input("Post number to reply to: ")
    # Input some text:
    posttext = input("Write some text: ")
    # Saves the input text to 'posttext.txt':
    file = open("/home/pi/10cv4/posttext.txt", "w")
    file.write(posttext)
    file.close()
    # Uses the global header & creates the data to be passed to the url:
    url = 'https://api.10centuries.org/content'
    data = {'reply_to': replytoid, 'content': posttext}
    response = requests.post(url, headers=headers, data=data)
    # Saves the server's response to 'serverresponse.txt':
    file = open("/home/pi/10cv4/serverresponse.txt", "w")
    file.write(response.text)
    file.close()
    # Displays the server's response:
    responsestatus = response.status_code
    if responsestatus == 200:
        print("Ok.")
    elif responsestatus != 200:
        print("Something went wrong. Are you logged in?")
    print("")

# Admin subroutines start here:

# Define 'Authorise' subroutine:
def authorise():
    # Define 'headertokenonly' and 'headers' as global variables, with actual values defined below:
    global headertokenonly
    global headers
    # Define 'headertokenonly' global authentication variable - read from /home/pi/10cv4/authtoken.txt,
    # ***The text file MUST exist and contain one line, ONLY the text of the auth token,***
    # It's returned from the API at the end of the 'Login' subroutine,
    # This header contains only the token - used when only an auth header is required,
    authtokenonlyfile = open("/home/pi/10cv4/authtoken.txt", "r")
    authtokenonly = authtokenonlyfile.read()
    headertokenonly = {'Authorization': authtokenonly}
    # Define 'headers' global variable - uses 'authtokenonly' from above,
    # This header is used throughout the application:
    headers = {'Authorization': authtokenonly, 'Content-Type': 'application/x-www-form-urlencoded'}

# Define the 'Login' subroutine:
def login():
    # Input account name:
    my_acctname = input("Username (email): ")
    # Input account password:
    my_acctpass = input("Password: ")
    # The login URL:
    url = 'https://api.10centuries.org/auth/login'
    loginheaders = {'Content-Type': 'application/x-www-form-urlencoded'}
    data = {'client_guid': my_client_guid, 'acctname': my_acctname, 'acctpass': my_acctpass}
    response = requests.post(url, headers=loginheaders, data=data)
    # Saves the server's response to 'loginresponse.txt':
    file = open("/home/pi/10cv4/loginresponse.txt", "w")
    file.write(response.text)
    file.close()
    # Displays the server's response onscreen - *all* of it:
    # Will be made better when I can extract data from the server responses.
    print(response.text)
    print("")
    # Have the *user* parse the Authentication Token and manually enter it, here:
    print("Please copy the Authentication Token text line - between {} - from above and paste it into the line below:")
    temptoken = input("Paste it here: ")
    file = open("/home/pi/10cv4/authtoken.txt", "w")
    file.write(temptoken)
    file.close()
    print("")
    # Re-authorise now:
    authorise()
    print("Re-authorised (but please check!)")
    print("")

# Define the 'Logout' subroutine:
def logout():
    # The logout URL:
    url = 'https://api.10centuries.org/auth/logout'
    response = requests.post(url, headers=headertokenonly)
    # Saves the server's response to 'logoutresponse.txt':
    file = open("/home/pi/10cv4/logoutresponse.txt", "w")
    file.write(response.text)
    file.close()
    # Displays the server's response onscreen - *all* of it:
    # Will be made better when I can extract data from the server responses.
    print(response.text)
    print("")
    print("Done - see logoutresponse.txt.")
    print("")

# Define the 'Sites' query subroutine:
def sites():
    # Uses the global header & creates the data to be passed to the url:
    url = 'https://api.10centuries.org/users/sites'
    response = requests.get(url, headers=headertokenonly)
    # Saves the server's response to 'serverresponse.txt':
    file = open("/home/pi/10cv4/serverresponse.txt", "w")
    file.write(response.text)
    file.close()
    # Displays the server's response onscreen - *all* of it:
    # Will be made better when I can extract data from the server responses.
    print(response.text)
    print("")
    print("Done - see serverresponse.txt.")
    print("")

# MAIN ROUTINE STARTS:

# AUTHENTICATION:

# Define 'my_client_guid' GUID variable - read from /home/pi/10cv4/10cv4guid.txt,
# ***The text file MUST exist and contain one line - ONLY the text of the Client Key,***
# It's obtained from your Admin page's https://admin.10centuries.org/apps/
# And must be added by hand!
myclientguidfile = open("/home/pi/10cv4/10cv4guid.txt", "r")
my_client_guid = myclientguidfile.read()
# Use the Authentication Token (now handled by 'authorise' subroutine):
authorise()

# MENU:

# The menu has no input validation outside valid options:
print("10cbazbt3 menu:")
print("  b = Blurb (social post)")
print("  p = Post (blog post)")
print("  m = Mentions")
print("  r = Reply")
print("")
print("  exit = Exit")
print("")
print("Admin:")
print("  sites =  Sites owned by user")
print("  Login =  Login (deletes current auth token!)")
print("  Logout = Logout (deletes current auth token!)")
print("")

choice = "Little Bobby Tables"
while choice != 'exit':
    choice = input("Choice? ")
    print("")
    if choice == 'b':
        blurb()
    elif choice == 'p':
        post()
    elif choice == 'm':
        mentions()
    elif choice == 'r':
        reply()
    elif choice == 'sites':
        sites()
    elif choice == 'Login':
        login()
    elif choice == 'Logout':
        logout()

# The menu exits here:
print("You chose 'exit': Goodbye!")
<|endoftext|>


<commit_before>import datetime

import jwe
import jwt
import aiohttp

from waterbutler.core import exceptions
from waterbutler.auth.osf import settings
from waterbutler.core.auth import (BaseAuthHandler,
                                    AuthType)

JWE_KEY = jwe.kdf(settings.JWE_SECRET.encode(), settings.JWE_SALT.encode())

class OsfAuthHandler(BaseAuthHandler):
    """Identity lookup via the Open Science Framework"""
    ACTION_MAP = {
        'put': 'upload',
        'get': 'download',
        'head': 'metadata',
        'delete': 'delete',
    }

    def build_payload(self, bundle, view_only=None, cookie=None):
        query_params = {}

        if cookie:
            bundle['cookie'] = cookie

        if view_only:
            # View only must go outside of the jwt
            query_params['view_only'] = view_only

        query_params['payload'] = jwe.encrypt(jwt.encode({
            'data': bundle,
            'exp': datetime.datetime.utcnow() + datetime.timedelta(seconds=settings.JWT_EXPIRATION)
        }, settings.JWT_SECRET, algorithm=settings.JWT_ALGORITHM), JWE_KEY)

        return query_params

    async def make_request(self, params, headers, cookies):
        try:
            response = await aiohttp.request(
                'get',
                settings.API_URL,
                params=params,
                headers=headers,
                cookies=cookies,
            )
        except aiohttp.errors.ClientError:
            raise exceptions.AuthError('Unable to connect to auth sever', code=503)

        if response.status != 200:
            try:
                data = await response.json()
            except ValueError:
                data = await response.read()
            raise exceptions.AuthError(data, code=response.status)

        try:
            raw = await response.json()
            signed_jwt = jwe.decrypt(raw['payload'].encode(), JWE_KEY)
            data = jwt.decode(signed_jwt, settings.JWT_SECRET, algorithm=settings.JWT_ALGORITHM, options={'require_exp': True})
            return data['data']
        except (jwt.InvalidTokenError, KeyError):
            raise exceptions.AuthError(data, code=response.status)

    async def fetch(self, request, bundle):
        """Used for v0"""
        headers = {'Content-Type': 'application/json'}

        if 'Authorization' in request.headers:
            headers['Authorization'] = request.headers['Authorization']

        cookie = request.query_arguments.get('cookie')
        if cookie:
            cookie = cookie[0].decode()

        view_only = request.query_arguments.get('view_only')
        if view_only:
            view_only = view_only[0].decode()

        payload = (await self.make_request(
            self.build_payload(bundle, cookie=cookie, view_only=view_only),
            headers,
            dict(request.cookies)
        ))

        payload['auth']['callback_url'] = payload['callback_url']
        return payload

    async def get(self, resource, provider, request, action=None, auth_type=AuthType.SOURCE,
                  path='', version=None):
        """Used for v1"""
        method = request.method.lower()

        if method == 'post' and action:
            post_action_map = {
                'copy': 'download' if auth_type is AuthType.SOURCE else 'upload',
                'rename': 'upload',
                'move': 'upload',
            }

            try:
                osf_action = post_action_map[action.lower()]
            except KeyError:
                raise exceptions.UnsupportedActionError(method, supported=post_action_map.keys())
        elif method == 'head' and settings.MFR_ACTION_HEADER in request.headers:
            mfr_action_map = {'render': 'render', 'export': 'export'}
            mfr_action = request.headers[settings.MFR_ACTION_HEADER].lower()
            try:
                osf_action = mfr_action_map[mfr_action]
            except KeyError:
                raise exceptions.UnsupportedActionError(mfr_action, supported=mfr_action_map.keys())
        else:
            try:
                osf_action = self.ACTION_MAP[method]
            except KeyError:
                raise exceptions.UnsupportedHTTPMethodError(method, supported=self.ACTION_MAP.keys())

        headers = {'Content-Type': 'application/json'}

        if 'Authorization' in request.headers:
            headers['Authorization'] = request.headers['Authorization']

        cookie = request.query_arguments.get('cookie')
        if cookie:
            cookie = cookie[0].decode()

        view_only = request.query_arguments.get('view_only')
        if view_only:
            # View only must go outside of the jwt
            view_only = view_only[0].decode()

        payload = (await self.make_request(
            self.build_payload({
                'nid': resource,
                'provider': provider,
                'action': osf_action,
                'path': path,
                'version': version,
            }, cookie=cookie, view_only=view_only),
            headers,
            dict(request.cookies)
        ))

        payload['auth']['callback_url'] = payload['callback_url']
        return payload
<commit_msg>relay MFR identifying header to OSF on auth<commit_after>import datetime

import jwe
import jwt
import aiohttp

from waterbutler.core import exceptions
from waterbutler.auth.osf import settings
from waterbutler.core.auth import (BaseAuthHandler,
                                   AuthType)
from waterbutler.settings import MFR_IDENTIFYING_HEADER

JWE_KEY = jwe.kdf(settings.JWE_SECRET.encode(), settings.JWE_SALT.encode())

class OsfAuthHandler(BaseAuthHandler):
    """Identity lookup via the Open Science Framework"""
    ACTION_MAP = {
        'put': 'upload',
        'get': 'download',
        'head': 'metadata',
        'delete': 'delete',
    }

    def build_payload(self, bundle, view_only=None, cookie=None):
        query_params = {}

        if cookie:
            bundle['cookie'] = cookie

        if view_only:
            # View only must go outside of the jwt
            query_params['view_only'] = view_only

        query_params['payload'] = jwe.encrypt(jwt.encode({
            'data': bundle,
            'exp': datetime.datetime.utcnow() + datetime.timedelta(seconds=settings.JWT_EXPIRATION)
        }, settings.JWT_SECRET, algorithm=settings.JWT_ALGORITHM), JWE_KEY)

        return query_params

    async def make_request(self, params, headers, cookies):
        try:
            response = await aiohttp.request(
                'get',
                settings.API_URL,
                params=params,
                headers=headers,
                cookies=cookies,
            )
        except aiohttp.errors.ClientError:
            raise exceptions.AuthError('Unable to connect to auth sever', code=503)

        if response.status != 200:
            try:
                data = await response.json()
            except ValueError:
                data = await response.read()
            raise exceptions.AuthError(data, code=response.status)

        try:
            raw = await response.json()
            signed_jwt = jwe.decrypt(raw['payload'].encode(), JWE_KEY)
            data = jwt.decode(signed_jwt, settings.JWT_SECRET, algorithm=settings.JWT_ALGORITHM, options={'require_exp': True})
            return data['data']
        except (jwt.InvalidTokenError, KeyError):
            raise exceptions.AuthError(data, code=response.status)

    async def fetch(self, request, bundle):
        """Used for v0"""
        headers = {'Content-Type': 'application/json'}

        if 'Authorization' in request.headers:
            headers['Authorization'] = request.headers['Authorization']

        cookie = request.query_arguments.get('cookie')
        if cookie:
            cookie = cookie[0].decode()

        view_only = request.query_arguments.get('view_only')
        if view_only:
            view_only = view_only[0].decode()

        payload = (await self.make_request(
            self.build_payload(bundle, cookie=cookie, view_only=view_only),
            headers,
            dict(request.cookies)
        ))

        payload['auth']['callback_url'] = payload['callback_url']
        return payload

    async def get(self, resource, provider, request, action=None, auth_type=AuthType.SOURCE,
                  path='', version=None):
        """Used for v1"""
        method = request.method.lower()

        if method == 'post' and action:
            post_action_map = {
                'copy': 'download' if auth_type is AuthType.SOURCE else 'upload',
                'rename': 'upload',
                'move': 'upload',
            }

            try:
                osf_action = post_action_map[action.lower()]
            except KeyError:
                raise exceptions.UnsupportedActionError(method, supported=post_action_map.keys())
        elif method == 'head' and settings.MFR_ACTION_HEADER in request.headers:
            mfr_action_map = {'render': 'render', 'export': 'export'}
            mfr_action = request.headers[settings.MFR_ACTION_HEADER].lower()
            try:
                osf_action = mfr_action_map[mfr_action]
            except KeyError:
                raise exceptions.UnsupportedActionError(mfr_action, supported=mfr_action_map.keys())
        else:
            try:
                osf_action = self.ACTION_MAP[method]
            except KeyError:
                raise exceptions.UnsupportedHTTPMethodError(method, supported=self.ACTION_MAP.keys())

        headers = {'Content-Type': 'application/json'}

        if 'Authorization' in request.headers:
            headers['Authorization'] = request.headers['Authorization']

        if MFR_IDENTIFYING_HEADER in request.headers:
            headers[MFR_IDENTIFYING_HEADER] = request.headers[MFR_IDENTIFYING_HEADER]

        cookie = request.query_arguments.get('cookie')
        if cookie:
            cookie = cookie[0].decode()

        view_only = request.query_arguments.get('view_only')
        if view_only:
            # View only must go outside of the jwt
            view_only = view_only[0].decode()

        payload = (await self.make_request(
            self.build_payload({
                'nid': resource,
                'provider': provider,
                'action': osf_action,
                'path': path,
                'version': version,
            }, cookie=cookie, view_only=view_only),
            headers,
            dict(request.cookies)
        ))

        payload['auth']['callback_url'] = payload['callback_url']
        return payload
<|endoftext|>


<commit_before>"""
Jhonatan da Silva
Last Updated version :
Tue Jan 31 22:35:35 2017
Number of code lines:
19
"""
# First we initiate the class
class Functions():
    def __init__(self):
        self.numFunctions = 3

    def firstFunction(self):
        print("I'm in the first function")

    def secondFunction(self):
        print("I'm in the second function")

    def numberFunctions(self):
        print(self.numFunctions)

F = Functions()

F.firstFunction()
F.secondFunction()
F.numberFunctions()
<commit_msg>update<commit_after>"""
Jhonatan da Silva
Last Updated version :
Tue Jan 31 22:35:52 2017
Number of code lines:
19
"""
# First we initiate the class
class Functions():
    def __init__(self):
        self.numFunctions = 3

    def firstFunction(self):
        print("I'm in the first function")

    def secondFunction(self):
        print("I'm in the second function")

    def numberFunctions(self):
        print(self.numFunctions)

F = Functions()

F.firstFunction()
F.secondFunction()
F.numberFunctions()
<|endoftext|>


<commit_before># encoding: utf-8
# Generic fabfile.py
#
# See README for instructions on how to use fabmanager.

import os
import datetime
import cStringIO

from fabric.api import *
from fabric.contrib import django
from fabric.contrib import files
from fabric.contrib import console

# Paths related to fabmanager
fabmanager_dir = os.path.dirname(os.path.abspath(__file__))
templates_dir  = os.path.join(fabmanager_dir, 'templates/fabmanager/')

# Environments - to be extended with ENVS.update()
ENVS = {}

# Python
GET_PYTHON_VERSION = "python -V 2>&1 | cut -f2 -d' ' | cut -f-2 -d."

# virtualenvwrapper
VIRTUALENVWRAPPER_SCRIPT = '/usr/local/bin/virtualenvwrapper.sh'
VIRTUALENVWRAPPER_PREFIX = 'export WORKON_HOME=%(workon)s && source %(script)s'

# Django strings should be interpolated by ENVS[project]
VIRTUALENV_DIR     = '%(workon)s/%(virtualenv)s'
SITE_PACKAGES_DIR  = 'lib/python%s/site-packages'
DJANGO_PROJECT_DIR = "%(workon)s/%(virtualenv)s/%(project)s"
DJANGO_PREFIX      = "export PYTHONPATH=%(workon)s/%(virtualenv)s:%(workon)s/%(virtualenv)s/%(project)s " \
                     "DJANGO_SETTINGS_MODULE=%(project)s.%(settings)s && " \
                     "source %(workon)s/%(virtualenv)s/bin/activate"

# Apache
CONFIG_DIR         = '%(workon)s/%(virtualenv)s/%(project)s/config'
MEDIA_DIR          = '%(workon)s/%(virtualenv)s/%(project)s/media'
STATIC_DIR         = '%(workon)s/%(virtualenv)s/%(project)s/static'
FAVICON_DIR        = '%(workon)s/%(virtualenv)s/%(project)s/static/img'
APACHE_CONF        = CONFIG_DIR+'/apache_%(virtualenv)s.conf'
WSGI_CONF          = CONFIG_DIR+'/wsgi_%(virtualenv)s.py'

# Aliases for common tasks at server
ALIASES = dict(
    gs='git status',
    gd='git diff',
    gl='git pull',
    gp='git push',
    gc='git commit -v -a',
    gb='git branch',
    gk='git checkout',
    gm='git checkout master',
    glog='git log --oneline --decorate',
    glogg='git log --oneline --decorate --graph',
    mng='django-admin.py',
)

# Setup commands

def _setup_environment(environment):
    """
    Sets up env variables for this session
    """
    env.environment = environment
    env.project = ENVS[environment]
    env.hosts = [env.project['host']]

def _require_environment():
    """Checks if env.environment and env.host exist"""
    require('environment', 'host', provided_by=ENVS.keys())

def _interpolate(string):
    """Interpolates string with dictionary provided by ENVS"""
    return string % env.project

def _virtualenvwrapper_prefix():
    """Prefix to be able to invoke virtualenvwrapper commands"""
    return VIRTUALENVWRAPPER_PREFIX % {
        'workon': env.project['workon'],
        'script': VIRTUALENVWRAPPER_SCRIPT,
    }

def _python_version():
    """Checks python version on remote virtualenv"""
    with settings(hide('commands'), warn_only=True):
        with prefix(_django_prefix()):
            result = run(GET_PYTHON_VERSION)
            if result.failed:
                abort('Could not determine Python version at virtualenv %s' % env.environment)
    return result

def _django_prefix():
    """Prefix to wrap commands with necessary virtualenv and variables"""
    return _interpolate(DJANGO_PREFIX)

def _django_project_dir():
    """Path to current project' directory"""
    return _interpolate(DJANGO_PROJECT_DIR)

def _parse_alias(command):
    """
    Mimics a bash alias: if command starts with a word that is present in ALIASES
    dictionary, substitutes that word for the value found, and maintains te rest.
    """
    words = command.split(' ')
    if not words or words[0] not in ALIASES.keys():
        return command
    else:
        return '%s %s' % (ALIASES[words[0]], ' '.join(words[1:]))

# Remote commands

def remote(command='gs'):
    """
    Issues a generic command at project's directory
    """
    _require_environment()
    with prefix(_django_prefix()):
        with cd(_django_project_dir()):
            run(_parse_alias(command))

def update():
    """
    Updates server from git pull
    """
    branch = env.project.get('git_branch', 'master')
    with settings(warn_only=True):
        remote('git pull origin %s && django-admin.py migrate && touch config/wsgi*' % branch)

def status():
    """
    Checks git log and status
    """
    remote('glogg -n 20 && echo "" && git status')

def python():
    """Tries to figure out Python version on server side"""
    _require_environment()
    print 'Python version on virtualenv %s: %s' % (env.environment, _python_version())

def backup():
    """
    Backup server's database and copy tar.gz to local ../backup dir
    """
    _require_environment()

    # Uses local settings to extract username/password to access remote database
    django.settings_module(env.project['settings'])
    from django.conf import settings
    database = settings.DATABASES['default']

    # Remote side
    with prefix(_django_prefix()):
        with cd(_django_project_dir()):
            # Creates dir to store backup, avoiding existing similar names
            dirname = '../backup/%s_%s' % (datetime.date.today().strftime('%Y%m%d'), env.environment)
            index = 0
            while files.exists(dirname) or files.exists('%s.tar.gz' % dirname):
                index += 1
                dirname += '.%s' % index
            run('mkdir -p %s' % dirname)

            # Backup
            run('mysqldump -u %s -p%s %s > %s/%s.sql' % (
                database['USER'],
                database['PASSWORD'],
                database['NAME'],
                dirname,
                database['NAME'],
            ))
            run('django-admin.py dumpdata auth > %s/auth.json' % dirname)
            run('cp -R media/uploads %s/' % dirname)
            run('cp -R media/newsletter %s/' % dirname)
            with hide('stdout'):
                run('tar -czvf %s.tar.gz %s/' % (dirname, dirname))
            run('rm -rf %s/' % dirname)

            # Download backup?
            if console.confirm('Download backup?'):
                get('%s.tar.gz' % dirname, '../backup')

def _generate_conf(conf_file, variables):
    """Generates conf file from template, and optionally saves it"""
    # File names
    filename_parts = conf_file.split('.')
    filename_parts.insert(1, env.environment)
    input_file = os.path.join(templates_dir, conf_file)
    output_file = 'config/%s_%s.%s' % tuple(filename_parts)

    # Generate conf file from template
    conf = ''
    output = cStringIO.StringIO()
    try:
        with open(input_file, 'r') as input:
            for line in input:
                output.write(line % variables)
        conf = output.getvalue()
    finally:
        output.close()

    # Shows conf file and optionally saves it
    print conf
    if console.confirm('Save to %s?' % output_file, default=False):
        with open(output_file, 'w') as output:
            output.write(conf)

def apache():
    """Generates Apache conf file. Requires: path to WSGI conf file."""
    _require_environment()

    # Dictionary for interpolating template
    config_dir        = _interpolate(CONFIG_DIR)
    host_aliases      = env.project.get('host_aliases', '')
    if host_aliases:
        host_aliases  = 'ServerAlias %s' % host_aliases

    variables = {
        'host':             env.project['host'],
        'host_aliases':     host_aliases,
        'static_admin_dir': '%s/%s/django/contrib/admin/media' % (_interpolate(VIRTUALENV_DIR), SITE_PACKAGES_DIR % _python_version()),
        'media_dir':        _interpolate(MEDIA_DIR),
        'static_dir':       _interpolate(STATIC_DIR),
        'favicon_dir':      _interpolate(FAVICON_DIR),
        'config_dir':       config_dir,
        'wsgi_file':        '%s/wsgi_%s.py' % (config_dir, env.environment),
    }
    _generate_conf('apache.conf', variables)

def wsgi():
    """Generates WSGI conf file"""
    _require_environment()

    # Dictionary for interpolating template
    variables = {
        'project': env.project['project'],
        'settings': env.project['settings'],
        'site_packages': SITE_PACKAGES_DIR % _python_version(),
        }
    _generate_conf('wsgi.py', variables)

def setup():
    """Sets up a new environment"""
    _require_environment()

    # Checks if needed conf files for this environment already exist
    if not os.path.exists('settings_%s.py' % env.environment):
        abort('There is no settings.py for %s - create one, and commit' % env.environment)
    if not os.path.exists('config/apache_%s.conf' % env.environment):
        abort('There is no Apache conf for %s - use task "apache" to generate one, and commit' % env.environment)
    if not os.path.exists('config/wsgi_%s.py' % env.environment):
        abort('There is no WSGI conf for %s - use task "wsgi" to generate one, and commit' % env.environment)

    # Configures remote server
    _setup_virtualenv()
    _clone_gitrepo()
    _setup_apache()
    # TODO: create more setup tasks:
    #   - setup MySQL
    #   - import MySQL database
    #   - create aux dirs (logs, etc.)
    #   - pip install
    #   - syncdb, migrate
    #   - collectstatic

def _setup_virtualenv():
    """Creates virtualenv for environment"""
    if files.exists(_interpolate(VIRTUALENV_DIR)):
        print 'virtualenv %s already exists' % env.environment
    else:
        with prefix(_virtualenvwrapper_prefix()):
            run(_interpolate('mkvirtualenv --no-site-packages %(virtualenv)s'))
            with hide('commands'):
                print 'virtualenv %s created with python %s\n' % (env.environment, run(GET_PYTHON_VERSION))

def _clone_gitrepo():
    """Clones project git repo into virtualenv"""
    if files.exists(_interpolate(DJANGO_PROJECT_DIR)):
#        print _interpolate('project %(project)s already exists')
        update()
    else:
        with cd(_interpolate(VIRTUALENV_DIR)):
            run(_interpolate('git clone %(git_repo)s %(project)s'))
            branch = env.project.get('git_branch', 'master')
            if branch != 'master':
                with cd(env.project['project']):
                    run('git fetch origin %s:%s' % (branch, branch))
                    run('git checkout %s' % branch)

def _setup_apache():
    """Configures Apache"""
    if files.exists(_interpolate('/etc/apache2/sites-enabled/%(virtualenv)s')):
        print 'Apache conf for %s already exists' % env.environment
    else:
        sudo(_interpolate('ln -s %s /etc/apache2/sites-enabled/%%(virtualenv)s' % APACHE_CONF))
        sudo('apache2ctl restart')<commit_msg>MySQL setup<commit_after># encoding: utf-8
# Generic fabfile.py
#
# See README for instructions on how to use fabmanager.

import os
import datetime
import cStringIO

from fabric.api import *
from fabric.contrib import django
from fabric.contrib import files
from fabric.contrib import console

from django.conf import settings as django_settings

# Paths related to fabmanager
fabmanager_dir = os.path.dirname(os.path.abspath(__file__))
templates_dir  = os.path.join(fabmanager_dir, 'templates/fabmanager/')

# Environments - to be extended with ENVS.update()
ENVS = {}

# Python
GET_PYTHON_VERSION = "python -V 2>&1 | cut -f2 -d' ' | cut -f-2 -d."

# virtualenvwrapper
VIRTUALENVWRAPPER_SCRIPT = '/usr/local/bin/virtualenvwrapper.sh'
VIRTUALENVWRAPPER_PREFIX = 'export WORKON_HOME=%(workon)s && source %(script)s'

# Django strings should be interpolated by ENVS[project]
VIRTUALENV_DIR     = '%(workon)s/%(virtualenv)s'
SITE_PACKAGES_DIR  = 'lib/python%s/site-packages'
DJANGO_PROJECT_DIR = "%(workon)s/%(virtualenv)s/%(project)s"
DJANGO_PREFIX      = "export PYTHONPATH=%(workon)s/%(virtualenv)s:%(workon)s/%(virtualenv)s/%(project)s " \
                     "DJANGO_SETTINGS_MODULE=%(project)s.%(settings)s && " \
                     "source %(workon)s/%(virtualenv)s/bin/activate"

# Apache
CONFIG_DIR         = '%(workon)s/%(virtualenv)s/%(project)s/config'
MEDIA_DIR          = '%(workon)s/%(virtualenv)s/%(project)s/media'
STATIC_DIR         = '%(workon)s/%(virtualenv)s/%(project)s/static'
FAVICON_DIR        = '%(workon)s/%(virtualenv)s/%(project)s/static/img'
APACHE_CONF        = CONFIG_DIR+'/apache_%(virtualenv)s.conf'
WSGI_CONF          = CONFIG_DIR+'/wsgi_%(virtualenv)s.py'

# MySQL
MYSQL_PREFIX       = 'mysql -u root -p -e %s'

# Aliases for common tasks at server
ALIASES = dict(
    gs='git status',
    gd='git diff',
    gl='git pull',
    gp='git push',
    gc='git commit -v -a',
    gb='git branch',
    gk='git checkout',
    gm='git checkout master',
    glog='git log --oneline --decorate',
    glogg='git log --oneline --decorate --graph',
    mng='django-admin.py',
)

# Setup commands

def _setup_environment(environment):
    """
    Sets up env variables for this session
    """
    env.environment = environment
    env.project = ENVS[environment]
    env.hosts = [env.project['host']]

def _require_environment():
    """Checks if env.environment and env.host exist"""
    require('environment', 'host', provided_by=ENVS.keys())

def _interpolate(string):
    """Interpolates string with dictionary provided by ENVS"""
    return string % env.project

def _virtualenvwrapper_prefix():
    """Prefix to be able to invoke virtualenvwrapper commands"""
    return VIRTUALENVWRAPPER_PREFIX % {
        'workon': env.project['workon'],
        'script': VIRTUALENVWRAPPER_SCRIPT,
    }

def _python_version():
    """Checks python version on remote virtualenv"""
    with settings(hide('commands', 'warnings'), warn_only=True):
        with prefix(_django_prefix()):
            result = run(GET_PYTHON_VERSION)
            if result.failed:
                abort('Could not determine Python version at virtualenv %s' % env.environment)
    return result

def _django_prefix():
    """Prefix to wrap commands with necessary virtualenv and variables"""
    return _interpolate(DJANGO_PREFIX)

def _django_project_dir():
    """Path to current project' directory"""
    return _interpolate(DJANGO_PROJECT_DIR)

def _parse_alias(command):
    """
    Mimics a bash alias: if command starts with a word that is present in ALIASES
    dictionary, substitutes that word for the value found, and maintains te rest.
    """
    words = command.split(' ')
    if not words or words[0] not in ALIASES.keys():
        return command
    else:
        return '%s %s' % (ALIASES[words[0]], ' '.join(words[1:]))

# Remote commands

def remote(command='gs'):
    """
    Issues a generic command at project's directory
    """
    _require_environment()
    with prefix(_django_prefix()):
        with cd(_django_project_dir()):
            run(_parse_alias(command))

def update():
    """
    Updates server from git pull
    """
    branch = env.project.get('git_branch', 'master')
    with settings(hide('warnings'), warn_only=True):
        remote('git pull origin %s && django-admin.py migrate && touch config/wsgi*' % branch)

def status():
    """
    Checks git log and status
    """
    remote('glogg -n 20 && echo "" && git status')

def python():
    """Tries to figure out Python version on server side"""
    _require_environment()
    print 'Python version on virtualenv %s: %s' % (env.environment, _python_version())

def backup():
    """
    Backup server's database and copy tar.gz to local ../backup dir
    TODO: create a restore task
    """
    _require_environment()

    # Uses local Django settings to extract username/password to access remote database
    django.settings_module(env.project['settings'])
    database = django_settings.DATABASES['default']

    # Remote side
    with prefix(_django_prefix()):
        with cd(_django_project_dir()):
            # Creates dir to store backup, avoiding existing similar names
            dirname = '../backup/%s_%s' % (datetime.date.today().strftime('%Y%m%d'), env.environment)
            index = 0
            while files.exists(dirname) or files.exists('%s.tar.gz' % dirname):
                index += 1
                dirname += '.%s' % index
            run('mkdir -p %s' % dirname)

            # Backup
            run('mysqldump -u %s -p%s %s > %s/%s.sql' % (
                database['USER'],
                database['PASSWORD'],
                database['NAME'],
                dirname,
                database['NAME'],
            ))
            run('django-admin.py dumpdata auth > %s/auth.json' % dirname)
            run('cp -R media/uploads %s/' % dirname)
            run('cp -R media/newsletter %s/' % dirname)
            with hide('stdout'):
                run('tar -czvf %s.tar.gz %s/' % (dirname, dirname))
            run('rm -rf %s/' % dirname)

            # Download backup?
            if console.confirm('Download backup?'):
                get('%s.tar.gz' % dirname, '../backup')

def _generate_conf(conf_file, variables):
    """Generates conf file from template, and optionally saves it"""
    # File names
    filename_parts = conf_file.split('.')
    filename_parts.insert(1, env.environment)
    input_file = os.path.join(templates_dir, conf_file)
    output_file = 'config/%s_%s.%s' % tuple(filename_parts)

    # Generate conf file from template
    conf = ''
    output = cStringIO.StringIO()
    try:
        with open(input_file, 'r') as input:
            for line in input:
                output.write(line % variables)
        conf = output.getvalue()
    finally:
        output.close()

    # Shows conf file and optionally saves it
    print conf
    if console.confirm('Save to %s?' % output_file, default=False):
        with open(output_file, 'w') as output:
            output.write(conf)

def apache():
    """Generates Apache conf file. Requires: path to WSGI conf file."""
    _require_environment()

    # Dictionary for interpolating template
    config_dir        = _interpolate(CONFIG_DIR)
    host_aliases      = env.project.get('host_aliases', '')
    if host_aliases:
        host_aliases  = 'ServerAlias %s' % host_aliases

    variables = {
        'host':             env.project['host'],
        'host_aliases':     host_aliases,
        'static_admin_dir': '%s/%s/django/contrib/admin/media' % (_interpolate(VIRTUALENV_DIR), SITE_PACKAGES_DIR % _python_version()),
        'media_dir':        _interpolate(MEDIA_DIR),
        'static_dir':       _interpolate(STATIC_DIR),
        'favicon_dir':      _interpolate(FAVICON_DIR),
        'config_dir':       config_dir,
        'wsgi_file':        '%s/wsgi_%s.py' % (config_dir, env.environment),
    }
    _generate_conf('apache.conf', variables)

def wsgi():
    """Generates WSGI conf file"""
    _require_environment()

    # Dictionary for interpolating template
    variables = {
        'project': env.project['project'],
        'settings': env.project['settings'],
        'site_packages': SITE_PACKAGES_DIR % _python_version(),
        }
    _generate_conf('wsgi.py', variables)

def setup():
    """Sets up a new environment"""
    _require_environment()

    # Checks if needed conf files for this environment already exist
    if not os.path.exists('settings_%s.py' % env.environment):
        abort('There is no settings.py for %s - create one, and commit' % env.environment)
    if not os.path.exists('config/apache_%s.conf' % env.environment):
        abort('There is no Apache conf for %s - use task "apache" to generate one, and commit' % env.environment)
    if not os.path.exists('config/wsgi_%s.py' % env.environment):
        abort('There is no WSGI conf for %s - use task "wsgi" to generate one, and commit' % env.environment)

    # Configures remote server
    _setup_virtualenv()
    _clone_gitrepo()
    _setup_apache()
    _setup_mysql()
    # TODO: create more setup tasks:
    #   - create aux dirs (logs, etc.)
    #   - pip install
    #   - syncdb, migrate
    #   - collectstatic

def _setup_virtualenv():
    """Creates virtualenv for environment"""
    if files.exists(_interpolate(VIRTUALENV_DIR)):
        print 'virtualenv %s already exists' % env.environment
    else:
        with prefix(_virtualenvwrapper_prefix()):
            run(_interpolate('mkvirtualenv --no-site-packages %(virtualenv)s'))
            with hide('commands'):
                print 'virtualenv %s created with python %s\n' % (env.environment, run(GET_PYTHON_VERSION))

def _clone_gitrepo():
    """Clones project git repo into virtualenv"""
    if files.exists(_interpolate(DJANGO_PROJECT_DIR)):
        print _interpolate('project %(project)s already exists, updating')
        update()
    else:
        with cd(_interpolate(VIRTUALENV_DIR)):
            run(_interpolate('git clone %(git_repo)s %(project)s'))
            branch = env.project.get('git_branch', 'master')
            if branch != 'master':
                with cd(env.project['project']):
                    run('git fetch origin %s:%s' % (branch, branch))
                    run('git checkout %s' % branch)

def _setup_apache():
    """Configures Apache"""
    if files.exists(_interpolate('/etc/apache2/sites-enabled/%(virtualenv)s')):
        print 'Apache conf for %s already exists' % env.environment
    else:
        sudo(_interpolate('ln -s %s /etc/apache2/sites-enabled/%%(virtualenv)s' % APACHE_CONF))
        sudo('apache2ctl restart')

def _setup_mysql():
    """Creates MySQL database according to env's settings.py"""
    # Uses local Django settings to extract username/password to access remote database
    django.settings_module(env.project['settings'])
    database = django_settings.DATABASES['default']

    # Create database & user, if not already there
    with settings(hide('warnings'), warn_only=True):
        result = run(MYSQL_PREFIX % "\"CREATE DATABASE %(NAME)s DEFAULT CHARACTER SET utf8;\"" % database)
        if result.succeeded:
            run(MYSQL_PREFIX % "\"GRANT ALL ON %(NAME)s.* TO '%(USER)s'@'localhost' IDENTIFIED BY '%(PASSWORD)s';\"" % database)
<|endoftext|>