<commit_before># -*- coding: utf-8 -*-

from __future__ import absolute_import

import uuid
import argparse
import logging

from moneta.cluster import MonetaCluster
from moneta.server import MonetaServer

def run():
    parser = argparse.ArgumentParser()
    parser.add_argument('--listen', nargs='?', default='127.0.0.1:32000', help='Listen host:port')
    parser.add_argument('--zookeeper', nargs='?', default='127.0.0.1:2181', help='Zookeeper hosts (comma-separated list of host:port items)')
    parser.add_argument('--nodename', nargs='?', default=uuid.uuid1().hex, help='Node name')
    parser.add_argument('--pool', nargs='?', default="default", help='Pool')
    parser.add_argument('--logfile', nargs='?', default=None, help='Log file')
    args = parser.parse_args()

    if args.logfile:
        logging.basicConfig(filename = args.logfile, format =  '%(asctime)s - %(name)s - %(levelname)s - %(message)s')
    else:
        logging.basicConfig()

    logging.getLogger('moneta').setLevel(logging.INFO)
    logging.getLogger('moneta.http').setLevel(logging.WARNING)

    logger = logging.getLogger('moneta')

    try:
        logger.debug('Starting')
        cluster = MonetaCluster(args.nodename, args.listen, args.zookeeper, pool = args.pool)
        server = MonetaServer(cluster, args.listen)
        logger.info('Started')

        server.run_forever()

    except BaseException:
        cluster.stop()
        logger.exception('An exception occurred. Terminating moneta.')
        exit(1)
<commit_msg>Improved log format + --loglevel option.<commit_after># -*- coding: utf-8 -*-

from __future__ import absolute_import

import uuid
import argparse
import logging

from moneta.cluster import MonetaCluster
from moneta.server import MonetaServer

logger = logging.getLogger('moneta')

def run():
    parser = argparse.ArgumentParser()
    parser.add_argument('--listen', nargs='?', default='127.0.0.1:32000', help='Listen host:port')
    parser.add_argument('--zookeeper', nargs='?', default='127.0.0.1:2181', help='Zookeeper hosts (comma-separated list of host:port items)')
    parser.add_argument('--nodename', nargs='?', default=uuid.uuid1().hex, help='Node name')
    parser.add_argument('--pool', nargs='?', default="default", help='Pool')
    parser.add_argument('--logfile', nargs='?', default=None, help='Log file')
    parser.add_argument('--loglevel', nargs='?', default="info", help='Log level', choices = ['debug', 'info', 'warning', 'error', 'critical', 'fatal'])
    args = parser.parse_args()

    if args.logfile:
        logging.basicConfig(filename = args.logfile, format =  '%(asctime)s [%(name)s] %(levelname)s: %(message)s')
    else:
        logging.basicConfig(format =  '%(asctime)s [%(name)s] %(levelname)s: %(message)s')

    loglevel = {
        'debug': logging.DEBUG,
        'info': logging.INFO,
        'warning': logging.WARNING,
        'error': logging.ERROR,
        'critical': logging.CRITICAL,
        'fatal': logging.FATAL
    }[args.loglevel]

    logger.setLevel(loglevel)

    try:
        logger.debug('Starting')
        cluster = MonetaCluster(args.nodename, args.listen, args.zookeeper, pool = args.pool)
        server = MonetaServer(cluster, args.listen)
        logger.info('Started')

        server.run_forever()

    except BaseException:
        cluster.stop()
        logger.exception('An exception occurred. Terminating moneta.')
        exit(1)
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
__author__ = 'peter'

import requests
from bs4 import BeautifulSoup
from cheekpouch import spawn

baseurl = 'http://bt.aisex.com/bt/'

def lemons():
    aisex = (baseurl + 'thread.php?fid=4&search=&page=' + str(i) for i in range(1, 0, -1))
    for url in aisex:
        r = requests.get(url)
        soup = BeautifulSoup(r.text, "html.parser")
        tags = soup.find_all('a', target="_blank")
        for tag in tags:
            if tag.parent.name == 'h3':
                yield baseurl + tag['href']

def harvest():
    for lemon in lemons():
        r = requests.get(lemon)
        soup = BeautifulSoup(r.text, "html.parser")
        tags = soup.find_all('a', target="_blank")
        for tag in tags:
            if tag.parent.id == 'read_tpc':
                print(baseurl + tag['href'])
                break

        #     for juice in juices:
        #         try:
        #             juice = Juice(key_name=lemon, image=juice['image'], download=juice['download'])
        #             juice.put()
        #         except BadValueError:
        #             logging.info(juice)
        #     bucket.append(lemon)
        # squeezed.put()

if __name__ == "__main__":
    harvest()
<commit_msg>userscript<commit_after># -*- coding: utf-8 -*-
__author__ = 'peter'

import requests
from bs4 import BeautifulSoup
from cheekpouch import spawn

baseurl = 'http://bt.aisex.com/bt/'

def lemons():
    aisex = (baseurl + 'thread.php?fid=4&search=&page=' + str(i) for i in range(1, 0, -1))
    for url in aisex:
        r = requests.get(url)
        soup = BeautifulSoup(r.text, "html.parser")
        tags = soup.find_all('a', target="_blank")
        for tag in tags:
            if tag.parent.name == 'h3':
                yield baseurl + tag['href']

def harvest():
    for lemon in lemons():
        r = requests.get(lemon)
        soup = BeautifulSoup(r.text, "html.parser")
        tags = soup.find_all('a', target="_blank")
        for tag in tags:
            if tag.parent['id'] == 'read_tpc':
                print(baseurl + tag['href'])
                break

        #     for juice in juices:
        #         try:
        #             juice = Juice(key_name=lemon, image=juice['image'], download=juice['download'])
        #             juice.put()
        #         except BadValueError:
        #             logging.info(juice)
        #     bucket.append(lemon)
        # squeezed.put()

if __name__ == "__main__":
    harvest()
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
__author__ = 'peter'

import requests
from bs4 import BeautifulSoup
from cheekpouch import spawn, r
import re

baseurl = 'http://bt.aisex.com/bt/'

def lemons():
    aisex = (baseurl + 'thread.php?fid=4&search=&page=' + str(i) for i in range(1, 0, -1))
    for url in aisex:
        response = requests.get(url)
        soup = BeautifulSoup(response.text, "html.parser")
        tags = soup.find_all('a', target="_blank")
        for tag in tags:
            if tag.parent.name == 'h3':
                yield baseurl + tag['href']

def harvest():
    for lemon in lemons():
        response = requests.get(lemon)
        soup = BeautifulSoup(response.text, "html.parser")
        tags = soup.find_all('a', href=re.compile("www.jandown.com"))
        for tag in tags:
            url = tag['href']
            if r.hgetall(url):
                pass
            else:
                spawn(url)
            break

if __name__ == "__main__":
    harvest()
<commit_msg>Redis Queue<commit_after># -*- coding: utf-8 -*-
__author__ = 'peter'

import requests
from bs4 import BeautifulSoup
from cheekpouch import spawn, r
import re
from rq import Queue

baseurl = 'http://bt.aisex.com/bt/'
q = Queue(connection=r)

def lemons():
    aisex = (baseurl + 'thread.php?fid=4&search=&page=' + str(i) for i in range(1, 0, -1))
    for url in aisex:
        response = requests.get(url)
        soup = BeautifulSoup(response.text, "html.parser")
        tags = soup.find_all('a', target="_blank")
        for tag in tags:
            if tag.parent.name == 'h3':
                yield baseurl + tag['href']

def harvest():
    for lemon in lemons():
        response = requests.get(lemon)
        soup = BeautifulSoup(response.text, "html.parser")
        tags = soup.find_all('a', href=re.compile("www.jandown.com"))
        for tag in tags:
            url = tag['href']
            if r.hgetall(url):
                pass
            else:
                q.enqueue(spawn, url)
            break

if __name__ == "__main__":
    harvest()
<|endoftext|>


<commit_before>import os
import sys
import logging
import ConfigParser

import cpuinfo
import cv2
import pyttsx
import vlc

if "X86_64" == cpuinfo.get_cpu_info()["arch"]:
    import getchar as interface

config = ConfigParser.ConfigParser()
tts_engine = pyttsx.init()
tts_engine.setProperty('voice', 'spanish-latin-am')
tts_engine.setProperty('rate', 95)
face_cascade = None
video = None
vlc_instance = None
player = None
stay_alive = True   # Keep the program running?
speaking = False    # Am I talking when I detect someone else prescence

def initialize():
    """Initialize program"""
    global video
    global vlc_instance
    global player
    global face_cascade
    config.readfp(open('../config.ini'))
    face_cascade = cv2.CascadeClassifier("../"
            + config.get('DEFAULT','facexml'))
    video = cv2.VideoCapture(0)
    # Create basic VLC instance
    vlc_instance = vlc.Instance()
    # Create VLC player
    player = vlc_instance.media_player_new()
    # TODO Make it load a playlist and set it up to play random
    promos = vlc_instance.media_new("file://"
            + os.environ["HOME"] + "/01.ogg")
    player.set_media(promos)

def get_frame():
    """Capture frames from camera"""
    success, frame = video.read()
    gray = cv2.cvtColor(frame, cv2.COLOR_BGR2GRAY)

    faces = face_cascade.detectMultiScale(
            gray,
            scaleFactor=1.1,
            minNeighbors=5,
            minSize=(30, 30),
            flags=cv2.cv.CV_HAAR_SCALE_IMAGE
            )

    ## Draw a rectangle around faces
    for (x, y, w, h) in faces:
       cv2.rectangle(frame, (x, y), (x+w, y+h), (0, 255, 0), 2)

    return frame

def jpeg_encode(frame):
    success, jpeg = cv2.imencode('.jpg', frame)

    return jpeg.tobytes()

def main():
    """Monroe waits for external sensors input and greet people"""
    global stay_alive
    log_init()                      # Initialize loging system
    logging.info("Starting")
    initialize()                    # Initilize global variables
    #read = interface.read_input() TODO replace it for something else

    while stay_alive:
        # Feel
        # Watch
        cv2.imshow('Video', get_frame())
        # Wait for input, TODO make it more generic loose from cv2
        r_input = cv2.waitKey(1) & 0xFF
        if r_input == ord('/'):
            stay_alive = False
            exit(stay_alive)
            break
        elif r_input == ord('1'):
            shout_out()
        elif r_input == ord('2'):
            if False == amIspeaking():
                tts_engine.say("Hola mundo!")
                tts_engine.runAndWait()

def amIspeaking():
    global speaking
    if speaking == True:
        return speaking
    else:
        speaking = True
        return False

def shout_out(snd_file=None):
    """Say something"""
    if False == amIspeaking():
        if (snd_file != None):
            speech = vlc_instance.media_new(snd_file)
            player.set_media(speech)

        player.play()
        speaking = False

def exit(flag):
    """Exit the app"""
    if (stay_alive != True):
        video.release()
        cv2.destroyAllWindows()
        logging.info("Bye!")
        sys.exit(0)

def log_init():
    """Initialize log"""
    logging.basicConfig(filename="monroe.log", level=logging.DEBUG)

if __name__ == "__main__":
    main()
<commit_msg>UPDATE he behaves better when talking<commit_after>import os
import sys
import logging
import ConfigParser

import cpuinfo
import cv2
import pyttsx
import vlc

if "X86_64" == cpuinfo.get_cpu_info()["arch"]:
    import getchar as interface

config = ConfigParser.ConfigParser()
tts_engine = pyttsx.init()
tts_engine.setProperty('voice', 'spanish-latin-am')
tts_engine.setProperty('rate', 95)
face_cascade = None
video = None
vlc_instance = None
player = None
stay_alive = True   # Keep the program running?
speaking = False    # Am I talking when I detect someone else prescence

def initialize():
    """Initialize program"""
    global video
    global vlc_instance
    global player
    global face_cascade
    config.readfp(open('../config.ini'))
    face_cascade = cv2.CascadeClassifier("../"
            + config.get('DEFAULT','facexml'))
    video = cv2.VideoCapture(0)
    # Create basic VLC instance
    vlc_instance = vlc.Instance()
    # Create VLC player
    player = vlc_instance.media_player_new()
    # TODO Make it load a playlist and set it up to play random
    promos = vlc_instance.media_new("file://"
            + os.environ["HOME"] + "/01.ogg")
    player.set_media(promos)

def get_frame():
    """Capture frames from camera"""
    success, frame = video.read()
    gray = cv2.cvtColor(frame, cv2.COLOR_BGR2GRAY)

    faces = face_cascade.detectMultiScale(
            gray,
            scaleFactor=1.1,
            minNeighbors=5,
            minSize=(30, 30),
            flags=cv2.cv.CV_HAAR_SCALE_IMAGE
            )

    ## Draw a rectangle around faces
    for (x, y, w, h) in faces:
       cv2.rectangle(frame, (x, y), (x+w, y+h), (0, 255, 0), 2)

    return frame

def jpeg_encode(frame):
    success, jpeg = cv2.imencode('.jpg', frame)

    return jpeg.tobytes()

def main():
    """Monroe waits for external sensors input and greet people"""
    global speaking
    global stay_alive
    log_init()                      # Initialize loging system
    logging.info("Starting")
    initialize()                    # Initilize global variables
    #read = interface.read_input() TODO replace it for something else

    while stay_alive:
        # Feel
        # Watch
        cv2.imshow('Video', get_frame())
        # Wait for input, TODO make it more generic loose from cv2
        r_input = cv2.waitKey(1) & 0xFF
        if r_input == ord('/'):
            stay_alive = False
            exit(stay_alive)
            break
        elif r_input == ord('.'):
            logging.info("Shut up!")
            tts_engine.stop()
            player.stop()
            speaking = False
        elif r_input == ord('1'):
            logging.info("Shout out!")
            shout_out()
        elif r_input == ord('2'):
            logging.info("Say something")
            if False == amIspeaking():
                tts_engine.say("Hola mundo!")
                tts_engine.runAndWait()
                speaking = False

def amIspeaking():
    global speaking
    if speaking == True:
        logging.info("I'am currently talking")
        return speaking
    else:
        speaking = True
        return False

def shout_out(snd_file=None):
    """Say something"""
    if False == amIspeaking():
        if (snd_file != None):
            speech = vlc_instance.media_new(snd_file)
            player.set_media(speech)

        player.play()
        speaking = False

def exit(flag):
    """Exit the app"""
    if (stay_alive != True):
        video.release()
        cv2.destroyAllWindows()
        logging.info("Bye!")
        sys.exit(0)

def log_init():
    """Initialize log"""
    logging.basicConfig(filename="monroe.log", level=logging.DEBUG)

if __name__ == "__main__":
    main()
<|endoftext|>


<commit_before># this file as been modified for use with dbt
import decimal
from typing import TypeVar, Generic

TV = TypeVar("TV")

class SerializableEncoder(Generic[TV]):
    @classmethod
    def _serialize(cls, value):
        raise NotImplementedError

    @classmethod
    def _deserialize(cls, value):
        raise NotImplementedError

class SerializableType:
    def _serialize(self):
        raise NotImplementedError

    @classmethod
    def _deserialize(cls, value):
        raise NotImplementedError

class SerializationStrategy:
    def _serialize(self, value):
        raise NotImplementedError

    def _deserialize(self, value):
        raise NotImplementedError

class RoundedDecimal(SerializationStrategy):
    def __init__(self, places=None, rounding=None):
        if places is not None:
            self.exp = decimal.Decimal((0, (1,), -places))
        else:
            self.exp = None
        self.rounding = rounding

    def _serialize(self, value) -> str:
        if self.exp:
            if self.rounding:
                return str(value.quantize(self.exp, rounding=self.rounding))
            else:
                return str(value.quantize(self.exp))
        else:
            return str(value)

    def _deserialize(self, value: str) -> decimal.Decimal:
        return decimal.Decimal(str(value))

__all__ = [
    "SerializableType",
    "SerializationStrategy",
    "RoundedDecimal",
]
<commit_msg>Update core/mashumaro/types.py<commit_after># this file has been modified for use with dbt
import decimal
from typing import TypeVar, Generic

TV = TypeVar("TV")

class SerializableEncoder(Generic[TV]):
    @classmethod
    def _serialize(cls, value):
        raise NotImplementedError

    @classmethod
    def _deserialize(cls, value):
        raise NotImplementedError

class SerializableType:
    def _serialize(self):
        raise NotImplementedError

    @classmethod
    def _deserialize(cls, value):
        raise NotImplementedError

class SerializationStrategy:
    def _serialize(self, value):
        raise NotImplementedError

    def _deserialize(self, value):
        raise NotImplementedError

class RoundedDecimal(SerializationStrategy):
    def __init__(self, places=None, rounding=None):
        if places is not None:
            self.exp = decimal.Decimal((0, (1,), -places))
        else:
            self.exp = None
        self.rounding = rounding

    def _serialize(self, value) -> str:
        if self.exp:
            if self.rounding:
                return str(value.quantize(self.exp, rounding=self.rounding))
            else:
                return str(value.quantize(self.exp))
        else:
            return str(value)

    def _deserialize(self, value: str) -> decimal.Decimal:
        return decimal.Decimal(str(value))

__all__ = [
    "SerializableType",
    "SerializationStrategy",
    "RoundedDecimal",
]
<|endoftext|>


<commit_before>class Message:
    """
    Represents a single message to be sent
    """

    def __init__(self, message_type):
        """
        Parameters
        :param message_type: str; one of statement, carousel, list, media
        """
        self.message_type = message_type
        self.suggestions = []

    def add_suggestion(self, suggestion):
        """
        Adds a suggestion to this message
        :param suggestion: Suggestion type.
        :return: self
        """
        self.suggestions.extend(suggestion)
        return self

class CarousalMessage(Message):
    """
    Represents a carousal message
    """

    def __init__(self):
        Message.__init__(self, "carousel")
        self.elements = []

    def add_carousal_element(self, element):
        self.elements.extend(element)

    def to_json(self):
        attr = {'messageType': self.message_type}
        if len(self.suggestions) > 0:
            attr['suggestionElements'] = [suggestion.to_json() for suggestion in self.suggestions]
        attr['carousalElements'] = [element.to_json() for element in self.elements]

class ListMessage(CarousalMessage):
    def __init__(self):
        CarousalMessage.__init__(self)
        self.cover = True
        self.message_type = "list"

class MediaMessage(Message):
    def __init__(self, media_url, media_type):
        Message.__init__(self, "media")
        self.media_url = media_url
        self.media_type = media_type

class TextMessage(Message):
    def __init__(self, text):
        Message.__init__(self, "statement")
        self.text = text
        self.buttons = []

    def add_button(self, button):
        self.buttons.extend(button)

class CarousalElement:
    def __init__(self, title, sub_title, image_url, click_url):
        self.image_url = image_url
        self.sub_title = sub_title
        self.title = title
        self.click_url = click_url
        self.buttons = []

    def add_button(self, button):
        self.buttons.extend(button)

class Button:
    def __init__(self, title, button_type):
        self.button_type = button_type
        self.title = title

class URLButton(Button):
    def __init__(self, title, url, height):
        Button.__init__(self, title, "URL")
        self.height = height
        self.url = url

class Suggestion:
    """
    Represents a suggestion, aka quick replies
    """

    def __init__(self, title, suggestion_type, payload, image_url):
        """
        Constructor

        :param title: str, The title of the suggestion
        :param suggestion_type: str, One of LOCATION or TEXT
        :param payload: str, The payload to be sent when someone clicks it
        :param image_url: str, The URL of the logo image
        """
        self.imageUrl = image_url
        self.payload = payload
        self.suggestion_type = suggestion_type
        self.title = title
<commit_msg>returning json<commit_after>import json

class Message:
    """
    Represents a single message to be sent
    """

    def __init__(self, message_type):
        """
        Parameters
        :param message_type: str; one of statement, carousel, list, media
        """
        self.message_type = message_type
        self.suggestions = []

    def add_suggestion(self, suggestion):
        """
        Adds a suggestion to this message
        :param suggestion: Suggestion type.
        :return: self
        """
        self.suggestions.extend(suggestion)
        return self

class CarousalMessage(Message):
    """
    Represents a carousal message
    """

    def __init__(self):
        Message.__init__(self, "carousel")
        self.elements = []

    def add_carousal_element(self, element):
        self.elements.extend(element)

    def to_json(self):
        attr = {'messageType': self.message_type}
        if len(self.suggestions) > 0:
            attr['suggestionElements'] = [suggestion.to_json() for suggestion in self.suggestions]
        attr['carousalElements'] = [element.to_json() for element in self.elements]
        return json.dumps(attr)

class ListMessage(CarousalMessage):
    def __init__(self):
        CarousalMessage.__init__(self)
        self.cover = True
        self.message_type = "list"

class MediaMessage(Message):
    def __init__(self, media_url, media_type):
        Message.__init__(self, "media")
        self.media_url = media_url
        self.media_type = media_type

class TextMessage(Message):
    def __init__(self, text):
        Message.__init__(self, "statement")
        self.text = text
        self.buttons = []

    def add_button(self, button):
        self.buttons.extend(button)

class CarousalElement:
    def __init__(self, title, sub_title, image_url, click_url):
        self.image_url = image_url
        self.sub_title = sub_title
        self.title = title
        self.click_url = click_url
        self.buttons = []

    def add_button(self, button):
        self.buttons.extend(button)

class Button:
    def __init__(self, title, button_type):
        self.button_type = button_type
        self.title = title

class URLButton(Button):
    def __init__(self, title, url, height):
        Button.__init__(self, title, "URL")
        self.height = height
        self.url = url

class Suggestion:
    """
    Represents a suggestion, aka quick replies
    """

    def __init__(self, title, suggestion_type, payload, image_url):
        """
        Constructor

        :param title: str, The title of the suggestion
        :param suggestion_type: str, One of LOCATION or TEXT
        :param payload: str, The payload to be sent when someone clicks it
        :param image_url: str, The URL of the logo image
        """
        self.imageUrl = image_url
        self.payload = payload
        self.suggestion_type = suggestion_type
        self.title = title
<|endoftext|>


<commit_before>from time import sleep
from PIL import Image
import os
import math
from .utils import *
from UpdatedImageProcessing import *
from .settings import GCSSettings
from .converter_functions import inverse_haversine, get_mission_json

def run_img_proc_process(logger_queue, location_log, targets_to_submit, interop_client_array):
    while True:
        if len(targets_to_submit) > 0:
            target_characteristics = targets_to_submit.pop(0)

            target_time = get_image_timestamp_from_metadata("static/imgs/" + target_characteristics["base_image_filename"])

            closest_time_index = 0
            least_time_difference = location_log[0]["epoch_time"]
            for index in range(len(location_log)):
                difference_in_times = target_time - location_log[closest_time_index]["epoch_time"]
                if abs(difference_in_times) <= least_time_difference:
                    closest_time_index = index
                    least_time_difference = difference_in_times
            drone_gps_location = location_log[closest_time_index]["current_location"]

            image = Image.open("static/imgs/" + target_characteristics["base_image_filename"])
            image_midpoint = (image.width / 2, image.height / 2)
            target_midpoint = ((target_characteristics["target_top_left"][0] + target_characteristics["target_bottom_right"][0]) / 2, (target_characteristics["target_top_left"][1] + target_characteristics["target_bottom_right"][1]) / 2)
            target_location = get_target_gps_location(image_midpoint, target_midpoint, drone_gps_location)
            target_characteristics["latitude"] = target_location.get_lat()
            target_characteristics["longitude"] = target_location.get_lon()

            original_image_path = "static/imgs/" + target_characteristics["base_image_filename"]
            cropped_target_path = "static/crops/" + str(len(os.listdir('static/crops'))) + ".jpg"
            cropped_target_data_path = "static/crops/" + str(len(os.listdir('static/crops'))) + ".json"
            crop_target(original_image_path, cropped_target_path, target_characteristics["target_top_left"], target_characteristics["target_bottom_right"])
            save_json_data(cropped_target_data_path, target_characteristics)

            if target_characteristics["type"] == "standard":
                interop_client_array[0].post_manual_standard_target(target_characteristics, cropped_target_path)
            elif target_characteristics["type"] == "emergent":
                interop_client_array[0].post_manual_emergent_target(target_characteristics, cropped_target_path)

        sleep(0.1)

def run_autonomous_img_proc_process(logger_queue, location_log, interop_client_array, img_proc_status, recieve_image_filenames):
    target_detected = []
    TARGET_MAP_PATH = "static/imgs/"
    AUTONOMOUS_IMAGE_PROCESSING_SAVE_PATH = "static/autonomous_crops"
    while True:
        current_target_map_name = recieve_image_filenames.recv()
        current_target_map_path = os.path.join(TARGET_MAP_PATH, current_target_map_name)

        combo_target_detection_result_list = TargetDetection.SingleTargetMapDetector.detect_single_target_map(current_target_map_path)
        single_target_crops = combo_target_detection_result_list[0]
        json_file = combo_target_detection_result_list[1]

        image = Image.open(current_target_map_path)
        json_file["target_map_center_location"] = (image.width / 2, image.height / 2)
        json_file["target_map_timestamp"] = get_image_timestamp_from_metadata(current_target_map_path)

        for index_in_single_target_crops in range(len(single_target_crops)):
            json_file["image_processing_results"][index_in_single_target_crops]["target_index"] = index_in_single_target_crops + 1

            current_crop_path = os.path.join(AUTONOMOUS_IMAGE_PROCESSING_SAVE_PATH, current_target_map_name + " - " + str(index_in_single_target_crops + 1) + ".png")
            single_target_crops[index_in_single_target_crops].save(current_crop_path)

            shape_type = ShapeDetection.ShapeClassificationTwo(current_crop_path).get_shape_type()
            json_file["image_processing_results"][index_in_single_target_crops]["target_shape_type"] = shape_type

            color_classifying_results = Classifiers.ColorClassifier(current_crop_path).get_color()
            shape_color = color_classifying_results[0]
            letter_color = color_classifying_results[1]
            json_file["image_processing_results"][index_in_single_target_crops]["target_shape_color"] = shape_color
            json_file["image_processing_results"][index_in_single_target_crops]["target_letter_color"] = letter_color

            # adds target location
            target_map_center_pixel_coordinates = json_file["target_map_center_location"]
            target_pixel_coordinates = json_file["image_processing_results"][index_in_single_target_crops]["target_location"]
            target_time = json_file["target_map_timestamp"]

            closest_time_index = 0
            least_time_difference = location_log[0]["epoch_time"]
            for index in range(len(location_log)):
                difference_in_times = target_time - location_log[closest_time_index]["epoch_time"]
                if abs(difference_in_times) <= least_time_difference:
                    closest_time_index = index
                    least_time_difference = difference_in_times

            drone_gps_location = location_log[closest_time_index]["current_location"]

            target_location = get_target_gps_location(target_map_center_pixel_coordinates, target_pixel_coordinates, drone_gps_location)
            json_file["image_processing_results"][index_in_single_target_crops]["latitude"] = target_location.get_lat()
            json_file["image_processing_results"][index_in_single_target_crops]["longitude"] = target_location.get_lon()

        with open(os.path.join(AUTONOMOUS_IMAGE_PROCESSING_SAVE_PATH, current_target_map_name[:-4] + ".json"), 'w') as fp:
            json.dump(json_file, fp, indent=4)
<commit_msg>image processing updates -- now checks target location<commit_after>from time import sleep
from PIL import Image
import os
import math
from .utils import *
from UpdatedImageProcessing import *
from .settings import GCSSettings
from .converter_functions import inverse_haversine, get_mission_json
from shapely.geometry import MultiPoint, Point

def run_img_proc_process(logger_queue, location_log, targets_to_submit, interop_client_array):
    while True:
        if len(targets_to_submit) > 0:
            target_characteristics = targets_to_submit.pop(0)

            target_time = get_image_timestamp_from_metadata("static/imgs/" + target_characteristics["base_image_filename"])

            closest_time_index = 0
            least_time_difference = location_log[0]["epoch_time"]
            for index in range(len(location_log)):
                difference_in_times = target_time - location_log[closest_time_index]["epoch_time"]
                if abs(difference_in_times) <= least_time_difference:
                    closest_time_index = index
                    least_time_difference = difference_in_times
            drone_gps_location = location_log[closest_time_index]["current_location"]

            image = Image.open("static/imgs/" + target_characteristics["base_image_filename"])
            image_midpoint = (image.width / 2, image.height / 2)
            target_midpoint = ((target_characteristics["target_top_left"][0] + target_characteristics["target_bottom_right"][0]) / 2, (target_characteristics["target_top_left"][1] + target_characteristics["target_bottom_right"][1]) / 2)
            target_location = get_target_gps_location(image_midpoint, target_midpoint, drone_gps_location)
            target_characteristics["latitude"] = target_location.get_lat()
            target_characteristics["longitude"] = target_location.get_lon()

            original_image_path = "static/imgs/" + target_characteristics["base_image_filename"]
            cropped_target_path = "static/crops/" + str(len(os.listdir('static/crops'))) + ".jpg"
            cropped_target_data_path = "static/crops/" + str(len(os.listdir('static/crops'))) + ".json"
            crop_target(original_image_path, cropped_target_path, target_characteristics["target_top_left"], target_characteristics["target_bottom_right"])
            save_json_data(cropped_target_data_path, target_characteristics)

            if target_characteristics["type"] == "standard":
                interop_client_array[0].post_manual_standard_target(target_characteristics, cropped_target_path)
            elif target_characteristics["type"] == "emergent":
                interop_client_array[0].post_manual_emergent_target(target_characteristics, cropped_target_path)

        sleep(0.1)

def run_autonomous_img_proc_process(logger_queue, location_log, interop_client_array, img_proc_status, recieve_image_filenames):
    TARGET_MAP_PATH = "static/imgs/"
    AUTONOMOUS_IMAGE_PROCESSING_SAVE_PATH = "static/autonomous_crops"
    while True:
        current_target_map_name = recieve_image_filenames.recv()
        current_target_map_path = os.path.join(TARGET_MAP_PATH, current_target_map_name)

        combo_target_detection_result_list = TargetDetection.SingleTargetMapDetector.detect_single_target_map(current_target_map_path)
        single_target_crops = combo_target_detection_result_list[0]
        json_file = combo_target_detection_result_list[1]

        image = Image.open(current_target_map_path)
        json_file["target_map_center_location"] = (image.width / 2, image.height / 2)
        json_file["target_map_timestamp"] = get_image_timestamp_from_metadata(current_target_map_path)

        for index_in_single_target_crops in range(len(single_target_crops)):
            json_file["image_processing_results"][index_in_single_target_crops]["target_index"] = index_in_single_target_crops + 1

            current_crop_path = os.path.join(AUTONOMOUS_IMAGE_PROCESSING_SAVE_PATH, current_target_map_name + " - " + str(index_in_single_target_crops + 1) + ".png")
            single_target_crops[index_in_single_target_crops].save(current_crop_path)

            # adds target location
            target_map_center_pixel_coordinates = json_file["target_map_center_location"]
            target_pixel_coordinates = json_file["image_processing_results"][index_in_single_target_crops]["target_location"]
            target_time = json_file["target_map_timestamp"]

            closest_time_index = 0
            least_time_difference = location_log[0]["epoch_time"]
            for index in range(len(location_log)):
                difference_in_times = abs(target_time - location_log[closest_time_index]["epoch_time"])
                if difference_in_times <= least_time_difference:
                    closest_time_index = index
                    least_time_difference = difference_in_times

            drone_gps_location = location_log[closest_time_index]["current_location"]
            target_location = get_target_gps_location(target_map_center_pixel_coordinates, target_pixel_coordinates, drone_gps_location)
            fly_zones = construct_fly_zone_polygon(interop_client_array)
            if (Point(target_location).within(fly_zones)) == False:
                 continue

            json_file["image_processing_results"][index_in_single_target_crops]["latitude"] = target_location.get_lat()
            json_file["image_processing_results"][index_in_single_target_crops]["longitude"] = target_location.get_lon()

            shape_type = ShapeDetection.ShapeClassificationTwo(current_crop_path).get_shape_type()
            json_file["image_processing_results"][index_in_single_target_crops]["target_shape_type"] = shape_type
            color_classifying_results = Classifiers.ColorClassifier(current_crop_path).get_color()
            shape_color = color_classifying_results[0]
            letter_color = color_classifying_results[1]
            json_file["image_processing_results"][index_in_single_target_crops]["target_shape_color"] = shape_color
            json_file["image_processing_results"][index_in_single_target_crops]["target_letter_color"] = letter_color

        with open(os.path.join(AUTONOMOUS_IMAGE_PROCESSING_SAVE_PATH, current_target_map_name[:-4] + ".json"), 'w') as fp:
            json.dump(json_file, fp, indent=4)

def construct_fly_zone_polygon(interop_client_array):
    mission_information_data = (get_mission_json(interop_client_array[0].get_active_mission(), interop_client_array[0].get_obstacles()))
    #mission_information_data["search_grid_points"]
    boundary_points = mission_information_data["fly_zones"]["boundary_pts"]
    point_list = []

    for point_count in range(boundary_points):
        point_list.append([boundary_points[point_count]["latitude"], boundary_points[point_count]["longitude"]])

    return MultiPoint(point_list).convex_hull
<|endoftext|>


<commit_before>#=======================================================================
# verilog_structural.py
#=======================================================================
# Tool to translate PyMTL Models into Verilog HDL.

import re
import sys
import collections

from ...model.signals import InPort, OutPort, Constant

#-----------------------------------------------------------------------
# header
#-----------------------------------------------------------------------
def header( model, symtab ):
  s    = title_bar.format( model.class_name )
  for name, value in model._args.items():
    value_str = '{}'.format( value )
    if 'instance at' in value_str:
      value_str = value_str.split(' at')[0] + '>'
    s += '// {}: {}'.format( name, value_str ) + endl

  dump_vcd = hasattr( model, 'vcd_file' ) and model.vcd_file != ''
  s   += '// dump-vcd: {}'.format( dump_vcd ) + endl

  s   += net_none + endl
  return s

#-----------------------------------------------------------------------
# port_declarations
#-----------------------------------------------------------------------
# Generate Verilog source for port declarations.
def port_declarations( model, symtab ):
  if not model.get_ports(): return ''
  regs = symtab[0]
  for port in model.get_ports():
    port._is_reg = port in regs
    if port._is_reg: regs.remove( port )
  port_list = [ port_decl( x ) for x in model.get_ports() ]
  s  = start_ports + endl
  s += port_delim.join( port_list ) + endl
  s += end_ports + endl
  s += endl
  return s

#------------------------------------------------------------------------
# wire_declarations
#-----------------------------------------------------------------------
# Generate Verilog source for wire declarations.
def wire_declarations( model, symtab ):
  if not model.get_wires(): return ''
  regs = symtab[0]
  wires = [ wire_decl( x ) for x in model.get_wires() if x not in regs ]
  if not wires: return ''
  s  = '  // wire declarations' + endl
  s += wire_delim.join( wires ) + wire_delim + endl
  s += endl
  return s

#-----------------------------------------------------------------------
# submodel_instances
#-----------------------------------------------------------------------
# Generate Verilog source for submodel instances.
def submodel_instances( model, symtab ):

  if not model.get_submodules(): return ''
  regs = symtab[0]

  s = ''
  for submodel in model.get_submodules():

    # Create strings for port connections
    submodel_name = mangle_name( submodel.name )
    temporaries = []
    connections = []
    for p in submodel.get_ports():
      port_name = mangle_name( p.name )
      temp_name = signal_to_str( p, None, model )

      # TODO: not needed since declared in create_declarations
      #if p in regs:
      #  regs.remove(p)
      #  temporaries.append( reg_to_str( p, None, model ) )
      if p not in regs:
        temporaries.append( wire_to_str( p, None, model ) )

      connections.append( connection.format( port_name, temp_name ) )

    connections = pretty_align( connections, '(' )

    # Print the temporaries
    s += '  // {} temporaries'.format( submodel_name ) + endl
    s += wire_delim.join( temporaries ) + wire_delim + endl

    # Print the submodule instantiation
    s += instance.format( submodel.class_name, submodel_name ) + endl
    s += tab + start_ports + endl
    s += port_delim.join( connections ) + endl
    s += tab + end_ports + endl
    s += endl

  return s

#-----------------------------------------------------------------------
# signal_assignments
#-----------------------------------------------------------------------
# Generate Verilog source for signal connections.
def signal_assignments( model, symtab ):
  if not model.get_connections(): return ''

  # Create all the assignment statements
  assignment_list = []
  for c in model.get_connections():
    dest = signal_to_str( c.dest_node, c.dest_slice, model )
    src  = signal_to_str( c.src_node,  c.src_slice,  model )
    assignment_list.append( assignment.format( dest, src ) )

  assignment_list = pretty_align( assignment_list, '=' )
  # Print them in alphabetically sorted order (prettier)
  s  = '  // signal connections' + endl
  s += endl.join( sorted( assignment_list ) )
  s += endl

  return s

#-----------------------------------------------------------------------
# verilog
#-----------------------------------------------------------------------
tab         = '  '
endl        = '\n'
div         = '//' + '-'*77
comment     = '// {}'
net_none    = '`default_nettype none'
net_wire    = '`default_nettype wire'
title_bar   = div + endl + comment + endl + div + endl
start_mod   = 'module {}'
end_mod     = 'endmodule // {}' + endl + net_wire
start_param = '#('
end_param   = ')'
start_ports = '('
end_ports   = ');'
instance    = tab + '{} {}'
connection  = tab + tab + '.{} ( {} )'
ioport_decl = tab + '{:6} {:4} [{:4}:0] {}'
signal_decl = tab + '{:6} [{:4}:0] {}'
port_delim  = ',' + endl
wire_delim  = ';' + endl
assignment  = tab + 'assign {} = {};'

#-----------------------------------------------------------------------
# port_decl
#-----------------------------------------------------------------------
def port_decl( port ):

  if   isinstance( port, InPort ):
    direction = 'input'
  elif isinstance( port, OutPort ):
    direction = 'output'

  type_ = 'reg' if port._is_reg else 'wire'
  nbits = port.nbits - 1
  name  = mangle_name( port.name )

  return ioport_decl.format( direction, type_, nbits, name )

#-----------------------------------------------------------------------
# wire_decl
#-----------------------------------------------------------------------
def wire_decl( port ):

  type_ = 'wire'
  nbits = port.nbits - 1
  name  = mangle_name( port.name )

  return signal_decl.format( type_, nbits, name )

#-----------------------------------------------------------------------
# pretty_align
#-----------------------------------------------------------------------
def pretty_align( assignment_list, char ):
  split = [ x.split( char ) for x in assignment_list ]
  pad   = max( [len(x) for x,y in split] )
  return [ "{0:<{1}}{2}{3}".format( x, pad, char, y ) for x,y in split ]

#-----------------------------------------------------------------------
# wire_to_str
#-----------------------------------------------------------------------
# utility function
# TODO: replace
def wire_to_str( port, slice_=None, parent=None ):
  return '  wire   [{:4}:0] {}'.format( port.nbits-1,
                                signal_to_str( port, slice_, parent ) )

#-----------------------------------------------------------------------
# reg_to_str
#-----------------------------------------------------------------------
def reg_to_str( port, slice_=None, parent=None ):
  return '  reg    [{:4}:0] {}'.format( port.nbits-1,
                                signal_to_str( port, slice_, parent ) )

#-----------------------------------------------------------------------
# mangle_name
#-----------------------------------------------------------------------
def mangle_name( name ):
  # Utility function
  def replacement_string( m ):
    return "${:03d}".format( int(m.group('idx')) )
  # Return the mangled name
  return re.sub( indexing, replacement_string, name.replace('.','_') )

# Regex to match list indexing
indexing = re.compile("(\[)(?P<idx>.*?)(\])")

#-----------------------------------------------------------------------
# signal_to_str
#-----------------------------------------------------------------------
# TODO: clean this up
def signal_to_str( node, addr, context ):

  # Special case constants
  if isinstance( node, Constant ): return node.name

  # If the node's parent module isn't the same as the current module
  # we need to prefix the signal name with the module name
  prefix = ''
  if node.parent != context and node.parent != None:
    prefix = '{}$'.format( mangle_name( node.parent.name ) )

  # If this is a slice, we need to provide the slice indexing
  suffix = ''
  if isinstance( addr, slice ):
    suffix = '[{}:{}]'.format( addr.stop - 1, addr.start )
  elif isinstance( addr, int ):
    suffix = '[{}]'.format( addr )

  # Return the string
  return prefix + mangle_name( node.name ) + suffix

#-----------------------------------------------------------------------
# create_declarations
#-----------------------------------------------------------------------
# TODO: clean this up!
def create_declarations( model, regs, ints, params, arrays ):

  scode = ''

  # Print the reg declarations
  if regs:
    scode += '  // register declarations\n'
    regs   = sorted( regs, key=lambda x: signal_to_str( x, None, model ) )

    for signal in regs:
      # If name also inferenced as an 'integer' type, declare as reg only
      if signal.parent == None and signal.name in ints:
        ints.remove( signal.name )
      scode += '  reg    [{:4}:0] {};\n' \
               .format( signal.nbits-1, signal_to_str( signal, None, model ))
    scode += '\n'

  # Print the localparam declarations
  if params:
    params = sorted( params, key=lambda x: x[0] )
    scode += '  // localparam declarations\n'
    for param, value in params:
      rhs    = "{}'d{}".format( value.nbits, value.uint() ) \
               if hasattr( value, 'nbits' ) else \
               "{}"    .format( value )
      scode += '  localparam {} = {};\n'.format( param, rhs )
    scode += '\n'

  # Print the int declarations
  if ints:
    ints = sorted( ints )
    scode += '  // loop variable declarations\n'
    for signal in ints:
      scode += '  integer {};\n'.format( signal )
    scode += '\n'

  return scode

#-----------------------------------------------------------------------
# array_declarations
#-----------------------------------------------------------------------
# TODO: clean this up!
def array_declarations( model, symtab ):

  arrays = symtab[-1]
  scode  = ''

  # Print the array declarations
  if arrays:
    arrays = sorted( arrays, key=lambda x: x.name )
    scode += '  // array declarations\n'
    for ports in arrays:

      first = ports[0]

      # Output Port
      if ports.is_lhs:
        scode += '  reg    [{:4}:0] {}[0:{}];\n'.format(
            first.nbits-1, ports.name, len(ports)-1)
        for i, port in enumerate(ports):
          scode += '  assign {0} = {1}[{2:3}];\n'.format(
              signal_to_str( port, None, model ), ports.name, i )

      # Input Port
      else:
        scode += '  wire   [{:4}:0] {}[0:{}];\n'.format(
            first.nbits-1, ports.name, len(ports)-1)
        for i, port in enumerate(ports):
          scode += '  assign {1}[{2:3}] = {0};\n'.format(
              signal_to_str( port, None, model ), ports.name, i )

    scode += '\n'

  return scode

<commit_msg>[pymtl.tools] rm [0:0] from 1-bit wire declarations<commit_after>#=======================================================================
# verilog_structural.py
#=======================================================================
# Tool to translate PyMTL Models into Verilog HDL.

import re
import sys
import collections

from ...model.signals import InPort, OutPort, Constant

#-----------------------------------------------------------------------
# header
#-----------------------------------------------------------------------
def header( model, symtab ):
  s    = title_bar.format( model.class_name )
  for name, value in model._args.items():
    value_str = '{}'.format( value )
    if 'instance at' in value_str:
      value_str = value_str.split(' at')[0] + '>'
    s += '// {}: {}'.format( name, value_str ) + endl

  dump_vcd = hasattr( model, 'vcd_file' ) and model.vcd_file != ''
  s   += '// dump-vcd: {}'.format( dump_vcd ) + endl

  s   += net_none + endl
  return s

#-----------------------------------------------------------------------
# port_declarations
#-----------------------------------------------------------------------
# Generate Verilog source for port declarations.
def port_declarations( model, symtab ):
  if not model.get_ports(): return ''
  regs = symtab[0]
  for port in model.get_ports():
    port._is_reg = port in regs
    if port._is_reg: regs.remove( port )
  port_list = [ port_decl( x ) for x in model.get_ports() ]
  s  = start_ports + endl
  s += port_delim.join( port_list ) + endl
  s += end_ports + endl
  s += endl
  return s

#------------------------------------------------------------------------
# wire_declarations
#-----------------------------------------------------------------------
# Generate Verilog source for wire declarations.
def wire_declarations( model, symtab ):
  if not model.get_wires(): return ''
  regs = symtab[0]
  wires = [ wire_decl( x ) for x in model.get_wires() if x not in regs ]
  if not wires: return ''
  s  = '  // wire declarations' + endl
  s += wire_delim.join( wires ) + wire_delim + endl
  s += endl
  return s

#-----------------------------------------------------------------------
# submodel_instances
#-----------------------------------------------------------------------
# Generate Verilog source for submodel instances.
def submodel_instances( model, symtab ):

  if not model.get_submodules(): return ''
  regs = symtab[0]

  s = ''
  for submodel in model.get_submodules():

    # Create strings for port connections
    submodel_name = mangle_name( submodel.name )
    temporaries = []
    connections = []
    for p in submodel.get_ports():
      port_name = mangle_name( p.name )
      temp_name = signal_to_str( p, None, model )

      # TODO: not needed since declared in create_declarations
      #if p in regs:
      #  regs.remove(p)
      #  temporaries.append( reg_to_str( p, None, model ) )
      if p not in regs:
        temporaries.append( wire_to_str( p, None, model ) )

      connections.append( connection.format( port_name, temp_name ) )

    connections = pretty_align( connections, '(' )

    # Print the temporaries
    s += '  // {} temporaries'.format( submodel_name ) + endl
    s += wire_delim.join( temporaries ) + wire_delim + endl

    # Print the submodule instantiation
    s += instance.format( submodel.class_name, submodel_name ) + endl
    s += tab + start_ports + endl
    s += port_delim.join( connections ) + endl
    s += tab + end_ports + endl
    s += endl

  return s

#-----------------------------------------------------------------------
# signal_assignments
#-----------------------------------------------------------------------
# Generate Verilog source for signal connections.
def signal_assignments( model, symtab ):
  if not model.get_connections(): return ''

  # Create all the assignment statements
  assignment_list = []
  for c in model.get_connections():
    dest = signal_to_str( c.dest_node, c.dest_slice, model )
    src  = signal_to_str( c.src_node,  c.src_slice,  model )
    assignment_list.append( assignment.format( dest, src ) )

  assignment_list = pretty_align( assignment_list, '=' )
  # Print them in alphabetically sorted order (prettier)
  s  = '  // signal connections' + endl
  s += endl.join( sorted( assignment_list ) )
  s += endl

  return s

#-----------------------------------------------------------------------
# verilog
#-----------------------------------------------------------------------
tab         = '  '
endl        = '\n'
div         = '//' + '-'*77
comment     = '// {}'
net_none    = '`default_nettype none'
net_wire    = '`default_nettype wire'
title_bar   = div + endl + comment + endl + div + endl
start_mod   = 'module {}'
end_mod     = 'endmodule // {}' + endl + net_wire
start_param = '#('
end_param   = ')'
start_ports = '('
end_ports   = ');'
instance    = tab + '{} {}'
connection  = tab + tab + '.{} ( {} )'
nbits_decl  = '[{:4}:0]'
onebit_decl = ' '*len(nbits_decl.format(0))
ioport_decl = tab + '{:6} {:4} {} {}'
signal_decl = tab + '{:6} {} {}'
port_delim  = ',' + endl
wire_delim  = ';' + endl
assignment  = tab + 'assign {} = {};'
def declare_bitwidth( nbits ):
  return nbits_decl.format( nbits ) if nbits else onebit_decl

#-----------------------------------------------------------------------
# port_decl
#-----------------------------------------------------------------------
def port_decl( port ):

  if   isinstance( port, InPort ):
    direction = 'input'
  elif isinstance( port, OutPort ):
    direction = 'output'

  type_ = 'reg' if port._is_reg else 'wire'
  nbits = port.nbits - 1
  name  = mangle_name( port.name )

  return ioport_decl.format( direction, type_, declare_bitwidth(nbits), name )

#-----------------------------------------------------------------------
# wire_decl
#-----------------------------------------------------------------------
def wire_decl( port ):

  type_ = 'wire'
  nbits = port.nbits - 1
  name  = mangle_name( port.name )

  return signal_decl.format( type_, declare_bitwidth(nbits), name )

#-----------------------------------------------------------------------
# pretty_align
#-----------------------------------------------------------------------
def pretty_align( assignment_list, char ):
  split = [ x.split( char ) for x in assignment_list ]
  pad   = max( [len(x) for x,y in split] )
  return [ "{0:<{1}}{2}{3}".format( x, pad, char, y ) for x,y in split ]

#-----------------------------------------------------------------------
# wire_to_str
#-----------------------------------------------------------------------
# utility function
# TODO: replace
def wire_to_str( port, slice_=None, parent=None ):
  return '  wire   {} {}'.format(
    declare_bitwidth(port.nbits-1),
    signal_to_str( port, slice_, parent )
  )

#-----------------------------------------------------------------------
# reg_to_str
#-----------------------------------------------------------------------
def reg_to_str( port, slice_=None, parent=None ):
  return '  reg    {} {}'.format(
    declare_bitwidth(port.nbits-1),
    signal_to_str( port, slice_, parent )
  )

#-----------------------------------------------------------------------
# mangle_name
#-----------------------------------------------------------------------
def mangle_name( name ):
  # Utility function
  def replacement_string( m ):
    return "${:03d}".format( int(m.group('idx')) )
  # Return the mangled name
  return re.sub( indexing, replacement_string, name.replace('.','_') )

# Regex to match list indexing
indexing = re.compile("(\[)(?P<idx>.*?)(\])")

#-----------------------------------------------------------------------
# signal_to_str
#-----------------------------------------------------------------------
# TODO: clean this up
def signal_to_str( node, addr, context ):

  # Special case constants
  if isinstance( node, Constant ): return node.name

  # If the node's parent module isn't the same as the current module
  # we need to prefix the signal name with the module name
  prefix = ''
  if node.parent != context and node.parent != None:
    prefix = '{}$'.format( mangle_name( node.parent.name ) )

  # If this is a slice, we need to provide the slice indexing
  suffix = ''
  if isinstance( addr, slice ):
    suffix = '[{}:{}]'.format( addr.stop - 1, addr.start )
  elif isinstance( addr, int ):
    suffix = '[{}]'.format( addr )

  # Return the string
  return prefix + mangle_name( node.name ) + suffix

#-----------------------------------------------------------------------
# create_declarations
#-----------------------------------------------------------------------
# TODO: clean this up!
def create_declarations( model, regs, ints, params, arrays ):

  scode = ''

  # Print the reg declarations
  if regs:
    scode += '  // register declarations\n'
    regs   = sorted( regs, key=lambda x: signal_to_str( x, None, model ) )

    for signal in regs:
      # If name also inferenced as an 'integer' type, declare as reg only
      if signal.parent == None and signal.name in ints:
        ints.remove( signal.name )
      scode += '  reg    [{:4}:0] {};\n' \
               .format( signal.nbits-1, signal_to_str( signal, None, model ))
    scode += '\n'

  # Print the localparam declarations
  if params:
    params = sorted( params, key=lambda x: x[0] )
    scode += '  // localparam declarations\n'
    for param, value in params:
      rhs    = "{}'d{}".format( value.nbits, value.uint() ) \
               if hasattr( value, 'nbits' ) else \
               "{}"    .format( value )
      scode += '  localparam {} = {};\n'.format( param, rhs )
    scode += '\n'

  # Print the int declarations
  if ints:
    ints = sorted( ints )
    scode += '  // loop variable declarations\n'
    for signal in ints:
      scode += '  integer {};\n'.format( signal )
    scode += '\n'

  return scode

#-----------------------------------------------------------------------
# array_declarations
#-----------------------------------------------------------------------
# TODO: clean this up!
def array_declarations( model, symtab ):

  arrays = symtab[-1]
  scode  = ''

  # Print the array declarations
  if arrays:
    arrays = sorted( arrays, key=lambda x: x.name )
    scode += '  // array declarations\n'
    for ports in arrays:

      first = ports[0]

      # Output Port
      if ports.is_lhs:
        scode += '  reg    [{:4}:0] {}[0:{}];\n'.format(
            first.nbits-1, ports.name, len(ports)-1)
        for i, port in enumerate(ports):
          scode += '  assign {0} = {1}[{2:3}];\n'.format(
              signal_to_str( port, None, model ), ports.name, i )

      # Input Port
      else:
        scode += '  wire   [{:4}:0] {}[0:{}];\n'.format(
            first.nbits-1, ports.name, len(ports)-1)
        for i, port in enumerate(ports):
          scode += '  assign {1}[{2:3}] = {0};\n'.format(
              signal_to_str( port, None, model ), ports.name, i )

    scode += '\n'

  return scode

<|endoftext|>


<commit_before>import re
from datetime import datetime, timedelta

from mongoengine import *

from core.config.mongoengine_extras import TimeDeltaField
from core.database import Node
from core.errors import TagValidationError

class Tag(Node):
    name = StringField(required=True, unique=True)
    count = IntField(required=True, default=0)
    created = DateTimeField(default=datetime.now)
    produces = ListField(ReferenceField("Tag", reverse_delete_rule=PULL))
    replaces = ListField(StringField())

    def info(self):
        i = {k: v for k, v in self._data.items() if k in ["name", "count", "created", "replaces"]}
        i['id'] = str(self.id)
        i['produces'] = [tag.name for tag in self.produces]
        return i

    def add_replaces(self, tags):
        if isinstance(tags, (str, unicode)):
            tags = [tags]
        self.replaces += list(set(tags + self.replaces))
        return self.save()

    def add_produces(self, tags):
        if isinstance(tags, (str, unicode)):
            tags = [Tag.get_or_create(name=tags)]
        else:
            tags = [Tag.get_or_create(name=t) for t in tags]

        self.produces = tags
        return self.save()

    def clean(self):
        self.name = re.sub("[^a-z0-9\-_]", "", self.name.lower())
        if not self.name:
            raise TagValidationError("{} is not a valid tag. Valid chars = [a-z0-9\\-_]".format(repr(self.name)))
        self.produces = list(set(self.produces))

    def __unicode__(self):
        return self.name

class ObservableTag(EmbeddedDocument):

    name = StringField(required=True)
    first_seen = DateTimeField(default=datetime.now)
    last_seen = DateTimeField(default=datetime.now)
    expiration = TimeDeltaField(default=timedelta(days=365))
    fresh = BooleanField(default=True)

    def __unicode__(self):
        return u"{} ({})".format(self.name, "fresh" if self.fresh else "old")

    def info(self):
        i = {k: v for k, v in self._data.items() if k in ["first_seen", "last_seen", "fresh", "name"]}
        return i
<commit_msg>Replace spaces with _<commit_after>import re
from datetime import datetime, timedelta

from mongoengine import *

from core.config.mongoengine_extras import TimeDeltaField
from core.database import Node
from core.errors import TagValidationError

class Tag(Node):
    name = StringField(required=True, unique=True)
    count = IntField(required=True, default=0)
    created = DateTimeField(default=datetime.now)
    produces = ListField(ReferenceField("Tag", reverse_delete_rule=PULL))
    replaces = ListField(StringField())

    def info(self):
        i = {k: v for k, v in self._data.items() if k in ["name", "count", "created", "replaces"]}
        i['id'] = str(self.id)
        i['produces'] = [tag.name for tag in self.produces]
        return i

    def add_replaces(self, tags):
        if isinstance(tags, (str, unicode)):
            tags = [tags]
        self.replaces += list(set(tags + self.replaces))
        return self.save()

    def add_produces(self, tags):
        if isinstance(tags, (str, unicode)):
            tags = [Tag.get_or_create(name=tags)]
        else:
            tags = [Tag.get_or_create(name=t) for t in tags]

        self.produces = tags
        return self.save()

    def clean(self):
        self.name = re.sub("[^a-z0-9\-_ ]", "", self.name.lower())
        self.name = re.sub(" ", "_", self.name)
        if not self.name:
            raise TagValidationError("{} is not a valid tag. Valid chars = [a-z0-9\\-_]".format(repr(self.name)))
        self.produces = list(set(self.produces))

    def __unicode__(self):
        return self.name

class ObservableTag(EmbeddedDocument):

    name = StringField(required=True)
    first_seen = DateTimeField(default=datetime.now)
    last_seen = DateTimeField(default=datetime.now)
    expiration = TimeDeltaField(default=timedelta(days=365))
    fresh = BooleanField(default=True)

    def __unicode__(self):
        return u"{} ({})".format(self.name, "fresh" if self.fresh else "old")

    def info(self):
        i = {k: v for k, v in self._data.items() if k in ["first_seen", "last_seen", "fresh", "name"]}
        return i
<|endoftext|>


<commit_before>"""\
Run this when links or keys are changed in the schema.

Examples

To update all keys and rels on the production server:

    %(prog)s production.ini

For the development.ini you must supply the paster app name:

    %(prog)s development.ini --app-name app

"""
import logging
import transaction
from ..storage import (
    DBSession,
    update_keys,
    update_rels,
)
from future.utils import itervalues
from pyramid.paster import get_app
from pyramid.traversal import resource_path

EPILOG = __doc__

logger = logging.getLogger(__name__)

DEFAULT_COLLECTIONS = [
    'dataset',
    'experiment'
]

def run(app, collections=None):
    root = app.root_factory(app)
    session = DBSession()
    if not collections:
        collections = DEFAULT_COLLECTIONS
    for collection_name in collections:
        collection = root[collection_name]
        collection_path = resource_path(collection)
        updated = 0
        errors = 0
        for count, item in enumerate(itervalues(collection)):
            path = resource_path(item)
            update = False
            sp = session.begin_nested()
            try:
                properties = item.upgrade_properties(finalize=False)
                unique_keys = item.unique_keys(properties)
                links = item.links(properties)
                keys_add, keys_remove = update_keys(item.model, unique_keys)
                if keys_add or keys_remove:
                    logger.debug('Updated keys: %s' % path)
                    update = True
                rels_add, rels_remove = update_rels(item.model, links)
                if rels_add or rels_remove:
                    logger.debug('Updated links: %s' % path)
                    update = True
            except Exception:
                logger.exception('Error updating keys: %s', path)
                sp.rollback()
                errors += 1
            else:
                sp.commit()
            if update:
                updated += 1
        logger.info('Collection %s: Updated %d of %d (errors %d)' %
                    (collection_path, updated, count + 1, errors))

def main():
    import argparse
    parser = argparse.ArgumentParser(
        description="Update links and keys", epilog=EPILOG,
        formatter_class=argparse.RawDescriptionHelpFormatter,
    )
    parser.add_argument('--item-type', action='append', help="Item type")
    parser.add_argument('--app-name', help="Pyramid app name in configfile")
    parser.add_argument('--abort', action='store_true', help="Rollback transaction")
    parser.add_argument('config_uri', help="path to configfile")
    args = parser.parse_args()

    logging.basicConfig()
    app = get_app(args.config_uri, args.app_name)
    # Loading app will have configured from config file. Reconfigure here:
    logging.getLogger('encoded').setLevel(logging.DEBUG)

    raised = False
    try:
        run(app, args.item_type)
    except:
        raised = True
        raise
    finally:
        if raised or args.abort:
            transaction.abort()
            logger.info('Rolled back.')
        else:
            transaction.commit()

if __name__ == '__main__':
    main()
<commit_msg>Remove dataset, experiment from update list<commit_after>"""\
Run this when links or keys are changed in the schema.

Examples

To update all keys and rels on the production server:

    %(prog)s production.ini

For the development.ini you must supply the paster app name:

    %(prog)s development.ini --app-name app

"""
import logging
import transaction
from ..storage import (
    DBSession,
    update_keys,
    update_rels,
)
from future.utils import itervalues
from pyramid.paster import get_app
from pyramid.traversal import resource_path

EPILOG = __doc__

logger = logging.getLogger(__name__)

DEFAULT_COLLECTIONS = [
]

def run(app, collections=None):
    root = app.root_factory(app)
    session = DBSession()
    if not collections:
        collections = DEFAULT_COLLECTIONS
    for collection_name in collections:
        collection = root[collection_name]
        collection_path = resource_path(collection)
        updated = 0
        errors = 0
        for count, item in enumerate(itervalues(collection)):
            path = resource_path(item)
            update = False
            sp = session.begin_nested()
            try:
                properties = item.upgrade_properties(finalize=False)
                unique_keys = item.unique_keys(properties)
                links = item.links(properties)
                keys_add, keys_remove = update_keys(item.model, unique_keys)
                if keys_add or keys_remove:
                    logger.debug('Updated keys: %s' % path)
                    update = True
                rels_add, rels_remove = update_rels(item.model, links)
                if rels_add or rels_remove:
                    logger.debug('Updated links: %s' % path)
                    update = True
            except Exception:
                logger.exception('Error updating keys: %s', path)
                sp.rollback()
                errors += 1
            else:
                sp.commit()
            if update:
                updated += 1
        logger.info('Collection %s: Updated %d of %d (errors %d)' %
                    (collection_path, updated, count + 1, errors))

def main():
    import argparse
    parser = argparse.ArgumentParser(
        description="Update links and keys", epilog=EPILOG,
        formatter_class=argparse.RawDescriptionHelpFormatter,
    )
    parser.add_argument('--item-type', action='append', help="Item type")
    parser.add_argument('--app-name', help="Pyramid app name in configfile")
    parser.add_argument('--abort', action='store_true', help="Rollback transaction")
    parser.add_argument('config_uri', help="path to configfile")
    args = parser.parse_args()

    logging.basicConfig()
    app = get_app(args.config_uri, args.app_name)
    # Loading app will have configured from config file. Reconfigure here:
    logging.getLogger('encoded').setLevel(logging.DEBUG)

    raised = False
    try:
        run(app, args.item_type)
    except:
        raised = True
        raise
    finally:
        if raised or args.abort:
            transaction.abort()
            logger.info('Rolled back.')
        else:
            transaction.commit()

if __name__ == '__main__':
    main()
<|endoftext|>


<commit_before>import json
import logging

from django.shortcuts import render
from django.core.mail import send_mail
from django.conf import settings
from django.views.decorators.csrf import csrf_exempt
from django.http import HttpResponse
from premailer import transform
from PIL import Image, ImageDraw

from .models import Upload

logger = logging.getLogger(__name__)
logger.setLevel(logging.DEBUG)

def index(request):
    return render(request, 'mosaico/index.html')

def editor(request):
    return render(request, 'mosaico/editor.html')

# mosaico views from https://github.com/voidlabs/mosaico/tree/master/backend

@csrf_exempt
def download(request):
    html = transform(request.POST['html'])
    action = request.POST['action']
    if action == 'download':
        filename = request.POST['filename']
        content_type = "text/html"
        content_disposition = "attachment; filename=%s" % filename
        response = HttpResponse(html, content_type=content_type)
        response['Content-Disposition'] = content_disposition
    elif action == 'email':
        to = request.POST['rcpt']
        subject = request.POST['subject']
        from_email = settings.DEFAULT_FROM_EMAIL
        send_mail(subject, "", from_email, [to], html_message=html, fail_silently=False)
        # TODO: return the mail ID here
        response = HttpResponse("OK: 250 OK id=12345")
    return response

@csrf_exempt
def upload(request):
    if request.method == 'POST':
        file = request.FILES.values()[0]
        upload = Upload(name=file.name, image=file)
        upload.save()
    uploads = Upload.objects.all()
    data = {'files': []}
    for upload in uploads:
        data['files'].append(upload.to_json_data())
    response = HttpResponse(json.dumps(data), content_type="application/json")
    return response

@csrf_exempt
def image(request):
    logger.debug("request.method: %r", request.method)
    if request.method == 'GET':
        method = request.GET['method']
        logger.debug("method: %r", method)
        params = request.GET['params'].split(',')
        logger.debug("params: %r", params)
        if method == 'placeholder':
            height, width = [size(p) for p in params]
            image = get_placeholder_image(height, width)
            response = HttpResponse(content_type="image/png")
            image.save(response, "PNG")
        elif method == 'cover':
            src = request.GET['src']
            height, width = [size(p) for p in params]
            for upload in Upload.objects.all():
                if upload.image.url == src:
                    break
            image = Image.open(upload.image.file)
            if not width:
                width = upload.image.width
            if not height:
                height = upload.image.height
            image.thumbnail((width, height), Image.ANTIALIAS)
            response = HttpResponse(content_type="image/%s" % image.format.lower())
            image.save(response, image.format)
        elif method == 'resize':
            src = request.GET['src']
            height, width = [size(p) for p in params]
            for upload in Upload.objects.all():
                if upload.image.url == src:
                    break
            image = Image.open(upload.image.file)
            if not width:
                width = upload.image.width
            if not height:
                height = upload.image.height
            image.resize((width, height))
            response = HttpResponse(content_type="image/%s" % image.format.lower())
        return response

def get_placeholder_image(width, height):
    image = Image.new("RGB", (width, height))
    draw = ImageDraw.Draw(image)
    draw.rectangle([(0, 0), (width, height)], fill=(0x70, 0x70, 0x70))
    # stripes
    x = 0
    y = 0
    size = 40
    while y < height:
        draw.polygon([(x, y), (x + size, y), (x + size*2, y + size), (x + size*2, y + size*2)], fill=(0x80, 0x80, 0x80))
        draw.polygon([(x, y + size), (x + size, y + size*2), (x, y + size*2)], fill=(0x80, 0x80, 0x80))
        x = x + size*2
        if (x > width):
            x = 0
            y = y + size*2
    return image

def size(size_txt):
    if size_txt == 'null':
        return None
    else:
        return int(size_txt)
<commit_msg>use only the path to lookup the upload<commit_after>import json
import logging
from urlparse import urlsplit

from django.shortcuts import render
from django.core.mail import send_mail
from django.conf import settings
from django.views.decorators.csrf import csrf_exempt
from django.http import HttpResponse
from premailer import transform
from PIL import Image, ImageDraw

from .models import Upload

logger = logging.getLogger(__name__)
logger.setLevel(logging.DEBUG)

def index(request):
    return render(request, 'mosaico/index.html')

def editor(request):
    return render(request, 'mosaico/editor.html')

# mosaico views from https://github.com/voidlabs/mosaico/tree/master/backend

@csrf_exempt
def download(request):
    html = transform(request.POST['html'])
    action = request.POST['action']
    if action == 'download':
        filename = request.POST['filename']
        content_type = "text/html"
        content_disposition = "attachment; filename=%s" % filename
        response = HttpResponse(html, content_type=content_type)
        response['Content-Disposition'] = content_disposition
    elif action == 'email':
        to = request.POST['rcpt']
        subject = request.POST['subject']
        from_email = settings.DEFAULT_FROM_EMAIL
        send_mail(subject, "", from_email, [to], html_message=html, fail_silently=False)
        # TODO: return the mail ID here
        response = HttpResponse("OK: 250 OK id=12345")
    return response

@csrf_exempt
def upload(request):
    if request.method == 'POST':
        file = request.FILES.values()[0]
        upload = Upload(name=file.name, image=file)
        upload.save()
    uploads = Upload.objects.all()
    data = {'files': []}
    for upload in uploads:
        data['files'].append(upload.to_json_data())
    response = HttpResponse(json.dumps(data), content_type="application/json")
    return response

@csrf_exempt
def image(request):
    logger.debug("request.method: %r", request.method)
    if request.method == 'GET':
        method = request.GET['method']
        logger.debug("method: %r", method)
        params = request.GET['params'].split(',')
        logger.debug("params: %r", params)
        if method == 'placeholder':
            height, width = [size(p) for p in params]
            image = get_placeholder_image(height, width)
            response = HttpResponse(content_type="image/png")
            image.save(response, "PNG")
        elif method == 'cover':
            src = request.GET['src']
            height, width = [size(p) for p in params]
            for upload in Upload.objects.all():
                if upload.image.url == src:
                    break
            image = Image.open(upload.image.file)
            if not width:
                width = upload.image.width
            if not height:
                height = upload.image.height
            image.thumbnail((width, height), Image.ANTIALIAS)
            response = HttpResponse(content_type="image/%s" % image.format.lower())
            image.save(response, image.format)
        elif method == 'resize':
            src = request.GET['src']
            path = urlsplit(src).path
            height, width = [size(p) for p in params]
            for upload in Upload.objects.all():
                if upload.image.url == path:
                    break
            image = Image.open(upload.image.file)
            if not width:
                width = upload.image.width
            if not height:
                height = upload.image.height
            image.resize((width, height))
            response = HttpResponse(content_type="image/%s" % image.format.lower())
        return response

def get_placeholder_image(width, height):
    image = Image.new("RGB", (width, height))
    draw = ImageDraw.Draw(image)
    draw.rectangle([(0, 0), (width, height)], fill=(0x70, 0x70, 0x70))
    # stripes
    x = 0
    y = 0
    size = 40
    while y < height:
        draw.polygon([(x, y), (x + size, y), (x + size*2, y + size), (x + size*2, y + size*2)], fill=(0x80, 0x80, 0x80))
        draw.polygon([(x, y + size), (x + size, y + size*2), (x, y + size*2)], fill=(0x80, 0x80, 0x80))
        x = x + size*2
        if (x > width):
            x = 0
            y = y + size*2
    return image

def size(size_txt):
    if size_txt == 'null':
        return None
    else:
        return int(size_txt)
<|endoftext|>


<commit_before>import json
from datetime import timedelta
from tornado import gen

from tornado.httpclient import AsyncHTTPClient, HTTPError
from tornado.log import app_log

from tornado.locks import BoundedSemaphore
from tornado.queues import Queue

from .base_filter import GitHubRepositoryDateFilter
from vizydrop.sdk.source import StreamingDataSource, SourceSchema
from vizydrop.utils import parse_link_header
from vizydrop.fields import *

# how many concurrent fetches can we do?
FETCH_CONCURRENCY = 10
# our maximum request time (in seconds)
MAXIMUM_REQ_TIME = 30

class GitHubCommitsSource(StreamingDataSource):
    class Meta:
        identifier = "commits"
        name = "Commits"
        tags = ["Commits", "Source Code", ]
        description = "List of commits"
        filter = GitHubRepositoryDateFilter

    class Schema(SourceSchema):
        date = DateField(name="Date of the commit")
        author = TextField(name="Author")
        added_files = NumberField(name="Added Files")
        deleted_files = NumberField(name="Deleted Files")
        modified_files = NumberField(name="Modified Files")
        additions = NumberField(name="Code Additions")
        deletions = NumberField(name="Code Deletions")

    @classmethod
    @gen.coroutine
    def get_data(cls, account, source_filter, limit=100, skip=0):
        """
        Gathers commit information from GH
        GET https://api.github.com/repos/:owner/:repo/commits
        Header: Accept: application/vnd.github.v3+json
        """
        if not account or not account.enabled:
            raise ValueError('cannot gather information without a valid account')
        client = AsyncHTTPClient()

        source_filter = GitHubRepositoryDateFilter(source_filter)

        if source_filter.repository is None:
            raise ValueError('required parameter projects missing')

        default_headers = {"Content-Type": "application/json", "Accept": "application/vnd.github.v3+json"}

        # first we grab our list of commits
        uri = "https://api.github.com/repos/{}/commits".format(source_filter.repository)
        qs = source_filter.get_qs()
        if qs != '':
            uri = uri + '?' + qs
        app_log.info("Starting retrieval of commit list for account {}".format(account._id))
        if limit is not None and limit <= 100:
            # we can handle our limit right here
            uri += "?per_page={}".format(limit)
        elif limit is None:
            uri += "?per_page=100"  # maximum number per page for GitHub API
        taken = 0

        queue = Queue()
        sem = BoundedSemaphore(FETCH_CONCURRENCY)
        done, working = set(), set()

        while uri is not None:
            app_log.info(
                "({}) Retrieving next page, received {} commits thus far".format(account._id, taken))
            req = account.get_request(uri, headers=default_headers)
            response = yield client.fetch(req)

            page_data = json.loads(response.body.decode('utf-8'))
            taken += page_data.__len__()
            for item in page_data:
                queue.put(item.get('url', None))

            if limit is None or taken < limit:
                # parse the Link header from GitHub (https://developer.github.com/v3/#pagination)
                links = parse_link_header(response.headers.get('Link', ''))
                uri = links.get('next', None)
            else:
                break
        app_log.info("({}) Commit list retrieved, fetching info for {} commits".format(account._id, taken))

        if queue.qsize() > 500:
            raise HTTPError(413, 'too many commits')

        # open our list
        cls.write('[')

        # our worker to actually fetch the info
        @gen.coroutine
        def fetch_url():
            current_url = yield queue.get()
            try:
                if current_url in working:
                    return
                page_no = working.__len__()
                app_log.info("Fetching page {}".format(page_no))
                working.add(current_url)
                req = account.get_request(current_url)
                client = AsyncHTTPClient()
                response = yield client.fetch(req)
                response_data = json.loads(response.body.decode('utf-8'))
                obj = {
                    'date': response_data['commit']['author']['date'],
                    'author': response_data['commit']['author']['name'],
                    'added_files': [file for file in response_data['files'] if file['status'] == 'added'].__len__(),
                    'deleted_files': [file for file in response_data['files'] if file['status'] == 'deleted'].__len__(),
                    'modified_files': [file for file in response_data['files'] if file['status'] == 'modified'].__len__(),
                    'additions': response_data['stats']['additions'],
                    'deletions': response_data['stats']['deletions']
                }
                if done.__len__() > 0:
                    cls.write(',')
                cls.write(json.dumps(obj))
                done.add(current_url)
                app_log.info("Page {} downloaded".format(page_no))

            finally:
                queue.task_done()
                sem.release()

        @gen.coroutine
        def worker():
            while True:
                yield sem.acquire()
                fetch_url()

        # start our concurrency worker
        worker()
        try:
            # wait until we're done
            yield queue.join(timeout=timedelta(seconds=MAXIMUM_REQ_TIME))
        except gen.TimeoutError:
            app_log.warning("Request exceeds maximum time, cutting response short")
        finally:
            # close our list
            cls.write(']')
        app_log.info("Finished retrieving commits for {}".format(account._id))
<commit_msg>Check for maximum commits now inside first loop<commit_after>import json
from datetime import timedelta
from tornado import gen

from tornado.httpclient import AsyncHTTPClient, HTTPError
from tornado.log import app_log

from tornado.locks import BoundedSemaphore
from tornado.queues import Queue

from .base_filter import GitHubRepositoryDateFilter
from vizydrop.sdk.source import StreamingDataSource, SourceSchema
from vizydrop.utils import parse_link_header
from vizydrop.fields import *

# how many concurrent fetches can we do?
FETCH_CONCURRENCY = 10
# our maximum request time (in seconds)
MAXIMUM_REQ_TIME = 30

class GitHubCommitsSource(StreamingDataSource):
    class Meta:
        identifier = "commits"
        name = "Commits"
        tags = ["Commits", "Source Code", ]
        description = "List of commits"
        filter = GitHubRepositoryDateFilter

    class Schema(SourceSchema):
        date = DateField(name="Date of the commit")
        author = TextField(name="Author")
        added_files = NumberField(name="Added Files")
        deleted_files = NumberField(name="Deleted Files")
        modified_files = NumberField(name="Modified Files")
        additions = NumberField(name="Code Additions")
        deletions = NumberField(name="Code Deletions")

    @classmethod
    @gen.coroutine
    def get_data(cls, account, source_filter, limit=100, skip=0):
        """
        Gathers commit information from GH
        GET https://api.github.com/repos/:owner/:repo/commits
        Header: Accept: application/vnd.github.v3+json
        """
        if not account or not account.enabled:
            raise ValueError('cannot gather information without a valid account')
        client = AsyncHTTPClient()

        source_filter = GitHubRepositoryDateFilter(source_filter)

        if source_filter.repository is None:
            raise ValueError('required parameter projects missing')

        default_headers = {"Content-Type": "application/json", "Accept": "application/vnd.github.v3+json"}

        # first we grab our list of commits
        uri = "https://api.github.com/repos/{}/commits".format(source_filter.repository)
        qs = source_filter.get_qs()
        if qs != '':
            uri = uri + '?' + qs
        app_log.info("Starting retrieval of commit list for account {}".format(account._id))
        if limit is not None and limit <= 100:
            # we can handle our limit right here
            uri += "?per_page={}".format(limit)
        elif limit is None:
            uri += "?per_page=100"  # maximum number per page for GitHub API
        taken = 0

        queue = Queue()
        sem = BoundedSemaphore(FETCH_CONCURRENCY)
        done, working = set(), set()

        while uri is not None:
            app_log.info(
                "({}) Retrieving next page, received {} commits thus far".format(account._id, taken))
            req = account.get_request(uri, headers=default_headers)
            response = yield client.fetch(req)

            page_data = json.loads(response.body.decode('utf-8'))
            taken += page_data.__len__()
            for item in page_data:
                queue.put(item.get('url', None))

            if limit is None or taken < limit:
                # parse the Link header from GitHub (https://developer.github.com/v3/#pagination)
                links = parse_link_header(response.headers.get('Link', ''))
                uri = links.get('next', None)
            else:
                break

            if queue.qsize() > 500:
                raise HTTPError(413, 'too many commits')
        app_log.info("({}) Commit list retrieved, fetching info for {} commits".format(account._id, taken))

        # open our list
        cls.write('[')

        # our worker to actually fetch the info
        @gen.coroutine
        def fetch_url():
            current_url = yield queue.get()
            try:
                if current_url in working:
                    return
                page_no = working.__len__()
                app_log.info("Fetching page {}".format(page_no))
                working.add(current_url)
                req = account.get_request(current_url)
                client = AsyncHTTPClient()
                response = yield client.fetch(req)
                response_data = json.loads(response.body.decode('utf-8'))
                obj = {
                    'date': response_data['commit']['author']['date'],
                    'author': response_data['commit']['author']['name'],
                    'added_files': [file for file in response_data['files'] if file['status'] == 'added'].__len__(),
                    'deleted_files': [file for file in response_data['files'] if file['status'] == 'deleted'].__len__(),
                    'modified_files': [file for file in response_data['files'] if file['status'] == 'modified'].__len__(),
                    'additions': response_data['stats']['additions'],
                    'deletions': response_data['stats']['deletions']
                }
                if done.__len__() > 0:
                    cls.write(',')
                cls.write(json.dumps(obj))
                done.add(current_url)
                app_log.info("Page {} downloaded".format(page_no))

            finally:
                queue.task_done()
                sem.release()

        @gen.coroutine
        def worker():
            while True:
                yield sem.acquire()
                fetch_url()

        # start our concurrency worker
        worker()
        try:
            # wait until we're done
            yield queue.join(timeout=timedelta(seconds=MAXIMUM_REQ_TIME))
        except gen.TimeoutError:
            app_log.warning("Request exceeds maximum time, cutting response short")
        finally:
            # close our list
            cls.write(']')
        app_log.info("Finished retrieving commits for {}".format(account._id))
<|endoftext|>