<commit_before>#!/usr/bin/python3

# Web Crawler: Follows and finds all links on a webpage using multiple
# threads for faster thread-based parallel processing. Also creates a
# graph to determine the connection of all the links visited

# Library Imports
# ------------------------------------------------------------------------

import re
import string
import os
import sys
import getopt
import httplib2
import requests
import urllib.request
from collections import defaultdict
from threading import Thread
from bs4 import BeautifulSoup

# Global Variables
# -------------------------------------------------------------------------

PROGRAM_NAME = os.path.basename(sys.argv[0])

STARTURLS = [
            'https://en.wikipedia.org/wiki/Web_crawler',
            'https://www.nd.edu/',
            'http://www.espn.com'
            'https://www3.nd.edu/~pbui/teaching/cse.30331.fa16/reading03.html'
            'http://stackoverflow.com/questions/927358/how-to-undo-last-commits-in-git'
            ]

DEPTH = 2
GETTEXT = False
GETFREQS = False

# Classes and Fuctions
# ------------------------------------------------------------------------

# Graph class to represent links to and fro different urls

class Graph(object):

    # Inits a graph object. If no dict is given, an empty dict will be used
    def __init__(self, graph=None):
        if graph == None:
            graph = {}
            freqs = {}
        self.__graph = graph
        self.__freqs = freqs

    def add_vertex(self, vertex):
        if vertex not in self.__graph:
            self.__graph[vertex] = []

    def add_edge(self, src, dst):
        if src in self.__graph and dst not in self.__graph[src]:
            self.__graph[src].append(dst)
            if dst in self.__freqs:
                self.__freqs[dst] += 1
            else:
                self.__freqs[dst] = 1
        elif src not in self.__graph:
            self.__graph[src] = [dst]
            if dst in self.__freqs:
                self.__freqs[dst] += 1
            else:
                self.__freqs[dst] = 1

    def __get_edges(self):
        edges = []
        for vertex in self.__graph:
            for neighbour in self.__graph[vertex]:
                if {neighbour, vertex} not in edges:
                    edges.append({vertex, neighbour})
        return edges

    def __str__(self):
        ret = ""
        for k in self.__freqs:
            ret += str(k) + "\t" + str(self.__freqs[k]) + "\n"
        ret = ret[:len(ret)-1]
        return ret

# Thread class to allow return values

class ThreadWithReturn(Thread):
    def __init__(self, target=None, args=()):
        Thread.__init__(self)
        self.target = target
        self.args = args
        self._return = None

    def run(self):
        if self.target is not None:
            self._return = self.target(*self.args)

    def join(self):
        Thread.join(self)
        return self._return

# Error function to display error messages

def error(message, exit_code = 1):
    print(message, file=sys.stderr)
    sys.exit(exit_code)

# Usage function for help messages

def usage(exit_code = 0):
    error('''Usage: {} [ [-t -f] -n DEPTH -u STARTURLSFILE]

    Options:

        -n DEPTH            Set the depth to follow links
        -u STARTURLSFILE    Provide a file containing one URL per line for the starting URLs
        -t                  Write the text of each URL to a .urlText.txt
        -f                  Write the number of URLs linking to a URL to .urlFreqs.txt
        -h                  Show this help message'''
        .format(PROGRAM_NAME, exit_code))

# Crawler function which gathers all links from a certain url

def crawler(url, graph):
    urls = []
    http = httplib2.Http(timeout=50)
    try:
        status, response = http.request(url)
    except:
        return crawled

    # Use BeautifulSoup to find all links on a page
    soup = BeautifulSoup(response.decode('utf-8', 'ignore'), 'lxml')
    for link in soup.find_all('a'):
        if link.has_attr('href') and link['href'].startswith("http"):
            graph.add_edge(url, link['href'])
            urls.append(link['href'])
    return urls

# Filters through text from soup and strips text of whitespace

def filterText(text):
    if text.parent.name in ['style', 'script', '[document]', 'head', 'title']:
        return False
    return True

# Prints out url with all text from url on one line

def textParser(url, f):
    line = url + " "
    http = httplib2.Http(timeout=10)
    try:
        status, response = http.request(url)
    except:
        return

    soup = BeautifulSoup(response.decode('utf-8', 'ignore'), 'lxml')
    # Format html and only print text from webpage:
    for word in soup.find_all(text=True):
        if filterText(word):
            pattern = re.compile('[\W_]+') #, re.UNICODE)
            pattern.sub('', word)
            filter(str.isalpha, word)
            word = word.lower()
            word = word.strip()
            word = " ".join(word.split())
            if not len(word) == 0:
                line += word + " "
    print(line, file=f)

# Parse command
# -----------------------------------------------------------------------

try:
    options, arguments = getopt.getopt(sys.argv[1:], "htfn:u:")
except getopt.GetoptError as e:
    error(e)

for option, value in options:
    if option == '-h':
        usage(0)
    elif option == '-t':
        GETTEXT = True
    elif option == '-f':
        GETFREQS = True
    elif option == '-n': # Option for setting the depth to follow links
        DEPTH = int(value)
    elif option == '-u': # Allows user to provide start URLs file
        STARTURLS = []
        try:
            with open(value, 'r') as startFile:
                for line in startFile:
                    STARTURLS.append(line)
        except IOError as e:
            error(e)
    else:
        usage(1)

if not len(arguments) == 0:
    usage(1)

if os.path.isfile("mapfile.txt"):
    os.remove("mapfile.txt")
# Main exection
# -----------------------------------------------------------------------

if __name__ == "__main__":
    graph = Graph()

    urls = set()
    crawled = set()

    for url in STARTURLS:
        urls.add(url)

    # Loop a certain depth
    for i in range(DEPTH):
        threads = []
        for url in urls:
            if url not in crawled:
                thread = ThreadWithReturn(target=crawler, args=(url,graph,))
                threads.append(thread)
                crawled.add(url)
        # Start and join the threads
        for thread in threads:
            thread.start()
        for thread in threads:
            found = thread.join()
            for url in found:
                urls.add(url)

    # Get the number of links to each URL
    if GETFREQS:
        urlsFile = open('.urlCounts.txt', 'w')
        print(graph, file=urlsFile)
        urlsFile.close()

    # Get the text for each page
    if GETTEXT:
        textFile = open('.urlText.txt', 'w')
        del threads[:]
        for url in urls:
            thread = Thread(target=textParser, args=(url,textFile,))
            threads.append(thread)
        for thread in threads:
            if not thread.is_alive():
                thread.start()
        for thread in threads:
            thread.join()

        textFile.close()
<commit_msg>Added option to turn off parallelism<commit_after>#!/usr/bin/python3

# Web Crawler: Follows and finds all links on a webpage using multiple
# threads for faster thread-based parallel processing. Also creates a
# graph to determine the connection of all the links visited

# Library Imports
# ------------------------------------------------------------------------

import re
import string
import os
import sys
import getopt
import httplib2
import requests
import urllib.request
from collections import defaultdict
from threading import Thread
from bs4 import BeautifulSoup

# Global Variables
# -------------------------------------------------------------------------

PROGRAM_NAME = os.path.basename(sys.argv[0])

STARTURLS = [
            'https://en.wikipedia.org/wiki/Web_crawler',
            'https://www.nd.edu/',
            'http://www.espn.com'
            'https://www3.nd.edu/~pbui/teaching/cse.30331.fa16/reading03.html'
            'http://stackoverflow.com/questions/927358/how-to-undo-last-commits-in-git'
            ]

DEPTH = 2
GETTEXT = False
GETFREQS = False
USETHREADS = True

# Classes and Fuctions
# ------------------------------------------------------------------------

# Graph class to represent links to and fro different urls

class Graph(object):

    # Inits a graph object. If no dict is given, an empty dict will be used
    def __init__(self, graph=None):
        if graph == None:
            graph = {}
            freqs = {}
        self.__graph = graph
        self.__freqs = freqs

    def add_vertex(self, vertex):
        if vertex not in self.__graph:
            self.__graph[vertex] = []

    def add_edge(self, src, dst):
        if src in self.__graph and dst not in self.__graph[src]:
            self.__graph[src].append(dst)
            if dst in self.__freqs:
                self.__freqs[dst] += 1
            else:
                self.__freqs[dst] = 1
        elif src not in self.__graph:
            self.__graph[src] = [dst]
            if dst in self.__freqs:
                self.__freqs[dst] += 1
            else:
                self.__freqs[dst] = 1

    def __get_edges(self):
        edges = []
        for vertex in self.__graph:
            for neighbour in self.__graph[vertex]:
                if {neighbour, vertex} not in edges:
                    edges.append({vertex, neighbour})
        return edges

    def __str__(self):
        ret = ""
        for k in self.__freqs:
            ret += str(k) + "\t" + str(self.__freqs[k]) + "\n"
        ret = ret[:len(ret)-1]
        return ret

# Thread class to allow return values

class ThreadWithReturn(Thread):
    def __init__(self, target=None, args=()):
        Thread.__init__(self)
        self.target = target
        self.args = args
        self._return = None

    def run(self):
        if self.target is not None:
            self._return = self.target(*self.args)

    def join(self):
        Thread.join(self)
        return self._return

# Error function to display error messages

def error(message, exit_code = 1):
    print(message, file=sys.stderr)
    sys.exit(exit_code)

# Usage function for help messages

def usage(exit_code = 0):
    error('''Usage: {} [ [-t -f] -n DEPTH -u STARTURLSFILE]

    Options:

        -n DEPTH            Set the depth to follow links
        -u STARTURLSFILE    Provide a file containing one URL per line for the starting URLs
        -t                  Write the text of each URL to a .urlText.txt
        -f                  Write the number of URLs linking to a URL to .urlFreqs.txt
        -p                  Turn off thread-based parallelism
        -h                  Show this help message'''
        .format(PROGRAM_NAME, exit_code))

# Crawler function which gathers all links from a certain url

def crawler(url, graph):
    urls = []
    http = httplib2.Http(timeout=50)
    try:
        status, response = http.request(url)
    except:
        return crawled

    # Use BeautifulSoup to find all links on a page
    soup = BeautifulSoup(response.decode('utf-8', 'ignore'), 'lxml')
    for link in soup.find_all('a'):
        if link.has_attr('href') and link['href'].startswith("http"):
            graph.add_edge(url, link['href'])
            urls.append(link['href'])
    return urls

# Filters through text from soup and strips text of whitespace

def filterText(text):
    if text.parent.name in ['style', 'script', '[document]', 'head', 'title']:
        return False
    return True

# Prints out url with all text from url on one line

def textParser(url, f):
    line = url + " "
    http = httplib2.Http(timeout=10)
    try:
        status, response = http.request(url)
    except:
        return

    soup = BeautifulSoup(response.decode('utf-8', 'ignore'), 'lxml')
    # Format html and only print text from webpage:
    for word in soup.find_all(text=True):
        if filterText(word):
            pattern = re.compile('[\W_]+') #, re.UNICODE)
            pattern.sub('', word)
            filter(str.isalpha, word)
            word = word.lower()
            word = word.strip()
            word = " ".join(word.split())
            if not len(word) == 0:
                line += word + " "
    print(line, file=f)

# Parse command
# -----------------------------------------------------------------------

try:
    options, arguments = getopt.getopt(sys.argv[1:], "htfpn:u:")
except getopt.GetoptError as e:
    error(e)

for option, value in options:
    if option == '-h':
        usage(0)
    elif option == '-t':
        GETTEXT = True
    elif option == '-f':
        GETFREQS = True
    elif option == '-n': # Option for setting the depth to follow links
        DEPTH = int(value)
    elif option == '-u': # Allows user to provide start URLs file
        STARTURLS = []
        try:
            with open(value, 'r') as startFile:
                for line in startFile:
                    STARTURLS.append(line)
        except IOError as e:
            error(e)
    elif option == '-p':
        USETHREADS = False
    else:
        usage(1)

if not len(arguments) == 0:
    usage(1)

if os.path.isfile("mapfile.txt"):
    os.remove("mapfile.txt")

# Main exection
# -----------------------------------------------------------------------

if __name__ == "__main__":
    graph = Graph()

    urls = set()
    crawled = set()

    for url in STARTURLS:
        urls.add(url)

    # Loop a certain depth
    for i in range(DEPTH):
        if USETHREADS:
            threads = []
            for url in urls:
                if url not in crawled:
                    thread = ThreadWithReturn(target=crawler, args=(url,graph,))
                    threads.append(thread)
                    crawled.add(url)
            # Start and join the threads
            for thread in threads:
                thread.start()
            for thread in threads:
                found = thread.join()
                for url in found:
                    urls.add(url)
        else:
            found = []
            for url in urls:
                if url not in crawled:
                    ret = crawler(url,graph)
                    for new in ret:
                        found.append(new)
                    crawled.add(url)
            for url in found:
                urls.add(url)

    # Get the number of links to each URL
    if GETFREQS:
        urlsFile = open('.urlCounts.txt', 'w')
        print(graph, file=urlsFile)
        urlsFile.close()

    # Get the text for each page
    if GETTEXT:
        textFile = open('.urlText.txt', 'w')
        if USETHREADS:
            del threads[:]
            for url in urls:
                thread = Thread(target=textParser, args=(url,textFile,))
                threads.append(thread)
            for thread in threads:
                if not thread.is_alive():
                    thread.start()
            for thread in threads:
                thread.join()
        else:
            for url in urls:
                textParser(url,textFile)

        textFile.close()
<|endoftext|>


<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-

# Copyright (c) 2015-2016:
#   Matthieu Estrada, ttamalfor@gmail.com
#
# This file is part of (AlignakApp).
#
# (AlignakApp) is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# (AlignakApp) is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with (AlignakApp).  If not, see <http://www.gnu.org/licenses/>.

"""
    App Synthesis manage widget for Synthesis QWidget.
"""

import datetime

from logging import getLogger

from alignak_app.synthesis.service import Service
from alignak_app.core.action_manager import ACK, DOWNTIME, PROCESS

try:
    __import__('PyQt5')
    from PyQt5.QtWidgets import QScrollArea  # pylint: disable=no-name-in-module
    from PyQt5.QtWidgets import QWidget, QVBoxLayout  # pylint: disable=no-name-in-module
    from PyQt5.QtWidgets import QGridLayout, QLabel  # pylint: disable=no-name-in-module
    from PyQt5.Qt import Qt  # pylint: disable=no-name-in-module
except ImportError:  # pragma: no cover
    from PyQt4.Qt import QScrollArea  # pylint: disable=import-error
    from PyQt4.Qt import QWidget, QVBoxLayout  # pylint: disable=import-error
    from PyQt4.Qt import QGridLayout, QLabel  # pylint: disable=import-error
    from PyQt4.Qt import Qt, QApplication  # pylint: disable=import-error

logger = getLogger(__name__)

class ServicesView(QWidget):
    """
        Class who create the Synthesis QWidget.
    """

    def __init__(self, action_manager, app_backend, parent=None):
        super(ServicesView, self).__init__(parent)
        self.setToolTip('Services View')
        self.layout = QVBoxLayout()
        self.setLayout(self.layout)
        self.current_host = None
        self.action_manager = action_manager
        self.app_backend = app_backend
        self.services_widget = []

    def display_services(self, services, host):  # pylint: disable=too-many-locals
        """
        Display services.

        :param services: services of a specific host from app_backend
        :type services: dict
        :param host: host to which the services belong
        :type host: dict
        """

        logger.info('Create Services View')
        self.current_host = host

        # Clean all items before
        for i in reversed(range(self.layout.count())):
            self.layout.itemAt(i).widget().setParent(None)

        service_title = QLabel('Services of ' + host['name'] + '')
        self.layout.addWidget(service_title)
        self.layout.setAlignment(service_title, Qt.AlignCenter)

        widget = QWidget()
        layout = QGridLayout()
        widget.setLayout(layout)

        # Reset pos and QWidget List
        self.services_widget = []
        pos = 0

        if not services:
            logger.warning('Services not Found ! ')

            output_service = QLabel('Services not available .... Search for a host before.')
            layout.addWidget(output_service, 0, 0)
        else:
            for service in services:
                # Create Service QWidget
                service_widget = Service()
                service_widget.initialize(service)
                service_widget.acknowledged.connect(self.add_acknowledge)
                service_widget.downtimed.connect(self.add_downtime)

                # Update buttons
                self.update_service_buttons(service_widget)
                layout.addWidget(service_widget, pos, 0)
                pos += 1

                # Add to QWidget list
                self.services_widget.append(service_widget)

        logger.debug('Number of services: ' + str(pos))

        scroll = QScrollArea()
        scroll.setWidgetResizable(True)
        scroll.setWidget(widget)

        self.layout.addWidget(scroll)
        self.show()

    def update_services(self):
        """
        Manage update of each Service QWidget in QWidget List

        """

        if self.services_widget:
            for service_widget in self.services_widget:
                updated_service = self.app_backend.get_service(
                    self.current_host['_id'],
                    service_widget.service['_id']
                )
                service_widget.update_service(updated_service)
                self.update_service_buttons(service_widget)

                QApplication.processEvents()

    def update_service_buttons(self, service):
        """
        Update buttons of the service who has emit action

        :param service: service QWidget
        :type service: Service
        """

        if 'OK' in service.service['ls_state']:
            service.acknowledge_btn.setEnabled(False)
        elif service.service['ls_acknowledged'] or \
                service.service['display_name'] in self.action_manager.acks_to_check['services']:
            service.acknowledge_btn.setEnabled(False)
        else:
            service.acknowledge_btn.setEnabled(True)

        if 'OK' in service.service['ls_state']:
            service.downtime_btn.setEnabled(False)
        if service.service['ls_downtimed'] or \
                service.service['display_name'] in \
                self.action_manager.downtimes_to_check['services']:
            service.downtime_btn.setEnabled(False)
        else:
            service.downtime_btn.setEnabled(True)

    def add_acknowledge(self):  # pragma: no cover, no testability
        """
        Handle action for "acknowledge_btn"

        """

        # Get service who emit SIGNAL
        service = self.sender().service

        if self.current_host:
            user = self.app_backend.get_user()

            comment = 'Service %s acknowledged by %s, from Alignak-app' % (
                service['name'],
                user['name']
            )

            data = {
                'action': 'add',
                'host': self.current_host['_id'],
                'service': service['_id'],
                'user': user['_id'],
                'comment': comment
            }

            post = self.app_backend.post(ACK, data)
            item_process = {
                'action': PROCESS,
                'name': service['name'],
                'post': post
            }
            self.action_manager.add_item(item_process)

            item_action = {
                'action': ACK,
                'host_id': self.current_host['_id'],
                'service_id': service['_id']
            }
            self.action_manager.add_item(item_action)

            self.sender().acknowledge_btn.setEnabled(False)

    def add_downtime(self):  # pragma: no cover, no testability
        """
        Handle action for "downtime_btn"

        """

        service = self.sender().service

        if self.current_host:
            user = self.app_backend.get_user()

            comment = 'Schedule downtime by %s, from Alignak-app' % user['name']

            start_time = datetime.datetime.now()
            end_time = start_time + datetime.timedelta(days=1)

            data = {
                'action': 'add',
                'host': self.current_host['_id'],
                'service': service['_id'],
                'user': user['_id'],
                'comment': comment,
                'start_time': start_time.timestamp(),
                'end_time': end_time.timestamp(),
                'fixed': True
            }

            post = self.app_backend.post(DOWNTIME, data)
            item_process = {
                'action': PROCESS,
                'name': service['name'],
                'post': post
            }
            self.action_manager.add_item(item_process)

            item_action = {
                'action': DOWNTIME,
                'host_id': self.current_host['_id'],
                'service_id': service['_id']
            }
            self.action_manager.add_item(item_action)

            self.sender().downtime_btn.setEnabled(False)
<commit_msg>Add missing import<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-

# Copyright (c) 2015-2016:
#   Matthieu Estrada, ttamalfor@gmail.com
#
# This file is part of (AlignakApp).
#
# (AlignakApp) is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# (AlignakApp) is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with (AlignakApp).  If not, see <http://www.gnu.org/licenses/>.

"""
    App Synthesis manage widget for Synthesis QWidget.
"""

import datetime

from logging import getLogger

from alignak_app.synthesis.service import Service
from alignak_app.core.action_manager import ACK, DOWNTIME, PROCESS

try:
    __import__('PyQt5')
    from PyQt5.QtWidgets import QScrollArea  # pylint: disable=no-name-in-module
    from PyQt5.QtWidgets import QWidget, QVBoxLayout  # pylint: disable=no-name-in-module
    from PyQt5.QtWidgets import QGridLayout, QLabel  # pylint: disable=no-name-in-module
    from PyQt5.Qt import Qt, QApplication  # pylint: disable=no-name-in-module
except ImportError:  # pragma: no cover
    from PyQt4.Qt import QScrollArea  # pylint: disable=import-error
    from PyQt4.Qt import QWidget, QVBoxLayout  # pylint: disable=import-error
    from PyQt4.Qt import QGridLayout, QLabel  # pylint: disable=import-error
    from PyQt4.Qt import Qt, QApplication  # pylint: disable=import-error

logger = getLogger(__name__)

class ServicesView(QWidget):
    """
        Class who create the Synthesis QWidget.
    """

    def __init__(self, action_manager, app_backend, parent=None):
        super(ServicesView, self).__init__(parent)
        self.setToolTip('Services View')
        self.layout = QVBoxLayout()
        self.setLayout(self.layout)
        self.current_host = None
        self.action_manager = action_manager
        self.app_backend = app_backend
        self.services_widget = []

    def display_services(self, services, host):  # pylint: disable=too-many-locals
        """
        Display services.

        :param services: services of a specific host from app_backend
        :type services: dict
        :param host: host to which the services belong
        :type host: dict
        """

        logger.info('Create Services View')
        self.current_host = host

        # Clean all items before
        for i in reversed(range(self.layout.count())):
            self.layout.itemAt(i).widget().setParent(None)

        service_title = QLabel('Services of ' + host['name'] + '')
        self.layout.addWidget(service_title)
        self.layout.setAlignment(service_title, Qt.AlignCenter)

        widget = QWidget()
        layout = QGridLayout()
        widget.setLayout(layout)

        # Reset pos and QWidget List
        self.services_widget = []
        pos = 0

        if not services:
            logger.warning('Services not Found ! ')

            output_service = QLabel('Services not available .... Search for a host before.')
            layout.addWidget(output_service, 0, 0)
        else:
            for service in services:
                # Create Service QWidget
                service_widget = Service()
                service_widget.initialize(service)
                service_widget.acknowledged.connect(self.add_acknowledge)
                service_widget.downtimed.connect(self.add_downtime)

                # Update buttons
                self.update_service_buttons(service_widget)
                layout.addWidget(service_widget, pos, 0)
                pos += 1

                # Add to QWidget list
                self.services_widget.append(service_widget)

        logger.debug('Number of services: ' + str(pos))

        scroll = QScrollArea()
        scroll.setWidgetResizable(True)
        scroll.setWidget(widget)

        self.layout.addWidget(scroll)
        self.show()

    def update_services(self):
        """
        Manage update of each Service QWidget in QWidget List

        """

        if self.services_widget:
            for service_widget in self.services_widget:
                updated_service = self.app_backend.get_service(
                    self.current_host['_id'],
                    service_widget.service['_id']
                )
                service_widget.update_service(updated_service)
                self.update_service_buttons(service_widget)

                QApplication.processEvents()

    def update_service_buttons(self, service):
        """
        Update buttons of the service who has emit action

        :param service: service QWidget
        :type service: Service
        """

        if 'OK' in service.service['ls_state']:
            service.acknowledge_btn.setEnabled(False)
        elif service.service['ls_acknowledged'] or \
                service.service['display_name'] in self.action_manager.acks_to_check['services']:
            service.acknowledge_btn.setEnabled(False)
        else:
            service.acknowledge_btn.setEnabled(True)

        if 'OK' in service.service['ls_state']:
            service.downtime_btn.setEnabled(False)
        if service.service['ls_downtimed'] or \
                service.service['display_name'] in \
                self.action_manager.downtimes_to_check['services']:
            service.downtime_btn.setEnabled(False)
        else:
            service.downtime_btn.setEnabled(True)

    def add_acknowledge(self):  # pragma: no cover, no testability
        """
        Handle action for "acknowledge_btn"

        """

        # Get service who emit SIGNAL
        service = self.sender().service

        if self.current_host:
            user = self.app_backend.get_user()

            comment = 'Service %s acknowledged by %s, from Alignak-app' % (
                service['name'],
                user['name']
            )

            data = {
                'action': 'add',
                'host': self.current_host['_id'],
                'service': service['_id'],
                'user': user['_id'],
                'comment': comment
            }

            post = self.app_backend.post(ACK, data)
            item_process = {
                'action': PROCESS,
                'name': service['name'],
                'post': post
            }
            self.action_manager.add_item(item_process)

            item_action = {
                'action': ACK,
                'host_id': self.current_host['_id'],
                'service_id': service['_id']
            }
            self.action_manager.add_item(item_action)

            self.sender().acknowledge_btn.setEnabled(False)

    def add_downtime(self):  # pragma: no cover, no testability
        """
        Handle action for "downtime_btn"

        """

        service = self.sender().service

        if self.current_host:
            user = self.app_backend.get_user()

            comment = 'Schedule downtime by %s, from Alignak-app' % user['name']

            start_time = datetime.datetime.now()
            end_time = start_time + datetime.timedelta(days=1)

            data = {
                'action': 'add',
                'host': self.current_host['_id'],
                'service': service['_id'],
                'user': user['_id'],
                'comment': comment,
                'start_time': start_time.timestamp(),
                'end_time': end_time.timestamp(),
                'fixed': True
            }

            post = self.app_backend.post(DOWNTIME, data)
            item_process = {
                'action': PROCESS,
                'name': service['name'],
                'post': post
            }
            self.action_manager.add_item(item_process)

            item_action = {
                'action': DOWNTIME,
                'host_id': self.current_host['_id'],
                'service_id': service['_id']
            }
            self.action_manager.add_item(item_action)

            self.sender().downtime_btn.setEnabled(False)
<|endoftext|>


<commit_before>#!/usr/bin/python
# -*- coding: utf-8 -*-
# copyright Julien TREBOSC 2012-2013
# check that variable PYTHONPATH points to the right folder for bruker.py
#      library
from __future__ import division

import numpy
import sys
import bruker
import math

# gestion des arguments
import argparse
parser = argparse.ArgumentParser(
  description='Add echoes in a qcpmg bruker experiment')
parser.add_argument('-l', '--lb', type=float,
          help='Lorentzian broadening applied to the decaying echo', default=0)
parser.add_argument('-g', '--gb', type=float,
                    help='Gaussian broadening applied to each echo', default=0)
parser.add_argument('-n', type=int, help='Number of echo to sum')
parser.add_argument('-c', type=float, help='qcpmg cycle in us')
parser.add_argument('-s', type=float,
   help='echo position from start of FID (digital filter excluded) in us')
parser.add_argument('--norm_noise', action='store_true',
   help='normalise noise level as function of number of echoes', default=False)
parser.add_argument('infile', help='Full path of the dataset to process')
group = parser.add_mutually_exclusive_group()
group.add_argument('-e', action='store_true', help='Sum only even echoes')
group.add_argument('-o', action='store_true', help='Sum only odd echoes')

args = parser.parse_args()
#print(args.__dict__)
dat = bruker.dataset(bruker.splitprocpath(args.infile))

# lire la fid et eliminer le filter digital (par defaut)
serfile = dat.readfid()

# calcule la duree d'un echo en points (a modifier selon le PP utilise)
dw = 1e6/float(dat.readacqpar("SW_h"))
P60 = float(dat.readacqpar("P 60")) # should store the cycle time
# in case cycle tiume is not stored in P60, this program uses a default
# calculation based on D3, D6, P4, P3
D3 = float(dat.readacqpar("D 3"))*1e6
D6 = float(dat.readacqpar("D 6"))*1e6
P4 = float(dat.readacqpar("P 4"))
P3 = float(dat.readacqpar("P 3"))
# cycle is set to a
if args.c : # one can provide the cycle in argument
    cycle = float(args.c)
elif P60 > 0: # pulse program should store status cycle in P60
    cycle = P60
else : # default behavior may depend on pulse program implementation
    cycle = 2*(D3+D6+2)+P4

# get the number of echoes
n_echoes = int(dat.readacqpar("L 22")) + 1
if args.n and (0 < args.n)  and (args.n <= n_echoes):
    n_echoes = args.n  # in argument is the real number of echoes

ppc = cycle/dw
npoints = int(round(cycle/dw))

#print(ppc, npoints, ppc-npoints)
if abs(ppc-npoints) > 0.001:
    print("Warning echo cycle is not multiple of dwell")
    roundChunk = True
else:
    roundChunk = False

# verifie si TD est coherent avec n_echoes
digFilLen = int(round(dat.getdigfilt()))
TD = int(dat.readacqpar("TD"))
# some times TD is too short to contain L22+1 echoes
# (mostly because of digital filter using some TD points)
if TD < npoints*2*n_echoes+2*digFilLen:
    n_echoes -= int(2*n_echoes - (TD-2*digFilLen)/npoints) + 1
    print("only %d echoes used" % (n_echoes,))

# reshape FID into 3D array (echo index, echo point index, Re/Im)
if not roundChunk:
    summed = serfile[0:npoints*2*n_echoes].reshape(n_echoes, npoints, 2)
else:
    tmp = serfile.reshape(len(serfile)//2, 2)
    summed = numpy.zeros((n_echoes, npoints, 2))
    for i in range(n_echoes):
        summed[i, :, :] += tmp[int(i*ppc+0.5):int(ppc*i+0.5) + npoints, :]

# create a gaussian apodization function
# time exp(-(a(t-center))**2) -> spectral exp(-(w/2a)**2)
#           half width at half maximum GB = a/pi * 2*sqrt(ln(2))
# hence a = GB
# center of gaussian depends on pulse sequence : it can be D6 or D6+D3

#create a gaussian array
if args.s:
    center = args.s
else:
    center =  (D6+D3)
center_point = center / cycle * npoints
GB = args.gb*math.pi/2.0/math.sqrt(math.log(2.0))
i = numpy.arange(npoints)
apod = numpy.exp(-(1e-6*dw*(i-center_point)*GB)**2)
apod = numpy.resize(apod, (2, npoints))
apod = numpy.swapaxes(apod, 0, 1)

# do the multiplication with gaussian and lorentzian broadening
LB = args.lb/2
SUME = numpy.zeros((npoints, 2))
SUMA = numpy.zeros((npoints, 2))

L_noise_weight = numpy.zeros(n_echoes)
for i in range(0, n_echoes):
    L_weight = numpy.exp(-LB*i*npoints*dw*1e-6)
    L_noise_weight[i] = L_weight
    if i % 2:
        SUME += summed[i, :, :] * apod * L_weight
    else:
        SUMA += summed[i, :, :] * apod * L_weight
if args.e: # keep only even echoes
    SUM = SUME
elif args.o: # keep only odd echoes
    SUM = SUMA
else:
    SUM = SUMA+SUME

if args.norm_noise:
#    print("normed for noise")
    SUM /= numpy.sqrt(L_noise_weight.sum())
else:
#    print("noise not normed")
    pass

# separate Re and Im
s1 = SUM[:, 0]
s2 = SUM[:, 1]

# Apply some zero filling (from SI) and put back digital filter
SI = int(dat.readprocpar("SI", False))
r1 = numpy.concatenate((numpy.zeros(digFilLen), s1,
            numpy.zeros(SI-len(s1)-digFilLen)))
r2 = numpy.concatenate((numpy.zeros(digFilLen), s2,
            numpy.zeros(SI-len(s2)-digFilLen)))
# ecrit les fichiers 1r 1i
dat.writespect1dri(r1, r2)

# set all optionnal processing parameters to 0
ProcOptions = {"WDW": ["LB", "GB", "SSB", "TM1", "TM2"],
         "PH_mod": ["PHC0", "PHC1"], "BC_mod": ["BCFW", "COROFFS"],
         "ME_mod": ["NCOEF", "LPBIN", "TDoff"], "FT_mod": ["FTSIZE"]}
for par in ProcOptions.keys():
    dat.writeprocpar(par, "0", True, dimension=1)
    for opt in ProcOptions[par]:
        dat.writeprocpar(opt, "0", True, dimension=1)

# write some status processed parameters in procs file so topspin can display
# and process the data properly
dat.writeprocpar("WDW", "1", True)
dat.writeprocpar("LB", str(args.gb), True)

dat.writeprocpar("AXUNIT", "s", True)
dat.writeprocpar("AXRIGHT", str(SI*dw*1e-6), True)

<commit_msg>BUG:qcpmgadd_:Fix n_echoes calculation when FID too small for L22+1 echoes<commit_after>#!/usr/bin/python
# -*- coding: utf-8 -*-
# copyright Julien TREBOSC 2012-2013
# check that variable PYTHONPATH points to the right folder for bruker.py
#      library
from __future__ import division

import numpy
import sys
import bruker
import math

# gestion des arguments
import argparse
parser = argparse.ArgumentParser(
  description='Add echoes in a qcpmg bruker experiment')
parser.add_argument('-l', '--lb', type=float,
          help='Lorentzian broadening applied to the decaying echo', default=0)
parser.add_argument('-g', '--gb', type=float,
                    help='Gaussian broadening applied to each echo', default=0)
parser.add_argument('-n', type=int, help='Number of echo to sum')
parser.add_argument('-c', type=float, help='qcpmg cycle in us')
parser.add_argument('-s', type=float,
   help='echo position from start of FID (digital filter excluded) in us')
parser.add_argument('--norm_noise', action='store_true',
   help='normalise noise level as function of number of echoes', default=False)
parser.add_argument('infile', help='Full path of the dataset to process')
group = parser.add_mutually_exclusive_group()
group.add_argument('-e', action='store_true', help='Sum only even echoes')
group.add_argument('-o', action='store_true', help='Sum only odd echoes')

args = parser.parse_args()
#print(args.__dict__)
dat = bruker.dataset(bruker.splitprocpath(args.infile))

# lire la fid et eliminer le filter digital (par defaut)
serfile = dat.readfid()

# calcule la duree d'un echo en points (a modifier selon le PP utilise)
dw = 1e6/float(dat.readacqpar("SW_h"))
P60 = float(dat.readacqpar("P 60")) # should store the cycle time
# in case cycle tiume is not stored in P60, this program uses a default
# calculation based on D3, D6, P4, P3
D3 = float(dat.readacqpar("D 3"))*1e6
D6 = float(dat.readacqpar("D 6"))*1e6
P4 = float(dat.readacqpar("P 4"))
P3 = float(dat.readacqpar("P 3"))
# cycle is set to a
if args.c : # one can provide the cycle in argument
    cycle = float(args.c)
elif P60 > 0: # pulse program should store status cycle in P60
    cycle = P60
else : # default behavior may depend on pulse program implementation
    cycle = 2*(D3+D6+2)+P4

# get the number of echoes: L22+1
n_echoes = int(dat.readacqpar("L 22")) + 1
if args.n and (0 < args.n)  and (args.n <= n_echoes):
    n_echoes = args.n  # in argument is the real number of echoes

# dwell point per cycle : should be integer but may not if improper acquisition parameter used
ppc = cycle/dw
# approximate number of dwell point per cycle
npoints = int(round(cycle/dw))

#print(ppc, npoints, ppc-npoints)
if abs(ppc-npoints) > 0.001:
    print("Warning echo cycle is not multiple of dwell")
    rounding_chunks = True
else:
    rounding_chunks = False

# verifie si TD est coherent avec n_echoes
digFilLen = int(round(dat.getdigfilt()))
TD = int(dat.readacqpar("TD"))
# sometimes TD is too short to contain L22+1 echoes
# (mostly because of digital filter using some TD points)
if TD < ppc*2*n_echoes+2*digFilLen:
    print("Warning FID cannot hold all echoes : only %d echoes used" % (n_echoes,))
    n_echoes = int((TD-2*digFilLen)/ppc/2)

# reshape FID into 3D array (echo index, echo point index, Re/Im)
if not rounding_chunks:
    summed = serfile[0:npoints*2*n_echoes].reshape(n_echoes, npoints, 2)
else:
    tmp = serfile.reshape(len(serfile)//2, 2)
    summed = numpy.zeros((n_echoes, npoints, 2))
    for i in range(n_echoes):
        summed[i, :, :] += tmp[int(i*ppc+0.5):int(ppc*i+0.5) + npoints, :]

# create a gaussian apodization function
# time exp(-(a(t-center))**2) -> spectral exp(-(w/2a)**2)
#           half width at half maximum GB = a/pi * 2*sqrt(ln(2))
# hence a = GB
# center of gaussian depends on pulse sequence : it can be D6 or D6+D3

#create a gaussian array
if args.s:
    center = args.s
else:
    center =  (D6+D3)
center_point = center / cycle * npoints
GB = args.gb*math.pi/2.0/math.sqrt(math.log(2.0))
i = numpy.arange(npoints)
apod = numpy.exp(-(1e-6*dw*(i-center_point)*GB)**2)
apod = numpy.resize(apod, (2, npoints))
apod = numpy.swapaxes(apod, 0, 1)

# do the multiplication with gaussian and lorentzian broadening
LB = args.lb/2
SUME = numpy.zeros((npoints, 2))
SUMA = numpy.zeros((npoints, 2))

L_noise_weight = numpy.zeros(n_echoes)
for i in range(0, n_echoes):
    L_weight = numpy.exp(-LB*i*npoints*dw*1e-6)
    L_noise_weight[i] = L_weight
    if i % 2:
        SUME += summed[i, :, :] * apod * L_weight
    else:
        SUMA += summed[i, :, :] * apod * L_weight
if args.e: # keep only even echoes
    SUM = SUME
elif args.o: # keep only odd echoes
    SUM = SUMA
else:
    SUM = SUMA+SUME

if args.norm_noise:
#    print("normed for noise")
    SUM /= numpy.sqrt(L_noise_weight.sum())
else:
#    print("noise not normed")
    pass

# separate Re and Im
s1 = SUM[:, 0]
s2 = SUM[:, 1]

# Apply some zero filling (from SI) and put back digital filter
SI = int(dat.readprocpar("SI", False))
r1 = numpy.concatenate((numpy.zeros(digFilLen), s1,
            numpy.zeros(SI-len(s1)-digFilLen)))
r2 = numpy.concatenate((numpy.zeros(digFilLen), s2,
            numpy.zeros(SI-len(s2)-digFilLen)))
# ecrit les fichiers 1r 1i
dat.writespect1dri(r1, r2)

# set all optionnal processing parameters to 0
ProcOptions = {"WDW": ["LB", "GB", "SSB", "TM1", "TM2"],
         "PH_mod": ["PHC0", "PHC1"], "BC_mod": ["BCFW", "COROFFS"],
         "ME_mod": ["NCOEF", "LPBIN", "TDoff"], "FT_mod": ["FTSIZE"]}
for par in ProcOptions.keys():
    dat.writeprocpar(par, "0", True, dimension=1)
    for opt in ProcOptions[par]:
        dat.writeprocpar(opt, "0", True, dimension=1)

# write some status processed parameters in procs file so topspin can display
# and process the data properly
dat.writeprocpar("WDW", "1", True)
dat.writeprocpar("LB", str(args.gb), True)

dat.writeprocpar("AXUNIT", "s", True)
dat.writeprocpar("AXRIGHT", str(SI*dw*1e-6), True)

<|endoftext|>


<commit_before>from osgeo import gdal
import numpy as np
import subprocess
import rasterio
import datetime
import sys
sys.path.append('../')
import constants_and_names as cn
import universal_util as uu

# Calculates a range of tile statistics
def aggregate_results(tile, pixel_count_dict):

    print "Processing {}".format(tile)

    # start time
    start = datetime.datetime.now()

    # Extracts the tile id and the tile type from the full tile name
    tile_id = uu.get_tile_id(tile)
    tile_type = uu.get_tile_type(tile)
    xmin, ymin, xmax, ymax = uu.coords(tile_id)

    print "  Converting {} to per-pixel values".format(tile)

    # Names of pixel area tile
    area_tile = '{0}_{1}.tif'.format(cn.pattern_pixel_area, tile_id)

    # Per-pixel value tile (intermediate output)
    per_pixel = '{0}_{1}_per_pixel.tif'.format(tile_id, tile_type)

    # Opens input tiles for rasterio
    in_src = rasterio.open(tile)
    pixel_area_src = rasterio.open(area_tile)

    # Grabs metadata about the tif, like its location/projection/cellsize
    kwargs = in_src.meta

    # Grabs the windows of the tile (stripes) so we can iterate over the entire tif without running out of memory
    windows = in_src.block_windows(1)

    kwargs.update(
        driver='GTiff',
        count=1,
        compress='lzw',
        nodata=0,
        dtype='float32'
    )

    # Opens the output tile, giving it the arguments of the input tiles
    per_pixel_dst = rasterio.open(per_pixel, 'w', **kwargs)

    # The number of pixels in the tile with values
    non_zero_pixels = 0

    # Iterates across the windows (1 pixel strips) of the input tile
    for idx, window in windows:

        # Creates windows for each input tile
        in_window = in_src.read(1, window=window)
        pixel_area_window = pixel_area_src.read(1, window=window)

        # Calculates the per-pixel value from the input tile value (/ha to /pixel)
        per_pixel_value = in_window * pixel_area_window / cn.m2_per_ha

        per_pixel_dst.write_band(1, per_pixel_value, window=window)

        # Adds the number of pixels with values in that window to the total for that tile
        non_zero_pixels = non_zero_pixels + np.count_nonzero(in_window)

    print "Pixels with values in {}: {}".format(tile, non_zero_pixels)

    print "  Calculating average per-pixel value in", tile

    avg_10km = '{0}_{1}_average.tif'.format(tile_id, tile_type)

    # cmd = ['gdalwarp', '-tap', '-tr', '{}'.format(str(0.096342599)), '{}'.format(str(0.096342599)),  '-co', 'COMPRESS=LZW',
    #        # '-te', str(xmin), str(ymin), str(xmax), str(ymax),
    #        per_pixel, avg_10km]

    dt = 'float32'
    # cmd = ['gdalwarp', '-t_srs', 'EPSG:4326', '-co', 'COMPRESS=LZW', '-tr', str(cn.aggreg_res), str(cn.aggreg_res), '-tap', '-te',
    #         str(xmin), str(ymin), str(xmax), str(ymax), '-r', 'average',
    #        '-dstnodata', '0', '-overwrite', '{}'.format(per_pixel), '{}'.format(avg_10km)]
    cmd = ['gdalwarp', '-co', 'COMPRESS=LZW', '-tr', '0.01', '0.01', '-overwrite', '-r', 'average',
           '00N_110E_all_drivers_t_CO2_ha_gross_emis_year_per_pixel.tif', 'test3.tif']
    subprocess.check_call(cmd)

    # uu.warp_to_Hansen('{0}_{1}_per_pixel.tif'.format(tile_id, tile_type), '{0}_{1}_average.tif'.format(tile_id, tile_type),
    #                   xmin, ymin, xmax, ymax, 'Byte')

    # Prints information about the tile that was just processed
    uu.end_of_fx_summary(start, tile_id, tile_type)<commit_msg>Working on aggregating output tiles to 10 km.<commit_after>from osgeo import gdal
import numpy as np
import subprocess
import rasterio
import datetime
import sys
sys.path.append('../')
import constants_and_names as cn
import universal_util as uu

# Calculates a range of tile statistics
def aggregate_results(tile, pixel_count_dict):

    print "Processing {}".format(tile)

    # start time
    start = datetime.datetime.now()

    # Extracts the tile id and the tile type from the full tile name
    tile_id = uu.get_tile_id(tile)
    tile_type = uu.get_tile_type(tile)
    xmin, ymin, xmax, ymax = uu.coords(tile_id)

    print "  Converting {} to per-pixel values".format(tile)

    # Names of pixel area tile
    area_tile = '{0}_{1}.tif'.format(cn.pattern_pixel_area, tile_id)

    # Per-pixel value tile (intermediate output)
    per_pixel = '{0}_{1}_per_pixel.tif'.format(tile_id, tile_type)

    # Opens input tiles for rasterio
    in_src = rasterio.open(tile)
    pixel_area_src = rasterio.open(area_tile)

    # Grabs metadata about the tif, like its location/projection/cellsize
    kwargs = in_src.meta

    # Grabs the windows of the tile (stripes) so we can iterate over the entire tif without running out of memory
    windows = in_src.block_windows(1)

    kwargs.update(
        driver='GTiff',
        count=1,
        compress='lzw',
        nodata=0,
        dtype='float32'
    )

    # Opens the output tile, giving it the arguments of the input tiles
    per_pixel_dst = rasterio.open(per_pixel, 'w', **kwargs)

    # The number of pixels in the tile with values
    non_zero_pixels = 0

    # # Iterates across the windows (1 pixel strips) of the input tile
    # for idx, window in windows:
    #
    #     # Creates windows for each input tile
    #     in_window = in_src.read(1, window=window)
    #     pixel_area_window = pixel_area_src.read(1, window=window)
    #
    #     # Calculates the per-pixel value from the input tile value (/ha to /pixel)
    #     per_pixel_value = in_window * pixel_area_window / cn.m2_per_ha
    #
    #     per_pixel_dst.write_band(1, per_pixel_value, window=window)
    #
    #     # Adds the number of pixels with values in that window to the total for that tile
    #     non_zero_pixels = non_zero_pixels + np.count_nonzero(in_window)

    print "Pixels with values in {}: {}".format(tile, non_zero_pixels)

    print "  Calculating average per-pixel value in", tile

    avg_10km = '{0}_{1}_average.tif'.format(tile_id, tile_type)

    # cmd = ['gdalwarp', '-tap', '-tr', '{}'.format(str(0.096342599)), '{}'.format(str(0.096342599)),  '-co', 'COMPRESS=LZW',
    #        # '-te', str(xmin), str(ymin), str(xmax), str(ymax),
    #        per_pixel, avg_10km]

    dt = 'float32'
    # cmd = ['gdalwarp', '-t_srs', 'EPSG:4326', '-co', 'COMPRESS=LZW', '-tr', str(cn.aggreg_res), str(cn.aggreg_res), '-tap', '-te',
    #         str(xmin), str(ymin), str(xmax), str(ymax), '-r', 'average',
    #        '-dstnodata', '0', '-overwrite', '{}'.format(per_pixel), '{}'.format(avg_10km)]
    # cmd = ['gdalwarp', '-co', 'COMPRESS=LZW', '-tr', '0.01', '0.01', '-overwrite', '-r', 'average',
    #        '00N_110E_all_drivers_t_CO2_ha_gross_emis_year_per_pixel.tif', 'test3.tif']
    cmd = ['gdalwarp', '-co', 'COMPRESS=LZW', '-tr', '0.01', '0.01', '-overwrite', '-r', 'average',
           '00N_110E_all_drivers_t_CO2_ha_gross_emis_year_per_pixel.tif', 'test4.tif']

    subprocess.check_call(cmd)

    # uu.warp_to_Hansen('{0}_{1}_per_pixel.tif'.format(tile_id, tile_type), '{0}_{1}_average.tif'.format(tile_id, tile_type),
    #                   xmin, ymin, xmax, ymax, 'Byte')

    # Prints information about the tile that was just processed
    uu.end_of_fx_summary(start, tile_id, tile_type)<|endoftext|>


<commit_before>from osgeo import gdal
import numpy as np
import subprocess
import rasterio
import datetime
import sys
sys.path.append('../')
import constants_and_names as cn
import universal_util as uu

# Calculates a range of tile statistics
def aggregate_results(tile, pixel_count_dict):

    print "Processing {}".format(tile)

    # start time
    start = datetime.datetime.now()

    # Extracts the tile id and the tile type from the full tile name
    tile_id = uu.get_tile_id(tile)
    tile_type = uu.get_tile_type(tile)
    xmin, ymin, xmax, ymax = uu.coords(tile_id)

    print "  Converting {} to per-pixel values".format(tile)

    # Names of pixel area tile
    area_tile = '{0}_{1}.tif'.format(cn.pattern_pixel_area, tile_id)

    # Per-pixel value tile (intermediate output)
    per_pixel = '{0}_{1}_per_pixel.tif'.format(tile_id, tile_type)

    # Opens input tiles for rasterio
    in_src = rasterio.open(tile)
    pixel_area_src = rasterio.open(area_tile)

    # Grabs metadata about the tif, like its location/projection/cellsize
    kwargs = in_src.meta

    # Grabs the windows of the tile (stripes) so we can iterate over the entire tif without running out of memory
    windows = in_src.block_windows(1)

    kwargs.update(
        driver='GTiff',
        count=1,
        compress='lzw',
        nodata=0,
        dtype='float32'
    )

    # Opens the output tile, giving it the arguments of the input tiles
    per_pixel_dst = rasterio.open(per_pixel, 'w', **kwargs)

    # The number of pixels in the tile with values
    non_zero_pixels = 0

    # # Iterates across the windows (1 pixel strips) of the input tile
    # for idx, window in windows:
    #
    #     # Creates windows for each input tile
    #     in_window = in_src.read(1, window=window)
    #     pixel_area_window = pixel_area_src.read(1, window=window)
    #
    #     # Calculates the per-pixel value from the input tile value (/ha to /pixel)
    #     per_pixel = in_window * pixel_area_window / cn.m2_per_ha
    #
    #     per_pixel_dst.write_band(1, per_pixel, window=window)
    #
    #     # Adds the number of pixels with values in that window to the total for that tile
    #     non_zero_pixels = non_zero_pixels + np.count_nonzero(in_window)

    print "Pixels with values in {}: {}".format(tile, non_zero_pixels)

    print "  Calculating average per-pixel value in", tile

    avg_10km = '{0}_{1}_average.tif'.format(tile_id, tile_type)

    # cmd = ['gdalwarp', '-tap', '-tr', '{}'.format(str(0.096342599)), '{}'.format(str(0.096342599)),  '-co', 'COMPRESS=LZW',
    #        # '-te', str(xmin), str(ymin), str(xmax), str(ymax),
    #        per_pixel, avg_10km]

    dt = 'float32'
    cmd = ['gdalwarp', '-t_srs', 'EPSG:4326', '-co', 'COMPRESS=LZW', '-tr', str(cn.aggreg_res), str(cn.aggreg_res), '-tap', '-te',
            str(xmin), str(ymin), str(xmax), str(ymax), '-dstnodata', '0', '-overwrite', per_pixel, avg_10km]
    # subprocess.check_call(cmd)

    uu.warp_to_Hansen(per_pixel, avg_10km, xmin, ymin, xmax, ymax, dt)

    # Prints information about the tile that was just processed
    uu.end_of_fx_summary(start, tile_id, tile_type)<commit_msg>Working on aggregating output tiles to 10 km.<commit_after>from osgeo import gdal
import numpy as np
import subprocess
import rasterio
import datetime
import sys
sys.path.append('../')
import constants_and_names as cn
import universal_util as uu

# Calculates a range of tile statistics
def aggregate_results(tile, pixel_count_dict):

    print "Processing {}".format(tile)

    # start time
    start = datetime.datetime.now()

    # Extracts the tile id and the tile type from the full tile name
    tile_id = uu.get_tile_id(tile)
    tile_type = uu.get_tile_type(tile)
    xmin, ymin, xmax, ymax = uu.coords(tile_id)

    print "  Converting {} to per-pixel values".format(tile)

    # Names of pixel area tile
    area_tile = '{0}_{1}.tif'.format(cn.pattern_pixel_area, tile_id)

    # Per-pixel value tile (intermediate output)
    per_pixel = '{0}_{1}_per_pixel.tif'.format(tile_id, tile_type)

    # Opens input tiles for rasterio
    in_src = rasterio.open(tile)
    pixel_area_src = rasterio.open(area_tile)

    # Grabs metadata about the tif, like its location/projection/cellsize
    kwargs = in_src.meta

    # Grabs the windows of the tile (stripes) so we can iterate over the entire tif without running out of memory
    windows = in_src.block_windows(1)

    kwargs.update(
        driver='GTiff',
        count=1,
        compress='lzw',
        nodata=0,
        dtype='float32'
    )

    # Opens the output tile, giving it the arguments of the input tiles
    per_pixel_dst = rasterio.open(per_pixel, 'w', **kwargs)

    # The number of pixels in the tile with values
    non_zero_pixels = 0

    # # Iterates across the windows (1 pixel strips) of the input tile
    # for idx, window in windows:
    #
    #     # Creates windows for each input tile
    #     in_window = in_src.read(1, window=window)
    #     pixel_area_window = pixel_area_src.read(1, window=window)
    #
    #     # Calculates the per-pixel value from the input tile value (/ha to /pixel)
    #     per_pixel = in_window * pixel_area_window / cn.m2_per_ha
    #
    #     per_pixel_dst.write_band(1, per_pixel, window=window)
    #
    #     # Adds the number of pixels with values in that window to the total for that tile
    #     non_zero_pixels = non_zero_pixels + np.count_nonzero(in_window)

    print "Pixels with values in {}: {}".format(tile, non_zero_pixels)

    print "  Calculating average per-pixel value in", tile

    avg_10km = '{0}_{1}_average.tif'.format(tile_id, tile_type)

    # cmd = ['gdalwarp', '-tap', '-tr', '{}'.format(str(0.096342599)), '{}'.format(str(0.096342599)),  '-co', 'COMPRESS=LZW',
    #        # '-te', str(xmin), str(ymin), str(xmax), str(ymax),
    #        per_pixel, avg_10km]

    dt = 'float32'
    cmd = ['gdalwarp', '-t_srs', 'EPSG:4326', '-co', 'COMPRESS=LZW', '-tr', str(cn.aggreg_res), str(cn.aggreg_res), '-tap', '-te',
            str(xmin), str(ymin), str(xmax), str(ymax), '-dstnodata', '0', '-overwrite', '{}'.format(per_pixel), '{}'.format(avg_10km)]
    subprocess.check_call(cmd)

    # uu.warp_to_Hansen(per_pixel, avg_10km, xmin, ymin, xmax, ymax, dt)

    # Prints information about the tile that was just processed
    uu.end_of_fx_summary(start, tile_id, tile_type)<|endoftext|>


<commit_before>### Calculates the net emissions over the study period, with units of CO2/ha on a pixel-by-pixel basis

import multiprocessing
import utilities
import net_emissions
import sys
sys.path.append('../')
import constants_and_names

### Need to update and install some packages on spot machine before running
### sudo pip install rasterio --upgrade

# biomass_tile_list = utilities.tile_list(constants_and_names.biomass_dir)
biomass_tile_list = ['10N_080W', '40N_120E'] # test tiles
# biomass_tile_list = ['40N_120E'] # test tiles
print biomass_tile_list

# # For downloading all tiles in the input folders
# download_list = [constants_and_names.cumul_gain_combo_dir, constants_and_names.emissions_total_dir]
#
# for input in download_list:
#     utilities.s3_folder_download('{}'.format(input), '.')

# For copying individual tiles to spot machine for testing
for tile in biomass_tile_list:

    utilities.s3_file_download('{0}{1}_{2}.tif'.format(constants_and_names.cumul_gain_combo_dir, constants_and_names.pattern_cumul_gain_combo, tile), '.')  # cumulative aboveand belowground carbon gain for all forest types
    utilities.s3_file_download('{0}{1}_{2}.tif'.format(constants_and_names.emissions_total_dir, tile, constants_and_names.pattern_emissions_total), '.')  # emissions from all drivers

# count = multiprocessing.cpu_count()
# pool = multiprocessing.Pool(count / 4)
# pool.map(net_emissions.net_calc, biomass_tile_list)

# For single processor use
for tile in biomass_tile_list:

    net_emissions.net_calc(tile)<commit_msg>Testing net emissions script.<commit_after>### Calculates the net emissions over the study period, with units of CO2/ha on a pixel-by-pixel basis

import multiprocessing
import utilities
import net_emissions
import sys
sys.path.append('../')
import constants_and_names

### Need to update and install some packages on spot machine before running
### sudo pip install rasterio --upgrade

# biomass_tile_list = utilities.tile_list(constants_and_names.biomass_dir)
biomass_tile_list = ['10N_080W', '40N_120E'] # test tiles
# biomass_tile_list = ['40N_120E'] # test tiles
print biomass_tile_list

# # For downloading all tiles in the input folders
# download_list = [constants_and_names.cumul_gain_combo_dir, constants_and_names.emissions_total_dir]
#
# for input in download_list:
#     utilities.s3_folder_download('{}'.format(input), '.')

# # For copying individual tiles to spot machine for testing
# for tile in biomass_tile_list:
#
#     utilities.s3_file_download('{0}{1}_{2}.tif'.format(constants_and_names.cumul_gain_combo_dir, constants_and_names.pattern_cumul_gain_combo, tile), '.')  # cumulative aboveand belowground carbon gain for all forest types
#     utilities.s3_file_download('{0}{1}_{2}.tif'.format(constants_and_names.emissions_total_dir, tile, constants_and_names.pattern_emissions_total), '.')  # emissions from all drivers

# count = multiprocessing.cpu_count()
# pool = multiprocessing.Pool(count / 4)
# pool.map(net_emissions.net_calc, biomass_tile_list)

# For single processor use
for tile in biomass_tile_list:

    net_emissions.net_calc(tile)<|endoftext|>


<commit_before>#!/usr/bin/python
# -*- coding: utf-8 -*-
#   moldynplot.StateProbFigureManager.py
#
#   Copyright (C) 2015 Karl T Debiec
#   All rights reserved.
#
#   This software may be modified and distributed under the terms of the
#   BSD license. See the LICENSE file for details.
"""
Generates one or more secondary structure figures to specifications in a
YAML file.
"""
################################### MODULES ###################################
from __future__ import absolute_import,division,print_function,unicode_literals
if __name__ == "__main__":
    __package__ = str("moldynplot")
    import moldynplot
from .myplotspec.FigureManager import FigureManager
################################### CLASSES ###################################
class SSFigureManager(FigureManager):
    """
    Manages the generation of secondary structure figures.
    """

    from .myplotspec.manage_defaults_presets import manage_defaults_presets
    from .myplotspec.manage_kwargs import manage_kwargs

    defaults = """
        draw_subplot:
          xlabel: Time (ns)
          ylabel_kw:
            va: center
          tick_params:
            left: off
            right: off
            bottom: off
            top: off
          y2ticks: []
          y2label_kw:
            rotation: 270
          grid: True
          grid_kw:
            b: True
            linestyle: '-'
            axis: x
        draw_dataset:
          heatmap_kw:
            cmap: !!python/object/apply:moldynplot.dssp_color_palette []
            edgecolors: none
            rasterized: True
    """

    available_presets = """
      dssp:
        class: content
        help: Dynamic secondary structure of proteins calculated by cpptraj
        draw_dataset:
          labels:
            0: None
            1: Parallel β Sheet
            2: Antiparallel β Sheet
            3: $3_{10}$ Helix
            4: α Helix
            5: π Helix
            6: Turn
            7: Bend
          heatmap_kw:
            vmin: 0
            vmax: 7
      notebook:
        class: target
        inherits: notebook
        draw_figure:
          left:       0.50
          sub_width:  5.00
          right:      0.25
          bottom:     1.00
          sub_height: 2.00
          hspace:     0.10
          top:        0.25
          title_kw:
            top: -0.1
          shared_legend: True
          shared_legend_kw:
            left:       0.50
            sub_width:  5.00
            right:      0.25
            bottom:     0.00
            sub_height: 0.50
            legend_kw:
              loc: 9
              numpoints: 1
              handletextpad: 0
              ncol: 4
        draw_subplot:
          y2label_kw:
            labelpad: 12
    """

    @manage_defaults_presets()
    @manage_kwargs()
    def draw_dataset(self, subplot, label=None, handles=None, verbose=1,
        debug=0, **kwargs):
        from os.path import expandvars
        import numpy as np
        import pandas
        from .myplotspec import multi_get_copy
        from sys import exit

        # Load infile
        infile = expandvars(kwargs.get("infile"))
        dt = kwargs.get("dt", 1)
        dataset = pandas.read_csv(infile, delim_whitespace=True,
          index_col=0)
        dataset.index *= dt / 1000
        dataset.index.names = ["time"]

        # Downsample
        downsample = kwargs.get("downsample", 1000)
        if downsample is not None:
            full_size = dataset.shape[0]
            reduced_size = int(full_size / downsample)
            reduced = pandas.DataFrame(0.0, index=range(0, reduced_size),
              columns = dataset.columns, dtype=np.int64)
            for i in range(1, reduced_size):
                reduced.loc[i] = dataset[
                  i*downsample:(i+1)*downsample].mode().loc[0]
            reduced.index *= (dt * downsample) / 1000
            reduced.index.names = ["time"]
            dataset = reduced

        heatmap = True
        if heatmap:
            heatmap_kw = multi_get_copy("heatmap_kw", kwargs, {})
            pcolormesh = subplot.pcolor(
              dataset.index.values,
              np.arange(len(dataset.columns)),
              dataset.T.values,
            **heatmap_kw)

        if handles is not None:
            cmap = pcolormesh.cmap
            labels = kwargs.get("labels")
            h_kw=dict(ls="none", marker="s", ms=5, mec="k")

            for i in sorted(labels.keys()):
                label = labels[i]
                handles[label] = subplot.plot((0),(0),
                  mfc=cmap(i / (len(labels) - 1)),
                  **h_kw)[0]

#################################### MAIN #####################################
if __name__ == "__main__":
    SSFigureManager().main()
<commit_msg>SSFigureManager: pandas -> pd<commit_after>#!/usr/bin/python
# -*- coding: utf-8 -*-
#   moldynplot.StateProbFigureManager.py
#
#   Copyright (C) 2015 Karl T Debiec
#   All rights reserved.
#
#   This software may be modified and distributed under the terms of the
#   BSD license. See the LICENSE file for details.
"""
Generates one or more secondary structure figures to specifications in a
YAML file.
"""
################################### MODULES ###################################
from __future__ import absolute_import,division,print_function,unicode_literals
if __name__ == "__main__":
    __package__ = str("moldynplot")
    import moldynplot
from .myplotspec.FigureManager import FigureManager
################################### CLASSES ###################################
class SSFigureManager(FigureManager):
    """
    Manages the generation of secondary structure figures.
    """

    from .myplotspec.manage_defaults_presets import manage_defaults_presets
    from .myplotspec.manage_kwargs import manage_kwargs

    defaults = """
        draw_subplot:
          xlabel: Time (ns)
          ylabel_kw:
            va: center
          tick_params:
            left: off
            right: off
            bottom: off
            top: off
          y2ticks: []
          y2label_kw:
            rotation: 270
          grid: True
          grid_kw:
            b: True
            linestyle: '-'
            axis: x
        draw_dataset:
          heatmap_kw:
            cmap: !!python/object/apply:moldynplot.dssp_color_palette []
            edgecolors: none
            rasterized: True
    """

    available_presets = """
      dssp:
        class: content
        help: Dynamic secondary structure of proteins calculated by cpptraj
        draw_dataset:
          labels:
            0: None
            1: Parallel β Sheet
            2: Antiparallel β Sheet
            3: $3_{10}$ Helix
            4: α Helix
            5: π Helix
            6: Turn
            7: Bend
          heatmap_kw:
            vmin: 0
            vmax: 7
      notebook:
        class: target
        inherits: notebook
        draw_figure:
          left:       0.50
          sub_width:  5.00
          right:      0.25
          bottom:     1.00
          sub_height: 2.00
          hspace:     0.10
          top:        0.25
          title_kw:
            top: -0.1
          shared_legend: True
          shared_legend_kw:
            left:       0.50
            sub_width:  5.00
            right:      0.25
            bottom:     0.00
            sub_height: 0.50
            legend_kw:
              loc: 9
              numpoints: 1
              handletextpad: 0
              ncol: 4
        draw_subplot:
          y2label_kw:
            labelpad: 12
    """

    @manage_defaults_presets()
    @manage_kwargs()
    def draw_dataset(self, subplot, downsample=None, label=None, handles=None,
        verbose=1, debug=0, **kwargs):
        from os.path import expandvars
        import numpy as np
        import pandas as pd
        from .myplotspec import multi_get_copy
        from sys import exit

        # Load infile
        infile = expandvars(kwargs.get("infile"))
        dt = kwargs.get("dt", 1)
        dataset = pd.read_csv(infile, delim_whitespace=True,
          index_col=0)
        dataset.index *= dt / 1000
        dataset.index.names = ["time"]

        # Downsample
        if downsample is not None:
            full_size = dataset.shape[0]
            reduced_size = int(full_size / downsample)
            reduced = pd.DataFrame(0.0, index=range(0, reduced_size),
              columns=dataset.columns, dtype=np.int64)
            for i in range(1, reduced_size):
                reduced.loc[i] = dataset[
                  i*downsample:(i+1)*downsample].mode().loc[0]
            reduced.index *= (dt * downsample) / 1000
            reduced.index.names = ["time"]
            dataset = reduced

        heatmap = True
        if heatmap:
            heatmap_kw = multi_get_copy("heatmap_kw", kwargs, {})
            pcolormesh = subplot.pcolor(
              dataset.index.values,
              np.arange(len(dataset.columns)),
              dataset.T.values,
            **heatmap_kw)

        if handles is not None:
            cmap = pcolormesh.cmap
            labels = kwargs.get("labels")
            h_kw=dict(ls="none", marker="s", ms=5, mec="k")

            for i in sorted(labels.keys()):
                label = labels[i]
                handles[label] = subplot.plot((0),(0),
                  mfc=cmap(i / (len(labels) - 1)),
                  **h_kw)[0]

#################################### MAIN #####################################
if __name__ == "__main__":
    SSFigureManager().main()
<|endoftext|>


<commit_before>import xml.etree.cElementTree as etree
from cStringIO import StringIO
from apertium_quality.core import whereis
from collections import defaultdict
from subprocess import *
import urllib

class RegressionTest(object):
	ns = "{http://www.mediawiki.org/xml/export-0.3/}"
	program = "apertium"

	def __init__(self, url, mode, directory="."):
		if not whereis(self.program):
			raise IOError("Cannot find `%s`. Check $PATH." % self.program)
		if not "Special:Export" in url:
			raise AttributeError("URL did not contain Special:Export.")
		self.mode = mode
		self.directory = directory
		self.tree = etree.parse(urllib.urlopen(url))
		self.passes = 0
		self.total = 0
		for e in self.tree.getroot().getiterator():
			if e.tag == self.ns + "title":
				self.title = e.text
			if e.tag == self.ns + "revision":
				self.revision = e[0].text # should be <id>
			if e.tag == self.ns + "text":
				self.text = e.text
		if not self.text:
			raise AttributeError("No text element?")
		self.tests = defaultdict(defaultdict)
		for i in self.text.split('\n'):
			if i[:4] == "* {{":
				x = i.strip("{}* ").split('|')
				self.tests[x[1]][x[2]] = x[3]
		self.out = StringIO()

	def run(self):
		for side in self.tests:
			self.out.write("Now testing: %s\n" % side)

			app = Popen([self.program, '-d', self.directory, self.mode],
				stdin=PIPE, stdout=PIPE, stderr=PIPE)
			args = '\n'.join(self.tests[side].keys()) + '\n'

			app.stdin.write(args.encode('utf-8'))
			self.results = app.communicate()[0].decode('utf-8').split('\n')
			print self.results

			for n, test in enumerate(self.tests[side].items()):
				self.out.write("%s\t  %s\n" % (self.mode, test[0].encode('utf-8')))
				print self.results[n].strip(), test
				if self.results[n].strip() == test[1].strip():
					self.out.write("WORKS\t  %s\n" % self.results[n].encode('utf-8'))
					self.passes += 1
				else:
					self.out.write("\t- %s\n" % test[1].encode('utf-8'))
					self.out.write("\t+ %s\n" % self.results[n].encode('utf-8'))
				self.total += 1

	def start(self):
		self.run()

	def get_output(self):
		print self.out.getvalue()
		print "%d/%d %d%" % (self.passes, self.total, self.passes / self.total * 100)
<commit_msg>What.<commit_after>import xml.etree.cElementTree as etree
from cStringIO import StringIO
from apertium_quality.core import whereis
from collections import defaultdict
from subprocess import *
from tempfile import NamedTemporaryFile
import urllib

class RegressionTest(object):
	ns = "{http://www.mediawiki.org/xml/export-0.3/}"
	program = "apertium"

	def __init__(self, url, mode, directory="."):
		if not whereis(self.program):
			raise IOError("Cannot find `%s`. Check $PATH." % self.program)
		if not "Special:Export" in url:
			raise AttributeError("URL did not contain Special:Export.")
		self.mode = mode
		self.directory = directory
		self.tree = etree.parse(urllib.urlopen(url))
		self.passes = 0
		self.total = 0
		for e in self.tree.getroot().getiterator():
			if e.tag == self.ns + "title":
				self.title = e.text
			if e.tag == self.ns + "revision":
				self.revision = e[0].text # should be <id>
			if e.tag == self.ns + "text":
				self.text = e.text
		if not self.text:
			raise AttributeError("No text element?")
		self.tests = defaultdict(defaultdict)
		for i in self.text.split('\n'):
			if i[:4] == "* {{":
				x = i.strip("{}* ").split('|')
				self.tests[x[1]][x[2]] = x[3]
		self.out = StringIO()

	def run(self):
		for side in self.tests:
			self.out.write("Now testing: %s\n" % side)
			tmp = NamedTemporaryFile(delete=False)
			args = '.\n'.join(self.tests[side].keys()).encode('utf-8') + '.\n'
			tmp.write(args)
			app = Popen([self.program, '-d', self.directory, self.mode, tmp.name],
				stdin=PIPE, stdout=PIPE, stderr=PIPE)
			#tmp = open('derp.txt', 'w'); tmp.write(args); tmp.close()
			self.results = app.communicate()[0].decode('utf-8').split('\n')

			print self.results
			print "Rst:",len(self.results),"Tst:",len(self.tests[side])
			for n, test in enumerate(self.tests[side].items()):
				self.out.write("%s\t  %s\n" % (self.mode, test[0].encode('utf-8')))
				if self.results[n].strip() == test[1].strip():
					self.out.write("WORKS\t  %s\n" % self.results[n].encode('utf-8'))
					self.passes += 1
				else:
					self.out.write("\t- %s\n" % test[1].encode('utf-8'))
					self.out.write("\t+ %s\n" % self.results[n].encode('utf-8'))
				self.total += 1

	def start(self):
		self.run()

	def get_output(self):
		print self.out.getvalue()
		print "%d/%d %d%" % (self.passes, self.total, self.passes / self.total * 100)
<|endoftext|>


<commit_before>from time import time, strftime
from numpy import savetxt, array
from agilentcounter import AgilentCounter
from agilentmultimeter import AgilentMultimeter

# Config settings
countergpib = 1
multigpib = 6
vrange = 1
vresolution = 0.0001

# Setting up frequency counter
counter = AgilentCounter(gpib = countergpib)
counter.reset()
counter.setupFast()
counter.write(":FUNC 'FREQ 1'")
counter.setupGating(0.1)

# Setting up multimeter
multi = AgilentMultimeter(gpib = multigpib)
multi.reset()

# Setting up output file
datafile = "beat_%s.log" %(strftime("%y%m%d_%H%M%S"))
out = file(datafile, 'a')
out.write("Time(UnixTime) BeatFrequency(Hz) Voltage(V)")

while True:
    try:
        start = time()
        counter.initMeasure()
        volts = multi.getDCVoltage(vrange,vresolution)
        freq = counter.getFreq()
        now = (time() + start) / 2
        result = array([[now, freq, volts]])
        savetxt(out, result)
        print "%f Hz / %f V" %(freq, volts)
    except (KeyboardInterrupt):
        break
out.close()
<commit_msg>frequency: add more comment<commit_after>from time import time, strftime
from numpy import savetxt, array
from agilentcounter import AgilentCounter
from agilentmultimeter import AgilentMultimeter

# Config settings
countergpib = 1
multigpib = 6
vrange = 1
vresolution = 0.0001

# Setting up frequency counter
counter = AgilentCounter(gpib = countergpib)
counter.reset()
counter.setupFast()
counter.write(":FUNC 'FREQ 1'")
counter.setupGating(0.1)

# Setting up multimeter
multi = AgilentMultimeter(gpib = multigpib)
multi.reset()

# Setting up output file
datafile = "beat_%s.log" %(strftime("%y%m%d_%H%M%S"))
out = file(datafile, 'a')
out.write("Time(UnixTime) BeatFrequency(Hz) Voltage(V)")

# Do logging until stopped by Ctrl-C
while True:
    try:
        start = time()
        counter.initMeasure()
        volts = multi.getDCVoltage(vrange,vresolution)
        freq = counter.getFreq()
        now = (time() + start) / 2
        result = array([[now, freq, volts]])
        savetxt(out, result)
        print "%f Hz / %f V" %(freq, volts)
    except (KeyboardInterrupt):
        break
out.close()
<|endoftext|>


<commit_before>import asyncio
import json

from dnslib import *

from PRCDNS.proxy_client import ProxyClient

class DNSServerProtocol(asyncio.Protocol):
    gFuncs = globals()
    peername = None

    def get_data(self, data):
        print(data)
        sz = struct.unpack(">H", data[:2])[0]
        if sz < len(data) - 2:
            raise Exception("Wrong size of TCP packet")
        elif sz > len(data) - 2:
            raise Exception("Too big TCP packet")
        return data[2:]

    def connection_made(self, transport):
        self.peername = transport.get_extra_info('peername')
        print('Connection from {}'.format(self.peername))
        self.transport = transport

    def data_received(self, data):
        data = self.get_data(data)
        request = DNSRecord.parse(data)
        print('Data received: {!r}'.format(request))

        loop1 = asyncio.new_event_loop()
        client = ProxyClient()
        # coros = asyncio.gather(*tasks)
        url = 'https://dns.google.com/resolve?name={}&edns_client_subnet={}/24'.format(str(request.q.qname),
                                                                                self.peername[0])
        google_dns_resp = loop1.run_until_complete(client.get(loop1, url))
        print(google_dns_resp)
        loop1.close()
        # google_dns_resp = """{"Status": 0,"TC": false,"RD": true,"RA": true,"AD": false,"CD": false,"Question":[ {"name": "img.alicdn.com.","type": 1}],"Answer":[ {"name": "img.alicdn.com.","type": 5,"TTL": 85493,"data": "img.alicdn.com.danuoyi.alicdn.com."},{"name": "img.alicdn.com.danuoyi.alicdn.com.","type": 1,"TTL": 59,"data": "111.32.130.108"},{"name": "img.alicdn.com.danuoyi.alicdn.com.","type": 1,"TTL": 59,"data": "111.32.130.109"}],"Additional":[],"edns_client_subnet": "223.72.90.0/24","Comment": "Response from danuoyinewns3.gds.alicdn.com.(121.42.1.129)"}"""
        resp = json.loads(google_dns_resp)
        a = request.reply()
        for answer in resp['Answer']:
            qTypeFunc = QTYPE[answer['type']]
            a.add_answer(RR(answer['name'], answer['type'], rdata=self.gFuncs[qTypeFunc](answer['data']),
                            ttl=answer['TTL']))
        print('Send: {!r}'.format(a))
        b_resp = a.pack()
        b_resp = struct.pack(">H", b_resp.__len__()) + b_resp
        self.transport.write(b_resp)

        print('Close the client socket')
        self.transport.close()

def main():
    loop = asyncio.get_event_loop()
    # Each client connection will create a new protocol instance
    coro = loop.create_server(lambda: DNSServerProtocol(), '127.0.0.1', 5353)
    server = loop.run_until_complete(coro)

    try:
        loop.run_forever()
    except KeyboardInterrupt:
        pass

    print('Close Server')
    server.close()
    loop.run_until_complete(server.wait_closed())
    loop.close()

if __name__ == "__main__":
    main()
<commit_msg>rm rocksdb<commit_after>import asyncio
import json

from dnslib import *

from PRCDNS.proxy_client import ProxyClient

class DNSServerProtocol(asyncio.Protocol):
    gFuncs = globals()
    peername = None

    def get_data(self, data):
        print(data)
        sz = struct.unpack(">H", data[:2])[0]
        if sz < len(data) - 2:
            raise Exception("Wrong size of TCP packet")
        elif sz > len(data) - 2:
            raise Exception("Too big TCP packet")
        return data[2:]

    def connection_made(self, transport):
        self.peername = transport.get_extra_info('peername')
        print('Connection from {}'.format(self.peername))
        self.transport = transport

    def data_received(self, data):
        data = self.get_data(data)
        request = DNSRecord.parse(data)
        print('Data received: {!r}'.format(request))

        loop1 = asyncio.new_event_loop()
        client = ProxyClient()
        # coros = asyncio.gather(*tasks)
        url = 'https://dns.google.com/resolve?name={}&edns_client_subnet={}/24'.format(str(request.q.qname),
                                                                                self.peername[0])
        google_dns_resp = loop1.run_until_complete(client.get(loop1, url))
        print(google_dns_resp)
        loop1.close()
        # google_dns_resp = """{"Status": 0,"TC": false,"RD": true,"RA": true,"AD": false,"CD": false,"Question":[ {"name": "img.alicdn.com.","type": 1}],"Answer":[ {"name": "img.alicdn.com.","type": 5,"TTL": 85493,"data": "img.alicdn.com.danuoyi.alicdn.com."},{"name": "img.alicdn.com.danuoyi.alicdn.com.","type": 1,"TTL": 59,"data": "111.32.130.108"},{"name": "img.alicdn.com.danuoyi.alicdn.com.","type": 1,"TTL": 59,"data": "111.32.130.109"}],"Additional":[],"edns_client_subnet": "223.72.90.0/24","Comment": "Response from danuoyinewns3.gds.alicdn.com.(121.42.1.129)"}"""
        resp = json.loads(google_dns_resp)
        a = request.reply()
        for answer in resp['Answer']:
            qTypeFunc = QTYPE[answer['type']]
            a.add_answer(RR(answer['name'], answer['type'], rdata=self.gFuncs[qTypeFunc](answer['data']),
                            ttl=answer['TTL']))
        print('Send: {!r}'.format(a))
        b_resp = a.pack()
        b_resp = struct.pack(">H", b_resp.__len__()) + b_resp
        self.transport.write(b_resp)

        print('Close the client socket')
        self.transport.close()

def main():
    loop = asyncio.get_event_loop()
    # Each client connection will create a new protocol instance
    coro = loop.create_server(lambda: DNSServerProtocol(), '0.0.0.0', 3535)
    server = loop.run_until_complete(coro)

    try:
        loop.run_forever()
    except KeyboardInterrupt:
        pass

    print('Close Server')
    server.close()
    loop.run_until_complete(server.wait_closed())
    loop.close()

if __name__ == "__main__":
    main()
<|endoftext|>


<commit_before>import os
from serial import Serial, SerialException
from channel import Channel

"""
	Sensor Class
	ToDo:
	- ggf. abstrakte Klasse mit Sensor und Logfile als Unterklassen
	- oder "FromFile" Parameter
	- Logger integrieren?
	- Teensy: Raspi- und Mac Unterschiede egalisieren
	- ggf. Teensy in eigene klasse auslagern

"""
class Sensor:

	def __init__(self):
		self.initialized = False
		self.initSerial()
		self.initChannels()

	def __del__(self):
		print "Close Teensy"
		self.teensy.close()

	def initSerial(self):

		try:
			for i in range(0, 5):
				devName = "/dev/ttyACM%d" % i
				if os.path.exists(devName):
					self.teensy = Serial(devName, 115200, timeout=10)
					return
			else:
				raise Exception("NoTeensy")
		except:
			raise Exception("NoTeensy")
		"""
		try:
			devName = "/dev/cu.usbmodem228731"
			if os.path.exists(devName):
				self.teensy = Serial(devName, 115200, timeout=0.1)
				return
			else:
				raise Exception("NoTeensy")
		except:
			raise Exception("NoTeensy")
		"""

	def checkTeensy(self):
		self.teensy.write("i")
		response = self.teensy.readline().strip()
		print "Init Teensy: %s" % response
		return response == "ok"

	def initChannels(self):
		self.checkTeensy()
		try:
			self.channels = []
			self.teensy.write("?")
			response = self.teensy.readline().strip()
			sensors = response.split(";")

			for sensor in sensors:
				if sensor != "":
					name, params = sensor.split(":")
					num, min, max = params.split(",")
					num = int(num)
					min, max = float(min), float(max)
					for i in range(num):
						self.channels.append(Channel(name, min, max, 100))

			self.initialized = True
		except:
			print "initChannels error"
			self.initialized = False

	def readData(self):
		if not self.initialized:
			self.initChannels()
		self.teensy.write("!")
		response = self.teensy.readline().strip()

		try:
			values = map(float, response.split(","))
			print values
			for i, v in enumerate(values):
				self.channels[i].putValue(v)
		except:
			raise
			self.initialized = False
<commit_msg>Start Cleanup, NumPy Smooth<commit_after>import os
from serial import Serial, SerialException
from channel import Channel

"""
	Sensor Class
	ToDo:
	- ggf. abstrakte Klasse mit Sensor und Logfile als Unterklassen
	- oder "FromFile" Parameter
	- Logger integrieren?
	- Teensy: Raspi- und Mac Unterschiede egalisieren
	- ggf. Teensy in eigene klasse auslagern

"""
class Sensor:

	def __init__(self):
		self.initialized = False
		self.initSerial()
		self.initChannels()

	def __del__(self):
		print "Close Teensy"
		self.teensy.close()

	def initSerial(self):

		try:
			for i in range(0, 5):
				devName = "/dev/ttyACM%d" % i
				if os.path.exists(devName):
					self.teensy = Serial(devName, 115200, timeout=10)
					return
			else:
				raise Exception("NoTeensy")
		except:
			raise Exception("NoTeensy")
		"""
		try:
			devName = "/dev/cu.usbmodem228731"
			if os.path.exists(devName):
				self.teensy = Serial(devName, 115200, timeout=0.1)
				return
			else:
				raise Exception("NoTeensy")
		except:
			raise Exception("NoTeensy")
		"""

	def checkTeensy(self):
		self.teensy.write("i")
		response = self.teensy.readline().strip()
		print "Init Teensy: %s" % response
		return response == "ok"

	def initChannels(self):
		self.checkTeensy()
		try:
			self.channels = []
			self.teensy.write("?")
			response = self.teensy.readline().strip()
			sensors = response.split(";")

			for sensor in sensors:
				if sensor != "":
					name, params = sensor.split(":")
					num, min, max = params.split(",")
					num = int(num)
					min, max = float(min), float(max)
					for i in range(num):
						self.channels.append(Channel(name, min, max, 100))

			self.initialized = True
		except:
			print "initChannels error"
			self.initialized = False

	def readData(self):
		if not self.initialized:
			self.initChannels()
		self.teensy.write("!")
		response = self.teensy.readline().strip()

		try:
			values = map(float, response.split(","))
			for i, v in enumerate(values):
				self.channels[i].putValue(v)
		except:
			raise
			self.initialized = False
<|endoftext|>


<commit_before>from django.contrib import admin
from blog.models import Post

class PostAdmin(admin.ModelAdmin):
    list_display  		= ('title', 'publish', 'status')
    list_filter   		= ('publish', 'status')
    search_fields 		= ('title', 'body', 'tease')
    prepopulated_fields = {'slug': ('title',)}

admin.site.register(Post, PostAdmin)
<commit_msg>removed vestiges of NFA<commit_after><|endoftext|>


<commit_before>#! /usr/bin/env Python3

import math

def Entropy(l):  # l is the list included times of each variable
    sumNum = sum(l)
    result = 0
    for i in l:
        if i == 0:
            result += 0
        else:
            result += - (i / sumNum) * math.log2(i / sumNum)

    return result

class Leaf():
    leaf = {}

    def __init__(self, **kwargs):
        self.leaf = kwargs

class Node():

    def __init__(self, tag):
        self.this = tag
        self.childList = []
        self.childNode = {}
        self.leaves = {}

    def addNode(self, chileWay, childNode):  # might be changed futrue
        self.childNode[chileWay] = childNode
        self.childList.append(chileWay)

    def addLeaf(self, chileWay, leaf):
        self.leaves[chileWay] = leaf
        self.childList.append(chileWay)

    def search(self, *attributions):
        pass

def main():

    a = Node("outlook?")
    b1 = Node("humidity?")
    b2 = Leaf("play"=4, "noPlay"=0)
    b3 = Node("windy?")

    c1 = Leaf("play"=2, "noPlay"=0)
    c2 = Leaf("play"=0, "noPlay"=3)
    c3 = Leaf("play"=0, "noPlay"=2)
    c4 = Leaf("play"=3, "noPlay"=0)

    a.addNode("sunny", b1)
    a.addNode("rain", b3)
    a.addLeaf("overcast", b2)

    b1.addLeaf("<=70", c1)
    b1.addLeaf(">70", c2)

    b3.addLeaf("true", c3)
    b3.addLeaf("false", c4)

    print(c1[play])

if __name__ == "__main__":
    print("")
<commit_msg>a bit, change the design in leaf<commit_after>#! /usr/bin/env Python3

import math

def Entropy(l):  # l is the list included times of each variable
    sumNum = sum(l)
    result = 0
    for i in l:
        if i == 0:
            result += 0
        else:
            result += - (i / sumNum) * math.log2(i / sumNum)

    return result

class Leaf():
    leaf = {}

    def __init__(self, dic):
        for key in dic:
            self.leaf[key] = dic[key]

class Node():

    def __init__(self, tag):
        self.this = tag
        self.childList = []
        self.childNode = {}
        self.leaves = {}

    def addNode(self, chileWay, childNode):  # might be changed futrue
        self.childNode[chileWay] = childNode
        self.childList.append(chileWay)

    def addLeaf(self, chileWay, leaf):
        self.leaves[chileWay] = leaf
        self.childList.append(chileWay)

    def search(self, *attributions):
        pass

def main():

    a = Node("outlook?")
    b1 = Node("humidity?")
    b2 = Leaf({"play": 4, "noPlay": 0})
    b3 = Node("windy?")

    c1 = Leaf({"play": 2, "noPlay": 0})
    c2 = Leaf({"play": 0, "noPlay": 3})
    c3 = Leaf({"play": 0, "noPlay": 2})
    c4 = Leaf({"play": 3, "noPlay": 0})

    a.addNode("sunny", b1)
    a.addNode("rain", b3)
    a.addLeaf("overcast", b2)

    b1.addLeaf("<=70", c1)
    b1.addLeaf(">70", c2)

    b3.addLeaf("true", c3)
    b3.addLeaf("false", c4)

    print(c1.leaf)

if __name__ == "__main__":
    print("")
<|endoftext|>


<commit_before>import os
from libtbx.containers import OrderedDict

class _ImagesetCache(dict):
  pass

imageset_cache = _ImagesetCache()

def load_imagesets(template, directory, id_image=None, image_range=None,
                   use_cache=True):
  global imageset_cache

  full_template_path = os.path.join(directory, template)
  if full_template_path not in imageset_cache or not use_cache:

    from dxtbx.datablock import DataBlockFactory
    from dxtbx.sweep_filenames import locate_files_matching_template_string
    paths = sorted(locate_files_matching_template_string(full_template_path))
    unhandled = []
    datablocks = DataBlockFactory.from_filenames(
      paths, verbose=False, unhandled=unhandled)
    assert len(unhandled) == 0, unhandled
    assert len(datablocks) == 1, len(datablocks)
    imagesets = datablocks[0].extract_sweeps()
    assert len(imagesets) > 0

    imageset_cache[full_template_path] = OrderedDict()

    for imageset in imagesets:
      scan = imageset.get_scan()
      _id_image = scan.get_image_range()[0]
      imageset_cache[full_template_path][_id_image] = imageset

  if id_image is not None:
    return [imageset_cache[full_template_path][id_image]]
  elif image_range is not None:
    for imageset in imageset_cache[full_template_path].values():
      scan = imageset.get_scan()
      scan_image_range = scan.get_image_range()
      if (image_range[0] >= scan_image_range[0] and
          image_range[1] <= scan_image_range[1]):
        return [imageset[
          scan_image_range[0]-image_range[0]:
          scan_image_range[1]-image_range[0] + 1]]
  return imageset_cache[full_template_path].values()
<commit_msg>more verbose assertions<commit_after>import os
from libtbx.containers import OrderedDict

class _ImagesetCache(dict):
  pass

imageset_cache = _ImagesetCache()

def load_imagesets(template, directory, id_image=None, image_range=None,
                   use_cache=True):
  global imageset_cache

  full_template_path = os.path.join(directory, template)
  if full_template_path not in imageset_cache or not use_cache:

    from dxtbx.datablock import DataBlockFactory
    from dxtbx.sweep_filenames import locate_files_matching_template_string
    paths = sorted(locate_files_matching_template_string(full_template_path))
    unhandled = []
    datablocks = DataBlockFactory.from_filenames(
      paths, verbose=False, unhandled=unhandled)
    assert len(unhandled) == 0, "unhandled image files identified: %s" % unhandled
    assert len(datablocks) == 1, "1 datablock expected, %d found" % len(datablocks)
    imagesets = datablocks[0].extract_sweeps()
    assert len(imagesets) > 0, "no imageset found"

    imageset_cache[full_template_path] = OrderedDict()

    for imageset in imagesets:
      scan = imageset.get_scan()
      _id_image = scan.get_image_range()[0]
      imageset_cache[full_template_path][_id_image] = imageset

  if id_image is not None:
    return [imageset_cache[full_template_path][id_image]]
  elif image_range is not None:
    for imageset in imageset_cache[full_template_path].values():
      scan = imageset.get_scan()
      scan_image_range = scan.get_image_range()
      if (image_range[0] >= scan_image_range[0] and
          image_range[1] <= scan_image_range[1]):
        return [imageset[
          scan_image_range[0]-image_range[0]:
          scan_image_range[1]-image_range[0] + 1]]
  return imageset_cache[full_template_path].values()
<|endoftext|>