<commit_before>#!/usr/bin/env python
# Copyright 2010 Google Inc. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import BaseHTTPServer
import errno
import logging
import socket
import SocketServer
import ssl
import time
import urlparse

import certutils
import daemonserver
import httparchive
import proxyshaper
import sslproxy

class HttpProxyError(Exception):
  """Module catch-all error."""
  pass

class HttpProxyServerError(HttpProxyError):
  """Raised for errors like 'Address already in use'."""
  pass

class HttpArchiveHandler(BaseHTTPServer.BaseHTTPRequestHandler):
  protocol_version = 'HTTP/1.1'  # override BaseHTTPServer setting

  # Since we do lots of small wfile.write() calls, turn on buffering.
  wbufsize = -1  # override StreamRequestHandler (a base class) setting

  def setup(self):
    """Override StreamRequestHandler method."""
    BaseHTTPServer.BaseHTTPRequestHandler.setup(self)
    if self.server.traffic_shaping_up_bps:
      self.rfile = proxyshaper.RateLimitedFile(
          self.server.get_active_request_count, self.rfile,
          self.server.traffic_shaping_up_bps)
    if self.server.traffic_shaping_down_bps:
      self.wfile = proxyshaper.RateLimitedFile(
          self.server.get_active_request_count, self.wfile,
          self.server.traffic_shaping_down_bps)

  # Make request handler logging match our logging format.
  def log_request(self, code='-', size='-'): pass
  def log_error(self, format, *args): logging.error(format, *args)
  def log_message(self, format, *args): logging.info(format, *args)

  def read_request_body(self):
    request_body = None
    length = int(self.headers.get('content-length', 0)) or None
    if length:
      request_body = self.rfile.read(length)
    return request_body

  def get_header_dict(self):
    return dict(self.headers.items())

  def get_archived_http_request(self):
    host = self.headers.get('host')
    if host is None:
      logging.error('Request without host header')
      return None

    parsed = urlparse.urlparse(self.path)
    params = ';%s' % parsed.params if parsed.params else ''
    query = '?%s' % parsed.query if parsed.query else ''
    fragment = '#%s' % parsed.fragment if parsed.fragment else ''
    full_path = '%s%s%s%s' % (parsed.path, params, query, fragment)

    return httparchive.ArchivedHttpRequest(
        self.command,
        host,
        full_path,
        self.read_request_body(),
        self.get_header_dict(),
        self.server.is_ssl)

  def send_archived_http_response(self, response):
    try:
      # We need to set the server name before we start the response.
      is_chunked = response.is_chunked()
      has_content_length = response.get_header('content-length') is not None
      self.server_version = response.get_header('server', 'WebPageReplay')
      self.sys_version = ''

      if response.version == 10:
        self.protocol_version = 'HTTP/1.0'

      # If we don't have chunked encoding and there is no content length,
      # we need to manually compute the content-length.
      if not is_chunked and not has_content_length:
        content_length = sum(len(c) for c in response.response_data)
        response.headers.append(('content-length', str(content_length)))

      is_replay = not self.server.http_archive_fetch.is_record_mode
      if is_replay and self.server.traffic_shaping_delay_ms:
        logging.debug('Using round trip delay: %sms',
                      self.server.traffic_shaping_delay_ms)
        time.sleep(self.server.traffic_shaping_delay_ms / 1000.0)
      if is_replay and self.server.use_delays:
        logging.debug('Using delays (ms): %s', response.delays)
        time.sleep(response.delays['headers'] / 1000.0)
        delays = response.delays['data']
      else:
        delays = [0] * len(response.response_data)
      self.send_response(response.status, response.reason)
      # TODO(mbelshe): This is lame - each write is a packet!
      for header, value in response.headers:
        if header in ('last-modified', 'expires'):
          self.send_header(header, response.update_date(value))
        elif header not in ('date', 'server'):
          self.send_header(header, value)
      self.end_headers()

      for chunk, delay in zip(response.response_data, delays):
        if delay:
          self.wfile.flush()
          time.sleep(delay / 1000.0)
        if is_chunked:
          # Write chunk length (hex) and data (e.g. "A\r\nTESSELATED\r\n").
          self.wfile.write('%x\r\n%s\r\n' % (len(chunk), chunk))
        else:
          self.wfile.write(chunk)
      if is_chunked:
        self.wfile.write('0\r\n\r\n')  # write final, zero-length chunk.
      self.wfile.flush()

      # TODO(mbelshe): This connection close doesn't seem to work.
      if response.version == 10:
        self.close_connection = 1

    except Exception, e:
      logging.error('Error sending response for %s%s: %s',
                    self.headers['host'], self.path, e)

  def handle_one_request(self):
    """Handle a single HTTP request."""
    try:
      self.raw_requestline = self.rfile.readline(65537)
      self.do_parse_and_handle_one_request()
    except socket.timeout, e:
      # A read or a write timed out.  Discard this connection
      self.log_error('Request timed out: %r', e)
      self.close_connection = 1
      return
    except socket.error, e:
      # Connection reset errors happen all the time due to the browser closing
      # without terminating the connection properly.  They can be safely
      # ignored.
      if e[0] != errno.ECONNRESET:
        raise

  def do_parse_and_handle_one_request(self):
    start_time = time.time()
    self.server.num_active_requests += 1
    request = None
    try:
      if len(self.raw_requestline) > 65536:
        self.requestline = ''
        self.request_version = ''
        self.command = ''
        self.send_error(414)
        return
      if not self.raw_requestline:
        self.close_connection = 1
        return
      if not self.parse_request():
        # An error code has been sent, just exit
        return

      try:
        request = self.get_archived_http_request()

        if request is None:
          self.send_error(500)
          return
        response = self.server.custom_handlers.handle(request)
        if not response:
          response = self.server.http_archive_fetch(request)
        if response:
          self.send_archived_http_response(response)
        else:
          self.send_error(404)
      finally:
        self.wfile.flush()  # Actually send the response if not already done.
    finally:
      request_time_ms = (time.time() - start_time) * 1000.0
      if request:
        logging.debug('Served: %s (%dms)', request, request_time_ms)
      self.server.total_request_time += request_time_ms
      self.server.num_active_requests -= 1

  def send_error(self, status, body=None):
    """Override the default send error with a version that doesn't unnecessarily
    close the connection.
    """
    response = httparchive.create_response(status, body=body)
    self.send_archived_http_response(response)

class HttpProxyServer(SocketServer.ThreadingMixIn,
                      BaseHTTPServer.HTTPServer,
                      daemonserver.DaemonServer):
  HANDLER = HttpArchiveHandler

  # Increase the request queue size. The default value, 5, is set in
  # SocketServer.TCPServer (the parent of BaseHTTPServer.HTTPServer).
  # Since we're intercepting many domains through this single server,
  # it is quite possible to get more than 5 concurrent requests.
  request_queue_size = 128

  # Don't prevent python from exiting when there is thread activity.
  daemon_threads = True

  def __init__(self, http_archive_fetch, custom_handlers,
               host='localhost', port=80, use_delays=False, is_ssl=False,
               protocol='HTTP',
               down_bandwidth='0', up_bandwidth='0', delay_ms='0'):
    """Start HTTP server.

    Args:
      host: a host string (name or IP) for the web proxy.
      port: a port string (e.g. '80') for the web proxy.
      use_delays: if True, add response data delays during replay.
      is_ssl: True iff proxy is using SSL.
      up_bandwidth: Upload bandwidth
      down_bandwidth: Download bandwidth
           Bandwidths measured in [K|M]{bit/s|Byte/s}. '0' means unlimited.
      delay_ms: Propagation delay in milliseconds. '0' means no delay.
    """
    try:
      BaseHTTPServer.HTTPServer.__init__(self, (host, port), self.HANDLER)
    except Exception, e:
      raise HttpProxyServerError('Could not start HTTPServer on port %d: %s' %
                                 (port, e))
    self.http_archive_fetch = http_archive_fetch
    self.custom_handlers = custom_handlers
    self.use_delays = use_delays
    self.is_ssl = is_ssl
    self.traffic_shaping_down_bps = proxyshaper.GetBitsPerSecond(down_bandwidth)
    self.traffic_shaping_up_bps = proxyshaper.GetBitsPerSecond(up_bandwidth)
    self.traffic_shaping_delay_ms = int(delay_ms)
    self.num_active_requests = 0
    self.total_request_time = 0
    self.protocol = protocol

    # Note: This message may be scraped. Do not change it.
    logging.warning(
        '%s server started on %s:%d' % (self.protocol, self.server_address[0],
                                        self.server_address[1]))

  def cleanup(self):
    try:
      self.shutdown()
    except KeyboardInterrupt:
      pass
    logging.info('Stopped %s server. Total time processing requests: %dms',
                 self.protocol, self.total_request_time)

  def get_active_request_count(self):
    return self.num_active_requests

class HttpsProxyServer(HttpProxyServer):
  """SSL server that generates certs for each host."""

  def __init__(self, http_archive_fetch, custom_handlers,
               https_root_ca_cert_path, **kwargs):
    self.ca_cert_path = https_root_ca_cert_path
    self.HANDLER = sslproxy.wrap_handler(HttpArchiveHandler)
    HttpProxyServer.__init__(self, http_archive_fetch, custom_handlers,
                             is_ssl=True, **kwargs)
    self.http_archive_fetch.http_archive.set_root_cert(https_root_ca_cert_path)

  def cleanup(self):
    try:
      self.shutdown()
    except KeyboardInterrupt:
      pass

class SingleCertHttpsProxyServer(HttpProxyServer):
  """SSL server."""

  def __init__(self, http_archive_fetch, custom_handlers,
               https_root_ca_cert_path, **kwargs):
    HttpProxyServer.__init__(self, http_archive_fetch, custom_handlers,
                             is_ssl=True, protocol='HTTPS', **kwargs)
    self.socket = ssl.wrap_socket(
        self.socket, certfile=https_root_ca_cert_path, server_side=True,
        do_handshake_on_connect=False)
    # Ancestor class, DaemonServer, calls serve_forever() during its __init__.

class HttpToHttpsProxyServer(HttpProxyServer):
  """Listens for HTTP requests but sends them to the target as HTTPS requests"""

  def __init__(self, http_archive_fetch, custom_handlers, **kwargs):
    HttpProxyServer.__init__(self, http_archive_fetch, custom_handlers,
                             is_ssl=True, protocol='HTTP-to-HTTPS', **kwargs)
<commit_msg>Fix protocol label for HTTPS proxy server. (Telemetry looks for this string.)<commit_after>#!/usr/bin/env python
# Copyright 2010 Google Inc. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import BaseHTTPServer
import errno
import logging
import socket
import SocketServer
import ssl
import time
import urlparse

import certutils
import daemonserver
import httparchive
import proxyshaper
import sslproxy

class HttpProxyError(Exception):
  """Module catch-all error."""
  pass

class HttpProxyServerError(HttpProxyError):
  """Raised for errors like 'Address already in use'."""
  pass

class HttpArchiveHandler(BaseHTTPServer.BaseHTTPRequestHandler):
  protocol_version = 'HTTP/1.1'  # override BaseHTTPServer setting

  # Since we do lots of small wfile.write() calls, turn on buffering.
  wbufsize = -1  # override StreamRequestHandler (a base class) setting

  def setup(self):
    """Override StreamRequestHandler method."""
    BaseHTTPServer.BaseHTTPRequestHandler.setup(self)
    if self.server.traffic_shaping_up_bps:
      self.rfile = proxyshaper.RateLimitedFile(
          self.server.get_active_request_count, self.rfile,
          self.server.traffic_shaping_up_bps)
    if self.server.traffic_shaping_down_bps:
      self.wfile = proxyshaper.RateLimitedFile(
          self.server.get_active_request_count, self.wfile,
          self.server.traffic_shaping_down_bps)

  # Make request handler logging match our logging format.
  def log_request(self, code='-', size='-'): pass
  def log_error(self, format, *args): logging.error(format, *args)
  def log_message(self, format, *args): logging.info(format, *args)

  def read_request_body(self):
    request_body = None
    length = int(self.headers.get('content-length', 0)) or None
    if length:
      request_body = self.rfile.read(length)
    return request_body

  def get_header_dict(self):
    return dict(self.headers.items())

  def get_archived_http_request(self):
    host = self.headers.get('host')
    if host is None:
      logging.error('Request without host header')
      return None

    parsed = urlparse.urlparse(self.path)
    params = ';%s' % parsed.params if parsed.params else ''
    query = '?%s' % parsed.query if parsed.query else ''
    fragment = '#%s' % parsed.fragment if parsed.fragment else ''
    full_path = '%s%s%s%s' % (parsed.path, params, query, fragment)

    return httparchive.ArchivedHttpRequest(
        self.command,
        host,
        full_path,
        self.read_request_body(),
        self.get_header_dict(),
        self.server.is_ssl)

  def send_archived_http_response(self, response):
    try:
      # We need to set the server name before we start the response.
      is_chunked = response.is_chunked()
      has_content_length = response.get_header('content-length') is not None
      self.server_version = response.get_header('server', 'WebPageReplay')
      self.sys_version = ''

      if response.version == 10:
        self.protocol_version = 'HTTP/1.0'

      # If we don't have chunked encoding and there is no content length,
      # we need to manually compute the content-length.
      if not is_chunked and not has_content_length:
        content_length = sum(len(c) for c in response.response_data)
        response.headers.append(('content-length', str(content_length)))

      is_replay = not self.server.http_archive_fetch.is_record_mode
      if is_replay and self.server.traffic_shaping_delay_ms:
        logging.debug('Using round trip delay: %sms',
                      self.server.traffic_shaping_delay_ms)
        time.sleep(self.server.traffic_shaping_delay_ms / 1000.0)
      if is_replay and self.server.use_delays:
        logging.debug('Using delays (ms): %s', response.delays)
        time.sleep(response.delays['headers'] / 1000.0)
        delays = response.delays['data']
      else:
        delays = [0] * len(response.response_data)
      self.send_response(response.status, response.reason)
      # TODO(mbelshe): This is lame - each write is a packet!
      for header, value in response.headers:
        if header in ('last-modified', 'expires'):
          self.send_header(header, response.update_date(value))
        elif header not in ('date', 'server'):
          self.send_header(header, value)
      self.end_headers()

      for chunk, delay in zip(response.response_data, delays):
        if delay:
          self.wfile.flush()
          time.sleep(delay / 1000.0)
        if is_chunked:
          # Write chunk length (hex) and data (e.g. "A\r\nTESSELATED\r\n").
          self.wfile.write('%x\r\n%s\r\n' % (len(chunk), chunk))
        else:
          self.wfile.write(chunk)
      if is_chunked:
        self.wfile.write('0\r\n\r\n')  # write final, zero-length chunk.
      self.wfile.flush()

      # TODO(mbelshe): This connection close doesn't seem to work.
      if response.version == 10:
        self.close_connection = 1

    except Exception, e:
      logging.error('Error sending response for %s%s: %s',
                    self.headers['host'], self.path, e)

  def handle_one_request(self):
    """Handle a single HTTP request."""
    try:
      self.raw_requestline = self.rfile.readline(65537)
      self.do_parse_and_handle_one_request()
    except socket.timeout, e:
      # A read or a write timed out.  Discard this connection
      self.log_error('Request timed out: %r', e)
      self.close_connection = 1
      return
    except socket.error, e:
      # Connection reset errors happen all the time due to the browser closing
      # without terminating the connection properly.  They can be safely
      # ignored.
      if e[0] != errno.ECONNRESET:
        raise

  def do_parse_and_handle_one_request(self):
    start_time = time.time()
    self.server.num_active_requests += 1
    request = None
    try:
      if len(self.raw_requestline) > 65536:
        self.requestline = ''
        self.request_version = ''
        self.command = ''
        self.send_error(414)
        return
      if not self.raw_requestline:
        self.close_connection = 1
        return
      if not self.parse_request():
        # An error code has been sent, just exit
        return

      try:
        request = self.get_archived_http_request()

        if request is None:
          self.send_error(500)
          return
        response = self.server.custom_handlers.handle(request)
        if not response:
          response = self.server.http_archive_fetch(request)
        if response:
          self.send_archived_http_response(response)
        else:
          self.send_error(404)
      finally:
        self.wfile.flush()  # Actually send the response if not already done.
    finally:
      request_time_ms = (time.time() - start_time) * 1000.0
      if request:
        logging.debug('Served: %s (%dms)', request, request_time_ms)
      self.server.total_request_time += request_time_ms
      self.server.num_active_requests -= 1

  def send_error(self, status, body=None):
    """Override the default send error with a version that doesn't unnecessarily
    close the connection.
    """
    response = httparchive.create_response(status, body=body)
    self.send_archived_http_response(response)

class HttpProxyServer(SocketServer.ThreadingMixIn,
                      BaseHTTPServer.HTTPServer,
                      daemonserver.DaemonServer):
  HANDLER = HttpArchiveHandler

  # Increase the request queue size. The default value, 5, is set in
  # SocketServer.TCPServer (the parent of BaseHTTPServer.HTTPServer).
  # Since we're intercepting many domains through this single server,
  # it is quite possible to get more than 5 concurrent requests.
  request_queue_size = 128

  # Don't prevent python from exiting when there is thread activity.
  daemon_threads = True

  def __init__(self, http_archive_fetch, custom_handlers,
               host='localhost', port=80, use_delays=False, is_ssl=False,
               protocol='HTTP',
               down_bandwidth='0', up_bandwidth='0', delay_ms='0'):
    """Start HTTP server.

    Args:
      host: a host string (name or IP) for the web proxy.
      port: a port string (e.g. '80') for the web proxy.
      use_delays: if True, add response data delays during replay.
      is_ssl: True iff proxy is using SSL.
      up_bandwidth: Upload bandwidth
      down_bandwidth: Download bandwidth
           Bandwidths measured in [K|M]{bit/s|Byte/s}. '0' means unlimited.
      delay_ms: Propagation delay in milliseconds. '0' means no delay.
    """
    try:
      BaseHTTPServer.HTTPServer.__init__(self, (host, port), self.HANDLER)
    except Exception, e:
      raise HttpProxyServerError('Could not start HTTPServer on port %d: %s' %
                                 (port, e))
    self.http_archive_fetch = http_archive_fetch
    self.custom_handlers = custom_handlers
    self.use_delays = use_delays
    self.is_ssl = is_ssl
    self.traffic_shaping_down_bps = proxyshaper.GetBitsPerSecond(down_bandwidth)
    self.traffic_shaping_up_bps = proxyshaper.GetBitsPerSecond(up_bandwidth)
    self.traffic_shaping_delay_ms = int(delay_ms)
    self.num_active_requests = 0
    self.total_request_time = 0
    self.protocol = protocol

    # Note: This message may be scraped. Do not change it.
    logging.warning(
        '%s server started on %s:%d' % (self.protocol, self.server_address[0],
                                        self.server_address[1]))

  def cleanup(self):
    try:
      self.shutdown()
    except KeyboardInterrupt:
      pass
    logging.info('Stopped %s server. Total time processing requests: %dms',
                 self.protocol, self.total_request_time)

  def get_active_request_count(self):
    return self.num_active_requests

class HttpsProxyServer(HttpProxyServer):
  """SSL server that generates certs for each host."""

  def __init__(self, http_archive_fetch, custom_handlers,
               https_root_ca_cert_path, **kwargs):
    self.ca_cert_path = https_root_ca_cert_path
    self.HANDLER = sslproxy.wrap_handler(HttpArchiveHandler)
    HttpProxyServer.__init__(self, http_archive_fetch, custom_handlers,
                             is_ssl=True, protocol='HTTPS', **kwargs)
    self.http_archive_fetch.http_archive.set_root_cert(https_root_ca_cert_path)

  def cleanup(self):
    try:
      self.shutdown()
    except KeyboardInterrupt:
      pass

class SingleCertHttpsProxyServer(HttpProxyServer):
  """SSL server."""

  def __init__(self, http_archive_fetch, custom_handlers,
               https_root_ca_cert_path, **kwargs):
    HttpProxyServer.__init__(self, http_archive_fetch, custom_handlers,
                             is_ssl=True, protocol='HTTPS', **kwargs)
    self.socket = ssl.wrap_socket(
        self.socket, certfile=https_root_ca_cert_path, server_side=True,
        do_handshake_on_connect=False)
    # Ancestor class, DaemonServer, calls serve_forever() during its __init__.

class HttpToHttpsProxyServer(HttpProxyServer):
  """Listens for HTTP requests but sends them to the target as HTTPS requests"""

  def __init__(self, http_archive_fetch, custom_handlers, **kwargs):
    HttpProxyServer.__init__(self, http_archive_fetch, custom_handlers,
                             is_ssl=True, protocol='HTTP-to-HTTPS', **kwargs)
<|endoftext|>


<commit_before># Author: Tony DiCola
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in
# all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
# THE SOFTWARE.

# Can enable debug output by uncommenting:
#import logging
#logging.basicConfig(level=logging.DEBUG)

from time import time, sleep
import random

import Adafruit_GPIO.SPI as SPI
import Adafruit_MAX31855.MAX31855 as MAX31855

# Define a function to convert celsius to fahrenheit.
def c_to_f(c):
        return c * 9.0 / 5.0 + 32.0

# Uncomment one of the blocks of code below to configure your Pi or BBB to use
# software or hardware SPI.

# Raspberry Pi software SPI configuration.
#CLK = 25
#CS  = 24
#DO  = 18
#sensor = MAX31855.MAX31855(CLK, CS, DO)

# Raspberry Pi hardware SPI configuration.
SPI_PORT   = 0
SPI_DEVICE = 0
sensor = MAX31855.MAX31855(spi=SPI.SpiDev(SPI_PORT, SPI_DEVICE))

# BeagleBone Black software SPI configuration.
#CLK = 'P9_12'
#CS  = 'P9_15'
#DO  = 'P9_23'
#sensor = MAX31855.MAX31855(CLK, CS, DO)

# BeagleBone Black hardware SPI configuration.
#SPI_PORT   = 1
#SPI_DEVICE = 0
#sensor = MAX31855.MAX31855(spi=SPI.SpiDev(SPI_PORT, SPI_DEVICE))

# Loop printing measurements every second.
print('Press Ctrl-C to quit.')
while True:
    temp = sensor.readTempC()
    internal = sensor.readInternalC()
    #internal = random.uniform(70.0, 120.0)

    print('{0:0.3F},0,{1:0.3F}'.format(time(), temp ))
    print ('{0:0.3F},internal,{1:0.3F}'.format(time(), internal ))
    #print('Thermocouple Temperature: {0:0.3F}*C / {1:0.3F}*F'.format(temp, c_to_f(temp)))
    #print('    Internal Temperature: {0:0.3F}*C / {1:0.3F}*F'.format(internal, c_to_f(internal)))

    #v = sensor._read32()
    #if v & 0x7:
    #    print "{0:0.3F},internal.error,{1:0.1F}".format(time(), float( v & 0x7))

    sleep(1.0)<commit_msg>Read random.<commit_after># Author: Tony DiCola
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in
# all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
# THE SOFTWARE.

# Can enable debug output by uncommenting:
#import logging
#logging.basicConfig(level=logging.DEBUG)

from time import time, sleep
import random

import Adafruit_GPIO.SPI as SPI
import Adafruit_MAX31855.MAX31855 as MAX31855

# Define a function to convert celsius to fahrenheit.
def c_to_f(c):
        return c * 9.0 / 5.0 + 32.0

# Uncomment one of the blocks of code below to configure your Pi or BBB to use
# software or hardware SPI.

# Raspberry Pi software SPI configuration.
#CLK = 25
#CS  = 24
#DO  = 18
#sensor = MAX31855.MAX31855(CLK, CS, DO)

# Raspberry Pi hardware SPI configuration.
SPI_PORT   = 0
SPI_DEVICE = 0
sensor = MAX31855.MAX31855(spi=SPI.SpiDev(SPI_PORT, SPI_DEVICE))

# BeagleBone Black software SPI configuration.
#CLK = 'P9_12'
#CS  = 'P9_15'
#DO  = 'P9_23'
#sensor = MAX31855.MAX31855(CLK, CS, DO)

# BeagleBone Black hardware SPI configuration.
#SPI_PORT   = 1
#SPI_DEVICE = 0
#sensor = MAX31855.MAX31855(spi=SPI.SpiDev(SPI_PORT, SPI_DEVICE))

# Loop printing measurements every second.
print('Press Ctrl-C to quit.')
while True:
    temp = sensor.readTempC()
    #internal = sensor.readInternalC()
    internal = random.uniform(70.0, 120.0)

    #print('{0:0.3F},0,{1:0.3F}'.format(time(), temp ))
    print ('{0:0.3F},internal,{1:0.3F}'.format(time(), internal ))
    #print('Thermocouple Temperature: {0:0.3F}*C / {1:0.3F}*F'.format(temp, c_to_f(temp)))
    #print('    Internal Temperature: {0:0.3F}*C / {1:0.3F}*F'.format(internal, c_to_f(internal)))

    #v = sensor._read32()
    #if v & 0x7:
    #    print "{0:0.3F},internal.error,{1:0.1F}".format(time(), float( v & 0x7))

    sleep(1.0)<|endoftext|>


<commit_before># Copyright 2015 Google Inc. All Rights Reserved.

# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at

#     http://www.apache.org/licenses/LICENSE-2.0

# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import unittest
import os.path
import datetime
import time
import openhtf
import mock
import sys

from openhtf.util import timeouts
from openhtf.util import configuration

class TestUtil(unittest.TestCase):
  def __init__(self, unittest_name):
    super(TestUtil, self).__init__(unittest_name)

  @classmethod
  def setUpClass(cls):
    TestUtil.time_out = 60
    TestUtil.time_sleep = 3
    TestUtil.polledtimeout = timeouts.PolledTimeout(TestUtil.time_out)
    TestUtil.testlog_dir = '/var/run/openhtf/unittest_log'
    if not os.path.exists(TestUtil.testlog_dir):
      os.makedirs(TestUtil.testlog_dir)

  @classmethod
  def tearDownClass(cls):
    pass

  def test_timeExpired_false(self):
    time.sleep(TestUtil.time_sleep)
    TestUtil.polledtimeout.Restart()
    sec = TestUtil.polledtimeout.seconds
    self.assertLessEqual(sec, TestUtil.time_out - TestUtil.time_sleep)
    self.assertFalse(TestUtil.polledtimeout.HasExpired())
    print "....test_timeExpired_false."

  def test_timeExpired_true(self):
    TestUtil.polledtimeout.Expire()
    self.assertTrue(TestUtil.polledtimeout.HasExpired())
    print "....test_timeExpired_true."

def testcase_runner(testlog_dir):
  testcase_file =  os.path.dirname(os.path.abspath(__file__))+'/testcases.txt'
  suite = unittest.TestSuite()

  try:
    f = open(testcase_file, "r")
  except IOError:
      print "WARNING: %s does not exist, quit test" % testcase_file

  tests = f.readlines()

  for test in tests:
    if test.startswith('test'):
      suite.addTest(TestUtil(test.strip()))

  test_result = unittest.TextTestRunner().run(suite)
  now =  datetime.datetime.now().strftime("%m-%d-%d_%H:%M:%S")
  classname = os.path.dirname(os.path.abspath(__file__)).split('/')[-1]

  if testlog_dir:
    log = testlog_dir + '/'+ classname.strip() + '_' + now

    with open(log, 'w') as f:
      f.write("====== Test Result =====\n")
      f.write(str(test_result))
      f.write("\n\n====== FAILE ======\n")
      f.write(str(test_result.failures))<commit_msg>remove import from test_util.py<commit_after># Copyright 2015 Google Inc. All Rights Reserved.

# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at

#     http://www.apache.org/licenses/LICENSE-2.0

# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import unittest
import os.path
import datetime
import time
import openhtf
import mock
import sys

from openhtf.util import timeouts

class TestUtil(unittest.TestCase):
  def __init__(self, unittest_name):
    super(TestUtil, self).__init__(unittest_name)

  @classmethod
  def setUpClass(cls):
    TestUtil.time_out = 60
    TestUtil.time_sleep = 3
    TestUtil.polledtimeout = timeouts.PolledTimeout(TestUtil.time_out)
    TestUtil.testlog_dir = '/var/run/openhtf/unittest_log'
    if not os.path.exists(TestUtil.testlog_dir):
      os.makedirs(TestUtil.testlog_dir)

  @classmethod
  def tearDownClass(cls):
    pass

  def test_timeExpired_false(self):
    time.sleep(TestUtil.time_sleep)
    TestUtil.polledtimeout.Restart()
    sec = TestUtil.polledtimeout.seconds
    self.assertLessEqual(sec, TestUtil.time_out - TestUtil.time_sleep)
    self.assertFalse(TestUtil.polledtimeout.HasExpired())
    print "....test_timeExpired_false."

  def test_timeExpired_true(self):
    TestUtil.polledtimeout.Expire()
    self.assertTrue(TestUtil.polledtimeout.HasExpired())
    print "....test_timeExpired_true."

def testcase_runner(testlog_dir):
  testcase_file =  os.path.dirname(os.path.abspath(__file__))+'/testcases.txt'
  suite = unittest.TestSuite()

  try:
    f = open(testcase_file, "r")
  except IOError:
      print "WARNING: %s does not exist, quit test" % testcase_file

  tests = f.readlines()

  for test in tests:
    if test.startswith('test'):
      suite.addTest(TestUtil(test.strip()))

  test_result = unittest.TextTestRunner().run(suite)
  now =  datetime.datetime.now().strftime("%m-%d-%d_%H:%M:%S")
  classname = os.path.dirname(os.path.abspath(__file__)).split('/')[-1]

  if testlog_dir:
    log = testlog_dir + '/'+ classname.strip() + '_' + now

    with open(log, 'w') as f:
      f.write("====== Test Result =====\n")
      f.write(str(test_result))
      f.write("\n\n====== FAILE ======\n")
      f.write(str(test_result.failures))<|endoftext|>


<commit_before># Copyright (C) 2020 The Android Open Source Project
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

from urllib.parse import urlparse
from typing import BinaryIO, Generator, List, Optional, Union

from .http import TraceProcessorHttp
from .loader import get_loader
from .protos import ProtoFactory
from .shell import load_shell

# Union of types supported for a trace which can be loaded by shell.
LoadableTrace = Union[None, str, BinaryIO, Generator[bytes, None, None]]

# Custom exception raised if any trace_processor functions return a
# response with an error defined
class TraceProcessorException(Exception):

  def __init__(self, message):
    super().__init__(message)

class TraceProcessor:

  # Values of these constants correspond to the QueryResponse message at
  # protos/perfetto/trace_processor/trace_processor.proto
  QUERY_CELL_INVALID_FIELD_ID = 0
  QUERY_CELL_NULL_FIELD_ID = 1
  QUERY_CELL_VARINT_FIELD_ID = 2
  QUERY_CELL_FLOAT64_FIELD_ID = 3
  QUERY_CELL_STRING_FIELD_ID = 4
  QUERY_CELL_BLOB_FIELD_ID = 5

  # This is the class returned to the user and contains one row of the
  # resultant query. Each column name is stored as an attribute of this
  # class, with the value corresponding to the column name and row in
  # the query results table.
  class Row(object):

    def __str__(self):
      return str(self.__dict__)

    def __repr__(self):
      return self.__dict__

  class QueryResultIterator:

    def __init__(self, column_names, batches):
      self.__column_names = column_names
      self.__column_count = 0
      self.__count = 0
      self.__cells = []
      self.__data_lists = [[], [], [], [], [], []]
      self.__data_lists_index = [0, 0, 0, 0, 0, 0]
      self.__current_index = 0

      # Iterate over all the batches and collect their
      # contents into lists based on the type of the batch
      batch_index = 0
      while True:
        # It's possible on some occasions that there are non UTF-8 characters
        # in the string_cells field. If this is the case, string_cells is
        # a bytestring which needs to be decoded (but passing ignore so that
        # we don't fail in decoding).
        strings_batch_str = batches[batch_index].string_cells
        try:
          strings_batch_str = strings_batch_str.decode('utf-8', 'ignore')
        except AttributeError:
          # AttributeError can occur when |strings_batch_str| is an str which
          # happens when everything in it is UTF-8 (protobuf automatically
          # does the conversion if it can).
          pass

        # Null-terminated strings in a batch are concatenated
        # into a single large byte array, so we split on the
        # null-terminator to get the individual strings
        strings_batch = strings_batch_str.split('\0')[:-1]
        self.__data_lists[TraceProcessor.QUERY_CELL_STRING_FIELD_ID].extend(
            strings_batch)
        self.__data_lists[TraceProcessor.QUERY_CELL_VARINT_FIELD_ID].extend(
            batches[batch_index].varint_cells)
        self.__data_lists[TraceProcessor.QUERY_CELL_FLOAT64_FIELD_ID].extend(
            batches[batch_index].float64_cells)
        self.__data_lists[TraceProcessor.QUERY_CELL_BLOB_FIELD_ID].extend(
            batches[batch_index].blob_cells)
        self.__cells.extend(batches[batch_index].cells)

        if batches[batch_index].is_last_batch:
          break
        batch_index += 1

      # If there are no rows in the query result, exit since
      # we won't be returning anything and don't need to go
      # through the process of generating the mapping between
      # columns and data_list indices
      if len(self.__cells) == 0:
        return

      # The count we collected so far was a count of all individual columns
      # in the query result, so we divide by the number of columns in a row
      # to get the number of rows
      self.__column_count = len(self.__column_names)
      self.__count = int(len(self.__cells) / self.__column_count)

      # Data integrity check - see that we have the expected amount of cells
      # for the number of rows that we need to return
      if len(self.__cells) % self.__column_count != 0:
        raise TraceProcessorException("Cell count " + str(len(self.__cells)) +
                                      " is not a multiple of column count " +
                                      str(len(self.__column_names)))

    # To use the query result as a populated Pandas dataframe, this
    # function must be called directly after calling query inside
    # TraceProcesor.
    def as_pandas_dataframe(self):
      try:
        import pandas as pd

        # Populate the dataframe with the query results
        rows = []
        for i in range(0, self.__count):
          row = []
          base_cell_index = i * self.__column_count
          for num in range(len(self.__column_names)):
            col_type = self.__cells[base_cell_index + num]
            if col_type == TraceProcessor.QUERY_CELL_INVALID_FIELD_ID:
              raise TraceProcessorException('Invalid cell type')

            if col_type == TraceProcessor.QUERY_CELL_NULL_FIELD_ID:
              row.append(None)
            else:
              col_index = self.__data_lists_index[col_type]
              self.__data_lists_index[col_type] += 1
              row.append(self.__data_lists[col_type][col_index])
          rows.append(row)

        df = pd.DataFrame(rows, columns=self.__column_names)
        return df.astype(object).where(df.notnull(),
                                       None).reset_index(drop=True)

      except ModuleNotFoundError:
        raise TraceProcessorException(
            'Python dependencies missing. Please pip3 install pandas numpy')

    def __len__(self):
      return self.__count

    def __iter__(self):
      return self

    def __next__(self):
      if self.__current_index == self.__count:
        raise StopIteration
      result = TraceProcessor.Row()
      base_cell_index = self.__current_index * self.__column_count
      for num, column_name in enumerate(self.__column_names):
        col_type = self.__cells[base_cell_index + num]
        if col_type == TraceProcessor.QUERY_CELL_INVALID_FIELD_ID:
          raise TraceProcessorException('Invalid cell type')
        if col_type != TraceProcessor.QUERY_CELL_NULL_FIELD_ID:
          col_index = self.__data_lists_index[col_type]
          self.__data_lists_index[col_type] += 1
          setattr(result, column_name, self.__data_lists[col_type][col_index])
        else:
          setattr(result, column_name, None)

      self.__current_index += 1
      return result

  def __init__(self,
               trace: LoadableTrace = None,
               addr: Optional[str] = None,
               bin_path: Optional[str] = None,
               unique_port: bool = True,
               verbose: bool = False,
               file_path: Optional[str] = None):
    """Create a trace processor instance.

    Args:
      trace: Trace to be loaded into the trace processor instance. One of
        three types of argument is supported:
        1) path to a trace file to open and read
        2) a file like object (file, io.BytesIO or similar) to read
        3) a generator yielding bytes
      addr: address of a running trace processor instance. For advanced
        use only.
      bin_path: path to a trace processor shell binary. For advanced use
        only.
      unique_port: whether the trace processor shell instance should be
        be started on a unique port. Only used when |addr| is not set.
        For advanced use only.
      verbose: whether trace processor shell should emit verbose logs;
        can be very spammy. For advanced use only.
      file_path (deprecated): path to a trace file to load. Please use
        |trace| instead of this field: specifying both will cause
        an exception to be thrown.
    """

    def create_tp_http():
      if addr:
        p = urlparse(addr)
        return TraceProcessorHttp(p.netloc if p.netloc else p.path)

      url, self.subprocess = load_shell(
          bin_path=bin_path, unique_port=unique_port, verbose=verbose)
      return TraceProcessorHttp(url)

    if trace and file_path:
      raise TraceProcessorException(
          "trace and file_path cannot both be specified.")

    self.http = create_tp_http()
    self.protos = ProtoFactory()

    if file_path:
      get_loader().parse_file(self.http, file_path)
    elif isinstance(trace, str):
      get_loader().parse_file(self.http, trace)
    elif hasattr(trace, 'read'):
      while True:
        chunk = trace.read()
        if not chunk:
          break
        self.http.parse(chunk)
      self.http.notify_eof()
    elif trace:
      for chunk in trace:
        self.http.parse(chunk)
      self.http.notify_eof()

  def query(self, sql: str):
    """Executes passed in SQL query using class defined HTTP API, and returns
    the response as a QueryResultIterator. Raises TraceProcessorException if
    the response returns with an error.

    Args:
      sql: SQL query written as a String

    Returns:
      A class which can iterate through each row of the results table. This
      can also be converted to a pandas dataframe by calling the
      as_pandas_dataframe() function after calling query.
    """
    response = self.http.execute_query(sql)
    if response.error:
      raise TraceProcessorException(response.error)

    return TraceProcessor.QueryResultIterator(response.column_names,
                                              response.batch)

  def metric(self, metrics: List[str]):
    """Returns the metrics data corresponding to the passed in trace metric.
    Raises TraceProcessorException if the response returns with an error.

    Args:
      metrics: A list of valid metrics as defined in TraceMetrics

    Returns:
      The metrics data as a proto message
    """
    response = self.http.compute_metric(metrics)
    if response.error:
      raise TraceProcessorException(response.error)

    metrics = self.protos.TraceMetrics()
    metrics.ParseFromString(response.metrics)
    return metrics

  def enable_metatrace(self):
    """Enable metatrace for the currently running trace_processor.
    """
    return self.http.enable_metatrace()

  def disable_and_read_metatrace(self):
    """Disable and return the metatrace formed from the currently running
    trace_processor. This must be enabled before attempting to disable. This
    returns the serialized bytes of the metatrace data directly. Raises
    TraceProcessorException if the response returns with an error.
    """
    response = self.http.disable_and_read_metatrace()
    if response.error:
      raise TraceProcessorException(response.error)

    return response.metatrace

  def __enter__(self):
    return self

  def __exit__(self, a, b, c):
    del a, b, c  # Unused.
    self.close()
    return False

  def close(self):
    if hasattr(self, 'subprocess'):
      self.subprocess.kill()
    self.http.conn.close()
<commit_msg>tp_py: fix missing chunk size for file reads am: 2c3a01f1e4 am: 1f2d4a5170<commit_after># Copyright (C) 2020 The Android Open Source Project
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

from urllib.parse import urlparse
from typing import BinaryIO, Generator, List, Optional, Union

from .http import TraceProcessorHttp
from .loader import get_loader
from .protos import ProtoFactory
from .shell import load_shell

# Union of types supported for a trace which can be loaded by shell.
LoadableTrace = Union[None, str, BinaryIO, Generator[bytes, None, None]]

# Custom exception raised if any trace_processor functions return a
# response with an error defined
class TraceProcessorException(Exception):

  def __init__(self, message):
    super().__init__(message)

class TraceProcessor:

  # Values of these constants correspond to the QueryResponse message at
  # protos/perfetto/trace_processor/trace_processor.proto
  QUERY_CELL_INVALID_FIELD_ID = 0
  QUERY_CELL_NULL_FIELD_ID = 1
  QUERY_CELL_VARINT_FIELD_ID = 2
  QUERY_CELL_FLOAT64_FIELD_ID = 3
  QUERY_CELL_STRING_FIELD_ID = 4
  QUERY_CELL_BLOB_FIELD_ID = 5

  # This is the class returned to the user and contains one row of the
  # resultant query. Each column name is stored as an attribute of this
  # class, with the value corresponding to the column name and row in
  # the query results table.
  class Row(object):

    def __str__(self):
      return str(self.__dict__)

    def __repr__(self):
      return self.__dict__

  class QueryResultIterator:

    def __init__(self, column_names, batches):
      self.__column_names = column_names
      self.__column_count = 0
      self.__count = 0
      self.__cells = []
      self.__data_lists = [[], [], [], [], [], []]
      self.__data_lists_index = [0, 0, 0, 0, 0, 0]
      self.__current_index = 0

      # Iterate over all the batches and collect their
      # contents into lists based on the type of the batch
      batch_index = 0
      while True:
        # It's possible on some occasions that there are non UTF-8 characters
        # in the string_cells field. If this is the case, string_cells is
        # a bytestring which needs to be decoded (but passing ignore so that
        # we don't fail in decoding).
        strings_batch_str = batches[batch_index].string_cells
        try:
          strings_batch_str = strings_batch_str.decode('utf-8', 'ignore')
        except AttributeError:
          # AttributeError can occur when |strings_batch_str| is an str which
          # happens when everything in it is UTF-8 (protobuf automatically
          # does the conversion if it can).
          pass

        # Null-terminated strings in a batch are concatenated
        # into a single large byte array, so we split on the
        # null-terminator to get the individual strings
        strings_batch = strings_batch_str.split('\0')[:-1]
        self.__data_lists[TraceProcessor.QUERY_CELL_STRING_FIELD_ID].extend(
            strings_batch)
        self.__data_lists[TraceProcessor.QUERY_CELL_VARINT_FIELD_ID].extend(
            batches[batch_index].varint_cells)
        self.__data_lists[TraceProcessor.QUERY_CELL_FLOAT64_FIELD_ID].extend(
            batches[batch_index].float64_cells)
        self.__data_lists[TraceProcessor.QUERY_CELL_BLOB_FIELD_ID].extend(
            batches[batch_index].blob_cells)
        self.__cells.extend(batches[batch_index].cells)

        if batches[batch_index].is_last_batch:
          break
        batch_index += 1

      # If there are no rows in the query result, exit since
      # we won't be returning anything and don't need to go
      # through the process of generating the mapping between
      # columns and data_list indices
      if len(self.__cells) == 0:
        return

      # The count we collected so far was a count of all individual columns
      # in the query result, so we divide by the number of columns in a row
      # to get the number of rows
      self.__column_count = len(self.__column_names)
      self.__count = int(len(self.__cells) / self.__column_count)

      # Data integrity check - see that we have the expected amount of cells
      # for the number of rows that we need to return
      if len(self.__cells) % self.__column_count != 0:
        raise TraceProcessorException("Cell count " + str(len(self.__cells)) +
                                      " is not a multiple of column count " +
                                      str(len(self.__column_names)))

    # To use the query result as a populated Pandas dataframe, this
    # function must be called directly after calling query inside
    # TraceProcesor.
    def as_pandas_dataframe(self):
      try:
        import pandas as pd

        # Populate the dataframe with the query results
        rows = []
        for i in range(0, self.__count):
          row = []
          base_cell_index = i * self.__column_count
          for num in range(len(self.__column_names)):
            col_type = self.__cells[base_cell_index + num]
            if col_type == TraceProcessor.QUERY_CELL_INVALID_FIELD_ID:
              raise TraceProcessorException('Invalid cell type')

            if col_type == TraceProcessor.QUERY_CELL_NULL_FIELD_ID:
              row.append(None)
            else:
              col_index = self.__data_lists_index[col_type]
              self.__data_lists_index[col_type] += 1
              row.append(self.__data_lists[col_type][col_index])
          rows.append(row)

        df = pd.DataFrame(rows, columns=self.__column_names)
        return df.astype(object).where(df.notnull(),
                                       None).reset_index(drop=True)

      except ModuleNotFoundError:
        raise TraceProcessorException(
            'Python dependencies missing. Please pip3 install pandas numpy')

    def __len__(self):
      return self.__count

    def __iter__(self):
      return self

    def __next__(self):
      if self.__current_index == self.__count:
        raise StopIteration
      result = TraceProcessor.Row()
      base_cell_index = self.__current_index * self.__column_count
      for num, column_name in enumerate(self.__column_names):
        col_type = self.__cells[base_cell_index + num]
        if col_type == TraceProcessor.QUERY_CELL_INVALID_FIELD_ID:
          raise TraceProcessorException('Invalid cell type')
        if col_type != TraceProcessor.QUERY_CELL_NULL_FIELD_ID:
          col_index = self.__data_lists_index[col_type]
          self.__data_lists_index[col_type] += 1
          setattr(result, column_name, self.__data_lists[col_type][col_index])
        else:
          setattr(result, column_name, None)

      self.__current_index += 1
      return result

  def __init__(self,
               trace: LoadableTrace = None,
               addr: Optional[str] = None,
               bin_path: Optional[str] = None,
               unique_port: bool = True,
               verbose: bool = False,
               file_path: Optional[str] = None):
    """Create a trace processor instance.

    Args:
      trace: Trace to be loaded into the trace processor instance. One of
        three types of argument is supported:
        1) path to a trace file to open and read
        2) a file like object (file, io.BytesIO or similar) to read
        3) a generator yielding bytes
      addr: address of a running trace processor instance. For advanced
        use only.
      bin_path: path to a trace processor shell binary. For advanced use
        only.
      unique_port: whether the trace processor shell instance should be
        be started on a unique port. Only used when |addr| is not set.
        For advanced use only.
      verbose: whether trace processor shell should emit verbose logs;
        can be very spammy. For advanced use only.
      file_path (deprecated): path to a trace file to load. Please use
        |trace| instead of this field: specifying both will cause
        an exception to be thrown.
    """

    def create_tp_http():
      if addr:
        p = urlparse(addr)
        return TraceProcessorHttp(p.netloc if p.netloc else p.path)

      url, self.subprocess = load_shell(
          bin_path=bin_path, unique_port=unique_port, verbose=verbose)
      return TraceProcessorHttp(url)

    if trace and file_path:
      raise TraceProcessorException(
          "trace and file_path cannot both be specified.")

    self.http = create_tp_http()
    self.protos = ProtoFactory()

    if file_path:
      get_loader().parse_file(self.http, file_path)
    elif isinstance(trace, str):
      get_loader().parse_file(self.http, trace)
    elif hasattr(trace, 'read'):
      while True:
        chunk = trace.read(32 * 1024 * 1024)
        if not chunk:
          break
        self.http.parse(chunk)
      self.http.notify_eof()
    elif trace:
      for chunk in trace:
        self.http.parse(chunk)
      self.http.notify_eof()

  def query(self, sql: str):
    """Executes passed in SQL query using class defined HTTP API, and returns
    the response as a QueryResultIterator. Raises TraceProcessorException if
    the response returns with an error.

    Args:
      sql: SQL query written as a String

    Returns:
      A class which can iterate through each row of the results table. This
      can also be converted to a pandas dataframe by calling the
      as_pandas_dataframe() function after calling query.
    """
    response = self.http.execute_query(sql)
    if response.error:
      raise TraceProcessorException(response.error)

    return TraceProcessor.QueryResultIterator(response.column_names,
                                              response.batch)

  def metric(self, metrics: List[str]):
    """Returns the metrics data corresponding to the passed in trace metric.
    Raises TraceProcessorException if the response returns with an error.

    Args:
      metrics: A list of valid metrics as defined in TraceMetrics

    Returns:
      The metrics data as a proto message
    """
    response = self.http.compute_metric(metrics)
    if response.error:
      raise TraceProcessorException(response.error)

    metrics = self.protos.TraceMetrics()
    metrics.ParseFromString(response.metrics)
    return metrics

  def enable_metatrace(self):
    """Enable metatrace for the currently running trace_processor.
    """
    return self.http.enable_metatrace()

  def disable_and_read_metatrace(self):
    """Disable and return the metatrace formed from the currently running
    trace_processor. This must be enabled before attempting to disable. This
    returns the serialized bytes of the metatrace data directly. Raises
    TraceProcessorException if the response returns with an error.
    """
    response = self.http.disable_and_read_metatrace()
    if response.error:
      raise TraceProcessorException(response.error)

    return response.metatrace

  def __enter__(self):
    return self

  def __exit__(self, a, b, c):
    del a, b, c  # Unused.
    self.close()
    return False

  def close(self):
    if hasattr(self, 'subprocess'):
      self.subprocess.kill()
    self.http.conn.close()
<|endoftext|>


<commit_before>"""
http://adventofcode.com/day/6

--- Day 6: Probably a Fire Hazard ---

Because your neighbors keep defeating you in the holiday house decorating
contest year after year, you've decided to deploy one million lights in a
1000x1000 grid.

Furthermore, because you've been especially nice this year, Santa has mailed you
instructions on how to display the ideal lighting configuration.

Lights in your grid are numbered from 0 to 999 in each direction; the lights at
each corner are at 0,0, 0,999, 999,999, and 999,0. The instructions include
whether to turn on, turn off, or toggle various inclusive ranges given as
coordinate pairs. Each coordinate pair represents opposite corners of a
rectangle, inclusive; a coordinate pair like 0,0 through 2,2 therefore refers to
9 lights in a 3x3 square. The lights all start turned off.

To defeat your neighbors this year, all you have to do is set up your lights by
doing the instructions Santa sent you in order.

For example:

    - turn on 0,0 through 999,999 would turn on (or leave on) every light.
    - toggle 0,0 through 999,0 would toggle the first line of 1000 lights,
      turning off the ones that were on, and turning on the ones that were off.
    - turn off 499,499 through 500,500 would turn off (or leave off) the middle
      four lights.

After following the instructions, how many lights are lit?

--- Part Two ---

You just finish implementing your winning light pattern when you realize you
mistranslated Santa's message from Ancient Nordic Elvish.

The light grid you bought actually has individual brightness controls; each
light can have a brightness of zero or more. The lights all start at zero.

The phrase turn on actually means that you should increase the brightness of
those lights by 1.

The phrase turn off actually means that you should decrease the brightness of
those lights by 1, to a minimum of zero.

The phrase toggle actually means that you should increase the brightness of
those lights by 2.

What is the total brightness of all lights combined after following Santa's
instructions?

For example:

    - turn on 0,0 through 0,0 would increase the total brightness by 1.
    - toggle 0,0 through 999,999 would increase the total brightness by 2000000.

"""
import re

import numpy as np

PATTERN = re.compile(r"([\w\s]+)\s(\d+),(\d+) through (\d+),(\d+)")

def test_parse_instruction():
    assert parse_instruction("turn on 0,0 through 999,999") == (
        "turn on",
        slice(0, 1000),
        slice(0, 1000),
    )
    assert parse_instruction("toggle 0,0 through 999,0") == (
        "toggle",
        slice(0, 1000),
        slice(0, 1),
    )
    assert parse_instruction("turn off 499,499 through 500,500") == (
        "turn off",
        slice(499, 501),
        slice(499, 501),
    )

def test_follow_instruction():
    orig_array = np.zeros((1000, 1000), dtype=np.bool)
    true_array = np.ones((1000, 1000), dtype=np.bool)

    new_array = follow_instruction("turn on 0,0 through 999,999", orig_array)
    assert new_array.all()
    new_array = follow_instruction("turn on 0,0 through 999,999", true_array)
    assert new_array.all()

    new_array = follow_instruction("turn off 499,499 through 500,500",
                                   orig_array)
    assert not new_array[499:500, 499:500].any()
    new_array = follow_instruction("turn off 499,499 through 500,500",
                                   true_array)
    assert not new_array[499:500, 499:500].any()

    new_array = follow_instruction("toggle 499,499 through 500,500", orig_array)
    assert new_array[499:500, 499:500].all()
    new_array = follow_instruction("toggle 499,499 through 500,500", true_array)
    assert not new_array[499:500, 499:500].any()

    new_array = follow_instruction("toggle 0,0 through 999,0", orig_array)
    assert new_array[:, 0].all()
    new_array = follow_instruction("toggle 0,0 through 999,0", true_array)
    assert not new_array[:, 0].any()

def test_follow_elvish():
    orig_array = np.zeros((1000, 1000))

    new_array = follow_elvish("turn off 499,499 through 500,500", orig_array)
    assert new_array.sum() == 0
    new_array = follow_elvish("turn on 499,499 through 500,500", orig_array)
    assert new_array.sum() == 4
    new_array = follow_elvish("toggle 499,499 through 500,500", orig_array)
    assert new_array.sum() == 8
    new_array = follow_elvish("turn on 0,0 through 0,0", orig_array)
    assert new_array.sum() == 1
    new_array = follow_elvish("toggle 0,0 through 999,999", orig_array)
    assert new_array.sum() == 2000000

def parse_instruction(instruction):
    match = re.search(PATTERN, instruction)
    command, xi, yi, xf, yf = match.groups()
    return command, slice(int(xi), int(xf)+1), slice(int(yi), int(yf)+1)

def follow_instruction(instruction, array):
    command, sx, sy = parse_instruction(instruction)
    new_array = array.copy()

    if command in ("turn off", "turn on"):
        new_array[sx, sy] = ["turn off", "turn on"].index(command)
    elif command == "toggle":
        new_array[sx, sy] ^= 1

    return new_array

def follow_elvish(instruction, array):
    command, sx, sy = parse_instruction(instruction)
    new_array = array.copy()

    new_array[sx, sy] += {"turn off": -1, "turn on": 1, "toggle": 2}[command]
    new_array[new_array < 0] = 0

    return new_array

def part_one():
    with open("inputs/day_06_input.txt", "r") as input_file:
        array = np.zeros((1000, 1000), dtype=np.bool)
        for instruction in input_file:
            command, sx, sy = parse_instruction(instruction)
            if command in ("turn off", "turn on"):
                array[sx, sy] = ["turn off", "turn on"].index(command)
            elif command == "toggle":
                array[sx, sy] ^= 1

    print("{} lights on".format(array.sum()))

def part_two():
    with open("inputs/day_06_input.txt", "r") as input_file:
        array = np.zeros((1000, 1000))
        for instruction in input_file:
            command, sx, sy = parse_instruction(instruction)
            array[sx, sy] += {"turn off": -1,
                              "turn on": 1,
                              "toggle": 2}[command]
            array[array < 0] = 0

    print("{} total brightness".format(array.sum()))

def main():
    with open("inputs/day_06_input.txt", "r") as input_file:
        grid = np.zeros((1000, 1000), dtype=np.bool)
        elvish = np.zeros((1000, 1000))
        for instruction in input_file:
            command, sx, sy = parse_instruction(instruction)
            if command in ("turn off", "turn on"):
                grid[sx, sy] = ["turn off", "turn on"].index(command)
            elif command == "toggle":
                grid[sx, sy] ^= 1
            elvish[sx, sy] += {"turn off": -1,
                               "turn on": 1,
                               "toggle": 2}[command]
            elvish[elvish < 0] = 0

    print("{} lights on".format(grid.sum()))
    print("{} total brightness".format(elvish.sum()))

if __name__ == "__main__":
    main()
<commit_msg>Use all functions.<commit_after>"""
http://adventofcode.com/day/6

--- Day 6: Probably a Fire Hazard ---

Because your neighbors keep defeating you in the holiday house decorating
contest year after year, you've decided to deploy one million lights in a
1000x1000 grid.

Furthermore, because you've been especially nice this year, Santa has mailed you
instructions on how to display the ideal lighting configuration.

Lights in your grid are numbered from 0 to 999 in each direction; the lights at
each corner are at 0,0, 0,999, 999,999, and 999,0. The instructions include
whether to turn on, turn off, or toggle various inclusive ranges given as
coordinate pairs. Each coordinate pair represents opposite corners of a
rectangle, inclusive; a coordinate pair like 0,0 through 2,2 therefore refers to
9 lights in a 3x3 square. The lights all start turned off.

To defeat your neighbors this year, all you have to do is set up your lights by
doing the instructions Santa sent you in order.

For example:

    - turn on 0,0 through 999,999 would turn on (or leave on) every light.
    - toggle 0,0 through 999,0 would toggle the first line of 1000 lights,
      turning off the ones that were on, and turning on the ones that were off.
    - turn off 499,499 through 500,500 would turn off (or leave off) the middle
      four lights.

After following the instructions, how many lights are lit?

--- Part Two ---

You just finish implementing your winning light pattern when you realize you
mistranslated Santa's message from Ancient Nordic Elvish.

The light grid you bought actually has individual brightness controls; each
light can have a brightness of zero or more. The lights all start at zero.

The phrase turn on actually means that you should increase the brightness of
those lights by 1.

The phrase turn off actually means that you should decrease the brightness of
those lights by 1, to a minimum of zero.

The phrase toggle actually means that you should increase the brightness of
those lights by 2.

What is the total brightness of all lights combined after following Santa's
instructions?

For example:

    - turn on 0,0 through 0,0 would increase the total brightness by 1.
    - toggle 0,0 through 999,999 would increase the total brightness by 2000000.

"""
import re

import numpy as np

PATTERN = re.compile(r"([\w\s]+)\s(\d+),(\d+) through (\d+),(\d+)")

def test_parse_instruction():
    assert parse_instruction("turn on 0,0 through 999,999") == (
        "turn on",
        slice(0, 1000),
        slice(0, 1000),
    )
    assert parse_instruction("toggle 0,0 through 999,0") == (
        "toggle",
        slice(0, 1000),
        slice(0, 1),
    )
    assert parse_instruction("turn off 499,499 through 500,500") == (
        "turn off",
        slice(499, 501),
        slice(499, 501),
    )

def test_follow_instruction():
    orig_array = np.zeros((1000, 1000), dtype=np.bool)
    true_array = np.ones((1000, 1000), dtype=np.bool)
    new_array = follow_instruction("turn on 0,0 through 999,999", orig_array)
    assert new_array.all()
    new_array = follow_instruction("turn on 0,0 through 999,999", true_array)
    assert new_array.all()

    orig_array = np.zeros((1000, 1000), dtype=np.bool)
    true_array = np.ones((1000, 1000), dtype=np.bool)
    new_array = follow_instruction("turn off 499,499 through 500,500",
                                   orig_array)
    assert not new_array[499:500, 499:500].any()
    new_array = follow_instruction("turn off 499,499 through 500,500",
                                   true_array)
    assert not new_array[499:500, 499:500].any()

    orig_array = np.zeros((1000, 1000), dtype=np.bool)
    true_array = np.ones((1000, 1000), dtype=np.bool)
    new_array = follow_instruction("toggle 499,499 through 500,500", orig_array)
    assert new_array[499:500, 499:500].all()
    new_array = follow_instruction("toggle 499,499 through 500,500", true_array)
    assert not new_array[499:500, 499:500].any()

    orig_array = np.zeros((1000, 1000), dtype=np.bool)
    true_array = np.ones((1000, 1000), dtype=np.bool)
    new_array = follow_instruction("toggle 0,0 through 999,0", orig_array)
    assert new_array[:, 0].all()
    new_array = follow_instruction("toggle 0,0 through 999,0", true_array)
    assert not new_array[:, 0].any()

def test_follow_elvish():
    orig_array = np.zeros((1000, 1000))
    new_array = follow_elvish("turn off 499,499 through 500,500", orig_array)
    assert new_array.sum() == 0
    orig_array = np.zeros((1000, 1000))
    new_array = follow_elvish("turn on 499,499 through 500,500", orig_array)
    assert new_array.sum() == 4
    orig_array = np.zeros((1000, 1000))
    new_array = follow_elvish("toggle 499,499 through 500,500", orig_array)
    assert new_array.sum() == 8
    orig_array = np.zeros((1000, 1000))
    new_array = follow_elvish("turn on 0,0 through 0,0", orig_array)
    assert new_array.sum() == 1
    orig_array = np.zeros((1000, 1000))
    new_array = follow_elvish("toggle 0,0 through 999,999", orig_array)
    assert new_array.sum() == 2000000

def parse_instruction(instruction):
    match = re.search(PATTERN, instruction)
    command, xi, yi, xf, yf = match.groups()
    return command, slice(int(xi), int(xf)+1), slice(int(yi), int(yf)+1)

def follow_instruction(instruction, array):
    command, sx, sy = parse_instruction(instruction)

    if command in ("turn off", "turn on"):
        array[sx, sy] = ["turn off", "turn on"].index(command)
    elif command == "toggle":
        array[sx, sy] ^= 1

    return array

def follow_elvish(instruction, array):
    command, sx, sy = parse_instruction(instruction)

    array[sx, sy] += {"turn off": -1, "turn on": 1, "toggle": 2}[command]
    array[array < 0] = 0

    return array

def part_one():
    with open("inputs/day_06_input.txt", "r") as input_file:
        array = np.zeros((1000, 1000), dtype=np.bool)
        for instruction in input_file:
            command, sx, sy = parse_instruction(instruction)
            if command in ("turn off", "turn on"):
                array[sx, sy] = ["turn off", "turn on"].index(command)
            elif command == "toggle":
                array[sx, sy] ^= 1

    print("{} lights on".format(array.sum()))

def part_two():
    with open("inputs/day_06_input.txt", "r") as input_file:
        array = np.zeros((1000, 1000))
        for instruction in input_file:
            command, sx, sy = parse_instruction(instruction)
            array[sx, sy] += {"turn off": -1,
                              "turn on": 1,
                              "toggle": 2}[command]
            array[array < 0] = 0

    print("{} total brightness".format(array.sum()))

def main():
    part_one()
    part_two()

if __name__ == "__main__":
    main()
<|endoftext|>


<commit_before># Copyright (c) 2013 The Pycroft Authors. See the AUTHORS file.
# This file is part of the Pycroft project and licensed under the terms of
# the Apache License, Version 2.0. See the LICENSE file for details.
from tests import FixtureDataTestBase
from tests.lib.fixtures.property_fixtures import UserData, PropertyGroupData,\
    PropertyData, MembershipData, TrafficGroupData

from pycroft.lib.property import create_membership, create_property,\
    create_property_group, create_traffic_group, delete_membership,\
    delete_property, delete_property_group, delete_traffic_group, _create_group,\
    _delete_group

from pycroft.model.property import TrafficGroup, PropertyGroup, Property,\
    Membership, Group
from pycroft.model.user import User
from pycroft.model import session

from datetime import datetime, timedelta
from sqlalchemy import Column, ForeignKey
from sqlalchemy.types import Integer

class Test_010_PropertyGroup(FixtureDataTestBase):
    datasets = [PropertyGroupData]

    def test_0010_create_property_group(self):
        name = "dummy_property_group2"

        property_group = create_property_group(name=name)

        self.assertIsNotNone(PropertyGroup.q.get(property_group.id))

        db_property_group = PropertyGroup.q.get(property_group.id)

        self.assertEqual(db_property_group.name, name)

        session.session.delete(db_property_group)
        session.session.commit()

    def test_0020_delete_property_group(self):
        del_property_group = delete_property_group(
            PropertyGroupData.dummy_property_group1.id)

        self.assertIsNone(PropertyGroup.q.get(del_property_group.id))

    def test_0025_delete_wrong_property_group(self):
        self.assertRaises(ValueError, delete_property_group,
            PropertyGroupData.dummy_property_group1.id + 100)

class Test_020_TrafficGroup(FixtureDataTestBase):
    datasets = [TrafficGroupData]

    def test_0010_create_traffic_group(self):
        name = "dummy_traffic_group2"
        traffic_limit = 100000

        traffic_group = create_traffic_group(name=name,
            traffic_limit=traffic_limit)

        self.assertIsNotNone(TrafficGroup.q.get(traffic_group.id))

        db_traffic_group = TrafficGroup.q.get(traffic_group.id)

        self.assertEqual(db_traffic_group.name, name)
        self.assertEqual(db_traffic_group.traffic_limit, traffic_limit)

        session.session.delete(db_traffic_group)
        session.session.commit()

    def test_0020_delete_traffic_group(self):
        del_traffic_group = delete_traffic_group(
            TrafficGroupData.dummy_traffic_group1.id)

        self.assertIsNone(TrafficGroup.q.get(del_traffic_group.id))

    def test_0025_delete_wrong_traffic_group(self):
        self.assertRaises(ValueError, delete_traffic_group,
            TrafficGroupData.dummy_traffic_group1.id + 100)

class Test_030_Membership(FixtureDataTestBase):
    datasets = [MembershipData, PropertyGroupData, UserData]

    def test_0010_create_membership(self):
        start_date = datetime.now()
        end_date = datetime.now() + timedelta(hours=1)
        group = PropertyGroup.q.first()
        user = User.q.first()

        membership = create_membership(start_date=start_date, end_date=end_date,
            group=group, user=user)

        self.assertIsNotNone(Membership.q.get(membership.id))

        db_membership = Membership.q.get(membership.id)

        self.assertEqual(db_membership.start_date, start_date)
        self.assertEqual(db_membership.end_date, end_date)
        self.assertEqual(db_membership.group, group)
        self.assertEqual(db_membership.user, user)

        session.session.delete(db_membership)
        session.session.commit()

    def test_0020_delete_membership(self):
        del_membership = delete_membership(MembershipData.dummy_membership1.id)

        self.assertIsNone(Membership.q.get(del_membership.id))

    def test_0025_delete_wrong_membership(self):
        self.assertRaises(ValueError, delete_membership,
            MembershipData.dummy_membership1.id + 100)

class Test_040_Property(FixtureDataTestBase):
    datasets = [PropertyGroupData, PropertyData]

    def test_0010_create_property(self):
        name = "dummy_property2"
        property_group = PropertyGroup.q.first()

        (_, property) = create_property(property_group=property_group,
                                        name=name, granted=True)

        self.assertIsNotNone(Property.q.get(property.id))

        db_property = Property.q.get(property.id)

        self.assertEqual(db_property.name, name)
        self.assertEqual(db_property.property_group, property_group)

        session.session.delete(db_property)
        session.session.commit()

    def test_0020_delete_property(self):
        property_name = PropertyData.dummy_property1.name
        group_id = PropertyData.dummy_property1.property_group.id

        (_, del_property) = delete_property(property_group_id=group_id,
                                            name=property_name)

        self.assertIsNone(Property.q.get(del_property.id))

    def test_0025_delete_wrong_property(self):
        property_name = PropertyData.dummy_property1.name
        group_id = PropertyData.dummy_property1.property_group.id
        empty_group_id = PropertyGroupData.dummy_property_group2.id

        self.assertRaises(ValueError, delete_property,
                          property_group_id=group_id,
                          name=property_name + "_fail")
        self.assertRaises(ValueError, delete_property,
                          property_group_id=group_id + 100,
                          name=property_name)
        self.assertRaises(ValueError, delete_property,
                          property_group_id=empty_group_id,
                          name=property_name)

class Test_050_MalformedGroup(FixtureDataTestBase):
    datasets = [UserData]

    class MalformedGroup(Group):
        id = Column(Integer, ForeignKey("group.id"), primary_key=True)
        __mapper_args__ = {'polymorphic_identity': 'malformed_group'}

    def test_0010_create_malformed_group(self):
        name = "malformed_group1"

        self.assertRaises(ValueError, _create_group, type='malformed_group',
            name=name, id=100)

    def test_0020_delete_malformed_group(self):
        malformed_group = Test_050_MalformedGroup.MalformedGroup(id=1000,
            name="malformed_group2")

        session.session.add(malformed_group)
        session.session.commit()

        self.assertRaises(ValueError, _delete_group, malformed_group.id)

        session.session.delete(malformed_group)
        session.session.commit()

<commit_msg>fix forgotten rename in test_property.py<commit_after># Copyright (c) 2014 The Pycroft Authors. See the AUTHORS file.
# This file is part of the Pycroft project and licensed under the terms of
# the Apache License, Version 2.0. See the LICENSE file for details.
from tests import FixtureDataTestBase
from tests.lib.fixtures.property_fixtures import UserData, PropertyGroupData,\
    PropertyData, MembershipData, TrafficGroupData

from pycroft.lib.property import create_membership, create_property,\
    create_property_group, create_traffic_group, delete_membership,\
    delete_property, delete_property_group, delete_traffic_group, _create_group,\
    _delete_group

from pycroft.model.property import TrafficGroup, PropertyGroup, Property,\
    Membership, Group
from pycroft.model.user import User
from pycroft.model import session

from datetime import datetime, timedelta
from sqlalchemy import Column, ForeignKey
from sqlalchemy.types import Integer

class Test_010_PropertyGroup(FixtureDataTestBase):
    datasets = [PropertyGroupData]

    def test_0010_create_property_group(self):
        name = "dummy_property_group2"

        property_group = create_property_group(name=name)

        self.assertIsNotNone(PropertyGroup.q.get(property_group.id))

        db_property_group = PropertyGroup.q.get(property_group.id)

        self.assertEqual(db_property_group.name, name)

        session.session.delete(db_property_group)
        session.session.commit()

    def test_0020_delete_property_group(self):
        del_property_group = delete_property_group(
            PropertyGroupData.dummy_property_group1.id)

        self.assertIsNone(PropertyGroup.q.get(del_property_group.id))

    def test_0025_delete_wrong_property_group(self):
        self.assertRaises(ValueError, delete_property_group,
            PropertyGroupData.dummy_property_group1.id + 100)

class Test_020_TrafficGroup(FixtureDataTestBase):
    datasets = [TrafficGroupData]

    def test_0010_create_traffic_group(self):
        name = "dummy_traffic_group2"
        traffic_limit = 100000

        traffic_group = create_traffic_group(name=name,
            traffic_limit=traffic_limit)

        self.assertIsNotNone(TrafficGroup.q.get(traffic_group.id))

        db_traffic_group = TrafficGroup.q.get(traffic_group.id)

        self.assertEqual(db_traffic_group.name, name)
        self.assertEqual(db_traffic_group.traffic_limit, traffic_limit)

        session.session.delete(db_traffic_group)
        session.session.commit()

    def test_0020_delete_traffic_group(self):
        del_traffic_group = delete_traffic_group(
            TrafficGroupData.dummy_traffic_group1.id)

        self.assertIsNone(TrafficGroup.q.get(del_traffic_group.id))

    def test_0025_delete_wrong_traffic_group(self):
        self.assertRaises(ValueError, delete_traffic_group,
            TrafficGroupData.dummy_traffic_group1.id + 100)

class Test_030_Membership(FixtureDataTestBase):
    datasets = [MembershipData, PropertyGroupData, UserData]

    def test_0010_create_membership(self):
        start_date = datetime.now()
        end_date = datetime.now() + timedelta(hours=1)
        group = PropertyGroup.q.first()
        user = User.q.first()

        membership = create_membership(start_date=start_date, end_date=end_date,
            group=group, user=user)

        self.assertIsNotNone(Membership.q.get(membership.id))

        db_membership = Membership.q.get(membership.id)

        self.assertEqual(db_membership.start_date, start_date)
        self.assertEqual(db_membership.end_date, end_date)
        self.assertEqual(db_membership.group, group)
        self.assertEqual(db_membership.user, user)

        session.session.delete(db_membership)
        session.session.commit()

    def test_0020_delete_membership(self):
        del_membership = delete_membership(MembershipData.dummy_membership1.id)

        self.assertIsNone(Membership.q.get(del_membership.id))

    def test_0025_delete_wrong_membership(self):
        self.assertRaises(ValueError, delete_membership,
            MembershipData.dummy_membership1.id + 100)

class Test_040_Property(FixtureDataTestBase):
    datasets = [PropertyGroupData, PropertyData]

    def test_0010_create_property(self):
        name = "dummy_property2"
        property_group = PropertyGroup.q.first()

        (_, property) = create_property(property_group=property_group,
                                        name=name, granted=True)

        self.assertIsNotNone(Property.q.get(property.id))

        db_property = Property.q.get(property.id)

        self.assertEqual(db_property.name, name)
        self.assertEqual(db_property.property_group, property_group)

        session.session.delete(db_property)
        session.session.commit()

    def test_0020_delete_property(self):
        property_name = PropertyData.dummy_property1.name
        group_id = PropertyData.dummy_property1.property_group.id

        (_, del_property) = delete_property(property_group_id=group_id,
                                            name=property_name)

        self.assertIsNone(Property.q.get(del_property.id))

    def test_0025_delete_wrong_property(self):
        property_name = PropertyData.dummy_property1.name
        group_id = PropertyData.dummy_property1.property_group.id
        empty_group_id = PropertyGroupData.dummy_property_group2.id

        self.assertRaises(ValueError, delete_property,
                          property_group_id=group_id,
                          name=property_name + "_fail")
        self.assertRaises(ValueError, delete_property,
                          property_group_id=group_id + 100,
                          name=property_name)
        self.assertRaises(ValueError, delete_property,
                          property_group_id=empty_group_id,
                          name=property_name)

class Test_050_MalformedGroup(FixtureDataTestBase):
    datasets = [UserData]

    class MalformedGroup(Group):
        id = Column(Integer, ForeignKey("group.id"), primary_key=True)
        __mapper_args__ = {'polymorphic_identity': 'malformed_group'}

    def test_0010_create_malformed_group(self):
        name = "malformed_group1"

        self.assertRaises(ValueError, _create_group,
            group_type='malformed_group',
            name=name, id=100)

    def test_0020_delete_malformed_group(self):
        malformed_group = Test_050_MalformedGroup.MalformedGroup(id=1000,
            name="malformed_group2")

        session.session.add(malformed_group)
        session.session.commit()

        self.assertRaises(ValueError, _delete_group, malformed_group.id)

        session.session.delete(malformed_group)
        session.session.commit()

<|endoftext|>


<commit_before>import numpy as np
import tensorflow as tf
import h5py
import time
import matplotlib
matplotlib.use('Agg')
import matplotlib.pyplot as plt

print('==> Experiment 1b')
filepath = '../taylorswift_out/small.mat'
print('==> Loading data from {}'.format(filepath))
# benchmark
t_start = time.time()

# reading data
f = h5py.File(filepath)
data_train = np.array(f.get('trainingSet'))
X_train = data_train[:, :-1]
y_train = data_train[:, -1].reshape(-1, 1)
data_test = np.array(f.get('testSet'))
X_test = data_test[:, :-1]
y_test = data_test[:, -1].reshape(-1, 1)
t_end = time.time()
print('--Time elapsed for loading data: {t:.2f} \
    seconds'.format(t = t_end - t_start))
del data_train, data_test, f
print('-- Number of training samples: {}'.format(X_train.shape[0]))
print('-- Number of test samples: {}'.format(X_test.shape[0]))

# Neural-network model set-up
# Functions for initializing neural nets parameters
def init_weight_variable(shape):
  initial = tf.truncated_normal(shape, stddev=0.1, dtype=tf.float64)
  return tf.Variable(initial)

def init_bias_variable(shape):
  initial = tf.constant(0.1, shape=shape, dtype=tf.float64)
  return tf.Variable(initial)

'''
	NN config parameters
'''
num_featuers = 121
hidden_layer_size = 20
num_classes = int(max(y_train.max(), y_test.max()) + 1)

# Transform labels into on-hot encoding form
y_train_OHEnc = tf.one_hot(y_train.copy(), num_classes)
y_test_OHEnc = tf.one_hot(y_test.copy(), num_classes)

# Set-up NN layers
x = tf.placeholder(tf.float64, [None, num_featuers])
W1 = init_weight_variable([num_featuers, hidden_layer_size])
b1 = init_bias_variable([hidden_layer_size])

# Hidden layer activation function: ReLU
h1 = tf.nn.relu(tf.matmul(x, W1) + b1)

W2 = init_weight_variable([hidden_layer_size, num_classes])
b2 = init_bias_variable([num_classes])

# Softmax layer (Output), dtype = float64
y = tf.matmul(h1, W2) + b2

# NN desired value (labels)
y_ = tf.placeholder(tf.float64, [None, num_classes])

# Loss function
cross_entropy = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(labels=y_, logits=y))
train_step = tf.train.AdamOptimizer(1e-4).minimize(cross_entropy)

sess = tf.InteractiveSession()

correct_prediction = tf.equal(tf.argmax(y, 1), tf.argmax(y_, 1))
accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float64))
sess.run(tf.global_variables_initializer())

y_train = sess.run(y_train_OHEnc)[:, 0, :]
y_test = sess.run(y_test_OHEnc)[:, 0, :]

'''
	Training config
'''
numTrainingVec = len(X_train)
batchSize = 1000
numEpochs = 300
print_freq = 5

train_acc_list = []
val_acc_list = []
train_err_list = []
val_err_list = []

# benchmark
t_start = time.time()
for epoch in range(numEpochs):
    for i in range(0,numTrainingVec,batchSize):

        # Batch Data
        batchEndPoint = min(i + batchSize, numTrainingVec)
        trainBatchData = X_train[i:batchEndPoint]
        trainBatchLabel = y_train[i:batchEndPoint]

        # train
        train_step.run(feed_dict={x: trainBatchData, y_: trainBatchLabel})

    # Print accuracy
    if (epoch + 1) % print_freq == 0:
        # evaluate accuracy and errors
        train_acc = accuracy.eval(feed_dict={x:X_train, y_: y_train})
        train_acc_list.append(train_acc)
        val_acc = accuracy.eval(feed_dict={x: X_test, y_: y_test})
        val_acc_list.append(val_acc)
        train_err = cross_entropy.eval(feed_dict={x: X_train, y_: y_train})
        train_err_list.append(train_err)
        val_err = cross_entropy.eval(feed_dict={x: X_test, y_: y_test})
        val_err_list.append(val_err)
        print("-- epoch: %d, training accuracy %g"%(epoch + 1, train_acc))

t_end = time.time()
print('--Time elapsed for training: {t:.2f} \
    seconds'.format(t = t_end - t_start))

# Reports
print('-- Training accuracy: {:.4f}'.format(train_acc_list[-1]))
print('-- Validation accuracy: {:.4f}'.format(val_acc_list[-1]))
print('-- Training error: {:.4E}'.format(train_err_list[-1]))
print('-- Validation error: {:.4E}'.format(val_err_list[-1]))

# Generating plots

print('==> Generating accuracy plot...')
x_list = range(0, print_freq * len(train_acc_list), print_freq)
plt.xlabel('Number of epochs')
plt.ylabel('Accuracy (%)')
plt.title('Accuracy vs Number of Epochs')
train_acc_plot, = plt.plot(x_list, train_acc_list, 'bo')
val_acc_plot, = plt.plot(x_list, val_acc_list, 'ro')
plt.legend((train_acc_plot, val_acc_plot), ('training', 'validation'), loc='best')
plt.savefig('exp1b_accuracy.png', format='png')
plt.close()

print('==> Generating error plot...')
plt.xlabel('Number of epochs')
plt.ylabel('Cross-Entropy Error')
plt.title('Error vs Number of Epochs')
train_err_plot, = plt.plot(x_list, train_err_list, 'bo')
val_err_plot, = plt.plot(x_list, val_err_list, 'ro')
plt.legend((train_err_plot, val_err_plot), ('training', 'validation'), loc='best')
plt.savefig('exp1b_error.png', format='png')
plt.close()

print('==> Done.')

<commit_msg>Neuron updates on #4<commit_after>import numpy as np
import tensorflow as tf
import h5py
import time
import matplotlib
matplotlib.use('Agg')
import matplotlib.pyplot as plt

print('==> Experiment 1b')
filepath = '../taylorswift_out/small.mat'
print('==> Loading data from {}'.format(filepath))
# benchmark
t_start = time.time()

# reading data
f = h5py.File(filepath)
data_train = np.array(f.get('trainingSet'))
X_train = data_train[:, :-1]
y_train = data_train[:, -1].reshape(-1, 1)
data_test = np.array(f.get('testSet'))
X_test = data_test[:, :-1]
y_test = data_test[:, -1].reshape(-1, 1)
t_end = time.time()
print('--Time elapsed for loading data: {t:.2f} \
    seconds'.format(t = t_end - t_start))
del data_train, data_test, f
print('-- Number of training samples: {}'.format(X_train.shape[0]))
print('-- Number of test samples: {}'.format(X_test.shape[0]))

# Neural-network model set-up
# Functions for initializing neural nets parameters
def init_weight_variable(shape):
  initial = tf.truncated_normal(shape, stddev=0.1, dtype=tf.float64)
  return tf.Variable(initial)

def init_bias_variable(shape):
  initial = tf.constant(0.1, shape=shape, dtype=tf.float64)
  return tf.Variable(initial)

'''
	NN config parameters
'''
num_featuers = 121
hidden_layer_size = 80
num_classes = int(max(y_train.max(), y_test.max()) + 1)

# Transform labels into on-hot encoding form
y_train_OHEnc = tf.one_hot(y_train.copy(), num_classes)
y_test_OHEnc = tf.one_hot(y_test.copy(), num_classes)

# Set-up NN layers
x = tf.placeholder(tf.float64, [None, num_featuers])
W1 = init_weight_variable([num_featuers, hidden_layer_size])
b1 = init_bias_variable([hidden_layer_size])

# Hidden layer activation function: ReLU
h1 = tf.nn.relu(tf.matmul(x, W1) + b1)

W2 = init_weight_variable([hidden_layer_size, num_classes])
b2 = init_bias_variable([num_classes])

# Softmax layer (Output), dtype = float64
y = tf.matmul(h1, W2) + b2

# NN desired value (labels)
y_ = tf.placeholder(tf.float64, [None, num_classes])

# Loss function
cross_entropy = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(labels=y_, logits=y))
train_step = tf.train.AdamOptimizer(1e-4).minimize(cross_entropy)

sess = tf.InteractiveSession()

correct_prediction = tf.equal(tf.argmax(y, 1), tf.argmax(y_, 1))
accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float64))
sess.run(tf.global_variables_initializer())

y_train = sess.run(y_train_OHEnc)[:, 0, :]
y_test = sess.run(y_test_OHEnc)[:, 0, :]

'''
	Training config
'''
numTrainingVec = len(X_train)
batchSize = 1000
numEpochs = 1200
print_freq = 5

train_acc_list = []
val_acc_list = []
train_err_list = []
val_err_list = []

# benchmark
t_start = time.time()
for epoch in range(numEpochs):
    for i in range(0,numTrainingVec,batchSize):

        # Batch Data
        batchEndPoint = min(i + batchSize, numTrainingVec)
        trainBatchData = X_train[i:batchEndPoint]
        trainBatchLabel = y_train[i:batchEndPoint]

        # train
        train_step.run(feed_dict={x: trainBatchData, y_: trainBatchLabel})

    # Print accuracy
    if (epoch + 1) % print_freq == 0:
        # evaluate accuracy and errors
        train_acc = accuracy.eval(feed_dict={x:X_train, y_: y_train})
        train_acc_list.append(train_acc)
        val_acc = accuracy.eval(feed_dict={x: X_test, y_: y_test})
        val_acc_list.append(val_acc)
        train_err = cross_entropy.eval(feed_dict={x: X_train, y_: y_train})
        train_err_list.append(train_err)
        val_err = cross_entropy.eval(feed_dict={x: X_test, y_: y_test})
        val_err_list.append(val_err)
        print("-- epoch: %d, training accuracy %g"%(epoch + 1, train_acc))

t_end = time.time()
print('--Time elapsed for training: {t:.2f} \
    seconds'.format(t = t_end - t_start))

# Reports
print('-- Training accuracy: {:.4f}'.format(train_acc_list[-1]))
print('-- Validation accuracy: {:.4f}'.format(val_acc_list[-1]))
print('-- Training error: {:.4E}'.format(train_err_list[-1]))
print('-- Validation error: {:.4E}'.format(val_err_list[-1]))

# Generating plots

print('==> Generating accuracy plot...')
x_list = range(0, print_freq * len(train_acc_list), print_freq)
plt.xlabel('Number of epochs')
plt.ylabel('Accuracy (%)')
plt.title('Accuracy vs Number of Epochs')
train_acc_plot, = plt.plot(x_list, train_acc_list, 'bo')
val_acc_plot, = plt.plot(x_list, val_acc_list, 'ro')
plt.legend((train_acc_plot, val_acc_plot), ('training', 'validation'), loc='best')
plt.savefig('exp1b_accuracy.png', format='png')
plt.close()

print('==> Generating error plot...')
plt.xlabel('Number of epochs')
plt.ylabel('Cross-Entropy Error')
plt.title('Error vs Number of Epochs')
train_err_plot, = plt.plot(x_list, train_err_list, 'bo')
val_err_plot, = plt.plot(x_list, val_err_list, 'ro')
plt.legend((train_err_plot, val_err_plot), ('training', 'validation'), loc='best')
plt.savefig('exp1b_error.png', format='png')
plt.close()

print('==> Done.')

<|endoftext|>