<commit_before>from __future__ import print_function
import os
import sys
import socket
import traceback
import struct
import cPickle
import base64

class SpecialLengths(object):
    PYTHON_EXCEPTION_THROWN = -3
    END_OF_STREAM = -4
    NULL = -5

class DataType(object):
    INT = 1
    LONG = 2
    FLOAT = 3
    DOUBLE = 4
    BLOB = 5

class Serializer(object):

    @staticmethod
    def read_long(stream):
        obj = stream.read(8)
        if not obj:
            raise EOFError
        return struct.unpack("!q", obj)[0]

    @staticmethod
    def read_float(stream):
        obj = stream.read(4)
        if not obj:
            raise EOFError
        return struct.unpack("!f", obj)[0]

    @staticmethod
    def read_double(stream):
        obj = stream.read(8)
        if not obj:
            raise EOFError
        return struct.unpack("!d", obj)[0]

    @staticmethod
    def read_int(stream):
        obj = stream.read(4)
        if not obj:
            raise EOFError
        return struct.unpack("!i", obj)[0]

    @staticmethod
    def write_int(value, stream):
        stream.write(struct.pack("!i", value))

    @staticmethod
    def write_float(value, stream):
        stream.write(struct.pack("!f", value))

    @staticmethod
    def write_double(value, stream):
        stream.write(struct.pack("!d", value))

    @staticmethod
    def write_long(value, stream):
        stream.write(struct.pack("!q", value))

class PickleSerializer(Serializer):

    @classmethod
    def read_item(cls, stream, item_type, length):
        obj = None
        if item_type == DataType.INT:
            obj = cls.read_int(stream)
        elif item_type == DataType.LONG:
            obj = cls.read_long(stream)
        elif item_type == DataType.FLOAT:
            obj = cls.read_float(stream)
        elif item_type == DataType.DOUBLE:
            obj = cls.read_double(stream)
        elif item_type == DataType.BLOB:
            obj = cls.loads(stream.read(length))
        return obj

    @classmethod
    def read_tuple(cls, stream, tuplesize):
        datalist = []
        for i in range(tuplesize):
            # first element read type
            element_type = cls.read_int(stream)
            # Second read the length
            length = cls.read_int(stream)

            if length == SpecialLengths.NULL or length == 0:
                datalist.append(0)
            # length is > 0, read the item now
            elif length > 0:
                obj = cls.read_item(stream, element_type, length)
                datalist.append(obj)
            else:
                raise ValueError("Invalid length for item.")

        return datalist

    @classmethod
    def write_with_length(cls, obj, stream, output_type):
        if output_type == DataType.INT:
            cls.write_int(DataType.INT, stream)
            cls.write_int(obj, stream)
        elif output_type == DataType.LONG:
            cls.write_int(DataType.LONG, stream)
            cls.write_long(obj, stream)
        elif output_type == DataType.FLOAT:
            cls.write_int(DataType.FLOAT, stream)
            cls.write_float(obj, stream)
        elif output_type == DataType.DOUBLE:
            cls.write_int(DataType.DOUBLE, stream)
            cls.write_double(obj, stream)
        elif output_type == DataType.BLOB:
            cls.write_int(DataType.BLOB, stream)
            cls.pickle_and_write(obj, stream)

    @classmethod
    def read_command(cls, stream):
        length = cls.read_int(stream)

        if length < 0:
            raise ValueError("Command cannot be less than zero.")
        s = stream.read(length)

        if len(s) < length:
            raise EOFError
        unenc = base64.urlsafe_b64decode(s)
        return cls.loads(unenc)

    @staticmethod
    def dumps(obj):
        protocol = 2
        return cPickle.dumps(obj, protocol)

    @staticmethod
    def loads(obj):
        return cPickle.loads(obj)

    @classmethod
    def pickle_and_write(cls, obj, stream):
        serialized = cls.dumps(obj)

        if serialized is None:
            raise ValueError("Serialized value should not be None.")
        elif len(serialized) > (1 << 31):
            raise ValueError("Cannot serialize object larger than 2G.")

        cls.write_int(len(serialized), stream)
        stream.write(serialized)

def main(in_file, out_file):
    pickle_serializer = PickleSerializer()
    try:
        func = pickle_serializer.read_command(in_file)
        tuple_size = pickle_serializer.read_int(in_file)
        output_type = pickle_serializer.read_int(in_file)
        is_flatmap = pickle_serializer.read_int(in_file)

        if tuple_size < 1:
            raise ValueError("Size of tuple should not be less than 1.")

        while True:
            num_tuples = pickle_serializer.read_int(in_file)
            if num_tuples == SpecialLengths.END_OF_STREAM:
                break

            tuple_list = []
            for j in range(num_tuples):
                tuple_list.append(pickle_serializer.read_tuple(in_file, tuple_size))

            retval = func(tuple_list)
            if is_flatmap > 0:
                count = len(retval)
                pickle_serializer.write_int(count, out_file)
                for i in range(count):
                    pickle_serializer.write_with_length(
                        retval[i], out_file, output_type)
            else:
                pickle_serializer.write_with_length(retval, out_file, output_type)

            out_file.flush()

    except Exception:
        try:
            pickle_serializer.write_int(SpecialLengths.PYTHON_EXCEPTION_THROWN, out_file)
            pickle_serializer.write_with_length(traceback.format_exc().encode(
                "utf-8"), out_file, DataType.BLOB)
            print(traceback.format_exc(), file=sys.stderr)
        except IOError:
            # JVM closed the socket
            print("IOError: \n{}".
                  format(traceback.format_exc()), file=sys.stderr)
        except Exception:
            print("Python worker process failed with exception:\n{}".
                  format(traceback.format_exc()), file=sys.stderr)
        exit(-1)

if __name__ == '__main__':
    # Read a local port to connect to from stdin
    port_number = int(sys.stdin.readline())
    sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
    sock.connect(("127.0.0.1", port_number))
    with os.fdopen(os.dup(sock.fileno()), "rb", 65536) as infile,\
            os.fdopen(os.dup(sock.fileno()), "wb", 65536) as outfile:
        main(infile, outfile)

<commit_msg>code review nitpicks<commit_after>from __future__ import print_function
import os
import sys
import socket
import traceback
import struct
import cPickle
import base64

class SpecialLengths(object):
    PYTHON_EXCEPTION_THROWN = -3
    END_OF_STREAM = -4
    NULL = -5

class DataType(object):
    INT = 1
    LONG = 2
    FLOAT = 3
    DOUBLE = 4
    BLOB = 5

class Serializer(object):

    @staticmethod
    def read_long(stream):
        obj = stream.read(8)
        if not obj:
            raise EOFError
        return struct.unpack("!q", obj)[0]

    @staticmethod
    def read_float(stream):
        obj = stream.read(4)
        if not obj:
            raise EOFError
        return struct.unpack("!f", obj)[0]

    @staticmethod
    def read_double(stream):
        obj = stream.read(8)
        if not obj:
            raise EOFError
        return struct.unpack("!d", obj)[0]

    @staticmethod
    def read_int(stream):
        obj = stream.read(4)
        if not obj:
            raise EOFError
        return struct.unpack("!i", obj)[0]

    @staticmethod
    def write_int(value, stream):
        stream.write(struct.pack("!i", value))

    @staticmethod
    def write_float(value, stream):
        stream.write(struct.pack("!f", value))

    @staticmethod
    def write_double(value, stream):
        stream.write(struct.pack("!d", value))

    @staticmethod
    def write_long(value, stream):
        stream.write(struct.pack("!q", value))

class PickleSerializer(Serializer):

    @classmethod
    def read_item(cls, stream, item_type, length):
        obj = None
        if item_type == DataType.INT:
            obj = cls.read_int(stream)
        elif item_type == DataType.LONG:
            obj = cls.read_long(stream)
        elif item_type == DataType.FLOAT:
            obj = cls.read_float(stream)
        elif item_type == DataType.DOUBLE:
            obj = cls.read_double(stream)
        elif item_type == DataType.BLOB:
            obj = cls.loads(stream.read(length))
        return obj

    @classmethod
    def read_tuple(cls, stream, tuplesize):
        datalist = []
        for _ in range(tuplesize):
            # first element read type
            element_type = cls.read_int(stream)
            # Second read the length
            length = cls.read_int(stream)

            if length == SpecialLengths.NULL or length == 0:
                datalist.append(0)
            # length is > 0, read the item now
            elif length > 0:
                obj = cls.read_item(stream, element_type, length)
                datalist.append(obj)
            else:
                raise ValueError("Invalid length for item.")

        return datalist

    @classmethod
    def write_with_length(cls, obj, stream, output_type):
        if output_type == DataType.INT:
            cls.write_int(DataType.INT, stream)
            cls.write_int(obj, stream)
        elif output_type == DataType.LONG:
            cls.write_int(DataType.LONG, stream)
            cls.write_long(obj, stream)
        elif output_type == DataType.FLOAT:
            cls.write_int(DataType.FLOAT, stream)
            cls.write_float(obj, stream)
        elif output_type == DataType.DOUBLE:
            cls.write_int(DataType.DOUBLE, stream)
            cls.write_double(obj, stream)
        elif output_type == DataType.BLOB:
            cls.write_int(DataType.BLOB, stream)
            cls.pickle_and_write(obj, stream)

    @classmethod
    def read_command(cls, stream):
        length = cls.read_int(stream)

        if length < 0:
            raise ValueError("Command length cannot be less than zero.")
        s = stream.read(length)

        if len(s) < length:
            raise EOFError
        unenc = base64.urlsafe_b64decode(s)
        return cls.loads(unenc)

    @staticmethod
    def dumps(obj):
        protocol = 2
        return cPickle.dumps(obj, protocol)

    @staticmethod
    def loads(obj):
        return cPickle.loads(obj)

    @classmethod
    def pickle_and_write(cls, obj, stream):
        serialized = cls.dumps(obj)

        if serialized is None:
            raise ValueError("Serialized value should not be None.")
        elif len(serialized) > (1 << 31):
            raise ValueError("Cannot serialize object larger than 2G.")

        cls.write_int(len(serialized), stream)
        stream.write(serialized)

def main(in_file, out_file):
    pickle_serializer = PickleSerializer()
    try:
        func = pickle_serializer.read_command(in_file)
        tuple_size = pickle_serializer.read_int(in_file)
        output_type = pickle_serializer.read_int(in_file)
        is_flatmap = pickle_serializer.read_int(in_file)

        if tuple_size < 1:
            raise ValueError("Size of tuple should not be less than 1.")

        while True:
            num_tuples = pickle_serializer.read_int(in_file)
            if num_tuples == SpecialLengths.END_OF_STREAM:
                break

            tuple_list = []
            for _ in range(num_tuples):
                tuple_list.append(pickle_serializer.read_tuple(in_file, tuple_size))

            retval = func(tuple_list)
            if is_flatmap > 0:
                count = len(retval)
                pickle_serializer.write_int(count, out_file)
                for i in range(count):
                    pickle_serializer.write_with_length(
                        retval[i], out_file, output_type)
            else:
                pickle_serializer.write_with_length(retval, out_file, output_type)

            out_file.flush()

    except Exception:
        try:
            pickle_serializer.write_int(SpecialLengths.PYTHON_EXCEPTION_THROWN, out_file)
            pickle_serializer.write_with_length(traceback.format_exc().encode(
                "utf-8"), out_file, DataType.BLOB)
            print(traceback.format_exc(), file=sys.stderr)
        except IOError:
            # JVM closed the socket
            print("IOError:\n{}".
                  format(traceback.format_exc()), file=sys.stderr)
        except Exception:
            print("Python worker process failed with exception:\n{}".
                  format(traceback.format_exc()), file=sys.stderr)
        sys.exit(-1)

if __name__ == '__main__':
    # Read a local port to connect to from stdin
    port_number = int(sys.stdin.readline())
    sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
    sock.connect(("127.0.0.1", port_number))
    with os.fdopen(os.dup(sock.fileno()), "rb", 65536) as infile,\
            os.fdopen(os.dup(sock.fileno()), "wb", 65536) as outfile:
        main(infile, outfile)

<|endoftext|>


<commit_before>"""
camera controller for tracking slime molds
University of Applied Sciences Potsdam (Germany) @idpotsdam
author: fabiantheblind
license: MIT
"""

import sys # we need this to exit if the GPIO lib is not present
import time # just for testing purpose
import datetime # for timestamps
import os # for path checking

# change these if you like
IMAGEFOLDER = "/home/pi/fhp/images/"
IMAGENAME = "slime"

# leave this alone
TIMESTAMP = 0
# Check if the libraries exist on this system
# if not throw a warning
try:
    import RPi.GPIO as GPIO
    print "GPIO lib is present. Moving on"
except ImportError:
    print "GPIO lib is not installed. Stopping program"
    print "Please install it with the commands:"
    print "$ sudo apt-get update"
    print "$ sudo apt-get install rpi.gpio"
    sys.exit()
try:
    import picamera
    print "picamera lib present moving on"
except ImportError:
    print "picamera lib is not installed. Stopping program"
    print "Please install it with the commands:"
    print "$ sudo apt-get update"
    print "$ sudo apt-get install python-picamera python3-picamera"
    sys.exit()

def lighton(led1, led2, led3, led4):
    GPIO.output(led1, GPIO.HIGH)
    GPIO.output(led2, GPIO.HIGH)
    GPIO.output(led3, GPIO.HIGH)
    GPIO.output(led4, GPIO.HIGH)

def lightoff(led1, led2, led3, led4):
    GPIO.output(led1, GPIO.LOW)
    GPIO.output(led2, GPIO.LOW)
    GPIO.output(led3, GPIO.LOW)
    GPIO.output(led4, GPIO.LOW)
##
#Add more LEDs as arguments
#
def setup(led1, led2, led3, led4):
    """Setup all that stuff"""
    print "setup GPIO"
    global TIMESTAMP
    GPIO.setwarnings(True)
    GPIO.setmode(GPIO.BOARD)
    GPIO.setup(led1, GPIO.OUT)
    GPIO.setup(led2, GPIO.OUT)
    GPIO.setup(led3, GPIO.OUT)
    GPIO.setup(led4, GPIO.OUT)
    if not os.path.exists(IMAGEFOLDER):
        print "folder does not exists. Create it"
        os.makedirs(IMAGEFOLDER)
    ts = time.time()
    TIMESTAMP = datetime.datetime.fromtimestamp(ts).strftime("%Y-%m-%d %H-%M-%S")

# now run all of that
if __name__ == '__main__':
    try:
        setup(11, 12, 15, 16)
        print "GPIO all set up"
        cam = picamera.PiCamera()
        # this is just for testing
        # the max resolution is
        # 2592 x 1944
        cam.resolution = (640, 480)
        # there cant be a preview when working remote via ssh
        # but if you are in the GUI you can preview the image
        # camera.start_preview()
        # time.sleep(5)
        filename = IMAGEFOLDER + IMAGENAME + TIMESTAMP + ".jpg"
        lighton(11, 12, 15, 16)
        time.sleep(0.2)
        cam.capture(filename)
        time.sleep(0.2)
        lightoff(11, 12, 15, 16)
        print "made a picture and wrote it to ", filename
        cam.close() # close the cam again
        # camera.stop_preview()
        # GPIO.cleanup()
    except KeyboardInterrupt:
    # here you put any code you want to run before the program
    # exits when you press CTRL+C
        print "KeyboardInterrupt"
        GPIO.cleanup() # this ensures a clean exit
    # except:
    # this catches ALL other exceptions including errors.
    # You won't get any error messages for debugging
    # so only use it once your code is working
        # print "Other error or exception occurred!"
    finally:
        print "end. clean up GPIO"
        GPIO.cleanup() # this ensures a clean exit

<commit_msg>added permission change<commit_after>"""
camera controller for tracking slime molds
University of Applied Sciences Potsdam (Germany) @idpotsdam
author: fabiantheblind
license: MIT
"""

import sys # we need this to exit if the GPIO lib is not present
import time # just for testing purpose
import datetime # for timestamps
import os # for path checking

# change these if you like
IMAGEFOLDER = "/home/pi/fhp/images/"
IMAGENAME = "slime"

# leave this alone
TIMESTAMP = 0

# Check if the libraries exist on this system
# if not throw a warning
try:
    import RPi.GPIO as GPIO
    print "GPIO lib is present. Moving on"
except ImportError:
    print "GPIO lib is not installed. Stopping program"
    print "Please install it with the commands:"
    print "$ sudo apt-get update"
    print "$ sudo apt-get install rpi.gpio"
    sys.exit()
try:
    import picamera
    print "picamera lib present moving on"
except ImportError:
    print "picamera lib is not installed. Stopping program"
    print "Please install it with the commands:"
    print "$ sudo apt-get update"
    print "$ sudo apt-get install python-picamera python3-picamera"
    sys.exit()

def lighton(led1, led2, led3, led4):
    GPIO.output(led1, GPIO.HIGH)
    GPIO.output(led2, GPIO.HIGH)
    GPIO.output(led3, GPIO.HIGH)
    GPIO.output(led4, GPIO.HIGH)

def lightoff(led1, led2, led3, led4):
    GPIO.output(led1, GPIO.LOW)
    GPIO.output(led2, GPIO.LOW)
    GPIO.output(led3, GPIO.LOW)
    GPIO.output(led4, GPIO.LOW)
##
#Add more LEDs as arguments
#
def setup(led1, led2, led3, led4):
    """Setup all that stuff"""
    print "setup GPIO"
    global TIMESTAMP
    GPIO.setwarnings(True)
    GPIO.setmode(GPIO.BOARD)
    GPIO.setup(led1, GPIO.OUT)
    GPIO.setup(led2, GPIO.OUT)
    GPIO.setup(led3, GPIO.OUT)
    GPIO.setup(led4, GPIO.OUT)
    if not os.path.exists(IMAGEFOLDER):
        print "folder does not exists. Create it"
        os.makedirs(IMAGEFOLDER)
    ts = time.time()
    TIMESTAMP = datetime.datetime.fromtimestamp(ts).strftime("%Y-%m-%d %H-%M-%S")

# now run all of that
if __name__ == '__main__':
    try:
        setup(11, 12, 15, 16)
        print "GPIO all set up"
        cam = picamera.PiCamera()
        # this is just for testing
        # the max resolution is
        # 2592 x 1944
        cam.resolution = (640, 480)
        # there cant be a preview when working remote via ssh
        # but if you are in the GUI you can preview the image
        # camera.start_preview()
        # time.sleep(5)
        filename = IMAGEFOLDER + IMAGENAME + TIMESTAMP + ".jpg"
        lighton(11, 12, 15, 16)
        time.sleep(0.2)
        cam.capture(filename)
        time.sleep(0.2)
        lightoff(11, 12, 15, 16)
        print "made a picture and wrote it to ", filename
        cam.close() # close the cam again
        # now set the permissions on the file so user pi can edit it
        os.chmod(filename, 0777)
        # camera.stop_preview()
        # GPIO.cleanup()
    except KeyboardInterrupt:
    # here you put any code you want to run before the program
    # exits when you press CTRL+C
        print "KeyboardInterrupt"
        GPIO.cleanup() # this ensures a clean exit
    # except:
    # this catches ALL other exceptions including errors.
    # You won't get any error messages for debugging
    # so only use it once your code is working
        # print "Other error or exception occurred!"
    finally:
        print "end. clean up GPIO"
        GPIO.cleanup() # this ensures a clean exit
<|endoftext|>


<commit_before>import web
from web import form
from solve_clue import CrypticClueSolver, split_clue_text
import threading

class index:
    def GET(self):
        # form = myform()
        answers = solver.answers_with_clues
        if answers == []:
            answers = ["Sorry, I couldn't find any answers"]
        elif answers is None:
            answers = [""]
        return render.index(answers[:200], form)

    def POST(self):
        # form = myform()
        if not form.validates():
            return render.index(["Sorry, something went wrong with that clue"], form)
        else:
            # phrases, answer = parse_clue_text(form.d.Clue)
            # answers = solve_phrases(phrases)[:50]
            phrases, lengths, pattern, answer = split_clue_text(form.d.Clue)
            if len(phrases) > 7:
                return render.index(["Sorry, I can't reliably handle clues longer than 7 phrases yet. Try grouping some words into phrases by putting an underscore instead of a space between them"], form)
            solver.setup(form.d.Clue)
            solver_thread = threading.Thread(target=solver.run)
            solver_thread.start()
            solver_thread.join()
            # answers = solve_clue_text(form.d.Clue)
            raise web.seeother('/')

class halt:
    def POST(self):
        print "trying to halt"
        solver.stop()
        raise web.seeother('/')

if __name__ == "__main__":
    render = web.template.render('crypticweb/templates/')

    urls = ('/', 'index',
            '/halt', 'halt')

    myform = form.Form(
        form.Textbox("Clue", size="100"))
    form = myform()
    with CrypticClueSolver() as solver:
        app = web.application(urls, globals())
        app.run()
<commit_msg>fix bug from file renaming<commit_after>import web
from web import form
from solve_clue import CrypticClueSolver, split_clue_text
import threading

class index:
    def GET(self):
        # form = myform()
        answers = solver.answers_with_clues
        if answers == []:
            answers = ["Sorry, I couldn't find any answers"]
        elif answers is None:
            answers = [""]
        return render.index(answers[:200], form)

    def POST(self):
        # form = myform()
        if not form.validates():
            return render.index(["Sorry, something went wrong with that clue"], form)
        else:
            # phrases, answer = parse_clue_text(form.d.Clue)
            # answers = solve_phrases(phrases)[:50]
            phrases, lengths, pattern, answer = split_clue_text(form.d.Clue)
            if len(phrases) > 7:
                return render.index(["Sorry, I can't reliably handle clues longer than 7 phrases yet. Try grouping some words into phrases by putting an underscore instead of a space between them"], form)
            solver.setup(form.d.Clue)
            solver_thread = threading.Thread(target=solver.run)
            solver_thread.start()
            solver_thread.join()
            # answers = solve_clue_text(form.d.Clue)
            raise web.seeother('/')

class halt:
    def POST(self):
        print "trying to halt"
        solver.stop()
        raise web.seeother('/')

if __name__ == "__main__":
    render = web.template.render('python/crypticweb/templates/')

    urls = ('/', 'index',
            '/halt', 'halt')

    myform = form.Form(
        form.Textbox("Clue", size="100"))
    form = myform()
    with CrypticClueSolver() as solver:
        app = web.application(urls, globals())
        app.run()
<|endoftext|>


<commit_before>#!/usr/bin/python
#
# Copyright 2015 Google Inc. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""This example demonstrates how to retrieve the latest report.

DBM users typically create scheduled reports, where the advertiser user either
the UI or API to create a query that will be used to generate reports based on
a daily, weekly, or monthly schedule.

In this example, we check to see if a daily query has run in the last 12 hours.
If it has, we download the data to a local CSV file. If no queryId is provided,
this will print a list of existing queries.
"""

import argparse
from datetime import datetime
from datetime import timedelta
import os
import urllib2
import util

# Optional filtering arguments.
parser = argparse.ArgumentParser(description='Downloads a report if it has '
                                 'been created in the given timeframe.')
parser.add_argument('--client_id', required=False,
                    help=('Your client ID from the Google Developers Console.'
                          'This should be provided along with the '
                          'client_secret the first time you run an example.'))
parser.add_argument('--client_secret', required=False,
                    help=('Your client secret from the Google Developers '
                          'Console. This should be provided along with the '
                          'client_id the first time you run an example.'))
parser.add_argument('--output_directory', default=(os.path.dirname(
    os.path.realpath(__file__))), help=('Path to the directory you want to '
                                        'save the report to.'))
parser.add_argument('--query_id', default=0, type=int,
                    help=('The id of a query used to generate a report.'))
parser.add_argument('--report_window', default=12, type=int,
                    help=('The age a report must be in hours at a maximum to '
                          'be considered fresh.'))

def main(doubleclick_bid_manager, output_dir, query_id, report_window):
  if query_id:
    # Call the API, getting the latest status for the passed queryId.
    response = (doubleclick_bid_manager.queries().getquery(queryId=query_id)
                .execute())
    try:
      # If it is recent enough...
      if (is_in_report_window(response['metadata']['latestReportRunTimeMs'],
                              report_window)):
        # Grab the report and write contents to a file.
        save_report_to_file(output_dir, response['queryId'],
                            (response['metadata']
                             ['googleCloudStoragePathForLatestReport']))
        print 'Download complete.'
      else:
        print('No reports for queryId "%s" in the last %s hours.' %
              (response['queryId'], report_window))
    except KeyError:
      print'No report found for queryId "%s".' % query_id
  else:
    # Call the API, getting a list of queries.
    response = doubleclick_bid_manager.queries().listqueries().execute()
    # Print queries out.
    print 'Id\t\tName'
    if 'queries' in response:
      for q in response['queries']:
        print '%s\t%s' % (q['queryId'], q['metadata']['title'])
    else:
      print 'No queries exist.'

def is_in_report_window(run_time_ms, report_window):
  """Determines if the given time in milliseconds is in the report window.

  Args:
    run_time_ms: str containing a time in milliseconds.
    report_window: int identifying the range of the report window in hours.
  Returns:
    A boolean indicating whether the given query's report run time is within
    the report window.
  """
  report_time = datetime.fromtimestamp(int((run_time_ms))/1000)
  earliest_time_in_range = datetime.now() - timedelta(hours=report_window)
  return report_time > earliest_time_in_range

def save_report_to_file(output_dir, query_id, report_url):
  """Saves the contents of the report_url to the given output directory.

  Args:
    output_dir: str containing the path to the directory you want to save to.
    query_id: str containing the Id of the query that generated the report.
    report_url: str containing the url to the generated report.
  """
  # Create formatter for output file path.
  if not os.path.isabs(output_dir):
    output_dir = os.path.expanduser(output_dir)
  output_fmt = output_dir + '/%s.csv'
  with(open(output_fmt % query_id, 'wb')) as handle:
    handle.write(urllib2.urlopen(report_url).read())

if __name__ == '__main__':
  args = parser.parse_args()
  # Retrieve the query id of the report we're downloading, or set to 0.
  QUERY_ID = args.query_id
  if not QUERY_ID:
    try:
      QUERY_ID = int(raw_input('Enter the query id or press enter to '
                               'list queries: '))
    except ValueError:
      QUERY_ID = 0

  main(util.get_service(
      client_id=args.client_id, client_secret=args.client_secret),
       args.output_directory, QUERY_ID, args.report_window)

<commit_msg>[Python] Updating get_latest_report.py<commit_after>#!/usr/bin/python
#
# Copyright 2015 Google Inc. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""This example demonstrates how to retrieve the latest report.

DBM users typically create scheduled reports, where the advertiser user either
the UI or API to create a query that will be used to generate reports based on
a daily, weekly, or monthly schedule.

In this example, we check to see if a daily query has run in the last 12 hours.
If it has, we download the data to a local CSV file. If no queryId is provided,
this will print a list of existing queries.
"""

import argparse
from contextlib import closing
from datetime import datetime
from datetime import timedelta
import os
import urllib2
import util

# Optional filtering arguments.
parser = argparse.ArgumentParser(description='Downloads a report if it has '
                                 'been created in the given timeframe.')
parser.add_argument('--client_id', required=False,
                    help=('Your client ID from the Google Developers Console.'
                          'This should be provided along with the '
                          'client_secret the first time you run an example.'))
parser.add_argument('--client_secret', required=False,
                    help=('Your client secret from the Google Developers '
                          'Console. This should be provided along with the '
                          'client_id the first time you run an example.'))
parser.add_argument('--output_directory', default=(os.path.dirname(
    os.path.realpath(__file__))), help=('Path to the directory you want to '
                                        'save the report to.'))
parser.add_argument('--query_id', default=0, type=int,
                    help=('The id of a query used to generate a report.'))
parser.add_argument('--report_window', default=12, type=int,
                    help=('The age a report must be in hours at a maximum to '
                          'be considered fresh.'))

def main(doubleclick_bid_manager, output_dir, query_id, report_window):
  if query_id:
    # Call the API, getting the latest status for the passed queryId.
    query = (doubleclick_bid_manager.queries().getquery(queryId=query_id)
                .execute())
    try:
      # If it is recent enough...
      if (is_in_report_window(query['metadata']['latestReportRunTimeMs'],
                              report_window)):
        if not os.path.isabs(output_dir):
          output_dir = os.path.expanduser(output_dir)

        # Grab the report and write contents to a file.
        report_url = query['metadata']['googleCloudStoragePathForLatestReport']
        output_file = '%s/%s.csv' % (output_dir, query['queryId'])
        with open(output_file, 'wb') as output:
          with closing(urllib2.urlopen(report_url)) as url:
            output.write(url.read())
        print 'Download complete.'
      else:
        print('No reports for queryId "%s" in the last %s hours.' %
              (query['queryId'], report_window))
    except KeyError:
      print 'No report found for queryId "%s".' % query_id
  else:
    # Call the API, getting a list of queries.
    response = doubleclick_bid_manager.queries().listqueries().execute()

    # Print queries out.
    print 'Id\t\tName'
    if 'queries' in response:
      for q in response['queries']:
        print '%s\t%s' % (q['queryId'], q['metadata']['title'])
    else:
      print 'No queries exist.'

def is_in_report_window(run_time_ms, report_window):
  """Determines if the given time in milliseconds is in the report window.

  Args:
    run_time_ms: str containing a time in milliseconds.
    report_window: int identifying the range of the report window in hours.
  Returns:
    A boolean indicating whether the given query's report run time is within
    the report window.
  """
  report_time = datetime.fromtimestamp(int((run_time_ms))/1000)
  earliest_time_in_range = datetime.now() - timedelta(hours=report_window)
  return report_time > earliest_time_in_range

if __name__ == '__main__':
  args = parser.parse_args()
  # Retrieve the query id of the report we're downloading, or set to 0.
  QUERY_ID = args.query_id
  if not QUERY_ID:
    try:
      QUERY_ID = int(raw_input('Enter the query id or press enter to '
                               'list queries: '))
    except ValueError:
      QUERY_ID = 0

  main(util.get_service(
      client_id=args.client_id, client_secret=args.client_secret),
       args.output_directory, QUERY_ID, args.report_window)

<|endoftext|>


<commit_before>#!/usr/bin/env python
"""The Kolobyte Python Library
This library is a collection of python code to make your life easier, more efficient,
and just help out with writing other code. Run `help("kolobyte")`, for documentation.

Tested on Python 2, sorry! Willing to accept testers using Python 3.
"""
import hashlib
import math
import os
import time
import itertools
import socket
import sys
import threading
import re
import select
import random
import string

class Shoe():
    """Socket wrapper to be netcat like.
    Inspired by shoe.rb from crowell @ https://github.com/crowell/shoe.rb

    Example:
        Check the test in test_kolobyte.py.
    """
    def __init__(self, host, port):
        self._socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
        self._socket.connect((host, port))
        self._socket.setblocking(0)

    def write(self, data):
        """Write `data` to a shoe.
        @note: Requires you to put a \n yourself.
        """
        self._socket.send(data)

    def read_for(self, secs):
        """Read from a shoe for `secs`
        """
        response = b""
        timeout = time.time() + secs
        while True:
            is_ready = select.select([self._socket], [], [], secs)
            try:
                if is_ready:
                    response += self._socket.recv(1)
            except:
                pass
            if timeout < time.time():
                break

        return response

    def read_until(self, mystr, is_regex=False):
        """Read from a shoe until a `mystr` appears. Accepts regex.
        """
        response = b""
        self._socket.setblocking(1)
        if is_regex:
            while not re.findall(mystr, response):
                response += self._socket.recv(1)
        else:
            while response.endswith(mystr) != True:
                response += self._socket.recv(1)
        self._socket.setblocking(1)
        return response

    def read_until_end(self, secs=1):
        """Read from a shoe until the end, defined by no recv for 1 second (default).
        """
        response = b""
        timeout = time.time() + secs
        while True:
            is_ready = select.select([self._socket], [], [], secs)
            try:
                if is_ready:
                    response += self._socket.recv(1)
            except:
                pass
            if timeout < time.time():
                break

        return response

    def tie(self):
        """Tie the shoe and make it interactive.
        """
        self._socket.settimeout(None)
        # Constantly read off the shoe.
        def _listen():
            while True:
                data = self._socket.recv(4096)
                print(data)

        t = threading.Thread(target = _listen)
        t.daemon = True
        t.start()

        # Constantly write any new input to the shoe.
        while True:
            payload = sys.stdin.readline()
            if payload == "":
                continue
            self.write(payload)

    def close(self):
        """Close the shoe
        """
        self._socket.close()

class PrimeSieve():
    """Sieve of Eratosthenes, finds prime #s up to n in O(nloglogn)
    Usage: primeSieve = PrimeSieve(500)
           primeSieve.primes[25] = 233
    """
    def __init__(self, n):
        """Init a sieve of primes up to `n`
        """
        # Assume [0,n) are all primes
        primes = [True for i in range(0,n)]
        for i in range(2,int(math.ceil(math.sqrt(n)))):
            if primes[i] is True:
                a = 0
                while (i**2 + a*i < n): # Remove every multiple of i
                    primes[i**2 + a*i] = False
                    a += 1

        self.primes = [i for i in range(2,n) if primes[i] is True]

class Singleton:
    """A non-thread-safe helper class to ease implementing singletons.
    This should be used as a decorator.

    The decorated class can define one `__init__` function that
    takes only the `self` argument. Other than that, there are
    no restrictions that apply to the decorated class.

    Limitations: The decorated class cannot be inherited from.

    Example:
    @Singleton
    class SharedSecretHolder:
        secret = "Foo"
        def __init__(self):
            print("SharedSecretHolder Singleton initalized!")
            self.secret = "It's a secret!"

    secret_holder1 = SharedSecretHolder.Instance()
    secret_holder2 = SharedSecretHolder.Instance()
    print(secret_holder2.secret)
    secret_holder1.secret = "The secret changed, but the other instance picked it up!"
    print(secret_holder2.secret)
    """
    def __init__(self, decorated):
        self._decorated = decorated

    def Instance(self):
        """Use to get the single instance of the Singleton. Upon first use, it calls the singleton's
        init function.
        """
        try:
            return self._instance
        except AttributeError:
            self._instance = self._decorated()
            return self._instance

    def __call__(self):
        raise TypeError('Singletons must be accessed through `Instance()`.')

    def __instancecheck__(self, inst):
        return isinstance(inst, self._decorated)

def md5(s):
    """Simpler md5sum a string
    """
    return hashlib.md5(s).hexdigest()

def isPrime(n):
    """Simple, and slow check if a number is prime or not
    """
    if n == 2 or n == 3:
        return True
    if n < 2 or n % 2 == 0:
        return False # 1, negative or even
    for i in xrange(3, int(math.sqrt(n))+1, 2):
        if n % i == 0:
            return False
    return True

def num(s):
    """Convert a string to an integer or float
    """
    try:
        return int(s)
    except ValueError:
        return float(s)

def rotn(s, n, letters="abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ"):
    """Rotate a string's characters by n
    """
    ans = ""
    for c in s:
        if not c.isalpha():
            ans += c
            continue
        ans += letters[(letters.index(c) + n) % len(letters)]
    return ans

def connect_pop3(mailserver, username, password):
    """Connect to a pop3 mailserver and return a handle to it
    """
    import poplib
    p = poplib.POP3(mailserver)
    p.user(username)
    p.pass_(password)

    return p

def dns_a_record(host):
    """Returns the DNS A-record of a host
    """
    try:
        import dns.resolver as dns
    except ImportError as e:
        print("[ERROR] Try pip install dnspython, or https://github.com/rthalley/dnspython")
        raise

    return dns.query(host,"A")

def send_smtp(mailserver, port, fromaddr, toaddr, subject, body):
    """Sends a message consisting of a `subject`, and `body`, from a `fromaddr`, to a `toaddr`.
    Spoofing is possible. Port must be a number.
    """
    import smtplib

    fromheader = 'From: %s\r\n' % fromaddr
    toheader = 'To: %s\r\n\r\n' % toaddr

    msg = '%s\n%s\n%s\n\n%s' % (fromheader, toheader, subject, body)

    s = smtplib.SMTP(mailserver, port)
    s.sendmail(fromaddr, toaddr, msg)
    s.quit()

def fork_exec(prog, args):
    """Lower level subprocess.Popen(). Uses fork() and execv() instead.
    Prog must be a string, and args must be a list.
    """
    pid = os.fork() # Returns 0 in the child, and the real pid in the parent
    if pid == 0:
        os.execv(prog, args)
    else:
        pass

def strxor(str1, str2):
    """Xors 2 strings character by character.
    """
    minlen = min(len(str1), len(str2))
    ans = ""
    for (c1, c2) in zip(str1[:minlen], str2[:minlen]):
        ans += chr(ord(c1) ^ ord(c2))
    return ans

def timeme(method):
    """@timeme decorator. Place before any function you want timed.

    Example:
        @timeme
        def print_after_three(s):
            time.sleep(3)
            return s
        print_after_three("testing timeme")
        >> ('print_after_three', 3003, 'ms')
    """
    def wrapper(*args, **kw):
        startTime = int(round(time.time() * 1000))
        result = method(*args, **kw)
        endTime = int(round(time.time() * 1000))

        print(method.__name__, endTime - startTime,'ms')
        return result

    return wrapper

def cyclic_pattern(maxlen, n = 3):
    """Generate the De Bruijn Sequence (a cyclic pattern) up to `maxlen` characters and subsequences
    of length `n`. Modified from github.com/longld/peda/.
    """
    charset = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz"
    k = len(charset)
    a = [0] * k * n
    sequence = []
    def db(t, p):
        if len(sequence) == maxlen:
            return

        if t > n:
            if n % p == 0:
                for j in range(1, p + 1):
                    sequence.append(charset[a[j]])
                    if len(sequence) == maxlen:
                        return
        else:
            a[t] = a[t - p]
            db(t + 1, p)
            for j in range(a[t - p] + 1, k):
                a[t] = j
                db(t + 1, t)
    db(1,1)
    return ''.join(sequence)

def cyclic_pattern_search(needle, size = 100000):
    """Search a the de bruijin cylic pattern for a `needle` and return its offset.
    """
    pattern = cyclic_pattern(size)
    pos = pattern.find(needle)
    return pos

def stack_overflower(padlen, noplen, payload = None):
    if payload is None:
        # Some default shellcode. I forget what it does...
        payload = "\xeb\x36\x31\xc0\x31\xdb\x31\xc9\x31\xd2\xb0\xa4\xcd\x80\x5e\x31\xc0\x88\x46\x07" \
                  "\x88\x46\x10\x88\x46\x1a\x89\x76\x1b\x8d\x5e\x08\x89\x5e\x1f\x8d\x5e\x11\x89\x5e" \
                  "\x23\x89\x46\x27\xb0\x0b\x89\xf3\x8d\x4e\x1b\x8d\x56\x27\xcd\x80\xe8\xc5\xff\xff" \
                  "\xff\x2f\x62\x69\x6e\x2f\x6e\x63\x23\x2d\x6c\x76\x70\x39\x39\x39\x39\x23\x2d\x65" \
                  "\x2f\x62\x69\x6e\x2f\x73\x68\x23\x41\x41\x41\x41\x42\x42\x42\x42\x43\x43\x43\x43" \
                  "\x44\x44\x44\x44"
    pad = cyclic_pattern(padlen)
    nopsled = "\x90"*noplen
    return pad+nopsled+payload

def bruteforce(charset, minlength, maxlength):
    """Efficient dumb bruteforcer generator.

    Example:
        # Generate every string 3 to 10 characters long, from the ascii_lowercase charset.
        for word in bruteforce(string.ascii_lowercase, 3, 10):
            print(word)
    """
    return (''.join(candidate)
        for candidate in itertools.chain.from_iterable(itertools.product(charset, repeat=i)
        for i in range(minlength, maxlength + 1)))

def entropy(data):
    """Calculates the Shannon entropy of data.
    """
    freqList = [0]*len(data)
    for b in range(256):
        ctr = 0.0
        for byte in data:
            if ord(byte) == b:
                ctr += 1
        freqList.append(float(ctr) / len(data))
    ent = 0.0
    for freq in freqList:
        if freq > 0:
            ent = ent + (freq * math.log(freq, 2))
    ent = -ent
    return ent

def pretty_json_dump(d):
    """Pretty dumps a dictionary as a JSON string
    """
    try:
        from flask import json
    except ImportError as e:
        print("[ERROR] Try pip install flask.")
        raise

    return json.dumps(d, indent=4, separators=(',',': '))

def random_string(n, max_size=False, char_pool=string.ascii_lowercase + string.digits):
    """Returns a random string of [char_pool] of size `n`.
    If max_size is True, then returns a random string of up to size `n`, minimum 1.
    """
    if max_size:
        n = random.randint(1, n)

    rand_str = ""
    for i in range(n):
        rand_c = random.choice(char_pool)
        rand_str += rand_c

    return rand_str

if __name__ == "__main__":
    help("kolobyte")
<commit_msg>Added byte_to_hex_str to kolobyte.py<commit_after>#!/usr/bin/env python
"""The Kolobyte Python Library
This library is a collection of python code to make your life easier, more efficient,
and just help out with writing other code. Run `help("kolobyte")`, for documentation.

Tested on Python 2, sorry! Willing to accept testers using Python 3.
"""
import hashlib
import math
import os
import time
import itertools
import socket
import sys
import threading
import re
import select
import random
import string

class Shoe():
    """Socket wrapper to be netcat like.
    Inspired by shoe.rb from crowell @ https://github.com/crowell/shoe.rb

    Example:
        Check the test in test_kolobyte.py.
    """
    def __init__(self, host, port):
        self._socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
        self._socket.connect((host, port))
        self._socket.setblocking(0)

    def write(self, data):
        """Write `data` to a shoe.
        @note: Requires you to put a \n yourself.
        """
        self._socket.send(data)

    def read_for(self, secs):
        """Read from a shoe for `secs`
        """
        response = b""
        timeout = time.time() + secs
        while True:
            is_ready = select.select([self._socket], [], [], secs)
            try:
                if is_ready:
                    response += self._socket.recv(1)
            except:
                pass
            if timeout < time.time():
                break

        return response

    def read_until(self, mystr, is_regex=False):
        """Read from a shoe until a `mystr` appears. Accepts regex.
        """
        response = b""
        self._socket.setblocking(1)
        if is_regex:
            while not re.findall(mystr, response):
                response += self._socket.recv(1)
        else:
            while response.endswith(mystr) != True:
                response += self._socket.recv(1)
        self._socket.setblocking(1)
        return response

    def read_until_end(self, secs=1):
        """Read from a shoe until the end, defined by no recv for 1 second (default).
        """
        response = b""
        timeout = time.time() + secs
        while True:
            is_ready = select.select([self._socket], [], [], secs)
            try:
                if is_ready:
                    response += self._socket.recv(1)
            except:
                pass
            if timeout < time.time():
                break

        return response

    def tie(self):
        """Tie the shoe and make it interactive.
        """
        self._socket.settimeout(None)
        # Constantly read off the shoe.
        def _listen():
            while True:
                data = self._socket.recv(4096)
                print(data)

        t = threading.Thread(target = _listen)
        t.daemon = True
        t.start()

        # Constantly write any new input to the shoe.
        while True:
            payload = sys.stdin.readline()
            if payload == "":
                continue
            self.write(payload)

    def close(self):
        """Close the shoe
        """
        self._socket.close()

class PrimeSieve():
    """Sieve of Eratosthenes, finds prime #s up to n in O(nloglogn)
    Usage: primeSieve = PrimeSieve(500)
           primeSieve.primes[25] = 233
    """
    def __init__(self, n):
        """Init a sieve of primes up to `n`
        """
        # Assume [0,n) are all primes
        primes = [True for i in range(0,n)]
        for i in range(2,int(math.ceil(math.sqrt(n)))):
            if primes[i] is True:
                a = 0
                while (i**2 + a*i < n): # Remove every multiple of i
                    primes[i**2 + a*i] = False
                    a += 1

        self.primes = [i for i in range(2,n) if primes[i] is True]

class Singleton:
    """A non-thread-safe helper class to ease implementing singletons.
    This should be used as a decorator.

    The decorated class can define one `__init__` function that
    takes only the `self` argument. Other than that, there are
    no restrictions that apply to the decorated class.

    Limitations: The decorated class cannot be inherited from.

    Example:
    @Singleton
    class SharedSecretHolder:
        secret = "Foo"
        def __init__(self):
            print("SharedSecretHolder Singleton initalized!")
            self.secret = "It's a secret!"

    secret_holder1 = SharedSecretHolder.Instance()
    secret_holder2 = SharedSecretHolder.Instance()
    print(secret_holder2.secret)
    secret_holder1.secret = "The secret changed, but the other instance picked it up!"
    print(secret_holder2.secret)
    """
    def __init__(self, decorated):
        self._decorated = decorated

    def Instance(self):
        """Use to get the single instance of the Singleton. Upon first use, it calls the singleton's
        init function.
        """
        try:
            return self._instance
        except AttributeError:
            self._instance = self._decorated()
            return self._instance

    def __call__(self):
        raise TypeError('Singletons must be accessed through `Instance()`.')

    def __instancecheck__(self, inst):
        return isinstance(inst, self._decorated)

def md5(s):
    """Simpler md5sum a string
    """
    return hashlib.md5(s).hexdigest()

def isPrime(n):
    """Simple, and slow check if a number is prime or not
    """
    if n == 2 or n == 3:
        return True
    if n < 2 or n % 2 == 0:
        return False # 1, negative or even
    for i in xrange(3, int(math.sqrt(n))+1, 2):
        if n % i == 0:
            return False
    return True

def num(s):
    """Convert a string to an integer or float
    """
    try:
        return int(s)
    except ValueError:
        return float(s)

def rotn(s, n, letters="abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ"):
    """Rotate a string's characters by n
    """
    ans = ""
    for c in s:
        if not c.isalpha():
            ans += c
            continue
        ans += letters[(letters.index(c) + n) % len(letters)]
    return ans

def connect_pop3(mailserver, username, password):
    """Connect to a pop3 mailserver and return a handle to it
    """
    import poplib
    p = poplib.POP3(mailserver)
    p.user(username)
    p.pass_(password)

    return p

def dns_a_record(host):
    """Returns the DNS A-record of a host
    """
    try:
        import dns.resolver as dns
    except ImportError as e:
        print("[ERROR] Try pip install dnspython, or https://github.com/rthalley/dnspython")
        raise

    return dns.query(host,"A")

def send_smtp(mailserver, port, fromaddr, toaddr, subject, body):
    """Sends a message consisting of a `subject`, and `body`, from a `fromaddr`, to a `toaddr`.
    Spoofing is possible. Port must be a number.
    """
    import smtplib

    fromheader = 'From: %s\r\n' % fromaddr
    toheader = 'To: %s\r\n\r\n' % toaddr

    msg = '%s\n%s\n%s\n\n%s' % (fromheader, toheader, subject, body)

    s = smtplib.SMTP(mailserver, port)
    s.sendmail(fromaddr, toaddr, msg)
    s.quit()

def fork_exec(prog, args):
    """Lower level subprocess.Popen(). Uses fork() and execv() instead.
    Prog must be a string, and args must be a list.
    """
    pid = os.fork() # Returns 0 in the child, and the real pid in the parent
    if pid == 0:
        os.execv(prog, args)
    else:
        pass

def strxor(str1, str2):
    """Xors 2 strings character by character.
    """
    minlen = min(len(str1), len(str2))
    ans = ""
    for (c1, c2) in zip(str1[:minlen], str2[:minlen]):
        ans += chr(ord(c1) ^ ord(c2))
    return ans

def timeme(method):
    """@timeme decorator. Place before any function you want timed.

    Example:
        @timeme
        def print_after_three(s):
            time.sleep(3)
            return s
        print_after_three("testing timeme")
        >> ('print_after_three', 3003, 'ms')
    """
    def wrapper(*args, **kw):
        startTime = int(round(time.time() * 1000))
        result = method(*args, **kw)
        endTime = int(round(time.time() * 1000))

        print(method.__name__, endTime - startTime,'ms')
        return result

    return wrapper

def cyclic_pattern(maxlen, n = 3):
    """Generate the De Bruijn Sequence (a cyclic pattern) up to `maxlen` characters and subsequences
    of length `n`. Modified from github.com/longld/peda/.
    """
    charset = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz"
    k = len(charset)
    a = [0] * k * n
    sequence = []
    def db(t, p):
        if len(sequence) == maxlen:
            return

        if t > n:
            if n % p == 0:
                for j in range(1, p + 1):
                    sequence.append(charset[a[j]])
                    if len(sequence) == maxlen:
                        return
        else:
            a[t] = a[t - p]
            db(t + 1, p)
            for j in range(a[t - p] + 1, k):
                a[t] = j
                db(t + 1, t)
    db(1,1)
    return ''.join(sequence)

def cyclic_pattern_search(needle, size = 100000):
    """Search a the de bruijin cylic pattern for a `needle` and return its offset.
    """
    pattern = cyclic_pattern(size)
    pos = pattern.find(needle)
    return pos

def stack_overflower(padlen, noplen, payload = None):
    if payload is None:
        # Some default shellcode. I forget what it does...
        payload = "\xeb\x36\x31\xc0\x31\xdb\x31\xc9\x31\xd2\xb0\xa4\xcd\x80\x5e\x31\xc0\x88\x46\x07" \
                  "\x88\x46\x10\x88\x46\x1a\x89\x76\x1b\x8d\x5e\x08\x89\x5e\x1f\x8d\x5e\x11\x89\x5e" \
                  "\x23\x89\x46\x27\xb0\x0b\x89\xf3\x8d\x4e\x1b\x8d\x56\x27\xcd\x80\xe8\xc5\xff\xff" \
                  "\xff\x2f\x62\x69\x6e\x2f\x6e\x63\x23\x2d\x6c\x76\x70\x39\x39\x39\x39\x23\x2d\x65" \
                  "\x2f\x62\x69\x6e\x2f\x73\x68\x23\x41\x41\x41\x41\x42\x42\x42\x42\x43\x43\x43\x43" \
                  "\x44\x44\x44\x44"
    pad = cyclic_pattern(padlen)
    nopsled = "\x90"*noplen
    return pad+nopsled+payload

def bruteforce(charset, minlength, maxlength):
    """Efficient dumb bruteforcer generator.

    Example:
        # Generate every string 3 to 10 characters long, from the ascii_lowercase charset.
        for word in bruteforce(string.ascii_lowercase, 3, 10):
            print(word)
    """
    return (''.join(candidate)
        for candidate in itertools.chain.from_iterable(itertools.product(charset, repeat=i)
        for i in range(minlength, maxlength + 1)))

def entropy(data):
    """Calculates the Shannon entropy of data.
    """
    freqList = [0]*len(data)
    for b in range(256):
        ctr = 0.0
        for byte in data:
            if ord(byte) == b:
                ctr += 1
        freqList.append(float(ctr) / len(data))
    ent = 0.0
    for freq in freqList:
        if freq > 0:
            ent = ent + (freq * math.log(freq, 2))
    ent = -ent
    return ent

def pretty_json_dump(d):
    """Pretty dumps a dictionary as a JSON string
    """
    try:
        from flask import json
    except ImportError as e:
        print("[ERROR] Try pip install flask.")
        raise

    return json.dumps(d, indent=4, separators=(',',': '))

def random_string(n, max_size=False, char_pool=string.ascii_lowercase + string.digits):
    """Returns a random string of [char_pool] of size `n`.
    If max_size is True, then returns a random string of up to size `n`, minimum 1.
    """
    if max_size:
        n = random.randint(1, n)

    rand_str = ""
    for i in range(n):
        rand_c = random.choice(char_pool)
        rand_str += rand_c

    return rand_str

def byte_to_hex_str(n):
    """Returns a 2 character hex string for a given byte value `n`.
    """
    if n > 255 or n < 0:
        raise ValueError()

    return '%02x' % n

if __name__ == "__main__":
    help("kolobyte")
<|endoftext|>


<commit_before>from __future__ import print_function
import numpy as np
from astropy.io import fits
from marvin import config
from marvin.api.api import Interaction

class Cube(object):

    def __init__(self, filename=None, plateifu=None, mangaid=None):
        args = [filename, plateifu, mangaid]
        errmsg = 'Enter filename, plateifu, or mangaid!'
        assert any(args), errmsg
        assert sum([bool(arg) for arg in args]) == 1, errmsg

        self.filename = filename
        self.mangaid = mangaid
        self.plateifu = plateifu

        # convert from mangaid to plateifu
        # print warning if multiple plateifus correspond to one mangaid
        # FIX
        mangaid_to_plateifu = {'1-209232': '8485-1901'}
        if self.mangaid:
            self.plateifu = mangaid_to_plateifu[self.mangaid]

        if config.mode == 'local':
            if self.filename:
                self._openFile()
            else:
                try:
                    # self._getCubeFromPlateIFU()
                    self._getCubeFromMangaID()
                except RuntimeError:
                    # convert from plateifu to filename (sdss_access)
                    self.filename = plateifu_to_filename(self.plateifu)
                    try:
                        self._openFile()
                    except IOError as e:
                        if config.download:
                            # download file via sdsssync
                            # self._openFile()
                            pass
                        else:
                            raise Exception('File does not exist locally') from e
        elif config.mode == 'remote':
            if self.filename:
                raise FileNotFoundError('Cannot open file remotely.')
            else:
                # use API
                pass

    def getSpectrum(self, x, y):
        ''' currently: x,y array indices
        ideally: x,y in arcsecond relative to cube center '''
        # spectrum = Spectrum(x,y)
        if config.mode == 'file':
            ''' local (client has a file) '''
            shape = self.flux.shape
            assert len(shape) == 3, 'Dimensions of flux not = 3'
            assert x < shape[2] and y < shape[1], 'Input x,y coordinates greater than flux dimensions'
            return self.flux[:, y, x]
        elif config.mode == 'db':
            ''' db means local (client) has db '''
            return self._cube.spaxels[0].flux
        else:
            ''' local (client) has nothing '''
            route = 'cubes/{0}/spectra/x={1}/y={2}/'.format(self.mangaid, x, y)
            results = Interaction(route, request_type='get')
            return results.getData(astype=np.array)

    def _openFile(self):
        try:
            self.hdu = fits.open(self.filename)
        except FileNotFoundError as e:
            raise # Exception('{0} does not exist. Please provide full file path.'.format(self.filename)) from e

    @property
    def flux(self):
        if config.mode == 'file':
            return self.hdu['FLUX'].data
        if config.mode == 'db':
            return None

    # Switch to _getCubeFromPlateIFU
    def _getCubeFromMangaID(self):
        ''' server-side code '''
        from ..db.database import db
        import sdss.internal.database.utah.mangadb.DataModelClasses as datadb
        session = db.Session()
        self._cube = session.query(datadb.Cube).join(datadb.PipelineInfo, datadb.PipelineVersion).filter(datadb.PipelineVersion.version == 'trunk', datadb.Cube.mangaid == self.mangaid).first()
        print('cube', self._cube, len(self._cube.spaxels))
        self.ifu = self._cube.ifu.name
        self.ra = self._cube.ra
        self.dec = self._cube.dec
        self.plate = self._cube.plate
<commit_msg>initial implementation of getSpectrum for files<commit_after>from __future__ import print_function
import numpy as np
from astropy.io import fits
from astropy import wcs
from marvin import config
from marvin.api.api import Interaction

class Cube(object):

    def __init__(self, filename=None, plateifu=None, mangaid=None):
        args = [filename, plateifu, mangaid]
        errmsg = 'Enter filename, plateifu, or mangaid!'
        assert any(args), errmsg
        assert sum([bool(arg) for arg in args]) == 1, errmsg

        self.filename = filename
        self.mangaid = mangaid
        self.plateifu = plateifu

        self.useDB = None

        # convert from mangaid to plateifu
        # print warning if multiple plateifus correspond to one mangaid
        # FIX
        mangaid_to_plateifu = {'1-209232': '8485-1901'}
        if self.mangaid:
            self.plateifu = mangaid_to_plateifu[self.mangaid]

        if config.mode == 'local':
            if self.filename:
                self._openFile()
            else:
                try:
                    # self._getCubeFromPlateIFU()
                    self._getCubeFromMangaID()
                except RuntimeError:
                    # convert from plateifu to filename (sdss_access)
                    self.filename = plateifu_to_filename(self.plateifu)
                    try:
                        self._openFile()
                    except IOError as e:
                        if config.download:
                            # download file via sdsssync
                            # self._openFile()
                            pass
                        else:
                            raise Exception('File does not exist locally') from e
        elif config.mode == 'remote':
            if self.filename:
                raise FileNotFoundError('Cannot open file remotely.')
            else:
                # use API
                pass

    def getSpectrum(self, x=None, y=None, ra=None, dec=None, ext='flux'):
        """Returns the flux spectrum for an spaxel.

        Parameters
        ----------
        x, y : int
            The spaxel coordinates relative to the centre of the cube.

        ra, dec : float
            The coordinates of the spaxel to return. The closest spaxel to
            those coordinates will be returned.

        ext : str
            The extension of the cube to use, either `'flux'`, `'ivar'`, or
            `'mask'`.

        Returns
        -------
        result : np.array
            An array with the spectrum for the input coordinates.

        """

        # Checks that we have the correct set of inputs.
        if x or y:
            assert not ra and not dec, 'Either use (x, y) or (ra, dec)'
            assert x and y, 'Specify both x and y'
            inputMode = 'pix'
        elif ra or dec:
            assert not x and not y, 'Either use (x, y) or (ra, dec)'
            assert ra and dec, 'Specify both ra and dec'
            inputMode = 'sky'
        else:
            raise ValueError('You need to specify (x, y) or (ra, dec)')

        assert isinstance(ext, basestring)
        ext = ext.lower()
        assert ext in ['flux', 'ivar', 'mask'], \
            'ext needs to be either \'flux\', \'ivar\', or \'mask\''

        if not self.useDB:
            cubeExt = self._hdu[ext.upper()]
            if inputMode == 'sky':
                cubeWCS = wcs.WCS(cubeExt.header)
                x, y = cubeWCS.wcs_sky2pix([ra, dec], 1)

            cubeShape = cubeExt.shape
            # TODO: assert that pix coordinates are in cube
            yMid, xMid = cubeShape[1:] / 2.
            return cubeExt.data[:, yMid - y, xMid + x]

        else:

            raise NotImplementedError(
                'getSpectrum from DB not yet implemented')

        # ''' currently: x,y array indices
        # ideally: x,y in arcsecond relative to cube center '''
        # # spectrum = Spectrum(x,y)
        # if config.mode == 'file':
        #     ''' local (client has a file) '''
        #     shape = self.flux.shape
        #     assert len(shape) == 3, 'Dimensions of flux not = 3'
        #     assert x < shape[2] and y < shape[1], 'Input x,y coordinates greater than flux dimensions'
        #     return self.flux[:, y, x]
        # elif config.mode == 'db':
        #     ''' db means local (client) has db '''
        #     return self._cube.spaxels[0].flux
        # else:
        #     ''' local (client) has nothing '''
        #     route = 'cubes/{0}/spectra/x={1}/y={2}/'.format(self.mangaid, x, y)
        #     results = Interaction(route, request_type='get')
        #     return results.getData(astype=np.array)

    def _openFile(self):

        if not os.path.exists(self.filename):
            raise ValueError('filename {0} cannot be found'
                             .format(self.filename))

        self._hdu = fits.open(self.filename)
        self.useDB = False

    @property
    def flux(self):
        if not self.useDB:
            return self._hdu['FLUX'].data
        else:
            return None

    # Switch to _getCubeFromPlateIFU
    def _getCubeFromMangaID(self):
        ''' server-side code '''
        from ..db.database import db
        import sdss.internal.database.utah.mangadb.DataModelClasses as datadb
        session = db.Session()
        self._cube = session.query(datadb.Cube).join(datadb.PipelineInfo, datadb.PipelineVersion).filter(datadb.PipelineVersion.version == 'trunk', datadb.Cube.mangaid == self.mangaid).first()
        print('cube', self._cube, len(self._cube.spaxels))
        self.ifu = self._cube.ifu.name
        self.ra = self._cube.ra
        self.dec = self._cube.dec
        self.plate = self._cube.plate
<|endoftext|>


<commit_before># Copyright 2020 The SQLFlow Authors. All rights reserved.
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import os
import pickle
import sys

import oss2
import six
import tensorflow as tf
from runtime.diagnostics import SQLFlowDiagnostic
from runtime.tensorflow import is_tf_estimator

# NOTE(typhoonzero): hard code bucket name "sqlflow-models" as the bucket to
# save models trained.
SQLFLOW_MODELS_BUCKET = "sqlflow-models"

def remove_bucket_prefix(oss_uri):
    return oss_uri.replace("oss://%s/" % SQLFLOW_MODELS_BUCKET, "")

def get_models_bucket():
    return get_bucket(SQLFLOW_MODELS_BUCKET)

def get_bucket(name, ak=None, sk=None, endpoint=None):
    if ak is None:
        ak = os.getenv("SQLFLOW_OSS_AK", "")
    if sk is None:
        sk = os.getenv("SQLFLOW_OSS_SK", "")
    if endpoint is None:
        endpoint = os.getenv("SQLFLOW_OSS_MODEL_ENDPOINT", "")
    if ak == "" or sk == "":
        raise ValueError("must configure SQLFLOW_OSS_AK and SQLFLOW_OSS_SK "
                         "when submitting to PAI")
    if endpoint == "":
        raise ValueError(
            "must configure SQLFLOW_OSS_MODEL_ENDPOINT when submitting to PAI")
    auth = oss2.Auth(ak, sk)
    bucket = oss2.Bucket(auth, endpoint, name)
    return bucket

def copyfileobj(source, dest, ak, sk, endpoint, bucket_name):
    '''
    copy_file_to_oss copies alocal file (source) to an object on OSS (dest),
    overwrite if the oss object exists.
    '''
    auth = oss2.Auth(ak, sk)
    bucket = oss2.Bucket(auth, endpoint, bucket_name)
    bucket.put_object_from_file(dest, source)

def get_oss_path_from_uri(oss_model_dir, file_name):
    # oss_model_dir is of format: oss://bucket/path/to/dir/
    assert (oss_model_dir.startswith("oss://"))
    oss_file_path = "/".join([oss_model_dir.rstrip("/"), file_name])
    return oss_file_path

def mkdir(bucket, oss_dir):
    assert (oss_dir.startswith("oss://"))
    if not oss_dir.endswith("/"):
        oss_dir = oss_dir + "/"
    path = remove_bucket_prefix(oss_dir)
    has_dir = True
    try:
        bucket.get_object_meta(path)
    except oss2.exceptions.NoSuchKey:
        has_dir = False
    except Exception as e:
        raise e

    if not has_dir:
        bucket.put_object(path, "")

def delete_oss_dir_recursive(bucket, directory):
    """
    Recursively delete a directory on the OSS

    Args:
        bucket: bucket on OSS
        directory (str): the directory to delete

    Returns:
        None.
    """
    if not directory.endswith("/"):
        raise SQLFlowDiagnostic("dir to delete must end with /")

    loc = bucket.list_objects(prefix=directory, delimiter="/")
    object_path_list = []
    for obj in loc.object_list:
        object_path_list.append(obj.key)

    # delete sub dir first
    if len(loc.prefix_list) > 0:
        for sub_prefix in loc.prefix_list:
            delete_oss_dir_recursive(bucket, sub_prefix)
    # empty list param will raise error
    if len(object_path_list) > 0:
        bucket.batch_delete_objects(object_path_list)

def save_dir(oss_model_dir, local_dir):
    '''
    Recursively upload local_dir under oss_model_dir
    '''
    bucket = get_models_bucket()
    for (root, dirs, files) in os.walk(local_dir, topdown=True):
        if not six.PY2:
            root = root.decode("utf-8")
        dst_dir = "/".join([oss_model_dir.rstrip("/"), root])
        mkdir(bucket, dst_dir)
        for file_name in files:
            if not six.PY2:
                file_name = file_name.decode("utf-8")
            curr_file_path = os.path.join(root, file_name)
            remote_file_path = "/".join([dst_dir.rstrip("/"), file_name])
            remote_file_path = remove_bucket_prefix(remote_file_path)
            bucket.put_object_from_file(remote_file_path, curr_file_path)

def load_dir(oss_model_dir):
    bucket = get_models_bucket()
    path = remove_bucket_prefix(oss_model_dir)
    prefix = "/".join(path.split("/")[:-1]) + "/"
    for obj in oss2.ObjectIterator(bucket, prefix=path):
        # remove prefix when writing to local, e.g.
        # remote: path/to/my/dir/
        # local: dir/
        if obj.key.endswith("/"):
            try:
                os.makedirs(obj.key.replace(prefix, ""))
            except Exception as e:
                sys.stderr.write("mkdir exception: %s\n" % str(e))
        else:
            bucket.get_object_to_file(obj.key, obj.key.replace(prefix, ""))

def save_file(oss_model_dir, local_file_name, oss_file_name=None):
    """
    Save the local file (file_name is a file under current directory)
    to OSS directory.

    Args:
        oss_model_dir (str): the OSS model directory. It is in the format
            of oss://bucket/path/to/dir/.
        local_file_name (str): the local file path.
        oss_file_name (str): the OSS file path to save. If None,
            use local_file_name as oss_file_name.

    Returns:
        None.
    """
    if oss_file_name is None:
        oss_file_name = local_file_name

    bucket = get_models_bucket()
    oss_path = get_oss_path_from_uri(oss_model_dir, oss_file_name)
    oss_path = remove_bucket_prefix(oss_path)

    mkdir(bucket, oss_model_dir)
    bucket.put_object_from_file(oss_path, local_file_name)

def save_string(oss_file_path, data):
    '''
    Save a string into an oss_file_path
    '''
    bucket = get_models_bucket()
    oss_dir = "/".join(oss_file_path.split("/")[:-1])
    mkdir(bucket, oss_dir)
    oss_file_path = remove_bucket_prefix(oss_file_path)
    bucket.put_object(oss_file_path, data)

def load_file(oss_model_dir, local_file_name, oss_file_name=None):
    """
    Load file from OSS to local directory.

    Args:
        oss_model_dir (str): the OSS model directory. It is in the format
            of oss://bucket/path/to/dir/.
        local_file_name (str): the local file path.
        oss_file_name (str): the OSS file path to load. If None,
            use local_file_name as oss_file_name.

    Returns:
        None.
    """
    if oss_file_name is None:
        oss_file_name = local_file_name

    oss_file_path = "/".join([oss_model_dir.rstrip("/"), oss_file_name])
    oss_file_path = remove_bucket_prefix(oss_file_path)
    bucket = get_models_bucket()
    bucket.get_object_to_file(oss_file_path, local_file_name)

def load_string(oss_file_path):
    data = load_bytes(oss_file_path)
    return data.decode("utf-8")

def load_bytes(oss_file_path):
    bucket = get_models_bucket()
    oss_file_path = remove_bucket_prefix(oss_file_path)
    return bucket.get_object(oss_file_path).read()

def save_metas(oss_model_dir, num_workers, file_name, *meta):
    '''
    Save model descriptions like the training SQL statements to OSS directory.
    Data are saved using pickle.
    it will report "can't pickle weakref objects" when using pickle.
    Args:
        oss_model_dir: OSS URI that the model will be saved to.
        *meta: python objects to be saved.
    Return:
        None
    '''
    if num_workers > 1:
        FLAGS = tf.app.flags.FLAGS
        if FLAGS.task_index != 0:
            print("skip saving model desc on workers other than worker 0")
            return
    oss_path = get_oss_path_from_uri(oss_model_dir, file_name)
    serialized = pickle.dumps(list(meta))
    save_string(oss_path, serialized)

    # write a file "file_name_estimator" to store the estimator name,
    # so we can determine if the estimator is BoostedTrees* when
    # explaining the model.
    estimator_file_name = "_".join([file_name, "estimator"])
    oss_path = get_oss_path_from_uri(oss_model_dir, estimator_file_name)
    save_string(oss_path, meta[0])

def load_metas(oss_model_dir, file_name):
    '''Load model meta which are saved by save_metas from OSS

    Args:
        oss_model_dir: OSS URI that the model meta saved to.
        file_name: meta data file name

    Returns:
        A list contains the saved python objects
    '''
    oss_path = "/".join([oss_model_dir.rstrip("/"), file_name])
    serialized = load_bytes(oss_path)
    return pickle.loads(serialized)

def load_oss_model(oss_model_dir, estimator):
    is_estimator = is_tf_estimator(estimator)
    # Keras single node is using h5 format to save the model, no need to deal
    # with export model format. Keras distributed mode will use estimator, so
    # this is also needed.
    if is_estimator:
        load_file(oss_model_dir, "exported_path")

    # NOTE(typhoonzero): directory "model_save" is hardcoded in
    # codegen/tensorflow/codegen.go
    load_dir(os.path.join(oss_model_dir, "model_save"))

def save_oss_model(oss_model_dir, model_name, is_estimator,
                   feature_column_names, feature_column_names_map,
                   feature_metas, label_meta, model_params,
                   feature_columns_code, num_workers):
    # Keras single node is using h5 format to save the model, no need to deal
    # with export model format. Keras distributed mode will use estimator, so
    # this is also needed.
    if is_estimator:
        with open("exported_path", "rb") as fn:
            saved_model_path = fn.read()
        save_dir(oss_model_dir, saved_model_path)
        save_file(oss_model_dir, "exported_path")
    else:
        if num_workers > 1:
            FLAGS = tf.app.flags.FLAGS
            if FLAGS.task_index == 0:
                save_file(oss_model_dir, "exported_path")
        else:
            save_dir(oss_model_dir, "model_save")

    save_metas(oss_model_dir, num_workers, "tensorflow_model_desc", model_name,
               feature_column_names, feature_column_names_map, feature_metas,
               label_meta, model_params, feature_columns_code)
<commit_msg>fix PAI py3 tests (#2930)<commit_after># Copyright 2020 The SQLFlow Authors. All rights reserved.
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import os
import pickle
import sys

import oss2
import tensorflow as tf
from runtime.diagnostics import SQLFlowDiagnostic
from runtime.tensorflow import is_tf_estimator

# NOTE(typhoonzero): hard code bucket name "sqlflow-models" as the bucket to
# save models trained.
SQLFLOW_MODELS_BUCKET = "sqlflow-models"

def remove_bucket_prefix(oss_uri):
    return oss_uri.replace("oss://%s/" % SQLFLOW_MODELS_BUCKET, "")

def get_models_bucket():
    return get_bucket(SQLFLOW_MODELS_BUCKET)

def get_bucket(name, ak=None, sk=None, endpoint=None):
    if ak is None:
        ak = os.getenv("SQLFLOW_OSS_AK", "")
    if sk is None:
        sk = os.getenv("SQLFLOW_OSS_SK", "")
    if endpoint is None:
        endpoint = os.getenv("SQLFLOW_OSS_MODEL_ENDPOINT", "")
    if ak == "" or sk == "":
        raise ValueError("must configure SQLFLOW_OSS_AK and SQLFLOW_OSS_SK "
                         "when submitting to PAI")
    if endpoint == "":
        raise ValueError(
            "must configure SQLFLOW_OSS_MODEL_ENDPOINT when submitting to PAI")
    auth = oss2.Auth(ak, sk)
    bucket = oss2.Bucket(auth, endpoint, name)
    return bucket

def copyfileobj(source, dest, ak, sk, endpoint, bucket_name):
    '''
    copy_file_to_oss copies alocal file (source) to an object on OSS (dest),
    overwrite if the oss object exists.
    '''
    auth = oss2.Auth(ak, sk)
    bucket = oss2.Bucket(auth, endpoint, bucket_name)
    bucket.put_object_from_file(dest, source)

def get_oss_path_from_uri(oss_model_dir, file_name):
    # oss_model_dir is of format: oss://bucket/path/to/dir/
    assert (oss_model_dir.startswith("oss://"))
    oss_file_path = "/".join([oss_model_dir.rstrip("/"), file_name])
    return oss_file_path

def mkdir(bucket, oss_dir):
    assert (oss_dir.startswith("oss://"))
    if not oss_dir.endswith("/"):
        oss_dir = oss_dir + "/"
    path = remove_bucket_prefix(oss_dir)
    has_dir = True
    try:
        bucket.get_object_meta(path)
    except oss2.exceptions.NoSuchKey:
        has_dir = False
    except Exception as e:
        raise e

    if not has_dir:
        bucket.put_object(path, "")

def delete_oss_dir_recursive(bucket, directory):
    """
    Recursively delete a directory on the OSS

    Args:
        bucket: bucket on OSS
        directory (str): the directory to delete

    Returns:
        None.
    """
    if not directory.endswith("/"):
        raise SQLFlowDiagnostic("dir to delete must end with /")

    loc = bucket.list_objects(prefix=directory, delimiter="/")
    object_path_list = []
    for obj in loc.object_list:
        object_path_list.append(obj.key)

    # delete sub dir first
    if len(loc.prefix_list) > 0:
        for sub_prefix in loc.prefix_list:
            delete_oss_dir_recursive(bucket, sub_prefix)
    # empty list param will raise error
    if len(object_path_list) > 0:
        bucket.batch_delete_objects(object_path_list)

def save_dir(oss_model_dir, local_dir):
    '''
    Recursively upload local_dir under oss_model_dir
    '''
    bucket = get_models_bucket()
    for (root, dirs, files) in os.walk(local_dir, topdown=True):
        dst_dir = "/".join([oss_model_dir.rstrip("/"), root])
        mkdir(bucket, dst_dir)
        for file_name in files:
            curr_file_path = os.path.join(root, file_name)
            remote_file_path = "/".join([dst_dir.rstrip("/"), file_name])
            remote_file_path = remove_bucket_prefix(remote_file_path)
            bucket.put_object_from_file(remote_file_path, curr_file_path)

def load_dir(oss_model_dir):
    bucket = get_models_bucket()
    path = remove_bucket_prefix(oss_model_dir)
    prefix = "/".join(path.split("/")[:-1]) + "/"
    for obj in oss2.ObjectIterator(bucket, prefix=path):
        # remove prefix when writing to local, e.g.
        # remote: path/to/my/dir/
        # local: dir/
        if obj.key.endswith("/"):
            try:
                os.makedirs(obj.key.replace(prefix, ""))
            except Exception as e:
                sys.stderr.write("mkdir exception: %s\n" % str(e))
        else:
            bucket.get_object_to_file(obj.key, obj.key.replace(prefix, ""))

def save_file(oss_model_dir, local_file_name, oss_file_name=None):
    """
    Save the local file (file_name is a file under current directory)
    to OSS directory.

    Args:
        oss_model_dir (str): the OSS model directory. It is in the format
            of oss://bucket/path/to/dir/.
        local_file_name (str): the local file path.
        oss_file_name (str): the OSS file path to save. If None,
            use local_file_name as oss_file_name.

    Returns:
        None.
    """
    if oss_file_name is None:
        oss_file_name = local_file_name

    bucket = get_models_bucket()
    oss_path = get_oss_path_from_uri(oss_model_dir, oss_file_name)
    oss_path = remove_bucket_prefix(oss_path)

    mkdir(bucket, oss_model_dir)
    bucket.put_object_from_file(oss_path, local_file_name)

def save_string(oss_file_path, data):
    '''
    Save a string into an oss_file_path
    '''
    bucket = get_models_bucket()
    oss_dir = "/".join(oss_file_path.split("/")[:-1])
    mkdir(bucket, oss_dir)
    oss_file_path = remove_bucket_prefix(oss_file_path)
    bucket.put_object(oss_file_path, data)

def load_file(oss_model_dir, local_file_name, oss_file_name=None):
    """
    Load file from OSS to local directory.

    Args:
        oss_model_dir (str): the OSS model directory. It is in the format
            of oss://bucket/path/to/dir/.
        local_file_name (str): the local file path.
        oss_file_name (str): the OSS file path to load. If None,
            use local_file_name as oss_file_name.

    Returns:
        None.
    """
    if oss_file_name is None:
        oss_file_name = local_file_name

    oss_file_path = "/".join([oss_model_dir.rstrip("/"), oss_file_name])
    oss_file_path = remove_bucket_prefix(oss_file_path)
    bucket = get_models_bucket()
    bucket.get_object_to_file(oss_file_path, local_file_name)

def load_string(oss_file_path):
    data = load_bytes(oss_file_path)
    return data.decode("utf-8")

def load_bytes(oss_file_path):
    bucket = get_models_bucket()
    oss_file_path = remove_bucket_prefix(oss_file_path)
    return bucket.get_object(oss_file_path).read()

def save_metas(oss_model_dir, num_workers, file_name, *meta):
    '''
    Save model descriptions like the training SQL statements to OSS directory.
    Data are saved using pickle.
    it will report "can't pickle weakref objects" when using pickle.
    Args:
        oss_model_dir: OSS URI that the model will be saved to.
        *meta: python objects to be saved.
    Return:
        None
    '''
    if num_workers > 1:
        FLAGS = tf.app.flags.FLAGS
        if FLAGS.task_index != 0:
            print("skip saving model desc on workers other than worker 0")
            return
    oss_path = get_oss_path_from_uri(oss_model_dir, file_name)
    serialized = pickle.dumps(list(meta))
    save_string(oss_path, serialized)

    # write a file "file_name_estimator" to store the estimator name,
    # so we can determine if the estimator is BoostedTrees* when
    # explaining the model.
    estimator_file_name = "_".join([file_name, "estimator"])
    oss_path = get_oss_path_from_uri(oss_model_dir, estimator_file_name)
    save_string(oss_path, meta[0])

def load_metas(oss_model_dir, file_name):
    '''Load model meta which are saved by save_metas from OSS

    Args:
        oss_model_dir: OSS URI that the model meta saved to.
        file_name: meta data file name

    Returns:
        A list contains the saved python objects
    '''
    oss_path = "/".join([oss_model_dir.rstrip("/"), file_name])
    serialized = load_bytes(oss_path)
    return pickle.loads(serialized)

def load_oss_model(oss_model_dir, estimator):
    is_estimator = is_tf_estimator(estimator)
    # Keras single node is using h5 format to save the model, no need to deal
    # with export model format. Keras distributed mode will use estimator, so
    # this is also needed.
    if is_estimator:
        load_file(oss_model_dir, "exported_path")

    # NOTE(typhoonzero): directory "model_save" is hardcoded in
    # codegen/tensorflow/codegen.go
    load_dir(os.path.join(oss_model_dir, "model_save"))

def save_oss_model(oss_model_dir, model_name, is_estimator,
                   feature_column_names, feature_column_names_map,
                   feature_metas, label_meta, model_params,
                   feature_columns_code, num_workers):
    # Keras single node is using h5 format to save the model, no need to deal
    # with export model format. Keras distributed mode will use estimator, so
    # this is also needed.
    if is_estimator:
        with open("exported_path", "rb") as fn:
            saved_model_path = fn.read()
        save_dir(oss_model_dir, saved_model_path)
        save_file(oss_model_dir, "exported_path")
    else:
        if num_workers > 1:
            FLAGS = tf.app.flags.FLAGS
            if FLAGS.task_index == 0:
                save_file(oss_model_dir, "exported_path")
        else:
            save_dir(oss_model_dir, "model_save")

    save_metas(oss_model_dir, num_workers, "tensorflow_model_desc", model_name,
               feature_column_names, feature_column_names_map, feature_metas,
               label_meta, model_params, feature_columns_code)
<|endoftext|>


<commit_before>"""
test_error_bars:
================

A module intended for use with Nose.

"""
from __future__ import absolute_import

from nose.tools import raises
from plotly.graph_objs import ErrorX, ErrorY
from plotly.exceptions import PlotlyDictKeyError

def test_instantiate_error_x():
    ErrorX()
    ErrorX(array=[1, 2, 3],
           arrayminus=[2, 1, 2],
           color='red',
           copy_ystyle=False,
           opacity=.4,
           symmetric=False,
           thickness=2,
           type='percent',
           value=1,
           valueminus=4,
           visible=True,
           width=5)

def test_instantiate_error_y():
    ErrorY()
    ErrorY(array=[1, 2, 3],
           arrayminus=[2, 1, 2],
           color='red',
           opacity=.4,
           symmetric=False,
           thickness=2,
           type='percent',
           value=1,
           valueminus=4,
           visible=True,
           width=5)

@raises(PlotlyDictKeyError)
def test_key_error():
    ErrorX(value=0.1, typ='percent', color='red')
<commit_msg>Update `test_error_bars.py`.<commit_after>"""
test_error_bars:
================

A module intended for use with Nose.

"""
from __future__ import absolute_import

from nose.tools import raises
from plotly.graph_objs import ErrorX, ErrorY
from plotly.exceptions import PlotlyDictKeyError

def test_instantiate_error_x():
    ErrorX()
    ErrorX(array=[1, 2, 3],
           arrayminus=[2, 1, 2],
           color='red',
           copy_ystyle=False,
           symmetric=False,
           thickness=2,
           type='percent',
           value=1,
           valueminus=4,
           visible=True,
           width=5)

def test_instantiate_error_y():
    ErrorY()
    ErrorY(array=[1, 2, 3],
           arrayminus=[2, 1, 2],
           color='red',
           symmetric=False,
           thickness=2,
           type='percent',
           value=1,
           valueminus=4,
           visible=True,
           width=5)

@raises(PlotlyDictKeyError)
def test_key_error():
    ErrorX(value=0.1, typ='percent', color='red')
<|endoftext|>