<commit_before>#! /usr/bin/env python

# -*- coding: utf-8 -*-

import pygtk
pygtk.require('2.0')
import gtk

class Region:

    def __init__(self):
        self.start_x = None
        self.start_y = None
        self.end_x = None
        self.end_y = None
        self.track_started = False
        self.track_ended = False

class Coords:

    def __init__(self):
        self.window = gtk.Window(gtk.WINDOW_TOPLEVEL)
        self.window.set_default_size(300,200)

        self.window.add_events(gtk.gdk.MOTION_NOTIFY | gtk.gdk.BUTTON_PRESS)
        self.window.connect("motion_notify_event", self.mouse_moved)
        self.window.connect("destroy", self.destroy)

        self.setup_widgets()

        self.window.show()

        self.root_window = None

    def main(self):
        gtk.main()

    def setup_widgets(self):
        start_label = gtk.Label("start")
        start_x_label = gtk.Label("x")
        start_y_label = gtk.Label("y")
        start_x = gtk.Label("this will be x")
        start_y = gtk.Label("this will be y")

        end_label = gtk.Label("end")
        end_x_label = gtk.Label("x")
        end_y_label = gtk.Label("y")
        end_x = gtk.Label("this will be x")
        end_y = gtk.Label("this will be y")

        width_label = gtk.Label("width")
        width = gtk.Label("this will be w")

        height_label = gtk.Label("height")
        height = gtk.Label("this will be h")

        start_button = gtk.Button("start")
        reset_button = gtk.Button("reset")

        hbox_one = gtk.HBox(gtk.TRUE, 10)
        hbox_one.pack_start(start_label)
        hbox_one.pack_start(start_x_label)
        hbox_one.pack_start(start_x)

        hbox_two = gtk.HBox(gtk.TRUE, 10)
        hbox_two.pack_start(start_y_label)
        hbox_two.pack_start(start_y)

        hbox_three = gtk.HBox(gtk.TRUE, 10)
        hbox_three.pack_start(end_label)
        hbox_three.pack_start(end_x_label)
        hbox_three.pack_start(end_x)

        hbox_four = gtk.HBox(gtk.TRUE, 10)
        hbox_four.pack_start(end_y_label)
        hbox_four.pack_start(end_y)

        hbox_five = gtk.HBox(gtk.TRUE, 10)
        hbox_five.pack_start(width_label)
        hbox_five.pack_start(width)
        hbox_five.pack_start(start_button)

        hbox_six = gtk.HBox(gtk.TRUE, 10)
        hbox_six.pack_start(height_label)
        hbox_six.pack_start(height)
        hbox_six.pack_start(reset_button)

        main_box = gtk.VBox(gtk.TRUE, 10)
        main_box.pack_start(hbox_one)
        main_box.pack_start(hbox_two)
        main_box.pack_start(hbox_three)
        main_box.pack_start(hbox_four)
        main_box.pack_start(hbox_five)
        main_box.pack_start(hbox_six)

        start_label.show()
        start_x_label.show()
        start_x.show()
        hbox_one.show()

        start_y_label.show()
        start_y.show()
        hbox_two.show()

        end_label.show()
        end_x_label.show()
        end_x.show()
        hbox_three.show()

        end_y_label.show()
        end_y.show()
        hbox_four.show()

        width_label.show()
        width.show()
        start_button.show()
        hbox_five.show()

        height_label.show()
        height.show()
        reset_button.show()
        hbox_six.show()

        self.window.add(main_box)
        main_box.show()

        self.start_button = start_button
        self.start_button.connect("clicked", self.start)

    def start(self, widget, data=None):
        region = Region()
        mask = gtk.gdk.POINTER_MOTION_MASK | gtk.gdk.BUTTON_PRESS_MASK | gtk.gdk.BUTTON_RELEASE_MASK
        self.root_window = gtk.gdk.get_default_root_window()
        result = gtk.gdk.pointer_grab(self.root_window, False, mask, None, None)
        self.root_window.add_filter(self.region, region)

    def region(self, event, region):
        x, y, flags = event.window.get_pointer()
        if 'GDK_BUTTON1_MASK' in flags.value_names \
                and region.track_started == False:
            region.start_x = x
            region.start_y = y
            region.track_started = True
            region.track_ended = False
            print 'started'
        if 'GDK_BUTTON1_MASK' not in flags.value_names \
                and region.track_started == True:
            region.end_x = x
            region.end_y = y
            region.track_ended = True
            region.track_started = False
            print 'ended'
            print region.start_x
            print region.start_y
            print region.end_x
            print region.end_y
        return gtk.gdk.FILTER_CONTINUE

    def mouse_moved(self, widget, data=None):
        #print gtk.gdk.display_get_default().get_pointer()
        pass

    def destroy(self, widget, data=None):
        gtk.main_quit()

if __name__ == "__main__":
    coords = Coords()
    coords.main()
<commit_msg>show coordinates. use only one region object<commit_after>#! /usr/bin/env python

# -*- coding: utf-8 -*-

import pygtk
pygtk.require('2.0')
import gtk

class Region:

    def __init__(self):
        self.start_x = None
        self.start_y = None
        self.end_x = None
        self.end_y = None
        self.track_started = False
        self.track_ended = False

    def get_width(self):
        if self.start_x is None or self.end_x is None:
            return 0

        return max(self.start_x, self.end_x) - min(self.start_x, self.end_x)

    def get_height(self):
        if self.start_y is None or self.end_y is None:
            return 0

        return max(self.start_y, self.end_y) - min(self.start_y, self.end_y)

class Coords:

    def __init__(self):
        self.window = gtk.Window(gtk.WINDOW_TOPLEVEL)
        self.window.set_default_size(300,200)

        self.window.add_events(gtk.gdk.MOTION_NOTIFY | gtk.gdk.BUTTON_PRESS)
        self.window.connect("motion_notify_event", self.mouse_moved)
        self.window.connect("destroy", self.destroy)

        self.setup_widgets()

        self.window.show()

        self.root_window = None

        self.region = Region()

    def main(self):
        gtk.main()

    def setup_widgets(self):
        start_label = gtk.Label("start")
        start_x_label = gtk.Label("x")
        start_y_label = gtk.Label("y")
        start_x = gtk.Label("this will be x")
        start_y = gtk.Label("this will be y")

        end_label = gtk.Label("end")
        end_x_label = gtk.Label("x")
        end_y_label = gtk.Label("y")
        end_x = gtk.Label("this will be x")
        end_y = gtk.Label("this will be y")

        width_label = gtk.Label("width")
        width = gtk.Label("this will be w")

        height_label = gtk.Label("height")
        height = gtk.Label("this will be h")

        start_button = gtk.Button("start")
        reset_button = gtk.Button("reset")

        hbox_one = gtk.HBox(gtk.TRUE, 10)
        hbox_one.pack_start(start_label)
        hbox_one.pack_start(start_x_label)
        hbox_one.pack_start(start_x)

        hbox_two = gtk.HBox(gtk.TRUE, 10)
        hbox_two.pack_start(start_y_label)
        hbox_two.pack_start(start_y)

        hbox_three = gtk.HBox(gtk.TRUE, 10)
        hbox_three.pack_start(end_label)
        hbox_three.pack_start(end_x_label)
        hbox_three.pack_start(end_x)

        hbox_four = gtk.HBox(gtk.TRUE, 10)
        hbox_four.pack_start(end_y_label)
        hbox_four.pack_start(end_y)

        hbox_five = gtk.HBox(gtk.TRUE, 10)
        hbox_five.pack_start(width_label)
        hbox_five.pack_start(width)
        hbox_five.pack_start(start_button)

        hbox_six = gtk.HBox(gtk.TRUE, 10)
        hbox_six.pack_start(height_label)
        hbox_six.pack_start(height)
        hbox_six.pack_start(reset_button)

        main_box = gtk.VBox(gtk.TRUE, 10)
        main_box.pack_start(hbox_one)
        main_box.pack_start(hbox_two)
        main_box.pack_start(hbox_three)
        main_box.pack_start(hbox_four)
        main_box.pack_start(hbox_five)
        main_box.pack_start(hbox_six)

        start_label.show()
        start_x_label.show()
        start_x.show()
        hbox_one.show()

        start_y_label.show()
        start_y.show()
        hbox_two.show()

        end_label.show()
        end_x_label.show()
        end_x.show()
        hbox_three.show()

        end_y_label.show()
        end_y.show()
        hbox_four.show()

        width_label.show()
        width.show()
        start_button.show()
        hbox_five.show()

        height_label.show()
        height.show()
        reset_button.show()
        hbox_six.show()

        self.window.add(main_box)
        main_box.show()

        self.start_button = start_button
        self.start_button.connect("clicked", self.start)

        reset_button.connect('clicked', self.reset)

        self.start_x = start_x
        self.start_y = start_y
        self.end_x = end_x
        self.end_y = end_y
        self.width = width
        self.height = height

    def reset(self, widget, data=None):
        print 'clicked'

    def start(self, widget, data=None):
        mask = gtk.gdk.POINTER_MOTION_MASK | gtk.gdk.BUTTON_PRESS_MASK | gtk.gdk.BUTTON_RELEASE_MASK
        self.root_window = gtk.gdk.get_default_root_window()
        result = gtk.gdk.pointer_grab(self.root_window, False, mask, None, None)
        self.root_window.add_filter(self.track_region, self.region)

    def track_region(self, event, region):
        x, y, flags = event.window.get_pointer()
        if 'GDK_BUTTON1_MASK' in flags.value_names \
                and region.track_started == False:
            region.start_x = x
            region.start_y = y
            region.track_started = True
            region.track_ended = False
        if 'GDK_BUTTON1_MASK' not in flags.value_names \
                and region.track_started == True:
            region.end_x = x
            region.end_y = y
            region.track_ended = True
            region.track_started = False
            gtk.gdk.pointer_ungrab()
            self.show_region(region)
        return gtk.gdk.FILTER_CONTINUE

    def show_region(self, region):
        self.start_x.set_text('%s' % region.start_x)
        self.start_y.set_text('%s' % region.start_y)
        self.end_x.set_text('%s' % region.end_x)
        self.end_y.set_text('%s' % region.end_y)
        self.width.set_text('%s' % region.get_width())
        self.height.set_text('%s' % region.get_height())

    def mouse_moved(self, widget, data=None):
        #print gtk.gdk.display_get_default().get_pointer()
        pass

    def destroy(self, widget, data=None):
        gtk.main_quit()

if __name__ == "__main__":
    coords = Coords()
    coords.main()
<|endoftext|>


<commit_before>#!/usr/bin/env python3
# -*- encoding: utf-8 -*-

"""
corpus.py generates a corpus file (outfile) out of a bunch of gunzipped xml
subtitle files in a directory and all its subdirectories.

Usage:
    corpus.py [-n=<n_threads>] [-v] <directory> <outfile>
    corpus.py -h | --help
    corpus.py --version

Options:
    -h, --help      Show this screen.
    -n=<n_threads>  Number of threads to use [default: 1].
    -v              Verbose output.

"""

__version__ = '0.1.0'

import fileinput
import re
import os
import time
import sys
import gzip
import multiprocessing

from docopt import docopt

# define rules for substitution
PATTERNS = (
        # remove xml tags
        re.compile('<.*?>', re.UNICODE+re.IGNORECASE),
        # remove everything within paranthesis
        re.compile('\(.*?\)', re.UNICODE),
        # remove everything within square brackets
        re.compile('\[.*?\]', re.UNICODE),
        )

def read_clean_gzfile(gz_file_path):
    """
    Generator that opens and reads a gunzipped xml subtitle file, while all
    xml tags and timestamps are removed.

    Yields
    ======
    line : non empty, cleaned line out of the xml subtitle file

    Raises
    ======
    FileNotFoundError : if file is not there.

    """
    with gzip.open(gz_file_path, "rt", encoding="utf-8-sig") as file_:
        for line in file_:
            result = line.strip().lstrip().rstrip()
            for pattern in PATTERNS:
                result = pattern.sub('', result)
            if not result:
                continue
            yield result + "\n"

def _job(filename):
    """Job for threads in multiprocessing."""
    lines = None
    not_found = None
    try:
        lines = list(read_clean_gzfile(filename))
        lines.append("---END.OF.DOCUMENT---\n")
    except FileNotFoundError:
        not_found = filename + "\n"
    return (lines, not_found)

def main(directory, outfile, *, n_threads=1, verbose=False):
    """
    Mein corpus.py program that starts the multiple processes and collects all
    the results.

    Parameters
    ==========
    directory : str
        use all gz-files in this directory and all subdirectories as input.
    outfile : str
        name of the outfile that will be created.
    n_threads : int
        number of threads to use.
    verbose : bool

    """

    if not os.path.isdir(directory):
        raise OSError("%s does not exist.")
    if os.path.isfile(outfile):
        raise OSError("%s exists. Please <outfile> needs to be new file name."
                      % outfile)
    # create file generator
    gz_files = [os.path.join(root, name)
             for root, dirs, files in os.walk(directory)
             for name in files
             if name.endswith((".gz",))]

    if verbose:
        print("Start processing %i files." % len(gz_files))
        start_time = time.time()
    not_founds = list()
    with multiprocessing.Pool(n_threads) as pool:
        with open(outfile, "wt") as result_file:
            progress_counter = 0
            n_files = len(gz_files)
            for lines, not_found in pool.imap_unordered(_job, gz_files):

                progress_counter += 1
                if verbose and progress_counter % 1000 == 0:
                    print("%i%% " % (progress_counter / n_files * 100), end="")
                    sys.stdout.flush()

                if lines is not None:
                    result_file.writelines(lines)
                elif not_found is not None:
                    not_founds.append(not_found)
                else:
                    raise NotImplementedError("This should never happend!")
    if verbose:
        duration = time.time() - start_time
        print("\nProcessed %i files. %i files where not found." %
              (len(gz_files), len(not_founds)))
        print("Processing took %.2f seconds (%ih%.2im)." % (duration, duration //
                                                          (60 * 60), duration // 60) )

    if not_founds:
        # prevent overwriting files
        file_name = outfile + ".not_found"
        nn = 1
        while True:
            if os.path.isfile(file_name):
                nn += 1
                file_name = outfile + ".not_found-" + str(nn)
                continue
            else:
                break

        with open(file_name, "wt") as not_found_file:
            not_found_file.writelines(not_founds)

if __name__ == "__main__":
    arguments = docopt(__doc__, version='corpus %s' % __version__)
    main(arguments['<directory>'],
         arguments['<outfile>'],
         n_threads=int(arguments['-n']),
         verbose=arguments['-v'])

<commit_msg>small improvement in corpus.py<commit_after>#!/usr/bin/env python3
# -*- encoding: utf-8 -*-

"""
corpus.py generates a corpus file (outfile) out of a bunch of gunzipped xml
subtitle files in a directory and all its subdirectories.

Usage:
    corpus.py [-n=N_THREADS] [-v] <directory> <outfile>
    corpus.py -h | --help
    corpus.py --version

Options:
    -h, --help      Show this screen.
    -n=N_THREADS    Number of threads to use [default: 1].
    -v              Verbose output.

"""

__version__ = '0.1.0'

import fileinput
import re
import os
import time
import sys
import gzip
import multiprocessing

from docopt import docopt

# define rules for substitution
PATTERNS = (
        # remove xml tags
        re.compile('<.*?>', re.UNICODE+re.IGNORECASE),
        # remove everything within paranthesis
        re.compile('\(.*?\)', re.UNICODE),
        # remove everything within square brackets
        re.compile('\[.*?\]', re.UNICODE),
        )

def read_clean_gzfile(gz_file_path):
    """
    Generator that opens and reads a gunzipped xml subtitle file, while all
    xml tags and timestamps are removed.

    Yields
    ======
    line : non empty, cleaned line out of the xml subtitle file

    Raises
    ======
    FileNotFoundError : if file is not there.

    """
    with gzip.open(gz_file_path, "rt", encoding="utf-8-sig") as file_:
        for line in file_:
            result = line.strip().lstrip().rstrip()
            for pattern in PATTERNS:
                result = pattern.sub('', result)
            if not result:
                continue
            yield result + "\n"

def _job(filename):
    """Job for threads in multiprocessing."""
    lines = None
    not_found = None
    try:
        lines = list(read_clean_gzfile(filename))
        lines.append("---END.OF.DOCUMENT---\n")
    except FileNotFoundError:
        not_found = filename + "\n"
    return (lines, not_found)

def main(directory, outfile, *, n_threads=1, verbose=False):
    """
    Mein corpus.py program that starts the multiple processes and collects all
    the results.

    Parameters
    ==========
    directory : str
        use all gz-files in this directory and all subdirectories as input.
    outfile : str
        name of the outfile that will be created.
    n_threads : int
        number of threads to use.
    verbose : bool

    """

    if not os.path.isdir(directory):
        raise OSError("%s does not exist.")
    if os.path.isfile(outfile):
        raise OSError("%s exists. Please <outfile> needs to be new file name."
                      % outfile)
    # create file generator
    gz_files = [os.path.join(root, name)
             for root, dirs, files in os.walk(directory)
             for name in files
             if name.endswith((".gz",))]

    if verbose:
        print("Start processing %i files." % len(gz_files))
        start_time = time.time()
    not_founds = list()
    with multiprocessing.Pool(n_threads) as pool:
        with open(outfile, "wt") as result_file:
            progress_counter = 0
            n_files = len(gz_files)
            for lines, not_found in pool.imap_unordered(_job, gz_files):

                progress_counter += 1
                if verbose and progress_counter % 1000 == 0:
                    print("%i%% " % (progress_counter / n_files * 100), end="")
                    sys.stdout.flush()

                if lines is not None:
                    result_file.writelines(lines)
                elif not_found is not None:
                    not_founds.append(not_found)
                else:
                    raise NotImplementedError("This should never happend!")
    if verbose:
        duration = time.time() - start_time
        print("\nProcessed %i files. %i files where not found." %
              (len(gz_files), len(not_founds)))
        print("Processing took %.2f seconds (%ih%.2im)." % (duration, duration //
                                                          (60 * 60), duration // 60) )

    if not_founds:
        # prevent overwriting files
        file_name = outfile + ".not_found"
        nn = 1
        while True:
            if os.path.isfile(file_name):
                nn += 1
                file_name = outfile + ".not_found-" + str(nn)
                continue
            else:
                break

        with open(file_name, "wt") as not_found_file:
            not_found_file.writelines(not_founds)

if __name__ == "__main__":
    arguments = docopt(__doc__, version='corpus %s' % __version__)
    main(arguments['<directory>'],
         arguments['<outfile>'],
         n_threads=int(arguments['-n']),
         verbose=arguments['-v'])

<|endoftext|>


<commit_before>import os
import time

from pymongo import MongoClient
from nltk.stem.wordnet import WordNetLemmatizer

connection = "mongodb://localhost:27030/"
dbTags = MongoClient(connection).Tags

reviews_cursor = dbTags.Reviews.find()
reviewsCount = reviews_cursor.count()
reviews_cursor.batch_size(5000)

lem = WordNetLemmatizer()

done = 0
start = time.time()

for review in reviews_cursor:
    nouns = []
    words = [word for word in review["words"] if word["pos"] in ["NN", "NNS"]]

    for word in words:
        nouns.append(lem.lemmatize(word["word"]))

    dbTags.Corpus.insert({
        "reviewId": review["reviewId"],
        "business": review["business"],
        "text": review["text"],
        "words": nouns
    })

    done += 1
    if done % 100 == 0:
        end = time.time()
        os.system('cls')
        print 'Done ' + str(done) + ' out of ' + str(reviewsCount) + ' in ' + str((end - start))<commit_msg>adapted corpus class to use external settings<commit_after>import os
import time

from pymongo import MongoClient
from nltk.stem.wordnet import WordNetLemmatizer

from settings import Settings

tags_collection = MongoClient(Settings.MONGO_CONNECTION_STRING)[Settings.TAGS_DATABASE][Settings.REVIEWS_COLLECTION]
corpus_collection = MongoClient(Settings.MONGO_CONNECTION_STRING)[Settings.TAGS_DATABASE][Settings.CORPUS_COLLECTION]

reviews_cursor = tags_collection.find()
reviewsCount = reviews_cursor.count()
reviews_cursor.batch_size(5000)

lem = WordNetLemmatizer()

done = 0
start = time.time()

for review in reviews_cursor:
    nouns = []
    words = [word for word in review["words"] if word["pos"] in ["NN", "NNS"]]

    for word in words:
        nouns.append(lem.lemmatize(word["word"]))

    corpus_collection.insert({
        "reviewId": review["reviewId"],
        "business": review["business"],
        "text": review["text"],
        "words": nouns
    })

    done += 1
    if done % 100 == 0:
        end = time.time()
        os.system('cls')
        print 'Done ' + str(done) + ' out of ' + str(reviewsCount) + ' in ' + str((end - start))<|endoftext|>


<commit_before>import keystoneclient.v2_0.client as ksclient
import glanceclient as gcclient
import novaclient.v1_1.client as nvclient

from Setting import getKsCreInfo, getNovaCreInfo

def create():
    ksCreInfo = getKsCreInfo()
    novaCreInfo = getNovaCreInfo()
    keystoneCli = ksclient.Client(**ksCreInfo)
    novaCli = nvclient.Client(**novaCreInfo)
    gcEndpoint = keystoneCli.service_catalog.get_urls(service_type='image')[0]
    glanceCli = gcclient.Client('1', gcEndpoint, token=keystoneCli.auth_token)

    images = glanceCli.images.list()
    image_create = None
    for image in images:
        if not image.name.find('ubuntu') == -1:
             image_create = image
             break

    flavor = novaCli.flavors.find(name='m1.micro')
    instance = novaCli.servers.create(name="test", image=image_create, flavor=flavor)
    print "starting spawn new instance......"
    print "new instance name : %s " % instance.name
    print "new instance id : %s " % instance.id
    print 'using "nova list" to check status'

if __name__ == '__main__':
    create()

<commit_msg>delete extra spaces<commit_after>import keystoneclient.v2_0.client as ksclient
import glanceclient as gcclient
import novaclient.v1_1.client as nvclient

from Setting import getKsCreInfo, getNovaCreInfo

def create():
    ksCreInfo = getKsCreInfo()
    novaCreInfo = getNovaCreInfo()
    keystoneCli = ksclient.Client(**ksCreInfo)
    novaCli = nvclient.Client(**novaCreInfo)
    gcEndpoint = keystoneCli.service_catalog.get_urls(service_type='image')[0]
    glanceCli = gcclient.Client('1', gcEndpoint, token=keystoneCli.auth_token)

    images = glanceCli.images.list()
    image_create = None
    for image in images:
        if not image.name.find('ubuntu') == -1:
             image_create = image
             break

    flavor = novaCli.flavors.find(name='m1.micro')
    instance = novaCli.servers.create(name="test", image=image_create, flavor=flavor)
    print "starting spawn new instance......"
    print "new instance name : %s " % instance.name
    print "new instance id : %s " % instance.id
    print 'using "nova list" to check status'

if __name__ == '__main__':
    create()
<|endoftext|>


<commit_before># Copyright 2017 Jonathan Anderson
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are met:
#
# 1. Redistributions of source code must retain the above copyright notice, this
# list of conditions and the following disclaimer.
#
# 2. Redistributions in binary form must reproduce the above copyright notice,
# this list of conditions and the following disclaimer in the documentation
# and/or other materials provided with the distribution.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

import base64
import config
import hashlib
import hmac as hm
import os

secret = os.environ.get("SECRET_KEY")
if secret is None:
    import random, warnings
    warnings.warn(
        'No SECRET_KEY in .env or environment; generating one-time key\n')

    secret = random.getrandbits(128)

def hmac(text):
    assert type(text) == str
    assert len(text) > 0

    h = hm.new(secret, text, hashlib.sha512)
    return base64.b32encode(h.digest()).lower()[:20]

def verify(text, mac):
    return (mac == hmac(text))
<commit_msg>Hash manually-provided secrets.<commit_after># Copyright 2017 Jonathan Anderson
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are met:
#
# 1. Redistributions of source code must retain the above copyright notice, this
# list of conditions and the following disclaimer.
#
# 2. Redistributions in binary form must reproduce the above copyright notice,
# this list of conditions and the following disclaimer in the documentation
# and/or other materials provided with the distribution.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

import base64
import config
import hashlib
import hmac as hm
import os

secret = os.environ.get("SECRET_KEY")
if secret is None:
    import random, warnings
    warnings.warn(
        'No SECRET_KEY in .env or environment; generating one-time key\n')

    secret = random.getrandbits(128)

else:
    secret = hashlib.sha512(secret).digest()

def hmac(text):
    assert type(text) == str
    assert len(text) > 0

    h = hm.new(secret, text, hashlib.sha512)
    return base64.b32encode(h.digest()).lower()[:20]

def verify(text, mac):
    return (mac == hmac(text))
<|endoftext|>


<commit_before>import hmac
import hashlib
import os
import binascii
import json
from hkdf import derive
from datetime import datetime, timedelta

_SIZE = 128
_HASH = hashlib.sha1

def generate_secret(size=_SIZE):
    return binascii.b2a_hex(os.urandom(size))[:size]

def sign(token, secret):
    if len(secret) != 128:
        raise ValueError("Invalid secret")
    token['signature'] = _signature(token, secret)
    return token

def _signature(token, secret):
    token = token.copy()
    if 'signature' in token:
        del token['signature']
    token = token.items()
    token.sort()
    return hmac.new(secret, json.dumps(token), _HASH).hexdigest()

def verify(token, secret):
    if len(secret) != 128:
        raise ValueError("Invalid secret")
    signature = token['signature']
    wanted = _signature(token, secret)
    if signature != wanted:
        raise ValueError('Invalid Token')

def create_header(token):
    return 'Authorization: MozToken %s' % json.dumps(token)

def extract_token(header):
    if not header.startswith('Authorization: MozToken '):
        raise ValueError("Invalid token")
    token = header[len('Authorization: MozToken '):]
    try:
        token = json.loads(token)
    except ValueError:
        raise ValueError("Invalid token")
    return token

def create_token(uid, secret, expires=None):
    if expires is None:
        expires = datetime.now() + timedelta(minutes=30)
    token = {'uid': uid, 'expires': expires.isoformat()}
    sign(token, secret)
    return token

if __name__ == '__main__':
    print('Creating a secret')
    secret = generate_secret()
    print(secret)

    print('Derive the client secret')
    dsecret = derive(secret, 128, salt="salted_value")
    print(dsecret)

    print('Derive the node server secret')
    dsecret2 = derive(secret, 128, salt="salted_value")
    print(dsecret2)

    print('========= SERVER ==========')
    print('Creating the signed token')
    token = create_token('123', dsecret)
    print token

    print('creating a header with it')
    header = create_header(token)
    print header

    print('========= NODE ==========')
    print('extracting the token from the header')

    token = extract_token(header)
    print header

    print "validating the signature"

    verify(token, dsecret)
<commit_msg>update the token server crypto proto with the last modifications on the wiki page<commit_after>import hmac
import hashlib
import os
import binascii
import json
from hkdf import derive
from datetime import datetime, timedelta
import time

_SIZE = 256
_HASH = hashlib.sha1

def random_value(size=_SIZE):
    return binascii.b2a_hex(os.urandom(size))[:size]

def sign(token, secret, key='signature'):
    if len(secret) != _SIZE:
        raise ValueError("Invalid secret")
    token[key] = _signature(token, secret, key)
    return token

def oauth_sign(token, secret):
    token['oauth_timestamp'] = time.time()
    token['oauth_nonce'] = random_value(128)
    token = sign(token, secret, key='oauth_signature')
    token['oauth_signature_method'] = 'HMAC-SHA1'  # XXX
    return token

def _signature(token, secret, key='signature'):
    token = token.copy()
    if key in token:
        del token[key]
    token = token.items()
    token.sort()
    return hmac.new(secret, json.dumps(token), _HASH).hexdigest()

def verify(token, secret, key="signature"):
    if len(secret) != _SIZE:
        raise ValueError("Invalid secret")
    signature = token[key]
    wanted = _signature(token, secret, key)
    if signature != wanted:
        raise ValueError('Invalid Token')

def create_header(token):
    return 'Authorization: MozToken %s' % json.dumps(token)

def extract_token(header):
    if not header.startswith('Authorization: MozToken '):
        raise ValueError("Invalid token")
    token = header[len('Authorization: MozToken '):]
    try:
        token = json.loads(token)
    except ValueError:
        raise ValueError("Invalid token")
    return token

def create_token(uid, secret, expires=None):
    if expires is None:
        expires = datetime.now() + timedelta(minutes=30)
    token = {'uid': uid, 'expires': time.mktime(expires.timetuple())}
    sign(token, secret)
    return token

def create_token_info(token, token_secret, salt, node, metadata=None):
    info = {
        'oauth_consumer_key': token,
        'oauth_consumer_secret': token_secret,
        'salt': salt,
        'app_node': node,
    }

    if metadata is not None:
        info['app_metadata'] = metadata

    return info

def get_secrets():
    master = random_value()
    sig_secret = derive(master, _SIZE, salt="SIGN")

    return master, sig_secret

def get_token(master, sig_secret, idx, node):
    salt = random_value()  # XXX define what's the salt lenght

    auth_token = create_token(idx, sig_secret)
    token_secret = derive(master, _SIZE, salt=salt)

    return create_token_info(auth_token, token_secret, salt, node)

if __name__ == '__main__':
    import sys

    master, sig_secret = get_secrets()
    for x in range(int(sys.argv[1])):
        print("Generates a token")
        print(get_token(master, sig_secret, 123, "phx2"))
<|endoftext|>


<commit_before>from cum import config, output, sanity
from natsort import humansorted
from shutil import copyfile
from sqlalchemy import (
    Boolean,
    Column,
    create_engine,
    DateTime,
    ForeignKey,
    Integer,
    String,
    Table
)
from sqlalchemy.ext.declarative import declarative_base
from sqlalchemy.orm import relationship, sessionmaker
from sqlalchemy.orm.exc import NoResultFound
from urllib.parse import urlparse
import click
import datetime
import os
import sqlalchemy.engine.url

Base = declarative_base()

group_table = Table(
    'group_association', Base.metadata,
    Column('chapter_id', Integer, ForeignKey('chapters.id')),
    Column('group_id', Integer, ForeignKey('groups.id'))
)

class Series(Base):
    __tablename__ = 'series'

    id = Column(Integer, primary_key=True)

    name = Column(String)
    alias = Column(String, unique=True)
    url = Column(String, unique=True)
    following = Column(Boolean, default=True)
    directory = Column(String)

    chapters = relationship("Chapter", backref="series")

    def __init__(self, series):
        self.name = series.name
        self.alias = series.alias
        self.url = series.url
        self.directory = series.directory

    @staticmethod
    def alias_lookup(alias):
        """Returns a DB object for a series by alias name. Prints an error if
        an invalid alias is specified.
        """
        try:
            s = (session.query(Series)
                 .filter_by(alias=alias, following=True)
                 .one())
        except NoResultFound:
            output.error('Could not find alias "{}"'.format(alias))
            exit(1)
        else:
            return s

    @property
    def last_added(self):
        """Returns the last time a chapter has been added to the series."""
        updates = sorted(self.chapters, reverse=True, key=lambda x: x.added_on)
        if updates:
            return updates[0].added_on
        else:
            return None

    @property
    def ordered_chapters(self):
        return humansorted(self.chapters, key=lambda x: x.chapter)

class Chapter(Base):
    __tablename__ = 'chapters'

    id = Column(Integer, primary_key=True)
    series_id = Column(Integer, ForeignKey('series.id'))

    # Downloaded has the value -1 for ignored chapters, 0 for new chapters and
    # 1 for downloaded chapters.
    downloaded = Column(Integer, default=0)
    chapter = Column(String)
    url = Column(String, unique=True)
    title = Column(String)
    added_on = Column(DateTime)

    groups = relationship('Group', secondary=group_table, backref='chapters')

    def __init__(self, chapter, series):
        self.series = series
        self.chapter = chapter.chapter
        self.title = chapter.title
        self.url = chapter.url
        self.added_on = datetime.datetime.now()

        self.groups = []
        for group in chapter.groups:
            try:
                g = session.query(Group).filter(Group.name == group).one()
            except NoResultFound:
                g = Group(group)
                session.add(g)
                session.commit()
            self.groups.append(g)

    @staticmethod
    def find_new(alias=None):
        """Return a list of new chapters as Chapter objects and applies human
        sorting to it. Accepts an optional 'alias' argument, which will filter
        the query.
        """
        query = session.query(Chapter).join(Series).filter(Series.following)
        if alias:
            query = query.filter(Series.alias == alias)
        query = query.filter(Chapter.downloaded == 0).all()
        return humansorted([x.to_object() for x in query],
                           key=lambda x: x.chapter)

    @property
    def group_tag(self):
        """Return a joined string of chapter's groups enclosed in brackets."""
        return ''.join(['[{}]'.format(x.name) for x in self.groups])

    @property
    def status(self):
        """Return the chapter's downloaded status as a one character flag."""
        if self.downloaded == 0:
            return 'n'
        elif self.downloaded == -1:
            return 'i'
        else:
            return ' '

    def to_object(self):
        """Turns a database entry into a chapter object for the respective
        site by parsing the URL.
        """
        parse = urlparse(self.url)
        kwargs = {'name': self.series.name,
                  'alias': self.series.alias,
                  'chapter': self.chapter,
                  'url': self.url,
                  'groups': self.groups,
                  'directory': self.series.directory}
        if parse.netloc == 'bato.to':
            from cum.scrapers.batoto import BatotoChapter
            return BatotoChapter(**kwargs)
        elif parse.netloc == 'dynasty-scans.com':
            from cum.scrapers.dynastyscans import DynastyScansChapter
            return DynastyScansChapter(**kwargs)
        elif parse.netloc == 'manga.madokami.com':
            from cum.scrapers.madokami import MadokamiChapter
            return MadokamiChapter(**kwargs)
        else:
            return None

class Group(Base):
    __tablename__ = 'groups'

    id = Column(Integer, primary_key=True)
    name = Column(String, unique=True)

    def __init__(self, name):
        self.name = name

    def __str__(self):
        return self.name

def backup_database():
    """Backs up the database file to a file called cum.db.bak."""
    db_path = os.path.join(config.cum_dir, 'cum.db')
    backup_path = os.path.join(config.cum_dir, 'cum.db.bak')
    copyfile(db_path, backup_path)

def initialize():
    global db_path, engine, session
    db_path = os.path.join(config.cum_dir, 'cum.db')
    db_url = sqlalchemy.engine.url.URL('sqlite', database=db_path)
    engine = create_engine(db_url)
    if not os.path.exists(db_path):
        Base.metadata.create_all(bind=engine)
    Session = sessionmaker(bind=engine)
    session = Session()

def test_database():
    """Runs a database sanity test."""
    sanity_tester = sanity.DatabaseSanity(Base, engine)
    sanity_tester.test()
    if sanity_tester.errors:
        for error in sanity_tester.errors:
            err_target, err_msg = str(error).split(' ', 1)
            message = ' '.join([click.style(err_target, bold=True), err_msg])
            output.warning(message)
        output.error('Database has failed sanity check; '
                     'run `cum repair-db` to repair database')
        exit(1)
<commit_msg>Fix last_added sorting bug.<commit_after>from cum import config, output, sanity
from natsort import humansorted
from shutil import copyfile
from sqlalchemy import (
    Boolean,
    Column,
    create_engine,
    DateTime,
    ForeignKey,
    Integer,
    String,
    Table
)
from sqlalchemy.ext.declarative import declarative_base
from sqlalchemy.orm import relationship, sessionmaker
from sqlalchemy.orm.exc import NoResultFound
from urllib.parse import urlparse
import click
import datetime
import os
import sqlalchemy.engine.url

Base = declarative_base()

group_table = Table(
    'group_association', Base.metadata,
    Column('chapter_id', Integer, ForeignKey('chapters.id')),
    Column('group_id', Integer, ForeignKey('groups.id'))
)

class Series(Base):
    __tablename__ = 'series'

    id = Column(Integer, primary_key=True)

    name = Column(String)
    alias = Column(String, unique=True)
    url = Column(String, unique=True)
    following = Column(Boolean, default=True)
    directory = Column(String)

    chapters = relationship("Chapter", backref="series")

    def __init__(self, series):
        self.name = series.name
        self.alias = series.alias
        self.url = series.url
        self.directory = series.directory

    @staticmethod
    def alias_lookup(alias):
        """Returns a DB object for a series by alias name. Prints an error if
        an invalid alias is specified.
        """
        try:
            s = (session.query(Series)
                 .filter_by(alias=alias, following=True)
                 .one())
        except NoResultFound:
            output.error('Could not find alias "{}"'.format(alias))
            exit(1)
        else:
            return s

    @property
    def last_added(self):
        """Returns the last time a chapter has been added to the series."""
        updates = sorted([x for x in self.chapters if x.added_on is not None],
                         reverse=True, key=lambda x: x.added_on)
        if updates:
            return updates[0].added_on
        else:
            return None

    @property
    def ordered_chapters(self):
        return humansorted(self.chapters, key=lambda x: x.chapter)

class Chapter(Base):
    __tablename__ = 'chapters'

    id = Column(Integer, primary_key=True)
    series_id = Column(Integer, ForeignKey('series.id'))

    # Downloaded has the value -1 for ignored chapters, 0 for new chapters and
    # 1 for downloaded chapters.
    downloaded = Column(Integer, default=0)
    chapter = Column(String)
    url = Column(String, unique=True)
    title = Column(String)
    added_on = Column(DateTime)

    groups = relationship('Group', secondary=group_table, backref='chapters')

    def __init__(self, chapter, series):
        self.series = series
        self.chapter = chapter.chapter
        self.title = chapter.title
        self.url = chapter.url
        self.added_on = datetime.datetime.now()

        self.groups = []
        for group in chapter.groups:
            try:
                g = session.query(Group).filter(Group.name == group).one()
            except NoResultFound:
                g = Group(group)
                session.add(g)
                session.commit()
            self.groups.append(g)

    @staticmethod
    def find_new(alias=None):
        """Return a list of new chapters as Chapter objects and applies human
        sorting to it. Accepts an optional 'alias' argument, which will filter
        the query.
        """
        query = session.query(Chapter).join(Series).filter(Series.following)
        if alias:
            query = query.filter(Series.alias == alias)
        query = query.filter(Chapter.downloaded == 0).all()
        return humansorted([x.to_object() for x in query],
                           key=lambda x: x.chapter)

    @property
    def group_tag(self):
        """Return a joined string of chapter's groups enclosed in brackets."""
        return ''.join(['[{}]'.format(x.name) for x in self.groups])

    @property
    def status(self):
        """Return the chapter's downloaded status as a one character flag."""
        if self.downloaded == 0:
            return 'n'
        elif self.downloaded == -1:
            return 'i'
        else:
            return ' '

    def to_object(self):
        """Turns a database entry into a chapter object for the respective
        site by parsing the URL.
        """
        parse = urlparse(self.url)
        kwargs = {'name': self.series.name,
                  'alias': self.series.alias,
                  'chapter': self.chapter,
                  'url': self.url,
                  'groups': self.groups,
                  'directory': self.series.directory}
        if parse.netloc == 'bato.to':
            from cum.scrapers.batoto import BatotoChapter
            return BatotoChapter(**kwargs)
        elif parse.netloc == 'dynasty-scans.com':
            from cum.scrapers.dynastyscans import DynastyScansChapter
            return DynastyScansChapter(**kwargs)
        elif parse.netloc == 'manga.madokami.com':
            from cum.scrapers.madokami import MadokamiChapter
            return MadokamiChapter(**kwargs)
        else:
            return None

class Group(Base):
    __tablename__ = 'groups'

    id = Column(Integer, primary_key=True)
    name = Column(String, unique=True)

    def __init__(self, name):
        self.name = name

    def __str__(self):
        return self.name

def backup_database():
    """Backs up the database file to a file called cum.db.bak."""
    db_path = os.path.join(config.cum_dir, 'cum.db')
    backup_path = os.path.join(config.cum_dir, 'cum.db.bak')
    copyfile(db_path, backup_path)

def initialize():
    global db_path, engine, session
    db_path = os.path.join(config.cum_dir, 'cum.db')
    db_url = sqlalchemy.engine.url.URL('sqlite', database=db_path)
    engine = create_engine(db_url)
    if not os.path.exists(db_path):
        Base.metadata.create_all(bind=engine)
    Session = sessionmaker(bind=engine)
    session = Session()

def test_database():
    """Runs a database sanity test."""
    sanity_tester = sanity.DatabaseSanity(Base, engine)
    sanity_tester.test()
    if sanity_tester.errors:
        for error in sanity_tester.errors:
            err_target, err_msg = str(error).split(' ', 1)
            message = ' '.join([click.style(err_target, bold=True), err_msg])
            output.warning(message)
        output.error('Database has failed sanity check; '
                     'run `cum repair-db` to repair database')
        exit(1)
<|endoftext|>


<commit_before>#!/usr/bin/pypy

from datetime import datetime
from csv import DictReader
from math import exp, log, sqrt
from pymmh3 import hash

# parameters #################################################################

train = 'train.txt'  # path to training file
test = 'test.txt'  # path to testing file

logbatch = 100000
dotest = True

D = 2 ** 24    # number of weights use for learning

signed = False    # Use signed hash? Set to False for to reduce number of hash calls

interaction = True

lambda1 = 0.
lambda2 = 0.

if interaction:
    alpha = .004  # learning rate for sgd optimization
else:
    alpha = .05   # learning rate for sgd optimization
adapt = 1.        # Use adagrad, sets it as power of adaptive factor. >1 will amplify adaptive measure and vice versa
fudge = .5        # Fudge factor

header = ['Label','i1','i2','i3','i4','i5','i6','i7','i8','i9','i10','i11','i12','i13','c1','c2','c3','c4','c5','c6','c7','c8','c9','c10','c11','c12','c13','c14','c15','c16','c17','c18','c19','c20','c21','c22','c23','c24','c25','c26']

# function definitions #######################################################

# A. Bounded logloss
# INPUT:
#     p: our prediction
#     y: real answer
# OUTPUT
#     logarithmic loss of p given y
def logloss(p, y):
    p = max(min(p, 1. - 10e-17), 10e-17)        # The bounds
    return -log(p) if y == 1. else -log(1. - p)

# B. Apply hash trick of the original csv row
# for simplicity, we treat both integer and categorical features as categorical
# INPUT:
#     csv_row: a csv dictionary, ex: {'Lable': '1', 'I1': '357', 'I2': '', ...}
#     D: the max index that we can hash to
# OUTPUT:
#     x: a list of indices that its value is 1
def get_x(csv_row, D):
    fullind = []
    for key, value in csv_row.items():
        s = key + '=' + value
        fullind.append(hash(s) % D) # weakest hash ever ?? Not anymore :P

    if interaction == True:
        indlist2 = []
        for i in range(len(fullind)):
            for j in range(i+1,len(fullind)):
                indlist2.append(fullind[i] ^ fullind[j]) # Creating interactions using XOR
        fullind = fullind + indlist2

    x = {}
    x[0] = 1  # 0 is the index of the bias term
    for index in fullind:
        if(not x.has_key(index)):
            x[index] = 0
        if signed:
            x[index] += (1 if (hash(str(index))%2)==1 else -1) # Disable for speed
        else:
            x[index] += 1

    return x  # x contains indices of features that have a value as number of occurences

# C. Get probability estimation on x
# INPUT:
#     x: features
#     w: weights
# OUTPUT:
#     probability of p(y = 1 | x; w)
def get_p(x, w):
    wTx = 0.
    for i, xi in x.items():
        wTx += w[i] * xi  # w[i] * x[i]
    return 1. / (1. + exp(-max(min(wTx, 50.), -50.)))  # bounded sigmoid

# D. Update given model
# INPUT:
#     w: weights
#     n: a counter that counts the number of times we encounter a feature
#        this is used for adaptive learning rate
#     x: feature
#     p: prediction of our model
#     y: answer
# OUTPUT:
#     w: updated model
#     n: updated count
def update_w(w, g, x, p, y):
    for i, xi in x.items():
        # alpha / (sqrt(g) + 1) is the adaptive learning rate heuristic
        # (p - y) * x[i] is the current gradient
        # note that in our case, if i in x then x[i] = 1
        delta = (p - y) * xi + lambda1 * ((-1.) if w[i] < 0. else 1.) + lambda2 * w[i]
        if adapt > 0:
            g[i] += delta ** 2
        w[i] -= delta * alpha / (sqrt(g[i]) ** adapt)  # Minimising log loss
    return w, g

# training and testing #######################################################

# initialize our model
w = [0.] * D  # weights
g = [fudge] * D  # sum of historical gradients

# start training a logistic regression model using on pass sgd
loss = 0.
lossb = 0.
for t, row in enumerate(DictReader(open(train), header, delimiter='\t')):
    y = 1. if row['Label'] == '1' else 0.

    del row['Label']  # can't let the model peek the answer

    # main training procedure
    # step 1, get the hashed features
    x = get_x(row, D)
    # step 2, get prediction
    p = get_p(x, w)

    # for progress validation, useless for learning our model
    lossx = logloss(p, y)
    loss += lossx
    lossb += lossx
    if t % logbatch == 0 and t > 1:
        print('%s\tencountered: %d\tcurrent whole logloss: %f\tcurrent batch logloss: %f' % (datetime.now(), t, loss/t, lossb/logbatch))
        lossb = 0.

    # step 3, update model with answer
    w, g = update_w(w, g, x, p, y)

if not dotest:
    exit()

# testing (build kaggle's submission file)
with open('submission.csv', 'w') as submission:
    submission.write('Id,Predicted\n')
    for t, row in enumerate(DictReader(open(test), header[1:], delimiter='\t')):
        x = get_x(row, D)
        p = get_p(x, w)
        submission.write('%d,%f\n' % (60000000+int(t), p))
<commit_msg>No regularisation for bias term.<commit_after>#!/usr/bin/pypy

from datetime import datetime
from csv import DictReader
from math import exp, log, sqrt
from pymmh3 import hash

# parameters #################################################################

train = 'train.txt'  # path to training file
test = 'test.txt'  # path to testing file

logbatch = 100000
dotest = True

D = 2 ** 24    # number of weights use for learning

signed = False    # Use signed hash? Set to False for to reduce number of hash calls

interaction = True

lambda1 = 0.
lambda2 = 0.

if interaction:
    alpha = .004  # learning rate for sgd optimization
else:
    alpha = .05   # learning rate for sgd optimization
adapt = 1.        # Use adagrad, sets it as power of adaptive factor. >1 will amplify adaptive measure and vice versa
fudge = .5        # Fudge factor

header = ['Label','i1','i2','i3','i4','i5','i6','i7','i8','i9','i10','i11','i12','i13','c1','c2','c3','c4','c5','c6','c7','c8','c9','c10','c11','c12','c13','c14','c15','c16','c17','c18','c19','c20','c21','c22','c23','c24','c25','c26']

# function definitions #######################################################

# A. Bounded logloss
# INPUT:
#     p: our prediction
#     y: real answer
# OUTPUT
#     logarithmic loss of p given y
def logloss(p, y):
    p = max(min(p, 1. - 10e-17), 10e-17)        # The bounds
    return -log(p) if y == 1. else -log(1. - p)

# B. Apply hash trick of the original csv row
# for simplicity, we treat both integer and categorical features as categorical
# INPUT:
#     csv_row: a csv dictionary, ex: {'Lable': '1', 'I1': '357', 'I2': '', ...}
#     D: the max index that we can hash to
# OUTPUT:
#     x: a list of indices that its value is 1
def get_x(csv_row, D):
    fullind = []
    for key, value in csv_row.items():
        s = key + '=' + value
        fullind.append(hash(s) % D) # weakest hash ever ?? Not anymore :P

    if interaction == True:
        indlist2 = []
        for i in range(len(fullind)):
            for j in range(i+1,len(fullind)):
                indlist2.append(fullind[i] ^ fullind[j]) # Creating interactions using XOR
        fullind = fullind + indlist2

    x = {}
    x[0] = 1  # 0 is the index of the bias term
    for index in fullind:
        if(not x.has_key(index)):
            x[index] = 0
        if signed:
            x[index] += (1 if (hash(str(index))%2)==1 else -1) # Disable for speed
        else:
            x[index] += 1

    return x  # x contains indices of features that have a value as number of occurences

# C. Get probability estimation on x
# INPUT:
#     x: features
#     w: weights
# OUTPUT:
#     probability of p(y = 1 | x; w)
def get_p(x, w):
    wTx = 0.
    for i, xi in x.items():
        wTx += w[i] * xi  # w[i] * x[i]
    return 1. / (1. + exp(-max(min(wTx, 50.), -50.)))  # bounded sigmoid

# D. Update given model
# INPUT:
#     w: weights
#     n: a counter that counts the number of times we encounter a feature
#        this is used for adaptive learning rate
#     x: feature
#     p: prediction of our model
#     y: answer
# OUTPUT:
#     w: updated model
#     n: updated count
def update_w(w, g, x, p, y):
    for i, xi in x.items():
        # alpha / (sqrt(g) + 1) is the adaptive learning rate heuristic
        # (p - y) * x[i] is the current gradient
        # note that in our case, if i in x then x[i] = 1
        delreg = (lambda1 * ((-1.) if w[i] < 0. else 1.) + lambda2 * w[i]) if i != 0 else 0.
        delta = (p - y) * xi + delreg
        if adapt > 0:
            g[i] += delta ** 2
        w[i] -= delta * alpha / (sqrt(g[i]) ** adapt)  # Minimising log loss
    return w, g

# training and testing #######################################################

# initialize our model
w = [0.] * D  # weights
g = [fudge] * D  # sum of historical gradients

# start training a logistic regression model using on pass sgd
loss = 0.
lossb = 0.
for t, row in enumerate(DictReader(open(train), header, delimiter='\t')):
    y = 1. if row['Label'] == '1' else 0.

    del row['Label']  # can't let the model peek the answer

    # main training procedure
    # step 1, get the hashed features
    x = get_x(row, D)
    # step 2, get prediction
    p = get_p(x, w)

    # for progress validation, useless for learning our model
    lossx = logloss(p, y)
    loss += lossx
    lossb += lossx
    if t % logbatch == 0 and t > 1:
        print('%s\tencountered: %d\tcurrent whole logloss: %f\tcurrent batch logloss: %f' % (datetime.now(), t, loss/t, lossb/logbatch))
        lossb = 0.

    # step 3, update model with answer
    w, g = update_w(w, g, x, p, y)

if not dotest:
    exit()

# testing (build kaggle's submission file)
with open('submission.csv', 'w') as submission:
    submission.write('Id,Predicted\n')
    for t, row in enumerate(DictReader(open(test), header[1:], delimiter='\t')):
        x = get_x(row, D)
        p = get_p(x, w)
        submission.write('%d,%f\n' % (60000000+int(t), p))
<|endoftext|>


<commit_before>#!/usr/bin/env pytho
# Copyright 2016 The MLab Signal Searcher Authors
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Look for evidence of performance fluctuating on a 24-hour cycle.

This module looks for evidence of daily periodicity in a timeseries. When a
network has performance that varies diurnally, then that is a strong suggestion
that the network is near overload, as network usage varies diurnally for most
eyeball networks.  In the MLab report on interconnections, it was found that
performance varying on a 24-hour basis was a leading indicator of severe
congestion problems.
"""

import datetime
import numpy
import report

# A tunable parameter to help adjust the sensitivity of this test. As the
# required SPIKE_RATIO grows, the test becomes less vulnerable to noise, but may
# miss some signals. Adjust this parameter up or down experimentally.
SPIKE_RATIO = 2.0

# A tunable parameter to indicate the lowest value that we might want to
# consider as a valid signal. Power below this value can't count as a signal,
# no matter how low the average might be. Adjust this parameter experimentally.
SQUELCH_THRESHOLD = .001

def _power_spike_at_24_hours(timeseries):
  """Looks for the presence of a 24-hour cycles in the timeseries.

  Looks to see how much power there is under the frequency corresponding to
  24 hours. A relatively large amount of power there would mean that the
  timeseries has a 24-hour component signal.

  Args:
      timeseries: A list with one datapoint per hour, in order.

  Returns:
      True if a strong 24-hour signal was found, False otherwise
  """
  if len(timeseries) < 24:
    return False

  # Perform an FFT
  spectrum = numpy.fft.rfft(timeseries)
  # Square the result to get the power curve
  power = [abs(x)**2 for x in spectrum]
  # Compare the value at the index corresponding to 24 hours to the average
  # value. If it is more than SPIKE_SIZE times the average value, then we will
  # count that as a a spike. We calculate the average after removing the DC
  # component of the signal.
  avg = numpy.average(power[1:])
  # Frequency at index N is N/len(array).  We want to find the power when the
  # frequency is 1/24 (one event per 24 hours, aka "daily"), which means we
  # need to look at index:
  #   N/len(array) = 1/24
  #           24*N = len(array)
  #              N = len(array) / 24
  day_index = int(len(timeseries) / 24.0)
  return ((power[day_index] > SQUELCH_THRESHOLD) and
          (power[day_index] / avg > SPIKE_RATIO))

def find_problems(timeseries):
  """Discover any problems relating to cyclic patterns in the data.

  Args:
      timeseries: the data read from MLab BigQuery

  Returns:
      a list of CycleProblems, possibly empty
  """
  problems_found = []
  for netblock, series in timeseries.iteritems():
    duration = datetime.timedelta(hours=len(series))
    for metric, metric_name in [('upload_speed', 'Upload speeds'),
                                ('download_speed', 'Download speeds'),
                                ('min_latency', 'Round-trip times')]:
      field_index = series[0]._fields.index(metric)
      assert field_index >= 0, 'Bad metric name %s' % metric
      single_metric_series = [d[field_index] for d in series]
      if _power_spike_at_24_hours(single_metric_series):
        problems_found.append(CycleProblem(netblock, duration, metric_name))
  return problems_found

class CycleProblem(report.Problem):
  """A report for the discovery of cyclic performance characteristics."""
  priority = 4  # all CycleProblems have the same priority for now

  def __init__(self, netblock, duration, metric):
    message = ('%s are fluctuating on a 24-hour cycle. This is frequently a '
               'leading indicator of congestion, and suggests that at least '
               'part of the network in question is underprovisioned for its '
               'peak load') % metric
    super(CycleProblem, self).__init__(netblock, duration,
                                       CycleProblem.priority, message)
<commit_msg>Stop removing the DC component of cycles.<commit_after>#!/usr/bin/env pytho
# Copyright 2016 The MLab Signal Searcher Authors
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Look for evidence of performance fluctuating on a 24-hour cycle.

This module looks for evidence of daily periodicity in a timeseries. When a
network has performance that varies diurnally, then that is a strong suggestion
that the network is near overload, as network usage varies diurnally for most
eyeball networks.  In the MLab report on interconnections, it was found that
performance varying on a 24-hour basis was a leading indicator of severe
congestion problems.
"""

import datetime
import numpy
import report

# A tunable parameter to help adjust the sensitivity of this test. As the
# required SPIKE_RATIO grows, the test becomes less vulnerable to noise, but may
# miss some signals. Adjust this parameter up or down experimentally.
SPIKE_RATIO = 2.0

# A tunable parameter to indicate the lowest value that we might want to
# consider as a valid signal. Power below this value can't count as a signal,
# no matter how low the average might be. Adjust this parameter experimentally.
SQUELCH_THRESHOLD = .001

def _power_spike_at_24_hours(timeseries):
  """Looks for the presence of a 24-hour cycles in the timeseries.

  Looks to see how much power there is under the frequency corresponding to
  24 hours. A relatively large amount of power there would mean that the
  timeseries has a 24-hour component signal.

  Args:
      timeseries: A list with one datapoint per hour, in order.

  Returns:
      True if a strong 24-hour signal was found, False otherwise
  """
  if len(timeseries) < 24:
    return False

  # Perform an FFT
  spectrum = numpy.fft.rfft(timeseries)
  # Square the result to get the power curve
  power = [abs(x)**2 for x in spectrum]
  # Compare the value at the index corresponding to 24 hours to the average
  # value. If it is more than SPIKE_RATIO times the average value, then we will
  # count that as a a spike.
  avg = numpy.average(power)
  # Frequency at index N is N/len(array).  We want to find the power when the
  # frequency is 1/24 (one event per 24 hours, aka "daily"), which means we
  # need to look at index:
  #   N/len(array) = 1/24
  #           24*N = len(array)
  #              N = len(array) / 24
  day_index = int(len(timeseries) / 24.0)
  return ((power[day_index] > SQUELCH_THRESHOLD) and
          (power[day_index] / avg > SPIKE_RATIO))

def find_problems(timeseries):
  """Discover any problems relating to cyclic patterns in the data.

  Args:
      timeseries: the data read from MLab BigQuery

  Returns:
      a list of CycleProblems, possibly empty
  """
  problems_found = []
  for netblock, series in timeseries.iteritems():
    duration = datetime.timedelta(hours=len(series))
    for metric, metric_name in [('upload_speed', 'Upload speeds'),
                                ('download_speed', 'Download speeds'),
                                ('min_latency', 'Round-trip times')]:
      field_index = series[0]._fields.index(metric)
      assert field_index >= 0, 'Bad metric name %s' % metric
      single_metric_series = [d[field_index] for d in series]
      if _power_spike_at_24_hours(single_metric_series):
        problems_found.append(CycleProblem(netblock, duration, metric_name))
  return problems_found

class CycleProblem(report.Problem):
  """A report for the discovery of cyclic performance characteristics."""
  priority = 4  # all CycleProblems have the same priority for now

  def __init__(self, netblock, duration, metric):
    message = ('%s are fluctuating on a 24-hour cycle. This is frequently a '
               'leading indicator of congestion, and suggests that at least '
               'part of the network in question is underprovisioned for its '
               'peak load') % metric
    super(CycleProblem, self).__init__(netblock, duration,
                                       CycleProblem.priority, message)
<|endoftext|>


<commit_before>#!/usr/bin/env python
################# TODO STATUS: ahora mismo se conecta el cliente directamente al servidor, hay que crear un servidor que haga el MITM, reenviando el trafico SSL recibido por ambos sockets y enviando el que el cliente del MITM decida
import socket
import SocketServer
import ssl
import threading
import sys

PUBLIC_ENDPOINT = ('', 4747)

INTERNAL_SSL_ENDPOINT = ('127.0.0.1', 65186)
INTERNAL_FORWARD_ENDPOINT = ('127.0.0.1', 65187)
RECV_BUFFER = 4096
KILL_THREADS_WHEN_MAIN_ENDS = True

SSL_VERSION = ssl.PROTOCOL_SSLv3

MITM_KEY_RECEIVED = "ok"
ENDPOINT_SERVER = 'S'
ENDPOINT_CLIENT = 'C'
MESSAGE_ENDPOINT_SEPARATOR = '-'

class MitmExchanger:
	__instance = None

	def __init__(self):
		self.mitms = {}
		self.counter = 0
		self.lock = threading.Lock()

	@classmethod
	def instance(cls):
		if not cls.__instance:
			cls.__instance = cls()
		return cls.__instance

	def put(self, mitm):
		with self.lock:
			key = self.counter
			self.counter += 1
		self.mitms[key] = mitm
		return key

	def pop(self, key):
		mitm = self.mitms[key]
		del self.mitms[key]
		return mitm

class Thread(threading.Thread):
	def __init__(self, *args, **dargs):
		super(Thread, self).__init__(*args, **dargs)
		self.daemon = KILL_THREADS_WHEN_MAIN_ENDS

class ThreadedTCPServer(SocketServer.ThreadingTCPServer):
	daemon_threads = KILL_THREADS_WHEN_MAIN_ENDS

	def start_background(self):
		Thread(target=self.serve_forever).start()

	def start_foreground(self):
		self.serve_forever()

class SslServerRequestHandler(SocketServer.BaseRequestHandler):
	def handle(self):
		self.ssl_socket = self.wrap_with_ssl_socket(self.request)
		handshake_completed = self.handle_ssl_handshake(self.ssl_socket)
		if handshake_completed:
			self.handle_http_request(self.ssl_socket)

	def wrap_with_ssl_socket(self, socket):
		return ssl.wrap_socket(socket, keyfile="certkey.pem", certfile="cert.pem", server_side=True, cert_reqs=ssl.CERT_NONE, ssl_version=SSL_VERSION, do_handshake_on_connect=False) # TODO force cbc block cipher

	def handle_ssl_handshake(self, ssl_socket):
		try:
			ssl_socket.do_handshake()
			return True
		except:
			return False

	def handle_http_request(self, socket):
		request = self.get_http_request(socket)
		has_valid_authentication = self.check_authentication(request)
		if has_valid_authentication:
			self.send_ok_response(socket)
		else:
			self.send_failure_response(socket)

	def get_http_request(self, socket):
		return socket.recv(RECV_BUFFER) # TODO

	def check_authentication(self, request):
		return self.check_http_authentication(request) and self.check_csrf_in_body(request)

	def check_http_authentication(self, request):
		return True # TODO

	def check_csrf_in_body(self, request):
		return True # TODO

	def send_ok_response(self, socket):
		socket.send("ok\n") # TODO

	def send_failure_response(self, socket):
		socket.send("error\n") # TODO

class MitmServerRequestHandler(SocketServer.BaseRequestHandler):
	def handle(self):
		mitm_key = self.get_mitm_key(self.request)
		mitm = self.get_mitm(mitm_key)
		server_socket = self.connect_to_server_socket()
		self.forward(server_socket, self.request, mitm)

	def get_mitm_key(self, socket):
		mitm_key = int(socket.recv(RECV_BUFFER))
		socket.send(MITM_KEY_RECEIVED)
		return mitm_key

	def get_mitm(self, mitm_key):
		return MitmExchanger.instance().pop(mitm_key)

	def connect_to_server_socket(self):
		return socket.create_connection(INTERNAL_SSL_ENDPOINT)

	def forward(self, server_socket, client_socket, mitm):
		forwarder_from_server = SocketStreamForwarder(server_socket, mitm.server_out_socket).forward()
		forwarder_from_client = SocketStreamForwarder(client_socket, mitm.client_out_socket).forward()
		forwarder_from_mitm = SocketStreamForwarder(mitm.recv_socket, mitm.send_socket(server_socket, client_socket)).forward()
		forwarder_from_server.wait()
		forwarder_from_client.wait()
		forwarder_from_mitm.wait()

class SocketStreamForwarder:
	def __init__(self, in_socket, out_socket):
		self.in_socket = in_socket
		self.out_socket = out_socket

	def forward(self):
		self.thread = Thread(target=self.forward_loop)
		self.thread.start()
		return self

	def forward_loop(self):
		read = self.in_socket.recv(RECV_BUFFER)
		while read:
			self.out_socket.sendall(read)
			read = self.in_socket.recv(RECV_BUFFER)

	def wait(self):
		self.thread.join()

class SslClientRequest:
	def __init__(self, initial_payload, path, credentials, body):
		self.initial_payload = initial_payload
		self.http_request = self.build_http_request(path, credentials, body)
		self.response = 'client-error' # TODO error?

	def run(self):
		socket = self.connect_to_server()
		self.send_initial_payload(socket, self.initial_payload)
		ssl_socket = self.wrap_with_ssl_socket(socket)
		valid_handshake = self.perform_ssl_handshake(ssl_socket)
		if valid_handshake:
			self.send_request(ssl_socket)
			self.response = self.get_response(ssl_socket)
		self.shutdown_socket(ssl_socket)

	def build_http_request(self, path, credentials, body):
		return "GET %s HTTP/1.0\r\nAuthorization: Basic %s\r\n\r\n%s" % (path, credentials, body)

	def connect_to_server(self):
		return socket.create_connection(INTERNAL_FORWARD_ENDPOINT)

	def send_initial_payload(self, socket, initial_payload):
		socket.sendall(initial_payload)
		ack = socket.recv(RECV_BUFFER)
		assert ack == MITM_KEY_RECEIVED

	def wrap_with_ssl_socket(self, socket):
		return ssl.wrap_socket(socket, server_side=False, cert_reqs=ssl.CERT_REQUIRED, ca_certs="cert.pem", ssl_version=SSL_VERSION, do_handshake_on_connect=False) # TODO include certificate file in ca_certs param, force cbc block cipher

	def perform_ssl_handshake(self, ssl_socket):
		try:
			ssl_socket.do_handshake()
			return True
		except:
			return False

	def send_request(self, socket):
		socket.sendall(self.http_request)

	def get_response(self, socket):
		return socket.recv(RECV_BUFFER) # TODO?

	def shutdown_socket(self, socket):
		socket.close()

class MitmSocketAggregator:
	def __init__(self, server_socket, client_socket, recv_socket, send_socket):
		self.server_out_socket = server_socket
		self.client_out_socket = client_socket
		self.recv_socket = recv_socket
		self._send_socket = send_socket

	def send_socket(self, server_socket, client_socket):
		self._send_socket.processor.set_router(EndpointRouter(server_socket, client_socket))
		return self._send_socket

class MitmInSocket:
	def __init__(self, rfile):
		self.rfile = rfile

	def recv(self, bufsize):
		return self.rfile.readline()

class MitmOutSocket:
	def __init__(self, processor):
		self.processor = processor

	def sendall(self, string):
		self.processor.process(string)

class FormatDataProcessor:
	def __init__(self, formatter, action):
		self.formatter = formatter
		self.action = action

	def process(self, string):
		formatted_data = self.formatter.format(string)
		self.action(formatted_data)

class EncodeAndEndpointFormatter:
	def __init__(self, endpoint, codec):
		self.endpoint = endpoint
		self.codec = codec

	def format(self, string):
		return self.endpoint + MESSAGE_ENDPOINT_SEPARATOR + self.codec.encode(string) + '\n'

class DecodeAndEndpointUnformatter:
	def __init__(self, codec):
		self.codec = codec

	def unformat(self, string):
		string = self.remove_trailing_newline(string)
		endpoint, encoded_string = self.split_by_separator(string)
		decoded_string = self.codec.decode(encoded_string)
		return (endpoint, decoded_string)

	def remove_trailing_newline(self, string):
		if string[-1] == '\n':
			string = string[:-1]
		return string

	def split_by_separator(self, string):
		return string.split(MESSAGE_ENDPOINT_SEPARATOR, 1)

class Base64Codec:
	def encode(self, string):
		return string.encode("base64").replace("\n", "")

	def decode(self, string):
		return string.decode("base64")

class UnformatAndRouteByEndpointDataProcessor:
	def __init__(self, unformatter):
		self.unformatter = unformatter

	def set_router(self, router):
		self.router = router

	def process(self, string):
		endpoint, text = self.unformatter.unformat(string)
		self.router.get(endpoint).sendall(text)

class EndpointRouter:
	def __init__(self, server_socket, client_socket):
		self.server_socket = server_socket
		self.client_socket = client_socket

	def get(self, endpoint):
		if endpoint == ENDPOINT_SERVER:
			return self.client_socket
		elif endpoint == ENDPOINT_CLIENT:
			return self.server_socket

class PublicServerRequestHandler(SocketServer.StreamRequestHandler):
	def handle(self):
		path, credentials, body = self.handle_initial_request()
		self.perform_https_connection(path, credentials, body)

	def handle_initial_request(self):
		request = self.rfile.readline()
		self.sanity_checks(request)
		path, body = self.split_request(request)
		csrf = self.get_csrf(body)
		credentials = self.get_credentials(csrf)
		return path, credentials, body

	def perform_https_connection(self, path, credentials, body):
		# TODO create a factory for this

		server_socket_codec = Base64Codec()
		server_socket_formatter = EncodeAndEndpointFormatter(ENDPOINT_SERVER, server_socket_codec)
		server_socket_action = self.wfile.write
		server_socket_processor = FormatDataProcessor(server_socket_formatter, server_socket_action)
		server_socket = MitmOutSocket(server_socket_processor)

		client_socket_codec = Base64Codec()
		client_socket_formatter = EncodeAndEndpointFormatter(ENDPOINT_CLIENT, client_socket_codec)
		client_socket_action = self.wfile.write
		client_socket_processor = FormatDataProcessor(client_socket_formatter, client_socket_action)
		client_socket = MitmOutSocket(client_socket_processor)

		recv_socket = MitmInSocket(self.rfile)

		send_socket_codec = Base64Codec()
		send_socket_unformatter = DecodeAndEndpointUnformatter(send_socket_codec)
		send_socket_processor = UnformatAndRouteByEndpointDataProcessor(send_socket_unformatter)
		send_socket = MitmOutSocket(send_socket_processor)

		mitm_socket = MitmSocketAggregator(server_socket, client_socket, recv_socket, send_socket)

		mitm_socket_key = MitmExchanger.instance().put(mitm_socket)

		ssl_client = SslClientRequest(str(mitm_socket_key), path, credentials, body)
		ssl_client.run()
		self.wfile.write(ssl_client.response)

	def sanity_checks(self, request):
		pass # TODO check post("[^"]", "[^"]");\n and raise exceptions if not match

	def split_request(self, request):
		path, end_position = self.get_substring_and_end_position(request, '"', '"', 0)
		body = self.get_substring(request, '"', '"', end_position)
		return path, body

	def get_csrf(self, body):
		return self.get_substring(body, "csrf=", "&")

	def get_credentials(self, csrf):
		return "pepe:pito" # TODO read from a file using the csrf

	def get_substring(self, string, before_start, after_end, offset = 0):
		return self.get_substring_and_end_position(string, before_start, after_end, offset)[0]

	def get_substring_and_end_position(self, string, before_start, after_end, offset):
		start_position = string.find(before_start, offset)
		if start_position == -1:
			pass # TODO raise?
		start_position += len(before_start)
		end_position = string.find(after_end, start_position)
		if end_position == -1:
			pass # TODO raise?
		return string[start_position:end_position], end_position + len(after_end)

if __name__ == "__main__":
	ssl_server = ThreadedTCPServer(INTERNAL_SSL_ENDPOINT, SslServerRequestHandler)
	ssl_server.start_background()

	forward_server = ThreadedTCPServer(INTERNAL_FORWARD_ENDPOINT, MitmServerRequestHandler)
	forward_server.start_background()

	public_server = ThreadedTCPServer(PUBLIC_ENDPOINT, PublicServerRequestHandler)
	public_server.start_foreground()

	# TODO

	forward_server.shutdown()
	ssl_server.shutdown()

<commit_msg>Avoid failing on error<commit_after>#!/usr/bin/env python
################# TODO STATUS: ahora mismo se conecta el cliente directamente al servidor, hay que crear un servidor que haga el MITM, reenviando el trafico SSL recibido por ambos sockets y enviando el que el cliente del MITM decida
import socket
import SocketServer
import ssl
import threading
import sys

PUBLIC_ENDPOINT = ('', 4747)

INTERNAL_SSL_ENDPOINT = ('127.0.0.1', 65186)
INTERNAL_FORWARD_ENDPOINT = ('127.0.0.1', 65187)
RECV_BUFFER = 4096
KILL_THREADS_WHEN_MAIN_ENDS = True

SSL_VERSION = ssl.PROTOCOL_SSLv3

MITM_KEY_RECEIVED = "ok"
ENDPOINT_SERVER = 'S'
ENDPOINT_CLIENT = 'C'
MESSAGE_ENDPOINT_SEPARATOR = '-'

class MitmExchanger:
	__instance = None

	def __init__(self):
		self.mitms = {}
		self.counter = 0
		self.lock = threading.Lock()

	@classmethod
	def instance(cls):
		if not cls.__instance:
			cls.__instance = cls()
		return cls.__instance

	def put(self, mitm):
		with self.lock:
			key = self.counter
			self.counter += 1
		self.mitms[key] = mitm
		return key

	def pop(self, key):
		mitm = self.mitms[key]
		del self.mitms[key]
		return mitm

class Thread(threading.Thread):
	def __init__(self, *args, **dargs):
		super(Thread, self).__init__(*args, **dargs)
		self.daemon = KILL_THREADS_WHEN_MAIN_ENDS

class ThreadedTCPServer(SocketServer.ThreadingTCPServer):
	daemon_threads = KILL_THREADS_WHEN_MAIN_ENDS

	def start_background(self):
		Thread(target=self.serve_forever).start()

	def start_foreground(self):
		self.serve_forever()

class SslServerRequestHandler(SocketServer.BaseRequestHandler):
	def handle(self):
		self.ssl_socket = self.wrap_with_ssl_socket(self.request)
		handshake_completed = self.handle_ssl_handshake(self.ssl_socket)
		if handshake_completed:
			self.handle_http_request(self.ssl_socket)

	def wrap_with_ssl_socket(self, socket):
		return ssl.wrap_socket(socket, keyfile="certkey.pem", certfile="cert.pem", server_side=True, cert_reqs=ssl.CERT_NONE, ssl_version=SSL_VERSION, do_handshake_on_connect=False) # TODO force cbc block cipher

	def handle_ssl_handshake(self, ssl_socket):
		try:
			ssl_socket.do_handshake()
			return True
		except:
			return False

	def handle_http_request(self, socket):
		request = self.get_http_request(socket)
		has_valid_authentication = self.check_authentication(request)
		if has_valid_authentication:
			self.send_ok_response(socket)
		else:
			self.send_failure_response(socket)

	def get_http_request(self, socket):
		return socket.recv(RECV_BUFFER) # TODO

	def check_authentication(self, request):
		return self.check_http_authentication(request) and self.check_csrf_in_body(request)

	def check_http_authentication(self, request):
		return True # TODO

	def check_csrf_in_body(self, request):
		return True # TODO

	def send_ok_response(self, socket):
		socket.send("ok\n") # TODO

	def send_failure_response(self, socket):
		socket.send("error\n") # TODO

class MitmServerRequestHandler(SocketServer.BaseRequestHandler):
	def handle(self):
		mitm_key = self.get_mitm_key(self.request)
		mitm = self.get_mitm(mitm_key)
		server_socket = self.connect_to_server_socket()
		self.forward(server_socket, self.request, mitm)

	def get_mitm_key(self, socket):
		mitm_key = int(socket.recv(RECV_BUFFER))
		socket.send(MITM_KEY_RECEIVED)
		return mitm_key

	def get_mitm(self, mitm_key):
		return MitmExchanger.instance().pop(mitm_key)

	def connect_to_server_socket(self):
		return socket.create_connection(INTERNAL_SSL_ENDPOINT)

	def forward(self, server_socket, client_socket, mitm):
		forwarder_from_server = SocketStreamForwarder(server_socket, mitm.server_out_socket).forward()
		forwarder_from_client = SocketStreamForwarder(client_socket, mitm.client_out_socket).forward()
		forwarder_from_mitm = SocketStreamForwarder(mitm.recv_socket, mitm.send_socket(server_socket, client_socket)).forward()
		forwarder_from_server.wait()
		forwarder_from_client.wait()
		forwarder_from_mitm.wait()

class SocketStreamForwarder:
	def __init__(self, in_socket, out_socket):
		self.in_socket = in_socket
		self.out_socket = out_socket

	def forward(self):
		self.thread = Thread(target=self.forward_loop)
		self.thread.start()
		return self

	def forward_loop(self):
		read = self.in_socket.recv(RECV_BUFFER)
		while read:
			self.out_socket.sendall(read)
			read = self.in_socket.recv(RECV_BUFFER)

	def wait(self):
		self.thread.join()

class SslClientRequest:
	def __init__(self, initial_payload, path, credentials, body):
		self.initial_payload = initial_payload
		self.http_request = self.build_http_request(path, credentials, body)
		self.response = 'client-error' # TODO error?

	def run(self):
		socket = self.connect_to_server()
		self.send_initial_payload(socket, self.initial_payload)
		ssl_socket = self.wrap_with_ssl_socket(socket)
		valid_handshake = self.perform_ssl_handshake(ssl_socket)
		if valid_handshake:
			self.send_request(ssl_socket)
			self.response = self.get_response(ssl_socket)
		self.shutdown_socket(ssl_socket)

	def build_http_request(self, path, credentials, body):
		return "GET %s HTTP/1.0\r\nAuthorization: Basic %s\r\n\r\n%s" % (path, credentials, body)

	def connect_to_server(self):
		return socket.create_connection(INTERNAL_FORWARD_ENDPOINT)

	def send_initial_payload(self, socket, initial_payload):
		socket.sendall(initial_payload)
		ack = socket.recv(RECV_BUFFER)
		assert ack == MITM_KEY_RECEIVED

	def wrap_with_ssl_socket(self, socket):
		return ssl.wrap_socket(socket, server_side=False, cert_reqs=ssl.CERT_REQUIRED, ca_certs="cert.pem", ssl_version=SSL_VERSION, do_handshake_on_connect=False) # TODO include certificate file in ca_certs param, force cbc block cipher

	def perform_ssl_handshake(self, ssl_socket):
		try:
			ssl_socket.do_handshake()
			return True
		except:
			return False

	def send_request(self, socket):
		socket.sendall(self.http_request)

	def get_response(self, socket):
		return socket.recv(RECV_BUFFER) # TODO?

	def shutdown_socket(self, socket):
		socket.close()

class MitmSocketAggregator:
	def __init__(self, server_socket, client_socket, recv_socket, send_socket):
		self.server_out_socket = server_socket
		self.client_out_socket = client_socket
		self.recv_socket = recv_socket
		self._send_socket = send_socket

	def send_socket(self, server_socket, client_socket):
		self._send_socket.processor.set_router(EndpointRouter(server_socket, client_socket))
		return self._send_socket

class MitmInSocket:
	def __init__(self, rfile):
		self.rfile = rfile

	def recv(self, bufsize):
		return self.rfile.readline()

class MitmOutSocket:
	def __init__(self, processor):
		self.processor = processor

	def sendall(self, string):
		try:
			self.processor.process(string)
		except:
			pass

class FormatDataProcessor:
	def __init__(self, formatter, action):
		self.formatter = formatter
		self.action = action

	def process(self, string):
		formatted_data = self.formatter.format(string)
		self.action(formatted_data)

class EncodeAndEndpointFormatter:
	def __init__(self, endpoint, codec):
		self.endpoint = endpoint
		self.codec = codec

	def format(self, string):
		return self.endpoint + MESSAGE_ENDPOINT_SEPARATOR + self.codec.encode(string) + '\n'

class DecodeAndEndpointUnformatter:
	def __init__(self, codec):
		self.codec = codec

	def unformat(self, string):
		string = self.remove_trailing_newline(string)
		endpoint, encoded_string = self.split_by_separator(string)
		decoded_string = self.codec.decode(encoded_string)
		return (endpoint, decoded_string)

	def remove_trailing_newline(self, string):
		if string[-1] == '\n':
			string = string[:-1]
		return string

	def split_by_separator(self, string):
		return string.split(MESSAGE_ENDPOINT_SEPARATOR, 1)

class Base64Codec:
	def encode(self, string):
		return string.encode("base64").replace("\n", "")

	def decode(self, string):
		return string.decode("base64")

class UnformatAndRouteByEndpointDataProcessor:
	def __init__(self, unformatter):
		self.unformatter = unformatter

	def set_router(self, router):
		self.router = router

	def process(self, string):
		endpoint, text = self.unformatter.unformat(string)
		self.router.get(endpoint).sendall(text)

class EndpointRouter:
	def __init__(self, server_socket, client_socket):
		self.server_socket = server_socket
		self.client_socket = client_socket

	def get(self, endpoint):
		if endpoint == ENDPOINT_SERVER:
			return self.client_socket
		elif endpoint == ENDPOINT_CLIENT:
			return self.server_socket

class PublicServerRequestHandler(SocketServer.StreamRequestHandler):
	def handle(self):
		path, credentials, body = self.handle_initial_request()
		self.perform_https_connection(path, credentials, body)

	def handle_initial_request(self):
		request = self.rfile.readline()
		self.sanity_checks(request)
		path, body = self.split_request(request)
		csrf = self.get_csrf(body)
		credentials = self.get_credentials(csrf)
		return path, credentials, body

	def perform_https_connection(self, path, credentials, body):
		# TODO create a factory for this

		server_socket_codec = Base64Codec()
		server_socket_formatter = EncodeAndEndpointFormatter(ENDPOINT_SERVER, server_socket_codec)
		server_socket_action = self.wfile.write
		server_socket_processor = FormatDataProcessor(server_socket_formatter, server_socket_action)
		server_socket = MitmOutSocket(server_socket_processor)

		client_socket_codec = Base64Codec()
		client_socket_formatter = EncodeAndEndpointFormatter(ENDPOINT_CLIENT, client_socket_codec)
		client_socket_action = self.wfile.write
		client_socket_processor = FormatDataProcessor(client_socket_formatter, client_socket_action)
		client_socket = MitmOutSocket(client_socket_processor)

		recv_socket = MitmInSocket(self.rfile)

		send_socket_codec = Base64Codec()
		send_socket_unformatter = DecodeAndEndpointUnformatter(send_socket_codec)
		send_socket_processor = UnformatAndRouteByEndpointDataProcessor(send_socket_unformatter)
		send_socket = MitmOutSocket(send_socket_processor)

		mitm_socket = MitmSocketAggregator(server_socket, client_socket, recv_socket, send_socket)

		mitm_socket_key = MitmExchanger.instance().put(mitm_socket)

		ssl_client = SslClientRequest(str(mitm_socket_key), path, credentials, body)
		ssl_client.run()
		self.wfile.write(ssl_client.response)

	def sanity_checks(self, request):
		pass # TODO check post("[^"]", "[^"]");\n and raise exceptions if not match

	def split_request(self, request):
		path, end_position = self.get_substring_and_end_position(request, '"', '"', 0)
		body = self.get_substring(request, '"', '"', end_position)
		return path, body

	def get_csrf(self, body):
		return self.get_substring(body, "csrf=", "&")

	def get_credentials(self, csrf):
		return "pepe:pito" # TODO read from a file using the csrf

	def get_substring(self, string, before_start, after_end, offset = 0):
		return self.get_substring_and_end_position(string, before_start, after_end, offset)[0]

	def get_substring_and_end_position(self, string, before_start, after_end, offset):
		start_position = string.find(before_start, offset)
		if start_position == -1:
			pass # TODO raise?
		start_position += len(before_start)
		end_position = string.find(after_end, start_position)
		if end_position == -1:
			pass # TODO raise?
		return string[start_position:end_position], end_position + len(after_end)

if __name__ == "__main__":
	ssl_server = ThreadedTCPServer(INTERNAL_SSL_ENDPOINT, SslServerRequestHandler)
	ssl_server.start_background()

	forward_server = ThreadedTCPServer(INTERNAL_FORWARD_ENDPOINT, MitmServerRequestHandler)
	forward_server.start_background()

	public_server = ThreadedTCPServer(PUBLIC_ENDPOINT, PublicServerRequestHandler)
	public_server.start_foreground()

	# TODO

	forward_server.shutdown()
	ssl_server.shutdown()

<|endoftext|>


<commit_before>import math
import numpy as np
from matplotlib import pyplot as plt
from matplotlib.patches import Circle
from tools import action as a
from tools import Simulation as Sim
from naoth import math2d as m2d
from tools import tools

class State:
    def __init__(self):
        self.pose = m2d.Pose2D()
        self.pose.translation = m2d.Vector2(4200, 0)
        self.pose.rotation = math.radians(0)

        self.ball_position = m2d.Vector2(100.0, 0.0)

        self.obstacle_list = ([])  # is in global coordinates

def draw_actions(actions_consequences, state, best_action):
    plt.clf()
    tools.draw_field()

    axes = plt.gca()
    axes.add_artist(Circle(xy=(state.pose.translation.x, state.pose.translation.y), radius=100, fill=False, edgecolor='white'))
    axes.text(0, 0, best_action, fontsize=12)

    x = np.array([])
    y = np.array([])

    for consequence in actions_consequences:
        for particle in consequence.positions():
            ball_pos = state.pose * particle.ball_pos  # transform in global coordinates

            x = np.append(x, [ball_pos.x])
            y = np.append(y, [ball_pos.y])

    plt.scatter(x, y, c='r', alpha=0.5)
    plt.pause(0.0001)

def main():
    state = State()

    no_action = a.Action("none", 0, 0, 0, 0)
    kick_short = a.Action("kick_short", 780, 150, 8.454482265522328, 6.992268841997358)
    sidekick_left = a.Action("sidekick_left", 750, 150, 86.170795364136380, 10.669170653645670)
    sidekick_right = a.Action("sidekick_right", 750, 150, -89.657943335302260, 10.553726275058064)

    action_list = [no_action, kick_short, sidekick_left, sidekick_right]
    # Todo Implement the particle filter here somehow
    # Todo change the directions of the best action/ or all actions
    # Sample is element of [0, 360)
    # Sample is evaluated by simulating the consequences and calculating the best actions
    # TODO: Problem: decide function gives only the best action and not a sorted list of actions
    # as far as I understand it we need that in order to resample the worst n samples and eventually converge
    # i dont know what the best way to do that is
    while True:
        actions_consequences = []
        # Simulate Consequences
        for action in action_list:
            single_consequence = a.ActionResults([])
            actions_consequences.append(Sim.simulate_consequences(action, single_consequence, state))

        # actions_consequences is now a list of ActionResults

        # Decide best action
        best_action = Sim.decide_smart(actions_consequences, state)

        draw_actions(actions_consequences, state, action_list[best_action].name)

if __name__ == "__main__":
    main()
<commit_msg>unfinished<commit_after>import math
import numpy as np
from matplotlib import pyplot as plt
from matplotlib.patches import Circle
from tools import action as a
from tools import Simulation as Sim
from naoth import math2d as m2d
from tools import tools

class State:
    def __init__(self):
        self.pose = m2d.Pose2D()
        self.pose.translation = m2d.Vector2(4200, 0)
        self.pose.rotation = math.radians(0)

        self.ball_position = m2d.Vector2(100.0, 0.0)

        self.obstacle_list = ([])  # is in global coordinates

def draw_actions(actions_consequences, state, best_action):
    plt.clf()
    tools.draw_field()

    axes = plt.gca()
    axes.add_artist(Circle(xy=(state.pose.translation.x, state.pose.translation.y), radius=100, fill=False, edgecolor='white'))
    axes.text(0, 0, best_action, fontsize=12)

    x = np.array([])
    y = np.array([])

    for consequence in actions_consequences:
        for particle in consequence.positions():
            ball_pos = state.pose * particle.ball_pos  # transform in global coordinates

            x = np.append(x, [ball_pos.x])
            y = np.append(y, [ball_pos.y])

    plt.scatter(x, y, c='r', alpha=0.5)
    plt.pause(0.0001)

def main():
    state = State()

    no_action = a.Action("none", 0, 0, 0, 0)
    kick_short = a.Action("kick_short", 780, 150, 8.454482265522328, 6.992268841997358)
    sidekick_left = a.Action("sidekick_left", 750, 150, 86.170795364136380, 10.669170653645670)
    sidekick_right = a.Action("sidekick_right", 750, 150, -89.657943335302260, 10.553726275058064)

    action_list = [no_action, kick_short, sidekick_left, sidekick_right]
    # Todo Implement the particle filter here somehow
    # Todo change the directions of the best action/ or all actions
    # Sample is element of [0, 360)
    # Sample is evaluated by simulating the consequences and calculating the best actions
    # TODO: Problem: decide function gives only the best action and not a sorted list of actions
    # as far as I understand it we need that in order to resample the worst n samples and eventually converge
    # i dont know what the best way to do that is

    # For Particle Filter:
    a.num_particles = 1
    # TODO: Create n random numbers here:
    while True:
        actions_consequences = []
        # Simulate Consequences

        single_consequence = a.ActionResults([])
        actions_consequences.append(Sim.simulate_consequences(kick_short, single_consequence, state))

        # actions_consequences is now a list of ActionResults

        # Decide best action
        best_action = Sim.decide_smart(actions_consequences, state)

        draw_actions(actions_consequences, state, action_list[best_action].name)

if __name__ == "__main__":
    main()
<|endoftext|>