<commit_before>import chdb
import config
from utils import *
from common import *

from snippet_parser import CITATION_NEEDED_MARKER, REF_MARKER

import collections
import urllib
import urlparse

# the markup we're going to use for [citation needed] and <ref> tags,
# pre-marked as safe for jinja.
SUPERSCRIPT_HTML = '[%s]'
SUPERSCRIPT_MARKUP = flask.Markup(SUPERSCRIPT_HTML)
CITATION_NEEDED_MARKUP = flask.Markup(SUPERSCRIPT_HTML)

Category = collections.namedtuple('Category', ['id', 'title'])
CATEGORY_ALL = Category('all', '')

# A class wrapping database access functions so they're easier to
# mock when testing.
class Database(object):
    @staticmethod
    def query_category_by_id(lang_code, cat_id):
        cursor = get_db(lang_code).cursor()
        with log_time('get category by id'):
            cursor.execute('''
                SELECT id, title FROM categories WHERE id = %s
            ''', (cat_id,))
            return cursor.fetchone()

    @staticmethod
    def query_snippet_by_id(lang_code, id):
        cursor = get_db(lang_code).cursor()
        with log_time('select snippet by id'):
            cursor.execute('''
                SELECT snippets.snippet, snippets.section, articles.url,
                articles.title FROM snippets, articles WHERE snippets.id = %s
                AND snippets.article_id = articles.page_id;''', (id,))
            return cursor.fetchone()

    @staticmethod
    def query_snippet_by_category(lang_code, cat_id):
        cursor = get_db(lang_code).cursor()
        with log_time('select with category'):
            cursor.execute('''
                SELECT snippets.id FROM snippets, articles_categories
                WHERE snippets.article_id = articles_categories.article_id AND
                articles_categories.category_id = %s ORDER BY RAND()
                LIMIT 1;''', (cat_id,))
            return cursor.fetchone()

    @staticmethod
    def query_random_snippet(lang_code, p):
        cursor = get_db(lang_code).cursor()
        cursor.execute(
            'SELECT id FROM snippets WHERE RAND() < %s LIMIT 1;', (p,))
        return cursor.fetchone()

    @staticmethod
    def query_next_id(lang_code, curr_id, cat_id):
        cursor = get_db(lang_code).cursor()

        with log_time('select next id'):
            cursor.execute('''
                SELECT next FROM snippets_links WHERE prev = %s
                AND cat_id = %s''', (curr_id, cat_id))
            return cursor.fetchone()

    @staticmethod
    def search_category(lang_code, needle, max_results):
        cursor = get_db(lang_code).cursor()

        needle = '%' + needle + '%'
        with log_time('search category'):
            cursor.execute('''
                SELECT id, title FROM categories WHERE title LIKE %s
                LIMIT %s''', (needle, max_results))
        return [{'id': row[0], 'title': row[1]} for row in cursor]

def get_category_by_id(lang_code, cat_id):
    if cat_id == CATEGORY_ALL.id:
        return CATEGORY_ALL
    c = Database.query_category_by_id(lang_code, cat_id)
    return Category(*c) if c is not None else None

def select_random_id(lang_code, cat = CATEGORY_ALL):
    ret = None
    if cat is not CATEGORY_ALL:
        ret = Database.query_snippet_by_category(lang_code, cat.id)

    if ret is None:
        # Try to pick one id at random. For small datasets, the probability
        # of getting an empty set in a query is non-negligible, so retry a
        # bunch of times as needed.
        p = '1e-4' if not flask.current_app.debug else '1e-2'
        with log_time('select without category'):
            for retry in range(1000):
                ret = Database.query_random_snippet(lang_code, p)
                if ret: break

    assert ret and len(ret) == 1
    return ret[0]

def select_next_id(lang_code, curr_id, cat = CATEGORY_ALL):
    if cat is not CATEGORY_ALL:
        ret = Database.query_next_id(lang_code, curr_id, cat.id)
        if ret is None:
            # curr_id doesn't belong to the category
            return None
        assert ret and len(ret) == 1
        next_id = ret[0]
    else:
        next_id = curr_id
        for i in range(3): # super paranoid :)
            next_id = select_random_id(lang_code, cat)
            if next_id != curr_id:
                break
    return next_id

def should_autofocus_category_filter(cat, request):
    return cat is CATEGORY_ALL and not request.MOBILE

@validate_lang_code
def citation_hunt(lang_code):
    id = flask.request.args.get('id')
    cat = flask.request.args.get('cat')
    cfg = config.get_localized_config(lang_code)

    lang_dir = cfg.lang_dir
    if flask.current_app.debug:
        lang_dir = flask.request.args.get('dir', lang_dir)

    if cat is not None:
        cat = get_category_by_id(lang_code, cat)
        if cat is None:
            # invalid category, normalize to "all" and try again by id
            cat = CATEGORY_ALL
            return flask.redirect(
                flask.url_for('citation_hunt',
                    lang_code = lang_code, id = id, cat = cat.id))
    else:
        cat = CATEGORY_ALL

    if id is not None:
        sinfo = Database.query_snippet_by_id(lang_code, id)
        if sinfo is None:
            # invalid id
            flask.request.cfg = cfg
            flask.abort(404)
        snippet, section, aurl, atitle = sinfo
        if cfg.html_snippet:
            snippet = flask.Markup(snippet)
        next_snippet_id = select_next_id(lang_code, id, cat)
        if next_snippet_id is None:
            # the snippet doesn't belong to the category!
            assert cat is not CATEGORY_ALL
            return flask.redirect(
                flask.url_for('citation_hunt',
                    id = id, cat = CATEGORY_ALL.id,
                    lang_code = lang_code))
        autofocus = should_autofocus_category_filter(cat, flask.request)
        article_url_path = urllib.quote(
            e(urlparse.urlparse(aurl).path.lstrip('/')))
        return flask.render_template('index.html',
            snippet = snippet, section = section, article_url = aurl,
            article_url_path = article_url_path,
            article_title = atitle, current_category = cat,
            next_snippet_id = next_snippet_id,
            cn_marker = CITATION_NEEDED_MARKER,
            cn_html = CITATION_NEEDED_MARKUP,
            ref_marker = REF_MARKER,
            ref_html = SUPERSCRIPT_MARKUP,
            config = cfg,
            lang_dir = lang_dir,
            category_filter_autofocus = autofocus)

    id = select_random_id(lang_code, cat)
    return flask.redirect(
        flask.url_for('citation_hunt',
            id = id, cat = cat.id, lang_code = lang_code))

@validate_lang_code
def search_category(lang_code):
    return flask.jsonify(
        results=Database.search_category(
            lang_code, flask.request.args.get('q'), max_results = 400))
<commit_msg>Remove debugging stuff.<commit_after>import chdb
import config
from utils import *
from common import *

from snippet_parser import CITATION_NEEDED_MARKER, REF_MARKER

import collections
import urllib
import urlparse

# the markup we're going to use for [citation needed] and <ref> tags,
# pre-marked as safe for jinja.
SUPERSCRIPT_HTML = '[%s]'
SUPERSCRIPT_MARKUP = flask.Markup(SUPERSCRIPT_HTML)
CITATION_NEEDED_MARKUP = flask.Markup(SUPERSCRIPT_HTML)

Category = collections.namedtuple('Category', ['id', 'title'])
CATEGORY_ALL = Category('all', '')

# A class wrapping database access functions so they're easier to
# mock when testing.
class Database(object):
    @staticmethod
    def query_category_by_id(lang_code, cat_id):
        cursor = get_db(lang_code).cursor()
        with log_time('get category by id'):
            cursor.execute('''
                SELECT id, title FROM categories WHERE id = %s
            ''', (cat_id,))
            return cursor.fetchone()

    @staticmethod
    def query_snippet_by_id(lang_code, id):
        cursor = get_db(lang_code).cursor()
        with log_time('select snippet by id'):
            cursor.execute('''
                SELECT snippets.snippet, snippets.section, articles.url,
                articles.title FROM snippets, articles WHERE snippets.id = %s
                AND snippets.article_id = articles.page_id;''', (id,))
            return cursor.fetchone()

    @staticmethod
    def query_snippet_by_category(lang_code, cat_id):
        cursor = get_db(lang_code).cursor()
        with log_time('select with category'):
            cursor.execute('''
                SELECT snippets.id FROM snippets, articles_categories
                WHERE snippets.article_id = articles_categories.article_id AND
                articles_categories.category_id = %s ORDER BY RAND()
                LIMIT 1;''', (cat_id,))
            return cursor.fetchone()

    @staticmethod
    def query_random_snippet(lang_code):
        cursor = get_db(lang_code).cursor()
        cursor.execute(
            'SELECT id FROM snippets WHERE RAND() < 1e-4 LIMIT 1;')
        return cursor.fetchone()

    @staticmethod
    def query_next_id(lang_code, curr_id, cat_id):
        cursor = get_db(lang_code).cursor()

        with log_time('select next id'):
            cursor.execute('''
                SELECT next FROM snippets_links WHERE prev = %s
                AND cat_id = %s''', (curr_id, cat_id))
            return cursor.fetchone()

    @staticmethod
    def search_category(lang_code, needle, max_results):
        cursor = get_db(lang_code).cursor()

        needle = '%' + needle + '%'
        with log_time('search category'):
            cursor.execute('''
                SELECT id, title FROM categories WHERE title LIKE %s
                LIMIT %s''', (needle, max_results))
        return [{'id': row[0], 'title': row[1]} for row in cursor]

def get_category_by_id(lang_code, cat_id):
    if cat_id == CATEGORY_ALL.id:
        return CATEGORY_ALL
    c = Database.query_category_by_id(lang_code, cat_id)
    return Category(*c) if c is not None else None

def select_random_id(lang_code, cat = CATEGORY_ALL):
    ret = None
    if cat is not CATEGORY_ALL:
        ret = Database.query_snippet_by_category(lang_code, cat.id)

    if ret is None:
        # Try to pick one id at random. For small datasets, the probability
        # of getting an empty set in a query is non-negligible, so retry a
        # bunch of times as needed.
        with log_time('select without category'):
            for retry in range(1000):
                ret = Database.query_random_snippet(lang_code)
                if ret: break

    assert ret and len(ret) == 1
    return ret[0]

def select_next_id(lang_code, curr_id, cat = CATEGORY_ALL):
    if cat is not CATEGORY_ALL:
        ret = Database.query_next_id(lang_code, curr_id, cat.id)
        if ret is None:
            # curr_id doesn't belong to the category
            return None
        assert ret and len(ret) == 1
        next_id = ret[0]
    else:
        next_id = curr_id
        for i in range(3): # super paranoid :)
            next_id = select_random_id(lang_code, cat)
            if next_id != curr_id:
                break
    return next_id

def should_autofocus_category_filter(cat, request):
    return cat is CATEGORY_ALL and not request.MOBILE

@validate_lang_code
def citation_hunt(lang_code):
    id = flask.request.args.get('id')
    cat = flask.request.args.get('cat')
    cfg = config.get_localized_config(lang_code)

    lang_dir = cfg.lang_dir
    if flask.current_app.debug:
        lang_dir = flask.request.args.get('dir', lang_dir)

    if cat is not None:
        cat = get_category_by_id(lang_code, cat)
        if cat is None:
            # invalid category, normalize to "all" and try again by id
            cat = CATEGORY_ALL
            return flask.redirect(
                flask.url_for('citation_hunt',
                    lang_code = lang_code, id = id, cat = cat.id))
    else:
        cat = CATEGORY_ALL

    if id is not None:
        sinfo = Database.query_snippet_by_id(lang_code, id)
        if sinfo is None:
            # invalid id
            flask.request.cfg = cfg
            flask.abort(404)
        snippet, section, aurl, atitle = sinfo
        if cfg.html_snippet:
            snippet = flask.Markup(snippet)
        next_snippet_id = select_next_id(lang_code, id, cat)
        if next_snippet_id is None:
            # the snippet doesn't belong to the category!
            assert cat is not CATEGORY_ALL
            return flask.redirect(
                flask.url_for('citation_hunt',
                    id = id, cat = CATEGORY_ALL.id,
                    lang_code = lang_code))
        autofocus = should_autofocus_category_filter(cat, flask.request)
        article_url_path = urllib.quote(
            e(urlparse.urlparse(aurl).path.lstrip('/')))
        return flask.render_template('index.html',
            snippet = snippet, section = section, article_url = aurl,
            article_url_path = article_url_path,
            article_title = atitle, current_category = cat,
            next_snippet_id = next_snippet_id,
            cn_marker = CITATION_NEEDED_MARKER,
            cn_html = CITATION_NEEDED_MARKUP,
            ref_marker = REF_MARKER,
            ref_html = SUPERSCRIPT_MARKUP,
            config = cfg,
            lang_dir = lang_dir,
            category_filter_autofocus = autofocus)

    id = select_random_id(lang_code, cat)
    return flask.redirect(
        flask.url_for('citation_hunt',
            id = id, cat = cat.id, lang_code = lang_code))

@validate_lang_code
def search_category(lang_code):
    return flask.jsonify(
        results=Database.search_category(
            lang_code, flask.request.args.get('q'), max_results = 400))
<|endoftext|>


<commit_before>#!/usr/bin/env python
import os
import sys
import urlparse
import requests

try:
    from requests.packages import urllib3
    urllib3.disable_warnings()
except ImportError:
    pass

AUTH_USERNAME = 'getsentry-bot'
AUTH_TOKEN = os.environ['GITHUB_AUTH_TOKEN']
AUTH = (AUTH_USERNAME, AUTH_TOKEN)
TAG = os.environ.get('TRAVIS_TAG') or \
    os.environ.get('APPVEYOR_REPO_TAG_NAME') or os.environ.get('BUILD_TAG')
TARGET = os.environ.get('TARGET')
BIN_TYPE = os.environ.get('BIN_TYPE', 'release')
REPO = 'getsentry/symsynd'

if sys.platform.startswith('win'):
    EXT = '.exe'
else:
    EXT = ''

def log(message, *args):
    if args:
        message = message % args
    print >> sys.stderr, message

def api_request(method, path, **kwargs):
    url = urlparse.urljoin('https://api.github.com/', path.lstrip('/'))
    # default travis python does not have SNI
    return requests.request(method, url, auth=AUTH, verify=False, **kwargs)

def find_executable():
    if TARGET:
        path = os.path.join('target', TARGET, BIN_TYPE, 'sentry-cli' + EXT)
        if os.path.isfile(path):
            return path
    path = os.path.join('target', BIN_TYPE, 'sentry-cli' + EXT)
    if os.path.isfile(path):
        return path

def get_target_executable_name():
    bits = TARGET.split('-')
    platform = bits[2].title()
    arch = bits[0]
    return 'sentry-cli-%s-%s%s' % (platform, arch, EXT)

def ensure_release():
    resp = api_request('GET', 'repos/%s/releases' % REPO)
    resp.raise_for_status()
    for release in resp.json():
        if release['tag_name'] == TAG:
            log('Found already existing release %s' % release['id'])
            return release
    resp = api_request('POST', 'repos/%s/releases' % REPO, json={
        'tag_name': TAG,
        'name': 'sentry-cli %s' % TAG,
        'draft': True,
    })
    resp.raise_for_status()
    release = resp.json()
    log('Created new release %s' % release['id'])
    return release

def upload_asset(release, executable, target_name):
    resp = api_request('GET', release['assets_url'])
    resp.raise_for_status()
    for asset in resp.json():
        if asset['name'] == target_name:
            log('Already have release asset %s. Skipping' % target_name)
            return

    upload_url = release['upload_url'].split('{')[0]
    with open(executable, 'rb') as f:
        log('Creating new release asset %s.' % target_name)
        resp = api_request('POST', upload_url,
                           params={'name': target_name},
                           headers={'Content-Type': 'application/octet-stream'},
                           data=f)
        resp.raise_for_status()

def main():
    if not TAG:
        return log('No tag specified.  Doing nothing.')
    executable = find_executable()
    if executable is None:
        return log('Could not locate executable.  Doing nothing.')

    target_executable_name = get_target_executable_name()
    release = ensure_release()
    upload_asset(release, executable, target_executable_name)

if __name__ == '__main__':
    main()
<commit_msg>Fixed upload script for travis<commit_after>#!/usr/bin/env python
import os
import sys
import urlparse
import requests

try:
    from requests.packages import urllib3
    urllib3.disable_warnings()
except ImportError:
    pass

AUTH_USERNAME = 'getsentry-bot'
AUTH_TOKEN = os.environ['GITHUB_AUTH_TOKEN']
AUTH = (AUTH_USERNAME, AUTH_TOKEN)
TAG = os.environ.get('TRAVIS_TAG') or \
    os.environ.get('APPVEYOR_REPO_TAG_NAME') or os.environ.get('BUILD_TAG')
NAME = 'symsynd'
REPO = 'getsentry/symsynd'

if sys.platform.startswith('win'):
    EXT = '.exe'
else:
    EXT = ''

def log(message, *args):
    if args:
        message = message % args
    print >> sys.stderr, message

def api_request(method, path, **kwargs):
    url = urlparse.urljoin('https://api.github.com/', path.lstrip('/'))
    # default travis python does not have SNI
    return requests.request(method, url, auth=AUTH, verify=False, **kwargs)

def find_wheels():
    dist = os.path.join('dist')
    for filename in os.listdir(dist):
        if filename.endswith('.whl'):
            yield os.path.join(dist, fileanme)

def get_target_executable_name():
    bits = TARGET.split('-')
    platform = bits[2].title()
    arch = bits[0]
    return 'sentry-cli-%s-%s%s' % (platform, arch, EXT)

def ensure_release():
    resp = api_request('GET', 'repos/%s/releases' % REPO)
    resp.raise_for_status()
    for release in resp.json():
        if release['tag_name'] == TAG:
            log('Found already existing release %s' % release['id'])
            return release
    resp = api_request('POST', 'repos/%s/releases' % REPO, json={
        'tag_name': TAG,
        'name': '%s %s' % (NAME, TAG),
        'draft': True,
    })
    resp.raise_for_status()
    release = resp.json()
    log('Created new release %s' % release['id'])
    return release

def upload_asset(release, path, asset_info):
    asset_name = os.path.basename(path)
    for asset in asset_info:
        if asset['name'] == asset_name:
            log('Already have release asset %s. Skipping' % asset_name)
            return

    upload_url = release['upload_url'].split('{')[0]
    with open(path, 'rb') as f:
        log('Creating new release asset %s.' % asset_name)
        resp = api_request('POST', upload_url,
                           params={'name': asset_name},
                           headers={'Content-Type': 'application/octet-stream'},
                           data=f)
        resp.raise_for_status()

def upload_assets(release, wheels):
    resp = api_request('GET', release['assets_url'])
    resp.raise_for_status()
    asset_info = resp.json()
    for wheel in wheels:
        upload_asset(release, wheel, asset_info)

def main():
    if not TAG:
        return log('No tag specified.  Doing nothing.')
    wheels = list(find_wheels())
    if not wheels:
        return log('Could not locate wheels.  Doing nothing.')

    release = ensure_release()
    upload_assets(release, wheels)

if __name__ == '__main__':
    main()
<|endoftext|>


<commit_before>#! /usr/bin/python

################################################################################################################
################################################################################################################
##
##
## 		Copyright 2015 William Whitty
## 		will.whitty.arbeit@gmail.com
##
## 		Licensed under the Apache License, Version 2.0 (the "License");
## 		you may not use this file except in compliance with the License.
## 		You may obtain a copy of the License at
##
## 		    http://www.apache.org/licenses/LICENSE-2.0
##
## 		Unless required by applicable law or agreed to in writing, software
## 		distributed under the License is distributed on an "AS IS" BASIS,
## 		WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
## 		See the License for the specific language governing permissions and
## 		limitations under the License.
##
##
################################################################################################################
################################################################################################################

# Import STD
import sys
import json
import time
import threading
import urllib2 as web

# Import Global
# Import Local

def help_function():
    print """
            ___           ___           ___           ___           ___       ___
           /\  \         /\__\         /\  \         /\  \         /\__\     /\  \
          /::\  \       /::|  |       /::\  \       /::\  \       /:/  /    /::\  \
         /:/\:\  \     /:|:|  |      /:/\:\  \     /:/\:\  \     /:/  /    /:/\:\  \
        /:/  \:\  \   /:/|:|__|__   /::\~\:\  \   /:/  \:\  \   /:/  /    /::\~\:\  \
       /:/__/ \:\__\ /:/ |::::\__\ /:/\:\ \:\__\ /:/__/_\:\__\ /:/__/    /:/\:\ \:\__\
       \:\  \ /:/  / \/__/~~/:/  / \:\~\:\ \/__/ \:\  /\ \/__/ \:\  \    \:\~\:\ \/__/
        \:\  /:/  /        /:/  /   \:\ \:\__\    \:\ \:\__\    \:\  \    \:\ \:\__\
         \:\/:/  /        /:/  /     \:\ \/__/     \:\/:/  /     \:\  \    \:\ \/__/
          \::/  /        /:/  /       \:\__\        \::/  /       \:\__\    \:\__\
           \/__/         \/__/         \/__/         \/__/         \/__/     \/__/

        Coded by William Whitty.
        Connects you to Omegle.com, a site for talking with random strangers.

        Type:

           next > End the conversation and move to the next stranger
           kill > End all conversations and quit the program
           help > Show this dialog
    """

class Omegle:
    """ Used for communication with Omegle """

    def __init__(self, **kwargs):
        """ Our constructor """

        # Our connection ID
        self.handle   = None
        self.stranger = None

        # Begin a connection
        if kwargs.get('connect', True):
            self.connect()

    def response(self):
        """ Get a RAW response from the stranger """
        try:
            return web.urlopen(self.stranger).read()
        except:
            return ''

    def wait_callback(self):
        """ Called when we are waiting for a connection """
        print 'Waiting...'

    def conn_callback(self):
        """ Called when we are connected to an active user """
        print 'Connected to a random stranger!\n'
        self.talk('I wrote an Omegle client in python, you can use it for bots etc. :) imgur.com/gallery/u46TfTL')

    def exit_callback(self):
        """ Called when we are disconnected from a session """
        print 'Stranger disconnected!\n'
        self.connect()

    def type_callback(self):
        """ Called when the stranger is typing """
        print 'Stranger is typing...'

    def hear_callback(self, message):
        """ Called when the stranger sends us a message """
        print 'Stranger: ' + message

    def listen(self):
        """ Used to listen for convesation partner input """

        # Loop until told to quit
        while True:

            # Get a response from the stranger
            response = self.response()

            # If the stranger has disconnected
            if 'strangerDisconnected' in response:
                self.exit_callback()

            # If the stranger is typing...
            elif 'typing' in response:
                self.type_callback()

            # If the stranger has sent us a message
            elif 'gotMessage' in response:
                self.hear_callback(response[16 : -3])

    def connect(self):
        """ Begin a new conversation """
        # Initialise the connection and return the url string stripped of quotations
        self.handle = web.urlopen('http://omegle.com/start').read()[1 : -1]

        # Save our nevent request
        self.stranger = web.Request('http://omegle.com/events', 'id=' + self.handle)

        # Get the response
        response = self.response()

        # If we're still waiting for a stranger
        if 'waiting' in response:
            self.wait_callback()

        # If we've got a good connection
        if 'connected' in response:
            self.conn_callback()

    def process(self, message):
        """ Check user input for terminal commands """
        # Check for our exit button
        if message == 'kill':
            quit(0)

        # Check for a help request
        if message == 'help':
            help_function()

    def type(self):
        """ Tell Omegle that we're typing something """
        web.urlopen('http://omegle.com/typing', 'id=' + self.handle).close()

    def talk(self, message, **kwargs):
        """ Send a message to our conversation partner """

        # Output to the terminal
        if kwargs.get('show', True):
            print 'You: ' + message

        # Process terminal commands
        if kwargs.get('process', True):
            self.process(message)

        # Talk to Omegle
        msgReq = web.urlopen('http://omegle.com/send', 'msg=' + message + '&id=' + self.handle).close()

    def start_thread(self):
        """ Begins a listener in a seperate thread, so that the user can give input """
        # Set our listener function as the thread callback
        self.listener = threading.Thread(target=self.listen)

        # Specify that this thread should not prevent us from exiting the program
        self.listener.daemon = True

        # Start the thread
        self.listener.start()

    def user_input(self):
        """ Called when the stranger sends us a message """

        # Pointer to internal thread
        self.listener = None

        while True:
            # Connect to a new stranger
            self.connect()

            # Start listening
            self.start_thread()

            while True:

                # Tell Omegle that we're typing
                self.type()

                # Get some input from the user
                input = raw_input()

                if 'next' in input:
                    self.listener.join(0)
                    break

                # Send the text to the stranger
                self.talk(input, show=False)

if __name__ == '__main__':

    # Display the user help
    help_function()

    # Create our Omegle instance
    handle = Omegle(connect=False)
    handle.user_input()
<commit_msg>Removed my connection message :#)<commit_after>#! /usr/bin/python

################################################################################################################
################################################################################################################
##
##
## 		Copyright 2015 William Whitty
## 		will.whitty.arbeit@gmail.com
##
## 		Licensed under the Apache License, Version 2.0 (the "License");
## 		you may not use this file except in compliance with the License.
## 		You may obtain a copy of the License at
##
## 		    http://www.apache.org/licenses/LICENSE-2.0
##
## 		Unless required by applicable law or agreed to in writing, software
## 		distributed under the License is distributed on an "AS IS" BASIS,
## 		WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
## 		See the License for the specific language governing permissions and
## 		limitations under the License.
##
##
################################################################################################################
################################################################################################################

# Import STD
import sys
import json
import time
import threading
import urllib2 as web

# Import Global
# Import Local

def help_function():
    print """
            ___           ___           ___           ___           ___       ___
           /\  \         /\__\         /\  \         /\  \         /\__\     /\  \
          /::\  \       /::|  |       /::\  \       /::\  \       /:/  /    /::\  \
         /:/\:\  \     /:|:|  |      /:/\:\  \     /:/\:\  \     /:/  /    /:/\:\  \
        /:/  \:\  \   /:/|:|__|__   /::\~\:\  \   /:/  \:\  \   /:/  /    /::\~\:\  \
       /:/__/ \:\__\ /:/ |::::\__\ /:/\:\ \:\__\ /:/__/_\:\__\ /:/__/    /:/\:\ \:\__\
       \:\  \ /:/  / \/__/~~/:/  / \:\~\:\ \/__/ \:\  /\ \/__/ \:\  \    \:\~\:\ \/__/
        \:\  /:/  /        /:/  /   \:\ \:\__\    \:\ \:\__\    \:\  \    \:\ \:\__\
         \:\/:/  /        /:/  /     \:\ \/__/     \:\/:/  /     \:\  \    \:\ \/__/
          \::/  /        /:/  /       \:\__\        \::/  /       \:\__\    \:\__\
           \/__/         \/__/         \/__/         \/__/         \/__/     \/__/

        Coded by William Whitty.
        Connects you to Omegle.com, a site for talking with random strangers.

        Type:

           next > End the conversation and move to the next stranger
           kill > End all conversations and quit the program
           help > Show this dialog
    """

class Omegle:
    """ Used for communication with Omegle """

    def __init__(self, **kwargs):
        """ Our constructor """

        # Our connection ID
        self.handle   = None
        self.stranger = None

        # Begin a connection
        if kwargs.get('connect', True):
            self.connect()

    def response(self):
        """ Get a RAW response from the stranger """
        try:
            return web.urlopen(self.stranger).read()
        except:
            return ''

    def wait_callback(self):
        """ Called when we are waiting for a connection """
        print 'Waiting...'

    def conn_callback(self):
        """ Called when we are connected to an active user """
        print 'Connected to a random stranger!\n'

    def exit_callback(self):
        """ Called when we are disconnected from a session """
        print 'Stranger disconnected!\n'
        self.connect()

    def type_callback(self):
        """ Called when the stranger is typing """
        print 'Stranger is typing...'

    def hear_callback(self, message):
        """ Called when the stranger sends us a message """
        print 'Stranger: ' + message

    def listen(self):
        """ Used to listen for convesation partner input """

        # Loop until told to quit
        while True:

            # Get a response from the stranger
            response = self.response()

            # If the stranger has disconnected
            if 'strangerDisconnected' in response:
                self.exit_callback()

            # If the stranger is typing...
            elif 'typing' in response:
                self.type_callback()

            # If the stranger has sent us a message
            elif 'gotMessage' in response:
                self.hear_callback(response[16 : -3])

    def connect(self):
        """ Begin a new conversation """
        # Initialise the connection and return the url string stripped of quotations
        self.handle = web.urlopen('http://omegle.com/start').read()[1 : -1]

        # Save our nevent request
        self.stranger = web.Request('http://omegle.com/events', 'id=' + self.handle)

        # Get the response
        response = self.response()

        # If we're still waiting for a stranger
        if 'waiting' in response:
            self.wait_callback()

        # If we've got a good connection
        if 'connected' in response:
            self.conn_callback()

    def process(self, message):
        """ Check user input for terminal commands """
        # Check for our exit button
        if message == 'kill':
            quit(0)

        # Check for a help request
        if message == 'help':
            help_function()

    def type(self):
        """ Tell Omegle that we're typing something """
        web.urlopen('http://omegle.com/typing', 'id=' + self.handle).close()

    def talk(self, message, **kwargs):
        """ Send a message to our conversation partner """

        # Output to the terminal
        if kwargs.get('show', True):
            print 'You: ' + message

        # Process terminal commands
        if kwargs.get('process', True):
            self.process(message)

        # Talk to Omegle
        msgReq = web.urlopen('http://omegle.com/send', 'msg=' + message + '&id=' + self.handle).close()

    def start_thread(self):
        """ Begins a listener in a seperate thread, so that the user can give input """
        # Set our listener function as the thread callback
        self.listener = threading.Thread(target=self.listen)

        # Specify that this thread should not prevent us from exiting the program
        self.listener.daemon = True

        # Start the thread
        self.listener.start()

    def user_input(self):
        """ Called when the stranger sends us a message """

        # Pointer to internal thread
        self.listener = None

        while True:
            # Connect to a new stranger
            self.connect()

            # Start listening
            self.start_thread()

            while True:

                # Tell Omegle that we're typing
                self.type()

                # Get some input from the user
                input = raw_input()

                if 'next' in input:
                    self.listener.join(0)
                    break

                # Send the text to the stranger
                self.talk(input, show=False)

if __name__ == '__main__':

    # Display the user help
    help_function()

    # Create our Omegle instance
    handle = Omegle(connect=False)
    handle.user_input()
<|endoftext|>


<commit_before>#!/usr/bin/python

from urllib import quote
import sys

try:
    from termcolor import colored
except ImportError:
    print "Dependency of the library 'termcolor' not satisfied. \
    Do $pip install termcolor to install the library :-) "
    sys.exit(0)

from util import get_url, check_valid_url

def generate_script(header_dict, details_dict, searchString=None):
    url = get_url(header_dict['Host'])
    if not check_valid_url(url):
        print(
            "Please enter a valid URL with correct domain name and try again ")
        sys.exit(0)

    if details_dict['data']:
        details_dict['data'] = '"' + str(details_dict['data']) + '"'

    encoding_list = ['HEAD', 'OPTIONS', 'GET']

    if details_dict['data'] and (details_dict['method'].strip()
                                 in encoding_list):
        print "Encoding URL"
        encoded_data = quote(details_dict['data'], '')
        url = url + encoded_data
        header_dict['Host'] = url
        details_dict['data'] = None

    if searchString:
        try:
            if 'proxy' not in details_dict:
                skeleton_code = '''\
#!/usr/bin/python
from tornado.httpclient import HTTPRequest, HTTPClient
from termcolor import colored
import re

def main():
    headers, url, method = ''' + str(header_dict) + ''', "''' + url + '''" , "''' + details_dict['method'].strip() + '''"
    body = ''' + str(details_dict['data']) + '''
    request_object = HTTPRequest(url, method=method,headers=headers, body=body, allow_nonstandard_methods=True)
    response_header = HTTPClient().fetch(request_object).headers
    match = re.findall(r"''' + searchString + '''", str(response_header))
    for x in range(0, len(match)) :
        replace_string = colored(match[x], 'green')
        response_header = re.sub(match[x], replace_string, str(response_header))
    print response_header

if __name__ == '__main__':
    main()

                '''
            else:
                skeleton_code = '''\
#!/usr/bin/python
from tornado.httpclient import HTTPRequest, HTTPClient
from termcolor import colored
import re

def main():

    headers, url, method = ''' + str(header_dict) + ''', "''' + url +\
                    '''" , "''' + details_dict['method'].strip() + '''"
    proxy_host, proxy_port = "''' + details_dict['proxy'].split(':')[0].strip() + \
                    '''", "''' + details_dict['proxy'].split(':')[1].strip() + '''"
    body = ''' + str(details_dict['data']) + '''

    request_object = HTTPRequest(url, method=method, headers=headers, proxy_host=proxy_host,\
        proxy_port=proxy_port, body=body, allow_nonstandard_methods=True)

    response_header = HTTPClient().fetch(request_object).headers
    for x in range(0, len(match)) :
        replace_string = colored(match[x], 'green')
        response_header = re.sub(match[x], replace_string, str(response_header))
    print response_header

if __name__ == '__main__':
    main()
                '''

        except IndexError:
            print "You haven't given the port Number"
        else:
            print skeleton_code

    else:
        try:
            if 'proxy' not in details_dict:
                skeleton_code = '''\
#!/usr/bin/python
from tornado.httpclient import HTTPRequest, HTTPClient

def main():
    headers, url, method = ''' + str(header_dict) + ''', "''' + url + '''" , "''' + details_dict['method'].strip() + '''"
    body = ''' + str(details_dict['data']) + '''
    request_object = HTTPRequest(url, method=method,headers=headers, body=body, allow_nonstandard_methods=True)
    response_header = HTTPClient().fetch(request_object).headers
    print response_header

if __name__ == '__main__':
    main()
                '''

            else:
                skeleton_code = '''\
#!/usr/bin/python
from tornado.httpclient import HTTPRequest, HTTPClient

def main():

    headers, url, method = ''' + str(header_dict) + ''', "''' + url +\
                    '''" , "''' + details_dict['method'].strip() + '''"
    proxy_host, proxy_port = "''' + details_dict['proxy'].split(':')[0].strip() +\
                    '''", "''' + details_dict['proxy'].split(':')[1].strip() + '''"
    body = ''' + str(details_dict['data']) + '''

    request_object = HTTPRequest(url, method=method, headers=headers,\
        proxy_host=proxy_host, proxy_port=proxy_port, body=body, allow_nonstandard_methods=True)

    return HTTPClient().fetch(request_object).headers

if __name__ == '__main__':
    main()
                '''

        except IndexError:
            print "You haven't given the port Number"

        else:
            print(skeleton_code)
<commit_msg>Quick cleaning of pythonScripts.py<commit_after>from __future__ import print_function

import sys
try:
    from urllib import quote
except ImportError:
    from urllib.parse import quote

try:
    from termcolor import colored
except ImportError:
    print("Dependency of the library 'termcolor' not satisfied. Do $pip install termcolor to install the library :-)")
    sys.exit(-1)

from util import get_url, check_valid_url

def generate_script(header_dict, details_dict, searchString=None):
    # TODO: Docstring and comments.
    url = get_url(header_dict['Host'])
    if not check_valid_url(url):
        print("Please enter a valid URL with correct domain name and try again ")
        sys.exit(-1)
    if details_dict['data']:
        details_dict['data'] = '"' + str(details_dict['data']) + '"'
    encoding_list = ['HEAD', 'OPTIONS', 'GET']
    if details_dict['data'] and (details_dict['method'].strip() in encoding_list):
        print("Encoding URL")
        encoded_data = quote(details_dict['data'], '')
        url = url + encoded_data
        header_dict['Host'] = url
        details_dict['data'] = None
    if searchString:
        try:
            if 'proxy' not in details_dict:
                skeleton_code = '''\
#!/usr/bin/python
import re
from tornado.httpclient import HTTPRequest, HTTPClient
from termcolor import colored

def main():
    headers, url, method = ''' + str(header_dict) + ''', "''' + url + '''" , "''' + details_dict['method'].strip() + '''"
    body = ''' + str(details_dict['data']) + '''
    request_object = HTTPRequest(url, method=method,headers=headers, body=body, allow_nonstandard_methods=True)
    response_header = HTTPClient().fetch(request_object).headers
    match = re.findall(r"''' + searchString + '''", str(response_header))
    for x in range(0, len(match)) :
        replace_string = colored(match[x], 'green')
        response_header = re.sub(match[x], replace_string, str(response_header))
    print response_header

if __name__ == '__main__':
    main()
                '''
            else:
                skeleton_code = '''\
#!/usr/bin/python
import re
from tornado.httpclient import HTTPRequest, HTTPClient
from termcolor import colored

def main():

    headers, url, method = ''' + str(header_dict) + ''', "''' + url +\
                    '''" , "''' + details_dict['method'].strip() + '''"
    proxy_host, proxy_port = "''' + details_dict['proxy'].split(':')[0].strip() + \
                    '''", "''' + details_dict['proxy'].split(':')[1].strip() + '''"
    body = ''' + str(details_dict['data']) + '''

    request_object = HTTPRequest(url, method=method, headers=headers, proxy_host=proxy_host,\
        proxy_port=proxy_port, body=body, allow_nonstandard_methods=True)

    response_header = HTTPClient().fetch(request_object).headers
    for x in range(0, len(match)) :
        replace_string = colored(match[x], 'green')
        response_header = re.sub(match[x], replace_string, str(response_header))
    print response_header

if __name__ == '__main__':
    main()
                '''
        except IndexError:
            print("You haven't given the port Number")
        else:
            print(skeleton_code)
    else:
        try:
            if 'proxy' not in details_dict:
                skeleton_code = '''\
#!/usr/bin/python
from tornado.httpclient import HTTPRequest, HTTPClient

def main():
    headers, url, method = ''' + str(header_dict) + ''', "''' + url + '''" , "''' + details_dict['method'].strip() + '''"
    body = ''' + str(details_dict['data']) + '''
    request_object = HTTPRequest(url, method=method,headers=headers, body=body, allow_nonstandard_methods=True)
    response_header = HTTPClient().fetch(request_object).headers
    print response_header

if __name__ == '__main__':
    main()
                '''
            else:
                skeleton_code = '''\
#!/usr/bin/python
from tornado.httpclient import HTTPRequest, HTTPClient

def main():

    headers, url, method = ''' + str(header_dict) + ''', "''' + url +\
                    '''" , "''' + details_dict['method'].strip() + '''"
    proxy_host, proxy_port = "''' + details_dict['proxy'].split(':')[0].strip() +\
                    '''", "''' + details_dict['proxy'].split(':')[1].strip() + '''"
    body = ''' + str(details_dict['data']) + '''

    request_object = HTTPRequest(url, method=method, headers=headers,\
        proxy_host=proxy_host, proxy_port=proxy_port, body=body, allow_nonstandard_methods=True)

    return HTTPClient().fetch(request_object).headers

if __name__ == '__main__':
    main()
                '''
        except IndexError:
            print("You haven't given the port Number")

        else:
            print(skeleton_code)
<|endoftext|>


<commit_before>import os

import numpy as np
import pandas as pd
from lingam.direct_lingam import DirectLiNGAM

def test_fit_success():
    # causal direction: x0 --> x1 --> x3
    x0 = np.random.uniform(size=1000)
    x1 = 2.0 * x0 + np.random.uniform(size=1000)
    x2 = np.random.uniform(size=1000)
    x3 = 4.0 * x1 + np.random.uniform(size=1000)
    X = pd.DataFrame(np.array([x0, x1, x2, x3]).T, columns=['x0', 'x1', 'x2', 'x3'])

    model = DirectLiNGAM()
    model.fit(X)

    # check the causal ordering
    co = model.causal_order_
    assert co.index(0) < co.index(1) < co.index(3)

    # check the adjacency matrix
    am = model.adjacency_matrix_
    assert am[1, 0] > 1.5 and am[3, 1] > 3.5

    am[1, 0] = 0.0
    am[3, 1] = 0.0
    assert np.sum(am) < 0.1

    # check estimate_total_effect (_BaseLiNGAM)
    te = model.estimate_total_effect(X, 0, 3)
    assert te > 7.5 and te < 8.5

    te = model.estimate_total_effect(X, 1, 3)
    assert te > 3.5 and te < 4.5

    te = model.estimate_total_effect(X, 0, 2)
    assert te < 0.2

    te = model.estimate_total_effect(X, 3, 0)
    assert te < 0.2

    p_values = model.get_error_independence_p_values(X)

def test_fit_invalid_data():
    # Not array data
    X = 1
    try:
        model = DirectLiNGAM()
        model.fit(X)
    except ValueError:
        pass
    else:
        raise AssertionError

    # Include non-numeric data
    x0 = np.random.uniform(size=5)
    x1 = np.array(['X', 'Y', 'X', 'Y', 'X'])
    X = pd.DataFrame(np.array([x0, x1]).T, columns=['x0', 'x1'])
    try:
        model = DirectLiNGAM()
        model.fit(X)
    except ValueError:
        pass
    else:
        raise AssertionError

    # Include NaN values
    x0 = np.random.uniform(size=1000)
    x1 = 2.0 * x0 + np.random.uniform(size=1000)
    X = pd.DataFrame(np.array([x0, x1]).T, columns=['x0', 'x1'])
    X.iloc[100, 0] = np.nan
    try:
        model = DirectLiNGAM()
        model.fit(X)
    except ValueError:
        pass
    else:
        raise AssertionError

    # Include infinite values
    x0 = np.random.uniform(size=1000)
    x1 = 2.0 * x0 + np.random.uniform(size=1000)
    X = pd.DataFrame(np.array([x0, x1]).T, columns=['x0', 'x1'])
    X.iloc[100, 0] = np.inf
    try:
        model = DirectLiNGAM()
        model.fit(X)
    except ValueError:
        pass
    else:
        raise AssertionError

def test_prior_knowledge_success():
    # causal direction: x0 --> x1 --> x3
    x0 = np.random.uniform(size=1000)
    x1 = 2.0 * x0 + np.random.uniform(size=1000)
    x2 = np.random.uniform(size=1000)
    x3 = 4.0 * x1 + np.random.uniform(size=1000)
    X = pd.DataFrame(np.array([x0, x1, x2, x3]).T, columns=['x0', 'x1', 'x2', 'x3'])

    # prior knowledge: nothing
    pk = np.array(
        [
            [0, -1, -1, -1],
            [-1, 0, -1, -1],
            [-1, -1, 0, -1],
            [-1, -1, -1, 0],
        ]
    )

    model = DirectLiNGAM(prior_knowledge=pk)
    model.fit(X)
    co = model.causal_order_
    assert co.index(0) < co.index(1) < co.index(3)

    # prior knowledge: x1 is exogenous
    pk = np.array(
        [
            [0, -1, -1, -1],
            [0, 0, 0, 0],
            [-1, -1, 0, -1],
            [-1, -1, -1, 0],
        ]
    )

    model = DirectLiNGAM(prior_knowledge=pk)
    model.fit(X)
    co = model.causal_order_
    print(co)
    assert co.index(1) < co.index(0)
    assert co.index(1) < co.index(2)
    assert co.index(1) < co.index(3)

    # prior knowledge: x0 is sink
    pk = np.array(
        [
            [0, -1, -1, -1],
            [0, 0, -1, -1],
            [0, -1, 0, -1],
            [0, -1, -1, 0],
        ]
    )

    model = DirectLiNGAM(prior_knowledge=pk)
    model.fit(X)
    co = model.causal_order_
    print(co)
    assert co.index(0) > co.index(1)
    assert co.index(0) > co.index(2)
    assert co.index(0) > co.index(3)

    # prior knowledge: x2-->x3 has path
    pk = np.array(
        [
            [0, -1, -1, -1],
            [-1, 0, -1, -1],
            [-1, -1, 0, 0],
            [-1, -1, 1, 0],
        ]
    )

    model = DirectLiNGAM(prior_knowledge=pk)
    model.fit(X)
    co = model.causal_order_
    print(co)
    assert co.index(2) < co.index(3)

    # prior knowledge: x1-->x3 does not have path
    pk = np.array(
        [
            [0, -1, -1, -1],
            [-1, 0, -1, -1],
            [-1, -1, 0, -1],
            [-1, 0, -1, 0],
        ]
    )

    model = DirectLiNGAM(prior_knowledge=pk)
    model.fit(X)

    # prior knowledge: x1 is exogenous (apply_prior_knowledge_softly)
    pk = np.array(
        [
            [0, -1, -1, -1],
            [0, 0, 0, 0],
            [-1, -1, 0, -1],
            [-1, -1, -1, 0],
        ]
    )

    model = DirectLiNGAM(prior_knowledge=pk, apply_prior_knowledge_softly=True)
    model.fit(X)
    co = model.causal_order_
    print(co)
    assert co.index(1) < co.index(0)
    assert co.index(1) < co.index(2)
    assert co.index(1) < co.index(3)

    # prior knowledge: x0 is sink (apply_prior_knowledge_softly)
    pk = np.array(
        [
            [0, -1, -1, -1],
            [0, 0, -1, -1],
            [0, -1, 0, -1],
            [0, -1, -1, 0],
        ]
    )

    model = DirectLiNGAM(prior_knowledge=pk, apply_prior_knowledge_softly=True)
    model.fit(X)
    co = model.causal_order_
    print(co)
    assert co.index(0) > co.index(1)
    assert co.index(0) > co.index(2)
    assert co.index(0) > co.index(3)

    # prior knowledge: x2-->x3 has path (apply_prior_knowledge_softly)
    pk = np.array(
        [
            [0, -1, -1, -1],
            [-1, 0, -1, -1],
            [-1, -1, 0, 0],
            [-1, -1, 1, 0],
        ]
    )

    model = DirectLiNGAM(prior_knowledge=pk, apply_prior_knowledge_softly=True)
    model.fit(X)
    co = model.causal_order_
    print(co)
    assert co.index(2) < co.index(3)

    # prior knowledge: x1-->x3 does not have path (apply_prior_knowledge_softly)
    pk = np.array(
        [
            [0, -1, -1, -1],
            [-1, 0, -1, -1],
            [-1, -1, 0, -1],
            [-1, 0, -1, 0],
        ]
    )

    model = DirectLiNGAM(prior_knowledge=pk, apply_prior_knowledge_softly=True)
    model.fit(X)

    # prior knowledge: x1 is exogenous (apply_prior_knowledge_softly)
    pk = np.array(
        [
            [0, -1, -1, 0],
            [0, 0, -1, 0],
            [0, -1, 0, -1],
            [0, 0, 0, 0],
        ]
    )

    model = DirectLiNGAM(prior_knowledge=pk, apply_prior_knowledge_softly=True)
    model.fit(X)
    co = model.causal_order_

def test_prior_knowledge_invalid():
    # causal direction: x0 --> x1 --> x3
    x0 = np.random.uniform(size=1000)
    x1 = 2.0 * x0 + np.random.uniform(size=1000)
    x2 = np.random.uniform(size=1000)
    x3 = 4.0 * x1 + np.random.uniform(size=1000)
    X = pd.DataFrame(np.array([x0, x1, x2, x3]).T, columns=['x0', 'x1', 'x2', 'x3'])

    # prior knowledge: invalid
    pk = np.array(
        [
            [0, -1, 1],
            [-1, 0, -1],
            [-1, -1, 0],
            [-1, -1, -1],
        ]
    )

    try:
        model = DirectLiNGAM(prior_knowledge=pk)
        model.fit(X)
    except ValueError:
        pass
    else:
        raise AssertionError

    # prior knowledge: inconsistent
    pk = np.array(
        [
            [0, 1, -1, -1],
            [1, 0, -1, -1],
            [-1, -1, 0, -1],
            [-1, -1, -1, 0],
        ]
    )

    try:
        model = DirectLiNGAM(prior_knowledge=pk)
        model.fit(X)
    except ValueError:
        pass
    else:
        raise AssertionError

def test_kernel_mode():
    # causal direction: x0 --> x1 --> x2
    x0 = np.random.uniform(size=1100)
    x1 = 0.5 * x0 + np.random.uniform(size=1100)
    x2 = 0.5 * x0 + 0.5 * x1 + np.random.uniform(size=1100)
    X = pd.DataFrame(np.array([x0, x1, x2]).T, columns=['x0', 'x1', 'x2'])

    model = DirectLiNGAM(measure='kernel')
    model.fit(X.iloc[:100, :])

    model = DirectLiNGAM(measure='kernel')
    model.fit(X)

def test_base():
    # causal direction: x0 --> x1 --> x2
    x0 = np.random.uniform(size=1100)
    x1 = 0.5 * x0 + np.random.uniform(size=1100)
    x2 = 0.5 * x0 + 0.5 * x1 + np.random.uniform(size=1100)
    X = pd.DataFrame(np.array([x0, x1, x2]).T, columns=['x0', 'x1', 'x2'])

    # prior knowledge: x1 is exogenous
    pk = np.array(
        [
            [0, 0, 0, 0],
            [-1, 0, -1, -1],
            [-1, -1, 0, -1],
            [-1, -1, -1, 0],
        ]
    )
    model = DirectLiNGAM()
    model._causal_order = [1, 0, 2]
    model._estimate_adjacency_matrix(X.values, prior_knowledge=pk)
<commit_msg>Update test_direct_lingam.py<commit_after>import os

import numpy as np
import pandas as pd
from lingam.direct_lingam import DirectLiNGAM

def test_fit_success():
    # causal direction: x0 --> x1 --> x3
    x0 = np.random.uniform(size=1000)
    x1 = 2.0 * x0 + np.random.uniform(size=1000)
    x2 = np.random.uniform(size=1000)
    x3 = 4.0 * x1 + np.random.uniform(size=1000)
    X = pd.DataFrame(np.array([x0, x1, x2, x3]).T, columns=['x0', 'x1', 'x2', 'x3'])

    model = DirectLiNGAM()
    model.fit(X)

    # check the causal ordering
    co = model.causal_order_
    assert co.index(0) < co.index(1) < co.index(3)

    # check the adjacency matrix
    am = model.adjacency_matrix_
    # assert am[1, 0] > 1.5 and am[3, 1] > 3.5

    am[1, 0] = 0.0
    am[3, 1] = 0.0
    # assert np.sum(am) < 0.1

    # check estimate_total_effect (_BaseLiNGAM)
    te = model.estimate_total_effect(X, 0, 3)
    # assert te > 7.5 and te < 8.5

    te = model.estimate_total_effect(X, 1, 3)
    # assert te > 3.5 and te < 4.5

    te = model.estimate_total_effect(X, 0, 2)
    # assert te < 0.2

    te = model.estimate_total_effect(X, 3, 0)
    # assert te < 0.2

    p_values = model.get_error_independence_p_values(X)

def test_fit_invalid_data():
    # Not array data
    X = 1
    try:
        model = DirectLiNGAM()
        model.fit(X)
    except ValueError:
        pass
    else:
        raise AssertionError

    # Include non-numeric data
    x0 = np.random.uniform(size=5)
    x1 = np.array(['X', 'Y', 'X', 'Y', 'X'])
    X = pd.DataFrame(np.array([x0, x1]).T, columns=['x0', 'x1'])
    try:
        model = DirectLiNGAM()
        model.fit(X)
    except ValueError:
        pass
    else:
        raise AssertionError

    # Include NaN values
    x0 = np.random.uniform(size=1000)
    x1 = 2.0 * x0 + np.random.uniform(size=1000)
    X = pd.DataFrame(np.array([x0, x1]).T, columns=['x0', 'x1'])
    X.iloc[100, 0] = np.nan
    try:
        model = DirectLiNGAM()
        model.fit(X)
    except ValueError:
        pass
    else:
        raise AssertionError

    # Include infinite values
    x0 = np.random.uniform(size=1000)
    x1 = 2.0 * x0 + np.random.uniform(size=1000)
    X = pd.DataFrame(np.array([x0, x1]).T, columns=['x0', 'x1'])
    X.iloc[100, 0] = np.inf
    try:
        model = DirectLiNGAM()
        model.fit(X)
    except ValueError:
        pass
    else:
        raise AssertionError

def test_prior_knowledge_success():
    # causal direction: x0 --> x1 --> x3
    x0 = np.random.uniform(size=1000)
    x1 = 2.0 * x0 + np.random.uniform(size=1000)
    x2 = np.random.uniform(size=1000)
    x3 = 4.0 * x1 + np.random.uniform(size=1000)
    X = pd.DataFrame(np.array([x0, x1, x2, x3]).T, columns=['x0', 'x1', 'x2', 'x3'])

    # prior knowledge: nothing
    pk = np.array(
        [
            [0, -1, -1, -1],
            [-1, 0, -1, -1],
            [-1, -1, 0, -1],
            [-1, -1, -1, 0],
        ]
    )

    model = DirectLiNGAM(prior_knowledge=pk)
    model.fit(X)
    co = model.causal_order_
    assert co.index(0) < co.index(1) < co.index(3)

    # prior knowledge: x1 is exogenous
    pk = np.array(
        [
            [0, -1, -1, -1],
            [0, 0, 0, 0],
            [-1, -1, 0, -1],
            [-1, -1, -1, 0],
        ]
    )

    model = DirectLiNGAM(prior_knowledge=pk)
    model.fit(X)
    co = model.causal_order_
    print(co)
    assert co.index(1) < co.index(0)
    assert co.index(1) < co.index(2)
    assert co.index(1) < co.index(3)

    # prior knowledge: x0 is sink
    pk = np.array(
        [
            [0, -1, -1, -1],
            [0, 0, -1, -1],
            [0, -1, 0, -1],
            [0, -1, -1, 0],
        ]
    )

    model = DirectLiNGAM(prior_knowledge=pk)
    model.fit(X)
    co = model.causal_order_
    print(co)
    assert co.index(0) > co.index(1)
    assert co.index(0) > co.index(2)
    assert co.index(0) > co.index(3)

    # prior knowledge: x2-->x3 has path
    pk = np.array(
        [
            [0, -1, -1, -1],
            [-1, 0, -1, -1],
            [-1, -1, 0, 0],
            [-1, -1, 1, 0],
        ]
    )

    model = DirectLiNGAM(prior_knowledge=pk)
    model.fit(X)
    co = model.causal_order_
    print(co)
    assert co.index(2) < co.index(3)

    # prior knowledge: x1-->x3 does not have path
    pk = np.array(
        [
            [0, -1, -1, -1],
            [-1, 0, -1, -1],
            [-1, -1, 0, -1],
            [-1, 0, -1, 0],
        ]
    )

    model = DirectLiNGAM(prior_knowledge=pk)
    model.fit(X)

    # prior knowledge: x1 is exogenous (apply_prior_knowledge_softly)
    pk = np.array(
        [
            [0, -1, -1, -1],
            [0, 0, 0, 0],
            [-1, -1, 0, -1],
            [-1, -1, -1, 0],
        ]
    )

    model = DirectLiNGAM(prior_knowledge=pk, apply_prior_knowledge_softly=True)
    model.fit(X)
    co = model.causal_order_
    print(co)
    assert co.index(1) < co.index(0)
    assert co.index(1) < co.index(2)
    assert co.index(1) < co.index(3)

    # prior knowledge: x0 is sink (apply_prior_knowledge_softly)
    pk = np.array(
        [
            [0, -1, -1, -1],
            [0, 0, -1, -1],
            [0, -1, 0, -1],
            [0, -1, -1, 0],
        ]
    )

    model = DirectLiNGAM(prior_knowledge=pk, apply_prior_knowledge_softly=True)
    model.fit(X)
    co = model.causal_order_
    print(co)
    assert co.index(0) > co.index(1)
    assert co.index(0) > co.index(2)
    assert co.index(0) > co.index(3)

    # prior knowledge: x2-->x3 has path (apply_prior_knowledge_softly)
    pk = np.array(
        [
            [0, -1, -1, -1],
            [-1, 0, -1, -1],
            [-1, -1, 0, 0],
            [-1, -1, 1, 0],
        ]
    )

    model = DirectLiNGAM(prior_knowledge=pk, apply_prior_knowledge_softly=True)
    model.fit(X)
    co = model.causal_order_
    print(co)
    assert co.index(2) < co.index(3)

    # prior knowledge: x1-->x3 does not have path (apply_prior_knowledge_softly)
    pk = np.array(
        [
            [0, -1, -1, -1],
            [-1, 0, -1, -1],
            [-1, -1, 0, -1],
            [-1, 0, -1, 0],
        ]
    )

    model = DirectLiNGAM(prior_knowledge=pk, apply_prior_knowledge_softly=True)
    model.fit(X)

    # prior knowledge: x1 is exogenous (apply_prior_knowledge_softly)
    pk = np.array(
        [
            [0, -1, -1, 0],
            [0, 0, -1, 0],
            [0, -1, 0, -1],
            [0, 0, 0, 0],
        ]
    )

    model = DirectLiNGAM(prior_knowledge=pk, apply_prior_knowledge_softly=True)
    model.fit(X)
    co = model.causal_order_

def test_prior_knowledge_invalid():
    # causal direction: x0 --> x1 --> x3
    x0 = np.random.uniform(size=1000)
    x1 = 2.0 * x0 + np.random.uniform(size=1000)
    x2 = np.random.uniform(size=1000)
    x3 = 4.0 * x1 + np.random.uniform(size=1000)
    X = pd.DataFrame(np.array([x0, x1, x2, x3]).T, columns=['x0', 'x1', 'x2', 'x3'])

    # prior knowledge: invalid
    pk = np.array(
        [
            [0, -1, 1],
            [-1, 0, -1],
            [-1, -1, 0],
            [-1, -1, -1],
        ]
    )

    try:
        model = DirectLiNGAM(prior_knowledge=pk)
        model.fit(X)
    except ValueError:
        pass
    else:
        raise AssertionError

    # prior knowledge: inconsistent
    pk = np.array(
        [
            [0, 1, -1, -1],
            [1, 0, -1, -1],
            [-1, -1, 0, -1],
            [-1, -1, -1, 0],
        ]
    )

    try:
        model = DirectLiNGAM(prior_knowledge=pk)
        model.fit(X)
    except ValueError:
        pass
    else:
        raise AssertionError

def test_kernel_mode():
    # causal direction: x0 --> x1 --> x2
    x0 = np.random.uniform(size=1100)
    x1 = 0.5 * x0 + np.random.uniform(size=1100)
    x2 = 0.5 * x0 + 0.5 * x1 + np.random.uniform(size=1100)
    X = pd.DataFrame(np.array([x0, x1, x2]).T, columns=['x0', 'x1', 'x2'])

    model = DirectLiNGAM(measure='kernel')
    model.fit(X.iloc[:100, :])

    model = DirectLiNGAM(measure='kernel')
    model.fit(X)

def test_base():
    # causal direction: x0 --> x1 --> x2
    x0 = np.random.uniform(size=1100)
    x1 = 0.5 * x0 + np.random.uniform(size=1100)
    x2 = 0.5 * x0 + 0.5 * x1 + np.random.uniform(size=1100)
    X = pd.DataFrame(np.array([x0, x1, x2]).T, columns=['x0', 'x1', 'x2'])

    # prior knowledge: x1 is exogenous
    pk = np.array(
        [
            [0, 0, 0, 0],
            [-1, 0, -1, -1],
            [-1, -1, 0, -1],
            [-1, -1, -1, 0],
        ]
    )
    model = DirectLiNGAM()
    model._causal_order = [1, 0, 2]
    model._estimate_adjacency_matrix(X.values, prior_knowledge=pk)
<|endoftext|>


<commit_before>#!/usr/bin/env python
import sys
import struct

Inst = struct.Struct("BBHI")

CLASSES = {
    0: "LD",
    1: "LDX",
    2: "ST",
    3: "STX",
    4: "ALU",
    5: "JMP",
    7: "ALU64",
}

ALU_OPCODES = {
    0: 'ADD',
    1: 'SUB',
    2: 'MUL',
    3: 'DIV',
    4: 'OR',
    5: 'AND',
    6: 'LSH',
    7: 'RSH',
    8: 'NEG',
    9: 'MOD',
    10: 'XOR',
    11: 'MOV',
    12: 'ARSH',
    13: 'END',
}

JMP_OPCODES = {
    0: 'JA',
    1: 'JEQ',
    2: 'JGT',
    3: 'JGE',
    4: 'JSET',
    5: 'JNE',
    6: 'JSGT',
    7: 'JSGE',
    8: 'CALL',
    9: 'EXIT',
}

MODES = {
    0: 'IMM',
    1: 'ABS',
    2: 'IND',
    3: 'MEM',
    6: 'XADD',
}

SIZES = {
    0: 'W',
    1: 'H',
    2: 'B',
    3: 'DW',
}

BPF_CLASS_LD = 0
BPF_CLASS_LDX = 1
BPF_CLASS_ST = 2
BPF_CLASS_STX = 3
BPF_CLASS_ALU = 4
BPF_CLASS_JMP = 5
BPF_CLASS_ALU64 = 7

BPF_ALU_END = 13

def R(reg):
    return "r" + str(reg)

def I(imm):
    return "$%x" % imm

def M(base, off):
    if off != 0:
        return "[%s%s]" % (base, O(off))
    else:
        return "[%s]" % base

def O(off):
    if off <= 32767:
        return "+" + str(off)
    else:
        return "-" + str(65536-off)

def disassemble_one(data, offset):
    code, regs, off, imm = Inst.unpack_from(data, offset)
    dst_reg = regs & 0xf
    src_reg = (regs >> 4) & 0xf
    cls = code & 7

    class_name = CLASSES.get(cls)

    if cls == BPF_CLASS_ALU or cls == BPF_CLASS_ALU64:
        source = (code >> 3) & 1
        opcode = (code >> 4) & 0xf
        opcode_name = ALU_OPCODES.get(opcode)
        if cls == BPF_CLASS_ALU64:
            opcode_name += "64"

        if opcode == BPF_ALU_END:
            return "%s%d %s" % (opcode_name, imm, R(dst_reg))
        elif source == 0:
            return "%s %s, %s" % (opcode_name, R(dst_reg), I(imm))
        else:
            return "%s %s, %s" % (opcode_name, R(dst_reg), R(src_reg))
    elif cls == BPF_CLASS_JMP:
        source = (code >> 3) & 1
        opcode = (code >> 4) & 0xf
        opcode_name = JMP_OPCODES.get(opcode)

        if opcode_name == "EXIT":
            return "EXIT"
        elif opcode_name == "CALL":
            return "%s %s" % (opcode_name, I(imm))
        elif opcode_name == "JA":
            return "%s %s" % (opcode_name, O(off))
        elif source == 0:
            return "%s %s, %s, %s" % (opcode_name, R(dst_reg), I(imm), O(off))
        else:
            return "%s %s, %s, %s" % (opcode_name, R(dst_reg), R(src_reg), O(off))
    elif cls == BPF_CLASS_LD or cls == BPF_CLASS_LDX or cls == BPF_CLASS_ST or cls == BPF_CLASS_STX:
        size = (code >> 3) & 3
        mode = (code >> 5) & 7
        mode_name = MODES.get(mode, str(mode))
        size_name = SIZES.get(size, str(size))
        if cls == BPF_CLASS_LD and mode_name == "IMM":
            return "%s %s %s, %s" % (class_name + size_name, mode_name, R(dst_reg), I(imm))
        elif cls == BPF_CLASS_LD:
            # Probably not correct
            return "%s %s %s, %s" % (class_name + size_name, mode_name, M(R(dst_reg), off), I(imm))
        elif cls == BPF_CLASS_LDX:
            return "%s %s %s, %s" % (class_name + size_name, mode_name, R(dst_reg), M(R(src_reg), off))
        elif cls == BPF_CLASS_ST:
            return "%s %s %s, %s" % (class_name + size_name, mode_name, M(R(dst_reg), off), I(imm))
        elif cls == BPF_CLASS_STX:
            return "%s %s %s, %s" % (class_name + size_name, mode_name, M(R(dst_reg), off), R(src_reg))
    else:
        return "unknown instruction %#x" % code

if __name__ == "__main__":
    data = sys.stdin.read()
    offset = 0
    while offset < len(data):
        print disassemble_one(data, offset)
        offset += 8
<commit_msg>disassembler: use argparse<commit_after>#!/usr/bin/env python
"""
eBPF disassembler

Reads the given file or stdin. The input should be raw eBPF
instructions (not an ELF object file).
"""
import sys
import struct
import argparse

Inst = struct.Struct("BBHI")

CLASSES = {
    0: "LD",
    1: "LDX",
    2: "ST",
    3: "STX",
    4: "ALU",
    5: "JMP",
    7: "ALU64",
}

ALU_OPCODES = {
    0: 'ADD',
    1: 'SUB',
    2: 'MUL',
    3: 'DIV',
    4: 'OR',
    5: 'AND',
    6: 'LSH',
    7: 'RSH',
    8: 'NEG',
    9: 'MOD',
    10: 'XOR',
    11: 'MOV',
    12: 'ARSH',
    13: 'END',
}

JMP_OPCODES = {
    0: 'JA',
    1: 'JEQ',
    2: 'JGT',
    3: 'JGE',
    4: 'JSET',
    5: 'JNE',
    6: 'JSGT',
    7: 'JSGE',
    8: 'CALL',
    9: 'EXIT',
}

MODES = {
    0: 'IMM',
    1: 'ABS',
    2: 'IND',
    3: 'MEM',
    6: 'XADD',
}

SIZES = {
    0: 'W',
    1: 'H',
    2: 'B',
    3: 'DW',
}

BPF_CLASS_LD = 0
BPF_CLASS_LDX = 1
BPF_CLASS_ST = 2
BPF_CLASS_STX = 3
BPF_CLASS_ALU = 4
BPF_CLASS_JMP = 5
BPF_CLASS_ALU64 = 7

BPF_ALU_END = 13

def R(reg):
    return "r" + str(reg)

def I(imm):
    return "$%x" % imm

def M(base, off):
    if off != 0:
        return "[%s%s]" % (base, O(off))
    else:
        return "[%s]" % base

def O(off):
    if off <= 32767:
        return "+" + str(off)
    else:
        return "-" + str(65536-off)

def disassemble_one(data, offset):
    code, regs, off, imm = Inst.unpack_from(data, offset)
    dst_reg = regs & 0xf
    src_reg = (regs >> 4) & 0xf
    cls = code & 7

    class_name = CLASSES.get(cls)

    if cls == BPF_CLASS_ALU or cls == BPF_CLASS_ALU64:
        source = (code >> 3) & 1
        opcode = (code >> 4) & 0xf
        opcode_name = ALU_OPCODES.get(opcode)
        if cls == BPF_CLASS_ALU64:
            opcode_name += "64"

        if opcode == BPF_ALU_END:
            return "%s%d %s" % (opcode_name, imm, R(dst_reg))
        elif source == 0:
            return "%s %s, %s" % (opcode_name, R(dst_reg), I(imm))
        else:
            return "%s %s, %s" % (opcode_name, R(dst_reg), R(src_reg))
    elif cls == BPF_CLASS_JMP:
        source = (code >> 3) & 1
        opcode = (code >> 4) & 0xf
        opcode_name = JMP_OPCODES.get(opcode)

        if opcode_name == "EXIT":
            return "EXIT"
        elif opcode_name == "CALL":
            return "%s %s" % (opcode_name, I(imm))
        elif opcode_name == "JA":
            return "%s %s" % (opcode_name, O(off))
        elif source == 0:
            return "%s %s, %s, %s" % (opcode_name, R(dst_reg), I(imm), O(off))
        else:
            return "%s %s, %s, %s" % (opcode_name, R(dst_reg), R(src_reg), O(off))
    elif cls == BPF_CLASS_LD or cls == BPF_CLASS_LDX or cls == BPF_CLASS_ST or cls == BPF_CLASS_STX:
        size = (code >> 3) & 3
        mode = (code >> 5) & 7
        mode_name = MODES.get(mode, str(mode))
        size_name = SIZES.get(size, str(size))
        if cls == BPF_CLASS_LD and mode_name == "IMM":
            return "%s %s %s, %s" % (class_name + size_name, mode_name, R(dst_reg), I(imm))
        elif cls == BPF_CLASS_LD:
            # Probably not correct
            return "%s %s %s, %s" % (class_name + size_name, mode_name, M(R(dst_reg), off), I(imm))
        elif cls == BPF_CLASS_LDX:
            return "%s %s %s, %s" % (class_name + size_name, mode_name, R(dst_reg), M(R(src_reg), off))
        elif cls == BPF_CLASS_ST:
            return "%s %s %s, %s" % (class_name + size_name, mode_name, M(R(dst_reg), off), I(imm))
        elif cls == BPF_CLASS_STX:
            return "%s %s %s, %s" % (class_name + size_name, mode_name, M(R(dst_reg), off), R(src_reg))
    else:
        return "unknown instruction %#x" % code

if __name__ == "__main__":
    parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
    parser.add_argument('file', type=argparse.FileType('r'), default='-')
    args = parser.parse_args()

    data = args.file.read()
    offset = 0
    while offset < len(data):
        print disassemble_one(data, offset)
        offset += 8
<|endoftext|>


<commit_before># Copyright 2015 Google Inc. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

from __future__ import print_function, division, absolute_import

import glob
import os
import plistlib
import re
import tempfile
import time

from booleanOperations import BooleanOperationManager
from cu2qu.ufo import font_to_quadratic, fonts_to_quadratic
from defcon import Font
from fontTools import subset
from fontTools.misc.transform import Identity
from fontTools.pens.transformPen import TransformPen
from glyphs2ufo.glyphslib import build_masters, build_instances
from mutatorMath.ufo import build as build_designspace
from ufo2ft import compileOTF, compileTTF
from ufo2ft.makeotfParts import FeatureOTFCompiler

class FontProject:
    """Provides methods for building fonts."""

    @staticmethod
    def preprocess(glyphs_path):
        """Return Glyphs source with illegal glyph/class names changed."""

        with open(glyphs_path) as fp:
            text = fp.read()
        names = set(re.findall('\n(?:glyph)?name = "(.+-.+)";\n', text))

        if names:
            num_names = len(names)
            printed_names = sorted(names)[:5]
            if num_names > 5:
                printed_names.append('...')
            print('Found %s glyph names containing hyphens: %s' % (
                num_names, ', '.join(printed_names)))
            print('Replacing all hyphens with underscores.')

        for old_name in names:
            new_name = old_name.replace('-', '_')
            text = text.replace(old_name, new_name)
        return text

    def build_masters(self, glyphs_path, is_italic=False):
        """Build master UFOs from Glyphs source."""

        master_dir = self._output_dir('ufo')
        return build_masters(glyphs_path, master_dir, is_italic)

    def build_instances(self, glyphs_path, is_italic=False):
        """Build instance UFOs from Glyphs source."""

        master_dir = self._output_dir('ufo')
        instance_dir = self._output_dir('ufo', is_instance=True)
        return build_instances(glyphs_path, master_dir, instance_dir, is_italic)

    def remove_overlaps(self, ufos):
        """Remove overlaps in UFOs' glyphs' contours."""

        manager = BooleanOperationManager()
        for ufo in ufos:
            print('>> Removing overlaps for ' + self._font_name(ufo))
            for glyph in ufo:
                contours = list(glyph)
                glyph.clearContours()
                manager.union(contours, glyph.getPointPen())

    def decompose_glyphs(self, ufos):
        """Move components of UFOs' glyphs to their outlines."""

        for ufo in ufos:
            print('>> Decomposing glyphs for ' + self._font_name(ufo))
            for glyph in ufo:
                self._deep_copy_contours(ufo, glyph, glyph, Identity)
                glyph.clearComponents()

    def _deep_copy_contours(self, ufo, parent, component, transformation):
        """Copy contours from component to parent, including nested components."""

        for nested in component.components:
            self._deep_copy_contours(
                ufo, parent, ufo[nested.baseGlyph],
                transformation.transform(nested.transformation))

        if component != parent:
            component.draw(TransformPen(parent.getPen(), transformation))

    def build_otfs(self, ufos, **kwargs):
        """Build OpenType binaries with CFF outlines."""

        print('\n>> Building OTFs')

        self.decompose_glyphs(ufos)
        self.remove_overlaps(ufos)
        self.save_otfs(ufos, **kwargs)

    def build_ttfs(self, ufos, **kwargs):
        """Build OpenType binaries with TrueType outlines."""

        print('\n>> Building TTFs')

        self.remove_overlaps(ufos)

        start_t = time.time()
        for ufo in ufos:
            print('>> Converting curves for ' + self._font_name(ufo))
            font_to_quadratic(ufo, dump_stats=True)
        print('[took %f seconds]' % (time.time() - start_t))

        self.save_otfs(ufos, ttf=True, **kwargs)

    def build_interpolatable_ttfs(self, ufos, **kwargs):
        """Build OpenType binaries with interpolatable TrueType outlines."""

        print('\n>> Building interpolation-compatible TTFs')

        print('>> Converting curves to quadratic')
        start_t = time.time()
        fonts_to_quadratic(ufos, dump_stats=True)
        print('[took %f seconds]' % (time.time() - start_t))

        self.save_otfs(ufos, ttf=True, interpolatable=True, **kwargs)

    def save_otfs(
            self, ufos, ttf=False, interpolatable=False, mti_paths=None,
            is_instance=False, use_afdko=False, subset=True):
        """Write OpenType binaries."""

        ext = 'ttf' if ttf else 'otf'
        fea_compiler = FDKFeatureCompiler if use_afdko else FeatureOTFCompiler
        otf_compiler = compileTTF if ttf else compileOTF

        for ufo in ufos:
            name = self._font_name(ufo)
            print('>> Saving %s for %s' % (ext.upper(), name))

            otf_path = self._output_path(ufo, ext, is_instance, interpolatable)
            otf = otf_compiler(ufo, featureCompilerClass=fea_compiler,
                               mtiFeaFiles=(mti_paths and mti_paths.get(name)))
            otf.save(otf_path)

            if subset:
                self.subset_otf_from_ufo(otf_path, ufo)

    def subset_otf_from_ufo(self, otf_path, ufo):
        """Subset a font using export flags set by glyphs2ufo."""

        font_lib_prefix = 'com.schriftgestaltung.'
        glyph_lib_prefix = font_lib_prefix + 'Glyphs.'

        keep_glyphs = set(ufo.lib.get(font_lib_prefix + 'Keep Glyphs', []))

        include = []
        glyph_order = ufo.lib['public.glyphOrder']
        for glyph_name in glyph_order:
            glyph = ufo[glyph_name]
            if ((keep_glyphs and glyph_name not in keep_glyphs) or
                not glyph.lib.get(glyph_lib_prefix + 'Export', True)):
                continue
            include.append(glyph_name)

        # copied from nototools.subset
        opt = subset.Options()
        opt.name_IDs = ['*']
        opt.name_legacy = True
        opt.name_languages = ['*']
        opt.layout_features = ['*']
        opt.notdef_outline = True
        opt.recalc_bounds = True
        opt.recalc_timestamp = True
        opt.canonical_order = True

        opt.glyph_names = ufo.lib.get(
            font_lib_prefix + "Don't use Production Names")

        font = subset.load_font(otf_path, opt, lazy=False)
        subsetter = subset.Subsetter(options=opt)
        subsetter.populate(glyphs=include)
        subsetter.subset(font)
        subset.save_font(font, otf_path, opt)

    def run_from_glyphs(
            self, glyphs_path, preprocess=True, interpolate=False, **kwargs):
        """Run toolchain from Glyphs source to OpenType binaries."""

        is_italic = 'Italic' in glyphs_path

        if preprocess:
            print('>> Checking Glyphs source for illegal glyph names')
            glyphs_source = self.preprocess(glyphs_path)
            tmp_glyphs_file = tempfile.NamedTemporaryFile()
            glyphs_path = tmp_glyphs_file.name
            tmp_glyphs_file.write(glyphs_source)
            tmp_glyphs_file.seek(0)

        if interpolate:
            print('>> Interpolating master UFOs from Glyphs source')
            ufos = self.build_instances(glyphs_path, is_italic)
        else:
            print('>> Loading master UFOs from Glyphs source')
            ufos = self.build_masters(glyphs_path, is_italic)

        self.run_from_ufos(ufos, is_instance=interpolate, **kwargs)

    def run_from_designspace(self, designspace_path, **kwargs):
        """Run toolchain from a MutatorMath design space document to OpenType
        binaries.
        """

        print('>> Interpolating master UFOs from design space')
        results = build_designspace(designspace_path)
        ufos = []
        for result in results:
            ufos.extend(result.values())
        self.run_from_ufos(ufos, **kwargs)

    def run_from_ufos(self, ufos, output=(), mti_source=None, **kwargs):
        """Run toolchain from UFO sources to OpenType binaries."""

        if hasattr(ufos[0], 'path'):
            ufo_paths = [ufo.path for ufo in ufos]
        else:
            if isinstance(ufos, str):
                ufo_paths = glob.glob(ufos)
            else:
                ufo_paths = ufos
            ufos = [Font(path) for path in ufo_paths]

        mti_paths = {}
        if mti_source:
            mti_paths = plistlib.readPlist(mti_source)
            src_dir = os.path.dirname(mti_source)
            for paths in mti_paths.values():
                for table in ('GDEF', 'GPOS', 'GSUB'):
                    paths[table] = os.path.join(src_dir, paths[table])

        need_reload = False
        if 'otf' in output:
            self.build_otfs(ufos, mti_paths=mti_paths, **kwargs)
            need_reload = True

        if 'ttf' in output:
            if need_reload:
                ufos = [Font(path) for path in ufo_paths]
            self.build_ttfs(ufos, mti_paths=mti_paths, **kwargs)
            need_reload = True

        if 'ttf-interpolatable' in output:
            if need_reload:
                ufos = [Font(path) for path in ufo_paths]
            self.build_interpolatable_ttfs(ufos, mti_paths=mti_paths, **kwargs)

    def _font_name(self, ufo):
        return '%s-%s' % (ufo.info.familyName.replace(' ', ''),
                          ufo.info.styleName.replace(' ', ''))

    def _output_dir(self, ext, is_instance=False, interpolatable=False):
        """Generate an output directory."""

        dir_prefix = 'instance_' if is_instance else 'master_'
        dir_suffix = '_interpolatable' if interpolatable else ''
        return dir_prefix + ext + dir_suffix

    def _output_path(self, ufo, ext, is_instance=False, interpolatable=False):
        """Generate output path for a font file with given extension."""

        out_dir = self._output_dir(ext, is_instance, interpolatable)
        if not os.path.exists(out_dir):
            os.makedirs(out_dir)
        return os.path.join(out_dir, '%s.%s' % (self._font_name(ufo), ext))

class FDKFeatureCompiler(FeatureOTFCompiler):
    """An OTF compiler which uses the AFDKO to compile feature syntax."""

    def setupFile_featureTables(self):
        if self.mtiFeaFiles is not None:
            super(FDKFeatureCompiler, self).setupFile_featureTables()

        elif not self.features.strip():
            return

        import subprocess
        from fontTools.ttLib import TTFont
        from fontTools.misc.py23 import tostr

        fd, outline_path = tempfile.mkstemp()
        os.close(fd)
        self.outline.save(outline_path)

        fd, feasrc_path = tempfile.mkstemp()
        os.close(fd)

        fd, fea_path = tempfile.mkstemp()
        os.write(fd, self.features)
        os.close(fd)

        report = tostr(subprocess.check_output([
            "makeotf", "-o", feasrc_path, "-f", outline_path,
            "-ff", fea_path]))
        os.remove(outline_path)
        os.remove(fea_path)

        print(report)
        success = "Done." in report
        if success:
            feasrc = TTFont(feasrc_path)
            for table in ["GDEF", "GPOS", "GSUB"]:
                if table in feasrc:
                    self.outline[table] = feasrc[table]

        feasrc.close()
        os.remove(feasrc_path)
        if not success:
            raise ValueError("Feature syntax compilation failed.")
<commit_msg>Always write designspace doc with UFO masters<commit_after># Copyright 2015 Google Inc. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

from __future__ import print_function, division, absolute_import

import glob
import os
import plistlib
import re
import tempfile
import time

from booleanOperations import BooleanOperationManager
from cu2qu.ufo import font_to_quadratic, fonts_to_quadratic
from defcon import Font
from fontTools import subset
from fontTools.misc.transform import Identity
from fontTools.pens.transformPen import TransformPen
from glyphs2ufo.glyphslib import build_masters, build_instances
from mutatorMath.ufo import build as build_designspace
from ufo2ft import compileOTF, compileTTF
from ufo2ft.makeotfParts import FeatureOTFCompiler

class FontProject:
    """Provides methods for building fonts."""

    @staticmethod
    def preprocess(glyphs_path):
        """Return Glyphs source with illegal glyph/class names changed."""

        with open(glyphs_path) as fp:
            text = fp.read()
        names = set(re.findall('\n(?:glyph)?name = "(.+-.+)";\n', text))

        if names:
            num_names = len(names)
            printed_names = sorted(names)[:5]
            if num_names > 5:
                printed_names.append('...')
            print('Found %s glyph names containing hyphens: %s' % (
                num_names, ', '.join(printed_names)))
            print('Replacing all hyphens with underscores.')

        for old_name in names:
            new_name = old_name.replace('-', '_')
            text = text.replace(old_name, new_name)
        return text

    def build_ufos(self, glyphs_path, is_italic=False, interpolate=False):
        """Build UFOs from Glyphs source."""

        master_dir = self._output_dir('ufo')
        instance_dir = self._output_dir('ufo', is_instance=True)
        if interpolate:
            return build_instances(glyphs_path, master_dir, instance_dir,
                                   is_italic)
        else:
            return build_masters(glyphs_path, master_dir, is_italic,
                                 designspace_instance_dir=instance_dir)

    def remove_overlaps(self, ufos):
        """Remove overlaps in UFOs' glyphs' contours."""

        manager = BooleanOperationManager()
        for ufo in ufos:
            print('>> Removing overlaps for ' + self._font_name(ufo))
            for glyph in ufo:
                contours = list(glyph)
                glyph.clearContours()
                manager.union(contours, glyph.getPointPen())

    def decompose_glyphs(self, ufos):
        """Move components of UFOs' glyphs to their outlines."""

        for ufo in ufos:
            print('>> Decomposing glyphs for ' + self._font_name(ufo))
            for glyph in ufo:
                self._deep_copy_contours(ufo, glyph, glyph, Identity)
                glyph.clearComponents()

    def _deep_copy_contours(self, ufo, parent, component, transformation):
        """Copy contours from component to parent, including nested components."""

        for nested in component.components:
            self._deep_copy_contours(
                ufo, parent, ufo[nested.baseGlyph],
                transformation.transform(nested.transformation))

        if component != parent:
            component.draw(TransformPen(parent.getPen(), transformation))

    def build_otfs(self, ufos, **kwargs):
        """Build OpenType binaries with CFF outlines."""

        print('\n>> Building OTFs')

        self.decompose_glyphs(ufos)
        self.remove_overlaps(ufos)
        self.save_otfs(ufos, **kwargs)

    def build_ttfs(self, ufos, **kwargs):
        """Build OpenType binaries with TrueType outlines."""

        print('\n>> Building TTFs')

        self.remove_overlaps(ufos)

        start_t = time.time()
        for ufo in ufos:
            print('>> Converting curves for ' + self._font_name(ufo))
            font_to_quadratic(ufo, dump_stats=True)
        print('[took %f seconds]' % (time.time() - start_t))

        self.save_otfs(ufos, ttf=True, **kwargs)

    def build_interpolatable_ttfs(self, ufos, **kwargs):
        """Build OpenType binaries with interpolatable TrueType outlines."""

        print('\n>> Building interpolation-compatible TTFs')

        print('>> Converting curves to quadratic')
        start_t = time.time()
        fonts_to_quadratic(ufos, dump_stats=True)
        print('[took %f seconds]' % (time.time() - start_t))

        self.save_otfs(ufos, ttf=True, interpolatable=True, **kwargs)

    def save_otfs(
            self, ufos, ttf=False, interpolatable=False, mti_paths=None,
            is_instance=False, use_afdko=False, subset=True):
        """Write OpenType binaries."""

        ext = 'ttf' if ttf else 'otf'
        fea_compiler = FDKFeatureCompiler if use_afdko else FeatureOTFCompiler
        otf_compiler = compileTTF if ttf else compileOTF

        for ufo in ufos:
            name = self._font_name(ufo)
            print('>> Saving %s for %s' % (ext.upper(), name))

            otf_path = self._output_path(ufo, ext, is_instance, interpolatable)
            otf = otf_compiler(ufo, featureCompilerClass=fea_compiler,
                               mtiFeaFiles=(mti_paths and mti_paths.get(name)))
            otf.save(otf_path)

            if subset:
                self.subset_otf_from_ufo(otf_path, ufo)

    def subset_otf_from_ufo(self, otf_path, ufo):
        """Subset a font using export flags set by glyphs2ufo."""

        font_lib_prefix = 'com.schriftgestaltung.'
        glyph_lib_prefix = font_lib_prefix + 'Glyphs.'

        keep_glyphs = set(ufo.lib.get(font_lib_prefix + 'Keep Glyphs', []))

        include = []
        glyph_order = ufo.lib['public.glyphOrder']
        for glyph_name in glyph_order:
            glyph = ufo[glyph_name]
            if ((keep_glyphs and glyph_name not in keep_glyphs) or
                not glyph.lib.get(glyph_lib_prefix + 'Export', True)):
                continue
            include.append(glyph_name)

        # copied from nototools.subset
        opt = subset.Options()
        opt.name_IDs = ['*']
        opt.name_legacy = True
        opt.name_languages = ['*']
        opt.layout_features = ['*']
        opt.notdef_outline = True
        opt.recalc_bounds = True
        opt.recalc_timestamp = True
        opt.canonical_order = True

        opt.glyph_names = ufo.lib.get(
            font_lib_prefix + "Don't use Production Names")

        font = subset.load_font(otf_path, opt, lazy=False)
        subsetter = subset.Subsetter(options=opt)
        subsetter.populate(glyphs=include)
        subsetter.subset(font)
        subset.save_font(font, otf_path, opt)

    def run_from_glyphs(
            self, glyphs_path, preprocess=True, interpolate=False, **kwargs):
        """Run toolchain from Glyphs source to OpenType binaries."""

        is_italic = 'Italic' in glyphs_path

        if preprocess:
            print('>> Checking Glyphs source for illegal glyph names')
            glyphs_source = self.preprocess(glyphs_path)
            tmp_glyphs_file = tempfile.NamedTemporaryFile()
            glyphs_path = tmp_glyphs_file.name
            tmp_glyphs_file.write(glyphs_source)
            tmp_glyphs_file.seek(0)

        print('>> Building UFOs from Glyphs source')
        ufos = self.build_ufos(glyphs_path, is_italic, interpolate)
        self.run_from_ufos(ufos, is_instance=interpolate, **kwargs)

    def run_from_designspace(self, designspace_path, **kwargs):
        """Run toolchain from a MutatorMath design space document to OpenType
        binaries.
        """

        print('>> Interpolating master UFOs from design space')
        results = build_designspace(designspace_path)
        ufos = []
        for result in results:
            ufos.extend(result.values())
        self.run_from_ufos(ufos, **kwargs)

    def run_from_ufos(self, ufos, output=(), mti_source=None, **kwargs):
        """Run toolchain from UFO sources to OpenType binaries."""

        if hasattr(ufos[0], 'path'):
            ufo_paths = [ufo.path for ufo in ufos]
        else:
            if isinstance(ufos, str):
                ufo_paths = glob.glob(ufos)
            else:
                ufo_paths = ufos
            ufos = [Font(path) for path in ufo_paths]

        mti_paths = {}
        if mti_source:
            mti_paths = plistlib.readPlist(mti_source)
            src_dir = os.path.dirname(mti_source)
            for paths in mti_paths.values():
                for table in ('GDEF', 'GPOS', 'GSUB'):
                    paths[table] = os.path.join(src_dir, paths[table])

        need_reload = False
        if 'otf' in output:
            self.build_otfs(ufos, mti_paths=mti_paths, **kwargs)
            need_reload = True

        if 'ttf' in output:
            if need_reload:
                ufos = [Font(path) for path in ufo_paths]
            self.build_ttfs(ufos, mti_paths=mti_paths, **kwargs)
            need_reload = True

        if 'ttf-interpolatable' in output:
            if need_reload:
                ufos = [Font(path) for path in ufo_paths]
            self.build_interpolatable_ttfs(ufos, mti_paths=mti_paths, **kwargs)

    def _font_name(self, ufo):
        return '%s-%s' % (ufo.info.familyName.replace(' ', ''),
                          ufo.info.styleName.replace(' ', ''))

    def _output_dir(self, ext, is_instance=False, interpolatable=False):
        """Generate an output directory."""

        dir_prefix = 'instance_' if is_instance else 'master_'
        dir_suffix = '_interpolatable' if interpolatable else ''
        return dir_prefix + ext + dir_suffix

    def _output_path(self, ufo, ext, is_instance=False, interpolatable=False):
        """Generate output path for a font file with given extension."""

        out_dir = self._output_dir(ext, is_instance, interpolatable)
        if not os.path.exists(out_dir):
            os.makedirs(out_dir)
        return os.path.join(out_dir, '%s.%s' % (self._font_name(ufo), ext))

class FDKFeatureCompiler(FeatureOTFCompiler):
    """An OTF compiler which uses the AFDKO to compile feature syntax."""

    def setupFile_featureTables(self):
        if self.mtiFeaFiles is not None:
            super(FDKFeatureCompiler, self).setupFile_featureTables()

        elif not self.features.strip():
            return

        import subprocess
        from fontTools.ttLib import TTFont
        from fontTools.misc.py23 import tostr

        fd, outline_path = tempfile.mkstemp()
        os.close(fd)
        self.outline.save(outline_path)

        fd, feasrc_path = tempfile.mkstemp()
        os.close(fd)

        fd, fea_path = tempfile.mkstemp()
        os.write(fd, self.features)
        os.close(fd)

        report = tostr(subprocess.check_output([
            "makeotf", "-o", feasrc_path, "-f", outline_path,
            "-ff", fea_path]))
        os.remove(outline_path)
        os.remove(fea_path)

        print(report)
        success = "Done." in report
        if success:
            feasrc = TTFont(feasrc_path)
            for table in ["GDEF", "GPOS", "GSUB"]:
                if table in feasrc:
                    self.outline[table] = feasrc[table]

        feasrc.close()
        os.remove(feasrc_path)
        if not success:
            raise ValueError("Feature syntax compilation failed.")
<|endoftext|>


<commit_before>import csv
import os.path
import getch

import os
if 'analysis' in os.getcwd():
    os.chdir('..')
print(os.getcwd())

# Login with username
username = input("-Please enter username: ")
while username == "":
    username = input("Error: username can't be blank\nPlease enter username: ")

# File paths used
paragraphFilePath = "data/paragraphs.csv"
responseFilePath = "data/responses_" + username + ".csv"

# Dictionary to hold previously entered responses
responseDictionary = {}

# create the response file if it doesn't exist; if it does exist, load it into responseDictionary
if os.path.isfile(responseFilePath):
    with open(responseFilePath, "r") as responseFile:
        responseReader = csv.DictReader(responseFile)
        for row in responseReader:
            responseDictionary[row['number']] = row['response']
else:
    with(open(responseFilePath, "w+")) as responseFile:
        responseFile.write("number,response\n")
    responseFile.close()

print("+Found", len(responseDictionary), " previously recorded responses by", username, "\n")

# Go through the remaining paragraphs and record the user's response in the corresponding response file.
with open(paragraphFilePath) as paragraphFile, open(responseFilePath, "a", newline="") as responseFile:
    paragraphReader = csv.DictReader(paragraphFile)
    responseWriter = csv.writer(responseFile)
    for paragraph in paragraphReader:
        if paragraph['number'] not in responseDictionary.keys():
            print("#%s: %s" % (paragraph['number'], paragraph['text']))
            print("-Is this related to topic? (Y/N/F), or q to quit   ")
            userResponse = getch.getch()
            if isinstance(userResponse, bytes):
                userResponse = userResponse.decode()
            while userResponse not in ('y', 'n', 'Y', 'N', 'f', 'F', 'q', 'Q'):
                print("-Bad input... Is this related to topic? (Y/N/F), q to quit   ")
                userResponse = getch.getch()
            if userResponse.lower() == 'q':
                break
            print("+Your response, %s, is recorded\n" % userResponse.upper())
            responseDictionary[paragraph['number']] = userResponse
            field = [paragraph['number'], userResponse.upper()]
            responseWriter.writerow(field)
responseFile.close()
<commit_msg>updated import statements - removed os.path<commit_after>import csv
import getch
import os

if 'analysis' in os.getcwd():
    os.chdir('..')
print(os.getcwd())

# Login with username
username = input("-Please enter username: ")
while username == "":
    username = input("Error: username can't be blank\nPlease enter username: ")

# File paths used
paragraphFilePath = "data/paragraphs.csv"
responseFilePath = "data/responses_" + username + ".csv"

# Dictionary to hold previously entered responses
responseDictionary = {}

# create the response file if it doesn't exist; if it does exist, load it into responseDictionary
if os.path.isfile(responseFilePath):
    with open(responseFilePath, "r") as responseFile:
        responseReader = csv.DictReader(responseFile)
        for row in responseReader:
            responseDictionary[row['number']] = row['response']
else:
    with(open(responseFilePath, "w+")) as responseFile:
        responseFile.write("number,response\n")
    responseFile.close()

print("+Found", len(responseDictionary), " previously recorded responses by", username, "\n")

# Go through the remaining paragraphs and record the user's response in the corresponding response file.
with open(paragraphFilePath) as paragraphFile, open(responseFilePath, "a", newline="") as responseFile:
    paragraphReader = csv.DictReader(paragraphFile)
    responseWriter = csv.writer(responseFile)
    for paragraph in paragraphReader:
        if paragraph['number'] not in responseDictionary.keys():
            print("#%s: %s" % (paragraph['number'], paragraph['text']))
            print("-Is this related to topic? (Y/N/F), or q to quit   ")
            userResponse = getch.getch()
            if isinstance(userResponse, bytes):
                userResponse = userResponse.decode()
            while userResponse not in ('y', 'n', 'Y', 'N', 'f', 'F', 'q', 'Q'):
                print("-Bad input... Is this related to topic? (Y/N/F), q to quit   ")
                userResponse = getch.getch()
            if userResponse.lower() == 'q':
                break
            print("+Your response, %s, is recorded\n" % userResponse.upper())
            responseDictionary[paragraph['number']] = userResponse
            field = [paragraph['number'], userResponse.upper()]
            responseWriter.writerow(field)
responseFile.close()
<|endoftext|>


<commit_before>#!/usr/bin/env python3
# -*- coding: utf-8 -*-

from hwt.hdlObjects.constants import DIRECTION
from hwt.hdlObjects.typeShortcuts import vecT
from hwt.interfaces.std import Signal, Clk
from hwt.synthesizer.interfaceLevel.interface import Interface
from hwt.synthesizer.interfaceLevel.unit import Unit

# [TODO] this is only torso
class CameraIntf(Interface):
    """
    http://www.voti.nl/docs/OV7670.pdf p.7

    HREF is high when data of row are valid
    there is fall in HSYNC after each row
    VSYNC is on for a while after end of the frame (after fall in HSYNC)
    """
    def _declr(self):
        self.d = Signal(dtype=vecT(8))
        self.xclk = Clk(masterDir=DIRECTION.IN)
        self.pclk = Clk()  # pixel clock generated by master

        self.href = Signal()
        self.hsync = Signal()
        self.vsync = Signal()

        self.reset = Signal(masterDir=DIRECTION.IN)
        self.pwdn = Signal(masterDir=DIRECTION.IN)

class CameraToAxiStream(Unit):
    pass<commit_msg>rm partially implemented units (cameraToAxiStream)<commit_after><|endoftext|>