<commit_before># gglogic boolean logic device simulations for ggmath

from ggmath import MathApp, _MathDynamic
from abc import ABCMeta, abstractmethod

class _BoolDevice(_MathDynamic, metaclass=ABCMeta):

    def __init__(self, mininputqty, *args, **kwargs):
        """
        Required Inputs

        * **mininputqty** minimum number of inputs possible

        Optional keyword arguments

        * **namedinputs** list of input names
        """
        self.In = [None]*mininputqty
        self.Enable = True
        namedinputs = kwargs.get('namedinputs', [])
        self._indict = {name:Eval(None) for name in namedinputs}

    @property
    def In(self):
        return self._input

    @In.setter
    def In(self, val):
        try:
            self._input = [self.Eval(v) for v in list(val)]
        except TypeError:
            self._input = [self.Eval(val)]

    # Enable attribute controls the "tri-state" of output
    @property
    def Enable(self):
        return self._enable

    @Enable.setter
    def Enable(self, val):
        self._enable = self.Eval(val)

    @abstractmethod
    def _getvalue(self):
        pass

    def __call__(self):
        if self.Enable:
            return self._getvalue()
        else:
            return None

    def __getattr__(self, name):
        return self._indict[name]

    def __setattr__(self, name, val):
        try:
            self._indict[name] = self.Eval(val)
        except AttributeError:
            print("boom")
            super().__setattr(name, val)

class _BoolOneInput(_BoolDevice):

    def __init__(self, *args, **kwargs):
        """
        No Required Inputs
        """
        super().__init__(1)

class _BoolMultiInput(_BoolDevice):

    def __init__(self, *args, **kwargs):
        """
        No Required Inputs
        """
        super().__init__(2)

class BoolNOT(_BoolOneInput):

    def _getvalue(self):
        inval = self.In[0]()
        if inval == None:
            return True  # equivalent to an "open" input
        else:
            return not inval

class BoolAND(_BoolMultiInput):

    def _getvalue(self):
        for v in self._input:
            if not v():
                return False
        return True

class TestDevice(_BoolOneInput):

    def __init__(self, *args, **kwargs):
        kwargs['namedinputs'] = ['in1', 'in2']
        super().__init__(*args, **kwargs)

    def out1(self):
        return self.in1 and self.in2

    def out2(self):
        return self.in1 or self.in2

# test code here
if __name__ == "__main__":

    from ggmath import GlassButton, LEDIndicator, MetalToggle

    IC1 = BoolNOT()
    IC2 = BoolAND()

    b1 = MetalToggle(1, (1,0))
    b2 = MetalToggle(1, (1,0.3))
    db1 = LEDIndicator((1.3,0), b1)
    db2 = LEDIndicator((1.3,0.3), b2)

    b3 = MetalToggle(1, (1,0.6))
    b4 = MetalToggle(1, (1,0.9))
    db1 = LEDIndicator((1.3,0.6), b3)
    db2 = LEDIndicator((1.3,0.9), b4)

    d2 = LEDIndicator((1.5,0.45), IC2)

    IC2.In = b1, b2
    IC2.In = IC2.In + [b3, b4]

    button = GlassButton(None, (0,0))
    LED = LEDIndicator((0,-1), IC1)
    IC1.In = button

    t1 = MetalToggle(1, (1,-1))
    t2 = MetalToggle(1, (1, -1.3))
    td = TestDevice()
    dt1 = LEDIndicator((1.3, -1), t1)
    dt2 = LEDIndicator((1.3, -1.3), t2)
    dtd1 = LEDIndicator((1.5, -1), td.out1)
    dtd2 = LEDIndicator((1.5, -1.3), td.out2)

    app = MathApp()
    app.run()<commit_msg>Updated from Brython Server: 7/26/2018 6:39:24 PM<commit_after># gglogic boolean logic device simulations for ggmath

from ggmath import MathApp, _MathDynamic
from abc import ABCMeta, abstractmethod

class _BoolDevice(_MathDynamic, metaclass=ABCMeta):

    def __init__(self, mininputqty, *args, **kwargs):
        """
        Required Inputs

        * **mininputqty** minimum number of inputs possible

        Optional keyword arguments

        * **namedinputs** list of input names
        """
        self.In = [None]*mininputqty
        self.Enable = True
        namedinputs = kwargs.get('namedinputs', [])
        self._indict = {name:Eval(None) for name in namedinputs}

    @property
    def In(self):
        return self._input

    @In.setter
    def In(self, val):
        try:
            self._input = [self.Eval(v) for v in list(val)]
        except TypeError:
            self._input = [self.Eval(val)]

    # Enable attribute controls the "tri-state" of output
    @property
    def Enable(self):
        return self._enable

    @Enable.setter
    def Enable(self, val):
        self._enable = self.Eval(val)

    @abstractmethod
    def _getvalue(self):
        pass

    def __call__(self):
        if self.Enable:
            return self._getvalue()
        else:
            return None

    def __getattr__(self, name):
        return self._indict[name]

    def __setattr__(self, name, val):
        try:
            self._indict[name] = self.Eval(val)
        except AttributeError:
            super().__setattr__(name, val)

class _BoolOneInput(_BoolDevice):

    def __init__(self, *args, **kwargs):
        """
        No Required Inputs
        """
        super().__init__(1)

class _BoolMultiInput(_BoolDevice):

    def __init__(self, *args, **kwargs):
        """
        No Required Inputs
        """
        super().__init__(2)

class BoolNOT(_BoolOneInput):

    def _getvalue(self):
        inval = self.In[0]()
        if inval == None:
            return True  # equivalent to an "open" input
        else:
            return not inval

class BoolAND(_BoolMultiInput):

    def _getvalue(self):
        for v in self._input:
            if not v():
                return False
        return True

class TestDevice(_BoolOneInput):

    def __init__(self, *args, **kwargs):
        kwargs['namedinputs'] = ['in1', 'in2']
        super().__init__(*args, **kwargs)

    def out1(self):
        return self.in1 and self.in2

    def out2(self):
        return self.in1 or self.in2

# test code here
if __name__ == "__main__":

    from ggmath import GlassButton, LEDIndicator, MetalToggle

    IC1 = BoolNOT()
    IC2 = BoolAND()

    b1 = MetalToggle(1, (1,0))
    b2 = MetalToggle(1, (1,0.3))
    db1 = LEDIndicator((1.3,0), b1)
    db2 = LEDIndicator((1.3,0.3), b2)

    b3 = MetalToggle(1, (1,0.6))
    b4 = MetalToggle(1, (1,0.9))
    db1 = LEDIndicator((1.3,0.6), b3)
    db2 = LEDIndicator((1.3,0.9), b4)

    d2 = LEDIndicator((1.5,0.45), IC2)

    IC2.In = b1, b2
    IC2.In = IC2.In + [b3, b4]

    button = GlassButton(None, (0,0))
    LED = LEDIndicator((0,-1), IC1)
    IC1.In = button

    t1 = MetalToggle(1, (1,-1))
    t2 = MetalToggle(1, (1, -1.3))
    td = TestDevice()
    dt1 = LEDIndicator((1.3, -1), t1)
    dt2 = LEDIndicator((1.3, -1.3), t2)
    dtd1 = LEDIndicator((1.5, -1), td.out1)
    dtd2 = LEDIndicator((1.5, -1.3), td.out2)

    app = MathApp()
    app.run()<|endoftext|>


<commit_before>import sqlite3
from os.path import join

from tenyksclient.client import Client, run_client
from tenyksclient.config import settings

class TenyksLeetPoints(Client):
    irc_message_filters = {
        'add_points': [r'give (?P<points>[0-9]*) points to (?P<user>.*)',
                       r'(?P<user>.*) \+(?P<points>[0-9]*)'],
        'remove_points': [r'remove (?P<points>[0-9]*) points from (?P<user>.*)',
                       r'(?P<user>.*) \-(?P<points>[0-9]*)'],
        'highscore': [r'highscore', r'scores', r'highscores', 'what are everyone\'s points?'],
    }
    direct_only = True

    def __init__(self, *args, **kwargs):
        super(TenyksLeetPoints, self).__init__(*args, **kwargs)
        self.create_tables(self.fetch_cursor())

    def fetch_cursor(self):
        db_file = '{name}.db'.format(name=self.name)
        conn = sqlite3.connect(join(settings.WORKING_DIR, db_file))
        return conn.cursor()

    def handle_add_points(self, data, match):
        match_dict = match.groupdict()

        if data['nick'] == match_dict['user']:
            self.send('{nick}: You cannot give points to yourself.'.format(
                nick=data['nick']), data)
            return

        if not self.conspirator_exists(self.fetch_cursor(), match_dict['user']):
            self.create_conspirator(self.fetch_cursor(), match_dict['user'])

        self.increment_points(self.fetch_cursor(), match_dict['user'],
            int(match_dict['points']))

        self.send('{nick}: Done.'.format(nick=data['nick']), data)

    def handle_remove_points(self, data, match):
        match_dict = match.groupdict()

        if data['nick'] == match_dict['user']:
            self.send('{nick}: You cannot remove points from yourself.'.format(
                nick=data['nick']), data)
            return

        if not self.conspirator_exists(self.fetch_cursor(), match_dict['user']):
            self.create_conspirator(self.fetch_cursor(), match_dict['user'])

        self.decrement_points(self.fetch_cursor(), match_dict['user'],
            int(match_dict['points']))

        self.send('{nick}: Done.'.format(nick=data['nick']), data)

    def handle_highscore(self, data, match):
        self.send("Highscores: ", data)
        for person in self.fetch_conspirators(self.fetch_cursor()):
            self.send("{0}: {1}".format(person[1], person[2]), data)

    def increment_points(self, cur, nick, points):
        result = cur.execute("""
            UPDATE points SET points = points.points + ?
            WHERE person = ?
        """, (points, nick))
        result.connection.commit()

    def decrement_points(self, cur, nick, points):
        result = cur.execute("""
            UPDATE points SET points = points.points - ?
            WHERE person = ?
        """, (points, nick))
        result.connection.commit()

    def fetch_conspirators(self, cur):
        result = cur.execute("""
            SELECT * FROM points
            ORDER BY points DESC;
        """)
        return result.fetchall()

    def conspirator_exists(self, cur, person_name):
        result = cur.execute("""
            SELECT * FROM points
            WHERE person = ?
        """, (person_name,))
        return result.fetchone() is not None

    def create_conspirator(self, cur, name, points=None):
        result = cur.execute("""
            INSERT INTO points (person, points)
            VALUES (?, ?)
        """, (name, points or 0))
        result.connection.commit()

    def create_tables(self, cur):
        table_sql = """
        CREATE TABLE IF NOT EXISTS points (
            id INTEGER PRIMARY KEY,
            person TEXT,
            points INTEGER
        );
        """
        cur.executescript(table_sql)

def main():
    run_client(TenyksLeetPoints)

if __name__ == '__main__':
    main()
<commit_msg>Fixed the terrible regular expressions<commit_after>import sqlite3
from os.path import join

from tenyksclient.client import Client, run_client
from tenyksclient.config import settings

class TenyksLeetPoints(Client):
    irc_message_filters = {
        'add_points': [r'^give (?P<points>[0-9]*) points to (?P<nick>[a-zA-Z0-9-_`]*)$',
                       r'^\+(?P<points>[0-9]*) to (?P<nick>[a-zA-Z0-9-_`]*)$'],
        'remove_points': [r'^remove (?P<points>[0-9]*) points from (?P<nick>[a-zA-Z0-9-_`]*)$',
                          r'^\-(?P<points>[0-9]*) to (?P<nick>[a-zA-Z0-9-_`]*)$'],
        'highscore': [r'highscore', r'scores', r'highscores', 'what are everyone\'s points?'],
    }
    direct_only = True

    def __init__(self, *args, **kwargs):
        super(TenyksLeetPoints, self).__init__(*args, **kwargs)
        self.create_tables(self.fetch_cursor())

    def fetch_cursor(self):
        db_file = '{name}.db'.format(name=self.name)
        conn = sqlite3.connect(join(settings.WORKING_DIR, db_file))
        return conn.cursor()

    def handle_add_points(self, data, match):
        match_dict = match.groupdict()

        if data['nick'] == match_dict['nick']:
            self.send('{nick}: You cannot give points to yourself.'.format(
                nick=data['nick']), data)
            return

        if not self.conspirator_exists(self.fetch_cursor(), match_dict['nick']):
            self.create_conspirator(self.fetch_cursor(), match_dict['nick'])

        self.increment_points(self.fetch_cursor(), match_dict['nick'],
            int(match_dict['points']))

        self.send('{nick}: Done.'.format(nick=data['nick']), data)

    def handle_remove_points(self, data, match):
        match_dict = match.groupdict()

        if data['nick'] == match_dict['nick']:
            self.send('{nick}: You cannot remove points from yourself.'.format(
                nick=data['nick']), data)
            return

        if not self.conspirator_exists(self.fetch_cursor(), match_dict['nick']):
            self.create_conspirator(self.fetch_cursor(), match_dict['nick'])

        self.decrement_points(self.fetch_cursor(), match_dict['nick'],
            int(match_dict['points']))

        self.send('{nick}: Done.'.format(nick=data['nick']), data)

    def handle_highscore(self, data, match):
        self.send("Highscores: ", data)
        for person in self.fetch_conspirators(self.fetch_cursor()):
            self.send("{0}: {1}".format(person[1], person[2]), data)

    def increment_points(self, cur, nick, points):
        result = cur.execute("""
            UPDATE points SET points = points.points + ?
            WHERE person = ?
        """, (points, nick))
        result.connection.commit()

    def decrement_points(self, cur, nick, points):
        result = cur.execute("""
            UPDATE points SET points = points.points - ?
            WHERE person = ?
        """, (points, nick))
        result.connection.commit()

    def fetch_conspirators(self, cur):
        result = cur.execute("""
            SELECT * FROM points
            ORDER BY points DESC;
        """)
        return result.fetchall()

    def conspirator_exists(self, cur, person_name):
        result = cur.execute("""
            SELECT * FROM points
            WHERE person = ?
        """, (person_name,))
        return result.fetchone() is not None

    def create_conspirator(self, cur, name, points=None):
        result = cur.execute("""
            INSERT INTO points (person, points)
            VALUES (?, ?)
        """, (name, points or 0))
        result.connection.commit()

    def create_tables(self, cur):
        table_sql = """
        CREATE TABLE IF NOT EXISTS points (
            id INTEGER PRIMARY KEY,
            person TEXT,
            points INTEGER
        );
        """
        cur.executescript(table_sql)

def main():
    run_client(TenyksLeetPoints)

if __name__ == '__main__':
    main()
<|endoftext|>


<commit_before>"""
=======================================
Visualizing the stock market structure
=======================================

This example employs several unsupervised learning techniques to extract
the stock market structure from variations in historical quotes.

The quantity that we use is the daily variation in quote price: quotes
that are linked tend to cofluctuate during a day.

.. _stock_market:

Learning a graph structure
--------------------------

We use sparse inverse covariance estimation to find which quotes are
correlated conditionally on the others. Specifically, sparse inverse
covariance gives us a graph, that is a list of connection. For each
symbol, the symbols that it is connected too are those useful to explain
its fluctuations.

Clustering
----------

We use clustering to group together quotes that behave similarly. Here,
amongst the :ref:`various clustering techniques <clustering>` available
in the scikit-learn, we use :ref:`affinity_propagation` as it does
not enforce equal-size clusters, and it can choose automatically the
number of clusters from the data.

Note that this gives us a different indication than the graph, as the
graph reflects conditional relations between variables, while the
clustering reflects marginal properties: variables clustered together can
be considered as having a similar impact at the level of the full stock
market.

Embedding in 2D space
---------------------

For visualization purposes, we need to lay out the different symbols on a
2D canvas. For this we use :ref:`manifold` techniques to retrieve 2D
embedding.

Visualization
-------------

The output of the 3 models are combined in a 2D graph where nodes
represents the stocks and edges the:

- cluster labels are used to define the color of the nodes
- the sparse covariance model is used to display the strength of the edges
- the 2D embedding is used to position the nodes in the plan

This example has a fair amount of visualization-related code, as
visualization is crucial here to display the graph. One of the challenge
is to position the labels minimizing overlap. For this we use an
heuristic based on the direction of the nearest neighbor along each
axis.
"""
print(__doc__)

# Author: Gael Varoquaux gael.varoquaux@normalesup.org
# License: BSD 3 clause

from datetime import datetime

import numpy as np
import matplotlib.pyplot as plt
from matplotlib.collections import LineCollection
from six.moves.urllib.request import urlopen
from six.moves.urllib.parse import urlencode
from sklearn import cluster, covariance, manifold

# #############################################################################
# Retrieve the data from Internet

def retry(f, n_attempts=3):
    "Wrapper function to retry function calls in case of exceptions"
    def wrapper(*args, **kwargs):
        for i in range(n_attempts):
            try:
                return f(*args, **kwargs)
            except Exception as e:
                if i == n_attempts - 1:
                    raise
    return wrapper

def quotes_historical_google(symbol, date1, date2):
    """Get the historical data from Google finance.

    Parameters
    ----------
    symbol : str
        Ticker symbol to query for, for example ``"DELL"``.
    date1 : datetime.datetime
        Start date.
    date2 : datetime.datetime
        End date.

    Returns
    -------
    X : array
        The columns are ``date`` -- datetime, ``open``, ``high``,
        ``low``, ``close`` and ``volume`` of type float.
    """
    params = urlencode({
        'q': symbol,
        'startdate': date1.strftime('%b %d, %Y'),
        'enddate': date2.strftime('%b %d, %Y'),
        'output': 'csv'
    })
    url = 'http://www.google.com/finance/historical?' + params
    response = urlopen(url)
    dtype = {
        'names': ['date', 'open', 'high', 'low', 'close', 'volume'],
        'formats': ['object', 'f4', 'f4', 'f4', 'f4', 'f4']
    }
    converters = {0: lambda s: datetime.strptime(s.decode(), '%d-%b-%y')}
    return np.genfromtxt(response, delimiter=',', skip_header=1,
                         dtype=dtype, converters=converters,
                         missing_values='-', filling_values=-1)

# Choose a time period reasonably calm (not too long ago so that we get
# high-tech firms, and before the 2008 crash)
d1 = datetime(2003, 1, 1)
d2 = datetime(2008, 1, 1)

symbol_dict = {
    'TOT': 'Total',
    'XOM': 'Exxon',
    'CVX': 'Chevron',
    'COP': 'ConocoPhillips',
    'VLO': 'Valero Energy',
    'MSFT': 'Microsoft',
    'IBM': 'IBM',
    'TWX': 'Time Warner',
    'CMCSA': 'Comcast',
    'CVC': 'Cablevision',
    'YHOO': 'Yahoo',
    'DELL': 'Dell',
    'HPQ': 'HP',
    'AMZN': 'Amazon',
    'TM': 'Toyota',
    'CAJ': 'Canon',
    'SNE': 'Sony',
    'F': 'Ford',
    'HMC': 'Honda',
    'NAV': 'Navistar',
    'NOC': 'Northrop Grumman',
    'BA': 'Boeing',
    'KO': 'Coca Cola',
    'MMM': '3M',
    'MCD': 'McDonald\'s',
    'PEP': 'Pepsi',
    'K': 'Kellogg',
    'UN': 'Unilever',
    'MAR': 'Marriott',
    'PG': 'Procter Gamble',
    'CL': 'Colgate-Palmolive',
    'GE': 'General Electrics',
    'WFC': 'Wells Fargo',
    'JPM': 'JPMorgan Chase',
    'AIG': 'AIG',
    'AXP': 'American express',
    'BAC': 'Bank of America',
    'GS': 'Goldman Sachs',
    'AAPL': 'Apple',
    'SAP': 'SAP',
    'CSCO': 'Cisco',
    'TXN': 'Texas Instruments',
    'XRX': 'Xerox',
    'WMT': 'Wal-Mart',
    'HD': 'Home Depot',
    'GSK': 'GlaxoSmithKline',
    'PFE': 'Pfizer',
    'SNY': 'Sanofi-Aventis',
    'NVS': 'Novartis',
    'KMB': 'Kimberly-Clark',
    'R': 'Ryder',
    'GD': 'General Dynamics',
    'RTN': 'Raytheon',
    'CVS': 'CVS',
    'CAT': 'Caterpillar',
    'DD': 'DuPont de Nemours'}

symbols, names = np.array(list(symbol_dict.items())).T

# retry is used because quotes_historical_google can temporarily fail
# for various reasons (e.g. empty result from Google API).
quotes = [
    retry(quotes_historical_google)(symbol, d1, d2) for symbol in symbols
]

close_prices = np.vstack([q['close'] for q in quotes])
open_prices = np.vstack([q['open'] for q in quotes])

# The daily variations of the quotes are what carry most information
variation = close_prices - open_prices

# #############################################################################
# Learn a graphical structure from the correlations
edge_model = covariance.GraphLassoCV()

# standardize the time series: using correlations rather than covariance
# is more efficient for structure recovery
X = variation.copy().T
X /= X.std(axis=0)
edge_model.fit(X)

# #############################################################################
# Cluster using affinity propagation

_, labels = cluster.affinity_propagation(edge_model.covariance_)
n_labels = labels.max()

for i in range(n_labels + 1):
    print('Cluster %i: %s' % ((i + 1), ', '.join(names[labels == i])))

# #############################################################################
# Find a low-dimension embedding for visualization: find the best position of
# the nodes (the stocks) on a 2D plane

# We use a dense eigen_solver to achieve reproducibility (arpack is
# initiated with random vectors that we don't control). In addition, we
# use a large number of neighbors to capture the large-scale structure.
node_position_model = manifold.LocallyLinearEmbedding(
    n_components=2, eigen_solver='dense', n_neighbors=6)

embedding = node_position_model.fit_transform(X.T).T

# #############################################################################
# Visualization
plt.figure(1, facecolor='w', figsize=(10, 8))
plt.clf()
ax = plt.axes([0., 0., 1., 1.])
plt.axis('off')

# Display a graph of the partial correlations
partial_correlations = edge_model.precision_.copy()
d = 1 / np.sqrt(np.diag(partial_correlations))
partial_correlations *= d
partial_correlations *= d[:, np.newaxis]
non_zero = (np.abs(np.triu(partial_correlations, k=1)) > 0.02)

# Plot the nodes using the coordinates of our embedding
plt.scatter(embedding[0], embedding[1], s=100 * d ** 2, c=labels,
            cmap=plt.cm.spectral)

# Plot the edges
start_idx, end_idx = np.where(non_zero)
# a sequence of (*line0*, *line1*, *line2*), where::
#            linen = (x0, y0), (x1, y1), ... (xm, ym)
segments = [[embedding[:, start], embedding[:, stop]]
            for start, stop in zip(start_idx, end_idx)]
values = np.abs(partial_correlations[non_zero])
lc = LineCollection(segments,
                    zorder=0, cmap=plt.cm.hot_r,
                    norm=plt.Normalize(0, .7 * values.max()))
lc.set_array(values)
lc.set_linewidths(15 * values)
ax.add_collection(lc)

# Add a label to each node. The challenge here is that we want to
# position the labels to avoid overlap with other labels
for index, (name, label, (x, y)) in enumerate(
        zip(names, labels, embedding.T)):

    dx = x - embedding[0]
    dx[index] = 1
    dy = y - embedding[1]
    dy[index] = 1
    this_dx = dx[np.argmin(np.abs(dy))]
    this_dy = dy[np.argmin(np.abs(dx))]
    if this_dx > 0:
        horizontalalignment = 'left'
        x = x + .002
    else:
        horizontalalignment = 'right'
        x = x - .002
    if this_dy > 0:
        verticalalignment = 'bottom'
        y = y + .002
    else:
        verticalalignment = 'top'
        y = y - .002
    plt.text(x, y, name, size=10,
             horizontalalignment=horizontalalignment,
             verticalalignment=verticalalignment,
             bbox=dict(facecolor='w',
                       edgecolor=plt.cm.spectral(label / float(n_labels)),
                       alpha=.6))

plt.xlim(embedding[0].min() - .15 * embedding[0].ptp(),
         embedding[0].max() + .10 * embedding[0].ptp(),)
plt.ylim(embedding[1].min() - .03 * embedding[1].ptp(),
         embedding[1].max() + .03 * embedding[1].ptp())

plt.show()
<commit_msg>More verbose output in plot_stock_market for debugging<commit_after>"""
=======================================
Visualizing the stock market structure
=======================================

This example employs several unsupervised learning techniques to extract
the stock market structure from variations in historical quotes.

The quantity that we use is the daily variation in quote price: quotes
that are linked tend to cofluctuate during a day.

.. _stock_market:

Learning a graph structure
--------------------------

We use sparse inverse covariance estimation to find which quotes are
correlated conditionally on the others. Specifically, sparse inverse
covariance gives us a graph, that is a list of connection. For each
symbol, the symbols that it is connected too are those useful to explain
its fluctuations.

Clustering
----------

We use clustering to group together quotes that behave similarly. Here,
amongst the :ref:`various clustering techniques <clustering>` available
in the scikit-learn, we use :ref:`affinity_propagation` as it does
not enforce equal-size clusters, and it can choose automatically the
number of clusters from the data.

Note that this gives us a different indication than the graph, as the
graph reflects conditional relations between variables, while the
clustering reflects marginal properties: variables clustered together can
be considered as having a similar impact at the level of the full stock
market.

Embedding in 2D space
---------------------

For visualization purposes, we need to lay out the different symbols on a
2D canvas. For this we use :ref:`manifold` techniques to retrieve 2D
embedding.

Visualization
-------------

The output of the 3 models are combined in a 2D graph where nodes
represents the stocks and edges the:

- cluster labels are used to define the color of the nodes
- the sparse covariance model is used to display the strength of the edges
- the 2D embedding is used to position the nodes in the plan

This example has a fair amount of visualization-related code, as
visualization is crucial here to display the graph. One of the challenge
is to position the labels minimizing overlap. For this we use an
heuristic based on the direction of the nearest neighbor along each
axis.
"""
from __future__ import print_function

# Author: Gael Varoquaux gael.varoquaux@normalesup.org
# License: BSD 3 clause

import sys
from datetime import datetime

import numpy as np
import matplotlib.pyplot as plt
from matplotlib.collections import LineCollection
from six.moves.urllib.request import urlopen
from six.moves.urllib.parse import urlencode
from sklearn import cluster, covariance, manifold

print(__doc__)

# #############################################################################
# Retrieve the data from Internet

def retry(f, n_attempts=3):
    "Wrapper function to retry function calls in case of exceptions"
    def wrapper(*args, **kwargs):
        for i in range(n_attempts):
            try:
                return f(*args, **kwargs)
            except Exception as e:
                if i == n_attempts - 1:
                    raise
    return wrapper

def quotes_historical_google(symbol, date1, date2):
    """Get the historical data from Google finance.

    Parameters
    ----------
    symbol : str
        Ticker symbol to query for, for example ``"DELL"``.
    date1 : datetime.datetime
        Start date.
    date2 : datetime.datetime
        End date.

    Returns
    -------
    X : array
        The columns are ``date`` -- datetime, ``open``, ``high``,
        ``low``, ``close`` and ``volume`` of type float.
    """
    params = urlencode({
        'q': symbol,
        'startdate': date1.strftime('%b %d, %Y'),
        'enddate': date2.strftime('%b %d, %Y'),
        'output': 'csv'
    })
    url = 'http://www.google.com/finance/historical?' + params
    response = urlopen(url)
    dtype = {
        'names': ['date', 'open', 'high', 'low', 'close', 'volume'],
        'formats': ['object', 'f4', 'f4', 'f4', 'f4', 'f4']
    }
    converters = {0: lambda s: datetime.strptime(s.decode(), '%d-%b-%y')}
    return np.genfromtxt(response, delimiter=',', skip_header=1,
                         dtype=dtype, converters=converters,
                         missing_values='-', filling_values=-1)

# Choose a time period reasonably calm (not too long ago so that we get
# high-tech firms, and before the 2008 crash)
d1 = datetime(2003, 1, 1)
d2 = datetime(2008, 1, 1)

symbol_dict = {
    'TOT': 'Total',
    'XOM': 'Exxon',
    'CVX': 'Chevron',
    'COP': 'ConocoPhillips',
    'VLO': 'Valero Energy',
    'MSFT': 'Microsoft',
    'IBM': 'IBM',
    'TWX': 'Time Warner',
    'CMCSA': 'Comcast',
    'CVC': 'Cablevision',
    'YHOO': 'Yahoo',
    'DELL': 'Dell',
    'HPQ': 'HP',
    'AMZN': 'Amazon',
    'TM': 'Toyota',
    'CAJ': 'Canon',
    'SNE': 'Sony',
    'F': 'Ford',
    'HMC': 'Honda',
    'NAV': 'Navistar',
    'NOC': 'Northrop Grumman',
    'BA': 'Boeing',
    'KO': 'Coca Cola',
    'MMM': '3M',
    'MCD': 'McDonald\'s',
    'PEP': 'Pepsi',
    'K': 'Kellogg',
    'UN': 'Unilever',
    'MAR': 'Marriott',
    'PG': 'Procter Gamble',
    'CL': 'Colgate-Palmolive',
    'GE': 'General Electrics',
    'WFC': 'Wells Fargo',
    'JPM': 'JPMorgan Chase',
    'AIG': 'AIG',
    'AXP': 'American express',
    'BAC': 'Bank of America',
    'GS': 'Goldman Sachs',
    'AAPL': 'Apple',
    'NYSE:SAP': 'SAP',
    'CSCO': 'Cisco',
    'TXN': 'Texas Instruments',
    'XRX': 'Xerox',
    'WMT': 'Wal-Mart',
    'HD': 'Home Depot',
    'GSK': 'GlaxoSmithKline',
    'PFE': 'Pfizer',
    'SNY': 'Sanofi-Aventis',
    'NVS': 'Novartis',
    'KMB': 'Kimberly-Clark',
    'R': 'Ryder',
    'GD': 'General Dynamics',
    'RTN': 'Raytheon',
    'CVS': 'CVS',
    'CAT': 'Caterpillar',
    'DD': 'DuPont de Nemours'}

symbols, names = np.array(list(symbol_dict.items())).T

# retry is used because quotes_historical_google can temporarily fail
# for various reasons (e.g. empty result from Google API).
quotes = []

for symbol in sorted(symbols):
    print('Fetching quote history for %r' % symbol, file=sys.stderr)
    quotes.append(retry(quotes_historical_google)(symbol, d1, d2))

close_prices = np.vstack([q['close'] for q in quotes])
open_prices = np.vstack([q['open'] for q in quotes])

# The daily variations of the quotes are what carry most information
variation = close_prices - open_prices

# #############################################################################
# Learn a graphical structure from the correlations
edge_model = covariance.GraphLassoCV()

# standardize the time series: using correlations rather than covariance
# is more efficient for structure recovery
X = variation.copy().T
X /= X.std(axis=0)
edge_model.fit(X)

# #############################################################################
# Cluster using affinity propagation

_, labels = cluster.affinity_propagation(edge_model.covariance_)
n_labels = labels.max()

for i in range(n_labels + 1):
    print('Cluster %i: %s' % ((i + 1), ', '.join(names[labels == i])))

# #############################################################################
# Find a low-dimension embedding for visualization: find the best position of
# the nodes (the stocks) on a 2D plane

# We use a dense eigen_solver to achieve reproducibility (arpack is
# initiated with random vectors that we don't control). In addition, we
# use a large number of neighbors to capture the large-scale structure.
node_position_model = manifold.LocallyLinearEmbedding(
    n_components=2, eigen_solver='dense', n_neighbors=6)

embedding = node_position_model.fit_transform(X.T).T

# #############################################################################
# Visualization
plt.figure(1, facecolor='w', figsize=(10, 8))
plt.clf()
ax = plt.axes([0., 0., 1., 1.])
plt.axis('off')

# Display a graph of the partial correlations
partial_correlations = edge_model.precision_.copy()
d = 1 / np.sqrt(np.diag(partial_correlations))
partial_correlations *= d
partial_correlations *= d[:, np.newaxis]
non_zero = (np.abs(np.triu(partial_correlations, k=1)) > 0.02)

# Plot the nodes using the coordinates of our embedding
plt.scatter(embedding[0], embedding[1], s=100 * d ** 2, c=labels,
            cmap=plt.cm.spectral)

# Plot the edges
start_idx, end_idx = np.where(non_zero)
# a sequence of (*line0*, *line1*, *line2*), where::
#            linen = (x0, y0), (x1, y1), ... (xm, ym)
segments = [[embedding[:, start], embedding[:, stop]]
            for start, stop in zip(start_idx, end_idx)]
values = np.abs(partial_correlations[non_zero])
lc = LineCollection(segments,
                    zorder=0, cmap=plt.cm.hot_r,
                    norm=plt.Normalize(0, .7 * values.max()))
lc.set_array(values)
lc.set_linewidths(15 * values)
ax.add_collection(lc)

# Add a label to each node. The challenge here is that we want to
# position the labels to avoid overlap with other labels
for index, (name, label, (x, y)) in enumerate(
        zip(names, labels, embedding.T)):

    dx = x - embedding[0]
    dx[index] = 1
    dy = y - embedding[1]
    dy[index] = 1
    this_dx = dx[np.argmin(np.abs(dy))]
    this_dy = dy[np.argmin(np.abs(dx))]
    if this_dx > 0:
        horizontalalignment = 'left'
        x = x + .002
    else:
        horizontalalignment = 'right'
        x = x - .002
    if this_dy > 0:
        verticalalignment = 'bottom'
        y = y + .002
    else:
        verticalalignment = 'top'
        y = y - .002
    plt.text(x, y, name, size=10,
             horizontalalignment=horizontalalignment,
             verticalalignment=verticalalignment,
             bbox=dict(facecolor='w',
                       edgecolor=plt.cm.spectral(label / float(n_labels)),
                       alpha=.6))

plt.xlim(embedding[0].min() - .15 * embedding[0].ptp(),
         embedding[0].max() + .10 * embedding[0].ptp(),)
plt.ylim(embedding[1].min() - .03 * embedding[1].ptp(),
         embedding[1].max() + .03 * embedding[1].ptp())

plt.show()
<|endoftext|>


<commit_before># Django settings for test_project project.

DEBUG = True
TEMPLATE_DEBUG = DEBUG

ADMINS = (
    # ('Your Name', 'your_email@example.com'),
)

MANAGERS = ADMINS

DATABASES = {
    'default': {
        'ENGINE': 'django.db.backends.postgresql_psycopg2', # Add 'postgresql_psycopg2', 'postgresql', 'mysql', 'sqlite3' or 'oracle'.
        'NAME': 'reversion',                      # Or path to database file if using sqlite3.
        'USER': 'postgres',                      # Not used with sqlite3.
        'PASSWORD': '',                  # Not used with sqlite3.
        'HOST': '',                      # Set to empty string for localhost. Not used with sqlite3.
        'PORT': '',                      # Set to empty string for default. Not used with sqlite3.
    }
}

# Local time zone for this installation. Choices can be found here:
# http://en.wikipedia.org/wiki/List_of_tz_zones_by_name
# although not all choices may be available on all operating systems.
# On Unix systems, a value of None will cause Django to use the same
# timezone as the operating system.
# If running in a Windows environment this must be set to the same as your
# system time zone.
TIME_ZONE = 'America/Chicago'

# Language code for this installation. All choices can be found here:
# http://www.i18nguy.com/unicode/language-identifiers.html
LANGUAGE_CODE = 'en-us'

SITE_ID = 1

# If you set this to False, Django will make some optimizations so as not
# to load the internationalization machinery.
USE_I18N = True

# If you set this to False, Django will not format dates, numbers and
# calendars according to the current locale
USE_L10N = True

# Absolute filesystem path to the directory that will hold user-uploaded files.
# Example: "/home/media/media.lawrence.com/media/"
MEDIA_ROOT = ''

# URL that handles the media served from MEDIA_ROOT. Make sure to use a
# trailing slash.
# Examples: "http://media.lawrence.com/media/", "http://example.com/media/"
MEDIA_URL = ''

# Absolute path to the directory static files should be collected to.
# Don't put anything in this directory yourself; store your static files
# in apps' "static/" subdirectories and in STATICFILES_DIRS.
# Example: "/home/media/media.lawrence.com/static/"
STATIC_ROOT = ''

# URL prefix for static files.
# Example: "http://media.lawrence.com/static/"
STATIC_URL = '/static/'

# URL prefix for admin static files -- CSS, JavaScript and images.
# Make sure to use a trailing slash.
# Examples: "http://foo.com/static/admin/", "/static/admin/".
ADMIN_MEDIA_PREFIX = '/static/admin/'

# Additional locations of static files
STATICFILES_DIRS = (
    # Put strings here, like "/home/html/static" or "C:/www/django/static".
    # Always use forward slashes, even on Windows.
    # Don't forget to use absolute paths, not relative paths.
)

# List of finder classes that know how to find static files in
# various locations.
STATICFILES_FINDERS = (
    'django.contrib.staticfiles.finders.FileSystemFinder',
    'django.contrib.staticfiles.finders.AppDirectoriesFinder',
#    'django.contrib.staticfiles.finders.DefaultStorageFinder',
)

# Make this unique, and don't share it with anybody.
SECRET_KEY = 'f-jyx)z&m_9k##)2l_d@@w9c^2_^x1+gy*skc6mr+m+s$%mljp'

# List of callables that know how to import templates from various sources.
TEMPLATE_LOADERS = (
    'django.template.loaders.filesystem.Loader',
    'django.template.loaders.app_directories.Loader',
#     'django.template.loaders.eggs.Loader',
)

MIDDLEWARE_CLASSES = (
    'django.middleware.common.CommonMiddleware',
    'django.contrib.sessions.middleware.SessionMiddleware',
    'django.middleware.csrf.CsrfViewMiddleware',
    'django.contrib.auth.middleware.AuthenticationMiddleware',
    'django.contrib.messages.middleware.MessageMiddleware',
    "reversion.middleware.RevisionMiddleware",
)

ROOT_URLCONF = 'test_project.urls'

TEMPLATE_DIRS = (
    # Put strings here, like "/home/html/django_templates" or "C:/www/django/templates".
    # Always use forward slashes, even on Windows.
    # Don't forget to use absolute paths, not relative paths.
)

INSTALLED_APPS = (
    'django.contrib.auth',
    'django.contrib.contenttypes',
    'django.contrib.sessions',
    'django.contrib.sites',
    'django.contrib.messages',
    'django.contrib.staticfiles',
    # Uncomment the next line to enable the admin:
    'django.contrib.admin',
    # Uncomment the next line to enable admin documentation:
    'django.contrib.admindocs',
    'reversion',
    'test_project.test_app',
    'south',
)

# A sample logging configuration. The only tangible logging
# performed by this configuration is to send an email to
# the site admins on every HTTP 500 error.
# See http://docs.djangoproject.com/en/dev/topics/logging for
# more details on how to customize your logging configuration.
LOGGING = {
    'version': 1,
    'disable_existing_loggers': False,
    'handlers': {
        'mail_admins': {
            'level': 'ERROR',
            'class': 'django.utils.log.AdminEmailHandler'
        }
    },
    'loggers': {
        'django.request': {
            'handlers': ['mail_admins'],
            'level': 'ERROR',
            'propagate': True,
        },
    }
}
<commit_msg>Switching back to sqlite3 backend for testing.<commit_after># Django settings for test_project project.

DEBUG = True
TEMPLATE_DEBUG = DEBUG

ADMINS = (
    # ('Your Name', 'your_email@example.com'),
)

MANAGERS = ADMINS

DATABASES = {
    'default': {
        'ENGINE': 'django.db.backends.sqlite3', # Add 'postgresql_psycopg2', 'postgresql', 'mysql', 'sqlite3' or 'oracle'.
        'NAME': 'test.db',                      # Or path to database file if using sqlite3.
        'USER': 'postgres',                      # Not used with sqlite3.
        'PASSWORD': '',                  # Not used with sqlite3.
        'HOST': '',                      # Set to empty string for localhost. Not used with sqlite3.
        'PORT': '',                      # Set to empty string for default. Not used with sqlite3.
    }
}

# Local time zone for this installation. Choices can be found here:
# http://en.wikipedia.org/wiki/List_of_tz_zones_by_name
# although not all choices may be available on all operating systems.
# On Unix systems, a value of None will cause Django to use the same
# timezone as the operating system.
# If running in a Windows environment this must be set to the same as your
# system time zone.
TIME_ZONE = 'America/Chicago'

# Language code for this installation. All choices can be found here:
# http://www.i18nguy.com/unicode/language-identifiers.html
LANGUAGE_CODE = 'en-us'

SITE_ID = 1

# If you set this to False, Django will make some optimizations so as not
# to load the internationalization machinery.
USE_I18N = True

# If you set this to False, Django will not format dates, numbers and
# calendars according to the current locale
USE_L10N = True

# Absolute filesystem path to the directory that will hold user-uploaded files.
# Example: "/home/media/media.lawrence.com/media/"
MEDIA_ROOT = ''

# URL that handles the media served from MEDIA_ROOT. Make sure to use a
# trailing slash.
# Examples: "http://media.lawrence.com/media/", "http://example.com/media/"
MEDIA_URL = ''

# Absolute path to the directory static files should be collected to.
# Don't put anything in this directory yourself; store your static files
# in apps' "static/" subdirectories and in STATICFILES_DIRS.
# Example: "/home/media/media.lawrence.com/static/"
STATIC_ROOT = ''

# URL prefix for static files.
# Example: "http://media.lawrence.com/static/"
STATIC_URL = '/static/'

# URL prefix for admin static files -- CSS, JavaScript and images.
# Make sure to use a trailing slash.
# Examples: "http://foo.com/static/admin/", "/static/admin/".
ADMIN_MEDIA_PREFIX = '/static/admin/'

# Additional locations of static files
STATICFILES_DIRS = (
    # Put strings here, like "/home/html/static" or "C:/www/django/static".
    # Always use forward slashes, even on Windows.
    # Don't forget to use absolute paths, not relative paths.
)

# List of finder classes that know how to find static files in
# various locations.
STATICFILES_FINDERS = (
    'django.contrib.staticfiles.finders.FileSystemFinder',
    'django.contrib.staticfiles.finders.AppDirectoriesFinder',
#    'django.contrib.staticfiles.finders.DefaultStorageFinder',
)

# Make this unique, and don't share it with anybody.
SECRET_KEY = 'f-jyx)z&m_9k##)2l_d@@w9c^2_^x1+gy*skc6mr+m+s$%mljp'

# List of callables that know how to import templates from various sources.
TEMPLATE_LOADERS = (
    'django.template.loaders.filesystem.Loader',
    'django.template.loaders.app_directories.Loader',
#     'django.template.loaders.eggs.Loader',
)

MIDDLEWARE_CLASSES = (
    'django.middleware.common.CommonMiddleware',
    'django.contrib.sessions.middleware.SessionMiddleware',
    'django.middleware.csrf.CsrfViewMiddleware',
    'django.contrib.auth.middleware.AuthenticationMiddleware',
    'django.contrib.messages.middleware.MessageMiddleware',
    "reversion.middleware.RevisionMiddleware",
)

ROOT_URLCONF = 'test_project.urls'

TEMPLATE_DIRS = (
    # Put strings here, like "/home/html/django_templates" or "C:/www/django/templates".
    # Always use forward slashes, even on Windows.
    # Don't forget to use absolute paths, not relative paths.
)

INSTALLED_APPS = (
    'django.contrib.auth',
    'django.contrib.contenttypes',
    'django.contrib.sessions',
    'django.contrib.sites',
    'django.contrib.messages',
    'django.contrib.staticfiles',
    # Uncomment the next line to enable the admin:
    'django.contrib.admin',
    # Uncomment the next line to enable admin documentation:
    'django.contrib.admindocs',
    'reversion',
    'test_project.test_app',
    'south',
)

# A sample logging configuration. The only tangible logging
# performed by this configuration is to send an email to
# the site admins on every HTTP 500 error.
# See http://docs.djangoproject.com/en/dev/topics/logging for
# more details on how to customize your logging configuration.
LOGGING = {
    'version': 1,
    'disable_existing_loggers': False,
    'handlers': {
        'mail_admins': {
            'level': 'ERROR',
            'class': 'django.utils.log.AdminEmailHandler'
        }
    },
    'loggers': {
        'django.request': {
            'handlers': ['mail_admins'],
            'level': 'ERROR',
            'propagate': True,
        },
    }
}
<|endoftext|>


<commit_before>import string
import sys

from state import *

def getStateName(line):
    colonLoc = string.find(line, ":")

    stateName = line[:colonLoc]

    return stateName

if __name__ == "__main__":
    sttm = SingleTapeTuringMachine(sys.argv[-1], ["_", "1", "H", "E"])
    args = sys.argv[1:-1]

    quiet = ("-q" in args)

    numSteps = float("Inf") # default value
    if ("-s" in args):
        numSteps = args[args.index("-s") + 1]

    output = None
    if ("-f" in args):
        output = open(args[args.index("-f") + 1], "w")

    sttm.run(quiet, numSteps, output)

def parseMachine(path, alphabet):
    with open(path, "r") as inp:
        tmLines = inp.readlines()

    stateDictionary = {"ACCEPT": SimpleState("ACCEPT", alphabet),
        "REJECT": SimpleState("REJECT", alphabet),
        "ERROR": SimpleState("ERROR", alphabet),
        "HALT": SimpleState("HALT", alphabet),
        "OUT": SimpleState("OUT", alphabet)}

    listOfRealStates = []

    # initialize state dictionary
    for line in tmLines[1:]:
        if line != "\n": # not a blank line
            lineSplit = string.split(line)

            if lineSplit[0] == "START":
                stateName = getStateName(line[6:])
                startState = State(stateName, None, alphabet)
                stateDictionary[stateName] = startState
                listOfRealStates.append(stateDictionary[stateName])
                startState.makeStartState()

            elif not lineSplit[0] in alphabet:
                stateName = getStateName(line)
                stateDictionary[stateName] = State(stateName, None, alphabet)
                listOfRealStates.append(stateDictionary[stateName])

    currentStateBeingModified = None

    # fill in state dictionary
    for line in tmLines[1:]:
        if line != "\n":
            lineSplit = string.split(line)

            if lineSplit[0] == "START":
                stateName = getStateName(line[6:])
                currentStateBeingModified = stateDictionary[stateName]

            elif not lineSplit[0] in alphabet:
                stateName = getStateName(line)
                currentStateBeingModified = stateDictionary[stateName]

            else:
                symbol = lineSplit[0]
                stateName = lineSplit[2][:-1]
                headMove = lineSplit[3][:-1]
                write = lineSplit[4]

                currentStateBeingModified.setNextState(symbol,
                    stateDictionary[stateName])
                currentStateBeingModified.setHeadMove(symbol, headMove)
                currentStateBeingModified.setWrite(symbol, write)

    return startState, stateDictionary

def stateDictionariesToLists(stateDictionary, alphabet, startState):
    simulationStates = {}
    for state in stateDictionary.itervalues():
        if state.isSimpleState():
            newState = state
        else:
            newState = SimulationState()
        simulationStates[state] = newState
    for state in stateDictionary.itervalues():
        if not state.isSimpleState():
            simulationStates[state]._initFromState(state, simulationStates)
    return simulationStates[startState]

class SingleTapeTuringMachine:
    def __init__(self, path, alphabet=["_", "1", "H", "E"]):
        self.state = None
        self.tape = Tape(None, alphabet[0])

        startState, stateDictionary = parseMachine(
                path, alphabet)
        startState = stateDictionariesToLists(
                stateDictionary, alphabet, startState)
        self.startState = startState

    def run(self, quiet=False, numSteps=float("Inf"), output=None):

        self.state = self.startState

        stepCounter = 0
        halted = False
        numSteps = float(numSteps)

        while stepCounter < numSteps:
            if not quiet:
                self.printTape(-2, 340, output)

            stepCounter += 1

            if self.state.isSimpleState():
                if self.state.stateName == "ERROR":
                    print "Turing machine threw error!"
                    halted = True
                    break

                if self.state.stateName == "ACCEPT":
                    print "Turing machine accepted after", stepCounter, "steps."
                    print len(self.tape.tapeDict), "squares of memory were used."
                    halted = True
                    break

                if self.state.stateName == "REJECT":
                    print "Turing machine rejected after", stepCounter, "steps."
                    print len(self.tape.tapeDict), "squares of memory were used."
                    halted = True
                    break

                if self.state.stateName == "HALT":
                    print "Turing machine halted after", stepCounter, "steps."
                    print len(self.tape.tapeDict), "squares of memory were used."
                    halted = True
                    break

                if self.state.stateName == "OUT":
                    print "Turing machine execution incomplete: reached out state."
                    print "Perhaps this Turing machine wants to be melded with another machine."

            symbolord = self.tape.readSymbolOrd()
            headmove = self.state.getHeadMove(symbolord)

            self.tape.writeSymbolOrd(self.state.getWrite(symbolord))
            self.tape.moveHead(headmove)
            self.state = self.state.getNextState(symbolord)

        if not halted:
            print "Turing machine ran for", numSteps, "steps without halting."

    def printTape(self, start, end, output):
        if output == None:

            print self.state.stateName

            self.tape.printTape(start, end)
#           print "--------------------------------------"
        else:
            output.write(self.state.stateName + "\n")

            self.tape.printTape(start, end, output)
#           output.write("--------------------------------------\n")

class SimulationState(object):
    def __init__(self):
        self.nextState = [None] * 256
        self.headMove = [""] * 256
        self.write = [0] * 256

    def _initFromState(self, realState, simulationStates):
        self.stateName = realState.stateName
        self.description = realState.description
        self.alphabet = realState.alphabet
        self.isStartState = realState.isStartState
        for symbol in self.alphabet:
            self.headMove[ord(symbol)] = realState.headMoveDict[symbol]
            self.write[ord(symbol)] = ord(realState.writeDict[symbol])
            self.nextState[ord(symbol)] = simulationStates[
                    realState.nextStateDict[symbol]]

    def getNextState(self, ordsymbol):
        return self.nextState[ordsymbol]

    def getHeadMove(self, ordsymbol):
        return self.headMove[ordsymbol]

    def getWrite(self, ordsymbol):
        return self.write[ordsymbol]

    def isSimpleState(self):
        return False

class Tape(object):
    # By convention the first symbol in the alphabet is the initial symbol
    def __init__(self, name, initSymbol):
        self.name = name
        self.headLoc = 0
        self.initSymbol = initSymbol
        self.initSymbolOrd = ord(initSymbol)
        self.tapePos = [self.initSymbolOrd]
        self.tapeNeg = []

    def readSymbol(self):
        return self._readSymbol(self.headLoc)

    def readSymbolOrd(self):
        return self._readSymbolOrd(self.headLoc)

    def _readSymbol(self, pos):
        return chr(self._readSymbolOrd(pos))

    def _readSymbolOrd(self, pos):
        try:
            if pos >= 0:
                return self.tapePos[pos]
            else:
                return self.tapeNeg[~pos]
        except IndexError:
            return self.initSymbolOrd

    def writeSymbol(self, symbol):
        return self.writeSymbolOrd(ord(symbol))

    def writeSymbolOrd(self, ordsymbol):
        if self.headLoc >= 0:
            self.tapePos[self.headLoc] = ordsymbol
        else:
            self.tapeNeg[~self.headLoc] = ordsymbol

    def moveHead(self, direction):
        if direction == "L":
            self.headLoc -= 1
            self.continueTape()

        elif direction == "R":
            self.headLoc += 1
            self.continueTape()

        elif direction == "-":
            pass
        else:
            print "bad head move", headmove
            raise

    def continueTape(self):
        if self.headLoc >= 0:
            assert 0 <= self.headLoc <= len(self.tapePos)
            if self.headLoc == len(self.tapePos):
                self.tapePos.append(self.initSymbolOrd)
        else:
            pos = ~self.headLoc
            assert 0 <= pos <= len(self.tapeNeg)
            if pos == len(self.tapeNeg):
                self.tapeNeg.append(self.initSymbolOrd)

    def printTape(self, start, end, output=None):
        out = self.getTapeOutput(start, end)
        if output == None:
            print out,
        else:
            output.write(out)

    def getTapeOutput(self, start, end):
        headString = []
        tapeString = []
        for i in range(start, end):
            if i == self.headLoc:
                headString.append("v")
            else:
                headString.append(" ")

            tapeString.append(self._readSymbol(i)[0])

        if not self.name == None:
            tapeString.append(" " + self.name)

        headString = "".join(headString)
        tapeString = "".join(tapeString)
        return headString + "\n" + tapeString + "\n"
<commit_msg>fix missing length function<commit_after>import string
import sys

from state import *

def getStateName(line):
    colonLoc = string.find(line, ":")

    stateName = line[:colonLoc]

    return stateName

if __name__ == "__main__":
    sttm = SingleTapeTuringMachine(sys.argv[-1], ["_", "1", "H", "E"])
    args = sys.argv[1:-1]

    quiet = ("-q" in args)

    numSteps = float("Inf") # default value
    if ("-s" in args):
        numSteps = args[args.index("-s") + 1]

    output = None
    if ("-f" in args):
        output = open(args[args.index("-f") + 1], "w")

    sttm.run(quiet, numSteps, output)

def parseMachine(path, alphabet):
    with open(path, "r") as inp:
        tmLines = inp.readlines()

    stateDictionary = {"ACCEPT": SimpleState("ACCEPT", alphabet),
        "REJECT": SimpleState("REJECT", alphabet),
        "ERROR": SimpleState("ERROR", alphabet),
        "HALT": SimpleState("HALT", alphabet),
        "OUT": SimpleState("OUT", alphabet)}

    listOfRealStates = []

    # initialize state dictionary
    for line in tmLines[1:]:
        if line != "\n": # not a blank line
            lineSplit = string.split(line)

            if lineSplit[0] == "START":
                stateName = getStateName(line[6:])
                startState = State(stateName, None, alphabet)
                stateDictionary[stateName] = startState
                listOfRealStates.append(stateDictionary[stateName])
                startState.makeStartState()

            elif not lineSplit[0] in alphabet:
                stateName = getStateName(line)
                stateDictionary[stateName] = State(stateName, None, alphabet)
                listOfRealStates.append(stateDictionary[stateName])

    currentStateBeingModified = None

    # fill in state dictionary
    for line in tmLines[1:]:
        if line != "\n":
            lineSplit = string.split(line)

            if lineSplit[0] == "START":
                stateName = getStateName(line[6:])
                currentStateBeingModified = stateDictionary[stateName]

            elif not lineSplit[0] in alphabet:
                stateName = getStateName(line)
                currentStateBeingModified = stateDictionary[stateName]

            else:
                symbol = lineSplit[0]
                stateName = lineSplit[2][:-1]
                headMove = lineSplit[3][:-1]
                write = lineSplit[4]

                currentStateBeingModified.setNextState(symbol,
                    stateDictionary[stateName])
                currentStateBeingModified.setHeadMove(symbol, headMove)
                currentStateBeingModified.setWrite(symbol, write)

    return startState, stateDictionary

def stateDictionariesToLists(stateDictionary, alphabet, startState):
    simulationStates = {}
    for state in stateDictionary.itervalues():
        if state.isSimpleState():
            newState = state
        else:
            newState = SimulationState()
        simulationStates[state] = newState
    for state in stateDictionary.itervalues():
        if not state.isSimpleState():
            simulationStates[state]._initFromState(state, simulationStates)
    return simulationStates[startState]

class SingleTapeTuringMachine:
    def __init__(self, path, alphabet=["_", "1", "H", "E"]):
        self.state = None
        self.tape = Tape(None, alphabet[0])

        startState, stateDictionary = parseMachine(
                path, alphabet)
        startState = stateDictionariesToLists(
                stateDictionary, alphabet, startState)
        self.startState = startState

    def run(self, quiet=False, numSteps=float("Inf"), output=None):

        self.state = self.startState

        stepCounter = 0
        halted = False
        numSteps = float(numSteps)

        while stepCounter < numSteps:
            if not quiet:
                self.printTape(-2, 340, output)

            stepCounter += 1

            if self.state.isSimpleState():
                if self.state.stateName == "ERROR":
                    print "Turing machine threw error!"
                    halted = True
                    break

                if self.state.stateName == "ACCEPT":
                    print "Turing machine accepted after", stepCounter, "steps."
                    print len(self.tape.tapeDict), "squares of memory were used."
                    halted = True
                    break

                if self.state.stateName == "REJECT":
                    print "Turing machine rejected after", stepCounter, "steps."
                    print self.tape.length(), "squares of memory were used."
                    halted = True
                    break

                if self.state.stateName == "HALT":
                    print "Turing machine halted after", stepCounter, "steps."
                    print len(self.tape.tapeDict), "squares of memory were used."
                    halted = True
                    break

                if self.state.stateName == "OUT":
                    print "Turing machine execution incomplete: reached out state."
                    print "Perhaps this Turing machine wants to be melded with another machine."

            symbolord = self.tape.readSymbolOrd()
            headmove = self.state.getHeadMove(symbolord)

            self.tape.writeSymbolOrd(self.state.getWrite(symbolord))
            self.tape.moveHead(headmove)
            self.state = self.state.getNextState(symbolord)

        if not halted:
            print "Turing machine ran for", numSteps, "steps without halting."

    def printTape(self, start, end, output):
        if output == None:

            print self.state.stateName

            self.tape.printTape(start, end)
#           print "--------------------------------------"
        else:
            output.write(self.state.stateName + "\n")

            self.tape.printTape(start, end, output)
#           output.write("--------------------------------------\n")

class SimulationState(object):
    def __init__(self):
        self.nextState = [None] * 256
        self.headMove = [""] * 256
        self.write = [0] * 256

    def _initFromState(self, realState, simulationStates):
        self.stateName = realState.stateName
        self.description = realState.description
        self.alphabet = realState.alphabet
        self.isStartState = realState.isStartState
        for symbol in self.alphabet:
            self.headMove[ord(symbol)] = realState.headMoveDict[symbol]
            self.write[ord(symbol)] = ord(realState.writeDict[symbol])
            self.nextState[ord(symbol)] = simulationStates[
                    realState.nextStateDict[symbol]]

    def getNextState(self, ordsymbol):
        return self.nextState[ordsymbol]

    def getHeadMove(self, ordsymbol):
        return self.headMove[ordsymbol]

    def getWrite(self, ordsymbol):
        return self.write[ordsymbol]

    def isSimpleState(self):
        return False

class Tape(object):
    # By convention the first symbol in the alphabet is the initial symbol
    def __init__(self, name, initSymbol):
        self.name = name
        self.headLoc = 0
        self.initSymbol = initSymbol
        self.initSymbolOrd = ord(initSymbol)
        self.tapePos = [self.initSymbolOrd]
        self.tapeNeg = []

    def length(self):
        return len(self.tapePos) + len(self.tapeNeg)

    def readSymbol(self):
        return self._readSymbol(self.headLoc)

    def readSymbolOrd(self):
        return self._readSymbolOrd(self.headLoc)

    def _readSymbol(self, pos):
        return chr(self._readSymbolOrd(pos))

    def _readSymbolOrd(self, pos):
        try:
            if pos >= 0:
                return self.tapePos[pos]
            else:
                return self.tapeNeg[~pos]
        except IndexError:
            return self.initSymbolOrd

    def writeSymbol(self, symbol):
        return self.writeSymbolOrd(ord(symbol))

    def writeSymbolOrd(self, ordsymbol):
        if self.headLoc >= 0:
            self.tapePos[self.headLoc] = ordsymbol
        else:
            self.tapeNeg[~self.headLoc] = ordsymbol

    def moveHead(self, direction):
        if direction == "L":
            self.headLoc -= 1
            self.continueTape()

        elif direction == "R":
            self.headLoc += 1
            self.continueTape()

        elif direction == "-":
            pass
        else:
            print "bad head move", headmove
            raise

    def continueTape(self):
        if self.headLoc >= 0:
            assert 0 <= self.headLoc <= len(self.tapePos)
            if self.headLoc == len(self.tapePos):
                self.tapePos.append(self.initSymbolOrd)
        else:
            pos = ~self.headLoc
            assert 0 <= pos <= len(self.tapeNeg)
            if pos == len(self.tapeNeg):
                self.tapeNeg.append(self.initSymbolOrd)

    def printTape(self, start, end, output=None):
        out = self.getTapeOutput(start, end)
        if output == None:
            print out,
        else:
            output.write(out)

    def getTapeOutput(self, start, end):
        headString = []
        tapeString = []
        for i in range(start, end):
            if i == self.headLoc:
                headString.append("v")
            else:
                headString.append(" ")

            tapeString.append(self._readSymbol(i)[0])

        if not self.name == None:
            tapeString.append(" " + self.name)

        headString = "".join(headString)
        tapeString = "".join(tapeString)
        return headString + "\n" + tapeString + "\n"
<|endoftext|>


<commit_before>"""Helper functions for retrieving data via the GitHub API.

auth() -----------> Return credentials for use in GitHub API calls.
auth_reset() -----> Reset authentication settings to default values.
auth_user() ------> Set GitHub user for subsequent API calls.

memberfields() ---> Get field values for a member/user.
members() --------> Get members of one or more organizations.
membersget() -----> Get member info from specified API endpoint.

pagination() -----> Parse values from 'link' HTTP header returned by GitHub API.

repofields() -----> Get field values for a repo.
repos() ----------> Get repo information for organization(s) or user(s).
reposget() -------> Get repo information from specified API endpoint.

verbose() --------> Set verbose mode on/off.
verbose_output() -> Display status info in verbose mode.

write_csv() ------> Write a list of namedtuples to a CSV file.
"""
import collections
import csv
import json
import os
import traceback

import requests

#------------------------------------------------------------------------------
class _settings: # pylint: disable=R0903
    """Used for global settings. Should not be accessed directly - e.g.,
    use the verbose() function to change _settings.verbose, or the user()
    function to change _settings.github_username/accesstoken.
    """
    verbose = False # default = verbose mode off
    github_username = None
    github_accesstoken = None

#------------------------------------------------------------------------------
def auth():
    """Credentials for basic authentication.

    Returns the tuple used for API calls. If the GitHub username and PAT have
    not been set, reads them from environment variables GitHubUser/GitHubPAT.
    """
    # Note that "auth() ->" is explcitly added to the verbose_output()
    # calls below because auth() is typically used inline from other
    # functions, so it isn't the caller in the call stack.

    if not _settings.github_username:
        auth_reset()

    username = _settings.github_username
    access_token = _settings.github_accesstoken

    verbose_output('auth() ->', 'username:', username)
    verbose_output('auth() ->',
                   'PAT:', access_token[0:2] + '...' + access_token[-2:])

    return (username, access_token)

#-------------------------------------------------------------------------------
def auth_reset():
    """Reset authorization settings to default values.

    Sets username and acccess_token to the values stored in the  GitHubUser and
    GitHubPAT environment variables.
    """
    _settings.github_username = os.getenv('GitHubUser')
    _settings.github_accesstoken = os.getenv('GitHubPAT')

#-------------------------------------------------------------------------------
def auth_user(username=None):
    """Set GitHub user for subsequent API calls.

    username = a GitHub username stored in github_users.json in the
               /private subfolder. If omitted, the GitHub user settings are
               initialized from the GitHubUser/GitHubPAT environment variables.
    """
    if username:
        with open('private/github_users.json', 'r') as jsonfile:
            github_users = json.load(jsonfile)
            _settings.github_username = username
            _settings.github_accesstoken = github_users[username]
    else:
        # no username specified, so reset to defaults
        auth_reset()

#-------------------------------------------------------------------------------
def memberfields(member_json, fields):
    """Get field values for a member/user.

    1st parameter = member's json representation as returned by GitHub API
    2nd parameter = list of names of desired fields

    Returns a namedtuple containing the desired fields and their values.
    """
    member_tuple = collections.namedtuple('member_tuple', ' '.join(fields))
    values = {}
    for fldname in fields:
        values[fldname] = member_json[fldname]
    return member_tuple(**values)

#-------------------------------------------------------------------------------
def members(org=None, fields=None, audit2fa=False):
    """Get members for one or more organizations.

    org = organization
    fields = list of field names to be returned; names must be the same as
             returned by the GitHub API
    audit2fa = whether to only return members with 2FA disabled. You must be
               authenticated via auth_user() as a member of the org(s) to use
               this option.

    Returns a list of namedtuple objects, one per member.
    """
    if not fields:
        # default fields to be returned if none specified
        fields = ['login', 'id', 'type', 'site_admin']

    # set endpoint suffix for audit2fa option
    suffix = '?filter=2fa_disabled' if audit2fa else ''

    memberlist = [] # the list that will be returned

    if isinstance(org, str):
        # one org ID passed as a string
        endpoint = 'https://api.github.com/orgs/' + org + '/members' + suffix
        memberlist.extend(membersget(endpoint, fields))
    else:
        # list of org IDs passed
        for orgid in org:
            endpoint = 'https://api.github.com/orgs/' + orgid + '/members' + suffix
            memberlist.extend(membersget(endpoint, fields))

    return memberlist

#------------------------------------------------------------------------------
def membersget(endpoint, fields):
    """Get member info from specified API endpoint.

    1st parameter = GitHub API endpoint2nd parameter = list of fields to be returned

    Returns a list of namedtuples containing the specified fields.
    """
    totpages = 0
    retval = [] # the list to be returned

    while True:
        response = requests.get(endpoint, auth=auth())
        if response.ok:
            totpages += 1
            thispage = json.loads(response.text)
            for member_json in thispage:
                retval.append(memberfields(member_json, fields))
        pagelinks = pagination(response)
        endpoint = pagelinks['nextURL']
        if not endpoint:
            break # there are no more results to process
        verbose_output('processing page {0} of {1}'. \
                       format(pagelinks['nextpage'], pagelinks['lastpage']))

    verbose_output('pages processed:', totpages)
    verbose_output('members returned:', len(retval))
    for header in ['X-RateLimit-Limit', 'X-RateLimit-Remaining']:
        verbose_output(header + ':', response.headers[header])

    return retval

#------------------------------------------------------------------------------
def pagination(link_header):
    """Parse values from the 'link' HTTP header returned by GitHub API.

    1st parameter = the 'link' HTTP header passed as a string, or a
                    response object returned by the requests module

    Returns a dictionary with entries for the URLs and page numbers parsed
    from the link string: firstURL, firstpage, prevURL, prevpage, nextURL,
    nextpage, lastURL, lastpage.
    """
    # initialize the dictionary
    retval = {'firstpage':0, 'firstURL':None, 'prevpage':0, 'prevURL':None,
              'nextpage':0, 'nextURL':None, 'lastpage':0, 'lastURL':None}

    if isinstance(link_header, str):
        link_string = link_header
    else:
        try:
            link_string = link_header.headers['Link']
        except KeyError:
            return retval # no Link HTTP header found, nothing to parse

    links = link_string.split(',') # each of these is '<url>; rel="type"'
    for link in links:
        linktype = link.split(';')[-1].split('=')[-1].strip()[1:-1]
        url = link.split(';')[0].strip()[1:-1]
        pageno = url.split('?')[-1].split('=')[-1].strip()
        retval[linktype + 'page'] = pageno
        retval[linktype + 'URL'] = url

    return retval

#-------------------------------------------------------------------------------
def repofields(repo_json, fields):
    """Get field values for a repo.

    1st parameter = repo's json representation as returned by GitHub API
    2nd parameter = list of names of desired fields

    Returns a namedtuple containing the desired fields and their values.
    """
    fldnames = [_.replace('.', '_') for _ in fields] # periods to underscores
    repo_tuple = collections.namedtuple('Repo', ' '.join(fldnames))
    values = {}
    for fldname in fields:
        if '.' in fldname:
            # special case - embedded field within a JSON object
            try:
                values[fldname.replace('.', '_')] = \
                    repo_json[fldname.split('.')[0]][fldname.split('.')[1]]
            except TypeError:
                values[fldname.replace('.', '_')] = None
        else:
            values[fldname] = repo_json[fldname]
    return repo_tuple(**values)

#-------------------------------------------------------------------------------
def repos(org=None, user=None, fields=None):
    """Get repo information for organization(s) or user(s).

    org = organization; either a string containing a single organization ID, or
          a list of organization IDs to be included
    user = username (ignored if org argument is provided); either a string
           containing a single organization ID, or a list of organization IDs
           to be included
    fields = list of field names to be returned; names must be the same as
             returned by the GitHub API. Embedded elements are also supported:
             for example, pass a field named 'license.name' and it will
             return the 'name' element of the 'license' entry for each repo.
             Namedtuples don't allow embedded periods in identifiers, so in
             this case the column name will be 'license_name'.

    Returns a list of namedtuple objects, one per repo.
    """
    if not fields:
        # default fields to be returned if none specified
        fields = ['full_name', 'watchers', 'forks', 'open_issues']

    repolist = [] # the list that will be returned

    if org:
        # get repos by org
        if isinstance(org, str):
            # one org ID passed as a string
            endpoint = 'https://api.github.com/orgs/' + org + '/repos'
            repolist.extend(reposget(endpoint, fields))
        else:
            # list of org IDs passed
            for orgid in org:
                endpoint = 'https://api.github.com/orgs/' + orgid + '/repos'
                repolist.extend(reposget(endpoint, fields))
    else:
        # get repos by user
        if isinstance(user, str):
            # one user ID passed as a string
            endpoint = 'https://api.github.com/users/' + user + '/repos'
            repolist.extend(reposget(endpoint, fields))
        else:
            # list of user IDs passed
            for userid in user:
                endpoint = 'https://api.github.com/users/' + userid + '/repos'
                repolist.extend(reposget(endpoint, fields))

    return repolist

#-------------------------------------------------------------------------------
def reposget(endpoint, fields):
    """Get repo information from specified API endpoint.

    1st parameter = GitHub API endpoint
    2nd parameter = list of fields to be returned

    Returns a list of namedtuples containing the specified fields.
    """
    totpages = 0
    retval = [] # the list to be returned

    # custom header to retrieve license info while License API is in preview
    headers = {'Accept': 'application/vnd.github.drax-preview+json'}

    while True:
        response = requests.get(endpoint, auth=auth(), headers=headers)
        if response.ok:
            totpages += 1
            thispage = json.loads(response.text)
            for repo_json in thispage:
                retval.append(repofields(repo_json, fields))
        pagelinks = pagination(response)
        endpoint = pagelinks['nextURL']
        if not endpoint:
            break # there are no more results to process
        verbose_output('processing page {0} of {1}'. \
                       format(pagelinks['nextpage'], pagelinks['lastpage']))

    verbose_output('pages processed:', totpages)
    verbose_output('repos returned:', len(retval))
    for header in ['X-RateLimit-Limit', 'X-RateLimit-Remaining']:
        verbose_output(header + ':', response.headers[header])

    return retval

#-------------------------------------------------------------------------------
def verbose(*args):
    """Set verbose mode on/off.

    1st parameter = True for verbose mode, False to turn verbose mode off.

    Returns the current verbose mode setting as True/False. To query the
    current setting, call verbose() with no parameters.
    """
    if len(args) == 1:
        _settings.verbose = args[0]
    return _settings.verbose

#-------------------------------------------------------------------------------
def verbose_output(*args):
    """Display status information in verbose mode.

    parameters = message to be displayed if verbose(True) is set.

    NOTE: can pass any number of parameters, which will be displayed as a single
    string delimited by spaces.
    """
    if not _settings.verbose:
        return # verbose mode is off, nothing to do

    # convert all to strings, to allow for non-string parameters
    string_args = [str(_) for _ in args]

    caller = traceback.format_stack()[1].split(',')[2].strip().split()[1]
    print(caller + '() ->', ' '.join(string_args))

#-------------------------------------------------------------------------------
def write_csv(listobj, filename):
    """Write a list of namedtuples to a CSV file.

    1st parameter = the list
    2nd parameter = name of CSV file to be written
    """
    csvfile = open(filename, 'w', newline='')
    csvwriter = csv.writer(csvfile, dialect='excel')
    header_row = listobj[0]._fields
    csvwriter.writerow(header_row)

    verbose_output('filename:', filename)
    verbose_output('columns:', header_row)

    for row in listobj:
        values = []
        for fldname in header_row:
            values.append(getattr(row, fldname))
        csvwriter.writerow(values)
    csvfile.close()

    verbose_output('total rows:', len(listobj))

#===============================================================================
#------------------------------------ TESTS ------------------------------------
#===============================================================================

#-------------------------------------------------------------------------------
def test_auth():
    """Simple test for auth() function.
    """
    print(auth())

#-------------------------------------------------------------------------------
def test_members():
    """Simple test for members() function. Also tests write_csv().
    """
    auth_user('msftgits')
    test_results = members(org='microsoft', audit2fa=True)
    for member in test_results:
        print(member)
    print('Total members:', len(test_results))

#-------------------------------------------------------------------------------
def test_repos():
    """Simple test for repos() function. Also tests write_csv().
    """
    oct_repos = repos(user=['octocat', 'dmahugh'],
                      fields=['full_name', 'license.name', 'license'])
    for repo in oct_repos:
        print(repo)
    print('Total repos: ', len(oct_repos))
    write_csv(oct_repos, 'OctocatRepos.csv')

#-------------------------------------------------------------------------------
def test_pagination():
    """Simple test for pagination() function.
    """
    testlinks = "<https://api.github.com/organizations/6154722/" + \
        "repos?page=2>; rel=\"next\", <https://api.github.com/" + \
        "organizations/6154722/repos?page=18>; rel=\"last\""
    print(pagination(testlinks))

# if running standalone, run tests ---------------------------------------------
if __name__ == "__main__":

    verbose(True) # turn on verbose mode
    #test_auth()
    test_members()
    #test_repos()
    #test_pagination()
<commit_msg>add org field to members() output<commit_after>"""Helper functions for retrieving data via the GitHub API.

auth() -----------> Return credentials for use in GitHub API calls.
auth_reset() -----> Reset authentication settings to default values.
auth_user() ------> Set GitHub user for subsequent API calls.

memberfields() ---> Get field values for a member/user.
members() --------> Get members of one or more organizations.
membersget() -----> Get member info for a specified organization.

pagination() -----> Parse values from 'link' HTTP header returned by GitHub API.

repofields() -----> Get field values for a repo.
repos() ----------> Get repo information for organization(s) or user(s).
reposget() -------> Get repo information from specified API endpoint.

verbose() --------> Set verbose mode on/off.
verbose_output() -> Display status info in verbose mode.

write_csv() ------> Write a list of namedtuples to a CSV file.
"""
import collections
import csv
import json
import os
import traceback

import requests

#------------------------------------------------------------------------------
class _settings: # pylint: disable=R0903
    """Used for global settings. Should not be accessed directly - e.g.,
    use the verbose() function to change _settings.verbose, or the user()
    function to change _settings.github_username/accesstoken.
    """
    verbose = False # default = verbose mode off
    github_username = None
    github_accesstoken = None

#------------------------------------------------------------------------------
def auth():
    """Credentials for basic authentication.

    Returns the tuple used for API calls. If the GitHub username and PAT have
    not been set, reads them from environment variables GitHubUser/GitHubPAT.
    """
    # Note that "auth() ->" is explcitly added to the verbose_output()
    # calls below because auth() is typically used inline from other
    # functions, so it isn't the caller in the call stack.

    if not _settings.github_username:
        auth_reset()

    username = _settings.github_username
    access_token = _settings.github_accesstoken

    verbose_output('auth() ->', 'username:', username)
    verbose_output('auth() ->',
                   'PAT:', access_token[0:2] + '...' + access_token[-2:])

    return (username, access_token)

#-------------------------------------------------------------------------------
def auth_reset():
    """Reset authorization settings to default values.

    Sets username and acccess_token to the values stored in the  GitHubUser and
    GitHubPAT environment variables.
    """
    _settings.github_username = os.getenv('GitHubUser')
    _settings.github_accesstoken = os.getenv('GitHubPAT')

#-------------------------------------------------------------------------------
def auth_user(username=None):
    """Set GitHub user for subsequent API calls.

    username = a GitHub username stored in github_users.json in the
               /private subfolder. If omitted, the GitHub user settings are
               initialized from the GitHubUser/GitHubPAT environment variables.
    """
    if username:
        with open('private/github_users.json', 'r') as jsonfile:
            github_users = json.load(jsonfile)
            _settings.github_username = username
            _settings.github_accesstoken = github_users[username]
    else:
        # no username specified, so reset to defaults
        auth_reset()

#-------------------------------------------------------------------------------
def memberfields(member_json, fields, org):
    """Get field values for a member/user.

    1st parameter = member's json representation as returned by GitHub API
    2nd parameter = list of names of desired fields
    3rd parameter = organization ID

    Returns a namedtuple containing the desired fields and their values.
    NOTE: in addition to the specified fields, always returns an 'org' field
    to distinguish between organizations in lists returned by members().
    """
    member_tuple = collections.namedtuple('member_tuple', 'org ' + ' '.join(fields))
    values = {}
    values['org'] = org
    for fldname in fields:
        values[fldname] = member_json[fldname]
    return member_tuple(**values)

#-------------------------------------------------------------------------------
def members(org=None, fields=None, audit2fa=False):
    """Get members for one or more organizations.

    org = organization
    fields = list of field names to be returned; names must be the same as
             returned by the GitHub API
    audit2fa = whether to only return members with 2FA disabled. You must be
               authenticated via auth_user() as a member of the org(s) to use
               this option.

    Returns a list of namedtuple objects, one per member.
    """
    if not fields:
        # default fields to be returned if none specified
        fields = ['login', 'id', 'type', 'site_admin']

    memberlist = [] # the list that will be returned

    if isinstance(org, str):
        # one org ID passed as a string
        memberlist.extend(membersget(org, fields, audit2fa))
    else:
        # list of org IDs passed
        for orgid in org:
            memberlist.extend(membersget(orgid, fields, audit2fa))

    return memberlist

#------------------------------------------------------------------------------
def membersget(org, fields, audit2fa):
    """Get member info for a specified organization.

    1st parameter = organization ID
    2nd parameter = list of fields to be returned
    3rd parameter = whether to only return members with 2FA disabled. You must
                    be authenticated via auth_user() as a member of the org(s)
                    to use this option.

    Returns a list of namedtuples containing the specified fields.
    """
    totpages = 0
    retval = [] # the list to be returned

    suffix = '?filter=2fa_disabled' if audit2fa else ''
    endpoint = 'https://api.github.com/orgs/' + org + '/members' + suffix

    while True:
        response = requests.get(endpoint, auth=auth())
        if response.ok:
            totpages += 1
            thispage = json.loads(response.text)
            for member_json in thispage:
                retval.append(memberfields(member_json, fields, org))
        pagelinks = pagination(response)
        endpoint = pagelinks['nextURL']
        if not endpoint:
            break # there are no more results to process
        verbose_output('processing page {0} of {1}'. \
                       format(pagelinks['nextpage'], pagelinks['lastpage']))

    verbose_output('pages processed:', totpages)
    verbose_output('members returned:', len(retval))
    for header in ['X-RateLimit-Limit', 'X-RateLimit-Remaining']:
        verbose_output(header + ':', response.headers[header])

    return retval

#------------------------------------------------------------------------------
def pagination(link_header):
    """Parse values from the 'link' HTTP header returned by GitHub API.

    1st parameter = the 'link' HTTP header passed as a string, or a
                    response object returned by the requests module

    Returns a dictionary with entries for the URLs and page numbers parsed
    from the link string: firstURL, firstpage, prevURL, prevpage, nextURL,
    nextpage, lastURL, lastpage.
    """
    # initialize the dictionary
    retval = {'firstpage':0, 'firstURL':None, 'prevpage':0, 'prevURL':None,
              'nextpage':0, 'nextURL':None, 'lastpage':0, 'lastURL':None}

    if isinstance(link_header, str):
        link_string = link_header
    else:
        try:
            link_string = link_header.headers['Link']
        except KeyError:
            return retval # no Link HTTP header found, nothing to parse

    links = link_string.split(',') # each of these is '<url>; rel="type"'
    for link in links:
        linktype = link.split(';')[-1].split('=')[-1].strip()[1:-1]
        url = link.split(';')[0].strip()[1:-1]
        pageno = url.split('?')[-1].split('=')[-1].strip()
        retval[linktype + 'page'] = pageno
        retval[linktype + 'URL'] = url

    return retval

#-------------------------------------------------------------------------------
def repofields(repo_json, fields):
    """Get field values for a repo.

    1st parameter = repo's json representation as returned by GitHub API
    2nd parameter = list of names of desired fields

    Returns a namedtuple containing the desired fields and their values.
    """
    fldnames = [_.replace('.', '_') for _ in fields] # periods to underscores
    repo_tuple = collections.namedtuple('Repo', ' '.join(fldnames))
    values = {}
    for fldname in fields:
        if '.' in fldname:
            # special case - embedded field within a JSON object
            try:
                values[fldname.replace('.', '_')] = \
                    repo_json[fldname.split('.')[0]][fldname.split('.')[1]]
            except TypeError:
                values[fldname.replace('.', '_')] = None
        else:
            values[fldname] = repo_json[fldname]
    return repo_tuple(**values)

#-------------------------------------------------------------------------------
def repos(org=None, user=None, fields=None):
    """Get repo information for organization(s) or user(s).

    org = organization; either a string containing a single organization ID, or
          a list of organization IDs to be included
    user = username (ignored if org argument is provided); either a string
           containing a single organization ID, or a list of organization IDs
           to be included
    fields = list of field names to be returned; names must be the same as
             returned by the GitHub API. Embedded elements are also supported:
             for example, pass a field named 'license.name' and it will
             return the 'name' element of the 'license' entry for each repo.
             Namedtuples don't allow embedded periods in identifiers, so in
             this case the column name will be 'license_name'.

    Returns a list of namedtuple objects, one per repo.
    """
    if not fields:
        # default fields to be returned if none specified
        fields = ['full_name', 'watchers', 'forks', 'open_issues']

    repolist = [] # the list that will be returned

    if org:
        # get repos by org
        if isinstance(org, str):
            # one org ID passed as a string
            endpoint = 'https://api.github.com/orgs/' + org + '/repos'
            repolist.extend(reposget(endpoint, fields))
        else:
            # list of org IDs passed
            for orgid in org:
                endpoint = 'https://api.github.com/orgs/' + orgid + '/repos'
                repolist.extend(reposget(endpoint, fields))
    else:
        # get repos by user
        if isinstance(user, str):
            # one user ID passed as a string
            endpoint = 'https://api.github.com/users/' + user + '/repos'
            repolist.extend(reposget(endpoint, fields))
        else:
            # list of user IDs passed
            for userid in user:
                endpoint = 'https://api.github.com/users/' + userid + '/repos'
                repolist.extend(reposget(endpoint, fields))

    return repolist

#-------------------------------------------------------------------------------
def reposget(endpoint, fields):
    """Get repo information from specified API endpoint.

    1st parameter = GitHub API endpoint
    2nd parameter = list of fields to be returned

    Returns a list of namedtuples containing the specified fields.
    """
    totpages = 0
    retval = [] # the list to be returned

    # custom header to retrieve license info while License API is in preview
    headers = {'Accept': 'application/vnd.github.drax-preview+json'}

    while True:
        response = requests.get(endpoint, auth=auth(), headers=headers)
        if response.ok:
            totpages += 1
            thispage = json.loads(response.text)
            for repo_json in thispage:
                retval.append(repofields(repo_json, fields))
        pagelinks = pagination(response)
        endpoint = pagelinks['nextURL']
        if not endpoint:
            break # there are no more results to process
        verbose_output('processing page {0} of {1}'. \
                       format(pagelinks['nextpage'], pagelinks['lastpage']))

    verbose_output('pages processed:', totpages)
    verbose_output('repos returned:', len(retval))
    for header in ['X-RateLimit-Limit', 'X-RateLimit-Remaining']:
        verbose_output(header + ':', response.headers[header])

    return retval

#-------------------------------------------------------------------------------
def verbose(*args):
    """Set verbose mode on/off.

    1st parameter = True for verbose mode, False to turn verbose mode off.

    Returns the current verbose mode setting as True/False. To query the
    current setting, call verbose() with no parameters.
    """
    if len(args) == 1:
        _settings.verbose = args[0]
    return _settings.verbose

#-------------------------------------------------------------------------------
def verbose_output(*args):
    """Display status information in verbose mode.

    parameters = message to be displayed if verbose(True) is set.

    NOTE: can pass any number of parameters, which will be displayed as a single
    string delimited by spaces.
    """
    if not _settings.verbose:
        return # verbose mode is off, nothing to do

    # convert all to strings, to allow for non-string parameters
    string_args = [str(_) for _ in args]

    caller = traceback.format_stack()[1].split(',')[2].strip().split()[1]
    print(caller + '() ->', ' '.join(string_args))

#-------------------------------------------------------------------------------
def write_csv(listobj, filename):
    """Write a list of namedtuples to a CSV file.

    1st parameter = the list
    2nd parameter = name of CSV file to be written
    """
    csvfile = open(filename, 'w', newline='')
    csvwriter = csv.writer(csvfile, dialect='excel')
    header_row = listobj[0]._fields
    csvwriter.writerow(header_row)

    verbose_output('filename:', filename)
    verbose_output('columns:', header_row)

    for row in listobj:
        values = []
        for fldname in header_row:
            values.append(getattr(row, fldname))
        csvwriter.writerow(values)
    csvfile.close()

    verbose_output('total rows:', len(listobj))

#===============================================================================
#------------------------------------ TESTS ------------------------------------
#===============================================================================

#-------------------------------------------------------------------------------
def test_auth():
    """Simple test for auth() function.
    """
    print(auth())

#-------------------------------------------------------------------------------
def test_members():
    """Simple test for members() function.
    """
    auth_user('msftgits')
    test_results = members(org=['bitstadium', 'liveservices'])
    for member in test_results:
        print(member)
    print('Total members:', len(test_results))

#-------------------------------------------------------------------------------
def test_repos():
    """Simple test for repos() function. Also tests write_csv().
    """
    oct_repos = repos(user=['octocat', 'dmahugh'],
                      fields=['full_name', 'license.name', 'license'])
    for repo in oct_repos:
        print(repo)
    print('Total repos: ', len(oct_repos))
    write_csv(oct_repos, 'data\OctocatRepos.csv')

#-------------------------------------------------------------------------------
def test_pagination():
    """Simple test for pagination() function.
    """
    testlinks = "<https://api.github.com/organizations/6154722/" + \
        "repos?page=2>; rel=\"next\", <https://api.github.com/" + \
        "organizations/6154722/repos?page=18>; rel=\"last\""
    print(pagination(testlinks))

# if running standalone, run tests ---------------------------------------------
if __name__ == "__main__":

    verbose(True) # turn on verbose mode
    #test_auth()
    test_members()
    #test_repos()
    #test_pagination()
<|endoftext|>