<commit_before>"""
===============================================================
Model selection with Probabilistic PCA and Factor Analysis (FA)
===============================================================

Probabilistic PCA and Factor Analysis are probabilistic models.
The consequence is that the likelihood of new data can be used
for model selection and covariance estimation.
Here we compare PCA and FA with cross-validation on low rank data corrupted
with homoscedastic noise (noise variance
is the same for each feature) or heteroscedastic noise (noise variance
is the different for each feature). In a second step we compare the model
likelihood to the likelihoods obtained from shrinkage covariance estimators.

One can observe that with homoscedastic noise both FA and PCA succeed
in recovering the size of the low rank subspace. The likelihood with PCA
is higher than FA in this case. However PCA fails and overestimates
the rank when heteroscedastic noise is present. Under appropriate
circumstances the low rank models are more likely than shrinkage models.

The automatic estimation from
Automatic Choice of Dimensionality for PCA. NIPS 2000: 598-604
by Thomas P. Minka is also compared.

"""

# Authors: Alexandre Gramfort
#          Denis A. Engemann
# License: BSD 3 clause

import numpy as np
import matplotlib.pyplot as plt
from scipy import linalg

from sklearn.decomposition import PCA, FactorAnalysis
from sklearn.covariance import ShrunkCovariance, LedoitWolf
from sklearn.model_selection import cross_val_score
from sklearn.model_selection import GridSearchCV

# #############################################################################
# Create the data

n_samples, n_features, rank = 1000, 50, 10
sigma = 1.0
rng = np.random.RandomState(42)
U, _, _ = linalg.svd(rng.randn(n_features, n_features))
X = np.dot(rng.randn(n_samples, rank), U[:, :rank].T)

# Adding homoscedastic noise
X_homo = X + sigma * rng.randn(n_samples, n_features)

# Adding heteroscedastic noise
sigmas = sigma * rng.rand(n_features) + sigma / 2.0
X_hetero = X + rng.randn(n_samples, n_features) * sigmas

# #############################################################################
# Fit the models

n_components = np.arange(0, n_features, 5)  # options for n_components

def compute_scores(X):
    pca = PCA(svd_solver="full")
    fa = FactorAnalysis()

    pca_scores, fa_scores = [], []
    for n in n_components:
        pca.n_components = n
        fa.n_components = n
        pca_scores.append(np.mean(cross_val_score(pca, X)))
        fa_scores.append(np.mean(cross_val_score(fa, X)))

    return pca_scores, fa_scores

def shrunk_cov_score(X):
    shrinkages = np.logspace(-2, 0, 30)
    cv = GridSearchCV(ShrunkCovariance(), {"shrinkage": shrinkages})
    return np.mean(cross_val_score(cv.fit(X).best_estimator_, X))

def lw_score(X):
    return np.mean(cross_val_score(LedoitWolf(), X))

for X, title in [(X_homo, "Homoscedastic Noise"), (X_hetero, "Heteroscedastic Noise")]:
    pca_scores, fa_scores = compute_scores(X)
    n_components_pca = n_components[np.argmax(pca_scores)]
    n_components_fa = n_components[np.argmax(fa_scores)]

    pca = PCA(svd_solver="full", n_components="mle")
    pca.fit(X)
    n_components_pca_mle = pca.n_components_

    print("best n_components by PCA CV = %d" % n_components_pca)
    print("best n_components by FactorAnalysis CV = %d" % n_components_fa)
    print("best n_components by PCA MLE = %d" % n_components_pca_mle)

    plt.figure()
    plt.plot(n_components, pca_scores, "b", label="PCA scores")
    plt.plot(n_components, fa_scores, "r", label="FA scores")
    plt.axvline(rank, color="g", label="TRUTH: %d" % rank, linestyle="-")
    plt.axvline(
        n_components_pca,
        color="b",
        label="PCA CV: %d" % n_components_pca,
        linestyle="--",
    )
    plt.axvline(
        n_components_fa,
        color="r",
        label="FactorAnalysis CV: %d" % n_components_fa,
        linestyle="--",
    )
    plt.axvline(
        n_components_pca_mle,
        color="k",
        label="PCA MLE: %d" % n_components_pca_mle,
        linestyle="--",
    )

    # compare with other covariance estimators
    plt.axhline(
        shrunk_cov_score(X),
        color="violet",
        label="Shrunk Covariance MLE",
        linestyle="-.",
    )
    plt.axhline(
        lw_score(X),
        color="orange",
        label="LedoitWolf MLE" % n_components_pca_mle,
        linestyle="-.",
    )

    plt.xlabel("nb of components")
    plt.ylabel("CV scores")
    plt.legend(loc="lower right")
    plt.title(title)

plt.show()
<commit_msg>ENH Impoving execution speed of plot_pca_vs_fa_model_selection.py by … (#21671)<commit_after>"""
===============================================================
Model selection with Probabilistic PCA and Factor Analysis (FA)
===============================================================

Probabilistic PCA and Factor Analysis are probabilistic models.
The consequence is that the likelihood of new data can be used
for model selection and covariance estimation.
Here we compare PCA and FA with cross-validation on low rank data corrupted
with homoscedastic noise (noise variance
is the same for each feature) or heteroscedastic noise (noise variance
is the different for each feature). In a second step we compare the model
likelihood to the likelihoods obtained from shrinkage covariance estimators.

One can observe that with homoscedastic noise both FA and PCA succeed
in recovering the size of the low rank subspace. The likelihood with PCA
is higher than FA in this case. However PCA fails and overestimates
the rank when heteroscedastic noise is present. Under appropriate
circumstances (choice of the number of components), the held-out
data is more likely for low rank models than for shrinkage models.

The automatic estimation from
Automatic Choice of Dimensionality for PCA. NIPS 2000: 598-604
by Thomas P. Minka is also compared.

"""

# Authors: Alexandre Gramfort
#          Denis A. Engemann
# License: BSD 3 clause

import numpy as np
import matplotlib.pyplot as plt
from scipy import linalg

from sklearn.decomposition import PCA, FactorAnalysis
from sklearn.covariance import ShrunkCovariance, LedoitWolf
from sklearn.model_selection import cross_val_score
from sklearn.model_selection import GridSearchCV

# #############################################################################
# Create the data

n_samples, n_features, rank = 500, 25, 5
sigma = 1.0
rng = np.random.RandomState(42)
U, _, _ = linalg.svd(rng.randn(n_features, n_features))
X = np.dot(rng.randn(n_samples, rank), U[:, :rank].T)

# Adding homoscedastic noise
X_homo = X + sigma * rng.randn(n_samples, n_features)

# Adding heteroscedastic noise
sigmas = sigma * rng.rand(n_features) + sigma / 2.0
X_hetero = X + rng.randn(n_samples, n_features) * sigmas

# #############################################################################
# Fit the models

n_components = np.arange(0, n_features, 5)  # options for n_components

def compute_scores(X):
    pca = PCA(svd_solver="full")
    fa = FactorAnalysis()

    pca_scores, fa_scores = [], []
    for n in n_components:
        pca.n_components = n
        fa.n_components = n
        pca_scores.append(np.mean(cross_val_score(pca, X)))
        fa_scores.append(np.mean(cross_val_score(fa, X)))

    return pca_scores, fa_scores

def shrunk_cov_score(X):
    shrinkages = np.logspace(-2, 0, 30)
    cv = GridSearchCV(ShrunkCovariance(), {"shrinkage": shrinkages})
    return np.mean(cross_val_score(cv.fit(X).best_estimator_, X))

def lw_score(X):
    return np.mean(cross_val_score(LedoitWolf(), X))

for X, title in [(X_homo, "Homoscedastic Noise"), (X_hetero, "Heteroscedastic Noise")]:
    pca_scores, fa_scores = compute_scores(X)
    n_components_pca = n_components[np.argmax(pca_scores)]
    n_components_fa = n_components[np.argmax(fa_scores)]

    pca = PCA(svd_solver="full", n_components="mle")
    pca.fit(X)
    n_components_pca_mle = pca.n_components_

    print("best n_components by PCA CV = %d" % n_components_pca)
    print("best n_components by FactorAnalysis CV = %d" % n_components_fa)
    print("best n_components by PCA MLE = %d" % n_components_pca_mle)

    plt.figure()
    plt.plot(n_components, pca_scores, "b", label="PCA scores")
    plt.plot(n_components, fa_scores, "r", label="FA scores")
    plt.axvline(rank, color="g", label="TRUTH: %d" % rank, linestyle="-")
    plt.axvline(
        n_components_pca,
        color="b",
        label="PCA CV: %d" % n_components_pca,
        linestyle="--",
    )
    plt.axvline(
        n_components_fa,
        color="r",
        label="FactorAnalysis CV: %d" % n_components_fa,
        linestyle="--",
    )
    plt.axvline(
        n_components_pca_mle,
        color="k",
        label="PCA MLE: %d" % n_components_pca_mle,
        linestyle="--",
    )

    # compare with other covariance estimators
    plt.axhline(
        shrunk_cov_score(X),
        color="violet",
        label="Shrunk Covariance MLE",
        linestyle="-.",
    )
    plt.axhline(
        lw_score(X),
        color="orange",
        label="LedoitWolf MLE" % n_components_pca_mle,
        linestyle="-.",
    )

    plt.xlabel("nb of components")
    plt.ylabel("CV scores")
    plt.legend(loc="lower right")
    plt.title(title)

plt.show()
<|endoftext|>


<commit_before>#!/usr/bin/python3
# -*- coding: utf-8 -*-

import argparse
import io
import logging
import sys
import requests
import urllib3
import tld
from tld import get_tld
from bs4 import BeautifulSoup
import html
from PyPDF2 import PdfFileReader, PdfFileWriter

def main():
    # See https://blog.quora.com/Launched-Customizable-Links for Quora's
    # launch post
    parser = argparse.ArgumentParser(description=("Get the linked title of " +
        "URLs (similar to Quora and Facebook)"))
    parser.add_argument("url", type=str, help="the URL")
    parser.add_argument("-f", "--format", type=str,
            help=("output format; accepted values are 'html', " +
            "'markdown', 'latex', 'mediawiki'"))
    parser.add_argument("-c", "--clean", action="store_true",
            help=("clean the title to remove the site name " +
            "if the title was obtained from an HTML title tag"))
    parser.add_argument("-v", "--verbose", action="store_const",
        dest="log_level", const=logging.DEBUG, help="enable debug messages")
    args = parser.parse_args()
    logging.basicConfig(level=args.log_level)

    url = args.url
    logging.debug("Trying first attempt")
    attempt_1 = try_url(url)
    if attempt_1["exit"]:
        logging.debug("First attempt succeeded!")
        print(attempt_1["text"], end="")
    else:
        logging.debug("First attempt failed; trying second attempt")
        attempt_2 = try_url("http://" + url)
        if attempt_2["exit"]:
            logging.debug("Second attempt succeeded!")
            print(attempt_2["text"], end="")
        else:
            logging.debug("Second attempted failed; using result " +
                "from first attempt")
            print(attempt_1["text"], end="")

def try_url(url):
    '''
    Return (Str, True) if succeeded; (Str, False) otherwise.
    '''
    result = {}
    analyte = analyze_url(url)
    if analyte:
        return {"text": analyte, "exit": True}
    try:
        user_agent = ("Mozilla/5.0 (X11; Linux i686; rv:31.0) Gecko/20100101 " +
            "Firefox/31.0 Iceweasel/31.5.0")
        headers = {"User-Agent": user_agent}
        response = requests.get(url, stream=True, headers=headers)
        url = response.url
        if  "text/html" in response.headers["content-type"] or "application/pdf" in response.headers["content-type"]:
            logging.debug("HTML page or PDF file detected")
            # <title> is probably in the first around 10MB
            doc = response.iter_content(chunk_size=1000000)
            data = next(doc)
            result["text"] = get_filetype_link(
                get_link_text(url, response.headers["content-type"], data=data),
                url,
                "markdown"
            )
        else:
            logging.debug("No HTML page detected")
            result["text"] = get_filetype_link(
                get_link_text(url, response.headers["content-type"]),
                url,
                "markdown"
            )
        result["exit"] = True
    except (requests.exceptions.MissingSchema, requests.exceptions.ConnectionError, requests.exceptions.InvalidSchema, urllib3.exceptions.LocationParseError):
        result["text"] = "[{url}]({url})".format(url=url)
        result["exit"] = False
    return result

def analyze_url(url):
    """
    Look just at the URL to see if a suitable title text can be found.  This
    method is much faster than actually visiting the URL to find the title
    element in the downloaded file. We want to do this for special sites like
    Facebook, which doesn't allow anonymous downloading of certain pages, like
    group pages.

    Args:
        url: A string that is a URL

    Returns:
        A string that is the title text to be used. If no suitable title text
        can be produced, return the empty string, "".
    """
    try:
        tl = get_tld(url)
    except tld.exceptions.TldBadUrl:
        try:
            tl = get_tld("http://" + url)
        except tld.exceptions.TldBadUrl:
            return ""
    if tl == "facebook.com" and "facebook.com/groups/" in url:
            return "Facebook group page post"
    return ""

def get_filetype_link(link_text, url, filetype):
    """
    Put the pieces together and produce a valid hyperlink for given output
    filetype.

    Args:
        link_text: A string representing the displayed or linked text when
            linking to something, e.g. "hello" in hello. This string should already be in
            the intended form; i.e. all HTML escapes should have been unescaped
            at this point.
        url: A string of the URL.
        filetype: A string of the output filetype. Accepted parameters are:
            "html", "markdown", "latex".

    Returns:
        A string that is a valid hyperlink for the specified output filetype.
    """
    if filetype == "markdown":
        # From http://pandoc.org/README.html#backslash-escapes
        special_chars = "\\`*_{}[]()>#+-.!"
        result = ""
        for c in link_text:
            if c in special_chars:
                result += "\\" + c
            else:
                result += c
        return "[{link_text}]({url})".format(link_text=result, url=url)
    if filetype == "html":
        return '{link_text}'.format(url=url, link_text=html.escape(link_text))
    if filetype == "mediawiki":
        return "[{url} {link_text}]".format(url=url, link_text=link_text)

def get_link_text(url, mime_type, data=None):
    '''
    Take URL, MIME type, and optional data to produce the link text.
    '''
    tld = get_tld(url)
    result = "File on " + tld
    if mime_type.startswith("image"):
        result = "Image on " + tld
    elif  "application/pdf" in mime_type:
        logging.debug("PDF detected")
        # I need seek() for some reason so convert from bytes
        data = io.BytesIO(data)
        # fix this later, but I always get a "PdfReadWarning: Xref table
        # not zero-indexed" which should only happen when the -v flag is
        # present
        import warnings
        warnings.filterwarnings("ignore")
        pdf = PdfFileReader(data, strict=True)
        # PyPDF2 somehow thinks many PDFs are encrypted with the empty
        # string, so deal with that
        if pdf.isEncrypted:
            pdf.decrypt('')
        result = pdf.getDocumentInfo().title
        #result = "PDF on " + tld
    elif "text/html" in mime_type:
        try:
            soup = BeautifulSoup(data, 'html.parser')
            meta = soup.find_all("meta")
            og_title_lst = []
            twitter_title_lst = []
            meta_title_lst = []
            for i in meta:
                if i.get("property") == "og:title":
                    og_title_lst.append(i.get("content"))
                elif i.get("property") == "twitter:title":
                    twitter_title_lst.append(i.get("content"))
                elif i.get("name") == "title":
                    meta_title_lst.append(i.get("content"))
            if og_title_lst:
                logging.debug("found og:title")
                result = og_title_lst[0].strip()
            elif twitter_title_lst:
                logging.debug("found twitter title")
                result = twitter_title_lst[0].strip()
            elif meta_title_lst:
                logging.debug("found meta name title")
                result = meta_title_lst[0].strip()
            elif soup.title and soup.title.string:
                logging.debug("found title tag")
                result = messy_title_parse(html.unescape(
                    soup.title.string))
            else:
                logging.debug("no title found; using default")
                result = "Page on " + tld
        except AttributeError:
            # Probably just empty title when trying to get
            # soup.title.string
            logging.debug("FIXME: this isn't supposed to happen")
            result = "Page on " + tld
    if len(result) > 255:
        result = result[:253] + " …"

    return result

def messy_title_parse(title, url=None):
    # Even if nothing works, at least we'll have a whitespace-sanitized
    # title
    logging.debug("Cleaning messy title")
    result = title.strip()
    hyphen_split = result.split(" - ")
    bar_split = result.split(" | ")
    em_dash_split = result.split(" — ")
    colon_split = result.split(": ")
    if len(hyphen_split) > 1:
        # So there is actually more than one part, so we just take the
        # first and we're done.  This is for titles like "Post Title -
        # Site Name"
        result = hyphen_split[0]
    elif len(em_dash_split) > 1:
        result = em_dash_split[0]
    elif len(bar_split) > 1:
        result = bar_split[0]
    elif len(colon_split) > 1:
        # For titles like "Site Name: Post Title"
        result = colon_split[-1]
    return result

if __name__ == "__main__":
    main()
<commit_msg>implement formats and latex<commit_after>#!/usr/bin/python3
# -*- coding: utf-8 -*-

import argparse
import io
import logging
import sys
import requests
import urllib3
import tld
from tld import get_tld
from bs4 import BeautifulSoup
import html
from PyPDF2 import PdfFileReader, PdfFileWriter

def main():
    # See https://blog.quora.com/Launched-Customizable-Links for Quora's
    # launch post
    parser = argparse.ArgumentParser(description=("Get the linked title of " +
        "URLs (similar to Quora and Facebook)"))
    parser.add_argument("url", type=str, help="the URL")
    parser.add_argument("-f", "--format", type=str,
            help=("output format; accepted values are 'html', " +
            "'markdown', 'latex', 'mediawiki'"))
    parser.add_argument("-c", "--clean", action="store_true",
            help=("clean the title to remove the site name " +
            "if the title was obtained from an HTML title tag"))
    parser.add_argument("-v", "--verbose", action="store_const",
        dest="log_level", const=logging.DEBUG, help="enable debug messages")
    args = parser.parse_args()
    logging.basicConfig(level=args.log_level)

    url = args.url
    logging.debug("Trying first attempt")
    attempt_1 = try_url(url, args.format)
    if attempt_1["exit"]:
        logging.debug("First attempt succeeded!")
        print(attempt_1["text"], end="")
    else:
        logging.debug("First attempt failed; trying second attempt")
        attempt_2 = try_url("http://" + url, args.format)
        if attempt_2["exit"]:
            logging.debug("Second attempt succeeded!")
            print(attempt_2["text"], end="")
        else:
            logging.debug("Second attempted failed; using result " +
                "from first attempt")
            print(attempt_1["text"], end="")

def try_url(url, fmt):
    '''
    Return (Str, True) if succeeded; (Str, False) otherwise.
    '''
    result = {}
    analyte = analyze_url(url)
    if analyte:
        return {"text": analyte, "exit": True}
    try:
        user_agent = ("Mozilla/5.0 (X11; Linux i686; rv:31.0) Gecko/20100101 " +
            "Firefox/31.0 Iceweasel/31.5.0")
        headers = {"User-Agent": user_agent}
        response = requests.get(url, stream=True, headers=headers)
        url = response.url
        if  "text/html" in response.headers["content-type"] or "application/pdf" in response.headers["content-type"]:
            logging.debug("HTML page or PDF file detected")
            # <title> is probably in the first around 10MB
            doc = response.iter_content(chunk_size=1000000)
            data = next(doc)
            result["text"] = get_filetype_link(
                get_link_text(url, response.headers["content-type"], data=data),
                url,
                fmt
            )
        else:
            logging.debug("No HTML page detected")
            result["text"] = get_filetype_link(
                get_link_text(url, response.headers["content-type"]),
                url,
                fmt
            )
        result["exit"] = True
    except (requests.exceptions.MissingSchema, requests.exceptions.ConnectionError, requests.exceptions.InvalidSchema, urllib3.exceptions.LocationParseError):
        result["text"] = "[{url}]({url})".format(url=url)
        result["exit"] = False
    return result

def analyze_url(url):
    """
    Look just at the URL to see if a suitable title text can be found.  This
    method is much faster than actually visiting the URL to find the title
    element in the downloaded file. We want to do this for special sites like
    Facebook, which doesn't allow anonymous downloading of certain pages, like
    group pages.

    Args:
        url: A string that is a URL

    Returns:
        A string that is the title text to be used. If no suitable title text
        can be produced, return the empty string, "".
    """
    try:
        tl = get_tld(url)
    except tld.exceptions.TldBadUrl:
        try:
            tl = get_tld("http://" + url)
        except tld.exceptions.TldBadUrl:
            return ""
    if tl == "facebook.com" and "facebook.com/groups/" in url:
            return "Facebook group page post"
    return ""

def get_filetype_link(link_text, url, filetype):
    """
    Put the pieces together and produce a valid hyperlink for given output
    filetype.

    Args:
        link_text: A string representing the displayed or linked text when
            linking to something, e.g. "hello" in hello. This string should already be in
            the intended form; i.e. all HTML escapes should have been unescaped
            at this point.
        url: A string of the URL.
        filetype: A string of the output filetype. Accepted parameters are:
            "html", "markdown", "latex".

    Returns:
        A string that is a valid hyperlink for the specified output filetype.
    """
    if filetype == "markdown":
        # From http://pandoc.org/README.html#backslash-escapes
        special_chars = "\\`*_{}[]()>#+-.!"
        result = ""
        for c in link_text:
            if c in special_chars:
                result += "\\" + c
            else:
                result += c
        return "[{link_text}]({url})".format(link_text=result, url=url)
    if filetype == "html":
        return '{link_text}'.format(url=url, link_text=html.escape(link_text))
    if filetype == "mediawiki":
        return "[{url} {link_text}]".format(url=url, link_text=link_text)
    if filetype == "latex":
        return ("\\href{%s}{%s}" % (url, link_text))

def get_link_text(url, mime_type, data=None):
    '''
    Take URL, MIME type, and optional data to produce the link text.
    '''
    tld = get_tld(url)
    result = "File on " + tld
    if mime_type.startswith("image"):
        result = "Image on " + tld
    elif  "application/pdf" in mime_type:
        logging.debug("PDF detected")
        # I need seek() for some reason so convert from bytes
        data = io.BytesIO(data)
        # fix this later, but I always get a "PdfReadWarning: Xref table
        # not zero-indexed" which should only happen when the -v flag is
        # present
        import warnings
        warnings.filterwarnings("ignore")
        pdf = PdfFileReader(data, strict=True)
        # PyPDF2 somehow thinks many PDFs are encrypted with the empty
        # string, so deal with that
        if pdf.isEncrypted:
            pdf.decrypt('')
        result = pdf.getDocumentInfo().title
        #result = "PDF on " + tld
    elif "text/html" in mime_type:
        try:
            soup = BeautifulSoup(data, 'html.parser')
            meta = soup.find_all("meta")
            og_title_lst = []
            twitter_title_lst = []
            meta_title_lst = []
            for i in meta:
                if i.get("property") == "og:title":
                    og_title_lst.append(i.get("content"))
                elif i.get("property") == "twitter:title":
                    twitter_title_lst.append(i.get("content"))
                elif i.get("name") == "title":
                    meta_title_lst.append(i.get("content"))
            if og_title_lst:
                logging.debug("found og:title")
                result = og_title_lst[0].strip()
            elif twitter_title_lst:
                logging.debug("found twitter title")
                result = twitter_title_lst[0].strip()
            elif meta_title_lst:
                logging.debug("found meta name title")
                result = meta_title_lst[0].strip()
            elif soup.title and soup.title.string:
                logging.debug("found title tag")
                result = messy_title_parse(html.unescape(
                    soup.title.string))
            else:
                logging.debug("no title found; using default")
                result = "Page on " + tld
        except AttributeError:
            # Probably just empty title when trying to get
            # soup.title.string
            logging.debug("FIXME: this isn't supposed to happen")
            result = "Page on " + tld
    if len(result) > 255:
        result = result[:253] + " …"

    return result

def messy_title_parse(title, url=None):
    # Even if nothing works, at least we'll have a whitespace-sanitized
    # title
    logging.debug("Cleaning messy title")
    result = title.strip()
    hyphen_split = result.split(" - ")
    bar_split = result.split(" | ")
    em_dash_split = result.split(" — ")
    colon_split = result.split(": ")
    if len(hyphen_split) > 1:
        # So there is actually more than one part, so we just take the
        # first and we're done.  This is for titles like "Post Title -
        # Site Name"
        result = hyphen_split[0]
    elif len(em_dash_split) > 1:
        result = em_dash_split[0]
    elif len(bar_split) > 1:
        result = bar_split[0]
    elif len(colon_split) > 1:
        # For titles like "Site Name: Post Title"
        result = colon_split[-1]
    return result

if __name__ == "__main__":
    main()
<|endoftext|>


<commit_before>#!/usr/bin/python3
# -*- coding: utf-8 -*-

import argparse
import io
import logging
import warnings
import sys
import requests
import urllib3
import tld
from tld import get_tld
from bs4 import BeautifulSoup
import html
import PyPDF2
from PyPDF2 import PdfFileReader, PdfFileWriter

def main():
    # See https://blog.quora.com/Launched-Customizable-Links for Quora's
    # launch post
    parser = argparse.ArgumentParser(description=("Get the linked title of " +
        "URLs (similar to Quora and Facebook)"))
    parser.add_argument("url", type=str, help="the URL")
    parser.add_argument("-f", "--format", type=str,
            help=("output format; accepted values are 'none', 'html', " +
            "'markdown', 'latex', 'mediawiki'"))
    parser.add_argument("-c", "--clean", action="store_true",
            help=("clean the title to remove the site name " +
            "if the title was obtained from an HTML title tag"))
    parser.add_argument("-v", "--verbose", action="store_const",
        dest="log_level", const=logging.DEBUG, help="enable debug messages")
    args = parser.parse_args()
    logging.basicConfig(level=args.log_level)

    url = args.url
    logging.debug("Trying first attempt")
    attempt_1 = try_url(url, args.format, clean=args.clean)
    if attempt_1["exit"]:
        logging.debug("First attempt succeeded!")
        print(attempt_1["text"], end="")
    else:
        logging.debug("First attempt failed; trying second attempt")
        attempt_2 = try_url("http://" + url, args.format, clean=args.clean)
        if attempt_2["exit"]:
            logging.debug("Second attempt succeeded!")
            print(attempt_2["text"], end="")
        else:
            logging.debug("Second attempted failed; using result " +
                "from first attempt")
            print(attempt_1["text"], end="")

def try_url(url, fmt, clean=False):
    '''
    Return (Str, True) if succeeded; (Str, False) otherwise.
    '''
    result = {}
    analyte = analyze_url(url)
    if analyte:
        return {"text": analyte, "exit": True}
    try:
        user_agent = ("Mozilla/5.0 (X11; Linux i686; rv:31.0) Gecko/20100101 " +
            "Firefox/31.0 Iceweasel/31.5.0")
        headers = {"User-Agent": user_agent}
        response = requests.get(url, stream=True, headers=headers)
        url = response.url
        if  ("text/html" in response.headers["content-type"] or
                "application/pdf" in response.headers["content-type"]):
            logging.debug("HTML page or PDF file detected")
            if "text/html" in response.headers["content-type"]:
                # <title> is probably in the first around 10MB of HTML
                # files, so we can download less here than in the case
                # for PDFs
                logging.debug("HTML detected")
                doc = response.iter_content(chunk_size=10000)
            else:
                # PDFs might require more downloading
                logging.debug("PDF detected")
                doc = response.iter_content(chunk_size=1000000)
            data = next(doc)
            result["text"] = get_filetype_link(
                get_link_text(url, response.headers["content-type"], data=data,
                    clean=clean),
                url,
                fmt
            )
        else:
            logging.debug("No HTML page detected")
            result["text"] = get_filetype_link(
                get_link_text(url, response.headers["content-type"]),
                url,
                fmt
            )
        result["exit"] = True
    except (requests.exceptions.MissingSchema,
            requests.exceptions.ConnectionError,
            requests.exceptions.InvalidSchema,
            requests.exceptions.InvalidURL,
            urllib3.exceptions.LocationParseError):
        # since it's not a valid URL, just return it
        result["text"] = url
        result["exit"] = False
    return result

def analyze_url(url):
    """
    Look just at the URL to see if a suitable title text can be found.  This
    method is much faster than actually visiting the URL to find the title
    element in the downloaded file. We want to do this for special sites like
    Facebook, which doesn't allow anonymous downloading of certain pages, like
    group pages.

    Args:
        url: A string that is a URL

    Returns:
        A string that is the title text to be used. If no suitable title text
        can be produced, return the empty string, "".
    """
    try:
        tl = get_tld(url)
    except (tld.exceptions.TldBadUrl, tld.exceptions.TldDomainNotFound):
        logging.debug("bad TLD; trying with http:// prefix")
        try:
            tl = get_tld("http://" + url)
        except (tld.exceptions.TldBadUrl, tld.exceptions.TldDomainNotFound):
            logging.debug("still bad TLD; giving up")
            return ""
    if tl == "facebook.com" and "facebook.com/groups/" in url:
            return "Facebook group page post"
    return ""

def get_filetype_link(link_text, url, filetype):
    """
    Put the pieces together and produce a valid hyperlink for given output
    filetype.

    Args:
        link_text: A string representing the displayed or linked text when
            linking to something, e.g. "hello" in hello. This string should already be in
            the intended form; i.e. all HTML escapes should have been unescaped
            at this point.
        url: A string of the URL.
        filetype: A string of the output filetype. Accepted parameters are:
            "none", "html", "markdown", "latex".

    Returns:
        A string that is a valid hyperlink for the specified output filetype.
    """
    if filetype == "markdown":
        # From http://pandoc.org/README.html#backslash-escapes
        # There is also the hyphen, "-", but I've removed that since
        # escaping it just prevents em- and en-dashes from forming (and
        # in most cases, this is what one wants)
        special_chars = "\\`*_{}[]()>#+.!"
        result = ""
        for c in link_text:
            if c in special_chars:
                result += "\\" + c
            else:
                result += c
        return "[{link_text}]({url})".format(link_text=result, url=url)
    if filetype == "html":
        return '{link_text}'.format(url=url,
                link_text=html.escape(link_text))
    if filetype == "mediawiki":
        return "[{url} {link_text}]".format(url=url,
                link_text=link_text)
    if filetype == "latex":
        # LaTeX is really sensitive about special characters so this
        # probably needs a lot of tweaking
        special_chars = "$&%{_#"
        result = ""
        for c in link_text:
            if c in special_chars:
                result += "\\" + c
            elif c == "\\":
                result += "\\textbackslash{}"
            elif c == "~":
                result += "\\textasciitilde{}"
            else:
                result += c
        clean_url = ""
        for c in url:
            if c in special_chars or c in "~":
                clean_url += "\\" + c
            elif c == "\\":
                clean_url += "{\\textbackslash}"
            else:
                clean_url += c
        return ("\\href{%s}{%s}" % (clean_url, result))
    else:
        return "{link_text}: {url}".format(url=url, link_text=link_text)

def get_link_text(url, mime_type, data=None, clean=False):
    '''
    Take URL, MIME type, and optional data to produce the link text.
    '''
    tld = get_tld(url)
    result = "File on " + tld
    if mime_type.startswith("image"):
        result = "Image on " + tld
    elif  "application/pdf" in mime_type:
        logging.debug("PDF detected")
        # I need seek() for some reason so convert from bytes
        data = io.BytesIO(data)
        # fix this later, but I always get a "PdfReadWarning: Xref table
        # not zero-indexed" which should only happen when the -v flag is
        # present
        warnings.filterwarnings("ignore")
        try:
            pdf = PdfFileReader(data, strict=True)
            # PyPDF2 somehow thinks many PDFs are encrypted with the empty
            # string, so deal with that
            if pdf.isEncrypted:
                pdf.decrypt('')
            result = pdf.getDocumentInfo().title
            if not result or result.strip() == "":
                result = "PDF on " + tld
        except PyPDF2.utils.PdfReadError:
            result = "PDF on " + tld
    elif "text/html" in mime_type:
        try:
            soup = BeautifulSoup(data, 'html.parser')
            meta = soup.find_all("meta")
            og_title_lst = []
            twitter_title_lst = []
            meta_title_lst = []
            schema_lst = []
            for i in meta:
                if i.get("property") == "og:title":
                    og_title_lst.append(i.get("content"))
                elif i.get("property") == "twitter:title":
                    twitter_title_lst.append(i.get("content"))
                elif i.get("name") == "title":
                    meta_title_lst.append(i.get("content"))
                elif i.get("itemprop") == "name":
                    schema_lst.append(i.get("content"))
            if og_title_lst:
                logging.debug("found og:title")
                result = og_title_lst[0].strip()
            elif twitter_title_lst:
                logging.debug("found twitter title")
                result = twitter_title_lst[0].strip()
            elif meta_title_lst:
                logging.debug("found meta name title")
                result = meta_title_lst[0].strip()
                if clean:
                    result = messy_title_parse(result, url)
            elif schema_lst:
                logging.debug("found schema title")
                result = schema_lst[0].strip()
            elif soup.title and soup.title.string:
                logging.debug("found title tag")
                result = html.unescape(soup.title.string)
                if clean:
                    result = messy_title_parse(result, url)
            else:
                logging.debug("no title found; using default")
                result = "Page on " + tld
        except AttributeError:
            # Probably just empty title when trying to get
            # soup.title.string
            logging.debug("FIXME: this isn't supposed to happen")
            result = "Page on " + tld
    if len(result) > 255:
        result = result[:253] + " …"

    return result

def messy_title_parse(title, url=None):
    # Even if nothing works, at least we'll have a whitespace-sanitized
    # title
    logging.debug("Cleaning messy title")
    result = title.strip()
    hyphen_split = result.split(" - ")
    bar_split = result.split(" | ")
    em_dash_split = result.split(" — ")
    colon_split = result.split(": ")
    if get_tld(url) in ["autoadmit.com", "xoxohth.com"]:
        return " - ".join(hyphen_split[1:])
    if len(hyphen_split) > 1:
        # So there is actually more than one part, so we just take the
        # first and we're done.  This is for titles like "Post Title -
        # Site Name"
        result = hyphen_split[0]
    elif len(em_dash_split) > 1:
        result = em_dash_split[0]
    elif len(bar_split) > 1:
        result = bar_split[0]
    elif len(colon_split) > 1:
        # For titles like "Site Name: Post Title"
        result = colon_split[-1]
    return result

if __name__ == "__main__":
    main()
<commit_msg>allow tex as a format (equivalent to latex)<commit_after>#!/usr/bin/python3
# -*- coding: utf-8 -*-

import argparse
import io
import logging
import warnings
import sys
import requests
import urllib3
import tld
from tld import get_tld
from bs4 import BeautifulSoup
import html
import PyPDF2
from PyPDF2 import PdfFileReader, PdfFileWriter

def main():
    # See https://blog.quora.com/Launched-Customizable-Links for Quora's
    # launch post
    parser = argparse.ArgumentParser(description=("Get the linked title of " +
        "URLs (similar to Quora and Facebook)"))
    parser.add_argument("url", type=str, help="the URL")
    parser.add_argument("-f", "--format", type=str,
            help=("output format; accepted values are 'none', 'html', " +
            "'markdown', 'tex', 'latex', 'mediawiki'"))
    parser.add_argument("-c", "--clean", action="store_true",
            help=("clean the title to remove the site name " +
            "if the title was obtained from an HTML title tag"))
    parser.add_argument("-v", "--verbose", action="store_const",
        dest="log_level", const=logging.DEBUG, help="enable debug messages")
    args = parser.parse_args()
    logging.basicConfig(level=args.log_level)

    url = args.url
    logging.debug("Trying first attempt")
    attempt_1 = try_url(url, args.format, clean=args.clean)
    if attempt_1["exit"]:
        logging.debug("First attempt succeeded!")
        print(attempt_1["text"], end="")
    else:
        logging.debug("First attempt failed; trying second attempt")
        attempt_2 = try_url("http://" + url, args.format, clean=args.clean)
        if attempt_2["exit"]:
            logging.debug("Second attempt succeeded!")
            print(attempt_2["text"], end="")
        else:
            logging.debug("Second attempted failed; using result " +
                "from first attempt")
            print(attempt_1["text"], end="")

def try_url(url, fmt, clean=False):
    '''
    Return (Str, True) if succeeded; (Str, False) otherwise.
    '''
    result = {}
    analyte = analyze_url(url)
    if analyte:
        return {"text": analyte, "exit": True}
    try:
        user_agent = ("Mozilla/5.0 (X11; Linux i686; rv:31.0) Gecko/20100101 " +
            "Firefox/31.0 Iceweasel/31.5.0")
        headers = {"User-Agent": user_agent}
        response = requests.get(url, stream=True, headers=headers)
        url = response.url
        if  ("text/html" in response.headers["content-type"] or
                "application/pdf" in response.headers["content-type"]):
            logging.debug("HTML page or PDF file detected")
            if "text/html" in response.headers["content-type"]:
                # <title> is probably in the first around 10MB of HTML
                # files, so we can download less here than in the case
                # for PDFs
                logging.debug("HTML detected")
                doc = response.iter_content(chunk_size=10000)
            else:
                # PDFs might require more downloading
                logging.debug("PDF detected")
                doc = response.iter_content(chunk_size=1000000)
            data = next(doc)
            result["text"] = get_filetype_link(
                get_link_text(url, response.headers["content-type"], data=data,
                    clean=clean),
                url,
                fmt
            )
        else:
            logging.debug("No HTML page detected")
            result["text"] = get_filetype_link(
                get_link_text(url, response.headers["content-type"]),
                url,
                fmt
            )
        result["exit"] = True
    except (requests.exceptions.MissingSchema,
            requests.exceptions.ConnectionError,
            requests.exceptions.InvalidSchema,
            requests.exceptions.InvalidURL,
            urllib3.exceptions.LocationParseError):
        # since it's not a valid URL, just return it
        result["text"] = url
        result["exit"] = False
    return result

def analyze_url(url):
    """
    Look just at the URL to see if a suitable title text can be found.  This
    method is much faster than actually visiting the URL to find the title
    element in the downloaded file. We want to do this for special sites like
    Facebook, which doesn't allow anonymous downloading of certain pages, like
    group pages.

    Args:
        url: A string that is a URL

    Returns:
        A string that is the title text to be used. If no suitable title text
        can be produced, return the empty string, "".
    """
    try:
        tl = get_tld(url)
    except (tld.exceptions.TldBadUrl, tld.exceptions.TldDomainNotFound):
        logging.debug("bad TLD; trying with http:// prefix")
        try:
            tl = get_tld("http://" + url)
        except (tld.exceptions.TldBadUrl, tld.exceptions.TldDomainNotFound):
            logging.debug("still bad TLD; giving up")
            return ""
    if tl == "facebook.com" and "facebook.com/groups/" in url:
            return "Facebook group page post"
    return ""

def get_filetype_link(link_text, url, filetype):
    """
    Put the pieces together and produce a valid hyperlink for given output
    filetype.

    Args:
        link_text: A string representing the displayed or linked text when
            linking to something, e.g. "hello" in hello. This string should already be in
            the intended form; i.e. all HTML escapes should have been unescaped
            at this point.
        url: A string of the URL.
        filetype: A string of the output filetype. Accepted parameters are:
            "none", "html", "markdown", "tex", "latex".

    Returns:
        A string that is a valid hyperlink for the specified output filetype.
    """
    if filetype == "markdown":
        # From http://pandoc.org/README.html#backslash-escapes
        # There is also the hyphen, "-", but I've removed that since
        # escaping it just prevents em- and en-dashes from forming (and
        # in most cases, this is what one wants)
        special_chars = "\\`*_{}[]()>#+.!"
        result = ""
        for c in link_text:
            if c in special_chars:
                result += "\\" + c
            else:
                result += c
        return "[{link_text}]({url})".format(link_text=result, url=url)
    if filetype == "html":
        return '{link_text}'.format(url=url,
                link_text=html.escape(link_text))
    if filetype == "mediawiki":
        return "[{url} {link_text}]".format(url=url,
                link_text=link_text)
    if filetype in ["latex", "tex"]:
        # LaTeX is really sensitive about special characters so this
        # probably needs a lot of tweaking
        special_chars = "$&%{_#"
        result = ""
        for c in link_text:
            if c in special_chars:
                result += "\\" + c
            elif c == "\\":
                result += "\\textbackslash{}"
            elif c == "~":
                result += "\\textasciitilde{}"
            else:
                result += c
        clean_url = ""
        for c in url:
            if c in special_chars or c in "~":
                clean_url += "\\" + c
            elif c == "\\":
                clean_url += "{\\textbackslash}"
            else:
                clean_url += c
        return ("\\href{%s}{%s}" % (clean_url, result))
    else:
        return "{link_text}: {url}".format(url=url, link_text=link_text)

def get_link_text(url, mime_type, data=None, clean=False):
    '''
    Take URL, MIME type, and optional data to produce the link text.
    '''
    tld = get_tld(url)
    result = "File on " + tld
    if mime_type.startswith("image"):
        result = "Image on " + tld
    elif  "application/pdf" in mime_type:
        logging.debug("PDF detected")
        # I need seek() for some reason so convert from bytes
        data = io.BytesIO(data)
        # fix this later, but I always get a "PdfReadWarning: Xref table
        # not zero-indexed" which should only happen when the -v flag is
        # present
        warnings.filterwarnings("ignore")
        try:
            pdf = PdfFileReader(data, strict=True)
            # PyPDF2 somehow thinks many PDFs are encrypted with the empty
            # string, so deal with that
            if pdf.isEncrypted:
                pdf.decrypt('')
            result = pdf.getDocumentInfo().title
            if not result or result.strip() == "":
                result = "PDF on " + tld
        except PyPDF2.utils.PdfReadError:
            result = "PDF on " + tld
    elif "text/html" in mime_type:
        try:
            soup = BeautifulSoup(data, 'html.parser')
            meta = soup.find_all("meta")
            og_title_lst = []
            twitter_title_lst = []
            meta_title_lst = []
            schema_lst = []
            for i in meta:
                if i.get("property") == "og:title":
                    og_title_lst.append(i.get("content"))
                elif i.get("property") == "twitter:title":
                    twitter_title_lst.append(i.get("content"))
                elif i.get("name") == "title":
                    meta_title_lst.append(i.get("content"))
                elif i.get("itemprop") == "name":
                    schema_lst.append(i.get("content"))
            if og_title_lst:
                logging.debug("found og:title")
                result = og_title_lst[0].strip()
            elif twitter_title_lst:
                logging.debug("found twitter title")
                result = twitter_title_lst[0].strip()
            elif meta_title_lst:
                logging.debug("found meta name title")
                result = meta_title_lst[0].strip()
                if clean:
                    result = messy_title_parse(result, url)
            elif schema_lst:
                logging.debug("found schema title")
                result = schema_lst[0].strip()
            elif soup.title and soup.title.string:
                logging.debug("found title tag")
                result = html.unescape(soup.title.string)
                if clean:
                    result = messy_title_parse(result, url)
            else:
                logging.debug("no title found; using default")
                result = "Page on " + tld
        except AttributeError:
            # Probably just empty title when trying to get
            # soup.title.string
            logging.debug("FIXME: this isn't supposed to happen")
            result = "Page on " + tld
    if len(result) > 255:
        result = result[:253] + " …"

    return result

def messy_title_parse(title, url=None):
    # Even if nothing works, at least we'll have a whitespace-sanitized
    # title
    logging.debug("Cleaning messy title")
    result = title.strip()
    hyphen_split = result.split(" - ")
    bar_split = result.split(" | ")
    em_dash_split = result.split(" — ")
    colon_split = result.split(": ")
    if get_tld(url) in ["autoadmit.com", "xoxohth.com"]:
        return " - ".join(hyphen_split[1:])
    if len(hyphen_split) > 1:
        # So there is actually more than one part, so we just take the
        # first and we're done.  This is for titles like "Post Title -
        # Site Name"
        result = hyphen_split[0]
    elif len(em_dash_split) > 1:
        result = em_dash_split[0]
    elif len(bar_split) > 1:
        result = bar_split[0]
    elif len(colon_split) > 1:
        # For titles like "Site Name: Post Title"
        result = colon_split[-1]
    return result

if __name__ == "__main__":
    main()
<|endoftext|>


<commit_before><commit_msg>NEW: script to compute fractional volume per label<commit_after><|endoftext|>


<commit_before>#!/bin/python3
import csv
import sys
import math
from collections import OrderedDict
from graphviz import Digraph
from cyclefind import CycleFind

class CycleDetect(object):
    def __init__(self):
        self.reset()

    def reset(self):
        self.graph = OrderedDict()
        self.origins = []
        self.delay = OrderedDict()
        self.limit = OrderedDict()
        self.cyclelimit = None

    def add_link(self, f, t, v, d=None, l=None):
        if f not in self.graph:
            self.graph[f] = {}
        if f not in self.delay:
            self.delay[f] = {}
        if f not in self.limit:
            self.limit[f] = {}
        if t not in self.graph[f]:
            self.graph[f][t] = 0.0
        self.graph[f][t] += v
        if d is not None:
            if t not in self.delay[f]:
                self.delay[f][t] = 0.0
            self.delay[f][t] += d
        if l is not None:
            if t not in self.limit[f]:
                self.limit[f][t] = l
            else:
                self.limit[f][t] = min(self.limit[f][t], l)

    def load(self, fplist):
        for fp in fplist:
            reader = csv.reader(fp, delimiter=',', quotechar='"')
            for row in reader:
                if len(row) == 0:
                    continue
                lformat = row[0]
                if len(row[0]) == 0 or row[0][0] == '#':
                    continue
                if lformat == 'origin':
                    f = row[1]
                    if f not in self.origins:
                        self.origins.append(f)
                if lformat == 'link':
                    f = row[1]
                    t = row[2]
                    v = float(row[3])
                    self.add_link(f, t, v)
                elif lformat == 'cycle-limit':
                    self.cyclelimit = int(row[1])
                elif lformat == 'node':
                    f = row[1]
                    if f not in self.graph:
                        self.graph[f] = {}
                elif lformat == 'bid-ask':
                    f = row[1]
                    t = row[2]
                    b = math.log(float(row[3]))
                    a = math.log(float(row[4]))
                    if len(row) > 5 and row[5] != "":
                        d = float(row[5])
                    else:
                        d = None
                    if len(row) > 6 and row[6] != "":
                        l = float(row[6])
                    else:
                        l = None
                    self.add_link(f, t, -b, d, l)
                    self.add_link(t, f, a, d, l)
                elif lformat == 'fee':
                    f = row[1]
                    t = row[2]
                    v = math.log(1.0 - float(row[3])/100.0)
                    if len(row) > 4 and row[4] != "":
                        d = float(row[4])
                    else:
                        d = None
                    if len(row) > 5 and row[5] != "":
                        l = float(row[5])
                    else:
                        l = None
                    self.add_link(f, t, -v, d, l)

    def graphviz(self, fp=None):
        if fp is not None:
            self.load(fp);
        nl = self.run()
        show = {}
        dot = Digraph(format='png')
        for ik, iv in self.graph.items():
            ikid = ik.replace(":", "__")
            dot.node(ikid, ik)
        for i in nl:
            i1 = i[0]
            for j in zip(i1, i1[1::]):
                ikid = j[0].replace(":", "__")
                jkid = j[1].replace(":", "__")
                if (ikid, jkid) not in show:
                    show[(ikid, jkid)] = True
                    dot.edge(ikid, jkid, color='red')
        for ik, iv in self.graph.items():
            for jk, jv in iv.items():
                ikid = ik.replace(":", "__")
                jkid = jk.replace(":", "__")
                if (ikid, jkid) not in show:
                    dot.edge(ikid, jkid)
        return dot.pipe()

    def run(self, fp=None):
        if fp is not None:
            self.load(fp)
        retval = []
        cf = CycleFind(self.graph, self.origins, cyclelimit=self.cyclelimit)
        cycles = cf.run()
        print("cycles", len(cycles))
        negative_cycle_lists = cf.filter_negative(cycles)
        print("neg-cycle", negative_cycle_lists)
        for i in negative_cycle_lists:
            total = 0.0
            d = 0.0
            l = None
            i1 = list(i)
            i1.append(i[0])
            for j in zip(i1, i1[1::]):
                total -= self.graph[j[0]][j[1]]
                d += self.delay[j[0]].get(j[1], 0.0)
                new_limit = self.limit[j[0]].get(j[1], None)
                if new_limit is not None:
                    if l is None or new_limit < l:
                        l = new_limit
            retval.append([i1, math.exp(total), d, l])
        return retval

    def format(self, cycle_list):
        retval = ''
        for i in cycle_list:
            retval += " -> ".join(i[0]) + "\n"
            retval += "Expected return: " + str(i[1]) + "\n"
            retval += "Delay: " + str(i[2]) + "\n"
            if i[3] is not None:
                retval += "Limit: " + str(i[3]) + "\n"
            retval += "\n"
        return retval

if __name__ == '__main__':
    cd = CycleDetect()
    for i in sys.argv[1:]:
        with open(i, 'r') as csvfile:
            print("Loading: ", i)
            cd.load([csvfile])
    cycles = cd.run()
    print(cycles)
    print(cd.format(cycles))
<commit_msg>add fixes<commit_after>#!/bin/python3
import csv
import sys
import math
from collections import OrderedDict
from graphviz import Digraph
from cyclefind import CycleFind

class CycleDetect(object):
    def __init__(self):
        self.reset()

    def reset(self):
        self.graph = OrderedDict()
        self.origins = []
        self.delay = OrderedDict()
        self.limit = OrderedDict()
        self.cyclelimit = None

    def add_link(self, f, t, v, d=None, l=None):
        if f not in self.graph:
            self.graph[f] = {}
        if f not in self.delay:
            self.delay[f] = {}
        if f not in self.limit:
            self.limit[f] = {}
        if t not in self.graph[f]:
            self.graph[f][t] = 0.0
        self.graph[f][t] += v
        if d is not None:
            if t not in self.delay[f]:
                self.delay[f][t] = 0.0
            self.delay[f][t] += d
        if l is not None:
            if t not in self.limit[f]:
                self.limit[f][t] = l
            else:
                self.limit[f][t] = min(self.limit[f][t], l)

    def load(self, fplist):
        tfee = []
        for fp in fplist:
            reader = csv.reader(fp, delimiter=',', quotechar='"')
            for row in reader:
                if len(row) == 0:
                    continue
                lformat = row[0]
                if len(row[0]) == 0 or row[0][0] == '#':
                    continue
                if lformat == 'origin':
                    f = row[1]
                    if f not in self.origins:
                        self.origins.append(f)
                if lformat == 'link':
                    f = row[1]
                    t = row[2]
                    v = float(row[3])
                    self.add_link(f, t, v)
                elif lformat == 'cycle-limit':
                    self.cyclelimit = int(row[1])
                elif lformat == 'node':
                    f = row[1]
                    if f not in self.graph:
                        self.graph[f] = {}
                elif lformat == 'bid-ask':
                    f = row[1]
                    t = row[2]
                    b = math.log(float(row[3]))
                    a = math.log(float(row[4]))
                    if len(row) > 5 and row[5] != "":
                        d = float(row[5])
                    else:
                        d = None
                    if len(row) > 6 and row[6] != "":
                        l = float(row[6])
                    else:
                        l = None
                    self.add_link(f, t, -b, d, l)
                    self.add_link(t, f, a, d, l)
                elif lformat == 'fee':
                    f = row[1]
                    t = row[2]
                    v = math.log(1.0 - float(row[3])/100.0)
                    if len(row) > 4 and row[4] != "":
                        d = float(row[4])
                    else:
                        d = None
                    if len(row) > 5 and row[5] != "":
                        l = float(row[5])
                    else:
                        l = None
                    self.add_link(f, t, -v, d, l)

    def graphviz(self, fp=None):
        if fp is not None:
            self.load(fp);
        nl = self.run()
        show = {}
        dot = Digraph(format='png')
        for ik, iv in self.graph.items():
            ikid = ik.replace(":", "__")
            dot.node(ikid, ik)
        for i in nl:
            i1 = i[0]
            for j in zip(i1, i1[1::]):
                ikid = j[0].replace(":", "__")
                jkid = j[1].replace(":", "__")
                if (ikid, jkid) not in show:
                    show[(ikid, jkid)] = True
                    dot.edge(ikid, jkid, color='red')
        for ik, iv in self.graph.items():
            for jk, jv in iv.items():
                ikid = ik.replace(":", "__")
                jkid = jk.replace(":", "__")
                if (ikid, jkid) not in show:
                    dot.edge(ikid, jkid)
        return dot.pipe()

    def run(self, fp=None):
        if fp is not None:
            self.load(fp)
        retval = []
        cf = CycleFind(self.graph, self.origins, cyclelimit=self.cyclelimit)
        cycles = cf.run()
        negative_cycle_lists = cf.filter_negative(cycles)
        for i in negative_cycle_lists:
            total = 0.0
            d = 0.0
            l = None
            i1 = list(i)
            i1.append(i[0])
            for j in zip(i1, i1[1::]):
                total -= self.graph[j[0]][j[1]]
                d += self.delay[j[0]].get(j[1], 0.0)
                new_limit = self.limit[j[0]].get(j[1], None)
                if new_limit is not None:
                    if l is None or new_limit < l:
                        l = new_limit
            retval.append([i1, math.exp(total), d, l])
        return retval

    def format(self, cycle_list):
        retval = ''
        for i in cycle_list:
            retval += " -> ".join(i[0]) + "\n"
            retval += "Expected return: " + str(i[1]) + "\n"
            retval += "Delay: " + str(i[2]) + "\n"
            if i[3] is not None:
                retval += "Limit: " + str(i[3]) + "\n"
            retval += "\n"
        return retval

if __name__ == '__main__':
    cd = CycleDetect()
    for i in sys.argv[1:]:
        with open(i, 'r') as csvfile:
            print("Loading: ", i)
            cd.load([csvfile])
    cycles = cd.run()
    print(cycles)
    print(cd.format(cycles))
<|endoftext|>


<commit_before>from math import sqrt, pi, sin, cos, atan2
import random
import itertools
import math
from typing import List, Dict, Set, Tuple, Any, Optional, Union # for mypy type checking

FLOAT_DIGITS = 8
EPSILON = 10**(-FLOAT_DIGITS)

def _sign(num):
    if num == 0:
        return 0
    return 1 if num > 0 else -1

class Pointlike(tuple):
    @property
    def rounded(self) -> "Pointlike":
        return self.__class__(round(q) for q in self)

    @property
    def position(self) -> "Pointlike":
        return self

    def distance_to(self, p: Union["Unit", "Point2", "Point3"]) -> Union[int, float]:
        p = p.position
        assert isinstance(p, Pointlike)
        if self == p:
            return 0
        return sqrt(sum(self.__class__((b-a)**2 for a, b in itertools.zip_longest(self, p, fillvalue=0))))

    def distance_to_point2(self, p2: "Point2") -> Union[int, float]:
        """ Same as the function above, but should be 3-4 times faster because of the dropped asserts and conversions and because it doesnt use a loop (itertools or zip). """
        return ((self[0]-p2[0])**2 + (self[1]-p2[1])**2)**0.5

    def _distance_squared(self, p2: "Point2") -> Union[int, float]:
        """ Function used to not take the square root as the distances will stay proportionally the same. This is to speed up the sorting process. """
        return ((self[0]-p2[0])**2 + (self[1]-p2[1])**2)

    def sort_by_distance(self, ps: Union["Units", List["Point2"]]):
        """ This returns the target points sorted. You should not pass a set or dict since those are not sortable. """
        if ps and all(isinstance(p, Point2) for p in ps):
            return sorted(ps, key=lambda p: self._distance_squared(p))
        return sorted(ps, key=lambda p: self.distance_to(p))

    def closest(self, ps: Union["Units", List["Point2"], Set["Point2"]]) -> Union["Unit", "Point2"]:
        """ This function assumes the 2d distance is meant """
        assert len(ps) > 0
        closest_distance_squared = math.inf
        for p2 in ps:
            p2pos = p2
            if not isinstance(p2pos, Point2):
                p2pos = p2.position
            distance = (self[0] - p2pos[0])**2 + (self[1] - p2pos[1])**2
            if distance < closest_distance_squared:
                closest_distance_squared = distance
                closest_element = p2
        return closest_element

    def distance_to_closest(self, ps: Union["Units", List["Point2"], Set["Point2"]]) -> Union[int, float]:
        """ This function assumes the 2d distance is meant """
        assert len(ps) > 0
        closest_distance_squared = math.inf
        for p2 in ps:
            if not isinstance(p2, Point2):
                p2 = p2.position
            distance = (self[0] - p2[0])**2 + (self[1] - p2[1])**2
            if distance < closest_distance_squared:
                closest_distance_squared = distance
        return closest_distance_squared**0.5

    def furthest(self, ps: Union["Units", List["Point2"], Set["Point2"]]) -> Union["Unit", "Pointlike"]:
        """ This function assumes the 2d distance is meant """
        assert len(ps) > 0
        furthest_distance_squared = 0
        for p2 in ps:
            p2pos = p2
            if not isinstance(p2pos, Point2):
                p2pos = p2.position
            distance = (self[0] - p2pos[0])**2 + (self[1] - p2pos[1])**2
            if furthest_distance_squared < distance:
                furthest_distance_squared = distance
                furthest_element = p2
        return furthest_element

    def distance_to_furthest(self, ps: Union["Units", List["Point2"], Set["Point2"]]) -> Union[int, float]:
        """ This function assumes the 2d distance is meant """
        assert len(ps) > 0
        furthest_distance_squared = 0
        for p2 in ps:
            if not isinstance(p2, Point2):
                p2 = p2.position
            distance = (self[0] - p2[0])**2 + (self[1] - p2[1])**2
            if furthest_distance_squared < distance:
                furthest_distance_squared = distance
        return furthest_distance_squared**0.5

    def offset(self, p) -> "Pointlike":
        return self.__class__(a+b for a, b in itertools.zip_longest(self, p[:len(self)], fillvalue=0))

    def unit_axes_towards(self, p):
        return self.__class__(_sign(b - a) for a, b in itertools.zip_longest(self, p[:len(self)], fillvalue=0))

    def towards(self, p: Union["Unit", "Pointlike"], distance: Union[int, float]=1, limit: bool=False) -> "Pointlike":
        assert self != p
        d = self.distance_to(p)
        if limit:
            distance = min(d, distance)
        return self.__class__(a + (b - a) / d * distance for a, b in itertools.zip_longest(self, p[:len(self)], fillvalue=0))

    def __eq__(self, other):
        if not isinstance(other, tuple):
            return False
        return all(abs(a - b) < EPSILON for a, b in itertools.zip_longest(self, other, fillvalue=0))

    def __hash__(self):
        return hash(tuple(int(c * FLOAT_DIGITS)  for c in self))

class Point2(Pointlike):
    @classmethod
    def from_proto(cls, data):
        return cls((data.x, data.y))

    @property
    def x(self) -> Union[int, float]:
        return self[0]

    @property
    def y(self) -> Union[int, float]:
        return self[1]

    @property
    def to2(self) -> "Point2":
        return Point2(self[:2])

    @property
    def to3(self) -> "Point3":
        return Point3((*self, 0))

    def random_on_distance(self, distance):
        if isinstance(distance, (tuple, list)): # interval
            distance = distance[0] + random.random() * (distance[1] - distance[0])

        assert distance > 0
        angle = random.random() * 2 * pi

        dx, dy = cos(angle), sin(angle)
        return Point2((self.x + dx * distance, self.y + dy * distance))

    def towards_with_random_angle(self, p: Union["Point2", "Point3"], distance: Union[int, float]=1, max_difference: Union[int, float]=(pi/4)) -> "Point2":
        tx, ty = self.to2.towards(p.to2, 1)
        angle = atan2(ty - self.y, tx - self.x)
        angle = (angle - max_difference) + max_difference * 2 * random.random()
        return Point2((self.x + cos(angle) * distance, self.y + sin(angle) * distance))

    def circle_intersection(self, p: "Point2", r: Union[int, float]) -> Set["Point2"]:
        """ self is point1, p is point2, r is the radius for circles originating in both points
        Used in ramp finding """
        assert self != p
        distanceBetweenPoints = self.distance_to(p)
        assert r > distanceBetweenPoints / 2
        # remaining distance from center towards the intersection, using pythagoras
        remainingDistanceFromCenter = (r**2 - (distanceBetweenPoints/2)**2)**0.5
        # center of both points
        offsetToCenter = Point2(((p.x - self.x) / 2, (p.y - self.y) / 2))
        center = self.offset(offsetToCenter)

        # stretch offset vector in the ratio of remaining distance from center to intersection
        vectorStretchFactor = remainingDistanceFromCenter / (distanceBetweenPoints / 2)
        v = offsetToCenter
        offsetToCenterStretched = Point2((v.x * vectorStretchFactor, v.y * vectorStretchFactor))

        # rotate vector by 90° and -90°
        vectorRotated1 = Point2((offsetToCenterStretched.y, -offsetToCenterStretched.x))
        vectorRotated2 = Point2((-offsetToCenterStretched.y, offsetToCenterStretched.x))
        intersect1 = center.offset(vectorRotated1)
        intersect2 = center.offset(vectorRotated2)
        return {intersect1, intersect2}

    @property
    def neighbors4(self) -> set:
        return {
            Point2((self.x - 1, self.y)),
            Point2((self.x + 1, self.y)),
            Point2((self.x, self.y - 1)),
            Point2((self.x, self.y + 1)),
        }

    @property
    def neighbors8(self) -> set:
        return self.neighbors4 | {
            Point2((self.x - 1, self.y - 1)),
            Point2((self.x - 1, self.y + 1)),
            Point2((self.x + 1, self.y - 1)),
            Point2((self.x + 1, self.y + 1)),
        }

    def negative_offset(self, other: "Point2") -> "Point2":
        return self.__class__((self.x - other.x, self.y - other.y))

    def __add__(self, other: "Point2") -> "Point2":
        return self.offset(other)

    def __sub__(self, other: "Point2") -> "Point2":
        return self.negative_offset(other)

    def __neg__(self) -> "Point2":
        return self.__class__(-a for a in self)

    def __abs__(self) -> Union[int, float]:
        return math.hypot(self.x, self.y)

    def __bool__(self) -> "Float":
        return self.x != 0 or self.y != 0

    def __mul__(self, other: Union[int, float, "Point2"]) -> "Point2":
        if isinstance(other, self.__class__):
            return self.__class__((self.x * other.x, self.y * other.y))
        return self.__class__((self.x * other, self.y * other))

    def __rmul__(self, other: Union[int, float, "Point2"]) -> "Point2":
        return self.__mul__(other)

    def __truediv__(self, other: Union[int, float, "Point2"]) -> "Point2":
        if isinstance(other, self.__class__):
            return self.__class__((self.x / other.x, self.y / other.y))
        return self.__class__((self.x / other, self.y / other))

    def is_same_as(self, other: "Point2", dist=0.1) -> "Bool":
        return self._distance_squared(other) <= dist ** 2

    def square_vector(self, other: "Point2") -> "Point2":
        return self.__class__((_sign(other.x - self.x), _sign(other.y - self.y)))

    def manhattan_distance(self, other: "Point2") -> Union[int, float]:
        return abs(other.x - self.x) + abs(other.y - self.y)

def center_point2(a: [Point2]) -> "Point2":
    s = Point2((0, 0))
    for p in a:
        s += p
    return s / len(a)

class Point3(Point2):
    @classmethod
    def from_proto(cls, data):
        return cls((data.x, data.y, data.z))

    @property
    def z(self) -> Union[int, float]:
        return self[2]

    @property
    def to3(self) -> "Point3":
        return Point3(self)

class Size(Point2):
    @property
    def width(self) -> Union[int, float]:
        return self[0]

    @property
    def height(self) -> Union[int, float]:
        return self[1]

class Rect(tuple):
    @classmethod
    def from_proto(cls, data):
        assert data.p0.x < data.p1.x and data.p0.y < data.p1.y
        return cls((data.p0.x, data.p0.y, data.p1.x - data.p0.x, data.p1.y - data.p0.y))

    @property
    def x(self) -> Union[int, float]:
        return self[0]

    @property
    def y(self) -> Union[int, float]:
        return self[1]

    @property
    def width(self) -> Union[int, float]:
        return self[2]

    @property
    def height(self) -> Union[int, float]:
        return self[3]

    @property
    def size(self) -> Size:
        return Size(self[2], self[3])

    @property
    def center(self) -> Point2:
        return Point2((self.x + self.width / 2, self.y + self.height / 2))

    def offset(self, p):
        return self.__class__((self[0]+p[0], self[1]+p[1], self[2], self[3]))
<commit_msg>Fix Point2 vector functions and operators<commit_after>from math import sqrt, pi, sin, cos, atan2
import random
import itertools
import math
from typing import List, Dict, Set, Tuple, Any, Optional, Union # for mypy type checking

FLOAT_DIGITS = 8
EPSILON = 10**(-FLOAT_DIGITS)

def _sign(num):
    if num == 0:
        return 0
    return 1 if num > 0 else -1

class Pointlike(tuple):
    @property
    def rounded(self) -> "Pointlike":
        return self.__class__(round(q) for q in self)

    @property
    def position(self) -> "Pointlike":
        return self

    def distance_to(self, p: Union["Unit", "Point2", "Point3"]) -> Union[int, float]:
        p = p.position
        assert isinstance(p, Pointlike)
        if self == p:
            return 0
        return sqrt(sum(self.__class__((b-a)**2 for a, b in itertools.zip_longest(self, p, fillvalue=0))))

    def distance_to_point2(self, p2: "Point2") -> Union[int, float]:
        """ Same as the function above, but should be 3-4 times faster because of the dropped asserts and conversions and because it doesnt use a loop (itertools or zip). """
        return ((self[0]-p2[0])**2 + (self[1]-p2[1])**2)**0.5

    def _distance_squared(self, p2: "Point2") -> Union[int, float]:
        """ Function used to not take the square root as the distances will stay proportionally the same. This is to speed up the sorting process. """
        return ((self[0]-p2[0])**2 + (self[1]-p2[1])**2)

    def sort_by_distance(self, ps: Union["Units", List["Point2"]]):
        """ This returns the target points sorted. You should not pass a set or dict since those are not sortable. """
        if ps and all(isinstance(p, Point2) for p in ps):
            return sorted(ps, key=lambda p: self._distance_squared(p))
        return sorted(ps, key=lambda p: self.distance_to(p))

    def closest(self, ps: Union["Units", List["Point2"], Set["Point2"]]) -> Union["Unit", "Point2"]:
        """ This function assumes the 2d distance is meant """
        assert len(ps) > 0
        closest_distance_squared = math.inf
        for p2 in ps:
            p2pos = p2
            if not isinstance(p2pos, Point2):
                p2pos = p2.position
            distance = (self[0] - p2pos[0])**2 + (self[1] - p2pos[1])**2
            if distance < closest_distance_squared:
                closest_distance_squared = distance
                closest_element = p2
        return closest_element

    def distance_to_closest(self, ps: Union["Units", List["Point2"], Set["Point2"]]) -> Union[int, float]:
        """ This function assumes the 2d distance is meant """
        assert len(ps) > 0
        closest_distance_squared = math.inf
        for p2 in ps:
            if not isinstance(p2, Point2):
                p2 = p2.position
            distance = (self[0] - p2[0])**2 + (self[1] - p2[1])**2
            if distance < closest_distance_squared:
                closest_distance_squared = distance
        return closest_distance_squared**0.5

    def furthest(self, ps: Union["Units", List["Point2"], Set["Point2"]]) -> Union["Unit", "Pointlike"]:
        """ This function assumes the 2d distance is meant """
        assert len(ps) > 0
        furthest_distance_squared = 0
        for p2 in ps:
            p2pos = p2
            if not isinstance(p2pos, Point2):
                p2pos = p2.position
            distance = (self[0] - p2pos[0])**2 + (self[1] - p2pos[1])**2
            if furthest_distance_squared < distance:
                furthest_distance_squared = distance
                furthest_element = p2
        return furthest_element

    def distance_to_furthest(self, ps: Union["Units", List["Point2"], Set["Point2"]]) -> Union[int, float]:
        """ This function assumes the 2d distance is meant """
        assert len(ps) > 0
        furthest_distance_squared = 0
        for p2 in ps:
            if not isinstance(p2, Point2):
                p2 = p2.position
            distance = (self[0] - p2[0])**2 + (self[1] - p2[1])**2
            if furthest_distance_squared < distance:
                furthest_distance_squared = distance
        return furthest_distance_squared**0.5

    def offset(self, p) -> "Pointlike":
        return self.__class__(a+b for a, b in itertools.zip_longest(self, p[:len(self)], fillvalue=0))

    def unit_axes_towards(self, p):
        return self.__class__(_sign(b - a) for a, b in itertools.zip_longest(self, p[:len(self)], fillvalue=0))

    def towards(self, p: Union["Unit", "Pointlike"], distance: Union[int, float]=1, limit: bool=False) -> "Pointlike":
        assert self != p
        d = self.distance_to(p)
        if limit:
            distance = min(d, distance)
        return self.__class__(a + (b - a) / d * distance for a, b in itertools.zip_longest(self, p[:len(self)], fillvalue=0))

    def __eq__(self, other):
        if not isinstance(other, tuple):
            return False
        return all(abs(a - b) < EPSILON for a, b in itertools.zip_longest(self, other, fillvalue=0))

    def __hash__(self):
        return hash(tuple(int(c * FLOAT_DIGITS)  for c in self))

class Point2(Pointlike):
    @classmethod
    def from_proto(cls, data):
        return cls((data.x, data.y))

    @property
    def x(self) -> Union[int, float]:
        return self[0]

    @property
    def y(self) -> Union[int, float]:
        return self[1]

    @property
    def to2(self) -> "Point2":
        return Point2(self[:2])

    @property
    def to3(self) -> "Point3":
        return Point3((*self, 0))

    def random_on_distance(self, distance):
        if isinstance(distance, (tuple, list)): # interval
            distance = distance[0] + random.random() * (distance[1] - distance[0])

        assert distance > 0
        angle = random.random() * 2 * pi

        dx, dy = cos(angle), sin(angle)
        return Point2((self.x + dx * distance, self.y + dy * distance))

    def towards_with_random_angle(self, p: Union["Point2", "Point3"], distance: Union[int, float]=1, max_difference: Union[int, float]=(pi/4)) -> "Point2":
        tx, ty = self.to2.towards(p.to2, 1)
        angle = atan2(ty - self.y, tx - self.x)
        angle = (angle - max_difference) + max_difference * 2 * random.random()
        return Point2((self.x + cos(angle) * distance, self.y + sin(angle) * distance))

    def circle_intersection(self, p: "Point2", r: Union[int, float]) -> Set["Point2"]:
        """ self is point1, p is point2, r is the radius for circles originating in both points
        Used in ramp finding """
        assert self != p
        distanceBetweenPoints = self.distance_to(p)
        assert r > distanceBetweenPoints / 2
        # remaining distance from center towards the intersection, using pythagoras
        remainingDistanceFromCenter = (r**2 - (distanceBetweenPoints/2)**2)**0.5
        # center of both points
        offsetToCenter = Point2(((p.x - self.x) / 2, (p.y - self.y) / 2))
        center = self.offset(offsetToCenter)

        # stretch offset vector in the ratio of remaining distance from center to intersection
        vectorStretchFactor = remainingDistanceFromCenter / (distanceBetweenPoints / 2)
        v = offsetToCenter
        offsetToCenterStretched = Point2((v.x * vectorStretchFactor, v.y * vectorStretchFactor))

        # rotate vector by 90° and -90°
        vectorRotated1 = Point2((offsetToCenterStretched.y, -offsetToCenterStretched.x))
        vectorRotated2 = Point2((-offsetToCenterStretched.y, offsetToCenterStretched.x))
        intersect1 = center.offset(vectorRotated1)
        intersect2 = center.offset(vectorRotated2)
        return {intersect1, intersect2}

    @property
    def neighbors4(self) -> set:
        return {
            Point2((self.x - 1, self.y)),
            Point2((self.x + 1, self.y)),
            Point2((self.x, self.y - 1)),
            Point2((self.x, self.y + 1)),
        }

    @property
    def neighbors8(self) -> set:
        return self.neighbors4 | {
            Point2((self.x - 1, self.y - 1)),
            Point2((self.x - 1, self.y + 1)),
            Point2((self.x + 1, self.y - 1)),
            Point2((self.x + 1, self.y + 1)),
        }

    def negative_offset(self, other: "Point2") -> "Point2":
        return self.__class__((self.x - other.x, self.y - other.y))

    def __add__(self, other: "Point2") -> "Point2":
        return self.offset(other)

    def __sub__(self, other: "Point2") -> "Point2":
        return self.negative_offset(other)

    def __neg__(self) -> "Point2":
        return self.__class__(-a for a in self)

    def __abs__(self) -> Union[int, float]:
        return math.hypot(self.x, self.y)

    def __bool__(self) -> bool:
        return self.x != 0 or self.y != 0

    def __mul__(self, other: Union[int, float, "Point2"]) -> "Point2":
        if isinstance(other, self.__class__):
            return self.__class__((self.x * other.x, self.y * other.y))
        return self.__class__((self.x * other, self.y * other))

    def __rmul__(self, other: Union[int, float, "Point2"]) -> "Point2":
        return self.__mul__(other)

    def __truediv__(self, other: Union[int, float, "Point2"]) -> "Point2":
        if isinstance(other, self.__class__):
            return self.__class__((self.x / other.x, self.y / other.y))
        return self.__class__((self.x / other, self.y / other))

    def is_same_as(self, other: "Point2", dist=0.1) -> bool:
        return self._distance_squared(other) <= dist ** 2

    def direction_vector(self, other: "Point2") -> "Point2":
        """ Converts a vector to a direction that can face vertically, horizontally or diagonal or be zero, e.g. (0, 0), (1, -1), (1, 0) """
        return self.__class__((_sign(other.x - self.x), _sign(other.y - self.y)))

    def manhattan_distance(self, other: "Point2") -> Union[int, float]:
        return abs(other.x - self.x) + abs(other.y - self.y)

    @staticmethod
    def center(a: ["Point2"]) -> "Point2":
        """ Returns the central point for points in list """
        s = Point2((0, 0))
        for p in a:
            s += p
        return s / len(a)

class Point3(Point2):
    @classmethod
    def from_proto(cls, data):
        return cls((data.x, data.y, data.z))

    @property
    def z(self) -> Union[int, float]:
        return self[2]

    @property
    def to3(self) -> "Point3":
        return Point3(self)

class Size(Point2):
    @property
    def width(self) -> Union[int, float]:
        return self[0]

    @property
    def height(self) -> Union[int, float]:
        return self[1]

class Rect(tuple):
    @classmethod
    def from_proto(cls, data):
        assert data.p0.x < data.p1.x and data.p0.y < data.p1.y
        return cls((data.p0.x, data.p0.y, data.p1.x - data.p0.x, data.p1.y - data.p0.y))

    @property
    def x(self) -> Union[int, float]:
        return self[0]

    @property
    def y(self) -> Union[int, float]:
        return self[1]

    @property
    def width(self) -> Union[int, float]:
        return self[2]

    @property
    def height(self) -> Union[int, float]:
        return self[3]

    @property
    def size(self) -> Size:
        return Size(self[2], self[3])

    @property
    def center(self) -> Point2:
        return Point2((self.x + self.width / 2, self.y + self.height / 2))

    def offset(self, p):
        return self.__class__((self[0]+p[0], self[1]+p[1], self[2], self[3]))
<|endoftext|>


<commit_before># Copyright (c) 2006,2007 Mitch Garnaat http://garnaat.org/
#
# Permission is hereby granted, free of charge, to any person obtaining a
# copy of this software and associated documentation files (the
# "Software"), to deal in the Software without restriction, including
# without limitation the rights to use, copy, modify, merge, publish, dis-
# tribute, sublicense, and/or sell copies of the Software, and to permit
# persons to whom the Software is furnished to do so, subject to the fol-
# lowing conditions:
#
# The above copyright notice and this permission notice shall be included
# in all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL-
# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT
# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
# IN THE SOFTWARE.

#
# Parts of this code were copied or derived from sample code supplied by AWS.
# The following notice applies to that code.
#
#  This software code is made available "AS IS" without warranties of any
#  kind.  You may copy, display, modify and redistribute the software
#  code either by itself or as incorporated into your code; provided that
#  you do not remove any proprietary notices.  Your use of this software
#  code is at your own risk and you waive any claim against Amazon
#  Digital Services, Inc. or its affiliates with respect to your use of
#  this software code. (c) 2006 Amazon Digital Services, Inc. or its
#  affiliates.

"""
Some handy utility functions used by several classes.
"""

import base64
import hmac
import re
import sha
import urllib

METADATA_PREFIX = 'x-amz-meta-'
AMAZON_HEADER_PREFIX = 'x-amz-'

# generates the aws canonical string for the given parameters
def canonical_string(method, path, headers, expires=None):
    interesting_headers = {}
    for key in headers:
        lk = key.lower()
        if lk in ['content-md5', 'content-type', 'date'] or lk.startswith(AMAZON_HEADER_PREFIX):
            interesting_headers[lk] = headers[key].strip()

    # these keys get empty strings if they don't exist
    if not interesting_headers.has_key('content-type'):
        interesting_headers['content-type'] = ''
    if not interesting_headers.has_key('content-md5'):
        interesting_headers['content-md5'] = ''

    # just in case someone used this.  it's not necessary in this lib.
    if interesting_headers.has_key('x-amz-date'):
        interesting_headers['date'] = ''

    # if you're using expires for query string auth, then it trumps date
    # (and x-amz-date)
    if expires:
        interesting_headers['date'] = str(expires)

    sorted_header_keys = interesting_headers.keys()
    sorted_header_keys.sort()

    buf = "%s\n" % method
    for key in sorted_header_keys:
        if key.startswith(AMAZON_HEADER_PREFIX):
            buf += "%s:%s\n" % (key, interesting_headers[key])
        else:
            buf += "%s\n" % interesting_headers[key]

    # don't include anything after the first ? in the resource...
    buf += "%s" % path.split('?')[0]

    # ...unless there is an acl or torrent parameter
    if re.search("[&?]acl($|=|&)", path):
        buf += "?acl"
    elif re.search("[&?]logging($|=|&)", path):
        buf += "?logging"
    elif re.search("[&?]torrent($|=|&)", path):
        buf += "?torrent"

    return buf

# computes the base64'ed hmac-sha hash of the canonical string and the secret
# access key, optionally urlencoding the result
def encode(aws_secret_access_key, str, urlencode=False):
    b64_hmac = base64.encodestring(hmac.new(aws_secret_access_key, str, sha).digest()).strip()
    if urlencode:
        return urllib.quote_plus(b64_hmac)
    else:
        return b64_hmac

def merge_meta(headers, metadata):
    final_headers = headers.copy()
    for k in metadata.keys():
        final_headers[METADATA_PREFIX + k] = metadata[k]

    return final_headers

def get_aws_metadata(headers):
    metadata = {}
    for hkey in headers.keys():
        if hkey.lower().startswith(METADATA_PREFIX):
            metadata[hkey[len(METADATA_PREFIX):]] = headers[hkey]
            del headers[hkey]
    return metadata

def get_instance_metadata(version='latest'):
    metadata = {}
    try:
        url = 'http://169.254.169.254/%s/meta-data/' % version
        s = urllib.urlopen(url)
        md_fields = s.read().split('\n')
        for md in md_fields:
            md_url = url + md
            s = urllib.urlopen(md_url)
            metadata[md] = s.read()
    except:
        print 'problem reading metadata'
    return metadata

def get_instance_userdata(version='latest', sep=None):
    user_data = None
    try:
        url = 'http://169.254.169.254/%s/user-data/' % version
        s = urllib.urlopen(url)
        user_data = s.read()
        if sep:
            l = user_data.split(sep)
            user_data = {}
            for nvpair in l:
                t = nvpair.split('=')
                user_data[t[0].strip()] = t[1].strip()
    except:
        print 'problem reading metadata'
    return user_data

<commit_msg>Moving the find_class method to utils to be shared by several modules.<commit_after># Copyright (c) 2006,2007 Mitch Garnaat http://garnaat.org/
#
# Permission is hereby granted, free of charge, to any person obtaining a
# copy of this software and associated documentation files (the
# "Software"), to deal in the Software without restriction, including
# without limitation the rights to use, copy, modify, merge, publish, dis-
# tribute, sublicense, and/or sell copies of the Software, and to permit
# persons to whom the Software is furnished to do so, subject to the fol-
# lowing conditions:
#
# The above copyright notice and this permission notice shall be included
# in all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL-
# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT
# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
# IN THE SOFTWARE.

#
# Parts of this code were copied or derived from sample code supplied by AWS.
# The following notice applies to that code.
#
#  This software code is made available "AS IS" without warranties of any
#  kind.  You may copy, display, modify and redistribute the software
#  code either by itself or as incorporated into your code; provided that
#  you do not remove any proprietary notices.  Your use of this software
#  code is at your own risk and you waive any claim against Amazon
#  Digital Services, Inc. or its affiliates with respect to your use of
#  this software code. (c) 2006 Amazon Digital Services, Inc. or its
#  affiliates.

"""
Some handy utility functions used by several classes.
"""

import base64
import hmac
import re
import sha
import urllib
import imp

METADATA_PREFIX = 'x-amz-meta-'
AMAZON_HEADER_PREFIX = 'x-amz-'

# generates the aws canonical string for the given parameters
def canonical_string(method, path, headers, expires=None):
    interesting_headers = {}
    for key in headers:
        lk = key.lower()
        if lk in ['content-md5', 'content-type', 'date'] or lk.startswith(AMAZON_HEADER_PREFIX):
            interesting_headers[lk] = headers[key].strip()

    # these keys get empty strings if they don't exist
    if not interesting_headers.has_key('content-type'):
        interesting_headers['content-type'] = ''
    if not interesting_headers.has_key('content-md5'):
        interesting_headers['content-md5'] = ''

    # just in case someone used this.  it's not necessary in this lib.
    if interesting_headers.has_key('x-amz-date'):
        interesting_headers['date'] = ''

    # if you're using expires for query string auth, then it trumps date
    # (and x-amz-date)
    if expires:
        interesting_headers['date'] = str(expires)

    sorted_header_keys = interesting_headers.keys()
    sorted_header_keys.sort()

    buf = "%s\n" % method
    for key in sorted_header_keys:
        if key.startswith(AMAZON_HEADER_PREFIX):
            buf += "%s:%s\n" % (key, interesting_headers[key])
        else:
            buf += "%s\n" % interesting_headers[key]

    # don't include anything after the first ? in the resource...
    buf += "%s" % path.split('?')[0]

    # ...unless there is an acl or torrent parameter
    if re.search("[&?]acl($|=|&)", path):
        buf += "?acl"
    elif re.search("[&?]logging($|=|&)", path):
        buf += "?logging"
    elif re.search("[&?]torrent($|=|&)", path):
        buf += "?torrent"

    return buf

# computes the base64'ed hmac-sha hash of the canonical string and the secret
# access key, optionally urlencoding the result
def encode(aws_secret_access_key, str, urlencode=False):
    b64_hmac = base64.encodestring(hmac.new(aws_secret_access_key, str, sha).digest()).strip()
    if urlencode:
        return urllib.quote_plus(b64_hmac)
    else:
        return b64_hmac

def merge_meta(headers, metadata):
    final_headers = headers.copy()
    for k in metadata.keys():
        final_headers[METADATA_PREFIX + k] = metadata[k]

    return final_headers

def get_aws_metadata(headers):
    metadata = {}
    for hkey in headers.keys():
        if hkey.lower().startswith(METADATA_PREFIX):
            metadata[hkey[len(METADATA_PREFIX):]] = headers[hkey]
            del headers[hkey]
    return metadata

def get_instance_metadata(version='latest'):
    metadata = {}
    try:
        url = 'http://169.254.169.254/%s/meta-data/' % version
        s = urllib.urlopen(url)
        md_fields = s.read().split('\n')
        for md in md_fields:
            md_url = url + md
            s = urllib.urlopen(md_url)
            metadata[md] = s.read()
    except:
        print 'problem reading metadata'
    return metadata

def get_instance_userdata(version='latest', sep=None):
    user_data = None
    try:
        url = 'http://169.254.169.254/%s/user-data/' % version
        s = urllib.urlopen(url)
        user_data = s.read()
        if sep:
            l = user_data.split(sep)
            user_data = {}
            for nvpair in l:
                t = nvpair.split('=')
                user_data[t[0].strip()] = t[1].strip()
    except:
        print 'problem reading metadata'
    return user_data

def find_class(module_name, class_name):
    modules = module_name.split('.')
    path = None
    for module_name in modules:
        fp, pathname, description = imp.find_module(module_name, path)
        module = imp.load_module(module_name, fp, pathname, description)
        if hasattr(module, '__path__'):
            path = module.__path__
    return getattr(module, class_name)

<|endoftext|>