<commit_before>"""Heuristic polynomial GCD algorithm (HEUGCD). """

from __future__ import print_function, division
from sympy.core.compatibility import xrange

HEU_GCD_MAX = 6

def heugcd(f, g):
    """
    Heuristic polynomial GCD in ``Z[X]``.

    Given univariate polynomials ``f`` and ``g`` in ``Z[X]``, returns
    their GCD and cofactors, i.e. polynomials ``h``, ``cff`` and ``cfg``
    such that::

          h = gcd(f, g), cff = quo(f, h) and cfg = quo(g, h)

    The algorithm is purely heuristic which means it may fail to compute
    the GCD. This will be signaled by raising an exception. In this case
    you will need to switch to another GCD method.

    The algorithm computes the polynomial GCD by evaluating polynomials
    ``f`` and ``g`` at certain points and computing (fast) integer GCD
    of those evaluations. The polynomial GCD is recovered from the integer
    image by interpolation. The evaluation proces reduces f and g variable
    by variable into a large integer. The final step is to verify if the
    interpolated polynomial is the correct GCD. This gives cofactors of
    the input polynomials as a side effect.

    Examples
    ========

    >>> from sympy.polys.heuristicgcd import heugcd
    >>> from sympy.polys import ring, ZZ

    >>> R, x,y, = ring("x,y", ZZ)

    >>> f = x**2 + 2*x*y + y**2
    >>> g = x**2 + x*y

    >>> h, cff, cfg = heugcd(f, g)
    >>> h, cff, cfg
    (x + y, x + y, x)

    >>> cff*h == f
    True
    >>> cfg*h == g
    True

    References
    ==========

    1. [Liao95]_

    """
    assert f.ring == g.ring and f.ring.domain.is_ZZ

    ring = f.ring
    x0 = ring.gens[0]
    domain = ring.domain

    gcd, f, g = f.extract_ground(g)

    f_norm = f.max_norm()
    g_norm = g.max_norm()

    B = domain(2*min(f_norm, g_norm) + 29)

    x = max(min(B, 99*domain.sqrt(B)),
            2*min(f_norm // abs(f.LC),
                  g_norm // abs(g.LC)) + 2)

    for i in xrange(0, HEU_GCD_MAX):
        ff = f.evaluate(x0, x)
        gg = g.evaluate(x0, x)

        if ff and gg:
            if ring.ngens == 1:
                h, cff, cfg = domain.cofactors(ff, gg)
            else:
                h, cff, cfg = heugcd(ff, gg)

            h = _gcd_interpolate(h, x, ring)
            h = h.primitive()[1]

            cff_, r = f.div(h)

            if not r:
                cfg_, r = g.div(h)

                if not r:
                    h = h.mul_ground(gcd)
                    return h, cff_, cfg_

            cff = _gcd_interpolate(cff, x, ring)

            h, r = f.div(cff)

            if not r:
                cfg_, r = g.div(h)

                if not r:
                    h = h.mul_ground(gcd)
                    return h, cff, cfg_

            cfg = _gcd_interpolate(cfg, x, ring)

            h, r = g.div(cfg)

            if not r:
                cff_, r = f.div(h)

                if not r:
                    h = h.mul_ground(gcd)
                    return h, cff_, cfg

        x = 73794*x * domain.sqrt(domain.sqrt(x)) // 27011

    raise HeuristicGCDFailed('no luck')

def _gcd_interpolate(h, x, ring):
    """Interpolate polynomial GCD from integer GCD. """
    f, i = ring.zero, 0

    # TODO: don't expose poly repr implementation details
    if ring.ngens == 1:
        while h:
            g = h % x
            if g > x // 2: g -= x
            h = (h - g) // x

            # f += X**i*g
            if g:
                f[(i,)] = g
            i += 1
    else:
        while h:
            g = h.trunc_ground(x)
            h = (h - g).quo_ground(x)

            # f += X**i*g
            if g:
                for monom, coeff in g.iterterms():
                    f[(i,) + monom] = coeff
            i += 1

    if f.LC < 0:
        return -f
    else:
        return  f
<commit_msg>Import not imported exception HeuristicGCDFailed HeuristicGCDFailed was being raised without importing it in polys/heuristicgcd.py.<commit_after>"""Heuristic polynomial GCD algorithm (HEUGCD). """

from __future__ import print_function, division
from sympy.core.compatibility import xrange
from sympy.polys.polyerrors import HeuristicGCDFailed

HEU_GCD_MAX = 6

def heugcd(f, g):
    """
    Heuristic polynomial GCD in ``Z[X]``.

    Given univariate polynomials ``f`` and ``g`` in ``Z[X]``, returns
    their GCD and cofactors, i.e. polynomials ``h``, ``cff`` and ``cfg``
    such that::

          h = gcd(f, g), cff = quo(f, h) and cfg = quo(g, h)

    The algorithm is purely heuristic which means it may fail to compute
    the GCD. This will be signaled by raising an exception. In this case
    you will need to switch to another GCD method.

    The algorithm computes the polynomial GCD by evaluating polynomials
    ``f`` and ``g`` at certain points and computing (fast) integer GCD
    of those evaluations. The polynomial GCD is recovered from the integer
    image by interpolation. The evaluation proces reduces f and g variable
    by variable into a large integer. The final step is to verify if the
    interpolated polynomial is the correct GCD. This gives cofactors of
    the input polynomials as a side effect.

    Examples
    ========

    >>> from sympy.polys.heuristicgcd import heugcd
    >>> from sympy.polys import ring, ZZ

    >>> R, x,y, = ring("x,y", ZZ)

    >>> f = x**2 + 2*x*y + y**2
    >>> g = x**2 + x*y

    >>> h, cff, cfg = heugcd(f, g)
    >>> h, cff, cfg
    (x + y, x + y, x)

    >>> cff*h == f
    True
    >>> cfg*h == g
    True

    References
    ==========

    1. [Liao95]_

    """
    assert f.ring == g.ring and f.ring.domain.is_ZZ

    ring = f.ring
    x0 = ring.gens[0]
    domain = ring.domain

    gcd, f, g = f.extract_ground(g)

    f_norm = f.max_norm()
    g_norm = g.max_norm()

    B = domain(2*min(f_norm, g_norm) + 29)

    x = max(min(B, 99*domain.sqrt(B)),
            2*min(f_norm // abs(f.LC),
                  g_norm // abs(g.LC)) + 2)

    for i in xrange(0, HEU_GCD_MAX):
        ff = f.evaluate(x0, x)
        gg = g.evaluate(x0, x)

        if ff and gg:
            if ring.ngens == 1:
                h, cff, cfg = domain.cofactors(ff, gg)
            else:
                h, cff, cfg = heugcd(ff, gg)

            h = _gcd_interpolate(h, x, ring)
            h = h.primitive()[1]

            cff_, r = f.div(h)

            if not r:
                cfg_, r = g.div(h)

                if not r:
                    h = h.mul_ground(gcd)
                    return h, cff_, cfg_

            cff = _gcd_interpolate(cff, x, ring)

            h, r = f.div(cff)

            if not r:
                cfg_, r = g.div(h)

                if not r:
                    h = h.mul_ground(gcd)
                    return h, cff, cfg_

            cfg = _gcd_interpolate(cfg, x, ring)

            h, r = g.div(cfg)

            if not r:
                cff_, r = f.div(h)

                if not r:
                    h = h.mul_ground(gcd)
                    return h, cff_, cfg

        x = 73794*x * domain.sqrt(domain.sqrt(x)) // 27011

    raise HeuristicGCDFailed('no luck')

def _gcd_interpolate(h, x, ring):
    """Interpolate polynomial GCD from integer GCD. """
    f, i = ring.zero, 0

    # TODO: don't expose poly repr implementation details
    if ring.ngens == 1:
        while h:
            g = h % x
            if g > x // 2: g -= x
            h = (h - g) // x

            # f += X**i*g
            if g:
                f[(i,)] = g
            i += 1
    else:
        while h:
            g = h.trunc_ground(x)
            h = (h - g).quo_ground(x)

            # f += X**i*g
            if g:
                for monom, coeff in g.iterterms():
                    f[(i,) + monom] = coeff
            i += 1

    if f.LC < 0:
        return -f
    else:
        return  f
<|endoftext|>


<commit_before>from dataclasses import dataclass
from ssl import CertificateError
from typing import Optional, List, cast

import cryptography
from cryptography.hazmat.primitives import hashes
from cryptography.x509 import ExtensionNotFound, ExtensionOID, Certificate
from nassl.ocsp_response import OcspResponseStatusEnum, OcspResponseNotTrustedError, OcspResponse

from sslyze.plugins.certificate_info.symantec import SymantecDistructTester
from sslyze.plugins.certificate_info.trust_stores.trust_store import TrustStore
from sslyze.plugins.certificate_info.trust_stores.trust_store_repository import TrustStoresRepository
from sslyze.plugins.certificate_info.certificate_utils import CertificateUtils

@dataclass(frozen=True)
class CertificateChainDeploymentAnalysisResult:

    leaf_certificate_subject_matches_hostname: bool
    leaf_certificate_has_must_staple_extension: bool
    leaf_certificate_is_ev: bool
    leaf_certificate_signed_certificate_timestamps_count: Optional[int]
    received_chain_contains_anchor_certificate: Optional[bool]
    received_chain_has_valid_order: bool
    verified_chain_has_sha1_signature: Optional[bool]
    verified_chain_has_legacy_symantec_anchor: Optional[bool]
    ocsp_response_is_trusted: Optional[bool]
    ocsp_response_status: Optional[OcspResponseStatusEnum]

class CertificateChainDeploymentAnalyzer:
    """Utility class for analyzing a certificate chain as deployed on a specific server.

    Useful for checking a server's certificate chain without having to use the CertificateInfoPlugin.
    """

    def __init__(
        self,
        server_hostname: str,
        received_chain: List[Certificate],
        verified_chain: Optional[List[Certificate]],
        trust_store_used_to_build_verified_chain: Optional[TrustStore],
        received_ocsp_response: Optional[OcspResponse],
    ) -> None:
        self.server_hostname = server_hostname
        self.received_certificate_chain = received_chain
        self.verified_certificate_chain = verified_chain
        self.trust_store_used_to_build_verified_chain = trust_store_used_to_build_verified_chain
        self.received_ocsp_response = received_ocsp_response

    def perform(self) -> CertificateChainDeploymentAnalysisResult:
        """Run the analysis.
        """
        leaf_cert = self.received_certificate_chain[0]

        # OCSP Must-Staple
        has_ocsp_must_staple = False
        try:
            tls_feature_ext = cast(
                cryptography.x509.TLSFeature, leaf_cert.extensions.get_extension_for_oid(ExtensionOID.TLS_FEATURE)
            )
            for feature_type in tls_feature_ext.value:
                if feature_type == cryptography.x509.TLSFeatureType.status_request:
                    has_ocsp_must_staple = True
                    break
        except ExtensionNotFound:
            pass

        # Received chain order
        is_chain_order_valid = True
        previous_issuer = None
        for index, cert in enumerate(self.received_certificate_chain):
            current_subject = cert.subject

            if index > 0:
                # Compare the current subject with the previous issuer in the chain
                if current_subject != previous_issuer:
                    is_chain_order_valid = False
                    break
            try:
                previous_issuer = cert.issuer
            except KeyError:
                # Missing issuer; this is okay if this is the last cert
                previous_issuer = "missing issuer {}".format(index)

        # Check if it is EV - we only have the EV OIDs for Mozilla
        is_leaf_certificate_ev = (
            TrustStoresRepository.get_default()
            .get_main_store()
            .is_certificate_extended_validation(self.received_certificate_chain[0])
        )

        # Check for Signed Timestamps
        number_of_scts: Optional[int] = 0
        try:
            # Look for the x509 extension
            sct_ext = leaf_cert.extensions.get_extension_for_oid(ExtensionOID.PRECERT_SIGNED_CERTIFICATE_TIMESTAMPS)

            if isinstance(sct_ext.value, cryptography.x509.UnrecognizedExtension):
                # The version of OpenSSL on the system is too old and can't parse the SCT extension
                number_of_scts = None

            # Count the number of entries in the extension
            number_of_scts = len(sct_ext.value)
        except ExtensionNotFound:
            pass

        # Check if the anchor was sent by the server
        has_anchor_in_certificate_chain = None
        if self.verified_certificate_chain:
            has_anchor_in_certificate_chain = self.verified_certificate_chain[-1] in self.received_certificate_chain

        # Check hostname validation
        try:
            CertificateUtils.certificate_matches_hostname(leaf_cert, self.server_hostname)
            certificate_matches_hostname = True
        except CertificateError:
            certificate_matches_hostname = False

        # Check if a SHA1-signed certificate is in the chain
        # Root certificates can still be signed with SHA1 so we only check leaf and intermediate certificates
        has_sha1_in_certificate_chain = None
        if self.verified_certificate_chain:
            has_sha1_in_certificate_chain = False
            for cert in self.verified_certificate_chain[:-1]:
                if isinstance(cert.signature_hash_algorithm, hashes.SHA1):
                    has_sha1_in_certificate_chain = True
                    break

        # Check if this is a distrusted Symantec-issued chain
        verified_chain_has_legacy_symantec_anchor = None
        if self.verified_certificate_chain:
            symantec_distrust_timeline = SymantecDistructTester.get_distrust_timeline(self.verified_certificate_chain)
            verified_chain_has_legacy_symantec_anchor = True if symantec_distrust_timeline else False

        # Check the OCSP response if there is one
        is_ocsp_response_trusted = None
        ocsp_response_status = None
        if self.received_ocsp_response:
            ocsp_response_status = self.received_ocsp_response.status
            if (
                self.trust_store_used_to_build_verified_chain
                and ocsp_response_status == OcspResponseStatusEnum.SUCCESSFUL
            ):
                try:
                    self.received_ocsp_response.verify(self.trust_store_used_to_build_verified_chain.path)
                    is_ocsp_response_trusted = True
                except OcspResponseNotTrustedError:
                    is_ocsp_response_trusted = False

        return CertificateChainDeploymentAnalysisResult(
            leaf_certificate_subject_matches_hostname=certificate_matches_hostname,
            leaf_certificate_has_must_staple_extension=has_ocsp_must_staple,
            leaf_certificate_is_ev=is_leaf_certificate_ev,
            leaf_certificate_signed_certificate_timestamps_count=number_of_scts,
            received_chain_contains_anchor_certificate=has_anchor_in_certificate_chain,
            received_chain_has_valid_order=is_chain_order_valid,
            verified_chain_has_sha1_signature=has_sha1_in_certificate_chain,
            verified_chain_has_legacy_symantec_anchor=verified_chain_has_legacy_symantec_anchor,
            ocsp_response_is_trusted=is_ocsp_response_trusted,
            ocsp_response_status=ocsp_response_status,
        )
<commit_msg>Fix type<commit_after>from dataclasses import dataclass
from ssl import CertificateError
from typing import Optional, List, cast

import cryptography
from cryptography.hazmat.primitives import hashes
from cryptography.x509 import ExtensionNotFound, ExtensionOID, Certificate
from nassl.ocsp_response import OcspResponseStatusEnum, OcspResponseNotTrustedError, OcspResponse

from sslyze.plugins.certificate_info.symantec import SymantecDistructTester
from sslyze.plugins.certificate_info.trust_stores.trust_store import TrustStore
from sslyze.plugins.certificate_info.trust_stores.trust_store_repository import TrustStoresRepository
from sslyze.plugins.certificate_info.certificate_utils import CertificateUtils

@dataclass(frozen=True)
class CertificateChainDeploymentAnalysisResult:

    leaf_certificate_subject_matches_hostname: bool
    leaf_certificate_has_must_staple_extension: bool
    leaf_certificate_is_ev: bool
    leaf_certificate_signed_certificate_timestamps_count: Optional[int]
    received_chain_contains_anchor_certificate: Optional[bool]
    received_chain_has_valid_order: bool
    verified_chain_has_sha1_signature: Optional[bool]
    verified_chain_has_legacy_symantec_anchor: Optional[bool]
    ocsp_response_is_trusted: Optional[bool]
    ocsp_response_status: Optional[OcspResponseStatusEnum]

class CertificateChainDeploymentAnalyzer:
    """Utility class for analyzing a certificate chain as deployed on a specific server.

    Useful for checking a server's certificate chain without having to use the CertificateInfoPlugin.
    """

    def __init__(
        self,
        server_hostname: str,
        received_chain: List[Certificate],
        verified_chain: Optional[List[Certificate]],
        trust_store_used_to_build_verified_chain: Optional[TrustStore],
        received_ocsp_response: Optional[OcspResponse],
    ) -> None:
        self.server_hostname = server_hostname
        self.received_certificate_chain = received_chain
        self.verified_certificate_chain = verified_chain
        self.trust_store_used_to_build_verified_chain = trust_store_used_to_build_verified_chain
        self.received_ocsp_response = received_ocsp_response

    def perform(self) -> CertificateChainDeploymentAnalysisResult:
        """Run the analysis.
        """
        leaf_cert = self.received_certificate_chain[0]

        # OCSP Must-Staple
        has_ocsp_must_staple = False
        try:
            tls_feature_ext = cast(
                cryptography.x509.TLSFeature, leaf_cert.extensions.get_extension_for_oid(ExtensionOID.TLS_FEATURE)
            )
            for feature_type in tls_feature_ext.value:
                if feature_type == cryptography.x509.TLSFeatureType.status_request:
                    has_ocsp_must_staple = True
                    break
        except ExtensionNotFound:
            pass

        # Received chain order
        is_chain_order_valid = True
        previous_issuer = None
        for index, cert in enumerate(self.received_certificate_chain):
            current_subject = cert.subject

            if index > 0:
                # Compare the current subject with the previous issuer in the chain
                if current_subject != previous_issuer:
                    is_chain_order_valid = False
                    break
            try:
                previous_issuer = cert.issuer
            except KeyError:
                # Missing issuer; this is okay if this is the last cert
                previous_issuer = None

        # Check if it is EV - we only have the EV OIDs for Mozilla
        is_leaf_certificate_ev = (
            TrustStoresRepository.get_default()
            .get_main_store()
            .is_certificate_extended_validation(self.received_certificate_chain[0])
        )

        # Check for Signed Timestamps
        number_of_scts: Optional[int] = 0
        try:
            # Look for the x509 extension
            sct_ext = leaf_cert.extensions.get_extension_for_oid(ExtensionOID.PRECERT_SIGNED_CERTIFICATE_TIMESTAMPS)
            if isinstance(sct_ext.value, cryptography.x509.UnrecognizedExtension):
                # The version of OpenSSL on the system is too old and can't parse the SCT extension
                number_of_scts = None

            # Count the number of entries in the extension
            sct_ext_value = cast(cryptography.x509.PrecertificateSignedCertificateTimestamps, sct_ext.value)
            number_of_scts = len(sct_ext_value)
        except ExtensionNotFound:
            pass

        # Check if the anchor was sent by the server
        has_anchor_in_certificate_chain = None
        if self.verified_certificate_chain:
            has_anchor_in_certificate_chain = self.verified_certificate_chain[-1] in self.received_certificate_chain

        # Check hostname validation
        try:
            CertificateUtils.certificate_matches_hostname(leaf_cert, self.server_hostname)
            certificate_matches_hostname = True
        except CertificateError:
            certificate_matches_hostname = False

        # Check if a SHA1-signed certificate is in the chain
        # Root certificates can still be signed with SHA1 so we only check leaf and intermediate certificates
        has_sha1_in_certificate_chain = None
        if self.verified_certificate_chain:
            has_sha1_in_certificate_chain = False
            for cert in self.verified_certificate_chain[:-1]:
                if isinstance(cert.signature_hash_algorithm, hashes.SHA1):
                    has_sha1_in_certificate_chain = True
                    break

        # Check if this is a distrusted Symantec-issued chain
        verified_chain_has_legacy_symantec_anchor = None
        if self.verified_certificate_chain:
            symantec_distrust_timeline = SymantecDistructTester.get_distrust_timeline(self.verified_certificate_chain)
            verified_chain_has_legacy_symantec_anchor = True if symantec_distrust_timeline else False

        # Check the OCSP response if there is one
        is_ocsp_response_trusted = None
        ocsp_response_status = None
        if self.received_ocsp_response:
            ocsp_response_status = self.received_ocsp_response.status
            if (
                self.trust_store_used_to_build_verified_chain
                and ocsp_response_status == OcspResponseStatusEnum.SUCCESSFUL
            ):
                try:
                    self.received_ocsp_response.verify(self.trust_store_used_to_build_verified_chain.path)
                    is_ocsp_response_trusted = True
                except OcspResponseNotTrustedError:
                    is_ocsp_response_trusted = False

        return CertificateChainDeploymentAnalysisResult(
            leaf_certificate_subject_matches_hostname=certificate_matches_hostname,
            leaf_certificate_has_must_staple_extension=has_ocsp_must_staple,
            leaf_certificate_is_ev=is_leaf_certificate_ev,
            leaf_certificate_signed_certificate_timestamps_count=number_of_scts,
            received_chain_contains_anchor_certificate=has_anchor_in_certificate_chain,
            received_chain_has_valid_order=is_chain_order_valid,
            verified_chain_has_sha1_signature=has_sha1_in_certificate_chain,
            verified_chain_has_legacy_symantec_anchor=verified_chain_has_legacy_symantec_anchor,
            ocsp_response_is_trusted=is_ocsp_response_trusted,
            ocsp_response_status=ocsp_response_status,
        )
<|endoftext|>


<commit_before>#!/usr/bin/env python3

import subprocess
import re

outfile = open('commands.md','w')

# tests:
test=[]
test.append( u"Usage: juju sync-tools [options]\n\nSummary:\ncopy tools from the official tool store into a local model\n\nOptions:\n--all  (= false)\n    copy all versions, not just the latest\n--destination (= \"\")\n    local destination directory\n--dev  (= false)\n    consider development versions as well as released ones\n    DEPRECATED: use --stream instead\n--dry-run  (= false)\n    don't copy, just print what would be copied\n--local-dir (= \"\")\n    local destination directory\n-m, --model (= \"\")\n    Model to operate in\n--public  (= false)\n    tools are for a public cloud, so generate mirrors information\n--source (= \"\")\n    local source directory\n--stream (= \"\")\n    simplestreams stream for which to sync metadata\n--version (= \"\")\n    copy a specific major[.minor] version\n\nDetails:\nThis copies the Juju tools tarball from the official tools store (located\nat https://streams.canonical.com/juju) into your model.\nThis is generally done when you want Juju to be able to run without having to\naccess the Internet. Alternatively you can specify a local directory as source.\n\nSometimes this is because the model does not have public access,\nand sometimes you just want to avoid having to access data outside of\nthe local cloud.\n")

pad='   '

# c = subprocess.check_output(['juju', 'help', 'commands']).splitlines()
# for command in c:

for t in test:
    p = re.compile(u'Usage:(.+?)\n\nSummary:\n(.+?)\n\nOptions:\n(.+?)Details:\n(.+)', re.DOTALL)
    #p = re.compile(ur'Usage:(.+?)\n\nSummary:\n(.+?)\n\nOptions:\n(.+?)Details:\n(.+)', re.DOTALL)
    #test_str = u"Usage: juju sync-tools [options]\n\nSummary:\ncopy tools from the official tool store into a local model\n\nOptions:\n--all  (= false)\n    copy all versions, not just the latest\n--destination (= \"\")\n    local destination directory\n--dev  (= false)\n    consider development versions as well as released ones\n    DEPRECATED: use --stream instead\n--dry-run  (= false)\n    don't copy, just print what would be copied\n--local-dir (= \"\")\n    local destination directory\n-m, --model (= \"\")\n    Model to operate in\n--public  (= false)\n    tools are for a public cloud, so generate mirrors information\n--source (= \"\")\n    local source directory\n--stream (= \"\")\n    simplestreams stream for which to sync metadata\n--version (= \"\")\n    copy a specific major[.minor] version\n\nDetails:\nThis copies the Juju tools tarball from the official tools store (located\nat https://streams.canonical.com/juju) into your model.\nThis is generally done when you want Juju to be able to run without having to\naccess the Internet. Alternatively you can specify a local directory as source.\n\nSometimes this is because the model does not have public access,\nand sometimes you just want to avoid having to access data outside of\nthe local cloud.\n"

    help=re.findall(p, t)[0]
    usage=pad+"**Usage:** `"+help[0]+"`\n\n"
    summary=pad+"**Summary:**\n\n"+pad+help[1]+"\n\n"
    options = pad+'**Options:**\n\n'
    option_lines = help[2].split('\n')

    for line in option_lines:
       if (line !=''):
          if (line[0]=='-'):
             options = options+pad+"_"+line.strip()+"_\n\n"
          else:
             options = options+pad+pad+line.strip()+"\n\n"
    outfile.write(usage+summary+options)
<commit_msg>added parsing for examples<commit_after>#!/usr/bin/env python3

import subprocess
import re

outfile = open('commands.md','w')

# tests:
test=[]
test.append( u"Usage: juju sync-tools [options]\n\nSummary:\ncopy tools from the official tool store into a local model\n\nOptions:\n--all  (= false)\n    copy all versions, not just the latest\n--destination (= \"\")\n    local destination directory\n--dev  (= false)\n    consider development versions as well as released ones\n    DEPRECATED: use --stream instead\n--dry-run  (= false)\n    don't copy, just print what would be copied\n--local-dir (= \"\")\n    local destination directory\n-m, --model (= \"\")\n    Model to operate in\n--public  (= false)\n    tools are for a public cloud, so generate mirrors information\n--source (= \"\")\n    local source directory\n--stream (= \"\")\n    simplestreams stream for which to sync metadata\n--version (= \"\")\n    copy a specific major[.minor] version\n\nDetails:\nThis copies the Juju tools tarball from the official tools store (located\nat https://streams.canonical.com/juju) into your model.\nThis is generally done when you want Juju to be able to run without having to\naccess the Internet. Alternatively you can specify a local directory as source.\n\nSometimes this is because the model does not have public access,\nand sometimes you just want to avoid having to access data outside of\nthe local cloud.\n")
test.append( u"Usage: juju bootstrap [options] <controller name> <cloud name>[/region]\n\nSummary:\nInitializes a cloud environment.\n\nOptions:\n--agent-version (= \"\")\n    Version of tools to use for Juju agents\n--auto-upgrade  (= false)\n    Upgrade to the latest patch release tools on first bootstrap\n--bootstrap-constraints  (= )\n    Specify bootstrap machine constraints\n--bootstrap-series (= \"\")\n    Specify the series of the bootstrap machine\n--config  (= )\n    Specify a controller configuration file, or one or more configuration\n    options\n    (--config config.yaml [--config key=value ...])\n--constraints  (= )\n    Set model constraints\n--credential (= \"\")\n    Credentials to use when bootstrapping\n-d, --default-model (= \"default\")\n    Name of the default hosted model for the controller\n--keep-broken  (= false)\n    Do not destroy the model if bootstrap fails\n--metadata-source (= \"\")\n    Local path to use as tools and/or metadata source\n--no-gui  (= false)\n    Do not install the Juju GUI in the controller when bootstrapping\n--to (= \"\")\n    Placement directive indicating an instance to bootstrap\n--upload-tools  (= false)\n    Upload local version of tools before bootstrapping\n\nDetails:\nInitialization consists of creating an 'admin' model and provisioning a\nmachine to act as controller.\nCredentials are set beforehand and are distinct from any other\nconfiguration (see `juju add-credential`).\nThe 'admin' model typically does not run workloads. It should remain\npristine to run and manage Juju's own infrastructure for the corresponding\ncloud. Additional (hosted) models should be created with `juju create-\nmodel` for workload purposes.\nNote that a 'default' model is also created and becomes the current model\nof the environment once the command completes. It can be discarded if\nother models are created.\nIf '--bootstrap-constraints' is used, its values will also apply to any\nfuture controllers provisioned for high availability (HA).\nIf '--constraints' is used, its values will be set as the default\nconstraints for all future workload machines in the model, exactly as if\nthe constraints were set with `juju set-model-constraints`.\nIt is possible to override constraints and the automatic machine selection\nalgorithm by assigning a \"placement directive\" via the '--to' option. This\ndictates what machine to use for the controller. This would typically be\nused with the MAAS provider ('--to <host>.maas').\nYou can change the default timeout and retry delays used during the\nbootstrap by changing the following settings in your configuration file\n(all values represent number of seconds):\n    # How long to wait for a connection to the controller\n    bootstrap-timeout: 600 # default: 10 minutes\n    # How long to wait between connection attempts to a controller\naddress.\n    bootstrap-retry-delay: 5 # default: 5 seconds\n    # How often to refresh controller addresses from the API server.\n    bootstrap-addresses-delay: 10 # default: 10 seconds\nPrivate clouds may need to specify their own custom image metadata and\ntools/agent. Use '--metadata-source' whose value is a local directory.\nThe value of '--agent-version' will become the default tools version to\nuse in all models for this controller. The full binary version is accepted\n(e.g.: 2.0.1-xenial-amd64) but only the numeric version (e.g.: 2.0.1) is\nused. Otherwise, by default, the version used is that of the client.\n\nExamples:\n    juju bootstrap mycontroller google\n    juju bootstrap --config=~/config-rs.yaml mycontroller rackspace\n    juju bootstrap --config agent-version=1.25.3 mycontroller aws\n    juju bootstrap --config bootstrap-timeout=1200 mycontroller azure\n\nSee also:\n    add-credentials\n    create-model\n    set-constraints")

pad='   '

# c = subprocess.check_output(['juju', 'help', 'commands']).splitlines()
# for command in c:

for t in test:
    p = re.compile(u'Usage:(.+?)\n\nSummary:\n(.+?)\n\nOptions:\n(.+?)Details:\n(.+)', re.DOTALL)
    #p = re.compile(ur'Usage:(.+?)\n\nSummary:\n(.+?)\n\nOptions:\n(.+?)Details:\n(.+)', re.DOTALL)
    #test_str = u"Usage: juju sync-tools [options]\n\nSummary:\ncopy tools from the official tool store into a local model\n\nOptions:\n--all  (= false)\n    copy all versions, not just the latest\n--destination (= \"\")\n    local destination directory\n--dev  (= false)\n    consider development versions as well as released ones\n    DEPRECATED: use --stream instead\n--dry-run  (= false)\n    don't copy, just print what would be copied\n--local-dir (= \"\")\n    local destination directory\n-m, --model (= \"\")\n    Model to operate in\n--public  (= false)\n    tools are for a public cloud, so generate mirrors information\n--source (= \"\")\n    local source directory\n--stream (= \"\")\n    simplestreams stream for which to sync metadata\n--version (= \"\")\n    copy a specific major[.minor] version\n\nDetails:\nThis copies the Juju tools tarball from the official tools store (located\nat https://streams.canonical.com/juju) into your model.\nThis is generally done when you want Juju to be able to run without having to\naccess the Internet. Alternatively you can specify a local directory as source.\n\nSometimes this is because the model does not have public access,\nand sometimes you just want to avoid having to access data outside of\nthe local cloud.\n"

    help=re.findall(p, t)[0]
    usage=pad+"**Usage:** `"+help[0]+"`\n\n"
    summary=pad+"**Summary:**\n\n"+pad+help[1]+"\n\n"
    options = pad+'**Options:**\n\n'
    option_lines = help[2].split('\n')

    for line in option_lines:
       if (line !=''):
          if (line[0]=='-'):
             options = options+pad+"_"+line.strip()+"_\n\n"
          else:
             options = options+pad+pad+line.strip()+"\n\n"

    details=help[3]
    examples=''
    also=''

    # search for see also section
    # if it exists, truncate details and process
    q= re.compile(u'See also:\n(.+?)$',re.DOTALL)
    x = re.search( q, details)
    if x:
      match=details[x.start()+10:].split('\n')
      details=details[:x.start()] # truncate the bit we matched
      also=pad+'**See also:**\n\n'

      for line in match:
          also = also+pad+pad+'`'+line.strip()+'`\n\n'

    # search for Examples section
    # if it exists, truncate details and process
    q= re.compile(u'Examples:\n(.+?)$',re.DOTALL)
    x = re.search( q, details)
    if x:
      match=details[x.start()+10:].split('\n')
      details=details[:x.start()] # truncate the bit we matched
      examples=pad+'**Examples:**\n\n'

      for line in match:
          if (line[0]==' '):
            examples=examples+pad+pad
          examples = examples+pad+line.strip()+'\n'

    outfile.write(usage+summary+options+details+examples+also)
<|endoftext|>


<commit_before>__all__ = ['VisualizationFrame']

import numpy as np
from sympy import Dummy, lambdify
from sympy.matrices.expressions import Identity
from sympy.physics.mechanics import Point, ReferenceFrame

from .shapes import Shape

class VisualizationFrame(object):
    """
    A VisualizationFrame represents an object that you want to visualize.
    It allows you to easily associate a reference frame and a point
    with a shape.

    A VisualizationFrame can be attached to only one Shape Object.
    It can be nested, i.e we can add/remove multiple visualization frames to
    one visualization frame. On adding the parent frame to the
    Scene object, all the children of the parent visualization frame
    are also added, and hence can be visualized and animated.

    A VisualizationFrame needs to have a ReferenceFrame, and a Point
    for it to form transformation matrices for visualization and
    animations.

    The ReferenceFrame and Point are required to be provided during
    initialization. They can be supplied in the form of any one of these:

    1)reference_frame, point argument.
    2)a RigidBody argument
    3)reference_frame, particle argument.

    In addition to these arguments, A shape argument is also required.
    """
    def __init__(self, *args):
        """
        To initialize a visualization frame a ReferenceFrame,
        Point, and Shape are required. These ReferenceFrame
        and Point can be passed provided in three ways:

        1) RigidBody: the RigidBody's frame and mass center are used.
        2) ReferenceFrame and a Particle: The Particle's Point is used.
        3) ReferenceFrame and a Point

        Parameters
        ==========
        name : str, optional
            Name assigned to VisualizationFrame, default is unnamed
        reference_frame : ReferenceFrame
            A reference_frame with respect to which all orientations of the
            shape takes place, during visualizations/animations.
        origin : Point
            A point with respect to which all the translations of the shape
            takes place, during visualizations/animations.
        rigidbody : RigidBody
            A rigidbody whose reference frame and mass center are to be
            assigned as reference_frame and origin of the
            VisualizationFrame.
        particle : Particle
            A particle whose point is assigned as origin of the
            VisualizationFrame.
        shape : Shape
            A shape to be attached to the VisualizationFrame

        Examples
        ========
        >>> from pydy.viz import VisualizationFrame, Sphere
        >>> from sympy.physics.mechanics import \
                               ReferenceFrame, Point, RigidBody, \
                                Particle, inertia
        >>> from sympy import symbols
        >>> I = ReferenceFrame('I')
        >>> O = Point('O')
        >>> shape = Sphere(5)
        >>> #initializing with reference frame, point
        >>> frame1 = VisualizationFrame('frame1', I, O, shape)
        >>> Ixx, Iyy, Izz, mass = symbols('Ixx Iyy Izz mass')
        >>> i = inertia(I, Ixx, Iyy, Izz)
        >>> rbody = RigidBody('rbody', O, I, mass, (inertia, O))
        >>> # Initializing with a rigidbody ..
        >>> frame2 = VisualizationFrame('frame2', rbody, shape)
        >>> Pa = Particle('Pa', O, mass)
        >>> #initializing with Particle, reference_frame ...
        >>> frame3 = VisualizationFrame('frame3', I, Pa, shape)
        """
        #Last arg should be a Shape ..
        if isinstance(args[-1], Shape):
            self._shape = args[-1]
        else:
            raise TypeError('''Please provide a valid shape object''')
        i = 0
        #If first arg is not str, name the visualization frame 'unnamed'
        if isinstance(args[i], str):
            self._name = args[i]
            i += 1
        else:
            self._name = 'unnamed'

        try:
            self._reference_frame = args[i].get_frame()
            self._origin = args[i].get_masscenter()

        except AttributeError:
            #It is not a rigidbody, hence this arg should be a
            #reference frame
            try:
                dcm = args[i]._dcm_dict
                self._reference_frame = args[i]
                i += 1
            except AttributeError:
                raise TypeError(''' A ReferenceFrame is to be supplied
                                   before a Particle/Point. ''')

            #Now next arg can either be a Particle or point
            try:
                self._origin = args[i].get_point()

            except AttributeError:
                self._origin = args[i]

    #setting attributes ..
    def __str__(self):
        return 'VisualizationFrame ' + self._name

    def __repr__(self):
        return 'VisualizationFrame'

    @property
    def name(self):
        """
        Name of the VisualizationFrame.
        """
        return self._name

    @name.setter
    def name(self, new_name):
        """
        Sets the name of the VisualizationFrame.

        """
        if not isinstance(new_name, str):
            raise TypeError('''Name should be a str object''')
        else:
            self._name = new_name

    @property
    def origin(self):
        """
        Origin of the VisualizationFrame,
        with respect to which all translational transformations
        take place.
        """
        return self._origin

    @origin.setter
    def origin(self, new_origin):
        """
        Sets the origin of the VisualizationFrame.
        """
        if not isinstance(new_origin, Point):
            raise TypeError('''origin should be a valid Point Object''')
        else:
            self._origin = new_origin

    @property
    def reference_frame(self):
        """
        reference_frame of the VisualizationFrame,
        with respect to which all rotational/orientational
        transformations take place.
        """
        return self._reference_frame

    @reference_frame.setter
    def reference_frame(self, new_reference_frame):
        if not isinstance(new_reference_frame, ReferenceFrame):
            raise TypeError('''reference_frame should be a valid
                                ReferenceFrame object.''')
        else:
            self._reference_frame = new_reference_frame

    @property
    def shape(self):
        """
        shape in the VisualizationFrame.
        A shape attached to the visualization frame.
        NOTE: Only one shape can be attached to a visualization frame.
        """
        return self._shape

    @shape.setter
    def shape(self, new_shape):
        """
        Sets the shape for VisualizationFrame.
        """
        if not isinstance(new_shape, Shape):
            raise TypeError('''shape should be a valid Shape object.''')
        else:
            self._shape = new_shape

    def generate_transformation_matrix(self, reference_frame, point):
        """Generates a symbolic transformation matrix, with respect to the
        provided reference frame and point.

        Parameters
        ==========
        reference_frame : ReferenceFrame
            A reference_frame with respect to which transformation matrix is
            generated.
        point : Point
            A point with respect to which transformation matrix is
            generated.

        Returns
        =======
        A 4 x 4 SymPy matrix, containing symbolic expressions describing the
        transformation as a function of time.

        """
        rotation_matrix = self.reference_frame.dcm(reference_frame)
        self._transform = Identity(4).as_mutable()
        self._transform[0:3, 0:3] = rotation_matrix[0:3, 0:3]

        _point_vector = self.origin.pos_from(point).express(reference_frame)

        self._transform[3, 0] = _point_vector.dot(reference_frame.x)
        self._transform[3, 1] = _point_vector.dot(reference_frame.y)
        self._transform[3, 2] = _point_vector.dot(reference_frame.z)
        return self._transform

    def generate_numeric_transform_function(self, dynamic_variables,
                                            constant_variables):
        """Returns a function which can compute the numerical values of the
        transformation matrix given the numerical dynamic variables (i.e.
        functions of time or states) and the numerical system constants.

        Parameters
        ==========
        dynamic_variables : list of sympy.Functions(time)
            All of the dynamic symbols used in defining the orientation and
            position of this visualization frame.
        constant_variables : list of sympy.Symbols
            All of the constants used in defining the orientation and
            position of this visualization frame.

        Returns
        =======
        numeric_transform : function
            A function which returns the numerical transformation matrix.

        """

        dummy_symbols = [Dummy() for i in dynamic_variables]
        dummy_dict = dict(zip(dynamic_variables, dummy_symbols))
        transform = self._transform.subs(dummy_dict)

        self._numeric_transform = lambdify(dummy_symbols +
                                           constant_variables, transform,
                                           modules="numpy")

        return self._numeric_transform

    def evaluate_transformation_matrix(self, dynamic_values, constant_values):
        """Returns the numerical transformation matrices for each time step.

        Parameters
        ----------
        dynamic_values : array_like, shape(m,) or shape(n, m)
            The m state values for each n time step.
        constant_values : array_like, shape(p,)
            The p constant parameter values of the system.

        Returns
        -------
        transform_matrix : numpy.array, shape(n, 4, 4)
            A 4 x 4 transformation matrix for each time step.

        """
        #If states is instance of numpy array, well and good.
        #else convert it to one:

        states = np.array(dynamic_values)
        if len(states.shape) > 1:
            n = states.shape[0]
            new = np.zeros((n, 4, 4))
            for i, time_instance in enumerate(states):
                args = np.hstack((time_instance, constant_values))
                new[i, :, :] = self._numeric_transform(*args)
        else:
            n = 1
            args = np.hstack((states, constant_values))
            new = self._numeric_transform(*args)

        self._visualization_matrix = new.reshape(n, 16)
        return self._visualization_matrix

    def generate_visualization_dict(self, constant_map={}):
        """Returns a dictionary of all the info required for the
        visualization of this frame.

        Before calling this method, all the transformation matrix generation
        methods should be called, or it will give an error.

        Parameters
        ==========
        constant_map : dictionary
            If the shape associated with this visualization frame has
            symbolic values for its geometric parameters, then you must
            supply a dictionary mapping the necessary SymPy symbols to
            Python floats.

        Returns
        =======
        data : dictionary
            The dictionary contains the following keys:
            name : string
                The name of the VisualizationFrame.
            shape : dictionary
                A dictionary generated from the associated Shape.
            simulation_matrix : list
                The N x 4 x 4 array provided as a list of lists of lists. N
                is the number of time steps and the 4 x 4 matrix at each
                time step represents the transformation matrix for animation
                purposes.

        """
        data = {}
        data['name'] = self.name
        data['shape'] = self.shape.generate_dict(constant_map=constant_map)
        try:
            data['simulation_matrix'] = self._visualization_matrix.tolist()
        except:
            raise RuntimeError("Please call the numerical ",
                               "transformation methods, ",
                               "before generating visualization dict")

        return data
<commit_msg>broke json generation method into two methods, one for generating scene information, and other for generating simulation information<commit_after>__all__ = ['VisualizationFrame']

import numpy as np
from sympy import Dummy, lambdify
from sympy.matrices.expressions import Identity
from sympy.physics.mechanics import Point, ReferenceFrame

from .shapes import Shape

class VisualizationFrame(object):
    """
    A VisualizationFrame represents an object that you want to visualize.
    It allows you to easily associate a reference frame and a point
    with a shape.

    A VisualizationFrame can be attached to only one Shape Object.
    It can be nested, i.e we can add/remove multiple visualization frames to
    one visualization frame. On adding the parent frame to the
    Scene object, all the children of the parent visualization frame
    are also added, and hence can be visualized and animated.

    A VisualizationFrame needs to have a ReferenceFrame, and a Point
    for it to form transformation matrices for visualization and
    animations.

    The ReferenceFrame and Point are required to be provided during
    initialization. They can be supplied in the form of any one of these:

    1)reference_frame, point argument.
    2)a RigidBody argument
    3)reference_frame, particle argument.

    In addition to these arguments, A shape argument is also required.
    """
    def __init__(self, *args):
        """
        To initialize a visualization frame a ReferenceFrame,
        Point, and Shape are required. These ReferenceFrame
        and Point can be passed provided in three ways:

        1) RigidBody: the RigidBody's frame and mass center are used.
        2) ReferenceFrame and a Particle: The Particle's Point is used.
        3) ReferenceFrame and a Point

        Parameters
        ==========
        name : str, optional
            Name assigned to VisualizationFrame, default is unnamed
        reference_frame : ReferenceFrame
            A reference_frame with respect to which all orientations of the
            shape takes place, during visualizations/animations.
        origin : Point
            A point with respect to which all the translations of the shape
            takes place, during visualizations/animations.
        rigidbody : RigidBody
            A rigidbody whose reference frame and mass center are to be
            assigned as reference_frame and origin of the
            VisualizationFrame.
        particle : Particle
            A particle whose point is assigned as origin of the
            VisualizationFrame.
        shape : Shape
            A shape to be attached to the VisualizationFrame

        Examples
        ========
        >>> from pydy.viz import VisualizationFrame, Sphere
        >>> from sympy.physics.mechanics import \
                               ReferenceFrame, Point, RigidBody, \
                                Particle, inertia
        >>> from sympy import symbols
        >>> I = ReferenceFrame('I')
        >>> O = Point('O')
        >>> shape = Sphere(5)
        >>> #initializing with reference frame, point
        >>> frame1 = VisualizationFrame('frame1', I, O, shape)
        >>> Ixx, Iyy, Izz, mass = symbols('Ixx Iyy Izz mass')
        >>> i = inertia(I, Ixx, Iyy, Izz)
        >>> rbody = RigidBody('rbody', O, I, mass, (inertia, O))
        >>> # Initializing with a rigidbody ..
        >>> frame2 = VisualizationFrame('frame2', rbody, shape)
        >>> Pa = Particle('Pa', O, mass)
        >>> #initializing with Particle, reference_frame ...
        >>> frame3 = VisualizationFrame('frame3', I, Pa, shape)
        """
        #Last arg should be a Shape ..
        if isinstance(args[-1], Shape):
            self._shape = args[-1]
        else:
            raise TypeError('''Please provide a valid shape object''')
        i = 0
        #If first arg is not str, name the visualization frame 'unnamed'
        if isinstance(args[i], str):
            self._name = args[i]
            i += 1
        else:
            self._name = 'unnamed'

        try:
            self._reference_frame = args[i].get_frame()
            self._origin = args[i].get_masscenter()

        except AttributeError:
            #It is not a rigidbody, hence this arg should be a
            #reference frame
            try:
                dcm = args[i]._dcm_dict
                self._reference_frame = args[i]
                i += 1
            except AttributeError:
                raise TypeError(''' A ReferenceFrame is to be supplied
                                   before a Particle/Point. ''')

            #Now next arg can either be a Particle or point
            try:
                self._origin = args[i].get_point()

            except AttributeError:
                self._origin = args[i]

    #setting attributes ..
    def __str__(self):
        return 'VisualizationFrame ' + self._name

    def __repr__(self):
        return 'VisualizationFrame'

    @property
    def name(self):
        """
        Name of the VisualizationFrame.
        """
        return self._name

    @name.setter
    def name(self, new_name):
        """
        Sets the name of the VisualizationFrame.

        """
        if not isinstance(new_name, str):
            raise TypeError('''Name should be a str object''')
        else:
            self._name = new_name

    @property
    def origin(self):
        """
        Origin of the VisualizationFrame,
        with respect to which all translational transformations
        take place.
        """
        return self._origin

    @origin.setter
    def origin(self, new_origin):
        """
        Sets the origin of the VisualizationFrame.
        """
        if not isinstance(new_origin, Point):
            raise TypeError('''origin should be a valid Point Object''')
        else:
            self._origin = new_origin

    @property
    def reference_frame(self):
        """
        reference_frame of the VisualizationFrame,
        with respect to which all rotational/orientational
        transformations take place.
        """
        return self._reference_frame

    @reference_frame.setter
    def reference_frame(self, new_reference_frame):
        if not isinstance(new_reference_frame, ReferenceFrame):
            raise TypeError('''reference_frame should be a valid
                                ReferenceFrame object.''')
        else:
            self._reference_frame = new_reference_frame

    @property
    def shape(self):
        """
        shape in the VisualizationFrame.
        A shape attached to the visualization frame.
        NOTE: Only one shape can be attached to a visualization frame.
        """
        return self._shape

    @shape.setter
    def shape(self, new_shape):
        """
        Sets the shape for VisualizationFrame.
        """
        if not isinstance(new_shape, Shape):
            raise TypeError('''shape should be a valid Shape object.''')
        else:
            self._shape = new_shape

    def generate_transformation_matrix(self, reference_frame, point):
        """Generates a symbolic transformation matrix, with respect to the
        provided reference frame and point.

        Parameters
        ==========
        reference_frame : ReferenceFrame
            A reference_frame with respect to which transformation matrix is
            generated.
        point : Point
            A point with respect to which transformation matrix is
            generated.

        Returns
        =======
        A 4 x 4 SymPy matrix, containing symbolic expressions describing the
        transformation as a function of time.

        """
        rotation_matrix = self.reference_frame.dcm(reference_frame)
        self._transform = Identity(4).as_mutable()
        self._transform[0:3, 0:3] = rotation_matrix[0:3, 0:3]

        _point_vector = self.origin.pos_from(point).express(reference_frame)

        self._transform[3, 0] = _point_vector.dot(reference_frame.x)
        self._transform[3, 1] = _point_vector.dot(reference_frame.y)
        self._transform[3, 2] = _point_vector.dot(reference_frame.z)
        return self._transform

    def generate_numeric_transform_function(self, dynamic_variables,
                                            constant_variables):
        """Returns a function which can compute the numerical values of the
        transformation matrix given the numerical dynamic variables (i.e.
        functions of time or states) and the numerical system constants.

        Parameters
        ==========
        dynamic_variables : list of sympy.Functions(time)
            All of the dynamic symbols used in defining the orientation and
            position of this visualization frame.
        constant_variables : list of sympy.Symbols
            All of the constants used in defining the orientation and
            position of this visualization frame.

        Returns
        =======
        numeric_transform : function
            A function which returns the numerical transformation matrix.

        """

        dummy_symbols = [Dummy() for i in dynamic_variables]
        dummy_dict = dict(zip(dynamic_variables, dummy_symbols))
        transform = self._transform.subs(dummy_dict)

        self._numeric_transform = lambdify(dummy_symbols +
                                           constant_variables, transform,
                                           modules="numpy")

        return self._numeric_transform

    def evaluate_transformation_matrix(self, dynamic_values, constant_values):
        """Returns the numerical transformation matrices for each time step.

        Parameters
        ----------
        dynamic_values : array_like, shape(m,) or shape(n, m)
            The m state values for each n time step.
        constant_values : array_like, shape(p,)
            The p constant parameter values of the system.

        Returns
        -------
        transform_matrix : numpy.array, shape(n, 4, 4)
            A 4 x 4 transformation matrix for each time step.

        """
        #If states is instance of numpy array, well and good.
        #else convert it to one:

        states = np.array(dynamic_values)
        if len(states.shape) > 1:
            n = states.shape[0]
            new = np.zeros((n, 4, 4))
            for i, time_instance in enumerate(states):
                args = np.hstack((time_instance, constant_values))
                new[i, :, :] = self._numeric_transform(*args)
        else:
            n = 1
            args = np.hstack((states, constant_values))
            new = self._numeric_transform(*args)

        self._visualization_matrix = new.reshape(n, 16)
        return self._visualization_matrix

    def generate_scene_dict(self, constant_map={}):
    """
    This method generates information for a static
    visualization in the initial conditions, in the form
    of dictionary. This contains shape information
    from `Shape.generate_dict()` followed by an
    init_orientation Key.

    Parameters
    ==========
    constant_map : dictionary
    Constant map is required when Shape contains sympy expressions.This
    dictionary maps sympy expressions/symbols to numerical values(floats)

    """
        _scene_dict = self.shape.generate_dict(constant_map=constant_map)
        _scene_dict["init_orientation"] = self._visualization_matrix.tolist()[0]
        _scene_dict["reference_frame_name"] = self._reference_frame
        _scene_dict["simulation_id"] = id(self)

        return _scene_dict

    def generate_simulation_dict(self):
    """
    Generates the simulation information for this visualization
    frame. It maps the simulation data information to the
    scene information via a unique id.

    """
        try:
            _simulation_dict[id(self)] = self._visualization_matrix.tolist()

        except:
            raise RuntimeError("Please call the numerical ",
                               "transformation methods, ",
                               "before generating visualization dict")

        return _simulation_dict

<|endoftext|>


<commit_before>from typing import TYPE_CHECKING

from django.conf import settings
from django.core.exceptions import ValidationError
from django_prices.utils.formatting import get_currency_fraction
from graphql.error import GraphQLError

if TYPE_CHECKING:
    from decimal import Decimal

def validate_one_of_args_is_in_query(arg1_name, arg1, arg2_name, arg2):
    if arg1 and arg2:
        raise GraphQLError(
            f"Argument '{arg1_name}' cannot be combined with '{arg2_name}'"
        )
    if not arg1 and not arg2:
        raise GraphQLError(
            f"Either '{arg1_name}'  or '{arg2_name}' argument is required"
        )

def validate_price_amount(value: "Decimal", currency: str = None):
    """Validate if price amount does not have too many decimal places.

    Price amount can't have more decimal places than currency allow to.
    """

    if not value:
        return
    if currency:
        currency_fraction = get_currency_fraction(currency)
    else:
        currency_fraction = get_currency_fraction(settings.DEFAULT_CURRENCY)
    value = value.normalize()
    if abs(value.as_tuple().exponent) > currency_fraction:
        raise ValidationError(
            f"Value cannot have more than {currency_fraction} decimal places."
        )
<commit_msg>Apply code review suggestions - update validate_price_amount validator<commit_after>from typing import TYPE_CHECKING, Optional

from django.conf import settings
from django.core.exceptions import ValidationError
from django_prices.utils.formatting import get_currency_fraction
from graphql.error import GraphQLError

if TYPE_CHECKING:
    from decimal import Decimal

def validate_one_of_args_is_in_query(arg1_name, arg1, arg2_name, arg2):
    if arg1 and arg2:
        raise GraphQLError(
            f"Argument '{arg1_name}' cannot be combined with '{arg2_name}'"
        )
    if not arg1 and not arg2:
        raise GraphQLError(
            f"Either '{arg1_name}'  or '{arg2_name}' argument is required"
        )

def validate_price_amount(value: Optional["Decimal"], currency: str = None):
    """Validate if price amount does not have too many decimal places.

    Price amount can't have more decimal places than currency allow to.
    Works only with decimal created from a string.
    """

    # check no needed when there is no value
    if not value:
        return

    currency_fraction = get_currency_fraction(currency or settings.DEFAULT_CURRENCY)
    value = value.normalize()
    if abs(value.as_tuple().exponent) > currency_fraction:
        raise ValidationError(
            f"Value cannot have more than {currency_fraction} decimal places."
        )
<|endoftext|>


<commit_before>import datetime
import feedparser
import logging
import lxml.html
import pytz
import requests
import socket
import time
import urllib2
import urlparse

from django.conf import settings
from django.db import transaction
from django.db.models import F
from django.utils import timezone

from django_push.subscriber.models import Subscription

from .. import __version__

USER_AGENT = 'FeedHQ/%s +https://github.org/brutasse/feedhq' % __version__

logger = logging.getLogger('feedupdater')

class FeedUpdater(object):

    def __init__(self, url, agent=' (1 subscriber)', feedparser=feedparser):
        self.url = url
        self.feedparser = feedparser
        self.updated = {}
        self.feeds = None
        feedparser.USER_AGENT = USER_AGENT
        feedparser.USER_AGENT += agent

    def update(self):
        self.get_feeds()
        self.get_entries()
        self.add_entries_to_feeds()
        self.handle_updated()
        self.remove_old_stuff()
        self.update_counts()

    def get_feeds(self):
        if self.feeds is None:
            from .models import Feed
            self.feeds = Feed.objects.filter(url=self.url, muted=False)

    def get_entries(self):
        """Populates self.entries and self.updated
        self.entries: a list of Entry objects, parsed from self.url
        self.updated: a dict of values to push to self.feeds
        """
        from .models import Entry
        socket.setdefaulttimeout(5)  # aggressive but otherwise
                                     # it can take ages
        feed = self.feeds[0]
        if feed.modified:
            modified = time.localtime(float(feed.modified))
            parsed_feed = self.feedparser.parse(feed.url, etag=feed.etag,
                                                modified=modified)
        else:
            parsed_feed = self.feedparser.parse(feed.url)

        if not 'status' in parsed_feed:
            logger.debug("No status in parsed feed, %s: %s" % (feed.pk,
                                                               feed.url))
            if feed.failed_attempts >= 20:
                logger.info("Feed failed 20 times, muting %s: %s" % (feed.pk,
                                                                     feed.url))
                self.feeds.update(muted=True)
            self.feeds.filter(url=feed.url).update(
                failed_attempts=F('failed_attempts') + 1
            )
            self.entries = []
            return
        self.feeds.update(failed_attempts=0)

        if parsed_feed.status == 301:  # permanent redirect
            self.updated['url'] = parsed_feed.href

        if parsed_feed.status == 410:  # Gone
            logger.info("Feed gone, %s: %s" % (feed.pk, feed.url))
            self.updated['muted'] = True
            self.entries = []
            return

        if parsed_feed.status == 304:  # Not modified
            logger.debug("Feed not modified, %s" % feed.url)
            self.entries = []
            return

        if ('link' in parsed_feed.feed and
            not feed.link == parsed_feed.feed.link):
            self.updated['link'] = parsed_feed.feed.link

        if ('title' in parsed_feed.feed and
            not feed.title == parsed_feed.feed.title):
            self.updated['title'] = parsed_feed.feed.title

        if 'etag' in parsed_feed:
            self.updated['etag'] = parsed_feed.etag
        if 'modified_parsed' in parsed_feed:
            timed = time.mktime(parsed_feed.modified_parsed)
            self.updated['modified'] = '%s' % timed

        if 'links' in parsed_feed.feed:
            for link in parsed_feed.feed.links:
                if link.rel == 'hub':
                    self.handle_hub(parsed_feed.href, link.href)

        self.entries = []
        for entry in parsed_feed.entries:
            if not 'link' in entry:
                continue
            title = entry.title
            if len(title) > 255:
                title = title[:252] + '...'
            parsed_entry = Entry(title=title)
            if 'description' in entry:
                parsed_entry.subtitle = entry.description
            if 'summary' in entry:
                parsed_entry.subtitle = entry.summary
            if 'content' in entry:  # this overrides the summary
                if entry.content:
                    parsed_entry.subtitle = ''
                    for content in entry.content:
                        parsed_entry.subtitle += content.value

            parsed_entry.subtitle = self.clean_content(
                parsed_entry.subtitle,
            )

            parsed_entry.link = entry.link
            if 'guid' in entry:
                parsed_guid = urlparse.urlparse(entry.guid)
                parsed_link = urlparse.urlparse(parsed_feed.feed.link)
                if (parsed_guid.scheme in ('http', 'https') and
                    parsed_guid.netloc == parsed_link.netloc):
                    parsed_entry.guid = entry.guid

            if not parsed_entry.id:
                # Update some fields only if the entry is a new one
                parsed_entry.date = self.get_date(entry)

            self.entries.append(parsed_entry)

    def handle_hub(self, topic_url, hub_url):
        """
        Initiates a PubSubHubbub subscription and
        renews the lease if necessary.
        """
        if settings.DEBUG or settings.TESTS:
            # Do not use PubSubHubbub on local development
            return

        subscriptions = Subscription.objects.filter(topic=topic_url)
        if not subscriptions.exists():
            logger.info("Subscribing to %s: %s" % (topic_url, hub_url))
            subscriptions = [Subscription.objects.subscribe(topic_url,
                                                            hub=hub_url)]

        for subscription in subscriptions:
            if subscription.lease_expiration is None:
                continue

            if subscription.lease_expiration < timezone.now():
                logger.info("Renewing lease for %s: %s" % (topic_url, hub_url))
                try:
                    subscription = Subscription.objects.subscribe(
                        subscription.topic,
                        subscription.hub
                    )
                except urllib2.URLError:
                    pass

    def get_date(self, entry):
        if 'published_parsed' in entry and entry.published_parsed is not None:
            field = entry.published_parsed
        elif 'updated_parsed' in entry and entry.updated_parsed is not None:
            field = entry.updated_parsed
        else:
            field = None

        if field is None:
            entry_date = timezone.now()
        else:
            entry_date = timezone.make_aware(
                datetime.datetime(*field[:6]),
                pytz.utc,
            )
            # Sometimes entries are published in the future. If they're
            # published, it's probably safe to adjust the date.
            if entry_date > timezone.now():
                entry_date = timezone.now()
        return entry_date

    @transaction.commit_on_success
    def add_entries_to_feeds(self):
        from .models import Entry
        for entry in self.entries:
            for feed in self.feeds:
                treshold = feed.get_treshold()
                if treshold is not None and entry.date < treshold:
                    # Skipping, it's too old
                    continue

                if not entry.link:
                    params = {'title__iexact': entry.title, 'feed': feed}
                else:
                    params = {'link__iexact': entry.link, 'feed': feed}

                try:
                    db_entry = Entry.objects.get(**params)
                except Entry.DoesNotExist:
                    db_entry = entry
                except Entry.MultipleObjectsReturned:
                    multiple = Entry.objects.filter(**params).order_by('date')
                    db_entry = multiple[0]
                    for e in multiple[1:]:
                        e.delete()

                if db_entry.permalink:
                    entry.permalink = db_entry.permalink
                else:
                    if entry.permalink:
                        db_entry.permalink = entry.permalink
                    elif not settings.TESTS:
                        # Try to use guid if possible
                        if hasattr(entry, 'guid'):
                            response = requests.head(entry.guid,
                                                     allow_redirects=True)
                            if response.status_code == 200:
                                resolved = response.url or ''
                                db_entry.permalink = entry.permalink = resolved

                        if (db_entry.link and
                            'feedproxy.google.com' in db_entry.link):
                            # Handling the FeedBurner redirection on
                            # behalf of the user
                            response = requests.head(db_entry.link,
                                                     allow_redirects=True)
                            resolved = response.url or ''
                            db_entry.permalink = entry.permalink = resolved

                if not db_entry.permalink:
                    db_entry.permalink = entry.permalink = entry.link

                db_entry.feed = feed
                db_entry.user = feed.category.user
                db_entry.save()
                feed.update_unread_count()

    def clean_content(self, content):
        page = lxml.html.fromstring('<div>%s</div>' % content)
        for element in page.iter('img'):
            if ('width="1"' in lxml.etree.tostring(element) or
                'width="0"' in lxml.etree.tostring(element)):
                # Tracking image -- deleting
                element.drop_tree()
        return lxml.etree.tostring(page)

    def handle_updated(self):
        self.feeds.update(**self.updated)

    def remove_old_stuff(self):
        """
        Gets rid of the `old` stuff, if the user doesn't want to keep the
        content in the archive.
        """
        from .models import Entry
        for feed in self.feeds:
            treshold = feed.get_treshold()
            if treshold is None:
                continue

            to_delete = Entry.objects.filter(feed=feed, date__lte=treshold)
            to_delete.delete()

    def update_counts(self):
        for feed in self.feeds:
            feed.update_unread_count()
<commit_msg>Handle entries with no title<commit_after>import datetime
import feedparser
import logging
import lxml.html
import pytz
import requests
import socket
import time
import urllib2
import urlparse

from django.conf import settings
from django.db import transaction
from django.db.models import F
from django.utils import timezone

from django_push.subscriber.models import Subscription

from .. import __version__

USER_AGENT = 'FeedHQ/%s +https://github.org/brutasse/feedhq' % __version__

logger = logging.getLogger('feedupdater')

class FeedUpdater(object):

    def __init__(self, url, agent=' (1 subscriber)', feedparser=feedparser):
        self.url = url
        self.feedparser = feedparser
        self.updated = {}
        self.feeds = None
        feedparser.USER_AGENT = USER_AGENT
        feedparser.USER_AGENT += agent

    def update(self):
        self.get_feeds()
        self.get_entries()
        self.add_entries_to_feeds()
        self.handle_updated()
        self.remove_old_stuff()
        self.update_counts()

    def get_feeds(self):
        if self.feeds is None:
            from .models import Feed
            self.feeds = Feed.objects.filter(url=self.url, muted=False)

    def get_entries(self):
        """Populates self.entries and self.updated
        self.entries: a list of Entry objects, parsed from self.url
        self.updated: a dict of values to push to self.feeds
        """
        from .models import Entry
        socket.setdefaulttimeout(5)  # aggressive but otherwise
                                     # it can take ages
        feed = self.feeds[0]
        if feed.modified:
            modified = time.localtime(float(feed.modified))
            parsed_feed = self.feedparser.parse(feed.url, etag=feed.etag,
                                                modified=modified)
        else:
            parsed_feed = self.feedparser.parse(feed.url)

        if not 'status' in parsed_feed:
            logger.debug("No status in parsed feed, %s: %s" % (feed.pk,
                                                               feed.url))
            if feed.failed_attempts >= 20:
                logger.info("Feed failed 20 times, muting %s: %s" % (feed.pk,
                                                                     feed.url))
                self.feeds.update(muted=True)
            self.feeds.filter(url=feed.url).update(
                failed_attempts=F('failed_attempts') + 1
            )
            self.entries = []
            return
        self.feeds.update(failed_attempts=0)

        if parsed_feed.status == 301:  # permanent redirect
            self.updated['url'] = parsed_feed.href

        if parsed_feed.status == 410:  # Gone
            logger.info("Feed gone, %s: %s" % (feed.pk, feed.url))
            self.updated['muted'] = True
            self.entries = []
            return

        if parsed_feed.status == 304:  # Not modified
            logger.debug("Feed not modified, %s" % feed.url)
            self.entries = []
            return

        if ('link' in parsed_feed.feed and
            not feed.link == parsed_feed.feed.link):
            self.updated['link'] = parsed_feed.feed.link

        if ('title' in parsed_feed.feed and
            not feed.title == parsed_feed.feed.title):
            self.updated['title'] = parsed_feed.feed.title

        if 'etag' in parsed_feed:
            self.updated['etag'] = parsed_feed.etag
        if 'modified_parsed' in parsed_feed:
            timed = time.mktime(parsed_feed.modified_parsed)
            self.updated['modified'] = '%s' % timed

        if 'links' in parsed_feed.feed:
            for link in parsed_feed.feed.links:
                if link.rel == 'hub':
                    self.handle_hub(parsed_feed.href, link.href)

        self.entries = []
        for entry in parsed_feed.entries:
            if not 'link' in entry:
                continue
            title = entry.title if 'title' in entry else '(No title)'
            if len(title) > 255:
                title = title[:252] + '...'
            parsed_entry = Entry(title=title)
            if 'description' in entry:
                parsed_entry.subtitle = entry.description
            if 'summary' in entry:
                parsed_entry.subtitle = entry.summary
            if 'content' in entry:  # this overrides the summary
                if entry.content:
                    parsed_entry.subtitle = ''
                    for content in entry.content:
                        parsed_entry.subtitle += content.value

            parsed_entry.subtitle = self.clean_content(
                parsed_entry.subtitle,
            )

            parsed_entry.link = entry.link
            if 'guid' in entry:
                parsed_guid = urlparse.urlparse(entry.guid)
                parsed_link = urlparse.urlparse(parsed_feed.feed.link)
                if (parsed_guid.scheme in ('http', 'https') and
                    parsed_guid.netloc == parsed_link.netloc):
                    parsed_entry.guid = entry.guid

            if not parsed_entry.id:
                # Update some fields only if the entry is a new one
                parsed_entry.date = self.get_date(entry)

            self.entries.append(parsed_entry)

    def handle_hub(self, topic_url, hub_url):
        """
        Initiates a PubSubHubbub subscription and
        renews the lease if necessary.
        """
        if settings.DEBUG or settings.TESTS:
            # Do not use PubSubHubbub on local development
            return

        subscriptions = Subscription.objects.filter(topic=topic_url)
        if not subscriptions.exists():
            logger.info("Subscribing to %s: %s" % (topic_url, hub_url))
            subscriptions = [Subscription.objects.subscribe(topic_url,
                                                            hub=hub_url)]

        for subscription in subscriptions:
            if subscription.lease_expiration is None:
                continue

            if subscription.lease_expiration < timezone.now():
                logger.info("Renewing lease for %s: %s" % (topic_url, hub_url))
                try:
                    subscription = Subscription.objects.subscribe(
                        subscription.topic,
                        subscription.hub
                    )
                except urllib2.URLError:
                    pass

    def get_date(self, entry):
        if 'published_parsed' in entry and entry.published_parsed is not None:
            field = entry.published_parsed
        elif 'updated_parsed' in entry and entry.updated_parsed is not None:
            field = entry.updated_parsed
        else:
            field = None

        if field is None:
            entry_date = timezone.now()
        else:
            entry_date = timezone.make_aware(
                datetime.datetime(*field[:6]),
                pytz.utc,
            )
            # Sometimes entries are published in the future. If they're
            # published, it's probably safe to adjust the date.
            if entry_date > timezone.now():
                entry_date = timezone.now()
        return entry_date

    @transaction.commit_on_success
    def add_entries_to_feeds(self):
        from .models import Entry
        for entry in self.entries:
            for feed in self.feeds:
                treshold = feed.get_treshold()
                if treshold is not None and entry.date < treshold:
                    # Skipping, it's too old
                    continue

                if not entry.link:
                    params = {'title__iexact': entry.title, 'feed': feed}
                else:
                    params = {'link__iexact': entry.link, 'feed': feed}

                try:
                    db_entry = Entry.objects.get(**params)
                except Entry.DoesNotExist:
                    db_entry = entry
                except Entry.MultipleObjectsReturned:
                    multiple = Entry.objects.filter(**params).order_by('date')
                    db_entry = multiple[0]
                    for e in multiple[1:]:
                        e.delete()

                if db_entry.permalink:
                    entry.permalink = db_entry.permalink
                else:
                    if entry.permalink:
                        db_entry.permalink = entry.permalink
                    elif not settings.TESTS:
                        # Try to use guid if possible
                        if hasattr(entry, 'guid'):
                            response = requests.head(entry.guid,
                                                     allow_redirects=True)
                            if response.status_code == 200:
                                resolved = response.url or ''
                                db_entry.permalink = entry.permalink = resolved

                        if (db_entry.link and
                            'feedproxy.google.com' in db_entry.link):
                            # Handling the FeedBurner redirection on
                            # behalf of the user
                            response = requests.head(db_entry.link,
                                                     allow_redirects=True)
                            resolved = response.url or ''
                            db_entry.permalink = entry.permalink = resolved

                if not db_entry.permalink:
                    db_entry.permalink = entry.permalink = entry.link

                db_entry.feed = feed
                db_entry.user = feed.category.user
                db_entry.save()
                feed.update_unread_count()

    def clean_content(self, content):
        page = lxml.html.fromstring('<div>%s</div>' % content)
        for element in page.iter('img'):
            if ('width="1"' in lxml.etree.tostring(element) or
                'width="0"' in lxml.etree.tostring(element)):
                # Tracking image -- deleting
                element.drop_tree()
        return lxml.etree.tostring(page)

    def handle_updated(self):
        self.feeds.update(**self.updated)

    def remove_old_stuff(self):
        """
        Gets rid of the `old` stuff, if the user doesn't want to keep the
        content in the archive.
        """
        from .models import Entry
        for feed in self.feeds:
            treshold = feed.get_treshold()
            if treshold is None:
                continue

            to_delete = Entry.objects.filter(feed=feed, date__lte=treshold)
            to_delete.delete()

    def update_counts(self):
        for feed in self.feeds:
            feed.update_unread_count()
<|endoftext|>