<commit_before>from zschema import *

zgrab_subj_issuer = SubRecord({
    "serial_number":ListOf(String()),
    "common_name":ListOf(String()),
    "country":ListOf(String()),
    "locality":ListOf(String()),
    "province":ListOf(String()),
    "street_address":ListOf(String()),
    "organization":ListOf(String()),
    "organizational_unit":ListOf(String()),
    "postal_code":ListOf(String()),
})

unknown_extension = SubRecord({
    "id":String(),
    "critical":Boolean(),
    "value":Binary(),
})

zgrab_parsed_certificate = SubRecord({
    "subject":zgrab_subj_issuer,
    "issuer":zgrab_subj_issuer,
    "version":Integer(),
    "serial_number":String(doc="Serial number as an unsigned decimal integer. Stored as string to support >uint lengths. Negative values are allowed."),
    "validity":SubRecord({
        "start":DateTime(doc="Timestamp of when certificate is first valid. Timezone is UTC."),
        "end":DateTime(doc="Timestamp of when certificate expires. Timezone is UTC.")
    }),
    "signature_algorithm":SubRecord({
        "name":String(),
        "oid":String(),
    }),
    "subject_key_info":SubRecord({
        "key_algorithm":SubRecord({
            "name":String(doc="Name of public key type, e.g., RSA or ECDSA. More information is available the named SubRecord (e.g., rsa_public_key)."),
            "oid":String(doc="OID of the public key on the certificate. This is helpful when an unknown type is present. This field is reserved and not current populated.")
         }),
        "rsa_public_key":SubRecord({
            "exponent":Long(),
            "modulus":Binary(),
            "length":Integer(doc="Bit-length of modulus.")
         }),
        "dsa_public_key":SubRecord({
            "p":Binary(),
            "q":Binary(),
            "g":Binary(),
            "y":Binary(),
        }),
        "ecdsa_public_key":SubRecord({
            "b":Binary(),
            "gx":Binary(),
            "gy":Binary(),
            "n":Binary(),
            "p":Binary(),
            "x":Binary(),
            "y":Binary(),
        })
    }),
    "extensions":SubRecord({
        "key_usage":SubRecord({
            "digital_signature":Boolean(),
            "certificate_sign":Boolean(),
            "crl_sign":Boolean(),
            "content_commitment":Boolean(),
            "key_encipherment":Boolean(),
            "value":Integer(),
            "data_encipherment":Boolean(),
            "key_agreement":Boolean(),
            "decipher_only":Boolean(),
            "encipher_only":Boolean(),
        }),
        "basic_constraints":SubRecord({
            "is_ca":Boolean(),
            "max_path_len":Integer(),
        }),
        "subject_alt_name":SubRecord({
            "dns_names":ListOf(String()),
            "email_addresses":ListOf(String()),
            "ip_addresses":ListOf(String()),
        }),
        "crl_distribution_points":ListOf(String()),
        "authority_key_id":String(), # is this actdually binary?
        "extended_key_usage":ListOf(Integer()),
        "certificate_policies":ListOf(String()),
        "authority_info_access":SubRecord({
            "ocsp_urls":ListOf(String()),
            "issuer_urls":ListOf(String())
        }),
        "name_constraints":SubRecord({
            "critical":Boolean(),
            "permitted_names":ListOf(String()),
        }),
    }),
    "unknown_extensions":ListOf(unknown_extension),
    "signature":SubRecord({
        "signature_algorithm":SubRecord({
            "name":String(),
            "oid":String(),
        }),
        "value":Binary(),
        "valid":Boolean(),
        "self_signed":Boolean(),
    }),
    "fingerprint_md5":Binary(),
    "fingerprint_sha1":Binary(),
    "fingerprint_sha256":Binary(),
})

zgrab_certificate = SubRecord({
    "raw":Binary(),
    "parsed":zgrab_parsed_certificate,
})

zgrab_tls = SubRecord({
    "client_hello":SubRecord({
        "random":Binary(),
        "extended_random":Binary(),
    }),
    "server_hello":SubRecord({
        "version":SubRecord({
            "name":String(),
            "value":Integer()
        }),
        "random":Binary(),
        "session_id": Binary(),
        "cipher_suite":SubRecord({
            "hex":String(),
            "name":String(),
            "value":Integer(),
        }),
        "compression_method":Integer(),
        "ocsp_stapling":Boolean(),
        "ticket":Boolean(),
        "secure_renegotiation":Boolean(),
        "heartbeat":Boolean(),
        "extended_random":Binary(),
    }),
    "server_certificates":SubRecord({
        "certificate":zgrab_certificate,
        "chain":ListOf(zgrab_certificate),
        "validation":SubRecord({
            "browser_trusted":Boolean(),
            "browser_error":String(),
            "matches_domain":Boolean(),
        }),
    }),
    "server_key_exchange":SubRecord({
        "ecdh_params":SubRecord({
            "curve_id":SubRecord({
                "name":String(),
                "id":Integer(),
            }),
            "server_public":SubRecord({
                "x":SubRecord({
                    "value":Binary(),
                    "length":Integer(),
                }),
                "y":SubRecord({
                    "value":Binary(),
                    "length":Integer(),
                }),
            }),
        }),
        "rsa_params":SubRecord({
            "exponent":Long(),
            "modulus":Binary(),
            "length":Integer(),
        }),
        "dh_params":SubRecord({
            "prime":SubRecord({
                "value":Binary(),
                "length":Integer(),
            }),
            "generator":SubRecord({
                "value":Binary(),
                "length":Integer(),
            }),
            "server_public":SubRecord({
                "value":Binary(),
                "length":Integer(),
           }),
        }),
        "signature":SubRecord({
            "raw":Binary(),
            "type":String(),
            "valid":Boolean(),
            "signature_and_hash_type":SubRecord({
                "signature_algorithm":String(),
                "hash_algorithm":String(),
            }),
            "tls_version":SubRecord({
                "name":String(),
                "value":Integer()
            }),
        }),
        "signature_error":String(),
    }),
    "server_finished":SubRecord({
        "verify_data":Binary()
    })
})

zgrab_base = Record({
    "ip":IPv4Address(required=True),
    "timestamp":DateTime(required=True),
    "domain":String(),
    "data":SubRecord({}),
    "error":String(),
    "error_component":String()
})

zgrab_banner = Record({
    "data":SubRecord({
        "banner":String()
    })
}, extends=zgrab_base)

register_schema("zgrab-ftp", zgrab_banner)

zgrab_tls_banner=Record({
    "data":SubRecord({
        "tls":zgrab_tls,
    })
}, extends=zgrab_banner)
register_schema("zgrab-imaps", zgrab_tls_banner)
register_schema("zgrab-pop3s", zgrab_tls_banner)

zgrab_starttls = Record({
    "data":SubRecord({
        "starttls":String(),
    })
}, extends=zgrab_tls_banner)
register_schema("zgrab-imap", zgrab_starttls)
register_schema("zgrab-pop3", zgrab_starttls)

zgrab_smtp = Record({
    "data":SubRecord({
        "ehlo":String(),
    })
}, extends=zgrab_starttls)
register_schema("zgrab-smtp", zgrab_smtp)

zgrab_https = Record({
    "data":SubRecord({
        "tls":zgrab_tls
    })
}, extends=zgrab_base)

register_schema("zgrab-https", zgrab_https)

zgrab_heartbleed = SubRecord({
    "heartbeat_enabled":Boolean(),
    "heartbleed_vulnerable":Boolean()
})

zgrab_https_heartbleed = Record({
    "data":SubRecord({
        "heartbleed":zgrab_heartbleed
    })
}, extends=zgrab_https)

register_schema("zgrab-https-heartbleed", zgrab_https_heartbleed)

zgrab_unknown_http_header = SubRecord({
    "key":String(),
    "value":String()
})

zgrab_http_headers = SubRecord({
    "access_control_allow_origin":String(),
    "accept_patch":String(),
    "accept_ranges":String(),
    "age":String(),
    "allow":String(),
    "cache_control":String(),
    "connection":String(),
    "content_disposition":String(),
    "content_encoding":String(),
    "content_language":String(),
    "content_length":String(),
    "content_location":String(),
    "content_md5":String(),
    "content_range":String(),
    "content_type":String(),
    "date":String(),
    "etag":String(),
    "expires":String(),
    "last_modified":String(),
    "link":String(),
    "location":String(),
    "p3p":String(),
    "pragma":String(),
    "proxy_authenticate":String(),
    "public_key_pins":String(),
    "refresh":String(),
    "retry_after":String(),
    "server":AnalyzedString(),
    "set_cookie":String(),
    "status":String(),
    "strict_transport_security":String(),
    "trailer":String(),
    "transfer_encoding":String(),
    "upgrade":String(),
    "vary":String(),
    "via":String(),
    "warning":String(),
    "www_authenticate":String(),
    "x_frame_options":String(),
    "x_xss_protection":String(),
    "content_security_policy":String(),
    "x_content_security_policy":String(),
    "x_webkit_csp":String(),
    "x_content_type_options":String(),
    "x_powered_by":String(),
    "x_ua_compatible":String(),
    "x_content_duration":String(),
    "x_real_ip":String(),
    "x_forwarded_for": String(),
    "proxy_agent":String(),
    "unknown":ListOf(zgrab_unknown_http_header)
})

zgrab_http_request = SubRecord({
    "method":String(),
    "endpoint":String(),
    "user_agent":String()
})

zgrab_http_response = SubRecord({
    "status_code":Integer(),
    "status_line":AnalyzedString(),
    "body":HTML(),
    "body_sha256": Binary(),
    "headers":zgrab_http_headers
})

zgrab_http = Record({
    "data":SubRecord({
      "http":SubRecord({
        "request":zgrab_http_request,
        "response":zgrab_http_response
      })
    })
}, extends=zgrab_base)

register_schema("zgrab-http", zgrab_http)

zgrab_http_proxy = Record({
    "data":SubRecord({
      "http":SubRecord({
        "connect_request":zgrab_http_request,
        "connect_response":zgrab_http_response
      })
    })
}, extends=zgrab_http)
register_schema("zgrab-proxy", zgrab_http_proxy)

zgrab_old_http = Record({
    "data":SubRecord({
        "write":String(),
        "read":String(),
    })
}, extends=zgrab_base)

register_schema("zgrab-old-http", zgrab_old_http)

zgrab_modbus = Record({
    "data":SubRecord({
        "modbus":SubRecord({
            "length":Integer(),
            "unit_id":Integer(),
            "function_code":Integer(),
            "raw_response":String(),
            "mei_response":SubRecord({
                "conformity_level":Integer(),
                "more_follows":Boolean(),
                "object_count":Integer(),
                "objects":SubRecord({
                    "product_code":String(),
                    "revision":String(),
                    "vendor":String(),
                    "oid.255":String(),
                    "vendor_url":String(),
                    "product_name":String(),
                    "model_name":String(),
                    "user_application_name":String(),
                }),
            }),
            "exception_response":SubRecord({
                "exception_function":Integer(),
                "exception_type":Integer(),
            }),
        }),
    }),
}, extends=zgrab_base)

register_schema("zgrab-modbus", zgrab_modbus)
<commit_msg>Remove oid.255 from schema.<commit_after>from zschema import *

zgrab_subj_issuer = SubRecord({
    "serial_number":ListOf(String()),
    "common_name":ListOf(String()),
    "country":ListOf(String()),
    "locality":ListOf(String()),
    "province":ListOf(String()),
    "street_address":ListOf(String()),
    "organization":ListOf(String()),
    "organizational_unit":ListOf(String()),
    "postal_code":ListOf(String()),
})

unknown_extension = SubRecord({
    "id":String(),
    "critical":Boolean(),
    "value":Binary(),
})

zgrab_parsed_certificate = SubRecord({
    "subject":zgrab_subj_issuer,
    "issuer":zgrab_subj_issuer,
    "version":Integer(),
    "serial_number":String(doc="Serial number as an unsigned decimal integer. Stored as string to support >uint lengths. Negative values are allowed."),
    "validity":SubRecord({
        "start":DateTime(doc="Timestamp of when certificate is first valid. Timezone is UTC."),
        "end":DateTime(doc="Timestamp of when certificate expires. Timezone is UTC.")
    }),
    "signature_algorithm":SubRecord({
        "name":String(),
        "oid":String(),
    }),
    "subject_key_info":SubRecord({
        "key_algorithm":SubRecord({
            "name":String(doc="Name of public key type, e.g., RSA or ECDSA. More information is available the named SubRecord (e.g., rsa_public_key)."),
            "oid":String(doc="OID of the public key on the certificate. This is helpful when an unknown type is present. This field is reserved and not current populated.")
         }),
        "rsa_public_key":SubRecord({
            "exponent":Long(),
            "modulus":Binary(),
            "length":Integer(doc="Bit-length of modulus.")
         }),
        "dsa_public_key":SubRecord({
            "p":Binary(),
            "q":Binary(),
            "g":Binary(),
            "y":Binary(),
        }),
        "ecdsa_public_key":SubRecord({
            "b":Binary(),
            "gx":Binary(),
            "gy":Binary(),
            "n":Binary(),
            "p":Binary(),
            "x":Binary(),
            "y":Binary(),
        })
    }),
    "extensions":SubRecord({
        "key_usage":SubRecord({
            "digital_signature":Boolean(),
            "certificate_sign":Boolean(),
            "crl_sign":Boolean(),
            "content_commitment":Boolean(),
            "key_encipherment":Boolean(),
            "value":Integer(),
            "data_encipherment":Boolean(),
            "key_agreement":Boolean(),
            "decipher_only":Boolean(),
            "encipher_only":Boolean(),
        }),
        "basic_constraints":SubRecord({
            "is_ca":Boolean(),
            "max_path_len":Integer(),
        }),
        "subject_alt_name":SubRecord({
            "dns_names":ListOf(String()),
            "email_addresses":ListOf(String()),
            "ip_addresses":ListOf(String()),
        }),
        "crl_distribution_points":ListOf(String()),
        "authority_key_id":String(), # is this actdually binary?
        "extended_key_usage":ListOf(Integer()),
        "certificate_policies":ListOf(String()),
        "authority_info_access":SubRecord({
            "ocsp_urls":ListOf(String()),
            "issuer_urls":ListOf(String())
        }),
        "name_constraints":SubRecord({
            "critical":Boolean(),
            "permitted_names":ListOf(String()),
        }),
    }),
    "unknown_extensions":ListOf(unknown_extension),
    "signature":SubRecord({
        "signature_algorithm":SubRecord({
            "name":String(),
            "oid":String(),
        }),
        "value":Binary(),
        "valid":Boolean(),
        "self_signed":Boolean(),
    }),
    "fingerprint_md5":Binary(),
    "fingerprint_sha1":Binary(),
    "fingerprint_sha256":Binary(),
})

zgrab_certificate = SubRecord({
    "raw":Binary(),
    "parsed":zgrab_parsed_certificate,
})

zgrab_tls = SubRecord({
    "client_hello":SubRecord({
        "random":Binary(),
        "extended_random":Binary(),
    }),
    "server_hello":SubRecord({
        "version":SubRecord({
            "name":String(),
            "value":Integer()
        }),
        "random":Binary(),
        "session_id": Binary(),
        "cipher_suite":SubRecord({
            "hex":String(),
            "name":String(),
            "value":Integer(),
        }),
        "compression_method":Integer(),
        "ocsp_stapling":Boolean(),
        "ticket":Boolean(),
        "secure_renegotiation":Boolean(),
        "heartbeat":Boolean(),
        "extended_random":Binary(),
    }),
    "server_certificates":SubRecord({
        "certificate":zgrab_certificate,
        "chain":ListOf(zgrab_certificate),
        "validation":SubRecord({
            "browser_trusted":Boolean(),
            "browser_error":String(),
            "matches_domain":Boolean(),
        }),
    }),
    "server_key_exchange":SubRecord({
        "ecdh_params":SubRecord({
            "curve_id":SubRecord({
                "name":String(),
                "id":Integer(),
            }),
            "server_public":SubRecord({
                "x":SubRecord({
                    "value":Binary(),
                    "length":Integer(),
                }),
                "y":SubRecord({
                    "value":Binary(),
                    "length":Integer(),
                }),
            }),
        }),
        "rsa_params":SubRecord({
            "exponent":Long(),
            "modulus":Binary(),
            "length":Integer(),
        }),
        "dh_params":SubRecord({
            "prime":SubRecord({
                "value":Binary(),
                "length":Integer(),
            }),
            "generator":SubRecord({
                "value":Binary(),
                "length":Integer(),
            }),
            "server_public":SubRecord({
                "value":Binary(),
                "length":Integer(),
           }),
        }),
        "signature":SubRecord({
            "raw":Binary(),
            "type":String(),
            "valid":Boolean(),
            "signature_and_hash_type":SubRecord({
                "signature_algorithm":String(),
                "hash_algorithm":String(),
            }),
            "tls_version":SubRecord({
                "name":String(),
                "value":Integer()
            }),
        }),
        "signature_error":String(),
    }),
    "server_finished":SubRecord({
        "verify_data":Binary()
    })
})

zgrab_base = Record({
    "ip":IPv4Address(required=True),
    "timestamp":DateTime(required=True),
    "domain":String(),
    "data":SubRecord({}),
    "error":String(),
    "error_component":String()
})

zgrab_banner = Record({
    "data":SubRecord({
        "banner":String()
    })
}, extends=zgrab_base)

register_schema("zgrab-ftp", zgrab_banner)

zgrab_tls_banner=Record({
    "data":SubRecord({
        "tls":zgrab_tls,
    })
}, extends=zgrab_banner)
register_schema("zgrab-imaps", zgrab_tls_banner)
register_schema("zgrab-pop3s", zgrab_tls_banner)

zgrab_starttls = Record({
    "data":SubRecord({
        "starttls":String(),
    })
}, extends=zgrab_tls_banner)
register_schema("zgrab-imap", zgrab_starttls)
register_schema("zgrab-pop3", zgrab_starttls)

zgrab_smtp = Record({
    "data":SubRecord({
        "ehlo":String(),
    })
}, extends=zgrab_starttls)
register_schema("zgrab-smtp", zgrab_smtp)

zgrab_https = Record({
    "data":SubRecord({
        "tls":zgrab_tls
    })
}, extends=zgrab_base)

register_schema("zgrab-https", zgrab_https)

zgrab_heartbleed = SubRecord({
    "heartbeat_enabled":Boolean(),
    "heartbleed_vulnerable":Boolean()
})

zgrab_https_heartbleed = Record({
    "data":SubRecord({
        "heartbleed":zgrab_heartbleed
    })
}, extends=zgrab_https)

register_schema("zgrab-https-heartbleed", zgrab_https_heartbleed)

zgrab_unknown_http_header = SubRecord({
    "key":String(),
    "value":String()
})

zgrab_http_headers = SubRecord({
    "access_control_allow_origin":String(),
    "accept_patch":String(),
    "accept_ranges":String(),
    "age":String(),
    "allow":String(),
    "cache_control":String(),
    "connection":String(),
    "content_disposition":String(),
    "content_encoding":String(),
    "content_language":String(),
    "content_length":String(),
    "content_location":String(),
    "content_md5":String(),
    "content_range":String(),
    "content_type":String(),
    "date":String(),
    "etag":String(),
    "expires":String(),
    "last_modified":String(),
    "link":String(),
    "location":String(),
    "p3p":String(),
    "pragma":String(),
    "proxy_authenticate":String(),
    "public_key_pins":String(),
    "refresh":String(),
    "retry_after":String(),
    "server":AnalyzedString(),
    "set_cookie":String(),
    "status":String(),
    "strict_transport_security":String(),
    "trailer":String(),
    "transfer_encoding":String(),
    "upgrade":String(),
    "vary":String(),
    "via":String(),
    "warning":String(),
    "www_authenticate":String(),
    "x_frame_options":String(),
    "x_xss_protection":String(),
    "content_security_policy":String(),
    "x_content_security_policy":String(),
    "x_webkit_csp":String(),
    "x_content_type_options":String(),
    "x_powered_by":String(),
    "x_ua_compatible":String(),
    "x_content_duration":String(),
    "x_real_ip":String(),
    "x_forwarded_for": String(),
    "proxy_agent":String(),
    "unknown":ListOf(zgrab_unknown_http_header)
})

zgrab_http_request = SubRecord({
    "method":String(),
    "endpoint":String(),
    "user_agent":String()
})

zgrab_http_response = SubRecord({
    "status_code":Integer(),
    "status_line":AnalyzedString(),
    "body":HTML(),
    "body_sha256": Binary(),
    "headers":zgrab_http_headers
})

zgrab_http = Record({
    "data":SubRecord({
      "http":SubRecord({
        "request":zgrab_http_request,
        "response":zgrab_http_response
      })
    })
}, extends=zgrab_base)

register_schema("zgrab-http", zgrab_http)

zgrab_http_proxy = Record({
    "data":SubRecord({
      "http":SubRecord({
        "connect_request":zgrab_http_request,
        "connect_response":zgrab_http_response
      })
    })
}, extends=zgrab_http)
register_schema("zgrab-proxy", zgrab_http_proxy)

zgrab_old_http = Record({
    "data":SubRecord({
        "write":String(),
        "read":String(),
    })
}, extends=zgrab_base)

register_schema("zgrab-old-http", zgrab_old_http)

zgrab_modbus = Record({
    "data":SubRecord({
        "modbus":SubRecord({
            "length":Integer(),
            "unit_id":Integer(),
            "function_code":Integer(),
            "raw_response":String(),
            "mei_response":SubRecord({
                "conformity_level":Integer(),
                "more_follows":Boolean(),
                "object_count":Integer(),
                "objects":SubRecord({
                    "product_code":String(),
                    "revision":String(),
                    "vendor":String(),
                    "vendor_url":String(),
                    "product_name":String(),
                    "model_name":String(),
                    "user_application_name":String(),
                }),
            }),
            "exception_response":SubRecord({
                "exception_function":Integer(),
                "exception_type":Integer(),
            }),
        }),
    }),
}, extends=zgrab_base)

register_schema("zgrab-modbus", zgrab_modbus)
<|endoftext|>


<commit_before>"""COUNTER 5 SUSHI support."""

import datetime
import logging
import time
import warnings

import pendulum
import requests

import pycounter.exceptions
from pycounter.helpers import convert_date_run
import pycounter.report

DEPRECATED_KEYS = {"requestor_email", "requestor_name", "customer_name"}

logger = logging.getLogger(__name__)

def _dates_from_filters(filters):
    """Convert report filters to start and end date

    Args:
        filters: a list of dicts containing SUSHI report filters

    Returns: tuple of start, end dates as datetime.date

    """

    converted_filters = {
        filter_["Name"]: datetime.datetime.strptime(filter_["Value"], "%Y-%m-%d").date()
        for filter_ in filters
        if filter_["Name"] in ("Begin_Date", "End_Date")
    }
    try:
        return converted_filters["Begin_Date"], converted_filters["End_Date"]
    except KeyError:  # pragma: no cover
        raise ValueError("filters must include a Begin_Date and End_Date")

def _raw_to_full(raw_report):
    """Convert a raw report to CounterReport.

    :param raw_report: raw report as dict decoded from JSON
    :return: a :class:`pycounter.report.CounterReport`
    """
    header = raw_report["Report_Header"]
    start_date, end_date = _dates_from_filters(header["Report_Filters"])
    date_run = header.get("Created")
    report_data = {
        "period": (start_date, end_date),
        "report_version": int(header["Release"]),
        "report_type": header["Report_ID"],
        "customer": header.get("Institution_Name", ""),
        "institutional_identifier": header.get("Customer_ID", ""),
        "metric": "FT Item Requests",  # FIXME: this is for COUNTER4 compatibility
        "date_run": pendulum.parse(date_run) if date_run else datetime.datetime.now(),
    }

    report = pycounter.report.CounterReport(**report_data)

    for item in raw_report["Report_Items"]:
        publisher_name = item.get("Publisher", "")
        platform = item.get("Platform", "")
        title = item["Title"]
        eissn = issn = doi = prop_id = ""
        # isbn = u""

        for identifier in item["Item_ID"]:
            if identifier["Type"] == "Print_ISSN":
                issn = identifier["Value"]
            elif identifier["Type"] == "Online_ISSN":
                eissn = identifier["Value"]
            # elif identifier["Type"] == "ISBN":
            #     isbn = identifier["Value"]
            elif identifier["Type"] == "DOI":
                doi = identifier["Value"]
            elif identifier["Type"] == "Proprietary_ID":
                prop_id = identifier["Value"]

        month_data = []

        for perform_item in item["Performance"]:
            item_date = convert_date_run(perform_item["Period"]["Begin_Date"])
            usage = None
            for inst in perform_item["Instance"]:
                if inst["Metric_Type"] == "Total_Item_Requests":
                    usage = inst["Count"]
            if usage is not None:
                month_data.append((item_date, int(usage)))

        if report.report_type.startswith("TR_J"):
            report.pubs.append(
                pycounter.report.CounterJournal(
                    title=title,
                    platform=platform,
                    publisher=publisher_name,
                    period=report.period,
                    metric=report.metric,
                    issn=issn,
                    eissn=eissn,
                    doi=doi,
                    proprietary_id=prop_id,
                    month_data=month_data,
                )
            )

    return report

def get_sushi_stats_raw(
    wsdl_url=None,
    start_date=None,
    end_date=None,
    requestor_id=None,
    customer_reference=None,
    report="TR_J1",
    release=5,
    sushi_dump=False,
    verify=True,
    url=None,
    **kwargs
):
    """Get SUSHI stats for a given site in dict (decoded from JSON) format.

    :param wsdl_url: (Deprecated; for backward compatibility with COUNTER 4 SUSHI
    code. Use `url` instead.) URL to API endpoint for this provider

    :param start_date: start date for report (must be first day of a month)

    :param end_date: end date for report (must be last day of a month)

    :param requestor_id: requestor ID as defined by SUSHI protocol

    :param customer_reference: customer reference number as defined by SUSHI
        protocol

    :param report: report type, values defined by SUSHI protocol

    :param release: COUNTER release (only 5 is supported in this module)

    :param sushi_dump: produces dump of JSON to DEBUG logger

    :param verify: bool: whether to verify SSL certificates

    :param url: str: URL to endpoint for this provider

    """
    _check_params(kwargs, release)
    if url is None and wsdl_url:  # pragma: no cover
        warnings.warn(
            DeprecationWarning(
                "wsdl_url argument to get_sushi_stats"
                "_raw is deprecated; use url instead"
            )
        )
        url = wsdl_url
    url_params = {"url": url, "report": report}
    req_params = {
        "customer_id": customer_reference,
        "begin_date": start_date,
        "end_date": end_date,
        "requestor_id": requestor_id,
    }

    response = requests.get(
        "{url}/reports/{report}".format(**url_params),
        params=req_params,
        headers={"User-Agent": "pycounter/%s" % pycounter.__version__},
        verify=verify,
    )

    if sushi_dump:  # pragma: no cover
        logger.debug(
            "SUSHI DUMP: request: %s \n\n response: %s",
            vars(response.request),
            response.content,
        )

    return response.json()

def _check_params(kwargs, release):
    """Warn about unnecessary/wrong params to get_sushi_stats_raw."""
    if release != 5:  # pragma: no cover
        raise pycounter.exceptions.SushiException(
            "The sushi5 module only supports " "release 5"
        )
    deprecated_args = set(kwargs) & DEPRECATED_KEYS
    if deprecated_args:  # pragma: no cover
        warnings.warn(
            DeprecationWarning(
                "The argument(s) %s are no longer used for SUSHI "
                "requests in COUNTER 5." % ", ".join(deprecated_args)
            )
        )
    unknown_args = set(kwargs) - DEPRECATED_KEYS
    if unknown_args:  # pragma: no cover
        warnings.warn(
            pycounter.exceptions.SushiWarning(
                "The arguments %s are not known for "
                "SUSHI requests in COUNTER 5." % ", ".join(deprecated_args)
            )
        )

def get_report(*args, **kwargs):
    """Get a usage report from a COUNTER 5 (RESTful) SUSHI server.

    returns a :class:`pycounter.report.CounterReport` object.

    parameters: see get_sushi_stats_raw

    :param no_delay: don't delay in retrying Report Queued
    """
    no_delay = kwargs.pop("no_delay", False)
    delay_amount = 0 if no_delay else 60
    while True:
        try:
            raw_report = get_sushi_stats_raw(*args, **kwargs)
            return _raw_to_full(raw_report)
        except pycounter.exceptions.ServiceBusyError:  # pragma: no cover
            print("Service busy, retrying in %d seconds" % delay_amount)
            time.sleep(delay_amount)
<commit_msg>refactor: move identifier handling to a separate function<commit_after>"""COUNTER 5 SUSHI support."""

import datetime
import logging
import time
import warnings

import pendulum
import requests

import pycounter.exceptions
from pycounter.helpers import convert_date_run
import pycounter.report

DEPRECATED_KEYS = {"requestor_email", "requestor_name", "customer_name"}

logger = logging.getLogger(__name__)

def _dates_from_filters(filters):
    """Convert report filters to start and end date

    Args:
        filters: a list of dicts containing SUSHI report filters

    Returns: tuple of start, end dates as datetime.date

    """

    converted_filters = {
        filter_["Name"]: datetime.datetime.strptime(filter_["Value"], "%Y-%m-%d").date()
        for filter_ in filters
        if filter_["Name"] in ("Begin_Date", "End_Date")
    }
    try:
        return converted_filters["Begin_Date"], converted_filters["End_Date"]
    except KeyError:  # pragma: no cover
        raise ValueError("filters must include a Begin_Date and End_Date")

def _get_identifiers(item):
    """Pull identifiers from an item into a dict."""
    identifiers = {"eissn": "", "issn": "", "doi": "", "prop_id": "", "isbn": ""}
    for identifier in item["Item_ID"]:
        if identifier["Type"] == "Print_ISSN":
            identifiers["issn"] = identifier["Value"]
        elif identifier["Type"] == "Online_ISSN":
            identifiers["eissn"] = identifier["Value"]
        elif identifier["Type"] == "ISBN":
            identifiers["eisnn"] = identifier["Value"]
        elif identifier["Type"] == "DOI":
            identifiers["doi"] = identifier["Value"]
        elif identifier["Type"] == "Proprietary_ID":
            identifiers["prop_id"] = identifier["Value"]

    return identifiers

def _raw_to_full(raw_report):
    """Convert a raw report to CounterReport.

    :param raw_report: raw report as dict decoded from JSON
    :return: a :class:`pycounter.report.CounterReport`
    """
    header = raw_report["Report_Header"]
    period = _dates_from_filters(header["Report_Filters"])
    date_run = header.get("Created")
    report = pycounter.report.CounterReport(
        period=period,
        report_version=int(header["Release"]),
        report_type=header["Report_ID"],
        customer=header.get("Institution_Name", ""),
        institutional_identifier=header.get("Customer_ID", ""),
        metric="FT Item Requests",  # FIXME: this is for COUNTER4 compatibility
        date_run=pendulum.parse(date_run) if date_run else datetime.datetime.now(),
    )

    for item in raw_report["Report_Items"]:
        publisher_name = item.get("Publisher", "")
        platform = item.get("Platform", "")
        title = item["Title"]

        identifiers = _get_identifiers(item)

        month_data = []

        for perform_item in item["Performance"]:
            item_date = convert_date_run(perform_item["Period"]["Begin_Date"])
            usage = None
            for inst in perform_item["Instance"]:
                if inst["Metric_Type"] == "Total_Item_Requests":
                    usage = inst["Count"]
            if usage is not None:
                month_data.append((item_date, int(usage)))

        if report.report_type.startswith("TR_J"):
            report.pubs.append(
                pycounter.report.CounterJournal(
                    title=title,
                    platform=platform,
                    publisher=publisher_name,
                    period=report.period,
                    metric=report.metric,
                    issn=identifiers["issn"],
                    eissn=identifiers["eissn"],
                    doi=identifiers["doi"],
                    proprietary_id=identifiers["prop_id"],
                    month_data=month_data,
                )
            )

    return report

def get_sushi_stats_raw(
    wsdl_url=None,
    start_date=None,
    end_date=None,
    requestor_id=None,
    customer_reference=None,
    report="TR_J1",
    release=5,
    sushi_dump=False,
    verify=True,
    url=None,
    **kwargs
):
    """Get SUSHI stats for a given site in dict (decoded from JSON) format.

    :param wsdl_url: (Deprecated; for backward compatibility with COUNTER 4 SUSHI
    code. Use `url` instead.) URL to API endpoint for this provider

    :param start_date: start date for report (must be first day of a month)

    :param end_date: end date for report (must be last day of a month)

    :param requestor_id: requestor ID as defined by SUSHI protocol

    :param customer_reference: customer reference number as defined by SUSHI
        protocol

    :param report: report type, values defined by SUSHI protocol

    :param release: COUNTER release (only 5 is supported in this module)

    :param sushi_dump: produces dump of JSON to DEBUG logger

    :param verify: bool: whether to verify SSL certificates

    :param url: str: URL to endpoint for this provider

    """
    _check_params(kwargs, release)
    if url is None and wsdl_url:  # pragma: no cover
        warnings.warn(
            DeprecationWarning(
                "wsdl_url argument to get_sushi_stats"
                "_raw is deprecated; use url instead"
            )
        )
        url = wsdl_url
    url_params = {"url": url, "report": report}
    req_params = {
        "customer_id": customer_reference,
        "begin_date": start_date,
        "end_date": end_date,
        "requestor_id": requestor_id,
    }

    response = requests.get(
        "{url}/reports/{report}".format(**url_params),
        params=req_params,
        headers={"User-Agent": "pycounter/%s" % pycounter.__version__},
        verify=verify,
    )

    if sushi_dump:  # pragma: no cover
        logger.debug(
            "SUSHI DUMP: request: %s \n\n response: %s",
            vars(response.request),
            response.content,
        )

    return response.json()

def _check_params(kwargs, release):
    """Warn about unnecessary/wrong params to get_sushi_stats_raw."""
    if release != 5:  # pragma: no cover
        raise pycounter.exceptions.SushiException(
            "The sushi5 module only supports " "release 5"
        )
    deprecated_args = set(kwargs) & DEPRECATED_KEYS
    if deprecated_args:  # pragma: no cover
        warnings.warn(
            DeprecationWarning(
                "The argument(s) %s are no longer used for SUSHI "
                "requests in COUNTER 5." % ", ".join(deprecated_args)
            )
        )
    unknown_args = set(kwargs) - DEPRECATED_KEYS
    if unknown_args:  # pragma: no cover
        warnings.warn(
            pycounter.exceptions.SushiWarning(
                "The arguments %s are not known for "
                "SUSHI requests in COUNTER 5." % ", ".join(deprecated_args)
            )
        )

def get_report(*args, **kwargs):
    """Get a usage report from a COUNTER 5 (RESTful) SUSHI server.

    returns a :class:`pycounter.report.CounterReport` object.

    parameters: see get_sushi_stats_raw

    :param no_delay: don't delay in retrying Report Queued
    """
    no_delay = kwargs.pop("no_delay", False)
    delay_amount = 0 if no_delay else 60
    while True:
        try:
            raw_report = get_sushi_stats_raw(*args, **kwargs)
            return _raw_to_full(raw_report)
        except pycounter.exceptions.ServiceBusyError:  # pragma: no cover
            print("Service busy, retrying in %d seconds" % delay_amount)
            time.sleep(delay_amount)
<|endoftext|>


<commit_before># Copyright (c) 2017 The Regents of the University of Michigan.
# All Rights Reserved. Licensed according to the terms of the Revised
# BSD License. See LICENSE.txt for details.

class LuhnNumber:

    def __init__ (self, number = None):
        self.__set_number(number)

    def get_check_digit (self):
        if self:
            return self.__get_check_digit_from_checkable_int()

        else:
            return None

    def __bool__ (self):
        return self.number is not None

    def __repr__ (self):
        return "<{} {}>".format(self.__class__.__name__,
                                repr(self.number))

    def __set_number (self, number):
        if isinstance(number, int):
            self.number = number

        elif isinstance(number, str):
            self.__try_to_extract_number_from_str(number)

        else:
            self.number = None

    def __try_to_extract_number_from_str (self, number):
        try:
            self.number = int(number)

        except ValueError:
            self.number = None

    def __get_check_digit_from_checkable_int (self):
        total = 0
        n = self.number

        while n > 0:
            total += self.__add_two_digits(n % 100)
            n //= 100

        return (9 * total) % 10

    def __add_two_digits (self, n):
        return (n // 10) + self.__rotate_digit(n % 10)

    def __rotate_digit (self, n):
        result = n * 2
        return result if result < 10 else result - 9

def is_luhn_checkable (number):
    if isinstance(number, str):
        return bool(number)

    else:
        return isinstance(number, int)

def get_check_digit (number = None):
    n = LuhnNumber(number)
    return n.get_check_digit()

def convert_to_int (number):
    try:
        return int(number)

    except ValueError:
        return 1

def verify_check_digit (number = None):
    if is_luhn_checkable(number):
        number = convert_to_int(number)

        return number % 10 == get_check_digit(number // 10)

    else:
        return False
<commit_msg>Internal method takes an argument now<commit_after># Copyright (c) 2017 The Regents of the University of Michigan.
# All Rights Reserved. Licensed according to the terms of the Revised
# BSD License. See LICENSE.txt for details.

class LuhnNumber:

    def __init__ (self, number = None):
        self.__set_number(number)

    def get_check_digit (self):
        if self:
            return self.__get_check_digit_from_int(self.number)

        else:
            return None

    def __bool__ (self):
        return self.number is not None

    def __repr__ (self):
        return "<{} {}>".format(self.__class__.__name__,
                                repr(self.number))

    def __set_number (self, number):
        if isinstance(number, int):
            self.number = number

        elif isinstance(number, str):
            self.__try_to_extract_number_from_str(number)

        else:
            self.number = None

    def __try_to_extract_number_from_str (self, number):
        try:
            self.number = int(number)

        except ValueError:
            self.number = None

    def __get_check_digit_from_int (self, n):
        total = 0

        while n > 0:
            total += self.__add_two_digits(n % 100)
            n //= 100

        return (9 * total) % 10

    def __add_two_digits (self, n):
        return (n // 10) + self.__rotate_digit(n % 10)

    def __rotate_digit (self, n):
        result = n * 2
        return result if result < 10 else result - 9

def is_luhn_checkable (number):
    if isinstance(number, str):
        return bool(number)

    else:
        return isinstance(number, int)

def get_check_digit (number = None):
    n = LuhnNumber(number)
    return n.get_check_digit()

def convert_to_int (number):
    try:
        return int(number)

    except ValueError:
        return 1

def verify_check_digit (number = None):
    if is_luhn_checkable(number):
        number = convert_to_int(number)

        return number % 10 == get_check_digit(number // 10)

    else:
        return False
<|endoftext|>


<commit_before>import os
import glob
import json
from functools import lru_cache
import yaml
import boto3
from botocore.exceptions import ClientError
import credstash

from .dynamo import Run
from . import config

def checkout_tasks():
    os.system('GIT_SSH_COMMAND="ssh -i deploy.key" '
              'git clone git@github.com:{}/{}.git --depth 1'.format(
                  config.GITHUB_USER, config.GITHUB_TASK_REPO
              ))

@lru_cache()
def get_credstash_env(name):
    env = {}
    prefix = name + '.'
    for key, value in credstash.getAllSecrets().items():
        if key.startswith(prefix):
            env[key.replace(prefix, '')] = value
    return env

def load_tasks(directory):
    environments = yaml.load(open(os.path.join(directory, 'environments.yaml')))

    tasks = []

    files = glob.glob(os.path.join(directory, 'tasks/*.yml'))
    for fn in files:
        with open(fn) as f:
            task = yaml.load(f)
            # environment can be a string and will be looked up in environments
            # or can be a dict and will be used as-is
            if isinstance(task['environment'], str):
                task['environment'] = environments[task['environment']]

            if task.get('credstash', None):
                task['environment'].update(get_credstash_env(task['credstash']))
            tasks.append(task)

    return tasks

def make_task(family,
              entrypoint,
              image,
              memory_soft=128,
              environment=None,
              verbose=False,
              force=False,
              # cpu=None,
              # memory=None,
              ):
    ecs = boto3.client('ecs')
    region = ecs.meta.region_name

    log_stream_prefix = family.lower()
    main_container = {
        'name': config.TASK_NAME,
        'image': image,
        'essential': True,
        'entryPoint': entrypoint,
        'memoryReservation': memory_soft,
        'logConfiguration': {
            "logDriver": "awslogs",
            "options": {
                "awslogs-group": config.LOG_GROUP,
                "awslogs-region": region,
                "awslogs-stream-prefix": log_stream_prefix
            }
        },
    }

    # TODO: add CPU/memory limits

    if environment:
        main_container['environment'] = [{'name': k, 'value': v}
                                         for k, v in environment.items()]

    create = False
    existing = None
    try:
        resp = ecs.describe_task_definition(taskDefinition=family)
        existing = resp['taskDefinition']
        for key in ('entryPoint', 'environment', 'image', 'name',
                    'memoryReservation', 'essential', 'logConfiguration'):

            # check if values differ for this key
            oldval = existing['containerDefinitions'][0][key]
            newval = main_container[key]
            if key == 'environment':
                s_oldval = sorted([tuple(i.items()) for i in oldval])
                s_newval = sorted([tuple(i.items()) for i in newval])
                differ = (s_oldval != s_newval)
            else:
                differ = (oldval != newval)

            if differ:
                create = True
                print('{}: changing {}: {} => {}'.format(family, key, oldval, newval))
    except ClientError:
        create = True

    if force and not create:
        print('{}: forced update'.format(family))
        create = True

    if create:
        account_id = boto3.client('sts').get_caller_identity().get('Account')
        # TODO: create this role?
        role_arn = 'arn:aws:iam::{}:role/{}'.format(account_id, 'ecs-fargate-bobsled')
        response = ecs.register_task_definition(
            family=family,
            containerDefinitions=[
                main_container
            ],
            cpu='256',
            memory='512',
            networkMode='awsvpc',
            executionRoleArn=role_arn,
            requiresCompatibilities=['EC2', 'FARGATE'],
        )
        return response
    elif existing:
        if verbose:
            print('{family}: definition matches {family}:{revision}'.format(**existing))
    else:
        print('{}: creating new task'.format(family))

def make_cron_rule(name, schedule, enabled, force=False, verbose=False):
    events = boto3.client('events')
    lamb = boto3.client('lambda')

    enabled = 'ENABLED' if enabled else 'DISABLED'
    create = False

    try:
        old_rule = events.describe_rule(Name=name)
        updating = []
        if schedule != old_rule['ScheduleExpression']:
            updating.append('schedule')
        if enabled != old_rule['State']:
            updating.append('enabled')
        if updating:
            print('{}: updating rule'.format(name), ' '.join(updating))
            create = True
    except ClientError:
        print('{}: creating new cron rule'.format(name), schedule)
        create = True

    if force:
        create = True

    # figure out full lambda arn
    account_id = boto3.client('sts').get_caller_identity().get('Account')
    region = events.meta.region_name
    lambda_arn = 'arn:aws:lambda:{}:{}:function:bobsled-dev'.format(region, account_id)

    if create:
        rule = events.put_rule(
            Name=name,
            ScheduleExpression=schedule,
            State=enabled,
            Description='run {} at {}'.format(name, schedule),
        )
        events.put_targets(
            Rule=name,
            Targets=[
                {
                    'Id': name + '-job',
                    'Arn': lambda_arn,
                    'Input': json.dumps({
                        'job': name,
                        'command': 'bobsled.tasks.run_task_handler',
                    })
                }
            ]
        )
        perm_statement_id = name + '-job-permission'
        try:
            lamb.add_permission(
                FunctionName=lambda_arn,
                StatementId=perm_statement_id,
                Action='lambda:InvokeFunction',
                Principal='events.amazonaws.com',
                SourceArn=rule['RuleArn'],
            )
        except ClientError as e:
            print(e)
            # don't recreate permission if it is already there
            pass
    elif verbose:
        print('{}: no schedule change'.format(name))

def publish_task_definitions(dirname, only=None, force=False, verbose=False):
    tasks = load_tasks(dirname)

    for task in tasks:
        # convert entrypoint to list, break on spaces if needed
        entrypoint = task['entrypoint']
        if not isinstance(entrypoint, list):
            entrypoint = entrypoint.split()

        # shortcut for only adding certain task definitions
        if only and task['name'] not in only:
            continue

        make_task(task['name'],
                  entrypoint,
                  image=task.get('image'),
                  memory_soft=task.get('memory_soft', 128),
                  environment=task.get('environment'),
                  force=force,
                  verbose=verbose,
                  )
        if task.get('cron'):
            make_cron_rule(task['name'],
                           'cron({})'.format(task['cron']),
                           task.get('enabled', True),
                           force=force,
                           verbose=verbose,
                           )

def run_task(task_name, started_by):
    ecs = boto3.client('ecs')

    print('running', task_name)

    taskdef = ecs.describe_task_definition(taskDefinition=task_name)

    response = ecs.run_task(
        cluster=config.CLUSTER_NAME,
        count=1,
        taskDefinition=task_name,
        startedBy=started_by,
        launchType='FARGATE',
        networkConfiguration={
            'awsvpcConfiguration': {
                'subnets': [config.SUBNET_ID],
                'securityGroups': [config.SECURITY_GROUP_ID],
                'assignPublicIp': 'ENABLED',
            }
        },
        # overrides={
        #    'containerOverrides': [
        #        {
        #            'name': 'string',
        #            'command': [
        #                'string',
        #            ],
        #            'environment': [
        #                {
        #                    'name': 'string',
        #                    'value': 'string'
        #                },
        #            ]
        #        },
        #    ],
        # },
    )

    Run(task_name,
        task_definition=taskdef['taskDefinition'],
        task_arn=response['tasks'][0]['taskArn'],
        ).save()
    return response

def run_task_handler(event, context):
    print(event, context)
    run_task(event['job'], 'lambda')
<commit_msg>support memory/cpu task options<commit_after>import os
import glob
import json
from functools import lru_cache
import yaml
import boto3
from botocore.exceptions import ClientError
import credstash

from .dynamo import Run
from . import config

def checkout_tasks():
    os.system('GIT_SSH_COMMAND="ssh -i deploy.key" '
              'git clone git@github.com:{}/{}.git --depth 1'.format(
                  config.GITHUB_USER, config.GITHUB_TASK_REPO
              ))

@lru_cache()
def get_credstash_env(name):
    env = {}
    prefix = name + '.'
    for key, value in credstash.getAllSecrets().items():
        if key.startswith(prefix):
            env[key.replace(prefix, '')] = value
    return env

def load_tasks(directory):
    environments = yaml.load(open(os.path.join(directory, 'environments.yaml')))

    tasks = []

    files = glob.glob(os.path.join(directory, 'tasks/*.yml'))
    for fn in files:
        with open(fn) as f:
            task = yaml.load(f)
            # environment can be a string and will be looked up in environments
            # or can be a dict and will be used as-is
            if isinstance(task['environment'], str):
                task['environment'] = environments[task['environment']]

            if task.get('credstash', None):
                task['environment'].update(get_credstash_env(task['credstash']))
            tasks.append(task)

    return tasks

def make_task(family,
              entrypoint,
              image,
              memory_soft=128,
              environment=None,
              verbose=False,
              force=False,
              cpu='256',
              memory='512',
              ):
    ecs = boto3.client('ecs')
    region = ecs.meta.region_name

    log_stream_prefix = family.lower()
    main_container = {
        'name': config.TASK_NAME,
        'image': image,
        'essential': True,
        'entryPoint': entrypoint,
        'memoryReservation': memory_soft,
        'logConfiguration': {
            "logDriver": "awslogs",
            "options": {
                "awslogs-group": config.LOG_GROUP,
                "awslogs-region": region,
                "awslogs-stream-prefix": log_stream_prefix
            }
        },
    }

    if environment:
        main_container['environment'] = [{'name': k, 'value': v}
                                         for k, v in environment.items()]

    create = False
    existing = None
    try:
        resp = ecs.describe_task_definition(taskDefinition=family)
        existing = resp['taskDefinition']

        if memory != existing['memory']:
            print('{}: changing memory: {} => {}'.format(family, existing['memory'], memory))
            create = True
        if cpu != existing['cpu']:
            print('{}: changing cpu: {} => {}'.format(family, existing['cpu'], cpu))
            create = True

        for key in ('entryPoint', 'environment', 'image', 'name',
                    'memoryReservation', 'essential', 'logConfiguration'):

            # check if values differ for this key
            oldval = existing['containerDefinitions'][0][key]
            newval = main_container[key]
            if key == 'environment':
                s_oldval = sorted([tuple(i.items()) for i in oldval])
                s_newval = sorted([tuple(i.items()) for i in newval])
                differ = (s_oldval != s_newval)
            else:
                differ = (oldval != newval)

            if differ:
                create = True
                print('{}: changing {}: {} => {}'.format(family, key, oldval, newval))
    except ClientError:
        create = True

    if force and not create:
        print('{}: forced update'.format(family))
        create = True

    if create:
        account_id = boto3.client('sts').get_caller_identity().get('Account')
        # TODO: create this role?
        role_arn = 'arn:aws:iam::{}:role/{}'.format(account_id, 'ecs-fargate-bobsled')
        response = ecs.register_task_definition(
            family=family,
            containerDefinitions=[
                main_container
            ],
            cpu=cpu,
            memory=memory,
            networkMode='awsvpc',
            executionRoleArn=role_arn,
            requiresCompatibilities=['EC2', 'FARGATE'],
        )
        return response
    elif existing:
        if verbose:
            print('{family}: definition matches {family}:{revision}'.format(**existing))
    else:
        print('{}: creating new task'.format(family))

def make_cron_rule(name, schedule, enabled, force=False, verbose=False):
    events = boto3.client('events')
    lamb = boto3.client('lambda')

    enabled = 'ENABLED' if enabled else 'DISABLED'
    create = False

    try:
        old_rule = events.describe_rule(Name=name)
        updating = []
        if schedule != old_rule['ScheduleExpression']:
            updating.append('schedule')
        if enabled != old_rule['State']:
            updating.append('enabled')
        if updating:
            print('{}: updating rule'.format(name), ' '.join(updating))
            create = True
    except ClientError:
        print('{}: creating new cron rule'.format(name), schedule)
        create = True

    if force:
        create = True

    # figure out full lambda arn
    account_id = boto3.client('sts').get_caller_identity().get('Account')
    region = events.meta.region_name
    lambda_arn = 'arn:aws:lambda:{}:{}:function:bobsled-dev'.format(region, account_id)

    if create:
        rule = events.put_rule(
            Name=name,
            ScheduleExpression=schedule,
            State=enabled,
            Description='run {} at {}'.format(name, schedule),
        )
        events.put_targets(
            Rule=name,
            Targets=[
                {
                    'Id': name + '-job',
                    'Arn': lambda_arn,
                    'Input': json.dumps({
                        'job': name,
                        'command': 'bobsled.tasks.run_task_handler',
                    })
                }
            ]
        )
        perm_statement_id = name + '-job-permission'
        try:
            lamb.add_permission(
                FunctionName=lambda_arn,
                StatementId=perm_statement_id,
                Action='lambda:InvokeFunction',
                Principal='events.amazonaws.com',
                SourceArn=rule['RuleArn'],
            )
        except ClientError as e:
            print(e)
            # don't recreate permission if it is already there
            pass
    elif verbose:
        print('{}: no schedule change'.format(name))

def publish_task_definitions(dirname, only=None, force=False, verbose=False):
    tasks = load_tasks(dirname)

    for task in tasks:
        # convert entrypoint to list, break on spaces if needed
        entrypoint = task['entrypoint']
        if not isinstance(entrypoint, list):
            entrypoint = entrypoint.split()

        # shortcut for only adding certain task definitions
        if only and task['name'] not in only:
            continue

        make_task(task['name'],
                  entrypoint,
                  image=task.get('image'),
                  memory_soft=task.get('memory_soft', 128),
                  environment=task.get('environment'),
                  force=force,
                  verbose=verbose,
                  memory=str(task.get('memory', '512')),
                  cpu=str(task.get('cpu', '256')),
                  )
        if task.get('cron'):
            make_cron_rule(task['name'],
                           'cron({})'.format(task['cron']),
                           task.get('enabled', True),
                           force=force,
                           verbose=verbose,
                           )

def run_task(task_name, started_by):
    ecs = boto3.client('ecs')

    print('running', task_name)

    taskdef = ecs.describe_task_definition(taskDefinition=task_name)

    response = ecs.run_task(
        cluster=config.CLUSTER_NAME,
        count=1,
        taskDefinition=task_name,
        startedBy=started_by,
        launchType='FARGATE',
        networkConfiguration={
            'awsvpcConfiguration': {
                'subnets': [config.SUBNET_ID],
                'securityGroups': [config.SECURITY_GROUP_ID],
                'assignPublicIp': 'ENABLED',
            }
        },
        # overrides={
        #    'containerOverrides': [
        #        {
        #            'name': 'string',
        #            'command': [
        #                'string',
        #            ],
        #            'environment': [
        #                {
        #                    'name': 'string',
        #                    'value': 'string'
        #                },
        #            ]
        #        },
        #    ],
        # },
    )

    Run(task_name,
        task_definition=taskdef['taskDefinition'],
        task_arn=response['tasks'][0]['taskArn'],
        ).save()
    return response

def run_task_handler(event, context):
    print(event, context)
    run_task(event['job'], 'lambda')
<|endoftext|>


<commit_before>import enum
from collections import namedtuple
from datetime import date

__all__ = ('Location', 'Result', 'ScheduleItem')

class Location(enum.Enum):
    Home = enum.auto()
    Away = enum.auto()
    Neutral = enum.auto()

class Result(enum.Enum):
    Win = enum.auto()
    Loss = enum.auto()

class Options(enum.Enum):
    OT1 = 1
    OT2 = 2
    OT3 = 3
    OT4 = 4
    OT5 = 5
    OT6 = 6
    Postseason = enum.auto()

Game = namedtuple(
    'Game',
    [
        'game_date',
        'opponent',
        'score',
        'location',
        'result',
        'city',
        'overtime',
        'postseaston'
    ]
)

ScheduleItemBase = namedtuple(
    'ScheduleItem',
    [
        'game_date',
        'away_team',
        'away_team_score',
        'home_team',
        'home_team_score',
        'options',
        'location',
        'city',
        'home_team_result',
        'away_team_result'
    ]
)

class ScheduleItem(ScheduleItemBase):
    """
    Represents a single line in the Games File found at KenPom.com

    Example:
    11/13/2016 Florida Atlantic        68 SIU Edwardsville        77 N   Honolulu, HI
    """

    def __new__(cls, *args, **kwargs):
        return super().__new__(cls, *args, **kwargs)

    @classmethod
    def date_from_string(cls, s):
        d = [int(val) for val in s.split('/')]
        return date(d[2], d[0], d[1])

    @classmethod
    def from_str(cls, in_str=''):
        away_team_score = int(in_str[34:38].strip())
        home_team_score = int(in_str[60:64].strip())

        if home_team_score > away_team_score:
            home_team_result = Result.Win
        else:
            home_team_result = Result.Loss

        if away_team_score > home_team_score:
            away_team_result = Result.Win
        else:
            away_team_result = Result.Loss

        options = in_str[64:69].strip()
        city = in_str[69:].strip()

        return ScheduleItem(
            game_date=cls.date_from_string(in_str[:11].strip()),
            away_team=in_str[11:34].strip(),
            away_team_score=away_team_score,
            home_team=in_str[38:60].strip(),
            home_team_score=home_team_score,
            options=options,
            location=Location.Neutral if options.casefold() == 'n' else '',
            city=city,
            home_team_result=home_team_result,
            away_team_result=away_team_result
        )
<commit_msg>Update available items in the __all__ property<commit_after>import enum
from collections import namedtuple
from datetime import date

__all__ = ('Game', 'Location', 'Options', 'Result', 'ScheduleItem')

class Location(enum.Enum):
    Home = enum.auto()
    Away = enum.auto()
    Neutral = enum.auto()

class Result(enum.Enum):
    Win = enum.auto()
    Loss = enum.auto()

class Options(enum.Enum):
    OT1 = 1
    OT2 = 2
    OT3 = 3
    OT4 = 4
    OT5 = 5
    OT6 = 6
    Postseason = enum.auto()

Game = namedtuple(
    'Game',
    [
        'game_date',
        'opponent',
        'score',
        'location',
        'result',
        'city',
        'overtime',
        'postseaston'
    ]
)

ScheduleItemBase = namedtuple(
    'ScheduleItem',
    [
        'game_date',
        'away_team',
        'away_team_score',
        'home_team',
        'home_team_score',
        'options',
        'location',
        'city',
        'home_team_result',
        'away_team_result'
    ]
)

class ScheduleItem(ScheduleItemBase):
    """
    Represents a single line in the Games File found at KenPom.com

    Example:
    11/13/2016 Florida Atlantic        68 SIU Edwardsville        77 N   Honolulu, HI
    """

    def __new__(cls, *args, **kwargs):
        return super().__new__(cls, *args, **kwargs)

    @classmethod
    def date_from_string(cls, s):
        d = [int(val) for val in s.split('/')]
        return date(d[2], d[0], d[1])

    @classmethod
    def from_str(cls, in_str=''):
        away_team_score = int(in_str[34:38].strip())
        home_team_score = int(in_str[60:64].strip())

        if home_team_score > away_team_score:
            home_team_result = Result.Win
        else:
            home_team_result = Result.Loss

        if away_team_score > home_team_score:
            away_team_result = Result.Win
        else:
            away_team_result = Result.Loss

        options = in_str[64:69].strip()
        city = in_str[69:].strip()

        return ScheduleItem(
            game_date=cls.date_from_string(in_str[:11].strip()),
            away_team=in_str[11:34].strip(),
            away_team_score=away_team_score,
            home_team=in_str[38:60].strip(),
            home_team_score=home_team_score,
            options=options,
            location=Location.Neutral if options.casefold() == 'n' else '',
            city=city,
            home_team_result=home_team_result,
            away_team_result=away_team_result
        )
<|endoftext|>


<commit_before>from __future__ import print_function

import os, sys
import argparse
import oath
import base64
from vipaccess.patharg import PathType
from vipaccess import provision as vp

EXCL_WRITE = 'x' if sys.version_info>=(3,3) else 'wx'

# http://stackoverflow.com/a/26379693/20789

def set_default_subparser(self, name, args=None):
    """default subparser selection. Call after setup, just before parse_args()
    name: is the name of the subparser to call by default
    args: if set is the argument list handed to parse_args()

    , tested with 2.7, 3.2, 3.3, 3.4
    it works with 2.6 assuming argparse is installed
    """
    subparser_found = False
    for arg in sys.argv[1:]:
        if arg in ['-h', '--help']:  # global help if no subparser
            break
    else:
        for x in self._subparsers._actions:
            if not isinstance(x, argparse._SubParsersAction):
                continue
            for sp_name in x._name_parser_map.keys():
                if sp_name in sys.argv[1:]:
                    subparser_found = True
        if not subparser_found:
            # insert default in first position, this implies no
            # global options without a sub_parsers specified
            if args is None:
                sys.argv.insert(1, name)
            else:
                args.insert(0, name)

argparse.ArgumentParser.set_default_subparser = set_default_subparser

########################################

def provision(p, args):
    print("Generating request...")
    request = vp.generate_request(token_model=args.token_model)
    print("Fetching provisioning response...")
    session = vp.requests.Session()
    response = vp.get_provisioning_response(request, session)
    print("Getting token from response...")
    otp_token = vp.get_token_from_response(response.content)
    print("Decrypting token...")
    otp_secret = vp.decrypt_key(otp_token['iv'], otp_token['cipher'])
    otp_secret_b32 = base64.b32encode(otp_secret).upper().decode('ascii')
    print("Checking token...")
    if not vp.check_token(otp_token['id'], otp_secret, session):
        print("WARNING: Something went wrong--the token could not be validated.\n",
              "    (check your system time; it differs from the server's by %d seconds)\n" % otp_token['timeskew'],
              file=sys.stderr)

    if args.print:
        otp_uri = vp.generate_otp_uri(otp_token, otp_secret, args.issuer)
        print('Credential created successfully:\n\t' + otp_uri)
        print("This credential expires on this date: " + otp_token['expiry'])
        print('\nYou will need the ID to register this credential: ' + otp_token['id'])
        print('\nYou can use oathtool to generate the same OTP codes')
        print('as would be produced by the official VIP Access apps:\n')
        print('    oathtool -d6 -b --totp    {}  # 6-digit code'''.format(otp_secret_b32))
        print('    oathtool -d6 -b --totp -v {}  # ... with extra information'''.format(otp_secret_b32))
    else:
        assert otp_token['digits']==6
        assert otp_token['algorithm']=='sha1'
        assert otp_token['period']==30
        os.umask(0o077) # stoken does this too (security)
        with open(os.path.expanduser(args.dotfile), EXCL_WRITE) as dotfile:
            dotfile.write('version 1\n')
            dotfile.write('secret %s\n' % otp_secret_b32)
            dotfile.write('id %s\n' % otp_token['id'])
            dotfile.write('expiry %s\n' % otp_token['expiry'])
        print('Credential created and saved successfully: ' + dotfile.name)
        print('You will need the ID to register this credential: ' + otp_token['id'])

def uri(p, args):
    if args.secret:
        d, secret = {}, args.secret
    else:
        with open(args.dotfile, "r") as dotfile:
            d = dict( l.strip().split(None, 1) for l in dotfile )
        if 'version' not in d:
            p.error('%s does not specify version' % args.dotfile)
        elif d['version'] != '1':
            p.error("%s specifies version %r, rather than expected '1'" % (args.dotfile, d['version']))
        elif 'secret' not in d:
            p.error('%s does not specify secret' % args.dotfile)
        secret = d['secret']

    try:
        key = oath.google_authenticator.lenient_b32decode(secret)
    except Exception as e:
        p.error('error interpreting secret as base32: %s' % e)
    print('Token URI:\n')
    print('    ' + vp.generate_otp_uri(d, key, args.issuer))

def show(p, args):
    if args.secret:
        secret = args.secret
    else:
        with open(args.dotfile, "r") as dotfile:
            d = dict( l.strip().split(None, 1) for l in dotfile )
        if 'version' not in d:
            p.error('%s does not specify version' % args.dotfile)
        elif d['version'] != '1':
            p.error("%s specifies version %r, rather than expected '1'" % (args.dotfile, d['version']))
        elif 'secret' not in d:
            p.error('%s does not specify secret' % args.dotfile)
        secret = d.get('secret')
        if args.verbose:
            if 'id' in d: print('Token ID: %s' % d['id'], file=sys.stderr)
            if 'expiry' in d: print('Token expiration: %s' % d['expiry'], file=sys.stderr)
            sys.stderr.write('\n')

    try:
        key = oath._utils.tohex( oath.google_authenticator.lenient_b32decode(secret) )
    except Exception as e:
        p.error('error interpreting secret as base32: %s' % e)
    print(oath.totp(key))

def main():
    p = argparse.ArgumentParser()

    class PrintAction(argparse.Action):
        def __call__(self, parser, namespace, values, option_string=None):
            setattr(namespace, 'print', True)
            setattr(namespace, 'dotfile', None)

    sp = p.add_subparsers(dest='cmd')
    pprov = sp.add_parser('provision', help='Provision a new VIP Access credential')
    pprov.set_defaults(func=provision)
    m = pprov.add_mutually_exclusive_group()
    m.add_argument('-p', '--print', action=PrintAction, nargs=0,
                   help="Print the new credential, but don't save it to a file")
    m.add_argument('-o', '--dotfile', type=PathType(type='file', exists=False), default=os.path.expanduser('~/.vipaccess'),
                   help="File in which to store the new credential (default ~/.vipaccess)")
    pprov.add_argument('-i', '--issuer', default="Symantec", action='store',
                       help="Specify the issuer name to use (default: Symantec)")
    pprov.add_argument('-t', '--token-model', default='VSST',
                      help="VIP Access token model. Normally VSST (desktop token, default) or VSMT (mobile token). "
                           "Some clients only accept one or the other. Other more obscure token types also exist: "
                           "https://support.symantec.com/en_US/article.TECH239895.html")

    pshow = sp.add_parser('show', help="Show the current 6-digit token")
    m = pshow.add_mutually_exclusive_group()
    m.add_argument('-s', '--secret',
                   help="Specify the token secret on the command line (base32 encoded)")
    m.add_argument('-f', '--dotfile', type=PathType(exists=True), default=os.path.expanduser('~/.vipaccess'),
                   help="File in which the credential is stored (default ~/.vipaccess)")
    pshow.add_argument('-i', '--issuer', default="Symantec", action='store',
                       help="Specify the issuer name to use (default: Symantec)")
    pshow.add_argument('-v', '--verbose', action='store_true')
    pshow.set_defaults(func=show)

    pshow = sp.add_parser('uri', help="Export the credential as a URI (otpauth://)")
    m = pshow.add_mutually_exclusive_group()
    m.add_argument('-s', '--secret',
                   help="Specify the token secret on the command line (base32 encoded)")
    m.add_argument('-f', '--dotfile', type=PathType(exists=True), default=os.path.expanduser('~/.vipaccess'),
                   help="File in which the credential is stored (default ~/.vipaccess)")
    pshow.add_argument('-v', '--verbose', action='store_true')
    pshow.set_defaults(func=uri)

    p.set_default_subparser('show')
    args = p.parse_args()
    return args.func(p, args)

if __name__=='__main__':
    main()
<commit_msg>URI arg parser: Use unique name for clarity and remove unused verbose arg<commit_after>from __future__ import print_function

import os, sys
import argparse
import oath
import base64
from vipaccess.patharg import PathType
from vipaccess import provision as vp

EXCL_WRITE = 'x' if sys.version_info>=(3,3) else 'wx'

# http://stackoverflow.com/a/26379693/20789

def set_default_subparser(self, name, args=None):
    """default subparser selection. Call after setup, just before parse_args()
    name: is the name of the subparser to call by default
    args: if set is the argument list handed to parse_args()

    , tested with 2.7, 3.2, 3.3, 3.4
    it works with 2.6 assuming argparse is installed
    """
    subparser_found = False
    for arg in sys.argv[1:]:
        if arg in ['-h', '--help']:  # global help if no subparser
            break
    else:
        for x in self._subparsers._actions:
            if not isinstance(x, argparse._SubParsersAction):
                continue
            for sp_name in x._name_parser_map.keys():
                if sp_name in sys.argv[1:]:
                    subparser_found = True
        if not subparser_found:
            # insert default in first position, this implies no
            # global options without a sub_parsers specified
            if args is None:
                sys.argv.insert(1, name)
            else:
                args.insert(0, name)

argparse.ArgumentParser.set_default_subparser = set_default_subparser

########################################

def provision(p, args):
    print("Generating request...")
    request = vp.generate_request(token_model=args.token_model)
    print("Fetching provisioning response...")
    session = vp.requests.Session()
    response = vp.get_provisioning_response(request, session)
    print("Getting token from response...")
    otp_token = vp.get_token_from_response(response.content)
    print("Decrypting token...")
    otp_secret = vp.decrypt_key(otp_token['iv'], otp_token['cipher'])
    otp_secret_b32 = base64.b32encode(otp_secret).upper().decode('ascii')
    print("Checking token...")
    if not vp.check_token(otp_token['id'], otp_secret, session):
        print("WARNING: Something went wrong--the token could not be validated.\n",
              "    (check your system time; it differs from the server's by %d seconds)\n" % otp_token['timeskew'],
              file=sys.stderr)

    if args.print:
        otp_uri = vp.generate_otp_uri(otp_token, otp_secret, args.issuer)
        print('Credential created successfully:\n\t' + otp_uri)
        print("This credential expires on this date: " + otp_token['expiry'])
        print('\nYou will need the ID to register this credential: ' + otp_token['id'])
        print('\nYou can use oathtool to generate the same OTP codes')
        print('as would be produced by the official VIP Access apps:\n')
        print('    oathtool -d6 -b --totp    {}  # 6-digit code'''.format(otp_secret_b32))
        print('    oathtool -d6 -b --totp -v {}  # ... with extra information'''.format(otp_secret_b32))
    else:
        assert otp_token['digits']==6
        assert otp_token['algorithm']=='sha1'
        assert otp_token['period']==30
        os.umask(0o077) # stoken does this too (security)
        with open(os.path.expanduser(args.dotfile), EXCL_WRITE) as dotfile:
            dotfile.write('version 1\n')
            dotfile.write('secret %s\n' % otp_secret_b32)
            dotfile.write('id %s\n' % otp_token['id'])
            dotfile.write('expiry %s\n' % otp_token['expiry'])
        print('Credential created and saved successfully: ' + dotfile.name)
        print('You will need the ID to register this credential: ' + otp_token['id'])

def uri(p, args):
    if args.secret:
        d, secret = {}, args.secret
    else:
        with open(args.dotfile, "r") as dotfile:
            d = dict( l.strip().split(None, 1) for l in dotfile )
        if 'version' not in d:
            p.error('%s does not specify version' % args.dotfile)
        elif d['version'] != '1':
            p.error("%s specifies version %r, rather than expected '1'" % (args.dotfile, d['version']))
        elif 'secret' not in d:
            p.error('%s does not specify secret' % args.dotfile)
        secret = d['secret']

    try:
        key = oath.google_authenticator.lenient_b32decode(secret)
    except Exception as e:
        p.error('error interpreting secret as base32: %s' % e)
    print('Token URI:\n')
    print('    ' + vp.generate_otp_uri(d, key, args.issuer))

def show(p, args):
    if args.secret:
        secret = args.secret
    else:
        with open(args.dotfile, "r") as dotfile:
            d = dict( l.strip().split(None, 1) for l in dotfile )
        if 'version' not in d:
            p.error('%s does not specify version' % args.dotfile)
        elif d['version'] != '1':
            p.error("%s specifies version %r, rather than expected '1'" % (args.dotfile, d['version']))
        elif 'secret' not in d:
            p.error('%s does not specify secret' % args.dotfile)
        secret = d.get('secret')
        if args.verbose:
            if 'id' in d: print('Token ID: %s' % d['id'], file=sys.stderr)
            if 'expiry' in d: print('Token expiration: %s' % d['expiry'], file=sys.stderr)
            sys.stderr.write('\n')

    try:
        key = oath._utils.tohex( oath.google_authenticator.lenient_b32decode(secret) )
    except Exception as e:
        p.error('error interpreting secret as base32: %s' % e)
    print(oath.totp(key))

def main():
    p = argparse.ArgumentParser()

    class PrintAction(argparse.Action):
        def __call__(self, parser, namespace, values, option_string=None):
            setattr(namespace, 'print', True)
            setattr(namespace, 'dotfile', None)

    sp = p.add_subparsers(dest='cmd')

    pprov = sp.add_parser('provision', help='Provision a new VIP Access credential')
    pprov.set_defaults(func=provision)
    m = pprov.add_mutually_exclusive_group()
    m.add_argument('-p', '--print', action=PrintAction, nargs=0,
                   help="Print the new credential, but don't save it to a file")
    m.add_argument('-o', '--dotfile', type=PathType(type='file', exists=False), default=os.path.expanduser('~/.vipaccess'),
                   help="File in which to store the new credential (default ~/.vipaccess)")
    pprov.add_argument('-i', '--issuer', default="Symantec", action='store',
                       help="Specify the issuer name to use (default: Symantec)")
    pprov.add_argument('-t', '--token-model', default='VSST',
                      help="VIP Access token model. Normally VSST (desktop token, default) or VSMT (mobile token). "
                           "Some clients only accept one or the other. Other more obscure token types also exist: "
                           "https://support.symantec.com/en_US/article.TECH239895.html")

    pshow = sp.add_parser('show', help="Show the current 6-digit token")
    m = pshow.add_mutually_exclusive_group()
    m.add_argument('-s', '--secret',
                   help="Specify the token secret on the command line (base32 encoded)")
    m.add_argument('-f', '--dotfile', type=PathType(exists=True), default=os.path.expanduser('~/.vipaccess'),
                   help="File in which the credential is stored (default ~/.vipaccess)")
    pshow.add_argument('-i', '--issuer', default="Symantec", action='store',
                       help="Specify the issuer name to use (default: Symantec)")
    pshow.add_argument('-v', '--verbose', action='store_true')
    pshow.set_defaults(func=show)

    puri = sp.add_parser('uri', help="Export the credential as a URI (otpauth://)")
    m = puri.add_mutually_exclusive_group()
    m.add_argument('-s', '--secret',
                   help="Specify the token secret on the command line (base32 encoded)")
    m.add_argument('-f', '--dotfile', type=PathType(exists=True), default=os.path.expanduser('~/.vipaccess'),
                   help="File in which the credential is stored (default ~/.vipaccess)")
    puri.set_defaults(func=uri)

    p.set_default_subparser('show')
    args = p.parse_args()
    return args.func(p, args)

if __name__=='__main__':
    main()
<|endoftext|>


<commit_before>import collections

from pycoin import ecdsa

from ..script import der, opcodes, tools

bytes_from_int = chr if bytes == str else lambda x: bytes([x])

def generate_default_placeholder_signature():
    order = ecdsa.generator_secp256k1.order()
    r, s = order - 1, order // 2
    return der.sigencode_der(r, s) + bytes_from_int(1)

DEFAULT_PLACEHOLDER_SIGNATURE = generate_default_placeholder_signature()

class ScriptType(object):
    def __init__(self):
        raise NotImplemented()

    @classmethod
    def subclasses(cls, skip_self=True):
        for c in cls.__subclasses__():
            for c1 in c.subclasses(skip_self=False):
                yield c1
            if not skip_self:
                yield cls

    @classmethod
    def from_address(cls, text, netcodes=None):
        for sc in cls.subclasses():
            try:
                st = sc.from_address(text, netcodes=netcodes)
                return st
            except Exception:
                pass

    @classmethod
    def from_script(cls, script):
        for sc in cls.subclasses():
            try:
                st = sc.from_script(script)
                return st
            except Exception:
                pass

    @classmethod
    def match(cls, script):
        template = cls.TEMPLATE
        r = collections.defaultdict(list)
        pc1 = pc2 = 0
        while 1:
            if pc1 == len(script) and pc2 == len(template):
                return r
            if pc1 >= len(script) or pc2 >= len(template):
                break
            opcode1, data1, pc1 = tools.get_opcode(script, pc1)
            opcode2, data2, pc2 = tools.get_opcode(template, pc2)
            l1 = 0 if data1 is None else len(data1)
            if opcode2 == opcodes.OP_PUBKEY:
                if l1 < 33 or l1 > 120:
                    break
                r["PUBKEY_LIST"].append(data1)
            elif opcode2 == opcodes.OP_PUBKEYHASH:
                if l1 != 160/8:
                    break
                r["PUBKEYHASH_LIST"].append(data1)
            elif opcode2 == opcodes.OP_NULLDATA:
                r["NULLDATA_LIST"].append(data1)
            elif (opcode1, data1) != (opcode2, data2):
                break
        raise ValueError("script doesn't match")

    def _create_script_signature(
            self, secret_exponent, signature_for_hash_type_f, signature_type, script):
        sign_value = signature_for_hash_type_f(signature_type, script)
        order = ecdsa.generator_secp256k1.order()
        r, s = ecdsa.sign(ecdsa.generator_secp256k1, secret_exponent, sign_value)
        if s + s > order:
            s = order - s
        return der.sigencode_der(r, s) + bytes_from_int(signature_type)

    def address(self, netcode=None):
        from pycoin.networks.default import get_current_netcode
        if netcode is None:
            netcode = get_current_netcode()
        return self.info()["address_f"](netcode)

    def solve(self, **kwargs):
        """
        The kwargs required depend upon the script type.
        """
        raise NotImplemented()
<commit_msg>Fix problem with "tx 14237b92d26850730ffab1bfb138121e487ddde444734ef195eb7928102bc939".<commit_after>import collections

from pycoin import ecdsa

from ..script import der, opcodes, tools

bytes_from_int = chr if bytes == str else lambda x: bytes([x])

def generate_default_placeholder_signature():
    order = ecdsa.generator_secp256k1.order()
    r, s = order - 1, order // 2
    return der.sigencode_der(r, s) + bytes_from_int(1)

DEFAULT_PLACEHOLDER_SIGNATURE = generate_default_placeholder_signature()

class ScriptType(object):
    def __init__(self):
        raise NotImplemented()

    @classmethod
    def subclasses(cls, skip_self=True):
        for c in cls.__subclasses__():
            for c1 in c.subclasses(skip_self=False):
                yield c1
            if not skip_self:
                yield cls

    @classmethod
    def from_address(cls, text, netcodes=None):
        for sc in cls.subclasses():
            try:
                st = sc.from_address(text, netcodes=netcodes)
                return st
            except Exception:
                pass

    @classmethod
    def from_script(cls, script):
        for sc in cls.subclasses():
            try:
                st = sc.from_script(script)
                return st
            except Exception:
                pass

    @classmethod
    def match(cls, script):
        template = cls.TEMPLATE
        r = collections.defaultdict(list)
        pc1 = pc2 = 0
        while 1:
            if pc1 == len(script) and pc2 == len(template):
                return r
            if pc1 >= len(script) or pc2 >= len(template):
                break
            opcode1, data1, pc1 = tools.get_opcode(script, pc1)
            opcode2, data2, pc2 = tools.get_opcode(template, pc2)
            l1 = 0 if data1 is None else len(data1)
            if opcode2 == opcodes.OP_PUBKEY:
                if l1 < 33 or l1 > 120:
                    break
                r["PUBKEY_LIST"].append(data1)
            elif opcode2 == opcodes.OP_PUBKEYHASH:
                if l1 != 160/8:
                    break
                r["PUBKEYHASH_LIST"].append(data1)
            elif opcode2 == opcodes.OP_NULLDATA:
                r["NULLDATA_LIST"].append(data1)
            elif (opcode1, data1) != (opcode2, data2):
                break
        raise ValueError("script doesn't match")

    def _create_script_signature(
            self, secret_exponent, signature_for_hash_type_f, signature_type, script):
        sign_value = signature_for_hash_type_f(signature_type, script)
        order = ecdsa.generator_secp256k1.order()
        r, s = ecdsa.sign(ecdsa.generator_secp256k1, secret_exponent, sign_value)
        if s + s > order:
            s = order - s
        return der.sigencode_der(r, s) + bytes_from_int(signature_type)

    def address(self, netcode=None):
        from pycoin.networks.default import get_current_netcode
        if netcode is None:
            netcode = get_current_netcode()
        return self.info().get("address_f", lambda n: "(unknown)")(netcode)

    def solve(self, **kwargs):
        """
        The kwargs required depend upon the script type.
        """
        raise NotImplemented()
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-

import re

import requests

from . import app

class Banned(requests.ConnectionError):

    def __init__(self):
        super(Banned, self).__init__('The connection was refused.')

class Session(requests.Session):

    def __init__(self, *args, **kwargs):
        super(Session, self).__init__(*args, **kwargs)
        self.headers['User-Agent'] = app.config['USER_AGENT']

    def request(self, *args, **kwargs):
        # set default timeout
        kwargs.setdefault('timeout', app.config['HTTP_TIMEOUT'])

        # by default we don't verify certificates
        kwargs.setdefault('verify', False)
        resp = super(Session, self).request(*args, **kwargs)

        # implicit exception raising on HTTP errors, can be turned off ad-hoc
        # by extra raise_for_status=False keyword argument
        if kwargs.get('raise_for_status', True):
            resp.raise_for_status()
        return resp

class CsfdSession(Session):
    """Dealing with various ČSFD's network issues and eventually
    trying to perform requests again.
    """

    def request(self, *args, **kwargs):
        try:
            return super(CsfdSession, self).request(*args, **kwargs)

        except requests.TooManyRedirects:
            return self.request(*args, **kwargs)
        except requests.HTTPError as e:
            if e.response.status_code in (502, 403):
                return self.request(*args, **kwargs)
            raise
        except requests.ConnectionError as e:
            if 'Connection refused' in unicode(e):
                raise Banned
            raise

session_map = (
    (re.compile(r'^https?://([^\.]+\.)?csfd\.cz'), CsfdSession),
)

def request(method, url, **kwargs):
    session_cls = Session

    for pattern, cls in session_map:
        if pattern.search(url):
            session_cls = cls
            break

    session = session_cls()
    return session.request(method=method, url=url, **kwargs)

def get(url, **kwargs):
    kwargs.setdefault('allow_redirects', True)
    return request('get', url, **kwargs)

def options(url, **kwargs):
    kwargs.setdefault('allow_redirects', True)
    return request('options', url, **kwargs)

def head(url, **kwargs):
    kwargs.setdefault('allow_redirects', False)
    return request('head', url, **kwargs)

def post(url, data=None, **kwargs):
    return request('post', url, data=data, **kwargs)

def put(url, data=None, **kwargs):
    return request('put', url, data=data, **kwargs)

def patch(url, data=None, **kwargs):
    return request('patch', url, data=data, **kwargs)

def delete(url, **kwargs):
    return request('delete', url, **kwargs)

from requests.exceptions import *  # NOQA
<commit_msg>Tuning the HTTP layer.<commit_after># -*- coding: utf-8 -*-

import re
import time
import random
from urlparse import urlparse

import requests

from . import app
from . import log

class Banned(requests.ConnectionError):

    def __init__(self):
        super(Banned, self).__init__('The connection was refused.')

class Session(requests.Session):

    def __init__(self, *args, **kwargs):
        super(Session, self).__init__(*args, **kwargs)
        self.headers['User-Agent'] = app.config['USER_AGENT']

    def request(self, method, url, *args, **kwargs):
        log.debug('HTTP: %s - %s.', method.upper(), urlparse(url).netloc)
        try:
            # set default timeout
            kwargs.setdefault('timeout', app.config['HTTP_TIMEOUT'])

            # by default we don't verify certificates
            kwargs.setdefault('verify', False)
            resp = super(Session, self).request(method, url, *args, **kwargs)

            # implicit exception raising on HTTP errors, can be turned
            # off ad-hoc by extra raise_for_status=False keyword argument
            if kwargs.get('raise_for_status', True):
                resp.raise_for_status()
            return resp

        except requests.ConnectionError as e:
            if 'Connection refused' in unicode(e):
                raise Banned
            raise

class CsfdSession(Session):
    """Deals with various ČSFD's network issues and eventually
    tries to perform the same requests again.
    """

    def wait(self):
        seconds = random.randrange(1, 5, 1)
        log.debug('HTTP: Waiting for %d seconds.', seconds)
        time.sleep(seconds)

    def request(self, *args, **kwargs):
        try:
            self.wait()
            return super(CsfdSession, self).request(*args, **kwargs)

        except requests.TooManyRedirects:
            return self.request(*args, **kwargs)
        except requests.HTTPError as e:
            if e.response.status_code in (502, 403):
                return self.request(*args, **kwargs)
            raise

session_map = (
    (re.compile(r'^https?://(www\.)?csfd\.cz'), CsfdSession),
)

def request(method, url, **kwargs):
    session_cls = Session

    for pattern, cls in session_map:
        if pattern.search(url):
            session_cls = cls
            break

    session = session_cls()
    return session.request(method=method, url=url, **kwargs)

def get(url, **kwargs):
    kwargs.setdefault('allow_redirects', True)
    return request('get', url, **kwargs)

def options(url, **kwargs):
    kwargs.setdefault('allow_redirects', True)
    return request('options', url, **kwargs)

def head(url, **kwargs):
    kwargs.setdefault('allow_redirects', False)
    return request('head', url, **kwargs)

def post(url, data=None, **kwargs):
    return request('post', url, data=data, **kwargs)

def put(url, data=None, **kwargs):
    return request('put', url, data=data, **kwargs)

def patch(url, data=None, **kwargs):
    return request('patch', url, data=data, **kwargs)

def delete(url, **kwargs):
    return request('delete', url, **kwargs)

from requests.exceptions import *  # NOQA
<|endoftext|>