<commit_before>"""Copyright 2018 Google LLC

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    https://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.Copyright 2018 Google LLC

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    https://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License."""

"""Json Web Signature (JWS) in compact serialization format.

Json Web Signature (Rfc7515) is a complicated standard with several dangerous
design options. Therefore, we will only implement a safe subset of it. JWS
Compact Serialization (https://tools.ietf.org/html/rfc7515#section-7.1) while
not ideal, is simple and safe if correctly implemented. We'll harden the API to
make it difficult to misuse.
"""

__author__ = "quannguyen@google.com (Quan Nguyen)"

import json

from cryptography import exceptions
from cryptography.hazmat.primitives.asymmetric import utils
import six

from google3.experimental.users.quannguyen.jwslib import util as jwsutil
from google3.experimental.users.quannguyen.jwslib.ecdsa_sign import EcdsaSign
from google3.experimental.users.quannguyen.jwslib.ecdsa_verify import EcdsaVerify
from google3.experimental.users.quannguyen.jwslib.hmac import Hmac
from google3.experimental.users.quannguyen.jwslib.rsa_sign import RsaSign
from google3.experimental.users.quannguyen.jwslib.rsa_verify import RsaVerify

class JwsPublicKeyVerify(object):
  """JWS Public Key Verifier which supports Rsa and Ecdsa signature schemes."""

  def __init__(self, jwk_set):
    """Constructor for JwsPublicKeyVerify.

    Args:
      jwk_set: a JwkSet.

    Raises:
      UnsupportedAlgorithm: if the algorihtm is not defined at
      https://tools.ietf.org/html/rfc7518#section-3.1 or if jwk is not Rsa or
      Ecdsa key.
    """
    self.verifiers = []
    for key in jwk_set.keys:
      if key.key_type == "RSA":
        self.verifiers.append((RsaVerify(key.pub_key, key.algorithm), key.kid))
      elif key.key_type == "EC":
        self.verifiers.append((EcdsaVerify(key.pub_key, key.algorithm),
                               key.kid))
      else:
        raise exceptions.UnsupportedAlgorithm(
            "Unsupported key type: %s" % (key.key_type))

  def verify(self, token):
    """Verifies whether the token is signed with the corresponding private key.

    Args:
      token: string, the JWS compact serialization token as defined at
        https://tools.ietf.org/html/rfc7515#section-7.1.

    Returns:
      True if the token was verified, false if not.
    """
    try:
      token_parts = token.split(".")
      if len(token_parts) != 3:
        return False
      [header, payload, sig] = [token_parts[0], token_parts[1], token_parts[2]]
      data = header + b"." + payload
      sig_bytes = jwsutil.urlsafe_b64decode(sig)
      header_json = json.loads(jwsutil.urlsafe_b64decode(header))
      if not header_json.get("alg", ""):
        return False
      header_kid = header_json.get("kid", "")
      # In practice, there is only 1 key that matches alg or kid.
      for (verifier, kid) in self.verifiers:
        found_candidate_verifier = False
        if header_kid:
          if header_kid == kid and header_json["alg"] == verifier.algorithm:
            found_candidate_verifier = True
        elif header_json["alg"] == verifier.algorithm:
          found_candidate_verifier = True
        if found_candidate_verifier:
          mod_sig_bytes = sig_bytes
          if header_json["alg"][:2] == "ES":
            # Jws's Ecdsa signature is a pair [r, s] while standard Ecdsa
            # signature is the DER encoding of [r, s].
            length = len(sig_bytes)
            if length % 2 != 0:
              return False
            [r, s] = [sig_bytes[0:length / 2], sig_bytes[length / 2:]]
            mod_sig_bytes = utils.encode_dss_signature(
                jwsutil.bytes_to_int(r), jwsutil.bytes_to_int(s))
          if verifier.verify(mod_sig_bytes, data):
            return True
      return False
    except:
      return False

class JwsPublicKeySign(object):
  """Jws public key signer that suppports both Ecdsa and Rsa signature schemes.
  """

  def __init__(self, jwk_set):
    """Constructor for JwsPublicKeySign.

    Args:
      jwk_set: a JwkSet.

    Raises:
      UnsupportedAlgorithm: if the algorihtm is not defined at
      https://tools.ietf.org/html/rfc7518#section-3.1 or if jwk is not Rsa or
      Ecdsa key.
    """
    if len(jwk_set.keys) != 1:
      raise exceptions.UnsupportedAlgorithm(
          "Do not support multiple keys in signer")
    key = jwk_set.keys[0]
    if key.key_type == "RSA":
      self.signer = RsaSign(key.priv_key, key.algorithm)
    elif key.key_type == "EC":
      self.signer = EcdsaSign(key.priv_key, key.algorithm)
    else:
      raise exceptions.UnsupportedAlgorithm(
          "Unknown key type: %s or algorithm: %s" % (key.key_type,
                                                     key.algorithm))
    if hasattr(key, "kid"):
      self.kid = key.kid
    self.algorithm = key.algorithm

  def sign(self, header, payload):
    """Computes the signed jws as defined at rfc7515#section-7.1.

    Args:
      header: bytes, the header to be signed.
      payload: bytes, the payload to be signed.

    Returns:
      base64url(header) || '.' || base64url(payload) || '.' ||
      base64url(signature), where the signature is computed over
      base64url(utf8(header)) || '.' || base64url(payload).
    """
    if not isinstance(header, six.binary_type) or not isinstance(
        payload, six.binary_type):
      raise TypeError("header and payload must be bytes.")
    signing_input = jwsutil.urlsafe_b64encode(
        header) + b"." + jwsutil.urlsafe_b64encode(payload)
    signature = self.signer.sign(signing_input)
    if self.algorithm[:2] == "ES":
      # Standard Ecdsa signature is the DER encoding of [r, s] while Jws's
      # singature is the concatenation of r and s.
      (r, s) = utils.decode_dss_signature(signature)
      curve_length = jwsutil.ecdsa_algorithm_to_curve_length(self.algorithm)
      signature = jwsutil.int_to_bytes(r, curve_length) + jwsutil.int_to_bytes(
          s, curve_length)
    return signing_input + b"." + jwsutil.urlsafe_b64encode(signature)

class JwsMacVerify(object):
  """Jws Mac Verifier that verifies message authentication code."""

  def __init__(self, jwk_set):
    """Constructor for JwsMacVerify.

    Args:
      jwk_set: a JwkSet.

    Raises:
      UnsupportedAlgorithm: if the algorihtm is not defined at
      https://tools.ietf.org/html/rfc7518#section-3.1 or if jwk is symmetric
      Hmac key.
    """
    self.verifiers = []
    for key in jwk_set.keys:
      if key.key_type == "oct":
        self.verifiers.append((Hmac(key.sym_key, key.algorithm), key.kid))
      else:
        raise exceptions.UnsupportedAlgorithm(
            "Unsupported key type: %s or unrecognized algorithm: %s" %
            (key.key_type, key.algorithm))

  def verify(self, token):
    """Verifies whether the token was authenticated with mac.

    Args:
      token: string, the JWS compact serialization token as defined at
        https://tools.ietf.org/html/rfc7515#section-7.1.

    Returns:
      True if the token was verified, false if not.
    """
    try:
      token_parts = token.split(".")
      if len(token_parts) != 3:
        return False
      [header, payload, mac] = [token_parts[0], token_parts[1], token_parts[2]]
      data = header + b"." + payload
      mac_bytes = jwsutil.urlsafe_b64decode(mac)
      header_json = json.loads(jwsutil.urlsafe_b64decode(header))
      if not header_json.get("alg", ""):
        return False
      header_kid = header_json.get("kid", "")
      # In practice, there is only 1 key that matches alg or kid.
      for (verifier, kid) in self.verifiers:
        found_candidate_verifier = False
        if header_kid:
          if header_kid == kid and header_json["alg"] == verifier.algorithm:
            found_candidate_verifier = True
        elif header_json["alg"] == verifier.algorithm:
          found_candidate_verifier = True
        if found_candidate_verifier:
          if verifier.verify_mac(mac_bytes, data):
            return True
      return False
    except:
      return False

class JwsMacAuthenticator(object):
  """Jws Mac Authenticator that authenticates jws token."""

  def __init__(self, jwk_set):
    """Constructor for JwsMacAuthenticator.

    Args:
      jwk_set: a JwkSet.

    Raises:
      UnsupportedAlgorithm: if the key.algorihtm is not defined at
      https://tools.ietf.org/html/rfc7518#section-3.1 or if jwk is not symmetric
      Hmac key.
    """
    if len(jwk_set.keys) != 1:
      raise exceptions.UnsupportedAlgorithm(
          "Do not support multiple keys in authenticator")
    key = jwk_set.keys[0]
    if key.key_type == "oct":
      self.mac = Hmac(key.sym_key, key.algorithm)
    else:
      raise exceptions.UnsupportedAlgorithm(
          "Unknown key type: %s or algorithm: %s" % (key.key_type,
                                                     key.algorithm))
    if hasattr(key, "kid"):
      self.kid = key.kid
    self.algorithm = key.algorithm

  def authenticate(self, header, payload):
    """Computes the authenticated jws as defined at rfc7515#section-7.1.

    Args:
      header: bytes, the header to be authenticated.
      payload: bytes, the payload to be authenticated.

    Returns:
      base64url(header) || '.' || base64url(payload) || '.' ||
      base64url(mac), where the mac is computed over
      base64url(utf8(header)) || '.' || base64url(payload).
    """
    if not isinstance(header, six.binary_type) or not isinstance(
        payload, six.binary_type):
      raise TypeError("header and payload must be bytes.")
    authenticating_input = jwsutil.urlsafe_b64encode(
        header) + b"." + jwsutil.urlsafe_b64encode(payload)
    mac_bytes = self.mac.compute_mac(authenticating_input)
    return authenticating_input + b"." + jwsutil.urlsafe_b64encode(mac_bytes)
<commit_msg>Test reformat<commit_after>"""
Copyright 2018 Google LLC

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    https://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.Copyright 2018 Google LLC

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    https://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
"""

"""Json Web Signature (JWS) in compact serialization format.

Json Web Signature (Rfc7515) is a complicated standard with several dangerous
design options. Therefore, we will only implement a safe subset of it. JWS
Compact Serialization (https://tools.ietf.org/html/rfc7515#section-7.1) while
not ideal, is simple and safe if correctly implemented. We'll harden the API to
make it difficult to misuse.
"""

__author__ = "quannguyen@google.com (Quan Nguyen)"

import json

from cryptography import exceptions
from cryptography.hazmat.primitives.asymmetric import utils
import six

from google3.experimental.users.quannguyen.jwslib import util as jwsutil
from google3.experimental.users.quannguyen.jwslib.ecdsa_sign import EcdsaSign
from google3.experimental.users.quannguyen.jwslib.ecdsa_verify import EcdsaVerify
from google3.experimental.users.quannguyen.jwslib.hmac import Hmac
from google3.experimental.users.quannguyen.jwslib.rsa_sign import RsaSign
from google3.experimental.users.quannguyen.jwslib.rsa_verify import RsaVerify

class JwsPublicKeyVerify(object):
  """JWS Public Key Verifier which supports Rsa and Ecdsa signature schemes."""

  def __init__(self, jwk_set):
    """Constructor for JwsPublicKeyVerify.

    Args:
      jwk_set: a JwkSet.

    Raises:
      UnsupportedAlgorithm: if the algorihtm is not defined at
      https://tools.ietf.org/html/rfc7518#section-3.1 or if jwk is not Rsa or
      Ecdsa key.
    """
    self.verifiers = []
    for key in jwk_set.keys:
      if key.key_type == "RSA":
        self.verifiers.append((RsaVerify(key.pub_key, key.algorithm), key.kid))
      elif key.key_type == "EC":
        self.verifiers.append((EcdsaVerify(key.pub_key, key.algorithm),
                               key.kid))
      else:
        raise exceptions.UnsupportedAlgorithm(
            "Unsupported key type: %s" % (key.key_type))

  def verify(self, token):
    """Verifies whether the token is signed with the corresponding private key.

    Args:
      token: string, the JWS compact serialization token as defined at
        https://tools.ietf.org/html/rfc7515#section-7.1.

    Returns:
      True if the token was verified, false if not.
    """
    try:
      token_parts = token.split(".")
      if len(token_parts) != 3:
        return False
      [header, payload, sig] = [token_parts[0], token_parts[1], token_parts[2]]
      data = header + b"." + payload
      sig_bytes = jwsutil.urlsafe_b64decode(sig)
      header_json = json.loads(jwsutil.urlsafe_b64decode(header))
      if not header_json.get("alg", ""):
        return False
      header_kid = header_json.get("kid", "")
      # In practice, there is only 1 key that matches alg or kid.
      for (verifier, kid) in self.verifiers:
        found_candidate_verifier = False
        if header_kid:
          if header_kid == kid and header_json["alg"] == verifier.algorithm:
            found_candidate_verifier = True
        elif header_json["alg"] == verifier.algorithm:
          found_candidate_verifier = True
        if found_candidate_verifier:
          mod_sig_bytes = sig_bytes
          if header_json["alg"][:2] == "ES":
            # Jws's Ecdsa signature is a pair [r, s] while standard Ecdsa
            # signature is the DER encoding of [r, s].
            length = len(sig_bytes)
            if length % 2 != 0:
              return False
            [r, s] = [sig_bytes[0:length / 2], sig_bytes[length / 2:]]
            mod_sig_bytes = utils.encode_dss_signature(
                jwsutil.bytes_to_int(r), jwsutil.bytes_to_int(s))
          if verifier.verify(mod_sig_bytes, data):
            return True
      return False
    except:
      return False

class JwsPublicKeySign(object):
  """Jws public key signer that suppports both Ecdsa and Rsa signature schemes.
  """

  def __init__(self, jwk_set):
    """Constructor for JwsPublicKeySign.

    Args:
      jwk_set: a JwkSet.

    Raises:
      UnsupportedAlgorithm: if the algorihtm is not defined at
      https://tools.ietf.org/html/rfc7518#section-3.1 or if jwk is not Rsa or
      Ecdsa key.
    """
    if len(jwk_set.keys) != 1:
      raise exceptions.UnsupportedAlgorithm(
          "Do not support multiple keys in signer")
    key = jwk_set.keys[0]
    if key.key_type == "RSA":
      self.signer = RsaSign(key.priv_key, key.algorithm)
    elif key.key_type == "EC":
      self.signer = EcdsaSign(key.priv_key, key.algorithm)
    else:
      raise exceptions.UnsupportedAlgorithm(
          "Unknown key type: %s or algorithm: %s" % (key.key_type,
                                                     key.algorithm))
    if hasattr(key, "kid"):
      self.kid = key.kid
    self.algorithm = key.algorithm

  def sign(self, header, payload):
    """Computes the signed jws as defined at rfc7515#section-7.1.

    Args:
      header: bytes, the header to be signed.
      payload: bytes, the payload to be signed.

    Returns:
      base64url(header) || '.' || base64url(payload) || '.' ||
      base64url(signature), where the signature is computed over
      base64url(utf8(header)) || '.' || base64url(payload).
    """
    if not isinstance(header, six.binary_type) or not isinstance(
        payload, six.binary_type):
      raise TypeError("header and payload must be bytes.")
    signing_input = jwsutil.urlsafe_b64encode(
        header) + b"." + jwsutil.urlsafe_b64encode(payload)
    signature = self.signer.sign(signing_input)
    if self.algorithm[:2] == "ES":
      # Standard Ecdsa signature is the DER encoding of [r, s] while Jws's
      # singature is the concatenation of r and s.
      (r, s) = utils.decode_dss_signature(signature)
      curve_length = jwsutil.ecdsa_algorithm_to_curve_length(self.algorithm)
      signature = jwsutil.int_to_bytes(r, curve_length) + jwsutil.int_to_bytes(
          s, curve_length)
    return signing_input + b"." + jwsutil.urlsafe_b64encode(signature)

class JwsMacVerify(object):
  """Jws Mac Verifier that verifies message authentication code."""

  def __init__(self, jwk_set):
    """Constructor for JwsMacVerify.

    Args:
      jwk_set: a JwkSet.

    Raises:
      UnsupportedAlgorithm: if the algorihtm is not defined at
      https://tools.ietf.org/html/rfc7518#section-3.1 or if jwk is symmetric
      Hmac key.
    """
    self.verifiers = []
    for key in jwk_set.keys:
      if key.key_type == "oct":
        self.verifiers.append((Hmac(key.sym_key, key.algorithm), key.kid))
      else:
        raise exceptions.UnsupportedAlgorithm(
            "Unsupported key type: %s or unrecognized algorithm: %s" %
            (key.key_type, key.algorithm))

  def verify(self, token):
    """Verifies whether the token was authenticated with mac.

    Args:
      token: string, the JWS compact serialization token as defined at
        https://tools.ietf.org/html/rfc7515#section-7.1.

    Returns:
      True if the token was verified, false if not.
    """
    try:
      token_parts = token.split(".")
      if len(token_parts) != 3:
        return False
      [header, payload, mac] = [token_parts[0], token_parts[1], token_parts[2]]
      data = header + b"." + payload
      mac_bytes = jwsutil.urlsafe_b64decode(mac)
      header_json = json.loads(jwsutil.urlsafe_b64decode(header))
      if not header_json.get("alg", ""):
        return False
      header_kid = header_json.get("kid", "")
      # In practice, there is only 1 key that matches alg or kid.
      for (verifier, kid) in self.verifiers:
        found_candidate_verifier = False
        if header_kid:
          if header_kid == kid and header_json["alg"] == verifier.algorithm:
            found_candidate_verifier = True
        elif header_json["alg"] == verifier.algorithm:
          found_candidate_verifier = True
        if found_candidate_verifier:
          if verifier.verify_mac(mac_bytes, data):
            return True
      return False
    except:
      return False

class JwsMacAuthenticator(object):
  """Jws Mac Authenticator that authenticates jws token."""

  def __init__(self, jwk_set):
    """Constructor for JwsMacAuthenticator.

    Args:
      jwk_set: a JwkSet.

    Raises:
      UnsupportedAlgorithm: if the key.algorihtm is not defined at
      https://tools.ietf.org/html/rfc7518#section-3.1 or if jwk is not symmetric
      Hmac key.
    """
    if len(jwk_set.keys) != 1:
      raise exceptions.UnsupportedAlgorithm(
          "Do not support multiple keys in authenticator")
    key = jwk_set.keys[0]
    if key.key_type == "oct":
      self.mac = Hmac(key.sym_key, key.algorithm)
    else:
      raise exceptions.UnsupportedAlgorithm(
          "Unknown key type: %s or algorithm: %s" % (key.key_type,
                                                     key.algorithm))
    if hasattr(key, "kid"):
      self.kid = key.kid
    self.algorithm = key.algorithm

  def authenticate(self, header, payload):
    """Computes the authenticated jws as defined at rfc7515#section-7.1.

    Args:
      header: bytes, the header to be authenticated.
      payload: bytes, the payload to be authenticated.

    Returns:
      base64url(header) || '.' || base64url(payload) || '.' ||
      base64url(mac), where the mac is computed over
      base64url(utf8(header)) || '.' || base64url(payload).
    """
    if not isinstance(header, six.binary_type) or not isinstance(
        payload, six.binary_type):
      raise TypeError("header and payload must be bytes.")
    authenticating_input = jwsutil.urlsafe_b64encode(
        header) + b"." + jwsutil.urlsafe_b64encode(payload)
    mac_bytes = self.mac.compute_mac(authenticating_input)
    return authenticating_input + b"." + jwsutil.urlsafe_b64encode(mac_bytes)
<|endoftext|>


<commit_before>#!/usr/bin/env python2.7
# -*- coding: utf-8 -*-
from urllib2 import Request, urlopen, URLError
from time import sleep          # we need sleep for a delay between readings
pc='%'                          # I find defining % as a variable avoids confusion

def scan():                     # we've put all the page scanning bits into a function
    req = Request(someurl)
    try:
        response = urlopen(req)
    except URLError as e:
        if hasattr(e, 'reason'):
            print 'We failed to reach a server.'
            print 'Reason: ', e.reason
        elif hasattr(e, 'code'):
            print 'The server couldn\'t fulfill the request.'
            print 'Error code: ', e.code
    else:
        the_page = response.readlines()
        project_name = someurl.split('/')[5].split('-')[0] # take project name from URL
        print '\033[34m\033[1m' + project_name + '\033[0m' # bold and blue title
        for line in the_page:
            if 'data-duration' in line:  # line 457
                time_left = float(line.split('"')[5][:-2])
                campaign_duration = float(line.split('"')[1])
                hours_into_campaign = (24 * campaign_duration) - time_left
                if time_left >= 24:
                    time_left_unit = "days"
                    time_left = str(int(time_left / 24))
                else:
                    time_left_unit = "hours"
                    time_left = str(time_left)

            if 'data-goal' in line:       # line 449
                words = line.split(" ")
                for word in words:
                    if 'data-goal' in word:
                        target = word.split('"')
                            # bold and yellow for labels, bold and white for figures
                        print '\033[33m\033[1mtarget:\033[0m \033[1m\033[37m£%.2f\033[0m' % float(target[1])
                    if 'data-percent-raised' in word:
                        percent = word.split('"')
                        print '\033[33m\033[1mpercentage raised:\033[0m \033[1m\033[37m%.2f%s\033[0m' % ((float(percent[1]) * 100) , pc)
                    if 'data-pledged' in word:
                        amount_raised = word.split('"')
                        print '\033[33m\033[1mTotal so far:\033[0m \033[1m\033[37m£%.2f\033[0m' % float(amount_raised[1])
        print '\033[33m\033[1mTime left:\033[0m \033[1m\033[37m%s %s\033[0m' % (time_left, time_left_unit)

        amount_per_hour = float(amount_raised[1]) / hours_into_campaign
        print '\033[33m\033[1m£/hr:\033[0m \033[1m\033[37m£%.2f \033[0m \n' % amount_per_hour

        # Now we'll have a continuous loop which calls our function for each URL we define
while True:
    someurl= 'https://www.kickstarter.com/projects/pimoroni/flotilla-for-raspberry-pi-making-for-everyone'
    scan()
    sleep(15)
<commit_msg>Update ks3.py<commit_after>#!/usr/bin/env python2.7
# -*- coding: utf-8 -*-
from urllib2 import Request, urlopen, URLError
from time import sleep          # we need sleep for a delay between readings
pc='%'                          # I find defining % as a variable avoids confusion

# Here we define the urls of the KS campaigns we want to track as a list
urls =['https://www.kickstarter.com/projects/pimoroni/flotilla-for-raspberry-pi-making-for-everyone',
       'https://www.kickstarter.com/projects/955730101/protocam-raspberry-pi-a-b-camera-module-add-on-boa']

def scan(someurl):                     # we've put all the page scanning bits into a function
    req = Request(someurl)
    try:
        response = urlopen(req)
    except URLError as e:
        if hasattr(e, 'reason'):
            print 'We failed to reach a server.'
            print 'Reason: ', e.reason
        elif hasattr(e, 'code'):
            print 'The server couldn\'t fulfill the request.'
            print 'Error code: ', e.code
    else:
        the_page = response.readlines()
        project_name = someurl.split('/')[5].split('-')[0] # take project name from URL
        print '\033[34m\033[1m' + project_name + '\033[0m' # bold and blue title
        for line in the_page:
            if 'data-duration' in line:  # line 457
                time_left = float(line.split('"')[5])
                campaign_duration = float(line.split('"')[1])
                hours_into_campaign = (24 * campaign_duration) - time_left
                if time_left >= 24:
                    time_left_unit = "days"
                    time_left = str(int(time_left / 24))
                else:
                    time_left_unit = "hours"
                    time_left = str(time_left)

            if 'data-goal' in line:       # line 449
                words = line.split(" ")
                for word in words:
                    if 'data-goal' in word:
                        target = word.split('"')
                            # bold and yellow for labels, bold and white for figures
                        print '\033[33m\033[1mtarget:\033[0m \033[1m\033[37m£%.2f\033[0m' % float(target[1])
                    if 'data-percent-raised' in word:
                        percent = word.split('"')
                        print '\033[33m\033[1mpercentage raised:\033[0m \033[1m\033[37m%.2f%s\033[0m' % ((float(percent[1]) * 100) , pc)
                    if 'data-pledged' in word:
                        amount_raised = word.split('"')
                        print '\033[33m\033[1mTotal so far:\033[0m \033[1m\033[37m£%.2f\033[0m' % float(amount_raised[1])
        print '\033[33m\033[1mTime left:\033[0m \033[1m\033[37m%s %s\033[0m' % (time_left, time_left_unit)

        amount_per_hour = float(amount_raised[1]) / hours_into_campaign
        print '\033[33m\033[1m£/hr:\033[0m \033[1m\033[37m£%.2f \033[0m \n' % amount_per_hour

while True:          # continuous loop which calls our function for each URL we define
    for url in urls:
        scan(url)
        sleep(15)
<|endoftext|>


<commit_before>#!/usr/bin/python

import time
import Adafruit_CharLCD as LCD

import argparse
import base64
import json
import httplib2

from googleapiclient import discovery
from oauth2client.client import GoogleCredentials

# Raspberry Pi pin configuration:
lcd_rs        = 25
lcd_en        = 24
lcd_d4        = 23
lcd_d5        = 17
lcd_d6        = 21
lcd_d7        = 22

# Define LCD column and row size for 16x2 LCD.
lcd_columns = 16
lcd_rows    = 2

# Initialize the LCD using the pins above.
lcd = LCD.Adafruit_CharLCD(lcd_rs, lcd_en, lcd_d4, lcd_d5, lcd_d6, lcd_d7,
                           lcd_columns, lcd_rows)

# Setting up Google Cloud Speech API
DISCOVERY_URL = ('https://{api}.googleapis.com/$discovery/rest?'
                 'version={apiVersion}')

def get_speech_service():
    '''
    Uses application default credentials to authorize google api requests
    Note that environment variables:
        GOOGLE_APPLICATION_CREDENTIALS must be set to the path of service credentials json
        GCLOUD_PROJECT must be set to the name of your project
        * I did this in the bash file *
    Returns:
        Google Cloud Speech API service
    '''
    credentials = GoogleCredentials.get_application_default().create_scoped(
        ['https://www.googleapis.com/auth/cloud-platform'])
    http = httplib2.Http()
    credentials.authorize(http)

    return discovery.build(
        'speech', 'v1beta1', http=http, discoveryServiceUrl=DISCOVERY_URL)

def main(speech_file):
    '''
    Transcribe the given audio file and prints it on the lcd

    Args:
        speech_file: the name of the audio file.
    '''
    with open(speech_file, 'rb') as speech:
        speech_content = base64.b64encode(speech.read())

    service = get_speech_service()
    service_request = service.speech().syncrecognize(
        body={
            'config': {
                'encoding': 'linear16',
                'sampleRate': 16000,
                'languageCode': 'en-US',
            },
            'audio': {
                'content': speech_content.decode('UTF-8')
                }
            })

    response = service_request.execute()

    # Unwrapping the json
    json_results = json.dumps(response)
    results = json.loads(json_results)['results']

    result_string = ''
    result_string_lcd = ''

    for result in results:
        alternatives = result['alternatives']

        for alternative in alternatives:
            transcript = alternative['transcript']

            for i in range(len(transcript)):
                result_string += transcript[i]

    num_new_lines = 0
    while num_new_lines < lcd_rows:
        num_new_lines += 1
        for col in range(lcd_columns):
            result_string_lcd += result_string[col+(num_new_lines*lcd_columns)]
        result_string_lcd += '\n'

    lcd.message(result_string_lcd)

if __name__ == '__main__':
    parser = argparse.ArgumentParser()
    parser.add_argument(
        'speech_file', help='/Users/roman/Desktop/audio.raw')
    args = parser.parse_args()
    main(args.speech_file)
<commit_msg>fixed bug causing too short messages to indexerror<commit_after>#!/usr/bin/python

import time
import Adafruit_CharLCD as LCD

import argparse
import base64
import json
import httplib2

from googleapiclient import discovery
from oauth2client.client import GoogleCredentials

# Raspberry Pi pin configuration:
lcd_rs        = 25
lcd_en        = 24
lcd_d4        = 23
lcd_d5        = 17
lcd_d6        = 21
lcd_d7        = 22

# Define LCD column and row size for 16x2 LCD.
lcd_columns = 16
lcd_rows    = 2

# Initialize the LCD using the pins above.
lcd = LCD.Adafruit_CharLCD(lcd_rs, lcd_en, lcd_d4, lcd_d5, lcd_d6, lcd_d7,
                           lcd_columns, lcd_rows)

# Setting up Google Cloud Speech API
DISCOVERY_URL = ('https://{api}.googleapis.com/$discovery/rest?'
                 'version={apiVersion}')

def get_speech_service():
    '''
    Uses application default credentials to authorize google api requests
    Note that environment variables:
        GOOGLE_APPLICATION_CREDENTIALS must be set to the path of service credentials json
        GCLOUD_PROJECT must be set to the name of your project
        * I did this in the bash file *
    Returns:
        Google Cloud Speech API service
    '''
    credentials = GoogleCredentials.get_application_default().create_scoped(
        ['https://www.googleapis.com/auth/cloud-platform'])
    http = httplib2.Http()
    credentials.authorize(http)

    return discovery.build(
        'speech', 'v1beta1', http=http, discoveryServiceUrl=DISCOVERY_URL)

def main(speech_file):
    '''
    Transcribe the given audio file and prints it on the lcd

    Args:
        speech_file: the name of the audio file.
    '''
    with open(speech_file, 'rb') as speech:
        speech_content = base64.b64encode(speech.read())

    service = get_speech_service()
    service_request = service.speech().syncrecognize(
        body={
            'config': {
                'encoding': 'linear16',
                'sampleRate': 16000,
                'languageCode': 'en-US',
            },
            'audio': {
                'content': speech_content.decode('UTF-8')
                }
            })

    response = service_request.execute()

    # Unwrapping the json
    json_results = json.dumps(response)
    results = json.loads(json_results)['results']

    result_string = ''
    result_string_lcd = ''

    for result in results:
        alternatives = result['alternatives']

        for alternative in alternatives:
            transcript = alternative['transcript']

            for i in range(len(transcript)):
                result_string += transcript[i]

    num_new_lines = 0
    while num_new_lines < lcd_rows:
        num_new_lines += 1
        for col in range(lcd_columns):
            if (col+(num_new_lines*lcd_columns)) < len(result_string_lcd)
                result_string_lcd += result_string[col+(num_new_lines*lcd_columns)]
        result_string_lcd += '\n'

    lcd.message(result_string_lcd)

if __name__ == '__main__':
    parser = argparse.ArgumentParser()
    parser.add_argument(
        'speech_file', help='/Users/roman/Desktop/audio.raw')
    args = parser.parse_args()
    main(args.speech_file)
<|endoftext|>


<commit_before>"""

# Support Code

## Standard Headers

"""
from __future__ import division,print_function
import  sys,random,math
sys.dont_write_bytecode = True
from settings import *
"""

## Simple, low-level stuff

### Maths Stuff

"""
def gt(x,y): return x > y
def lt(x,y): return x < y
"""

An accumulator for numbers.

"""
class N:
  "An Accumulator for numbers"
  def __init__(i): i.n = i.m2 = i.mu = 0.0
  def sd(i)       : return (i.m2/(i.n - 1))**0.5
  def __iadd__(i,x):
    i.n   += 1
    delta  = x - i.mu
    i.mu  += delta*1.0/i.n
    i.m2  += delta*(x - i.mu)
    return i
"""

### Random stuff.

"""
by   = lambda x: random.uniform(0,x)
seed = random.seed
any  = random.choice
"""

### List Handling Tricks

"""
def first(lst): return lst[0]
def second(lst): return lst[1]
def third(lst): return lst[2]
"""

### Printing Stuff

Print without newline:

"""
def say(*lst): print(*lst,end="")
"""

Print a list of numbers without an excess of decimal places:

"""
def gs(lst) : return [g(x) for x in lst]
def g(x)    :
  txt = '%g' % x
  return int(txt) if int(x) == x else float(txt)
"""

Pretty print a dictionary:

"""
def showd(d):
  def one(k,v):
    if isinstance(v,list):
      v = gs(v)
    if isinstance(v,float):
      return ":%s %g" % (k,v)
    return ":%s %s" % (k,v)
  return ' '.join([one(k,v) for k,v in
                    sorted(d.items())
                     if not "_" in k])
"""

## Decorator to run code at Start-up

"""
def go(f):
  "A decorator that runs code at load time."
  print("\n# ---|", f.__name__,"|-----------------")
  if f.__doc__: print("#", f.__doc__)
  f()
"""

## Handling command line options.

Convert command line to a function call.
e.g. if the file lib.py ends with

    if __name__ == '__main__':eval(todo())

then

    python lib.py myfun :a 1 :b fred

results in a call to  _myfun(a=1,b='fred')_.

"""
def todo(com="print(The._logo,'WHERE (2.0) you at?')"):
  import sys
  if len(sys.argv) < 2: return com
  def strp(x): return isinstance(x,basestring)
  def wrap(x): return "'%s'"%x if strp(x) else str(x)
  def oneTwo(lst):
    while lst: yield lst.pop(0), lst.pop(0)
  def value(x):
    try:    return eval(x)
    except: return x
  def two(x,y): return x[1:] +"="+wrap(value(y))
  twos = [two(x,y) for x,y in oneTwo(sys.argv[2:])]
  return sys.argv[1]+'(**dict('+ ','.join(twos)+'))'

"""

## More interesting, low-level stuff

"""
def timing(f,repeats=10):
  "How long does 'f' take to run?"
  import time
  time1 = time.clock()
  for _ in range(repeats):
    f()
  return (time.clock() - time1)*1.0/repeats
"""

## Data Completion Tool

"""
def data(indep=[], less=[], more=[], _rows=[]):
  nindep= len(indep)
  ndep  = len(less) + len(more)
  m= o(lo={}, hi={}, w={},
       eval  = lambda m,it : True,
       _rows = [o(cells=r,score=0,scored=False)
                for r in _rows],
       names = indep+less+more)
  m.decisions  = [x for x in range(nindep)]
  m.objectives = [nindep+ x- 1 for x in range(ndep)]
  m.cols       = m.decisions + m.objectives
  for x in m.decisions :
    m.w[x]=  1
  for y,_ in enumerate(less) :
    m.w[x+y]   = -1
  for z,_ in enumerate(more) :
    m.w[x+y+z] =  1
  for x in m.cols:
    all = sorted(row.cells[x] for row in m._rows)
    m.lo[x] = all[0]
    m.hi[x] = all[-1]
  return m
"""

## Start-up Actions

"""
if __name__ == '__main__': eval(todo())
<commit_msg>stuff<commit_after>"""

# Support Code

## Standard Headers

"""
from __future__ import division,print_function
import  sys,random,math
sys.dont_write_bytecode = True
from settings import *
"""

## Simple, low-level stuff

### Maths Stuff

"""
def gt(x,y): return x > y
def lt(x,y): return x < y
"""

An accumulator for numbers.

"""
class N:
  "An Accumulator for numbers"
  def __init__(i): i.n = i.m2 = i.mu = 0.0
  def sd(i)       : return (i.m2/(i.n - 1))**0.5
  def __iadd__(i,x):
    i.n   += 1
    delta  = x - i.mu
    i.mu  += delta*1.0/i.n
    i.m2  += delta*(x - i.mu)
    return i
"""

### Random stuff.

"""
by   = lambda x: random.uniform(0,x)
seed = random.seed
any  = random.choice
"""

### List Handling Tricks

"""
def first(lst): return lst[0]
def second(lst): return lst[1]
def third(lst): return lst[2]
"""

### Printing Stuff

Print without newline:

"""
def say(*lst): print(*lst,end="")
"""

Print a list of numbers without an excess of decimal places:

"""
def gs(lst) : return [g(x) for x in lst]
def g(x)    :
  txt = '%g' % x
  return int(txt) if int(x) == x else float(txt)
"""

Pretty print a dictionary:

"""
def showd(d):
  def one(k,v):
    if isinstance(v,list):
      v = gs(v)
    if isinstance(v,float):
      return ":%s %g" % (k,v)
    return ":%s %s" % (k,v)
  return ' '.join([one(k,v) for k,v in
                    sorted(d.items())
                     if not "_" in k])
"""

## Decorator to run code at Start-up

"""
def go(f):
  "A decorator that runs code at load time."
  print("\n# ---|", f.__name__,"|-----------------")
  if f.__doc__: print("#", f.__doc__)
  f()
"""

## Handling command line options.

Convert command line to a function call.
e.g. if the file lib.py ends with

    if __name__ == '__main__':eval(todo())

then

    python lib.py myfun :a 1 :b fred

results in a call to  _myfun(a=1,b='fred')_.

"""
def todo(com="print(The._logo,'WHERE (2.0) you at?')"):
  import sys
  if len(sys.argv) < 2: return com
  def strp(x): return isinstance(x,basestring)
  def wrap(x): return "'%s'"%x if strp(x) else str(x)
  def oneTwo(lst):
    while lst: yield lst.pop(0), lst.pop(0)
  def value(x):
    try:    return eval(x)
    except: return x
  def two(x,y): return x[1:] +"="+wrap(value(y))
  twos = [two(x,y) for x,y in oneTwo(sys.argv[2:])]
  return sys.argv[1]+'(**dict('+ ','.join(twos)+'))'
"""

## More interesting, low-level stuff

"""
def timing(f,repeats=10):
  "How long does 'f' take to run?"
  import time
  time1 = time.clock()
  for _ in range(repeats):
    f()
  return (time.clock() - time1)*1.0/repeats
"""

## Data Completion Tool

Fills in some details on a table of data. For example,

     def nasa93():
       vl=1;l=2;n=3;h=4;vh=5;xh=6
       return data(indep= [
                     'Prec', 'Flex', 'Resl', 'Team', 'Pmat', 'rely', 'data', 'cplx', 'ruse',
                     'docu', 'time', 'stor', 'pvol', 'acap', 'pcap', 'pcon', 'aexp', 'plex',
                     'ltex', 'tool', 'site', 'sced', 'kloc'],
                   less = ['effort', 'defects', 'months'],
                   _rows=[
                      [h,h,h,vh,h,h,l,h,n,n,n,n,l,n,n,n,n,n,h,n,n,l,25.9,117.6,808,15.3],
                      [h,h,h,vh,h,h,l,h,n,n,n,n,l,n,n,n,n,n,h,n,n,l,24.6,117.6,767,15.0],
                      [h,h,h,vh,h,h,l,h,n,n,n,n,l,n,n,n,n,n,h,n,n,l,7.7,31.2,240,10.1],
     ...

Adds in information on _cols_, _decisions_, _hi,lo_, etc:

    {	:cols [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11,
             12, 13, 14, 15, 16, 17, 18, 19, 20, 21,
             22, 22, 23, 24]
 	    :decisions [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10,
                  11, 12, 13, 14, 15, 16, 17, 18,
                  19, 20, 21, 22]
 	    :eval <function <lambda> at 0x7f3f825bea28>
 	    :hi {0: 4, 1: 4, 2: 4, 3: 5, 4: 4, 5: 5, 6: 5,
           7: 6, 8: 3, 9: 3, 10: 6, 11: 6, 12: 4, 13: 5,
           14: 5, 15: 3, 16: 5, 17: 4, 18: 4, 19: 4,
           20: 3, 21: 3, 22: 980, 23: 8211, 24: 50961}
 	    :lo {0: 4, 1: 4, 2: 4, 3: 5, 4: 2, 5: 2, 6: 2,
           7: 2, 8: 3, 9: 3, 10: 3, 11: 3, 12: 2,
           13: 3, 14: 3, 15: 3, 16: 2, 17: 1, 18: 1,
            19: 3, 20: 3, 21: 2, 22: 0.9, 23: 8.4, 24: 28}
 	    :names ['Prec', 'Flex', 'Resl', 'Team', 'Pmat',
              'rely', 'data', 'cplx', 'ruse', 'docu',
              'time', 'stor', 'pvol', 'acap', 'pcap',
              'pcon', 'aexp', 'plex', 'ltex', 'tool',
              'site', 'sced', 'kloc', 'effort',
              'defects', 'months']
 	    :objectives [22, 23, 24]
 	    :w {0: 1, 1: 1, 2: 1, 3: 1, 4: 1, 5: 1, 6: 1,
          7: 1, 8: 1, 9: 1, 10: 1, 11: 1, 12: 1, 13: 1,
          14: 1, 15: 1, 16: 1, 17: 1, 18: 1, 19: 1,
          20: 1, 21: 1, 22: -1, 23: -1, 24: -1}
    }

"""
def data(indep=[], less=[], more=[], _rows=[]):
  nindep= len(indep)
  ndep  = len(less) + len(more)
  m= o(lo={}, hi={}, w={},
       eval  = lambda m,it : True,
       _rows = [o(cells=r,score=0,scored=False)
                for r in _rows],
       names = indep+less+more)
  m.decisions  = [x for x in range(nindep)]
  m.objectives = [nindep+ x- 1 for x in range(ndep)]
  m.cols       = m.decisions + m.objectives
  for x in m.decisions :
    m.w[x]=  1
  for y,_ in enumerate(less) :
    m.w[x+y]   = -1
  for z,_ in enumerate(more) :
    m.w[x+y+z] =  1
  for x in m.cols:
    all = sorted(row.cells[x] for row in m._rows)
    m.lo[x] = all[0]
    m.hi[x] = all[-1]
  return m
"""

## Start-up Actions

"""
if __name__ == '__main__': eval(todo())
<|endoftext|>


<commit_before>#!/usr/bin/env python3
# coding: utf-8

import sys; sys.dont_write_bytecode = True

import collections
import contextlib
import datetime
import itertools
import logging
import os
import pathlib
import sqlite3
import tempfile

import click
import requests

__version__ = "0.1a"

# Database functions.
# ------------------------------------------------------------------------------

class ConnectionWrapper:
    "Database connection wrapper."

    def __init__(self, connection):
        self.connection = connection

    def __enter__(self):
        return self

    def cursor(self):
        "Gets database cursor wrapper."

        return CursorWrapper(self.connection.cursor())

    def __exit__(self, exc_type, exc_value, traceback):
        self.connection.close()

class CursorWrapper:
    "Database cursor wrapper class."

    def __init__(self, cursor):
        self.cursor = cursor

    def __enter__(self):
        return self

    def initialize_database(self):
        "Initializes empty database."

        self.cursor.execute("""
            create table options (
                name text not null primary key,
                integer_value integer,
                real_value real,
                text_value text,
                blob_value blob
            )""")
        self.cursor.execute("""
            create table posts (
                key text not null primary key,
                timestamp integer not null,
                title text null,
                text_ text not null
            )""")

    def upsert_option(self, name, value):
        "Inserts or updates option."
        logging.info("Setting option `%s` to `%s`.", name, value)
        row = self.make_option_row(name, value)
        try:
            self.cursor.execute("""
                insert into options (integer_value, real_value, text_value, blob_value, name)
                values (?, ?, ?, ?, ?)
            """, row)
        except sqlite3.IntegrityError:
            self.cursor.execute("""
                update options
                set integer_value = ?, real_value = ?, text_value = ?, blob_value = ? where name = ?
            """, row)

    def make_option_row(self, name, value):
        "Gets option row by value."
        return (as_(value, int), as_(value, float), as_(value, str), as_(value, bytes), name)

    def insert_post(self, key, timestamp, title, text):
        "Insert new post."
        self.cursor.execute("insert into posts values (?, ?, ?, ?)", (key, get_timestamp(), title, text))

    def update_post(self, key, timestamp, title, text):
        "Updates existing post."
        self.cursor.execute(
            "update posts set timestamp = ?, title = ?, text = ? where key = ?", (
            timestamp, title, text, key,
        ))

    def upsert_post(self, key, timestamp, title, text):
        "Inserts new post or updates if exists."
        try:
            self.insert_post(key, timestamp, title, text)
        except sqlite3.IntegrityError:
            self.update_post(key, timestamp, title, text)

    def __exit__(self, exc_type, exc_value, traceback):
        if not exc_type:
            self.cursor.connection.commit()
        else:
            self.cursor.connection.rollback()
        self.cursor.connection.close()

# Utilities.
# ------------------------------------------------------------------------------

def as_(value, type):
    return value if isinstance(value, type) else None

def urlify(title):
    "Gets post key by title."
    return title.lower().replace(" ", "-")

def get_text(editor, key, text=""):
    "Execute editor and read user input."

    fd, path = tempfile.mkstemp(prefix="lje-{}-".format(key), suffix=".txt", text=True)
    try:
        with os.fdopen(fd, "wt", encoding="utf-8") as fp:
            fp.write(text)
        os.system("{0} \"{1}\"".format(editor, path))
        with open(path, "rt", encoding="utf-8") as fp:
            return fp.read()
    finally:
        pathlib.Path(path).unlink()

def get_timestamp():
    "Gets UTC timestamp."
    return int((datetime.datetime.utcnow() - datetime.datetime(1970, 1, 1)).total_seconds())

# Common options, arguments and types.
# ------------------------------------------------------------------------------

class AliasedGroup(click.Group):

    def get_command(self, ctx, name):
        command = click.Group.get_command(self, ctx, name)
        if command is not None:
            return command
        matches = [command for command in self.list_commands(ctx) if command.startswith(name)]
        if not matches:
            return None
        if len(matches) == 1:
            return click.Group.get_command(self, ctx, matches[0])
        ctx.fail("`{0}` is not a command. Did you mean one of these? {1}".format(
            name, ", ".join(matches)))

class SQLiteType(click.ParamType):
    name = "sqlite"

    def __init__(self, exists):
        super()
        self.exists = exists

    def convert(self, value, param, ctx):
        if pathlib.Path(value).exists():
            if not self.exists:
                raise click.UsageError("database already exists")
        else:
            if self.exists:
                raise click.UsageError("existing database is expected")
        return sqlite3.connect(value)

editor_option = click.option(
    "-e", "--editor",
    default="env editor",
    help="Editor command.",
    metavar="<editor>",
    show_default=True,
)

existing_database_argument = click.argument("database", metavar="<database>", type=SQLiteType(exists=True))
new_database_argument = click.argument("database", metavar="<database>", type=SQLiteType(exists=False))

# Build command.
# ------------------------------------------------------------------------------

@click.command(short_help="Builds blog.")
@existing_database_argument
def build(database):
    pass

# Init command.
# ------------------------------------------------------------------------------

@click.command(short_help="Initialize new blog.")
@new_database_argument
@click.option("--name", help="Your name.", metavar="<name>", prompt=True, required=True)
@click.option("--email", help="Your email.", metavar="<email>", prompt=True, required=True)
@click.option("--title", help="Blog title.", metavar="<title>", prompt=True, required=True)
@click.option("--url", help="Blog URL.", metavar="<url>", prompt=True, required=True)
def init(database, name, email, title, url):
    with ConnectionWrapper(database) as connection, connection.cursor() as cursor:
        cursor.initialize_database()
        cursor.upsert_option("author.email", email)
        cursor.upsert_option("author.name", name)
        cursor.upsert_option("blog.title", title)
        cursor.upsert_option("blog.url", url)

# Compose command.
# ------------------------------------------------------------------------------

@click.command(short_help="Compose new article.")
@existing_database_argument
@editor_option
@click.option("--key", default=None, help="Post key. Example: my-first-post.", metavar="<key>")
@click.option("--title", help="Post title.", metavar="<title>", prompt=True, required=True)
@click.option("--tag", help="Post tag.", metavar="<tag>", multiple=True)
def compose(database, editor, key, title, tag):
    key = key or urlify(title)
    text = get_text(editor, key)
    with ConnectionWrapper(database) as connection, connection.cursor() as cursor:
        cursor.insert_post(key, get_timestamp(), title, text)

# Edit command.
# ------------------------------------------------------------------------------

@click.command(short_help="Edit existing article.")
@editor_option
def edit(editor):
    pass

# Publish command.
# ------------------------------------------------------------------------------

@click.command(short_help="Publish draft.")
def publish():
    pass

# Unpublish command.
# ------------------------------------------------------------------------------

@click.command(short_help="Unpublish post.")
def unpublish():
    pass

# Options command group.
# ------------------------------------------------------------------------------

@click.group(short_help="Get or set blog options.")
def options():
    pass

# Get option command.
# ------------------------------------------------------------------------------

@click.command("get", short_help="Get option.")
def get_option():
    pass

# Set option command.
# ------------------------------------------------------------------------------

@click.command("set", short_help="Set option.")
def set_option():
    pass

# List options command.
# ------------------------------------------------------------------------------

@click.command("list", short_help="List all options.")
def list_options():
    pass

# Import command group.
# ------------------------------------------------------------------------------

@click.group("import", short_help="Import blog.")
def import_():
    """
    Imports another source into a new Љ blog.
    """
    pass

# Import from Tumblr.
# ------------------------------------------------------------------------------

@click.command("tumblr", short_help="Import from tumblr.")
@new_database_argument
@click.argument("hostname", metavar="<hostname>")
def import_tumblr(database, hostname):
    """
    Imports blog from Tumblr.

    \b
    Example:
    \b
        lje.py import tumblr myblog.db eigenein.tumblr.com
    """

    session = requests.Session()
    imported_posts = 0

    with ConnectionWrapper(database) as connection, connection.cursor() as cursor:
        cursor.initialize_database()

        response = tumblr_get(session, "info", hostname)

        cursor.upsert_option("author.name", response["blog"]["name"])
        cursor.upsert_option("blog.title", response["blog"]["title"])
        cursor.upsert_option("blog.url", response["blog"]["url"])

        for offset in itertools.count(0, 20):
            response = tumblr_get(session, "posts/text", hostname, filter="raw", offset=offset, limit=20)
            if offset >= response["total_posts"]:
                break
            for post in response["posts"]:
                cursor.upsert_post(post["slug"], post["timestamp"], post["title"], post["body"])
                imported_posts += 1
                logging.info("Imported: %s.", post["slug"])

    logging.info("Imported posts: %d.", imported_posts)

def tumblr_get(session, method, hostname, **params):
    "Makes request to Tumblr API."

    params["api_key"] = "x4OpEVw3OfxdUXA46aCXh3M308SMRKCA6LklBnSzMNvKOCXMFD"
    url = "http://api.tumblr.com/v2/blog/{}/{}".format(hostname, method)
    response = session.get(url, params=params)
    response.raise_for_status()
    response = response.json()
    return response["response"]

# Version command.
# ------------------------------------------------------------------------------

@click.command()
def version():
    """Print version and exit."""
    print(__version__)

# Entry point.
# ------------------------------------------------------------------------------

@click.group(cls=AliasedGroup)
def main():
    """
    Љ is a small and easy static blog generator.
    """
    logging.basicConfig(format="%(message)s", level=logging.INFO, stream=sys.stderr)

if __name__ == "__main__":
    options.add_command(get_option)
    options.add_command(set_option)
    options.add_command(list_options)
    import_.add_command(import_tumblr)
    main.add_command(build)
    main.add_command(init)
    main.add_command(compose)
    main.add_command(edit)
    main.add_command(publish)
    main.add_command(unpublish)
    main.add_command(options)
    main.add_command(import_)
    main.add_command(version)
    main()
<commit_msg>Added help<commit_after>#!/usr/bin/env python3
# coding: utf-8

import sys; sys.dont_write_bytecode = True

import collections
import contextlib
import datetime
import itertools
import logging
import os
import pathlib
import sqlite3
import tempfile

import click
import requests

__version__ = "0.1a"

# Database functions.
# ------------------------------------------------------------------------------

class ConnectionWrapper:
    "Database connection wrapper."

    def __init__(self, connection):
        self.connection = connection

    def __enter__(self):
        return self

    def cursor(self):
        "Gets database cursor wrapper."

        return CursorWrapper(self.connection.cursor())

    def __exit__(self, exc_type, exc_value, traceback):
        self.connection.close()

class CursorWrapper:
    "Database cursor wrapper class."

    def __init__(self, cursor):
        self.cursor = cursor

    def __enter__(self):
        return self

    def initialize_database(self):
        "Initializes empty database."

        self.cursor.execute("""
            create table options (
                name text not null primary key,
                integer_value integer,
                real_value real,
                text_value text,
                blob_value blob
            )""")
        self.cursor.execute("""
            create table posts (
                key text not null primary key,
                timestamp integer not null,
                title text null,
                text_ text not null
            )""")

    def upsert_option(self, name, value):
        "Inserts or updates option."
        logging.info("Setting option `%s` to `%s`.", name, value)
        row = self.make_option_row(name, value)
        try:
            self.cursor.execute("""
                insert into options (integer_value, real_value, text_value, blob_value, name)
                values (?, ?, ?, ?, ?)
            """, row)
        except sqlite3.IntegrityError:
            self.cursor.execute("""
                update options
                set integer_value = ?, real_value = ?, text_value = ?, blob_value = ? where name = ?
            """, row)

    def make_option_row(self, name, value):
        "Gets option row by value."
        return (as_(value, int), as_(value, float), as_(value, str), as_(value, bytes), name)

    def insert_post(self, key, timestamp, title, text):
        "Insert new post."
        self.cursor.execute("insert into posts values (?, ?, ?, ?)", (key, get_timestamp(), title, text))

    def update_post(self, key, timestamp, title, text):
        "Updates existing post."
        self.cursor.execute(
            "update posts set timestamp = ?, title = ?, text = ? where key = ?", (
            timestamp, title, text, key,
        ))

    def upsert_post(self, key, timestamp, title, text):
        "Inserts new post or updates if exists."
        try:
            self.insert_post(key, timestamp, title, text)
        except sqlite3.IntegrityError:
            self.update_post(key, timestamp, title, text)

    def __exit__(self, exc_type, exc_value, traceback):
        if not exc_type:
            self.cursor.connection.commit()
        else:
            self.cursor.connection.rollback()
        self.cursor.connection.close()

# Utilities.
# ------------------------------------------------------------------------------

def as_(value, type):
    return value if isinstance(value, type) else None

def urlify(title):
    "Gets post key by title."
    return title.lower().replace(" ", "-")

def get_text(editor, key, text=""):
    "Execute editor and read user input."

    fd, path = tempfile.mkstemp(prefix="lje-{}-".format(key), suffix=".txt", text=True)
    try:
        with os.fdopen(fd, "wt", encoding="utf-8") as fp:
            fp.write(text)
        os.system("{0} \"{1}\"".format(editor, path))
        with open(path, "rt", encoding="utf-8") as fp:
            return fp.read()
    finally:
        pathlib.Path(path).unlink()

def get_timestamp():
    "Gets UTC timestamp."
    return int((datetime.datetime.utcnow() - datetime.datetime(1970, 1, 1)).total_seconds())

# Common options, arguments and types.
# ------------------------------------------------------------------------------

class AliasedGroup(click.Group):

    def get_command(self, ctx, name):
        command = click.Group.get_command(self, ctx, name)
        if command is not None:
            return command
        matches = [command for command in self.list_commands(ctx) if command.startswith(name)]
        if not matches:
            return None
        if len(matches) == 1:
            return click.Group.get_command(self, ctx, matches[0])
        ctx.fail("`{0}` is not a command. Did you mean one of these? {1}".format(
            name, ", ".join(matches)))

class SQLiteType(click.ParamType):
    name = "sqlite"

    def __init__(self, exists):
        super()
        self.exists = exists

    def convert(self, value, param, ctx):
        if pathlib.Path(value).exists():
            if not self.exists:
                raise click.UsageError("database already exists")
        else:
            if self.exists:
                raise click.UsageError("existing database is expected")
        return sqlite3.connect(value)

editor_option = click.option(
    "-e", "--editor",
    default="env editor",
    help="Editor command.",
    metavar="<editor>",
    show_default=True,
)

existing_database_argument = click.argument("database", metavar="<database>", type=SQLiteType(exists=True))
new_database_argument = click.argument("database", metavar="<database>", type=SQLiteType(exists=False))

# Build command.
# ------------------------------------------------------------------------------

@click.command(short_help="Builds blog.")
@existing_database_argument
def build(database):
    pass

# Init command.
# ------------------------------------------------------------------------------

@click.command(short_help="Initialize new blog.")
@new_database_argument
@click.option("--name", help="Your name.", metavar="<name>", prompt=True, required=True)
@click.option("--email", help="Your email.", metavar="<email>", prompt=True, required=True)
@click.option("--title", help="Blog title.", metavar="<title>", prompt=True, required=True)
@click.option("--url", help="Blog URL.", metavar="<url>", prompt=True, required=True)
def init(database, name, email, title, url):
    with ConnectionWrapper(database) as connection, connection.cursor() as cursor:
        cursor.initialize_database()
        cursor.upsert_option("author.email", email)
        cursor.upsert_option("author.name", name)
        cursor.upsert_option("blog.title", title)
        cursor.upsert_option("blog.url", url)

# Compose command.
# ------------------------------------------------------------------------------

@click.command(short_help="Compose new article.")
@existing_database_argument
@editor_option
@click.option("--key", default=None, help="Post key. Example: my-first-post.", metavar="<key>")
@click.option("--title", help="Post title.", metavar="<title>", prompt=True, required=True)
@click.option("--tag", help="Post tag.", metavar="<tag>", multiple=True)
def compose(database, editor, key, title, tag):
    key = key or urlify(title)
    text = get_text(editor, key)
    with ConnectionWrapper(database) as connection, connection.cursor() as cursor:
        cursor.insert_post(key, get_timestamp(), title, text)

# Edit command.
# ------------------------------------------------------------------------------

@click.command(short_help="Edit existing article.")
@editor_option
def edit(editor):
    pass

# Publish command.
# ------------------------------------------------------------------------------

@click.command(short_help="Publish draft.")
def publish():
    pass

# Unpublish command.
# ------------------------------------------------------------------------------

@click.command(short_help="Unpublish post.")
def unpublish():
    pass

# Options command group.
# ------------------------------------------------------------------------------

@click.group(short_help="Get or set blog options.")
def options():
    pass

# Get option command.
# ------------------------------------------------------------------------------

@click.command("get", short_help="Get option.")
def get_option():
    pass

# Set option command.
# ------------------------------------------------------------------------------

@click.command("set", short_help="Set option.")
def set_option():
    pass

# List options command.
# ------------------------------------------------------------------------------

@click.command("list", short_help="List all options.")
def list_options():
    pass

# Import command group.
# ------------------------------------------------------------------------------

@click.group("import", short_help="Import blog.")
def import_():
    """
    Imports another source into a new Љ blog.
    """
    pass

# Import from Tumblr.
# ------------------------------------------------------------------------------

@click.command("tumblr", short_help="Import from tumblr.")
@new_database_argument
@click.argument("hostname", metavar="<hostname>")
def import_tumblr(database, hostname):
    """
    Imports blog from Tumblr.

    At the moment text posts are imported only.

    \b
    Example:
    \b
        lje.py import tumblr myblog.db eigenein.tumblr.com
    """

    session = requests.Session()
    imported_posts = 0

    with ConnectionWrapper(database) as connection, connection.cursor() as cursor:
        cursor.initialize_database()

        response = tumblr_get(session, "info", hostname)

        cursor.upsert_option("author.name", response["blog"]["name"])
        cursor.upsert_option("blog.title", response["blog"]["title"])
        cursor.upsert_option("blog.url", response["blog"]["url"])

        for offset in itertools.count(0, 20):
            response = tumblr_get(session, "posts/text", hostname, filter="raw", offset=offset, limit=20)
            if offset >= response["total_posts"]:
                break
            for post in response["posts"]:
                cursor.upsert_post(post["slug"], post["timestamp"], post["title"], post["body"])
                imported_posts += 1
                logging.info("Imported: %s.", post["slug"])

    logging.info("Imported posts: %d.", imported_posts)

def tumblr_get(session, method, hostname, **params):
    "Makes request to Tumblr API."

    params["api_key"] = "x4OpEVw3OfxdUXA46aCXh3M308SMRKCA6LklBnSzMNvKOCXMFD"
    url = "http://api.tumblr.com/v2/blog/{}/{}".format(hostname, method)
    response = session.get(url, params=params)
    response.raise_for_status()
    response = response.json()
    return response["response"]

# Version command.
# ------------------------------------------------------------------------------

@click.command()
def version():
    """Print version and exit."""
    print(__version__)

# Entry point.
# ------------------------------------------------------------------------------

@click.group(cls=AliasedGroup)
def main():
    """
    Љ is a small and easy static blog generator.
    """
    logging.basicConfig(format="%(message)s", level=logging.INFO, stream=sys.stderr)

if __name__ == "__main__":
    options.add_command(get_option)
    options.add_command(set_option)
    options.add_command(list_options)
    import_.add_command(import_tumblr)
    main.add_command(build)
    main.add_command(init)
    main.add_command(compose)
    main.add_command(edit)
    main.add_command(publish)
    main.add_command(unpublish)
    main.add_command(options)
    main.add_command(import_)
    main.add_command(version)
    main()
<|endoftext|>


<commit_before>import os
import os.path as op
import shutil
import re

import polib

from . import pygettext
from .util import modified_after, dedupe

LC_MESSAGES = 'LC_MESSAGES'

def get_langs(folder):
    return [name for name in os.listdir(folder) if op.isdir(op.join(folder, name))]

def files_with_ext(folder, ext):
    return [op.join(folder, fn) for fn in os.listdir(folder) if fn.endswith(ext)]

def generate_pot(folders, outpath, keywords):
    pyfiles = []
    for folder in folders:
        for root, dirs, filenames in os.walk(folder):
            keep = [fn for fn in filenames if fn.endswith('.py')]
            pyfiles += [op.join(root, fn) for fn in keep]
    pygettext.main(pyfiles, outpath=outpath, keywords=keywords)

def compile_all_po(base_folder):
    langs = get_langs(base_folder)
    for lang in langs:
        pofolder = op.join(base_folder, lang, LC_MESSAGES)
        pofiles = files_with_ext(pofolder, '.po')
        for pofile in pofiles:
            p = polib.pofile(pofile)
            p.save_as_mofile(pofile[:-3] + '.mo')

def merge_locale_dir(target, mergeinto):
    langs = get_langs(target)
    for lang in langs:
        if not op.exists(op.join(mergeinto, lang)):
            continue
        mofolder = op.join(target, lang, LC_MESSAGES)
        mofiles = files_with_ext(mofolder, '.mo')
        for mofile in mofiles:
            shutil.copy(mofile, op.join(mergeinto, lang, LC_MESSAGES))

def merge_pots_into_pos(folder):
    # We're going to take all pot files in `folder` and for each lang, merge it with the po file
    # with the same name.
    potfiles = files_with_ext(folder, '.pot')
    for potfile in potfiles:
        refpot = polib.pofile(potfile)
        refname = op.splitext(op.basename(potfile))[0]
        for lang in get_langs(folder):
            po = polib.pofile(op.join(folder, lang, LC_MESSAGES, refname + '.po'))
            po.merge(refpot)
            po.save()

#--- Cocoa
def all_lproj_paths(folder):
    return files_with_ext(folder, '.lproj')

def all_xib_strings(lprojpath):
    xibnames = [op.splitext(op.basename(fn))[0] for fn in os.listdir(lprojpath) if fn.endswith('.xib')]
    stringspaths = [op.join(lprojpath, fn + '.strings') for fn in xibnames]
    return [path for path in stringspaths if op.exists(path)]

def escape_cocoa_strings(s):
    return s.replace('\\', '\\\\').replace('"', '\\"').replace('\n', '\\n')

def unescape_cocoa_strings(s):
    return s.replace('\\\\', '\\').replace('\\"', '"').replace('\\n', '\n')

def strings2pot(target, dest):
    with open(target, 'rt', encoding='utf-8') as fp:
        contents = fp.read()
    # We're reading an en.lproj file. We only care about the righthand part of the translation.
    re_trans = re.compile(r'".*" = "(.*)";')
    strings = re_trans.findall(contents)
    po = polib.pofile(dest)
    for s in dedupe(strings):
        s = unescape_cocoa_strings(s)
        entry = po.find(s)
        if entry is None:
            entry = polib.POEntry(msgid=s)
            po.append(entry)
        # we don't know or care about a line number so we put 0
        entry.occurrences.append((target, '0'))
    po.save()

def allstrings2pot(lprojpath, dest, excludes=None):
    allstrings = files_with_ext(lprojpath, '.strings')
    if excludes:
        allstrings = [p for p in allstrings if op.splitext(op.basename(p))[0] not in excludes]
    for strings_path in allstrings:
        strings2pot(strings_path, dest)

def strings2po(en_strings, tr_strings, dest):
    # This function is a temporary function to get translation information from XIB strings files
    # into .po files. Because XIB strings look like "123.title" = "foobar";, we need the reference
    # english strings file to know what the english title of object 123 actually is. We could rely
    # on the comment just before every entry, but it seems like newlines and other special
    # characters are ommited from the comment, so it messes up our stuff.
    # Once all cocoa strings have been translated and I only receive translations in the new .po
    # format, this function won't be needed anymore.
    # If the strings you're working with is a non-XIB one, pass None in en_strings.
    print("Processing {} with {} onto {}".format(tr_strings, en_strings, dest))
    re_trans = re.compile(r'"(.*)" = "(.*)";')
    with open(tr_strings, 'rt', encoding='utf-8') as fp:
        contents = fp.read()
    # ref2tr = {123.title: translated}
    ref2tr = {ref: unescape_cocoa_strings(tr) for ref, tr in re_trans.findall(contents)}
    if en_strings:
        with open(en_strings, 'rt', encoding='utf-8') as fp:
            contents = fp.read()
        # ref2en = {123.title: en}
        ref2en = {ref: unescape_cocoa_strings(en) for ref, en in re_trans.findall(contents)}
    else:
        ref2en = {x: x for x in ref2tr}
    po = polib.pofile(dest)
    print("Untranslated entries: {}".format(len(po.untranslated_entries())))
    count = 0
    for ref, en in ref2en.items():
        translated = ref2tr.get(ref)
        if not translated:
            continue
        entry = po.find(en)
        if entry is None:
            print("WARGNING: {} not found in dest .po".format(en))
            continue
        if entry.msgstr:
            if entry.msgstr != translated:
                print("WARNING: {} != {} for {}".format(translated, entry.msgstr, en))
        else:
            entry.msgstr = translated
            count += 1
    print("Updated {} translations".format(count))
    po.save()

def allstrings2po(enlproj, trlproj, dest, excludes=None):
    allstrings = files_with_ext(trlproj, '.strings')
    for stringsfile in allstrings:
        basename = op.splitext(op.basename(stringsfile))[0]
        if excludes and basename in excludes:
            continue
        enstrings = op.join(enlproj, basename + '.strings')
        strings2po(enstrings, stringsfile, dest)

def po2strings(pofile, en_strings, dest):
    # Takes en_strings and replace all righthand parts of "foo" = "bar"; entries with translations
    # in pofile, then puts the result in dest.
    po = polib.pofile(pofile)
    if not modified_after(pofile, dest):
        return
    print("Creating {} from {}".format(dest, pofile))
    with open(en_strings, 'rt', encoding='utf-8') as fp:
        contents = fp.read()
    re_trans = re.compile(r'(?<= = ").*(?=";\n)')
    def repl(match):
        s = match.group(0)
        unescaped = unescape_cocoa_strings(s)
        entry = po.find(unescaped)
        if entry is None:
            print("WARNING: Could not find entry '{}' in .po file".format(s))
            return s
        trans = entry.msgstr
        return escape_cocoa_strings(trans) if trans else s
    contents = re_trans.sub(repl, contents)
    with open(dest, 'wt', encoding='utf-8') as fp:
        fp.write(contents)

def po2allxibstrings(pofile, en_lproj, dest_lproj):
    if not op.exists(dest_lproj):
        os.mkdir(dest_lproj)
    allstrings = all_xib_strings(en_lproj)
    for strings in allstrings:
        deststrings = op.join(dest_lproj, op.basename(strings))
        po2strings(pofile, strings, deststrings)

#--- Qt
def unescape_xml(s):
    return s.replace('"', '"').replace(''', '\'').replace('<', '<').replace('>', '>').replace('&', '&')

def ts2po(tsfile, dest):
    # This function is a temporary function to get translation information from Qt's ts files.
    print("Processing {} onto {}".format(tsfile, dest))
    re_trans = re.compile(r'<source>(.*?)</source>[\n\r\t ]*?<translation>(.*?)</translation>', re.MULTILINE|re.DOTALL)
    with open(tsfile, 'rt', encoding='utf-8') as fp:
        contents = fp.read()
    # ref2tr = {123.title: translated}
    en2tr = {unescape_xml(en): unescape_xml(tr) for en, tr in re_trans.findall(contents)}
    po = polib.pofile(dest)
    print("Untranslated entries: {}".format(len(po.untranslated_entries())))
    count = 0
    for en, translated in en2tr.items():
        entry = po.find(en)
        if entry is None:
            continue
        if entry.msgstr:
            if entry.msgstr != translated:
                print("WARNING: {} != {} for {}".format(translated, entry.msgstr, en))
        else:
            entry.msgstr = translated
            count += 1
    print("Updated {} translations".format(count))
    po.save()
<commit_msg>Don't allow duplicate occurrences in loc.strings2pot()<commit_after>import os
import os.path as op
import shutil
import re

import polib

from . import pygettext
from .util import modified_after, dedupe

LC_MESSAGES = 'LC_MESSAGES'

def get_langs(folder):
    return [name for name in os.listdir(folder) if op.isdir(op.join(folder, name))]

def files_with_ext(folder, ext):
    return [op.join(folder, fn) for fn in os.listdir(folder) if fn.endswith(ext)]

def generate_pot(folders, outpath, keywords):
    pyfiles = []
    for folder in folders:
        for root, dirs, filenames in os.walk(folder):
            keep = [fn for fn in filenames if fn.endswith('.py')]
            pyfiles += [op.join(root, fn) for fn in keep]
    pygettext.main(pyfiles, outpath=outpath, keywords=keywords)

def compile_all_po(base_folder):
    langs = get_langs(base_folder)
    for lang in langs:
        pofolder = op.join(base_folder, lang, LC_MESSAGES)
        pofiles = files_with_ext(pofolder, '.po')
        for pofile in pofiles:
            p = polib.pofile(pofile)
            p.save_as_mofile(pofile[:-3] + '.mo')

def merge_locale_dir(target, mergeinto):
    langs = get_langs(target)
    for lang in langs:
        if not op.exists(op.join(mergeinto, lang)):
            continue
        mofolder = op.join(target, lang, LC_MESSAGES)
        mofiles = files_with_ext(mofolder, '.mo')
        for mofile in mofiles:
            shutil.copy(mofile, op.join(mergeinto, lang, LC_MESSAGES))

def merge_pots_into_pos(folder):
    # We're going to take all pot files in `folder` and for each lang, merge it with the po file
    # with the same name.
    potfiles = files_with_ext(folder, '.pot')
    for potfile in potfiles:
        refpot = polib.pofile(potfile)
        refname = op.splitext(op.basename(potfile))[0]
        for lang in get_langs(folder):
            po = polib.pofile(op.join(folder, lang, LC_MESSAGES, refname + '.po'))
            po.merge(refpot)
            po.save()

#--- Cocoa
def all_lproj_paths(folder):
    return files_with_ext(folder, '.lproj')

def all_xib_strings(lprojpath):
    xibnames = [op.splitext(op.basename(fn))[0] for fn in os.listdir(lprojpath) if fn.endswith('.xib')]
    stringspaths = [op.join(lprojpath, fn + '.strings') for fn in xibnames]
    return [path for path in stringspaths if op.exists(path)]

def escape_cocoa_strings(s):
    return s.replace('\\', '\\\\').replace('"', '\\"').replace('\n', '\\n')

def unescape_cocoa_strings(s):
    return s.replace('\\\\', '\\').replace('\\"', '"').replace('\\n', '\n')

def strings2pot(target, dest):
    with open(target, 'rt', encoding='utf-8') as fp:
        contents = fp.read()
    # We're reading an en.lproj file. We only care about the righthand part of the translation.
    re_trans = re.compile(r'".*" = "(.*)";')
    strings = re_trans.findall(contents)
    po = polib.pofile(dest)
    for s in dedupe(strings):
        s = unescape_cocoa_strings(s)
        entry = po.find(s)
        if entry is None:
            entry = polib.POEntry(msgid=s)
            po.append(entry)
        # we don't know or care about a line number so we put 0
        entry.occurrences.append((target, '0'))
        entry.occurrences = dedupe(entry.occurrences)
    po.save()

def allstrings2pot(lprojpath, dest, excludes=None):
    allstrings = files_with_ext(lprojpath, '.strings')
    if excludes:
        allstrings = [p for p in allstrings if op.splitext(op.basename(p))[0] not in excludes]
    for strings_path in allstrings:
        strings2pot(strings_path, dest)

def strings2po(en_strings, tr_strings, dest):
    # This function is a temporary function to get translation information from XIB strings files
    # into .po files. Because XIB strings look like "123.title" = "foobar";, we need the reference
    # english strings file to know what the english title of object 123 actually is. We could rely
    # on the comment just before every entry, but it seems like newlines and other special
    # characters are ommited from the comment, so it messes up our stuff.
    # Once all cocoa strings have been translated and I only receive translations in the new .po
    # format, this function won't be needed anymore.
    # If the strings you're working with is a non-XIB one, pass None in en_strings.
    print("Processing {} with {} onto {}".format(tr_strings, en_strings, dest))
    re_trans = re.compile(r'"(.*)" = "(.*)";')
    with open(tr_strings, 'rt', encoding='utf-8') as fp:
        contents = fp.read()
    # ref2tr = {123.title: translated}
    ref2tr = {ref: unescape_cocoa_strings(tr) for ref, tr in re_trans.findall(contents)}
    if en_strings:
        with open(en_strings, 'rt', encoding='utf-8') as fp:
            contents = fp.read()
        # ref2en = {123.title: en}
        ref2en = {ref: unescape_cocoa_strings(en) for ref, en in re_trans.findall(contents)}
    else:
        ref2en = {x: x for x in ref2tr}
    po = polib.pofile(dest)
    print("Untranslated entries: {}".format(len(po.untranslated_entries())))
    count = 0
    for ref, en in ref2en.items():
        translated = ref2tr.get(ref)
        if not translated:
            continue
        entry = po.find(en)
        if entry is None:
            print("WARGNING: {} not found in dest .po".format(en))
            continue
        if entry.msgstr:
            if entry.msgstr != translated:
                print("WARNING: {} != {} for {}".format(translated, entry.msgstr, en))
        else:
            entry.msgstr = translated
            count += 1
    print("Updated {} translations".format(count))
    po.save()

def allstrings2po(enlproj, trlproj, dest, excludes=None):
    allstrings = files_with_ext(trlproj, '.strings')
    for stringsfile in allstrings:
        basename = op.splitext(op.basename(stringsfile))[0]
        if excludes and basename in excludes:
            continue
        enstrings = op.join(enlproj, basename + '.strings')
        strings2po(enstrings, stringsfile, dest)

def po2strings(pofile, en_strings, dest):
    # Takes en_strings and replace all righthand parts of "foo" = "bar"; entries with translations
    # in pofile, then puts the result in dest.
    po = polib.pofile(pofile)
    if not modified_after(pofile, dest):
        return
    print("Creating {} from {}".format(dest, pofile))
    with open(en_strings, 'rt', encoding='utf-8') as fp:
        contents = fp.read()
    re_trans = re.compile(r'(?<= = ").*(?=";\n)')
    def repl(match):
        s = match.group(0)
        unescaped = unescape_cocoa_strings(s)
        entry = po.find(unescaped)
        if entry is None:
            print("WARNING: Could not find entry '{}' in .po file".format(s))
            return s
        trans = entry.msgstr
        return escape_cocoa_strings(trans) if trans else s
    contents = re_trans.sub(repl, contents)
    with open(dest, 'wt', encoding='utf-8') as fp:
        fp.write(contents)

def po2allxibstrings(pofile, en_lproj, dest_lproj):
    if not op.exists(dest_lproj):
        os.mkdir(dest_lproj)
    allstrings = all_xib_strings(en_lproj)
    for strings in allstrings:
        deststrings = op.join(dest_lproj, op.basename(strings))
        po2strings(pofile, strings, deststrings)

#--- Qt
def unescape_xml(s):
    return s.replace('"', '"').replace(''', '\'').replace('<', '<').replace('>', '>').replace('&', '&')

def ts2po(tsfile, dest):
    # This function is a temporary function to get translation information from Qt's ts files.
    print("Processing {} onto {}".format(tsfile, dest))
    re_trans = re.compile(r'<source>(.*?)</source>[\n\r\t ]*?<translation>(.*?)</translation>', re.MULTILINE|re.DOTALL)
    with open(tsfile, 'rt', encoding='utf-8') as fp:
        contents = fp.read()
    # ref2tr = {123.title: translated}
    en2tr = {unescape_xml(en): unescape_xml(tr) for en, tr in re_trans.findall(contents)}
    po = polib.pofile(dest)
    print("Untranslated entries: {}".format(len(po.untranslated_entries())))
    count = 0
    for en, translated in en2tr.items():
        entry = po.find(en)
        if entry is None:
            continue
        if entry.msgstr:
            if entry.msgstr != translated:
                print("WARNING: {} != {} for {}".format(translated, entry.msgstr, en))
        else:
            entry.msgstr = translated
            count += 1
    print("Updated {} translations".format(count))
    po.save()
<|endoftext|>


<commit_before>import sys
import time

def uniout(str = '', threadData = '', level = 0, method = 'STDOUT'):
	if method == 'STDOUT':
		_stdout(str, level)
	elif method == 'DELETE':
		_delete(threadData)
	elif method == 'BLOCK':
		_block(threadData)
	elif method == 'ERROR':
		_error(str)

	return

def _stdout(str, level):
	_output(str)

	return

def _delete(threadData):
	str = ''\
		+ '{\n'\
		+ '    "type" : "delete",\n'\
		+ '    "data" : {\n'\
		+ '        "time" : "' + time.asctime() + '",\n'\
		+ '        "title" : "' + threadData['title'].encode('utf-8') + '",\n'\
		+ '        "author" : "' + threadData['author'].encode('utf-8') + '",\n'\
		+ '        "abstract" : "' + threadData['abstract'].encode('utf-8') + '",\n'\
		+ '    }\n'\
		+ '},\n'

	_output(str, stream = 'HISTORY')

	return

def _block(threadData):
	str = ''\
		+ '{\n'\
		+ '    "type" : "block",\n'\
		+ '    "data" : {\n'\
		+ '        "time" : "' + time.asctime() + '",\n'\
		+ '        "author" : "' + threadData['author'].encode('utf-8') + '",\n'\
		+ '    }\n'\
		+ '},\n'\

	_output(str, stream = 'HISTORY')

	return

def _error(str):
	_output(str, stream = 'ERROR')

	return

def _output(str, stream = 'STDOUT'):
	if stream == 'STDOUT':
		print str
	elif stream == 'HISTORY':
		with open('history.log', 'a') as f:
			f.write(str)
	elif stream == 'ERROR':
		with open('error.log', 'a') as f:
			f.write(str)

	return

def test():
	stdStr = 'this is stdout'
	threadData = {
		'title' : 'title',
		'author' : 'author',
		'abstract' : 'abstract'
	}
	errorStr = 'this is error'

	#uniout(stdStr)
	uniout(threadData = threadData, method = 'HISTORY')
	#uniout(threadData = threadData, method = 'BLOCK')
	#uniout(errorStr, method = 'ERROR')

test()
<commit_msg>use colorama to print colorful texts<commit_after>#--*-- coding:utf-8 --*--
import sys
import time
import colorama

__level__ = {
	'DEBUG'  : 50,
	'DEFAULT': 40,
	'SIMPLE' : 30,

	'INFO':40,
	'SUCCESS':40,
	'ERROR':30
}

class log(object):
	"""docstring for log"""
	def __init__(self, logmethod, logtype, level):
		self.method = logmethod
		self.level = level
		self.type = logtype

		if 'console' in logmethod:
			colorama.init(autoreset = True)

	def setLevel(self, level):
		self.level = __level__['level']

	def setOutputFile(self, filename):
		self.filename = filename

	def setMySQL(self, host, username, password, database):
		self.mysql['host'] = host
		self.mysql['username'] = username
		self.mysql['password'] = password
		self.mysql['database'] = database

	def log(self, message, level):
		if 'console' in self.method:
			self.__ToConsole__(message, level)

		if 'file' in self.method:
			self.__ToFile__(message, level)

	def __ToConsole__(self, message, level):
		if level == 'SUCCESS':
			sys.stdout.write(colorama.Style.BRIGHT + colorama.Fore.GREEN +  u'【调试】 ' + message)
			sys.stdout.write('\n')
			sys.stdout.flush()
		if level == 'DEBUG':
			sys.stdout.write(colorama.Style.BRIGHT + colorama.Fore.YELLOW +  u'【调试】 ' + message)
			sys.stdout.write('\n')
			sys.stdout.flush()
		if level == 'INFO':
			sys.stdout.write(colorama.Style.BRIGHT + colorama.Fore.WHITE + u'【信息】 ' + message)
			sys.stdout.write('\n')
			sys.stdout.flush()
		if level == 'ERROR':
			sys.stdout.write(colorama.Style.BRIGHT + colorama.Fore.RED + u'【错误】 ' + message)
			sys.stdout.write('\n')
			sys.stdout.flush()

	def __ToFile__(self, message, level):
		print '[FILE]' + message

	def __ToMySQL__(self, message, level):
		print u'当前版本暂不支持数据库'

test = log('console', 'INFO','DEFAULT')
test.log(u'这是普通信息', 'INFO')
test.log(u'这是错误信息', 'ERROR')
test.log(u'调试信息啊', 'DEBUG')
test.log(u'成功啦！！！', 'SUCCESS')

# def loginit(level = 'DEFAULT', method = ['console']):
# 	__logConfig__['LogLevel']     = level
# 	__logConfig__['OutputMethod'] = method

# 	if

# def uniout(str = '', threadData = '', level = 0, method = 'STDOUT'):
# 	if method == 'STDOUT':
# 		_stdout(str, level)
# 	elif method == 'DELETE':
# 		_delete(threadData)
# 	elif method == 'BLOCK':
# 		_block(threadData)
# 	elif method == 'ERROR':
# 		_error(str)

# 	return

# def _stdout(str, level):
# 	_output(str)

# 	return

# def _delete(threadData):
# 	str = ''\
# 		+ '{\n'\
# 		+ '    "type" : "delete",\n'\
# 		+ '    "data" : {\n'\
# 		+ '        "time" : "' + time.asctime() + '",\n'\
# 		+ '        "title" : "' + threadData['title'].encode('utf-8') + '",\n'\
# 		+ '        "author" : "' + threadData['author'].encode('utf-8') + '",\n'\
# 		+ '        "abstract" : "' + threadData['abstract'].encode('utf-8') + '",\n'\
# 		+ '    }\n'\
# 		+ '},\n'

# 	_output(str, stream = 'HISTORY')

# 	return

# def _block(threadData):
# 	str = ''\
# 		+ '{\n'\
# 		+ '    "type" : "block",\n'\
# 		+ '    "data" : {\n'\
# 		+ '        "time" : "' + time.asctime() + '",\n'\
# 		+ '        "author" : "' + threadData['author'].encode('utf-8') + '",\n'\
# 		+ '    }\n'\
# 		+ '},\n'\

# 	_output(str, stream = 'HISTORY')

# 	return

# def _error(str):
# 	_output(str, stream = 'ERROR')

# 	return

# def _output(str, stream = 'STDOUT'):
# 	if stream == 'STDOUT':
# 		print str
# 	elif stream == 'HISTORY':
# 		with open('history.log', 'a') as f:
# 			f.write(str)
# 	elif stream == 'ERROR':
# 		with open('error.log', 'a') as f:
# 			f.write(str)

# 	return

# def test():
# 	stdStr = 'this is stdout'
# 	threadData = {
# 		'title' : 'title',
# 		'author' : 'author',
# 		'abstract' : 'abstract'
# 	}
# 	errorStr = 'this is error'

# 	#uniout(stdStr)
# 	uniout(threadData = threadData, method = 'HISTORY')
# 	#uniout(threadData = threadData, method = 'BLOCK')
# 	#uniout(errorStr, method = 'ERROR')

# test()
<|endoftext|>


<commit_before>import itertools
import logging
import sys

import gflags
import numpy as np
from theano import tensor as T
import theano

from rembed import util
from rembed.data.boolean import import_binary_bracketed_data as import_data
from rembed.stack import HardStack

FLAGS = gflags.FLAGS

def build_model(vocab_size, seq_length, inputs, vs):
    # Build hard stack which scans over input sequence.
    stack = HardStack(
        FLAGS.embedding_dim, vocab_size, seq_length, FLAGS.num_composition_layers, vs, X=inputs)

    # Extract top element of final stack timestep.
    embeddings = stack.final_stack[:, 0].reshape((-1, FLAGS.embedding_dim))

    # Feed forward through a single output layer
    logits = util.Linear(
        embeddings, FLAGS.embedding_dim, 2, vs, use_bias=True)

    return logits

def build_cost(logits, targets):
    # Clip gradients coming from the cost function.
    logits = theano.gradient.grad_clip(
        logits, -1 * FLAGS.clipping_max_norm, FLAGS.clipping_max_norm)

    predicted_dist = T.nnet.softmax(logits)
    costs = T.nnet.categorical_crossentropy(predicted_dist, targets)
    cost = costs.mean()
    return cost

def tokens_to_ids(vocabulary, dataset):
    """Replace strings in original boolean dataset with token IDs."""

    for example in dataset:
        example["op_sequence"] = [vocabulary[token]
                                  for token in example["op_sequence"]]
    return dataset

def crop_and_pad(dataset, length):
    # NOTE: This can probably be done faster in NumPy if it winds up making a
    # difference.
    for example in dataset:
        padding_amount = length - len(example["op_sequence"])
        if padding_amount < 0:
            print "Cropping len " + str(len(example["op_sequence"]))
            example["op_sequence"] = example["op_sequence"][-padding_amount:-1]
        else:
            example["op_sequence"] = [0] * \
                padding_amount + example["op_sequence"]
    return dataset

def load_data():
    dataset = import_data.import_binary_bracketed_data(FLAGS.data_path)

    # Build vocabulary from data
    # TODO(SB): Use a fixed vocab file in case this takes especially long.
    vocabulary = {import_data.REDUCE_OP: -1,
                  '*PADDING*': 0}
    types = set(itertools.chain.from_iterable([example["op_sequence"]
                                               for example in dataset]))
    types.remove(import_data.REDUCE_OP)
    vocabulary.update({type: i + 1 for i, type in enumerate(types)})

    # Convert token sequences to integer sequences
    dataset = tokens_to_ids(vocabulary, dataset)
    dataset = crop_and_pad(dataset, FLAGS.seq_length)
    X = np.array([example["op_sequence"] for example in dataset],
                 dtype=np.int32)
    y = np.array([0 if example["label"] == "F" else 1 for example in dataset],
                 dtype=np.int32)

    logging.info("Loaded %i examples to sequences of length %i",
                 len(dataset), FLAGS.seq_length)

    # Build batched data iterator.
    # TODO(SB): Add shuffling.
    def data_iter():
        size = FLAGS.batch_size
        start = -1 * size

        # TODO Don't be lazy and drop remainder of examples that don't fit into
        # a final batch [No need to do that if we shuffle. -SB]
        while True:
            start += size
            if start > len(X):
                start = 0
            yield X[start:start + size], y[start:start + size]

    return data_iter(), len(vocabulary) - 1, FLAGS.seq_length

def train():
    data_iter, vocab_size, seq_length = load_data()

    X = T.imatrix("X")
    y = T.ivector("y")
    lr = T.scalar("lr")

    logging.info("Building model.")
    vs = util.VariableStore(
        default_initializer=util.UniformInitializer(FLAGS.init_range))
    logits = build_model(vocab_size, seq_length, X, vs)
    xent_cost = build_cost(logits, y)

    # L2 regularization
    # TODO(SB): Don't naively L2ify the embedding matrix. It'll break on NL.
    l2_cost = 0.0
    for value in vs.vars.values():
        l2_cost += FLAGS.l2_lambda * T.sum(T.sqr(value))
    total_cost = xent_cost + l2_cost

    new_values = util.SGD(total_cost, vs.vars.values(), lr)
    update_fn = theano.function(
        [X, y, lr], [total_cost, xent_cost, l2_cost], updates=new_values)

    for step in range(FLAGS.training_steps):
        X_batch, y_batch = data_iter.next()
        total_cost_value, xent_cost_value, l2_cost_value = update_fn(
            X_batch, y_batch, FLAGS.learning_rate)
        if step % FLAGS.statistics_interval_steps == 0:
            print "Step: %i\tCost: %f %f %f" % (step, total_cost_value, xent_cost_value, l2_cost_value)

if __name__ == '__main__':
    logging.basicConfig(level=logging.DEBUG)

    # Data settings
    gflags.DEFINE_string("data_path", None, "")
    gflags.DEFINE_integer("seq_length", 29, "")

    # Model architecture settings
    gflags.DEFINE_integer("embedding_dim", 5, "")
    gflags.DEFINE_integer("num_composition_layers", 1, "")

    # Optimization settings
    gflags.DEFINE_integer("training_steps", 50000, "")
    gflags.DEFINE_integer("batch_size", 32, "")
    gflags.DEFINE_float("learning_rate", 0.5, "")
    gflags.DEFINE_float("clipping_max_norm", 1.0, "")
    gflags.DEFINE_float("l2_lambda", 0.00001, "")
    gflags.DEFINE_float("init_range", 0.2, "")

    # Display settings
    gflags.DEFINE_integer("statistics_interval_steps", 50, "")

    # Parse command line flags
    FLAGS(sys.argv)

    train()
<commit_msg>Shuffle data after each epoch.<commit_after>import itertools
import logging
import random
import sys

import gflags
import numpy as np
from theano import tensor as T
import theano

from rembed import util
from rembed.data.boolean import import_binary_bracketed_data as import_data
from rembed.stack import HardStack

FLAGS = gflags.FLAGS

def build_model(vocab_size, seq_length, inputs, vs):
    # Build hard stack which scans over input sequence.
    stack = HardStack(
        FLAGS.embedding_dim, vocab_size, seq_length, FLAGS.num_composition_layers, vs, X=inputs)

    # Extract top element of final stack timestep.
    embeddings = stack.final_stack[:, 0].reshape((-1, FLAGS.embedding_dim))

    # Feed forward through a single output layer
    logits = util.Linear(
        embeddings, FLAGS.embedding_dim, 2, vs, use_bias=True)

    return logits

def build_cost(logits, targets):
    # Clip gradients coming from the cost function.
    logits = theano.gradient.grad_clip(
        logits, -1 * FLAGS.clipping_max_norm, FLAGS.clipping_max_norm)

    predicted_dist = T.nnet.softmax(logits)
    costs = T.nnet.categorical_crossentropy(predicted_dist, targets)
    cost = costs.mean()
    return cost

def tokens_to_ids(vocabulary, dataset):
    """Replace strings in original boolean dataset with token IDs."""

    for example in dataset:
        example["op_sequence"] = [vocabulary[token]
                                  for token in example["op_sequence"]]
    return dataset

def crop_and_pad(dataset, length):
    # NOTE: This can probably be done faster in NumPy if it winds up making a
    # difference.
    for example in dataset:
        padding_amount = length - len(example["op_sequence"])
        if padding_amount < 0:
            print "Cropping len " + str(len(example["op_sequence"]))
            example["op_sequence"] = example["op_sequence"][-padding_amount:-1]
        else:
            example["op_sequence"] = [0] * \
                padding_amount + example["op_sequence"]
    return dataset

def load_data():
    dataset = import_data.import_binary_bracketed_data(FLAGS.data_path)

    # Build vocabulary from data
    # TODO(SB): Use a fixed vocab file in case this takes especially long.
    vocabulary = {import_data.REDUCE_OP: -1,
                  '*PADDING*': 0}
    types = set(itertools.chain.from_iterable([example["op_sequence"]
                                               for example in dataset]))
    types.remove(import_data.REDUCE_OP)
    vocabulary.update({type: i + 1 for i, type in enumerate(types)})

    # Convert token sequences to integer sequences
    dataset = tokens_to_ids(vocabulary, dataset)
    dataset = crop_and_pad(dataset, FLAGS.seq_length)
    X = np.array([example["op_sequence"] for example in dataset],
                 dtype=np.int32)
    y = np.array([0 if example["label"] == "F" else 1 for example in dataset],
                 dtype=np.int32)

    logging.info("Loaded %i examples to sequences of length %i",
                 len(dataset), FLAGS.seq_length)

    # Build batched data iterator.
    # TODO(SB): Add shuffling.
    def data_iter():
        size = FLAGS.batch_size
        start = -1 * size
        order = range(len(X))
        random.shuffle(order)

        while True:
            start += size
            if start > len(X):
                start = 0
                random.shuffle(order)
            batch_indices = order[start:start + size]
            yield X[batch_indices], y[batch_indices]

    return data_iter(), len(vocabulary) - 1, FLAGS.seq_length

def train():
    data_iter, vocab_size, seq_length = load_data()

    X = T.imatrix("X")
    y = T.ivector("y")
    lr = T.scalar("lr")

    logging.info("Building model.")
    vs = util.VariableStore(
        default_initializer=util.UniformInitializer(FLAGS.init_range))
    logits = build_model(vocab_size, seq_length, X, vs)
    xent_cost = build_cost(logits, y)

    # L2 regularization
    # TODO(SB): Don't naively L2ify the embedding matrix. It'll break on NL.
    l2_cost = 0.0
    for value in vs.vars.values():
        l2_cost += FLAGS.l2_lambda * T.sum(T.sqr(value))
    total_cost = xent_cost + l2_cost

    new_values = util.SGD(total_cost, vs.vars.values(), lr)
    update_fn = theano.function(
        [X, y, lr], [total_cost, xent_cost, l2_cost], updates=new_values)

    for step in range(FLAGS.training_steps):
        X_batch, y_batch = data_iter.next()
        total_cost_value, xent_cost_value, l2_cost_value = update_fn(
            X_batch, y_batch, FLAGS.learning_rate)
        if step % FLAGS.statistics_interval_steps == 0:
            print "Step: %i\tCost: %f %f %f" % (step, total_cost_value, xent_cost_value, l2_cost_value)

if __name__ == '__main__':
    logging.basicConfig(level=logging.DEBUG)

    # Data settings
    gflags.DEFINE_string("data_path", None, "")
    gflags.DEFINE_integer("seq_length", 29, "")

    # Model architecture settings
    gflags.DEFINE_integer("embedding_dim", 5, "")
    gflags.DEFINE_integer("num_composition_layers", 1, "")

    # Optimization settings
    gflags.DEFINE_integer("training_steps", 50000, "")
    gflags.DEFINE_integer("batch_size", 32, "")
    gflags.DEFINE_float("learning_rate", 0.1, "")
    gflags.DEFINE_float("clipping_max_norm", 1.0, "")
    gflags.DEFINE_float("l2_lambda", 1e-6, "")
    gflags.DEFINE_float("init_range", 0.1, "")

    # Display settings
    gflags.DEFINE_integer("statistics_interval_steps", 50, "")

    # Parse command line flags
    FLAGS(sys.argv)

    train()
<|endoftext|>