<commit_before>from __future__ import division
from __future__ import unicode_literals
from __future__ import print_function
from __future__ import absolute_import

import logging
import os
import re
from typing import Any, Dict, List, Optional, Text

from rasa_nlu import utils
from rasa_nlu.featurizers import Featurizer
from rasa_nlu.training_data import Message
from rasa_nlu.training_data import TrainingData
from rasa_nlu.components import Component
from rasa_nlu.config import RasaNLUModelConfig
from rasa_nlu.model import Metadata

logger = logging.getLogger(__name__)

class CountVectorsFeaturizer(Featurizer):
    """Bag of words featurizer

        Creates bag-of-words representation of intent features
        using sklearn's `CountVectorizer`.
        All tokens which consist only of digits (e.g. 123 and 99
        but not ab12d) will be represented by a single feature.

        To use the idea of Subword Semantic Hashing
        from https://arxiv.org/abs/1810.07150
        set analyzer` to `char_wb`.
    """

    name = "intent_featurizer_count_vectors"

    provides = ["text_features"]

    requires = []

    defaults = {
        # the parameters are taken from
        # sklearn's CountVectorizer

        # whether to use word or character n-grams
        # 'char_wb' creates character n-grams inside word boundaries
        # n-grams at the edges of words are padded with space.
        "analyzer": 'word',  # use 'char' or 'char_wb' for character

        # regular expression for tokens
        # only used if analyzer == 'word'
        "token_pattern": r'(?u)\b\w\w+\b',

        # remove accents during the preprocessing step
        "strip_accents": None,  # {'ascii', 'unicode', None}

        # list of stop words
        "stop_words": None,  # string {'english'}, list, or None (default)

        # min document frequency of a word to add to vocabulary
        # float - the parameter represents a proportion of documents
        # integer - absolute counts
        "min_df": 1,  # float in range [0.0, 1.0] or int

        # max document frequency of a word to add to vocabulary
        # float - the parameter represents a proportion of documents
        # integer - absolute counts
        "max_df": 1.0,  # float in range [0.0, 1.0] or int

        # set range of ngrams to be extracted
        "min_ngram": 1,  # int
        "max_ngram": 1,  # int

        # limit vocabulary size
        "max_features": None,  # int or None

        # if convert all characters to lowercase
        "lowercase": True,  # bool

        # handling Out-Of-Vacabulary (OOV) words
        # will be converted to lowercase if lowercase is True
        "OOV_token": None,  # string or None
        "OOV_words": []  # string or list of strings
    }

    @classmethod
    def required_packages(cls):
        # type: () -> List[Text]
        return ["sklearn"]

    def _load_count_vect_params(self):
        # set analyzer
        self.analyzer = self.component_config['analyzer']

        # regular expression for tokens
        self.token_pattern = self.component_config['token_pattern']

        # remove accents during the preprocessing step
        self.strip_accents = self.component_config['strip_accents']

        # list of stop words
        self.stop_words = self.component_config['stop_words']

        # min number of word occurancies in the document to add to vocabulary
        self.min_df = self.component_config['min_df']

        # max number (fraction if float) of word occurancies
        # in the document to add to vocabulary
        self.max_df = self.component_config['max_df']

        # set ngram range
        self.min_ngram = self.component_config['min_ngram']
        self.max_ngram = self.component_config['max_ngram']

        # limit vocabulary size
        self.max_features = self.component_config['max_features']

        # if convert all characters to lowercase
        self.lowercase = self.component_config['lowercase']

    # noinspection PyPep8Naming
    def _load_OOV_params(self):
        self.OOV_token = self.component_config['OOV_token']

        self.OOV_words = self.component_config['OOV_words']
        if self.OOV_words and not self.OOV_token:
            logger.error("The list OOV_words={} was given, but "
                         "OOV_token was not. OOV words are ignored."
                         "".format(self.OOV_words))
            self.OOV_words = []

        if self.lowercase and self.OOV_token:
            # convert to lowercase
            self.OOV_token = self.OOV_token.lower()
            if self.OOV_words:
                self.OOV_words = [w.lower() for w in self.OOV_words]

    def __init__(self, component_config=None):
        """Construct a new count vectorizer using the sklearn framework."""

        super(CountVectorsFeaturizer, self).__init__(component_config)

        # parameters for sklearn's CountVectorizer
        self._load_count_vect_params()

        # handling Out-Of-Vacabulary (OOV) words
        self._load_OOV_params()

        if self.analyzer != 'word':
            if self.OOV_token is not None:
                logger.warning("Analyzer is set to character, "
                               "provided OOV word token will be ignored.")
            if self.stop_words is not None:
                logger.warning("Analyzer is set to character, "
                               "provided stop words will be ignored.")
            if self.max_ngram == 1:
                logger.warning("Analyzer is set to character, "
                               "but max n-gram is set to 1. "
                               "It means that the vocabulary will "
                               "contain single letters only.")

        # declare class instance for CountVectorizer
        self.vect = None

    def _tokenizer(self, text):
        """Override tokenizer in CountVectorizer"""
        text = re.sub(r'\b[0-9]+\b', '__NUMBER__', text)

        token_pattern = re.compile(self.token_pattern)
        tokens = token_pattern.findall(text)

        if self.OOV_token:
            if hasattr(self.vect, 'vocabulary_'):
                # CountVectorizer is trained, process for prediction
                if self.OOV_token in self.vect.vocabulary_:
                    tokens = [
                        t if t in self.vect.vocabulary_.keys()
                        else self.OOV_token for t in tokens
                    ]
            elif self.OOV_words:
                # CountVectorizer is not trained, process for train
                tokens = [
                    self.OOV_token if t in self.OOV_words else t
                    for t in tokens
                ]

        return tokens

    @staticmethod
    def _get_message_text(message):
        if message.get("spacy_doc"):  # if lemmatize is possible
            return ' '.join([t.lemma_ for t in message.get("spacy_doc")])
        elif message.get("tokens"):  # if directly tokens is provided
            return ' '.join([t.text for t in message.get("tokens")])
        else:
            return message.text

    # noinspection PyPep8Naming
    def _check_OOV_present(self, examples):
        if self.OOV_token and not self.OOV_words:
            for t in examples:
                if self.OOV_token in t or (
                        self.lowercase and self.OOV_token in t.lower()):
                    return
            logger.warning("OOV_token='{}' was given, but it is not present "
                           "in the training data. All unseen words "
                           "will be ignored during prediction."
                           "".format(self.OOV_token))

    def train(self, training_data, cfg=None, **kwargs):
        # type: (TrainingData, RasaNLUModelConfig, Any) -> None
        """Take parameters from config and
            construct a new count vectorizer using the sklearn framework."""
        from sklearn.feature_extraction.text import CountVectorizer

        spacy_nlp = kwargs.get("spacy_nlp")
        if spacy_nlp is not None:
            # create spacy lemma_ for OOV_words
            self.OOV_words = [t.lemma_
                              for w in self.OOV_words
                              for t in spacy_nlp(w)]

        self.vect = CountVectorizer(token_pattern=self.token_pattern,
                                    strip_accents=self.strip_accents,
                                    lowercase=self.lowercase,
                                    stop_words=self.stop_words,
                                    ngram_range=(self.min_ngram,
                                                 self.max_ngram),
                                    max_df=self.max_df,
                                    min_df=self.min_df,
                                    max_features=self.max_features,
                                    tokenizer=self._tokenizer,
                                    analyzer=self.analyzer)

        lem_exs = [self._get_message_text(example)
                   for example in training_data.intent_examples]

        self._check_OOV_present(lem_exs)

        try:
            # noinspection PyPep8Naming
            X = self.vect.fit_transform(lem_exs).toarray()
        except ValueError:
            self.vect = None
            return

        for i, example in enumerate(training_data.intent_examples):
            # create bag for each example
            example.set("text_features",
                        self._combine_with_existing_text_features(example,
                                                                  X[i]))

    def process(self, message, **kwargs):
        # type: (Message, **Any) -> None
        if self.vect is None:
            logger.error("There is no trained CountVectorizer: "
                         "component is either not trained or "
                         "didn't receive enough training data")
        else:
            message_text = self._get_message_text(message)

            bag = self.vect.transform([message_text]).toarray().squeeze()
            message.set("text_features",
                        self._combine_with_existing_text_features(message,
                                                                  bag))

    def persist(self, model_dir):
        # type: (Text) -> Dict[Text, Any]
        """Persist this model into the passed directory.
        Returns the metadata necessary to load the model again."""

        featurizer_file = os.path.join(model_dir, self.name + ".pkl")
        utils.pycloud_pickle(featurizer_file, self)
        return {"featurizer_file": self.name + ".pkl"}

    @classmethod
    def load(cls,
             model_dir=None,  # type: Text
             model_metadata=None,  # type: Metadata
             cached_component=None,  # type: Optional[Component]
             **kwargs  # type: Any
             ):
        # type: (...) -> CountVectorsFeaturizer

        meta = model_metadata.for_component(cls.name)

        if model_dir and meta.get("featurizer_file"):
            file_name = meta.get("featurizer_file")
            featurizer_file = os.path.join(model_dir, file_name)
            return utils.pycloud_unpickle(featurizer_file)
        else:
            logger.warning("Failed to load featurizer. Maybe path {} "
                           "doesn't exist".format(os.path.abspath(model_dir)))
            return CountVectorsFeaturizer(meta)
<commit_msg>rephrase comment<commit_after>from __future__ import division
from __future__ import unicode_literals
from __future__ import print_function
from __future__ import absolute_import

import logging
import os
import re
from typing import Any, Dict, List, Optional, Text

from rasa_nlu import utils
from rasa_nlu.featurizers import Featurizer
from rasa_nlu.training_data import Message
from rasa_nlu.training_data import TrainingData
from rasa_nlu.components import Component
from rasa_nlu.config import RasaNLUModelConfig
from rasa_nlu.model import Metadata

logger = logging.getLogger(__name__)

class CountVectorsFeaturizer(Featurizer):
    """Bag of words featurizer

        Creates bag-of-words representation of intent features
        using sklearn's `CountVectorizer`.
        All tokens which consist only of digits (e.g. 123 and 99
        but not ab12d) will be represented by a single feature.

        Set analyzer` to `char_wb`
        to use the idea of Subword Semantic Hashing
        from https://arxiv.org/abs/1810.07150.
    """

    name = "intent_featurizer_count_vectors"

    provides = ["text_features"]

    requires = []

    defaults = {
        # the parameters are taken from
        # sklearn's CountVectorizer

        # whether to use word or character n-grams
        # 'char_wb' creates character n-grams inside word boundaries
        # n-grams at the edges of words are padded with space.
        "analyzer": 'word',  # use 'char' or 'char_wb' for character

        # regular expression for tokens
        # only used if analyzer == 'word'
        "token_pattern": r'(?u)\b\w\w+\b',

        # remove accents during the preprocessing step
        "strip_accents": None,  # {'ascii', 'unicode', None}

        # list of stop words
        "stop_words": None,  # string {'english'}, list, or None (default)

        # min document frequency of a word to add to vocabulary
        # float - the parameter represents a proportion of documents
        # integer - absolute counts
        "min_df": 1,  # float in range [0.0, 1.0] or int

        # max document frequency of a word to add to vocabulary
        # float - the parameter represents a proportion of documents
        # integer - absolute counts
        "max_df": 1.0,  # float in range [0.0, 1.0] or int

        # set range of ngrams to be extracted
        "min_ngram": 1,  # int
        "max_ngram": 1,  # int

        # limit vocabulary size
        "max_features": None,  # int or None

        # if convert all characters to lowercase
        "lowercase": True,  # bool

        # handling Out-Of-Vacabulary (OOV) words
        # will be converted to lowercase if lowercase is True
        "OOV_token": None,  # string or None
        "OOV_words": []  # string or list of strings
    }

    @classmethod
    def required_packages(cls):
        # type: () -> List[Text]
        return ["sklearn"]

    def _load_count_vect_params(self):
        # set analyzer
        self.analyzer = self.component_config['analyzer']

        # regular expression for tokens
        self.token_pattern = self.component_config['token_pattern']

        # remove accents during the preprocessing step
        self.strip_accents = self.component_config['strip_accents']

        # list of stop words
        self.stop_words = self.component_config['stop_words']

        # min number of word occurancies in the document to add to vocabulary
        self.min_df = self.component_config['min_df']

        # max number (fraction if float) of word occurancies
        # in the document to add to vocabulary
        self.max_df = self.component_config['max_df']

        # set ngram range
        self.min_ngram = self.component_config['min_ngram']
        self.max_ngram = self.component_config['max_ngram']

        # limit vocabulary size
        self.max_features = self.component_config['max_features']

        # if convert all characters to lowercase
        self.lowercase = self.component_config['lowercase']

    # noinspection PyPep8Naming
    def _load_OOV_params(self):
        self.OOV_token = self.component_config['OOV_token']

        self.OOV_words = self.component_config['OOV_words']
        if self.OOV_words and not self.OOV_token:
            logger.error("The list OOV_words={} was given, but "
                         "OOV_token was not. OOV words are ignored."
                         "".format(self.OOV_words))
            self.OOV_words = []

        if self.lowercase and self.OOV_token:
            # convert to lowercase
            self.OOV_token = self.OOV_token.lower()
            if self.OOV_words:
                self.OOV_words = [w.lower() for w in self.OOV_words]

    def __init__(self, component_config=None):
        """Construct a new count vectorizer using the sklearn framework."""

        super(CountVectorsFeaturizer, self).__init__(component_config)

        # parameters for sklearn's CountVectorizer
        self._load_count_vect_params()

        # handling Out-Of-Vacabulary (OOV) words
        self._load_OOV_params()

        if self.analyzer != 'word':
            if self.OOV_token is not None:
                logger.warning("Analyzer is set to character, "
                               "provided OOV word token will be ignored.")
            if self.stop_words is not None:
                logger.warning("Analyzer is set to character, "
                               "provided stop words will be ignored.")
            if self.max_ngram == 1:
                logger.warning("Analyzer is set to character, "
                               "but max n-gram is set to 1. "
                               "It means that the vocabulary will "
                               "contain single letters only.")

        # declare class instance for CountVectorizer
        self.vect = None

    def _tokenizer(self, text):
        """Override tokenizer in CountVectorizer"""
        text = re.sub(r'\b[0-9]+\b', '__NUMBER__', text)

        token_pattern = re.compile(self.token_pattern)
        tokens = token_pattern.findall(text)

        if self.OOV_token:
            if hasattr(self.vect, 'vocabulary_'):
                # CountVectorizer is trained, process for prediction
                if self.OOV_token in self.vect.vocabulary_:
                    tokens = [
                        t if t in self.vect.vocabulary_.keys()
                        else self.OOV_token for t in tokens
                    ]
            elif self.OOV_words:
                # CountVectorizer is not trained, process for train
                tokens = [
                    self.OOV_token if t in self.OOV_words else t
                    for t in tokens
                ]

        return tokens

    @staticmethod
    def _get_message_text(message):
        if message.get("spacy_doc"):  # if lemmatize is possible
            return ' '.join([t.lemma_ for t in message.get("spacy_doc")])
        elif message.get("tokens"):  # if directly tokens is provided
            return ' '.join([t.text for t in message.get("tokens")])
        else:
            return message.text

    # noinspection PyPep8Naming
    def _check_OOV_present(self, examples):
        if self.OOV_token and not self.OOV_words:
            for t in examples:
                if self.OOV_token in t or (
                        self.lowercase and self.OOV_token in t.lower()):
                    return
            logger.warning("OOV_token='{}' was given, but it is not present "
                           "in the training data. All unseen words "
                           "will be ignored during prediction."
                           "".format(self.OOV_token))

    def train(self, training_data, cfg=None, **kwargs):
        # type: (TrainingData, RasaNLUModelConfig, Any) -> None
        """Take parameters from config and
            construct a new count vectorizer using the sklearn framework."""
        from sklearn.feature_extraction.text import CountVectorizer

        spacy_nlp = kwargs.get("spacy_nlp")
        if spacy_nlp is not None:
            # create spacy lemma_ for OOV_words
            self.OOV_words = [t.lemma_
                              for w in self.OOV_words
                              for t in spacy_nlp(w)]

        self.vect = CountVectorizer(token_pattern=self.token_pattern,
                                    strip_accents=self.strip_accents,
                                    lowercase=self.lowercase,
                                    stop_words=self.stop_words,
                                    ngram_range=(self.min_ngram,
                                                 self.max_ngram),
                                    max_df=self.max_df,
                                    min_df=self.min_df,
                                    max_features=self.max_features,
                                    tokenizer=self._tokenizer,
                                    analyzer=self.analyzer)

        lem_exs = [self._get_message_text(example)
                   for example in training_data.intent_examples]

        self._check_OOV_present(lem_exs)

        try:
            # noinspection PyPep8Naming
            X = self.vect.fit_transform(lem_exs).toarray()
        except ValueError:
            self.vect = None
            return

        for i, example in enumerate(training_data.intent_examples):
            # create bag for each example
            example.set("text_features",
                        self._combine_with_existing_text_features(example,
                                                                  X[i]))

    def process(self, message, **kwargs):
        # type: (Message, **Any) -> None
        if self.vect is None:
            logger.error("There is no trained CountVectorizer: "
                         "component is either not trained or "
                         "didn't receive enough training data")
        else:
            message_text = self._get_message_text(message)

            bag = self.vect.transform([message_text]).toarray().squeeze()
            message.set("text_features",
                        self._combine_with_existing_text_features(message,
                                                                  bag))

    def persist(self, model_dir):
        # type: (Text) -> Dict[Text, Any]
        """Persist this model into the passed directory.
        Returns the metadata necessary to load the model again."""

        featurizer_file = os.path.join(model_dir, self.name + ".pkl")
        utils.pycloud_pickle(featurizer_file, self)
        return {"featurizer_file": self.name + ".pkl"}

    @classmethod
    def load(cls,
             model_dir=None,  # type: Text
             model_metadata=None,  # type: Metadata
             cached_component=None,  # type: Optional[Component]
             **kwargs  # type: Any
             ):
        # type: (...) -> CountVectorsFeaturizer

        meta = model_metadata.for_component(cls.name)

        if model_dir and meta.get("featurizer_file"):
            file_name = meta.get("featurizer_file")
            featurizer_file = os.path.join(model_dir, file_name)
            return utils.pycloud_unpickle(featurizer_file)
        else:
            logger.warning("Failed to load featurizer. Maybe path {} "
                           "doesn't exist".format(os.path.abspath(model_dir)))
            return CountVectorsFeaturizer(meta)
<|endoftext|>


<commit_before>import binascii
import os
import rsa
import struct
import subprocess
import sys
from config import *

def u8(data):
	return struct.pack(">B", data)

def u16(data):
	return struct.pack(">H", data)

def u32(data):
	return struct.pack(">I", data)

def u32_littleendian(data):
	return struct.pack("<I", data)

if (len(sys.argv) != 4):
	print "Usage: sign.py <mode> <input file> <output file>"
	print "\n"
	print "Mode can either be set to dec (not-encrypted file) or enc (encrypted file)."
	sys.exit(1)

input_file = sys.argv[2]
input_file2 = sys.argv[2] + "2"
output_file = sys.argv[3]

if os.path.exists(output_file):
	os.remove(output_file)

if (sys.argv[1] == "dec"):
	with open(input_file, "rb") as source_file:
		crc32 = format(binascii.crc32(source_file.read()) & 0xFFFFFFFF, '08x')

	filesize = u32(os.path.getsize(input_file) + 12)

	with open(input_file, "rb") as source_file:
		with open(input_file2, "w+") as dest_file:
			read = source_file.read()
			version = u32(0)
			dest_file.write(version)
			dest_file.write(filesize)
			dest_file.write(binascii.unhexlify(crc32))
			dest_file.write(read)

	lz77 = subprocess.call(["mono", "--runtime=v4.0.30319", dsdecmp_path + "/DSDecmp.exe", "-c", "lz10", input_file2, output_file])

	with open(output_file, "rb") as source_file:
		read = source_file.read()

	with open(rsa_key_path, "rb") as source_file:
		private_key_data = source_file.read()

	private_key = rsa.PrivateKey.load_pkcs1(private_key_data, "PEM")

	signature = rsa.sign(read, private_key, "SHA-1")

	with open(output_file, "w+") as dest_file:
		dest_file.write(binascii.unhexlify(hex(int(0))[2:].zfill(128)))
		dest_file.write(signature)
		dest_file.write(read)

	os.remove(input_file)

elif (sys.argv[1] == "enc"):
	with open(input_file, "rb") as source_file:
		read = source_file.read()

	with open(rsa_key_path, "rb") as source_file:
		private_key_data = source_file.read()

	private_key = rsa.PrivateKey.load_pkcs1(private_key_data, "PEM")

	signature = rsa.sign(read, private_key, "SHA-1")

	with open(aes_key_path, "rb") as source_file:
		key_data = binascii.hexlify(source_file.read())

	iv_data = binascii.b2a_hex(os.urandom(16))

	encrypt_aes = subprocess.call(["openssl", "enc", "-aes-128-ofb", "-in", input_file, "-out", output_file, "-K", key_data, "-iv", iv_data])

	with open(output_file, "rb") as source_file:
		read = source_file.read()

	with open(output_file, "w+") as dest_file:
		dest_file.write(binascii.unhexlify("574332340000000100000000010000000000000000000000000000000000000000000000000000000000000000000000"))
		dest_file.write(binascii.unhexlify(iv_data))
		dest_file.write(signature)
		dest_file.write(read)
<commit_msg>Fixed lzss<commit_after>import binascii
import os
import rsa
import struct
import subprocess
import sys
from config import *

def u8(data):
	return struct.pack(">B", data)

def u16(data):
	return struct.pack(">H", data)

def u32(data):
	return struct.pack(">I", data)

def u32_littleendian(data):
	return struct.pack("<I", data)

if (len(sys.argv) != 4):
	print "Usage: sign.py <mode> <input file> <output file>"
	print "\n"
	print "Mode can either be set to dec (not-encrypted file) or enc (encrypted file)."
	sys.exit(1)

input_file = sys.argv[2]
input_file2 = sys.argv[2] + "2"
output_file = sys.argv[3]

if os.path.exists(output_file):
	os.remove(output_file)

if (sys.argv[1] == "dec"):
	with open(input_file, "rb") as source_file:
		crc32 = format(binascii.crc32(source_file.read()) & 0xFFFFFFFF, '08x')

	filesize = u32(os.path.getsize(input_file) + 12)

	with open(input_file, "rb") as source_file:
		with open(input_file2, "w+") as dest_file:
			read = source_file.read()
			version = u32(0)
			dest_file.write(version)
			dest_file.write(filesize)
			dest_file.write(binascii.unhexlify(crc32))
			dest_file.write(read)

	lz77 = subprocess.call([lzss_path + "lzss", "-evf", input_file2, output_file])

	with open(output_file, "rb") as source_file:
		read = source_file.read()

	with open(rsa_key_path, "rb") as source_file:
		private_key_data = source_file.read()

	private_key = rsa.PrivateKey.load_pkcs1(private_key_data, "PEM")

	signature = rsa.sign(read, private_key, "SHA-1")

	with open(output_file, "w+") as dest_file:
		dest_file.write(binascii.unhexlify(hex(int(0))[2:].zfill(128)))
		dest_file.write(signature)
		dest_file.write(read)

	os.remove(input_file)

elif (sys.argv[1] == "enc"):
	with open(input_file, "rb") as source_file:
		read = source_file.read()

	with open(rsa_key_path, "rb") as source_file:
		private_key_data = source_file.read()

	private_key = rsa.PrivateKey.load_pkcs1(private_key_data, "PEM")

	signature = rsa.sign(read, private_key, "SHA-1")

	with open(aes_key_path, "rb") as source_file:
		key_data = binascii.hexlify(source_file.read())

	iv_data = binascii.b2a_hex(os.urandom(16))

	encrypt_aes = subprocess.call(["openssl", "enc", "-aes-128-ofb", "-in", input_file, "-out", output_file, "-K", key_data, "-iv", iv_data])

	with open(output_file, "rb") as source_file:
		read = source_file.read()

	with open(output_file, "w+") as dest_file:
		dest_file.write(binascii.unhexlify("574332340000000100000000010000000000000000000000000000000000000000000000000000000000000000000000"))
		dest_file.write(binascii.unhexlify(iv_data))
		dest_file.write(signature)
		dest_file.write(read)
<|endoftext|>


<commit_before>import sublime
import sublime_plugin

import sys
import struct
import subprocess
import os
import atexit
import shlex

running_server = None

class HGServer(object):
    """I drive a Mercurial command server (Mercurial>=1.9).

    Mercurial command server protocol: http://mercurial.selenic.com/wiki/CommandServer
    """
    def __init__(self, hg_exe='hg'):
        global running_server
        if not running_server or running_server.stdin.closed:
            startupinfo = None
            if os.name == 'nt':
                # Hide the child process window on Windows.
                startupinfo = subprocess.STARTUPINFO()
                startupinfo.dwFlags |= subprocess.STARTF_USESHOWWINDOW

            self.server = subprocess.Popen(
                                    [hg_exe, "serve", "--cmdserver", "pipe"],
                                    stdin=subprocess.PIPE, stdout=subprocess.PIPE,
                                    stderr=subprocess.PIPE,
                                    startupinfo=startupinfo
                                    )

            # Is this needed?
            atexit.register(self.shut_down)

            self.receive_greeting()
            self.encoding = self.get_encoding()

            running_server = self.server
            return

        # Reuse existing server.
        self.server = running_server
        self.encoding = self.get_encoding()

    def receive_greeting(self):
        try:
            channel, data = self.read_data()
        except struct.error:
            err = self.server.stderr.read()
            raise EnvironmentError(err)
        except EnvironmentError:
            err = self.server.stderr.read()
            EnvironmentError(err)
            raise

        try:
            caps, enc = data.split('\n')
        except ValueError:
            # Means the server isn't returning the promised data, so the
            # environment is wrong.
            print "SublimeHg:err: SublimeHg requires Mercurial>=1.9. (Probable cause.)"
            raise EnvironmentError("SublimeHg requires Mercurial>=1.9")

        caps = ', '.join(caps.split()[1:])
        print "SublimeHg:inf:", "Capabilities:", caps
        print "SublimeHg:inf: Encoding:", enc.split()[1]

    def read_data(self):
        channel, length = struct.unpack('>cI', self.server.stdout.read(5))
        if channel in 'IL':
            return channel, length

        return channel, self.server.stdout.read(length)

    def write_block(self, cmd, *args):
        self.server.stdin.write(cmd + '\n')

        if args:
            data = '\0'.join(args)
            length = struct.pack('>l', len(data))
            self.server.stdin.write(length + data)
            self.server.stdin.flush()

    def run_command(self, *args):
        if len(args) == 1 and ' ' in args[0]:
            args = shlex.split(args[0])

        if args[0] == 'hg':
            print "SublimeHg:inf: Stripped superfluous 'hg' from '%s'" % ' '.join(args)
            args = args[1:]

        print "SublimeHg:inf: Sending command '%s' as %s" % (' '.join(args), args)
        self.write_block('runcommand', *args)

        rv = ''
        while True:
            channel, line = self.read_data()
            if channel == 'o':
                rv += line
            elif channel == 'r':
                print "SublimeHg:inf: Return value: %s" % struct.unpack('>l', line)[0]
                return rv[:-1]
            else:
                print "SublimeHg:err: " + line
                #  XXX Ask user for more input instead of blowing up.
                rv = "Could not complete operation. Was your command complete?"
                self.shut_down()
                return rv

    def get_encoding(self):
        self.write_block('getencoding')
        return self.read_data()[1]

    def shut_down(self):
        print "SublimeHg:inf: Shutting down HG server..."
        if not self.server.stdin.closed:
            self.server.stdin.close()

class HgCmdLineCommand(sublime_plugin.TextCommand):

    def configure(self):
        s = sublime.load_settings('Global.sublime-settings')
        user_exe = s.get('packages.sublime_hg.hg_exe')
        self.hg_exe = user_exe or 'hg'

    def run(self, edit, cmd=None):
        self.edit = edit
        if not cmd:
            self.view.window().show_input_panel('Hg command:', 'status', self.on_done, None, None)
            return
        self.on_done(cmd)

    def on_done(self, s):
        old_cd = os.getcwd()
        os.chdir(os.path.dirname(self.view.file_name()))

        if not hasattr(self, 'hg_exe'):
            self.configure()

        try:
            hgs = HGServer(self.hg_exe)
        except EnvironmentError, e:
            sublime.status_message("SublimeHg:err:" + str(e))
            os.chdir(old_cd)
            return

        try:
            # Where's the encoding done here?
            data = hgs.run_command(s.encode(hgs.encoding))
            p = self.view.window().get_output_panel('hgs')
            p.insert(self.edit, 0, data)
            self.view.window().run_command('show_panel', {'panel': 'output.hgs'})
        except UnicodeDecodeError, e:
            print "Oops (funny characters!)..."
            print e
        finally:
            os.chdir(old_cd)

# TODO: Add serve and start in a separate process?
SUBLIMEHG_CMDS = sorted([
    "add", "addremove", "annotate", "archive", "backout", "bisect",
    "bookmarks", "branch", "branches", "bundle", "cat", "clone", "commit",
    "copy", "diff", "export", "forget", "grep", "heads", "help", "identify",
    "import", "incoming", "init", "locate", "log", "manifest", "merge",
    "outgoing", "parents", "paths", "pull", "push", "recover", "remove",
    "rename", "resolve", "revert", "rollback", "root", "showconfig", "status",
    "summary", "tag", "tags", "tip", "unbundle", "update", "verify", "version"
    ])

class HgCommand(sublime_plugin.TextCommand):

    def run(self, edit):
        self.edit = edit
        self.view.window().show_quick_panel(SUBLIMEHG_CMDS, self.on_done)

    def on_done(self, s):
        if s == -1: return

        if SUBLIMEHG_CMDS[s] == 'commit':
            self.view.run_command("hg_commit")
            return
        elif SUBLIMEHG_CMDS[s] == 'commit (this file)':
            fn = self.view.file_name()
            if not fn: return
            self.view.run_command("hg_commit", {"what": fn})
            return

        self.view.run_command("hg_cmd_line", {"cmd": SUBLIMEHG_CMDS[s]})

class HgCommit(sublime_plugin.TextCommand):
    def run(self, edit, what=''):
        self.what = what
        self.view.window().show_input_panel("Hg commit message:", '', self.on_done, None, None)

    def on_done(self, s):
        if ' ' in self.what:
            self.view.run_command("hg_cmd_line", {"cmd":"commit '%s' -m \"%s\"" % (self.what, s)})
            return
        self.view.run_command("hg_cmd_line", {"cmd":"commit %s -m \"%s\"" % (self.what, s)})
<commit_msg>Fixed bug.<commit_after>import sublime
import sublime_plugin

import sys
import struct
import subprocess
import os
import atexit
import shlex

running_server = None

class HGServer(object):
    """I drive a Mercurial command server (Mercurial>=1.9).

    Mercurial command server protocol: http://mercurial.selenic.com/wiki/CommandServer
    """
    def __init__(self, hg_exe='hg'):
        global running_server
        if not running_server or running_server.stdin.closed:
            startupinfo = None
            if os.name == 'nt':
                # Hide the child process window on Windows.
                startupinfo = subprocess.STARTUPINFO()
                startupinfo.dwFlags |= subprocess.STARTF_USESHOWWINDOW

            self.server = subprocess.Popen(
                                    [hg_exe, "serve", "--cmdserver", "pipe"],
                                    stdin=subprocess.PIPE, stdout=subprocess.PIPE,
                                    stderr=subprocess.PIPE,
                                    startupinfo=startupinfo
                                    )

            # Is this needed?
            atexit.register(self.shut_down)

            self.receive_greeting()
            self.encoding = self.get_encoding()

            running_server = self.server
            return

        # Reuse existing server.
        self.server = running_server
        self.encoding = self.get_encoding()

    def receive_greeting(self):
        try:
            channel, data = self.read_data()
        except struct.error:
            err = self.server.stderr.read()
            raise EnvironmentError(err)
        except EnvironmentError:
            err = self.server.stderr.read()
            EnvironmentError(err)
            raise

        try:
            caps, enc = data.split('\n')
        except ValueError:
            # Means the server isn't returning the promised data, so the
            # environment is wrong.
            print "SublimeHg:err: SublimeHg requires Mercurial>=1.9. (Probable cause.)"
            raise EnvironmentError("SublimeHg requires Mercurial>=1.9")

        caps = ', '.join(caps.split()[1:])
        print "SublimeHg:inf:", "Capabilities:", caps
        print "SublimeHg:inf: Encoding:", enc.split()[1]

    def read_data(self):
        channel, length = struct.unpack('>cI', self.server.stdout.read(5))
        if channel in 'IL':
            return channel, length

        return channel, self.server.stdout.read(length)

    def write_block(self, cmd, *args):
        self.server.stdin.write(cmd + '\n')

        if args:
            data = '\0'.join(args)
            length = struct.pack('>l', len(data))
            self.server.stdin.write(length + data)
            self.server.stdin.flush()

    def run_command(self, *args):
        if len(args) == 1 and ' ' in args[0]:
            args = shlex.split(args[0])

        if args[0] == 'hg':
            print "SublimeHg:inf: Stripped superfluous 'hg' from '%s'" % ' '.join(args)
            args = args[1:]

        print "SublimeHg:inf: Sending command '%s' as %s" % (' '.join(args), args)
        self.write_block('runcommand', *args)

        rv = ''
        while True:
            channel, line = self.read_data()
            if channel == 'o':
                rv += line
            elif channel == 'r':
                print "SublimeHg:inf: Return value: %s" % struct.unpack('>l', line)[0]
                return rv[:-1]
            else:
                print "SublimeHg:err: " + line
                #  XXX Ask user for more input instead of blowing up.
                rv = "Could not complete operation. Was your command complete?"
                self.shut_down()
                return rv

    def get_encoding(self):
        self.write_block('getencoding')
        return self.read_data()[1]

    def shut_down(self):
        print "SublimeHg:inf: Shutting down HG server..."
        if not self.server.stdin.closed:
            self.server.stdin.close()

class HgCmdLineCommand(sublime_plugin.TextCommand):

    def configure(self):
        s = sublime.load_settings('Global.sublime-settings')
        user_exe = s.get('packages.sublime_hg.hg_exe')
        self.hg_exe = user_exe or 'hg'

    def run(self, edit, cmd=None):
        self.edit = edit
        if not cmd:
            self.view.window().show_input_panel('Hg command:', 'status', self.on_done, None, None)
            return
        self.on_done(cmd)

    def on_done(self, s):
        old_cd = os.getcwd()
        os.chdir(os.path.dirname(self.view.file_name()))

        if not hasattr(self, 'hg_exe'):
            self.configure()

        try:
            hgs = HGServer(self.hg_exe)
        except EnvironmentError, e:
            sublime.status_message("SublimeHg:err:" + str(e))
            os.chdir(old_cd)
            return

        try:
            # Where's the encoding done here?
            data = hgs.run_command(s.encode(hgs.encoding))
            p = self.view.window().get_output_panel('hgs')
            p.insert(self.edit, 0, data.decode(hgs.encoding))
            self.view.window().run_command('show_panel', {'panel': 'output.hgs'})
        except UnicodeDecodeError, e:
            print "Oops (funny characters!)..."
            print e
        finally:
            os.chdir(old_cd)

# TODO: Add serve and start in a separate process?
SUBLIMEHG_CMDS = sorted([
    "add", "addremove", "annotate", "archive", "backout", "bisect",
    "bookmarks", "branch", "branches", "bundle", "cat", "clone", "commit",
    "copy", "diff", "export", "forget", "grep", "heads", "help", "identify",
    "import", "incoming", "init", "locate", "log", "manifest", "merge",
    "outgoing", "parents", "paths", "pull", "push", "recover", "remove",
    "rename", "resolve", "revert", "rollback", "root", "showconfig", "status",
    "summary", "tag", "tags", "tip", "unbundle", "update", "verify", "version"
    ])

class HgCommand(sublime_plugin.TextCommand):

    def run(self, edit):
        self.edit = edit
        self.view.window().show_quick_panel(SUBLIMEHG_CMDS, self.on_done)

    def on_done(self, s):
        if s == -1: return

        if SUBLIMEHG_CMDS[s] == 'commit':
            self.view.run_command("hg_commit")
            return
        elif SUBLIMEHG_CMDS[s] == 'commit (this file)':
            fn = self.view.file_name()
            if not fn: return
            self.view.run_command("hg_commit", {"what": fn})
            return

        self.view.run_command("hg_cmd_line", {"cmd": SUBLIMEHG_CMDS[s]})

class HgCommit(sublime_plugin.TextCommand):
    def run(self, edit, what=''):
        self.what = what
        self.view.window().show_input_panel("Hg commit message:", '', self.on_done, None, None)

    def on_done(self, s):
        if ' ' in self.what:
            self.view.run_command("hg_cmd_line", {"cmd":"commit '%s' -m \"%s\"" % (self.what, s)})
            return
        self.view.run_command("hg_cmd_line", {"cmd":"commit %s -m \"%s\"" % (self.what, s)})
<|endoftext|>


<commit_before>#!/usr/bin/python3

import os
import re
import sys
import json
import arrow
import markdown
import unicodedata
from collections import namedtuple
from bs4 import BeautifulSoup

def get_properties(properties_file="properties.json"):
    if os.path.exists(properties_file):
        with open(properties_file) as f:
            return json.loads(f.read())

def generate_pages(properties):
    proj_root = properties.get('root', '/')
    homepage = properties.get('homepage', 'index.html')
    searchpage = properties.get('searchpage', 'search.html')
    site_title = properties.get('site_title', 'PyQuo')
    ts_frmt = properties.get('timestamp_format', 'dddd DD MMMM, YYYY')
    directory = properties.get('source_directory', 'markdown')
    css = properties.get('css')

    all_pages = []
    categories = []
    count = 0

    for root, dirs, files in os.walk(directory):
        valid_files = [file for file in files if not
                       (file.endswith('~') or file.startswith('.'))]

        for valid_file in valid_files:
            with open(os.path.join(root, valid_file), 'r') as input_file:
                md = markdown.Markdown(extensions=['markdown.extensions.meta'])
                markdown_text = input_file.read()
                html = md.convert(markdown_text)
                text = ''.join(BeautifulSoup(html).findAll(text=True))
            if md.Meta == {}:
                continue

            meta = namedtuple('meta', [])
            meta.title = md.Meta['title'][0]
            meta.slug = slugify(meta.title)
            meta.categories = [
                category for category in set([cat.replace(" ", "").lower()
                for cat in md.Meta['category'][0].split(',')])]
            meta.authors = md.Meta['authors'][0]
            meta.date = md.Meta['date'][0]
            meta.tags = md.Meta['tags'][0].split(',')
            meta.header_image = md.Meta['headerimage'][0]
            meta.publish = True if md.Meta['publish'][0].lower() in [
                'true', 'yes'] else False
            meta.content = html
            meta.index = re.findall("\w+", text.lower())
            categories.extend(meta.categories)

            if meta.publish is True:
                generate_static_page(site_title, homepage, searchpage, meta,
                                     css, ts_frmt, proj_root)
                all_pages.append(meta)
                count += 1
        all_cats = [category for category in set(categories)]
        print('{} pages generated with the following categories: {}.'
              .format(count, ', '.join(all_cats)))
        all_cats, tags, chronology, index, word_cloud =\
            extract_site_wide_metadata(all_pages)
        generate_front_page(site_title, homepage, searchpage, all_cats, tags,
                            chronology, word_cloud, proj_root, css, ts_frmt)
        generate_search_page(site_title, homepage, searchpage, index, css,
                             proj_root)

def extract_site_wide_metadata(all_pages):
    """Get site-wide metadata.

    - Arrange titles by inverse date order, with link to location.
    - Generate tag cloud (dict of tag and number of times used).
    - Generate calendar so can click on a date and get link.
    """

    categories = []
    tags = {}
    chronology = {}
    index = {}
    for page in all_pages:
        for cat in page.categories:
            category = cat.lower()
            if category not in categories:
                categories.append(category)

            for tag in page.tags:
                if tag not in tags:
                    tags[tag] = 1
                else:
                    tags[tag] += 1

            if page.date not in chronology:
                link = "{}/{}.html".format(category, page.slug)

                if page.date != '':
                    timestamp = arrow.get(page.date)
                else:
                    timestamp = arrow.now()
                add_to_chronology_dict(
                    chronology, timestamp, (page.title, link))

            for idx in sorted(page.index + [tag.lower().strip() for tag in
                              page.tags if tag != ""]):
                word = idx.strip()
                link = chronology[timestamp][1]
                if word not in index:
                    index[word] = [link]
                else:
                    if link not in index[word]:
                        index[word].append(link)
    words_used = [(idx, len(index[idx])) for idx in index]
    word_cloud = sorted(words_used, key=lambda x: x[1], reverse=True)

    return categories, tags, chronology, index, word_cloud

def add_to_chronology_dict(chronology, timestamp, value):
    if timestamp in chronology:
        for num in range(1, 3600):
            one_second_later = timestamp.replace(seconds =+ num)
            if one_second_later not in chronology:
                timestamp = one_second_later
                break
    chronology[timestamp] = value

def generate_front_page(site_title, homepage, searchpage, categories, tags,
                        chronology, word_cloud, proj_root, css, ts_frmt):
    with open(homepage, 'w') as op_file:
        print('', file=op_file)
        print('    ', file=op_file)
        print('
{}
'.format(site_title), file=op_file)
        print('        ', file=op_file)
        print('Home | Search'
              .format(proj_root, homepage, searchpage), file=op_file)
        print('        ', file=op_file)
        print('        <link rel="stylesheet" type="text/css" media="screen"' +
              ' href="{}" />'.format(css), file=op_file)
        print('        <title>{}</title>'.format(site_title), file=op_file)
        print('    ', file=op_file)
        print('    ', file=op_file)

        for cat in sorted(categories, reverse=True):
            category = cat.lower()
            print('
{}
'.format(category), file=op_file)
            print('
', file=op_file)

            for timestamp, (title, link) in sorted(chronology.items(),
                                                   reverse=True):
                if category == link.split('/')[0]:
                    print('
{}
'
                          .format(link, title), file=op_file)
            print('
', file=op_file)

        # TODO: Add some kind of javascript tag-cloud, word-cloud and category
        # list here.

        print('    ', file=op_file)
        print('', file=op_file)

    print(homepage + ' generated.')

def generate_search_page(site_title, homepage, searchpage, index, css,
                         proj_root):
    with open(searchpage, 'w') as op_file:
        print('', file=op_file)
        print('    ', file=op_file)
        print('
{}
'.format(site_title), file=op_file)
        print('        ', file=op_file)
        print('        <link rel="stylesheet" type="text/css" media="screen"' +
              ' href="{}" />'.format(css), file=op_file)
        print('        <title>Search</title>', file=op_file)
        print('Home | Search'
              .format(proj_root, homepage, searchpage), file=op_file)
        print('        ', file=op_file)
        print('    ', file=op_file)
        print('    ', file=op_file)

        print('
', file=op_file)
        for word, links in sorted(index.items()):
            print('
{}'.format(word), file=op_file)
            print('
', file=op_file)
            for link in links:
                print('
{0}
'.format(link),
                      file=op_file)
            print('
', file=op_file)
        print('
', file=op_file)

        print('    ', file=op_file)
        print('', file=op_file)

        # TODO: Add some javascript wizardry so this displays a search box and
        # only presents the list of pages when that word is entered into the box
        # (everything else should be hidden).

    print(searchpage + ' generated.')

def generate_static_page(site_title, homepage, searchpage, meta, css, ts_frmt,
                         proj_root, media_dir="../media"):
    for cat in meta.categories:
        category = cat.lower()
        create_dir_if_absent(category)

        with open(os.path.join(category, meta.slug) + '.html', 'w') as op_file:
            print('', file=op_file)
            print('    <title>{}</title>'.format(meta.title), file=op_file)
            print('    ', file=op_file)
            print('    ', file=op_file)
            print('
{}
'.format(site_title), file=op_file)
            print('        ', file=op_file)
            print('        <link rel="stylesheet" type="text/css"' +
                  ' media="screen" href="{}" />'.format(css), file=op_file)
            print('        <title>{}</title>'.format(meta.title), file=op_file)
            print('Home | Search'
                  .format(proj_root, homepage, searchpage), file=op_file)
            print('        ', file=op_file)
            print('    ', file=op_file)
            print('
', file=op_file)
            if meta.header_image != "":
                print('
', file=op_file)
                img_path = os.path.join(media_dir, meta.header_image)
                print(''.format(img_path), file=op_file)
                print('
', file=op_file)
            print('
{}
'.format(meta.title), file=op_file)
            print(    meta.content, file=op_file)
            print('
', file=op_file)
            print('

', file=op_file)
            date = \
                arrow.get(meta.date).format(ts_frmt) if meta.date != '' else ''

            print('
{}, {}
'.format(meta.authors, date),
                  file=op_file)
            print('', file=op_file)

def slugify(value):
    """
    Converts to lowercase, removes non-word characters (alphanumerics and
    underscores) and converts spaces to hyphens. Also strips leading and
    trailing whitespace.
    """
    normalized_value = unicodedata.normalize(
        'NFKD', value).encode('ascii', 'ignore').decode('ascii')
    stripped_value = re.sub('[^\w\s-]', '', normalized_value).strip().lower()
    return re.sub('[-\s]+', '-', stripped_value)

def create_dir_if_absent(folder):
    if not os.path.exists(folder):
        os.makedirs(folder)

def main():
    properties = get_properties("properties.json")
    generate_pages(properties=properties)

if __name__ == "__main__":
    sys.exit(main())
<commit_msg>Added category to static page title<commit_after>#!/usr/bin/python3

import os
import re
import sys
import json
import arrow
import markdown
import unicodedata
from collections import namedtuple
from bs4 import BeautifulSoup

def get_properties(properties_file="properties.json"):
    if os.path.exists(properties_file):
        with open(properties_file) as f:
            return json.loads(f.read())

def generate_pages(properties):
    proj_root = properties.get('root', '/')
    homepage = properties.get('homepage', 'index.html')
    searchpage = properties.get('searchpage', 'search.html')
    site_title = properties.get('site_title', 'PyQuo')
    ts_frmt = properties.get('timestamp_format', 'dddd DD MMMM, YYYY')
    directory = properties.get('source_directory', 'markdown')
    css = properties.get('css')

    all_pages = []
    categories = []
    count = 0

    for root, dirs, files in os.walk(directory):
        valid_files = [file for file in files if not
                       (file.endswith('~') or file.startswith('.'))]

        for valid_file in valid_files:
            with open(os.path.join(root, valid_file), 'r') as input_file:
                md = markdown.Markdown(extensions=['markdown.extensions.meta'])
                markdown_text = input_file.read()
                html = md.convert(markdown_text)
                text = ''.join(BeautifulSoup(html).findAll(text=True))
            if md.Meta == {}:
                continue

            meta = namedtuple('meta', [])
            meta.title = md.Meta['title'][0]
            meta.slug = slugify(meta.title)
            meta.categories = [
                category for category in set([cat.replace(" ", "").lower()
                for cat in md.Meta['category'][0].split(',')])]
            meta.authors = md.Meta['authors'][0]
            meta.date = md.Meta['date'][0]
            meta.tags = md.Meta['tags'][0].split(',')
            meta.header_image = md.Meta['headerimage'][0]
            meta.publish = True if md.Meta['publish'][0].lower() in [
                'true', 'yes'] else False
            meta.content = html
            meta.index = re.findall("\w+", text.lower())
            categories.extend(meta.categories)

            if meta.publish is True:
                generate_static_page(site_title, homepage, searchpage, meta,
                                     css, ts_frmt, proj_root)
                all_pages.append(meta)
                count += 1
        all_cats = [category for category in set(categories)]
        print('{} pages generated with the following categories: {}.'
              .format(count, ', '.join(all_cats)))
        all_cats, tags, chronology, index, word_cloud =\
            extract_site_wide_metadata(all_pages)
        generate_front_page(site_title, homepage, searchpage, all_cats, tags,
                            chronology, word_cloud, proj_root, css, ts_frmt)
        generate_search_page(site_title, homepage, searchpage, index, css,
                             proj_root)

def extract_site_wide_metadata(all_pages):
    """Get site-wide metadata.

    - Arrange titles by inverse date order, with link to location.
    - Generate tag cloud (dict of tag and number of times used).
    - Generate calendar so can click on a date and get link.
    """

    categories = []
    tags = {}
    chronology = {}
    index = {}
    for page in all_pages:
        for cat in page.categories:
            category = cat.lower()
            if category not in categories:
                categories.append(category)

            for tag in page.tags:
                if tag not in tags:
                    tags[tag] = 1
                else:
                    tags[tag] += 1

            if page.date not in chronology:
                link = "{}/{}.html".format(category, page.slug)

                if page.date != '':
                    timestamp = arrow.get(page.date)
                else:
                    timestamp = arrow.now()
                add_to_chronology_dict(
                    chronology, timestamp, (page.title, link))

            for idx in sorted(page.index + [tag.lower().strip() for tag in
                              page.tags if tag != ""]):
                word = idx.strip()
                link = chronology[timestamp][1]
                if word not in index:
                    index[word] = [link]
                else:
                    if link not in index[word]:
                        index[word].append(link)
    words_used = [(idx, len(index[idx])) for idx in index]
    word_cloud = sorted(words_used, key=lambda x: x[1], reverse=True)

    return categories, tags, chronology, index, word_cloud

def add_to_chronology_dict(chronology, timestamp, value):
    if timestamp in chronology:
        for num in range(1, 3600):
            one_second_later = timestamp.replace(seconds =+ num)
            if one_second_later not in chronology:
                timestamp = one_second_later
                break
    chronology[timestamp] = value

def generate_front_page(site_title, homepage, searchpage, categories, tags,
                        chronology, word_cloud, proj_root, css, ts_frmt):
    with open(homepage, 'w') as op_file:
        print('', file=op_file)
        print('    ', file=op_file)
        print('
{}
'.format(site_title), file=op_file)
        print('        ', file=op_file)
        print('Home | Search'
              .format(proj_root, homepage, searchpage), file=op_file)
        print('        ', file=op_file)
        print('        <link rel="stylesheet" type="text/css" media="screen"' +
              ' href="{}" />'.format(css), file=op_file)
        print('        <title>{}</title>'.format(site_title), file=op_file)
        print('    ', file=op_file)
        print('    ', file=op_file)

        for cat in sorted(categories, reverse=True):
            category = cat.lower()
            print('
{}
'.format(category), file=op_file)
            print('
', file=op_file)

            for timestamp, (title, link) in sorted(chronology.items(),
                                                   reverse=True):
                if category == link.split('/')[0]:
                    print('
{}
'
                          .format(link, title), file=op_file)
            print('
', file=op_file)

        # TODO: Add some kind of javascript tag-cloud, word-cloud and category
        # list here.

        print('    ', file=op_file)
        print('', file=op_file)

    print(homepage + ' generated.')

def generate_search_page(site_title, homepage, searchpage, index, css,
                         proj_root):
    with open(searchpage, 'w') as op_file:
        print('', file=op_file)
        print('    ', file=op_file)
        print('
{}
'.format(site_title), file=op_file)
        print('        ', file=op_file)
        print('        <link rel="stylesheet" type="text/css" media="screen"' +
              ' href="{}" />'.format(css), file=op_file)
        print('        <title>Search</title>', file=op_file)
        print('Home | Search'
              .format(proj_root, homepage, searchpage), file=op_file)
        print('        ', file=op_file)
        print('    ', file=op_file)
        print('    ', file=op_file)

        print('
', file=op_file)
        for word, links in sorted(index.items()):
            print('
{}'.format(word), file=op_file)
            print('
', file=op_file)
            for link in links:
                print('
{0}
'.format(link),
                      file=op_file)
            print('
', file=op_file)
        print('
', file=op_file)

        print('    ', file=op_file)
        print('', file=op_file)

        # TODO: Add some javascript wizardry so this displays a search box and
        # only presents the list of pages when that word is entered into the box
        # (everything else should be hidden).

    print(searchpage + ' generated.')

def generate_static_page(site_title, homepage, searchpage, meta, css, ts_frmt,
                         proj_root, media_dir="../media"):
    for cat in meta.categories:
        category = cat.lower()
        create_dir_if_absent(category)

        with open(os.path.join(category, meta.slug) + '.html', 'w') as op_file:
            print('', file=op_file)
            print('    <title>{} ({})</title>'.format(meta.title, category),
                  file=op_file)
            print('    ', file=op_file)
            print('    ', file=op_file)
            print('
{}
'.format(site_title), file=op_file)
            print('        ', file=op_file)
            print('        <link rel="stylesheet" type="text/css"' +
                  ' media="screen" href="{}" />'.format(css), file=op_file)
            print('        <title>{}</title>'.format(meta.title), file=op_file)
            print('Home | Search'
                  .format(proj_root, homepage, searchpage), file=op_file)
            print('        ', file=op_file)
            print('    ', file=op_file)
            print('
', file=op_file)
            if meta.header_image != "":
                print('
', file=op_file)
                img_path = os.path.join(media_dir, meta.header_image)
                print(''.format(img_path), file=op_file)
                print('
', file=op_file)
            print('
{}
'.format(meta.title), file=op_file)
            print(    meta.content, file=op_file)
            print('
', file=op_file)
            print('

', file=op_file)
            date = \
                arrow.get(meta.date).format(ts_frmt) if meta.date != '' else ''

            print('
{}, {}
'.format(meta.authors, date),
                  file=op_file)
            print('', file=op_file)

def slugify(value):
    """
    Converts to lowercase, removes non-word characters (alphanumerics and
    underscores) and converts spaces to hyphens. Also strips leading and
    trailing whitespace.
    """
    normalized_value = unicodedata.normalize(
        'NFKD', value).encode('ascii', 'ignore').decode('ascii')
    stripped_value = re.sub('[^\w\s-]', '', normalized_value).strip().lower()
    return re.sub('[-\s]+', '-', stripped_value)

def create_dir_if_absent(folder):
    if not os.path.exists(folder):
        os.makedirs(folder)

def main():
    properties = get_properties("properties.json")
    generate_pages(properties=properties)

if __name__ == "__main__":
    sys.exit(main())
<|endoftext|>


<commit_before># Copyright (c) 2008-2016 Szczepan Faber, Serhiy Oplakanets, Herr Kaste
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in
# all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
# THE SOFTWARE.

from collections import deque
import inspect
import functools

from . import invocation
from . import signature
from . import utils
from .mock_registry import mock_registry

__all__ = ['mock']

class _Dummy(object):
    # We spell out `__call__` here for convenience. All other magic methods
    # must be configured before use, but we want `mock`s to be callable by
    # default.
    def __call__(self, *args, **kwargs):
        return self.__getattr__('__call__')(*args, **kwargs)

def remembered_invocation_builder(mock, method_name, *args, **kwargs):
    invoc = invocation.RememberedInvocation(mock, method_name)
    return invoc(*args, **kwargs)

class Mock(object):
    def __init__(self, mocked_obj, strict=True, spec=None):
        self.mocked_obj = mocked_obj
        self.strict = strict
        self.spec = spec

        self.invocations = deque()
        self.stubbed_invocations = deque()

        self.original_methods = {}
        self._signatures_store = {}

    def remember(self, invocation):
        self.invocations.appendleft(invocation)

    def finish_stubbing(self, stubbed_invocation):
        self.stubbed_invocations.appendleft(stubbed_invocation)

    def clear_invocations(self):
        self.invocations = deque()

    # STUBBING

    def get_original_method(self, method_name):
        if not self.spec:
            return None
        return self.spec.__dict__.get(method_name)

    def set_method(self, method_name, new_method):
        setattr(self.mocked_obj, method_name, new_method)

    def replace_method(self, method_name, original_method):

        def new_mocked_method(*args, **kwargs):
            # we throw away the first argument, if it's either self or cls
            if (inspect.ismethod(new_mocked_method) or
                    inspect.isclass(self.mocked_obj) and
                    not isinstance(new_mocked_method, staticmethod)):
                args = args[1:]

            return remembered_invocation_builder(
                self, method_name, *args, **kwargs)

        new_mocked_method.__name__ = method_name
        if original_method:
            new_mocked_method.__doc__ = original_method.__doc__
            new_mocked_method.__wrapped__ = original_method
            try:
                new_mocked_method.__module__ = original_method.__module__
            except AttributeError:
                pass

        if (inspect.ismethod(original_method)):
            new_mocked_method = utils.newmethod(
                new_mocked_method, self.mocked_obj)
        if isinstance(original_method, staticmethod):
            new_mocked_method = staticmethod(new_mocked_method)
        elif isinstance(original_method, classmethod):
            new_mocked_method = classmethod(new_mocked_method)
        elif (inspect.isclass(self.mocked_obj) and  # TBC: Inner classes
                inspect.isclass(original_method)):
            new_mocked_method = staticmethod(new_mocked_method)

        self.set_method(method_name, new_mocked_method)

    def stub(self, method_name):
        try:
            self.original_methods[method_name]
        except KeyError:
            original_method = self.get_original_method(method_name)
            self.original_methods[method_name] = original_method

            self.replace_method(method_name, original_method)

    def unstub(self):
        while self.original_methods:
            method_name, original_method = self.original_methods.popitem()
            # If original_method is None, we *added* it to mocked_obj, so we
            # must delete it here.
            # If we mocked an instance, our mocked function will actually hide
            # the one on its class, so we delete as well.
            if (not original_method or not inspect.isclass(self.mocked_obj) and
                    inspect.ismethod(original_method)):
                delattr(self.mocked_obj, method_name)
            else:
                self.set_method(method_name, original_method)

    # SPECCING

    def has_method(self, method_name):
        if self.spec:
            return hasattr(self.spec, method_name)
        else:
            return True

    def get_signature(self, method_name):
        if self.spec is None:
            return None

        try:
            return self._signatures_store[method_name]
        except KeyError:
            sig = signature.get_signature(self.spec, method_name)
            self._signatures_store[method_name] = sig
            return sig

class _OMITTED(object):
    def __repr__(self):
        return 'OMITTED'

OMITTED = _OMITTED()

def mock(config_or_spec=None, spec=None, strict=OMITTED):
    """Create 'empty' objects ('Mocks').

    Will create an empty unconfigured object, that you can pass
    around. All interactions (method calls) will be recorded and can be
    verified using :func:`verify` et.al.

    A plain `mock()` will be not `strict`, and thus all methods regardless
    of the arguments will return ``None``.

    .. note:: Technically all attributes will return an internal interface.
        Because of that a simple ``if mock().foo:`` will surprisingly pass.

    If you set strict to ``True``: ``mock(strict=True)`` all unexpected
    interactions will raise an error instead.

    You configure a mock using :func:`when`, :func:`when2` or :func:`expect`.
    You can also very conveniently just pass in a dict here::

        response = mock({'text': 'ok', 'raise_for_status': lambda: None})

    You can also create an empty Mock which is specced against a given
    `spec`: ``mock(requests.Response)``. These mock are by default strict,
    thus they raise if you want to stub a method, the spec does not implement.
    Mockito will also match the function signature.

    You can pre-configure a specced mock as well::

        response = mock({'json': lambda: {'status': 'Ok'}},
                        spec=requests.Response)

    Mocks are by default callable. Configure the callable behavior using
    `when`::

        dummy = mock()
        when(dummy).__call_(1).thenReturn(2)

    All other magic methods must be configured this way or they will raise an
    AttributeError.

    See :func:`verify` to verify your interactions after usage.

    """

    if type(config_or_spec) is dict:
        config = config_or_spec
    else:
        config = {}
        spec = config_or_spec

    if strict is OMITTED:
        strict = False if spec is None else True

    class Dummy(_Dummy):
        if spec:
            __class__ = spec  # make isinstance work

        def __getattr__(self, method_name):
            if strict:
                raise AttributeError(
                    "'Dummy' has no attribute %r configured" % method_name)
            return functools.partial(
                remembered_invocation_builder, theMock, method_name)

        def __repr__(self):
            name = 'Dummy'
            if spec:
                name += spec.__name__
            return "<%s id=%s>" % (name, id(self))

    # That's a tricky one: The object we will return is an *instance* of our
    # Dummy class, but the mock we register will point and patch the class.
    # T.i. so that magic methods (`__call__` etc.) can be configured.
    obj = Dummy()
    theMock = Mock(Dummy, strict=strict, spec=spec)

    for n, v in config.items():
        if inspect.isfunction(v):
            invocation.StubbedInvocation(theMock, n)(Ellipsis).thenAnswer(v)
        else:
            setattr(obj, n, v)

    mock_registry.register(obj, theMock)
    return obj
<commit_msg>Explicit check for None for `self.spec`<commit_after># Copyright (c) 2008-2016 Szczepan Faber, Serhiy Oplakanets, Herr Kaste
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in
# all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
# THE SOFTWARE.

from collections import deque
import inspect
import functools

from . import invocation
from . import signature
from . import utils
from .mock_registry import mock_registry

__all__ = ['mock']

class _Dummy(object):
    # We spell out `__call__` here for convenience. All other magic methods
    # must be configured before use, but we want `mock`s to be callable by
    # default.
    def __call__(self, *args, **kwargs):
        return self.__getattr__('__call__')(*args, **kwargs)

def remembered_invocation_builder(mock, method_name, *args, **kwargs):
    invoc = invocation.RememberedInvocation(mock, method_name)
    return invoc(*args, **kwargs)

class Mock(object):
    def __init__(self, mocked_obj, strict=True, spec=None):
        self.mocked_obj = mocked_obj
        self.strict = strict
        self.spec = spec

        self.invocations = deque()
        self.stubbed_invocations = deque()

        self.original_methods = {}
        self._signatures_store = {}

    def remember(self, invocation):
        self.invocations.appendleft(invocation)

    def finish_stubbing(self, stubbed_invocation):
        self.stubbed_invocations.appendleft(stubbed_invocation)

    def clear_invocations(self):
        self.invocations = deque()

    # STUBBING

    def get_original_method(self, method_name):
        if self.spec is None:
            return None

        return self.spec.__dict__.get(method_name)

    def set_method(self, method_name, new_method):
        setattr(self.mocked_obj, method_name, new_method)

    def replace_method(self, method_name, original_method):

        def new_mocked_method(*args, **kwargs):
            # we throw away the first argument, if it's either self or cls
            if (inspect.ismethod(new_mocked_method) or
                    inspect.isclass(self.mocked_obj) and
                    not isinstance(new_mocked_method, staticmethod)):
                args = args[1:]

            return remembered_invocation_builder(
                self, method_name, *args, **kwargs)

        new_mocked_method.__name__ = method_name
        if original_method:
            new_mocked_method.__doc__ = original_method.__doc__
            new_mocked_method.__wrapped__ = original_method
            try:
                new_mocked_method.__module__ = original_method.__module__
            except AttributeError:
                pass

        if (inspect.ismethod(original_method)):
            new_mocked_method = utils.newmethod(
                new_mocked_method, self.mocked_obj)
        if isinstance(original_method, staticmethod):
            new_mocked_method = staticmethod(new_mocked_method)
        elif isinstance(original_method, classmethod):
            new_mocked_method = classmethod(new_mocked_method)
        elif (inspect.isclass(self.mocked_obj) and  # TBC: Inner classes
                inspect.isclass(original_method)):
            new_mocked_method = staticmethod(new_mocked_method)

        self.set_method(method_name, new_mocked_method)

    def stub(self, method_name):
        try:
            self.original_methods[method_name]
        except KeyError:
            original_method = self.get_original_method(method_name)
            self.original_methods[method_name] = original_method

            self.replace_method(method_name, original_method)

    def unstub(self):
        while self.original_methods:
            method_name, original_method = self.original_methods.popitem()
            # If original_method is None, we *added* it to mocked_obj, so we
            # must delete it here.
            # If we mocked an instance, our mocked function will actually hide
            # the one on its class, so we delete as well.
            if (not original_method or not inspect.isclass(self.mocked_obj) and
                    inspect.ismethod(original_method)):
                delattr(self.mocked_obj, method_name)
            else:
                self.set_method(method_name, original_method)

    # SPECCING

    def has_method(self, method_name):
        if self.spec is None:
            return True

        return hasattr(self.spec, method_name)

    def get_signature(self, method_name):
        if self.spec is None:
            return None

        try:
            return self._signatures_store[method_name]
        except KeyError:
            sig = signature.get_signature(self.spec, method_name)
            self._signatures_store[method_name] = sig
            return sig

class _OMITTED(object):
    def __repr__(self):
        return 'OMITTED'

OMITTED = _OMITTED()

def mock(config_or_spec=None, spec=None, strict=OMITTED):
    """Create 'empty' objects ('Mocks').

    Will create an empty unconfigured object, that you can pass
    around. All interactions (method calls) will be recorded and can be
    verified using :func:`verify` et.al.

    A plain `mock()` will be not `strict`, and thus all methods regardless
    of the arguments will return ``None``.

    .. note:: Technically all attributes will return an internal interface.
        Because of that a simple ``if mock().foo:`` will surprisingly pass.

    If you set strict to ``True``: ``mock(strict=True)`` all unexpected
    interactions will raise an error instead.

    You configure a mock using :func:`when`, :func:`when2` or :func:`expect`.
    You can also very conveniently just pass in a dict here::

        response = mock({'text': 'ok', 'raise_for_status': lambda: None})

    You can also create an empty Mock which is specced against a given
    `spec`: ``mock(requests.Response)``. These mock are by default strict,
    thus they raise if you want to stub a method, the spec does not implement.
    Mockito will also match the function signature.

    You can pre-configure a specced mock as well::

        response = mock({'json': lambda: {'status': 'Ok'}},
                        spec=requests.Response)

    Mocks are by default callable. Configure the callable behavior using
    `when`::

        dummy = mock()
        when(dummy).__call_(1).thenReturn(2)

    All other magic methods must be configured this way or they will raise an
    AttributeError.

    See :func:`verify` to verify your interactions after usage.

    """

    if type(config_or_spec) is dict:
        config = config_or_spec
    else:
        config = {}
        spec = config_or_spec

    if strict is OMITTED:
        strict = False if spec is None else True

    class Dummy(_Dummy):
        if spec:
            __class__ = spec  # make isinstance work

        def __getattr__(self, method_name):
            if strict:
                raise AttributeError(
                    "'Dummy' has no attribute %r configured" % method_name)
            return functools.partial(
                remembered_invocation_builder, theMock, method_name)

        def __repr__(self):
            name = 'Dummy'
            if spec:
                name += spec.__name__
            return "<%s id=%s>" % (name, id(self))

    # That's a tricky one: The object we will return is an *instance* of our
    # Dummy class, but the mock we register will point and patch the class.
    # T.i. so that magic methods (`__call__` etc.) can be configured.
    obj = Dummy()
    theMock = Mock(Dummy, strict=strict, spec=spec)

    for n, v in config.items():
        if inspect.isfunction(v):
            invocation.StubbedInvocation(theMock, n)(Ellipsis).thenAnswer(v)
        else:
            setattr(obj, n, v)

    mock_registry.register(obj, theMock)
    return obj
<|endoftext|>