<commit_before># coding: utf-8
import glob
from time import time
from preprocess_text import corpus
from sklearn.feature_extraction.text import TfidfVectorizer, CountVectorizer
from sklearn.decomposition import NMF, LatentDirichletAllocation
from nltk import word_tokenize
from nltk.stem import WordNetLemmatizer
import re
import pickle

NYT_CORPUS_PATH = '/opt/nlp_shared/corpora/NytCorpora/NYTCorpus/'

# Compute topics for Bill Clinton's terms
YEARS = range(1993, 2002)

N_FEATURES = 1000
N_TOPICS = 100
N_TOP_WORDS = 100

corpora_all = []

class LemmaTokenizer(object):
        def __init__(self):
            self.wnl = WordNetLemmatizer()
        def __call__(self, doc):
            return [self.wnl.lemmatize(t) for t in word_tokenize(doc)]

def print_top_words(model, feature_names, N_TOP_WORDS):
    for topic_idx, topic in enumerate(model.components_):
        print("Topic #%d:" % topic_idx)
        words_for_topic = [feature_names[i] for i in topic.argsort()[:-N_TOP_WORDS - 1:-1]]
        words_for_topic_filtered = []
        for word in words_for_topic:
            if re.match('[^a-zA-Z]+', word) is None:
                words_for_topic_filtered.append(word)
        print(" ".join(words_for_topic_filtered))
        print()

# returns an nmf_model given a corpora
def build_nmf_model(corpora, num_features=N_FEATURES, num_topics=N_TOPICS):
    dataset = []
    for corpus_for_day in corpora:
        for article in corpus_for_day:
            dataset.append('\n'.join(article.sents))

    tfidf_vectorizer = TfidfVectorizer(max_df=0.95, min_df=2,
                                       max_features=num_features.
                                       tokenizer=LemmaTokenizer(),
                                       stop_words='english')
    tfidf = tfidf_vectorizer.fit_transform(dataset)
    nmf = NMF(n_components=num_topics, random_state=1, alpha=.1, l1_ratio=.5).fit(tfidf)
    tfidf_feature_names = tfidf_vectorizer.get_feature_names()
    return (nmf, tfidf_feature_names)

if __name__ == '__main__':
    for year in YEARS:
        corpora_files_for_year = glob.glob("{}{}*industrial*".format(NYT_CORPUS_PATH, year))
        corpora_for_year = []
        for corpus_path in corpora_files_for_year:
            corpus_date = corpus_path.split('/')[-1][:10]
            corpus_for_date = corpus.Corpus.load(NYT_CORPUS_PATH, corpus_date)
            corpora_for_year.append(corpus_for_date)
        print("Found articles for {} days in the year {}".format(len(corpora_for_year), year))

        corpora_all += corpora_for_year

    dataset = []
    for corpus_for_day in corpora_all:
        for article in corpus_for_day:
            dataset.append('\n'.join(article.sents))

    N_SAMPLES=len(dataset)

    average_num_sentences = 0.0
    for article in dataset:
        average_num_sentences += article.count('\n')
    average_num_sentences /= len(dataset)
    print("average number of sentences in the {} articles is {}".format(len(dataset), average_num_sentences))

    data_samples = []
    data_samples = dataset[:N_SAMPLES]

    # Use tf-idf features for NMF.
    print("Extracting tf-idf features for NMF...")
    tfidf_vectorizer = TfidfVectorizer(max_df=0.95, min_df=2,
                                       max_features=N_FEATURES,
                                       tokenizer=LemmaTokenizer(),
                                       stop_words='english')
    t0 = time()
    tfidf = tfidf_vectorizer.fit_transform(data_samples)
    print("done in %0.3fs." % (time() - t0))

    # Fit the NMF model
    print("Fitting the NMF model with tf-idf features, "
          "N_SAMPLES=%d and N_FEATURES=%d..."
          % (N_SAMPLES, N_FEATURES))
    t0 = time()
    nmf = NMF(n_components=N_TOPICS, random_state=1,
              alpha=.1, l1_ratio=.5).fit(tfidf)
    print("done in %0.3fs." % (time() - t0))

    print("\nTopics in NMF model:")
    tfidf_feature_names = tfidf_vectorizer.get_feature_names()
    #print_top_words(nmf, tfidf_feature_names, N_TOP_WORDS)
<commit_msg>fixed nmf and added file saving/loading<commit_after># coding: utf-8
import glob
from time import time
from preprocess_text import corpus
from sklearn.feature_extraction.text import TfidfVectorizer, CountVectorizer
from sklearn.decomposition import NMF, LatentDirichletAllocation
from nltk import word_tokenize
from nltk.stem import WordNetLemmatizer
import re
import pickle

NYT_CORPUS_PATH = '/opt/nlp_shared/corpora/NytCorpora/NYTCorpus/'

# Compute topics for Bill Clinton's terms
YEARS = range(1993, 2002)

N_FEATURES = 1000
N_TOPICS = 100
N_TOP_WORDS = 100

corpora_all = []

class LemmaTokenizer(object):
        def __init__(self):
            self.wnl = WordNetLemmatizer()
        def __call__(self, doc):
            output = []
            for t in word_tokenize(doc):
                if re.match('[^a-zA-Z]+', t):
                    output.append(self.wnl.lemmatize(t))

def get_top_words(model, feature_names, n=0):
    topics = []
    for topic_idx, topic in enumerate(model.components_):
        print("Topic #%d:" % topic_idx)
        words_for_topic_filtered = []
        words_for_topic = [feature_names[i] for i in topic.argsort()[:-n - 1:-1]]
        for word in words_for_topic:
            if re.match('[^a-zA-Z]+', word) is None:
                words_for_topic_filtered.append(word)
    topics.append(words_for_topic_filtered)
    return topics

# returns an nmf_model given a corpora
def build_nmf_model(corpora, num_features=N_FEATURES, num_topics=N_TOPICS):
    dataset = []
    for date, corpus_for_day in corpora.items():
        for article in corpus_for_day:
            dataset.append('\n'.join(article.sents))

    tfidf_vectorizer = TfidfVectorizer(max_df=0.95, min_df=2,
                                       max_features=num_features,
                                       tokenizer=LemmaTokenizer(),
                                       stop_words='english')
    tfidf = tfidf_vectorizer.fit_transform(dataset)
    nmf = NMF(n_components=num_topics, random_state=1, alpha=.1, l1_ratio=.5).fit(tfidf)
    tfidf_feature_names = tfidf_vectorizer.get_feature_names()
    return (nmf, tfidf_feature_names)

def write_model_to_disk(nmf_file_name, features_file_name, num_features=N_FEATURES, num_topics=N_TOPICS):
    nmf, feature_names = build_nmf_model(corpora, num_features, num_topics)
    pickle.dump(feature_names, features_file_name)
    pickle.dump(nmf, nmf_file_name)

def load_model_from_disk(nmf_file_name, features_file_name):
    return (pickle.load(nmf_file_name), pickle.load(features_file_name))

if __name__ == '__main__':
    for year in YEARS:
        corpora_files_for_year = glob.glob("{}{}*industrial*".format(NYT_CORPUS_PATH, year))
        corpora_for_year = []
        for corpus_path in corpora_files_for_year:
            corpus_date = corpus_path.split('/')[-1][:10]
            corpus_for_date = corpus.Corpus.load(NYT_CORPUS_PATH, corpus_date)
            corpora_for_year.append(corpus_for_date)
        print("Found articles for {} days in the year {}".format(len(corpora_for_year), year))

        corpora_all += corpora_for_year

    dataset = []
    for corpus_for_day in corpora_all:
        for article in corpus_for_day:
            dataset.append('\n'.join(article.sents))

    N_SAMPLES=len(dataset)

    average_num_sentences = 0.0
    for article in dataset:
        average_num_sentences += article.count('\n')
    average_num_sentences /= len(dataset)
    print("average number of sentences in the {} articles is {}".format(len(dataset), average_num_sentences))

    data_samples = []
    data_samples = dataset[:N_SAMPLES]

    # Use tf-idf features for NMF.
    print("Extracting tf-idf features for NMF...")
    tfidf_vectorizer = TfidfVectorizer(max_df=0.95, min_df=2,
                                       max_features=N_FEATURES,
                                       tokenizer=LemmaTokenizer(),
                                       stop_words='english')
    t0 = time()
    tfidf = tfidf_vectorizer.fit_transform(data_samples)
    print("done in %0.3fs." % (time() - t0))

    # Fit the NMF model
    print("Fitting the NMF model with tf-idf features, "
          "N_SAMPLES=%d and N_FEATURES=%d..."
          % (N_SAMPLES, N_FEATURES))
    t0 = time()
    nmf = NMF(n_components=N_TOPICS, random_state=1,
              alpha=.1, l1_ratio=.5).fit(tfidf)
    print("done in %0.3fs." % (time() - t0))

    print("\nTopics in NMF model:")
    tfidf_feature_names = tfidf_vectorizer.get_feature_names()
    get_top_words(nmf, tfidf_feature_names, N_TOP_WORDS)
<|endoftext|>


<commit_before>from setuptools import setup, find_packages

__author__ = "James King"
__email__ = "james@agentultra.com"
__version__ = "0.1.1"
__license__ = "MIT"

setup(
    name="Horton",
    version=__version__,
    packages=find_packages(),

    extras_require = {
        'pygame': ["pygame"],
    },

    author=__author__,
    author_email=__email__,
    license=__license__,
    description="A library for building grid-based simulations.",
    classifiers=[
        "Development Status :: 3 - Alpha",
        "License :: OSI Approved :: MIT License",
        "Topic :: Software Development :: Libraries",
    ]
)
<commit_msg>Updated setup.py<commit_after>from setuptools import setup, find_packages

__author__ = "James King"
__email__ = "james@agentultra.com"
__version__ = "0.1.1"
__license__ = "MIT"
__description__ = """
A library of grids and other fine amusements. Contains a Grid and
Grid-like data structures and optional modules for rendering them with
pygame, creating cellular-automata simulations, games, and such
things.
"""

setup(
    name="Horton",
    version=__version__,
    packages=find_packages(),

    extras_require = {
        'pygame': ["pygame"],
    },

    author=__author__,
    author_email=__email__,
    license=__license__,
    description=__description__,
    classifiers=[
        "Development Status :: 3 - Alpha",
        "Inteded Audience :: Developers",
        "License :: OSI Approved :: MIT License",
        "Topic :: Artistic Software",
        "Topic :: Games/Entertainment",
        "Topic :: Software Development :: Libraries",
    ]
)
<|endoftext|>


<commit_before># coding: utf-8

# TODO Implement real object parser and attribute parsers

class Object(object):
	""" This object type is some kind of magic: It acts as a dictionary and a
	list, therefore an implementation using a trie would be the best idea.
	Unfortunately, lists are fast enough for our purpose, so we perform linear
	searches. If you have time, implement is as tree. And don't implement it as
	dict. """

	def __init__(self, data=None):
		self.data = []
		if data:
			self.extend(data)

	def extend(self, ex):
		""" expand has also multiple behaviours depending on given data types. If
		ex is a list of tuples, then this is simply appended as key-value-pairs to
		the structure. Otherwise, if ex is a dict, then __setitem__ will be used on
		the values of struct """

		if isinstance(ex, list):
			# at first we will check the whole struture of ex, if we encounter any
			# error, we will raise an exception, so we are never in undefined state
			for off, kvpair in zip(range(len(ex)), ex):
				if not isinstance(kvpair, tuple):
					raise ValueError("offset {}: expected entry to be tuple, got {}".format(off, type(kvpair)))
				if len(kvpair) != 2:
					raise ValueError("offset {}: expected tuple to have two values, got {}".format(off, len(kvpair)))
				if not isinstance(kvpair[0], str):
					raise ValueError("offset {}: expected key to be str, got {}".format(off, type(kvpair[0])))
				if not isinstance(kvpair[1], str):
					raise ValueError("offset {}: expected value to be str, got {}".format(off, type(kvpair[0])))

			self.data.extend(ex)
			return
		elif isinstance(ex, dict):
			# the same for dicts, we will check the given dict and then we will
			# add the values to the structure
			for key, value in ex.items():
				if not isinstance(key, str):
					raise ValueError("expected key to be str, got {}: {}".format(type(key), key))
				if not isinstance(value, str):
					raise ValueError("key {}: expected value to be str, got {}".format(key, type(value)))

			for key, value in ex.items():
				self[key] = value
			return
		raise TypeError("Expected ex to be dict or list, got {}".format(type(ex)))

	def __getitem__(self, key):
		if isinstance(key, str):
			try:
				return self.get(key)[0][1]
			except IndexError:
				raise KeyError(repr(key))
		elif isinstance(key, int):
			return self.data[key]

		raise TypeError("Expected key to be str or int, got {}".format(type(key)))

	def __len__(self):
		return len(self.data)

	def __contains__(self, key):
		""" __contains__ checks whether a given key of type str is present in the
		structure or not. """
		if isinstance(key, str):
			return len(self.get(key)) > 0

		raise TypeError("Expected key to be str, got {}".format(type(key)))

	def __setitem__(self, key, value):
		""" __setitem__ has special behaviour on this object type: If key is an
		instance of str, then it will remove entry with "key" and append key and
		value to end. In that case, value can be iterable, so multiple new entries
		will be emmitted. Otherwise, if key is a simple number, then it works as
		in lists: It will simply replace the entry with value. In that case, value
		needs to be a tuple of (k, v), where k is the string key and v is the
		string value. """
		if isinstance(key, str):
			if not isinstance(value, list) and not isinstance(value, str):
				raise TypeError("Expected value to be str or list, got {}".format(type(key)))
			del self[key]
			if isinstance(value, list):
				for v in value:
					self.add(key, v)
			else:
				self.add(key, value)
			return
		elif isinstance(key, int):
			if not isinstance(value, tuple):
				raise TypeError("Expected value to be tuple, got {}".format(type(key)))
			self.data[key] = value
			return

		raise TypeError("Expected key to be str or int, got {}".format(type(key)))

	def __delitem__(self, key):
		""" __delitem__ has also special semantics. If key is an instance of str,
		then this will delete any entry with the given key, otherwise if key is an
		instance of int, then this will delete the n'th entry. """
		if isinstance(key, str):
			while key in self.keys():
				del self.data[self.index(key)]
			return

		del self.data[key]

	def __iter__(self):
		return iter(self.data)

	def __repr__(self):
		return repr(self.data)

	def get(self, key):
		return [kvpair for kvpair in self.data if kvpair[0] == key]

	def add(self, k, v):
		return self.data.append((k, v))

	def remove(self, k):
		return self.data.remove(k)

	def index(self, key):
		""" Return index of first entry for key. """
		for kvpair, offset in zip(self.data, range(len(self.data))):
			if kvpair[0] == key:
				return offset
		raise ValueError("{} not found".format(key))

	def keys(self):
		""" Return list of all keys. """
		return list(set([kvpair[0] for kvpair in self.data]))

	def pretty_print(self, kv_padding=8):
		left_padding = sorted([len(k) for k in self.keys()], reverse=True)[0] + kv_padding
		result = []

		for key, value in self:
			result.append(key)
			result.append(":")
			result.append(" " * (left_padding - len(key) - 1))
			value_lines = value.split("\n")
			result.append(value_lines[0])
			for value_line in value_lines[1:]:
				result.append("\n")
				result.append(" " * left_padding)
				result.append(value_line)
			result.append("\n")

		return "".join(result)

	def __str__(self):
		return self.pretty_print()

	def __hash__(self):
		return hash((self.type, self.primary_key))

	@property
	def type(self):
		return self.data[0][0]

	@property
	def primary_key(self):
		return self.data[0][1]

	@property
	def spec(self):
		return (self.type, self.primary_key)

	@classmethod
	def from_string(cls, string):
		return cls(parse_rpsl(string.split("\n")))

def parse_rpsl(lines):
	""" Simple RPSL "parser", which expects a list of lines as input """
	result = []

	for line in lines:
		# remove any comments (text after % and #)
		line = line.split("%")[0]
		line = line.split("#")[0]

		# continue if line is empty
		if not line.strip():
			continue

		# check for line continuations
		if line[0] == ' ':
			entry = result.pop()
			value = "\n".join([entry[1], line.strip()])
			result.append((entry[0], value))
			continue

		key, value = line.split(":", 1)

		result.append((key.strip(), value.strip()))

	return result

try:
	import netaddr

	def inetnum_range(inetnum):

		if isinstance(inetnum, Object):
			inetnum = inetnum.primary_key

		return netaddr.IPRange(*[ipr.strip() for ipr in inetnum.split("-", 1)])

	def inetnum_cidrs(inetnum):
		""" Return all CIDRs included in given inetnum object. """

		try:
			return netaddr.IPNetwork(inetnum.primary_key)
		except (netaddr.core.AddrFormatError, ValueError):
			if "-" in inetnum.primary_key:
				return inetnum_range(inetnum).cidrs()
			else:
				raise
except LoadError:
	pass
<commit_msg>bugfix<commit_after># coding: utf-8

# TODO Implement real object parser and attribute parsers

class Object(object):
	""" This object type is some kind of magic: It acts as a dictionary and a
	list, therefore an implementation using a trie would be the best idea.
	Unfortunately, lists are fast enough for our purpose, so we perform linear
	searches. If you have time, implement is as tree. And don't implement it as
	dict. """

	def __init__(self, data=None):
		self.data = []
		if data:
			self.extend(data)

	def extend(self, ex):
		""" expand has also multiple behaviours depending on given data types. If
		ex is a list of tuples, then this is simply appended as key-value-pairs to
		the structure. Otherwise, if ex is a dict, then __setitem__ will be used on
		the values of struct """

		if isinstance(ex, list):
			# at first we will check the whole struture of ex, if we encounter any
			# error, we will raise an exception, so we are never in undefined state
			for off, kvpair in zip(range(len(ex)), ex):
				if not isinstance(kvpair, tuple):
					raise ValueError("offset {}: expected entry to be tuple, got {}".format(off, type(kvpair)))
				if len(kvpair) != 2:
					raise ValueError("offset {}: expected tuple to have two values, got {}".format(off, len(kvpair)))
				if not isinstance(kvpair[0], str):
					raise ValueError("offset {}: expected key to be str, got {}".format(off, type(kvpair[0])))
				if not isinstance(kvpair[1], str):
					raise ValueError("offset {}: expected value to be str, got {}".format(off, type(kvpair[0])))

			self.data.extend(ex)
			return
		elif isinstance(ex, dict):
			# the same for dicts, we will check the given dict and then we will
			# add the values to the structure
			for key, value in ex.items():
				if not isinstance(key, str):
					raise ValueError("expected key to be str, got {}: {}".format(type(key), key))
				if not isinstance(value, str):
					raise ValueError("key {}: expected value to be str, got {}".format(key, type(value)))

			for key, value in ex.items():
				self[key] = value
			return
		raise TypeError("Expected ex to be dict or list, got {}".format(type(ex)))

	def __getitem__(self, key):
		if isinstance(key, str):
			try:
				return self.get(key)[0][1]
			except IndexError:
				raise KeyError(repr(key))
		elif isinstance(key, int):
			return self.data[key]

		raise TypeError("Expected key to be str or int, got {}".format(type(key)))

	def __len__(self):
		return len(self.data)

	def __contains__(self, key):
		""" __contains__ checks whether a given key of type str is present in the
		structure or not. """
		if isinstance(key, str):
			return len(self.get(key)) > 0

		raise TypeError("Expected key to be str, got {}".format(type(key)))

	def __setitem__(self, key, value):
		""" __setitem__ has special behaviour on this object type: If key is an
		instance of str, then it will remove entry with "key" and append key and
		value to end. In that case, value can be iterable, so multiple new entries
		will be emmitted. Otherwise, if key is a simple number, then it works as
		in lists: It will simply replace the entry with value. In that case, value
		needs to be a tuple of (k, v), where k is the string key and v is the
		string value. """
		if isinstance(key, str):
			if not isinstance(value, list) and not isinstance(value, str):
				raise TypeError("Expected value to be str or list, got {}".format(type(key)))
			del self[key]
			if isinstance(value, list):
				for v in value:
					self.add(key, v)
			else:
				self.add(key, value)
			return
		elif isinstance(key, int):
			if not isinstance(value, tuple):
				raise TypeError("Expected value to be tuple, got {}".format(type(key)))
			self.data[key] = value
			return

		raise TypeError("Expected key to be str or int, got {}".format(type(key)))

	def __delitem__(self, key):
		""" __delitem__ has also special semantics. If key is an instance of str,
		then this will delete any entry with the given key, otherwise if key is an
		instance of int, then this will delete the n'th entry. """
		if isinstance(key, str):
			while key in self.keys():
				del self.data[self.index(key)]
			return

		del self.data[key]

	def __iter__(self):
		return iter(self.data)

	def __repr__(self):
		return repr(self.data)

	def get(self, key):
		return [kvpair for kvpair in self.data if kvpair[0] == key]

	def add(self, k, v):
		return self.data.append((k, v))

	def remove(self, k):
		return self.data.remove(k)

	def index(self, key):
		""" Return index of first entry for key. """
		for kvpair, offset in zip(self.data, range(len(self.data))):
			if kvpair[0] == key:
				return offset
		raise ValueError("{} not found".format(key))

	def keys(self):
		""" Return list of all keys. """
		return list(set([kvpair[0] for kvpair in self.data]))

	def pretty_print(self, kv_padding=8):
		left_padding = sorted([len(k) for k in self.keys()], reverse=True)[0] + kv_padding
		result = []

		for key, value in self:
			result.append(key)
			result.append(":")
			result.append(" " * (left_padding - len(key) - 1))
			value_lines = value.split("\n")
			result.append(value_lines[0])
			for value_line in value_lines[1:]:
				result.append("\n")
				result.append(" " * left_padding)
				result.append(value_line)
			result.append("\n")

		return "".join(result)

	def __str__(self):
		return self.pretty_print()

	def __hash__(self):
		return hash((self.type, self.primary_key))

	@property
	def type(self):
		return self.data[0][0]

	@property
	def primary_key(self):
		return self.data[0][1]

	@property
	def spec(self):
		return (self.type, self.primary_key)

	@classmethod
	def from_string(cls, string):
		return cls(parse_rpsl(string.split("\n")))

def parse_rpsl(lines):
	""" Simple RPSL "parser", which expects a list of lines as input """
	result = []

	for line in lines:
		# remove any comments (text after % and #)
		line = line.split("%")[0]
		line = line.split("#")[0]

		# continue if line is empty
		if not line.strip():
			continue

		# check for line continuations
		if line[0] == ' ':
			entry = result.pop()
			value = "\n".join([entry[1], line.strip()])
			result.append((entry[0], value))
			continue

		key, value = line.split(":", 1)

		result.append((key.strip(), value.strip()))

	return result

try:
	import netaddr

	def inetnum_range(inetnum):

		if isinstance(inetnum, Object):
			inetnum = inetnum.primary_key

		return netaddr.IPRange(*[ipr.strip() for ipr in inetnum.split("-", 1)])

	def inetnum_cidrs(inetnum):
		""" Return all CIDRs included in given inetnum object. """

		try:
			return [netaddr.IPNetwork(inetnum.primary_key)]
		except (netaddr.core.AddrFormatError, ValueError):
			if "-" in inetnum.primary_key:
				return inetnum_range(inetnum).cidrs()
			else:
				raise
except LoadError:
	pass
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
#  Copyright 2013 Takeshi KOMIYA
#
#  Licensed under the Apache License, Version 2.0 (the "License");
#  you may not use this file except in compliance with the License.
#  You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
#  Unless required by applicable law or agreed to in writing, software
#  distributed under the License is distributed on an "AS IS" BASIS,
#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#  See the License for the specific language governing permissions and
#  limitations under the License.

import os
import re
import sys
import json
import yaml
import socket
import signal
import urllib
import tempfile
from glob import glob
from time import sleep
from shutil import copyfile, copytree, rmtree
from datetime import datetime

__all__ = ['Elasticsearch', 'skipIfNotFound']

SEARCH_PATHS = ['/usr/share/elasticsearch']
DEFAULT_SETTINGS = dict(auto_start=2,
                        base_dir=None,
                        elasticsearch_home=None,
                        pid=None,
                        port=None,
                        copy_data_from=None)

class Elasticsearch(object):
    def __init__(self, **kwargs):
        self.settings = dict(DEFAULT_SETTINGS)
        self.settings.update(kwargs)
        self.pid = None
        self._owner_pid = os.getpid()
        self._use_tmpdir = False

        if self.base_dir:
            if self.base_dir[0] != '/':
                self.settings['base_dir'] = os.path.join(os.getcwd(), self.base_dir)
        else:
            self.settings['base_dir'] = tempfile.mkdtemp()
            self._use_tmpdir = True

        if self.elasticsearch_home is None:
            self.settings['elasticsearch_home'] = find_elasticsearch_home()

        user_config = self.settings.get('elasticsearch_yaml')
        with open(os.path.join(self.elasticsearch_home, 'config', 'elasticsearch.yml')) as fd:
            self.settings['elasticsearch_yaml'] = yaml.load(fd.read()) or {}
            self.settings['elasticsearch_yaml']['path.data'] = os.path.join(self.base_dir, 'data')
            self.settings['elasticsearch_yaml']['path.logs'] = os.path.join(self.base_dir, 'logs')
            self.settings['elasticsearch_yaml']['cluster.name'] = generate_cluster_name()
            self.settings['elasticsearch_yaml']['discovery.zen.ping.multicast.enabled'] = False

            if self.port:
                self.settings['elasticsearch_yaml']['http.port'] = self.port

            if user_config:
                for key, value in user_config.items():
                    self.settings['elasticsearch_yaml'][key] = value

        if self.auto_start:
            if self.auto_start >= 2:
                self.setup()

            self.start()

    def __del__(self):
        self.stop()

    def __enter__(self):
        return self

    def __exit__(self, *args):
        self.stop()

    def __getattr__(self, name):
        if name in self.settings:
            return self.settings[name]
        else:
            raise AttributeError("'Elasticsearch' object has no attribute '%s'" % name)

    def dsn(self, **kwargs):
        return {'hosts': ['127.0.0.1:%d' % self.elasticsearch_yaml['http.port']]}

    def setup(self):
        # copy data files
        if self.copy_data_from:
            try:
                copytree(self.copy_data_from, os.path.join(self.base_dir, 'data'))
                os.chmod(os.path.join(self.base_dir, 'data'), 0o700)
                if self.elasticsearch_yaml['cluster.name']:
                    indexdir = os.listdir(self.copy_data_from)[0]
                    os.rename(os.path.join(self.base_dir, 'data', indexdir),
                              os.path.join(self.base_dir, 'data', self.elasticsearch_yaml['cluster.name']))
            except Exception as exc:
                raise RuntimeError("could not copytree %s to %s: %r" %
                                   (self.copy_data_from, os.path.join(self.base_dir, 'data'), exc))

        # (re)create directory structure
        for subdir in ['data', 'logs']:
            path = os.path.join(self.base_dir, subdir)
            if not os.path.exists(path):
                os.makedirs(path)
                os.chmod(path, 0o700)

        # conf directory
        for filename in os.listdir(self.elasticsearch_home):
            srcpath = os.path.join(self.elasticsearch_home, filename)
            destpath = os.path.join(self.base_dir, filename)
            if not os.path.exists(destpath):
                if filename in ['lib', 'plugins']:
                    os.symlink(srcpath, destpath)
                elif os.path.isdir(srcpath):
                    copytree(srcpath, destpath)
                else:
                    copyfile(srcpath, destpath)

    def prestart(self):
        # assign port to elasticsearch
        self.settings['elasticsearch_yaml']['http.port'] = self.port or get_unused_port()

        # generate cassandra.yaml
        with open(os.path.join(self.base_dir, 'config', 'elasticsearch.yml'), 'wt') as fd:
            fd.write(yaml.dump(self.elasticsearch_yaml, default_flow_style=False))

    def start(self):
        if self.pid:
            return  # already started

        self.prestart()

        logger = open(os.path.join(self.base_dir, 'logs', 'elasticsearch-launch.log'), 'wt')
        pid = os.fork()
        if pid == 0:
            os.dup2(logger.fileno(), sys.__stdout__.fileno())
            os.dup2(logger.fileno(), sys.__stderr__.fileno())

            try:
                elasticsearch_bin = os.path.join(self.base_dir, 'bin', 'elasticsearch')
                os.execl(elasticsearch_bin, elasticsearch_bin)
            except Exception as exc:
                raise RuntimeError('failed to launch elasticsearch: %r' % exc)
        else:
            logger.close()

            exec_at = datetime.now()
            while True:
                if os.waitpid(pid, os.WNOHANG)[0] != 0:
                    raise RuntimeError("*** failed to launch elasticsearch ***\n" + self.read_log())

                if self.is_connection_available():
                    break

                if (datetime.now() - exec_at).seconds > 20.0:
                    self.stop()
                    raise RuntimeError("*** failed to launch elasticsearch (timeout) ***\n" + self.read_log())

                sleep(0.1)

            self.pid = pid

    def stop(self, _signal=signal.SIGTERM):
        self.terminate(_signal)
        self.cleanup()

    def terminate(self, _signal=signal.SIGTERM):
        if self.pid is None:
            return  # not started

        if self._owner_pid != os.getpid():
            return  # could not stop in child process

        try:
            os.kill(self.pid, _signal)
            killed_at = datetime.now()
            while (os.waitpid(self.pid, os.WNOHANG)):
                if (datetime.now() - killed_at).seconds > 10.0:
                    os.kill(self.pid, signal.SIGKILL)
                    raise RuntimeError("*** failed to shutdown elasticsearch (timeout) ***\n" + self.read_log())

                sleep(0.1)
        except OSError:
            pass

        self.pid = None

    def cleanup(self):
        if self.pid is not None:
            return

        if self._use_tmpdir and os.path.exists(self.base_dir):
            rmtree(self.base_dir, ignore_errors=True)
            self._use_tmpdir = False

    def read_log(self):
        try:
            with open(os.path.join(self.base_dir, 'logs', 'elasticsearch-launch.log')) as log:
                return log.read()
        except Exception as exc:
            raise RuntimeError("failed to open file:logs/elasticsearch-launch.log: %r" % exc)

    def is_connection_available(self):
        try:
            url = 'http://127.0.0.1:%d/_cluster/health' % self.elasticsearch_yaml['http.port']
            ret = json.loads(urllib.urlopen(url).read())
            if ret['status'] in ('green', 'yellow'):
                return True
            else:
                return False
        except Exception:
            return False

def skipIfNotInstalled(arg=None):
    if sys.version_info < (2, 7):
        from unittest2 import skipIf
    else:
        from unittest import skipIf

    def decorator(fn, path=arg):
        if path:
            cond = not os.path.exists(path)
        else:
            try:
                find_elasticsearch_home()  # raise exception if not found
                cond = False
            except:
                cond = True  # not found

        return skipIf(cond, "Elasticsearch not found")(fn)

    if callable(arg):  # execute as simple decorator
        return decorator(arg, None)
    else:  # execute with path argument
        return decorator

skipIfNotFound = skipIfNotInstalled

def strip_version(dir):
    m = re.search('(\d+)\.(\d+)\.(\d+)', dir)
    if m is None:
        return None
    else:
        return tuple([int(ver) for ver in m.groups()])

def find_elasticsearch_home():
    elasticsearch_home = os.environ.get('ES_HOME')
    if elasticsearch_home and os.path.exists(os.path.join(elasticsearch_home, 'bin', 'elasticsearch')):
        return elasticsearch_home

    for path in SEARCH_PATHS:
        if os.path.exists(os.path.join(path, 'bin', 'elasticsearch')):
            return path

    # search newest elasticsearch-x.x.x directory
    globbed = glob("/usr/local/*elasticsearch*") + glob("*elasticsearch*")
    elasticsearch_dirs = [os.path.abspath(dir) for dir in globbed if os.path.isdir(dir)]
    if elasticsearch_dirs:
        return sorted(elasticsearch_dirs, key=strip_version)[-1]

    raise RuntimeError("could not find ES_HOME")

def get_unused_port():
    sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
    sock.bind(('localhost', 0))
    _, port = sock.getsockname()
    sock.close()

    return port

def generate_cluster_name():
    import string
    import random

    return ''.join([random.choice(string.ascii_letters) for i in range(6)])
<commit_msg>Add /usr/local/Cellar to search paths (for homebrew)<commit_after># -*- coding: utf-8 -*-
#  Copyright 2013 Takeshi KOMIYA
#
#  Licensed under the Apache License, Version 2.0 (the "License");
#  you may not use this file except in compliance with the License.
#  You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
#  Unless required by applicable law or agreed to in writing, software
#  distributed under the License is distributed on an "AS IS" BASIS,
#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#  See the License for the specific language governing permissions and
#  limitations under the License.

import os
import re
import sys
import json
import yaml
import socket
import signal
import urllib
import tempfile
from glob import glob
from time import sleep
from shutil import copyfile, copytree, rmtree
from datetime import datetime

__all__ = ['Elasticsearch', 'skipIfNotFound']

SEARCH_PATHS = ['/usr/share/elasticsearch']
DEFAULT_SETTINGS = dict(auto_start=2,
                        base_dir=None,
                        elasticsearch_home=None,
                        pid=None,
                        port=None,
                        copy_data_from=None)

class Elasticsearch(object):
    def __init__(self, **kwargs):
        self.settings = dict(DEFAULT_SETTINGS)
        self.settings.update(kwargs)
        self.pid = None
        self._owner_pid = os.getpid()
        self._use_tmpdir = False

        if self.base_dir:
            if self.base_dir[0] != '/':
                self.settings['base_dir'] = os.path.join(os.getcwd(), self.base_dir)
        else:
            self.settings['base_dir'] = tempfile.mkdtemp()
            self._use_tmpdir = True

        if self.elasticsearch_home is None:
            self.settings['elasticsearch_home'] = find_elasticsearch_home()

        user_config = self.settings.get('elasticsearch_yaml')
        with open(os.path.join(self.elasticsearch_home, 'config', 'elasticsearch.yml')) as fd:
            self.settings['elasticsearch_yaml'] = yaml.load(fd.read()) or {}
            self.settings['elasticsearch_yaml']['path.data'] = os.path.join(self.base_dir, 'data')
            self.settings['elasticsearch_yaml']['path.logs'] = os.path.join(self.base_dir, 'logs')
            self.settings['elasticsearch_yaml']['cluster.name'] = generate_cluster_name()
            self.settings['elasticsearch_yaml']['discovery.zen.ping.multicast.enabled'] = False

            if self.port:
                self.settings['elasticsearch_yaml']['http.port'] = self.port

            if user_config:
                for key, value in user_config.items():
                    self.settings['elasticsearch_yaml'][key] = value

        if self.auto_start:
            if self.auto_start >= 2:
                self.setup()

            self.start()

    def __del__(self):
        self.stop()

    def __enter__(self):
        return self

    def __exit__(self, *args):
        self.stop()

    def __getattr__(self, name):
        if name in self.settings:
            return self.settings[name]
        else:
            raise AttributeError("'Elasticsearch' object has no attribute '%s'" % name)

    def dsn(self, **kwargs):
        return {'hosts': ['127.0.0.1:%d' % self.elasticsearch_yaml['http.port']]}

    def setup(self):
        # copy data files
        if self.copy_data_from:
            try:
                copytree(self.copy_data_from, os.path.join(self.base_dir, 'data'))
                os.chmod(os.path.join(self.base_dir, 'data'), 0o700)
                if self.elasticsearch_yaml['cluster.name']:
                    indexdir = os.listdir(self.copy_data_from)[0]
                    os.rename(os.path.join(self.base_dir, 'data', indexdir),
                              os.path.join(self.base_dir, 'data', self.elasticsearch_yaml['cluster.name']))
            except Exception as exc:
                raise RuntimeError("could not copytree %s to %s: %r" %
                                   (self.copy_data_from, os.path.join(self.base_dir, 'data'), exc))

        # (re)create directory structure
        for subdir in ['data', 'logs']:
            path = os.path.join(self.base_dir, subdir)
            if not os.path.exists(path):
                os.makedirs(path)
                os.chmod(path, 0o700)

        # conf directory
        for filename in os.listdir(self.elasticsearch_home):
            srcpath = os.path.join(self.elasticsearch_home, filename)
            destpath = os.path.join(self.base_dir, filename)
            if not os.path.exists(destpath):
                if filename in ['lib', 'plugins']:
                    os.symlink(srcpath, destpath)
                elif os.path.isdir(srcpath):
                    copytree(srcpath, destpath)
                else:
                    copyfile(srcpath, destpath)

        # rewrite elasticsearch.in.sh (for homebrew)
        with open(os.path.join(self.base_dir, 'bin', 'elasticsearch.in.sh'), 'r+t') as fd:
            body = re.sub('ES_HOME=.*', '', fd.read())
            fd.seek(0)
            fd.write(body)

    def prestart(self):
        # assign port to elasticsearch
        self.settings['elasticsearch_yaml']['http.port'] = self.port or get_unused_port()

        # generate cassandra.yaml
        with open(os.path.join(self.base_dir, 'config', 'elasticsearch.yml'), 'wt') as fd:
            fd.write(yaml.dump(self.elasticsearch_yaml, default_flow_style=False))

    def start(self):
        if self.pid:
            return  # already started

        self.prestart()

        logger = open(os.path.join(self.base_dir, 'logs', 'elasticsearch-launch.log'), 'wt')
        pid = os.fork()
        if pid == 0:
            os.dup2(logger.fileno(), sys.__stdout__.fileno())
            os.dup2(logger.fileno(), sys.__stderr__.fileno())

            try:
                elasticsearch_bin = os.path.join(self.base_dir, 'bin', 'elasticsearch')
                os.execl(elasticsearch_bin, elasticsearch_bin)
            except Exception as exc:
                raise RuntimeError('failed to launch elasticsearch: %r' % exc)
        else:
            logger.close()

            exec_at = datetime.now()
            while True:
                if os.waitpid(pid, os.WNOHANG)[0] != 0:
                    raise RuntimeError("*** failed to launch elasticsearch ***\n" + self.read_log())

                if self.is_connection_available():
                    break

                if (datetime.now() - exec_at).seconds > 20.0:
                    self.stop()
                    raise RuntimeError("*** failed to launch elasticsearch (timeout) ***\n" + self.read_log())

                sleep(0.1)

            self.pid = pid

    def stop(self, _signal=signal.SIGTERM):
        self.terminate(_signal)
        self.cleanup()

    def terminate(self, _signal=signal.SIGTERM):
        if self.pid is None:
            return  # not started

        if self._owner_pid != os.getpid():
            return  # could not stop in child process

        try:
            os.kill(self.pid, _signal)
            killed_at = datetime.now()
            while (os.waitpid(self.pid, os.WNOHANG)):
                if (datetime.now() - killed_at).seconds > 10.0:
                    os.kill(self.pid, signal.SIGKILL)
                    raise RuntimeError("*** failed to shutdown elasticsearch (timeout) ***\n" + self.read_log())

                sleep(0.1)
        except OSError:
            pass

        self.pid = None

    def cleanup(self):
        if self.pid is not None:
            return

        if self._use_tmpdir and os.path.exists(self.base_dir):
            rmtree(self.base_dir, ignore_errors=True)
            self._use_tmpdir = False

    def read_log(self):
        try:
            with open(os.path.join(self.base_dir, 'logs', 'elasticsearch-launch.log')) as log:
                return log.read()
        except Exception as exc:
            raise RuntimeError("failed to open file:logs/elasticsearch-launch.log: %r" % exc)

    def is_connection_available(self):
        try:
            url = 'http://127.0.0.1:%d/_cluster/health' % self.elasticsearch_yaml['http.port']
            ret = json.loads(urllib.urlopen(url).read())
            if ret['status'] in ('green', 'yellow'):
                return True
            else:
                return False
        except Exception:
            return False

def skipIfNotInstalled(arg=None):
    if sys.version_info < (2, 7):
        from unittest2 import skipIf
    else:
        from unittest import skipIf

    def decorator(fn, path=arg):
        if path:
            cond = not os.path.exists(path)
        else:
            try:
                find_elasticsearch_home()  # raise exception if not found
                cond = False
            except:
                cond = True  # not found

        return skipIf(cond, "Elasticsearch not found")(fn)

    if callable(arg):  # execute as simple decorator
        return decorator(arg, None)
    else:  # execute with path argument
        return decorator

skipIfNotFound = skipIfNotInstalled

def strip_version(dir):
    m = re.search('(\d+)\.(\d+)\.(\d+)', dir)
    if m is None:
        return None
    else:
        return tuple([int(ver) for ver in m.groups()])

def find_elasticsearch_home():
    elasticsearch_home = os.environ.get('ES_HOME')
    if elasticsearch_home and os.path.exists(os.path.join(elasticsearch_home, 'bin', 'elasticsearch')):
        return elasticsearch_home

    for path in SEARCH_PATHS:
        if os.path.exists(os.path.join(path, 'bin', 'elasticsearch')):
            return path

    # search newest elasticsearch-x.x.x directory
    globbed = (glob("/usr/local/*elasticsearch*") +
               glob("*elasticsearch*") +
               glob("/usr/local/Cellar/elasticsearch/*/libexec"))
    elasticsearch_dirs = [os.path.abspath(dir) for dir in globbed if os.path.isdir(dir)]
    if elasticsearch_dirs:
        return sorted(elasticsearch_dirs, key=strip_version)[-1]

    raise RuntimeError("could not find ES_HOME")

def get_unused_port():
    sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
    sock.bind(('localhost', 0))
    _, port = sock.getsockname()
    sock.close()

    return port

def generate_cluster_name():
    import string
    import random

    return ''.join([random.choice(string.ascii_letters) for i in range(6)])
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-

import os
import base64
import mimetypes

def get_abs_path_url(path):
    """ Returns the absolute url for a given local path.
    """
    return "file://%s" % os.path.abspath(path)

def get_path_url(abs_path, relative=False):
    """ Returns an absolute or relative path url from an absolute path.
    """
    if relative:
        return get_rel_path_url(abs_path)
    else:
        return get_abs_path_url(abs_path)

def get_rel_path_url(path, base_path=os.getcwd()):
    """ Returns a relative path from the absolute one passed as argument.
    """
    return os.path.relpath(path)

def encode_image_from_url(url, source_path):
    if not url or url.startswith('data:') or url.startswith('file://'):
        return False

    if (url.startswith('http://') or url.startswith('https://')):
        return False

    real_path = url if os.path.isabs(url) else os.path.join(source_path, url)

    if not os.path.exists(real_path):
        print('%s was not found, skipping' % url)
        return False

    mime_type, encoding = mimetypes.guess_type(real_path)

    if not mime_type:
        print('Unrecognized mime type for %s, skipping' % url)
        return False

    try:
        with open(real_path, 'rb') as image_file:
            image_contents = image_file.read()
            encoded_image = base64.b64encode(image_contents)
    except IOError:
        return False
    except Exception:
        return False

    return u"data:%s;base64,%s" % (mime_type, encoded_image.decode())
<commit_msg>Simplify file path generation in utils<commit_after># -*- coding: utf-8 -*-

import os
import base64
import mimetypes

def get_path_url(path, relative=False):
    """ Returns an absolute or relative path url given a path
    """
    if relative:
        return os.path.relpath(path)
    else:
        return 'file://%s' % os.path.abspath(path)

def encode_image_from_url(url, source_path):
    if not url or url.startswith('data:') or url.startswith('file://'):
        return False

    if (url.startswith('http://') or url.startswith('https://')):
        return False

    real_path = url if os.path.isabs(url) else os.path.join(source_path, url)

    if not os.path.exists(real_path):
        print('%s was not found, skipping' % url)
        return False

    mime_type, encoding = mimetypes.guess_type(real_path)

    if not mime_type:
        print('Unrecognized mime type for %s, skipping' % url)
        return False

    try:
        with open(real_path, 'rb') as image_file:
            image_contents = image_file.read()
            encoded_image = base64.b64encode(image_contents)
    except IOError:
        return False
    except Exception:
        return False

    return u"data:%s;base64,%s" % (mime_type, encoded_image.decode())
<|endoftext|>


<commit_before>from optparse import make_option
from django.core.management.base import BaseCommand
from corehq.apps.hqcase.dbaccessors import get_case_ids_in_domain
from corehq.apps.tzmigration import set_migration_started, \
    set_migration_complete
from corehq.apps.tzmigration.timezonemigration import prepare_planning_db, \
    get_planning_db, get_planning_db_filepath, delete_planning_db, \
    prepare_case_json, FormJsonDiff, commit_plan
from corehq.util.dates import iso_string_to_datetime
from couchforms.dbaccessors import get_form_ids_by_type

def _is_datetime(string):
    if not isinstance(string, basestring):
        return False
    try:
        iso_string_to_datetime(string)
    except (ValueError, OverflowError, TypeError):
        return False
    else:
        return True

class Command(BaseCommand):
    option_list = BaseCommand.option_list + (
        make_option('--BEGIN', action='store_true', default=False),
        make_option('--COMMIT', action='store_true', default=False),
        make_option('--ABORT', action='store_true', default=False),
        make_option('--prepare', action='store_true', default=False),
        make_option('--prepare-case-json', action='store_true', default=False),
        make_option('--blow-away', action='store_true', default=False),
        make_option('--stats', action='store_true', default=False),
        make_option('--show-diffs', action='store_true', default=False),
        make_option('--play', action='store_true', default=False),
    )

    @staticmethod
    def require_only_option(sole_option, options):
        assert all(option is False for key, option in options.items()
                   if option != sole_option)

    def handle(self, domain, **options):
        filepath = get_planning_db_filepath(domain)
        self.stdout.write('Using file {}\n'.format(filepath))
        if options['BEGIN']:
            self.require_only_option('BEGIN', options)
            set_migration_started(domain)
        if options['ABORT']:
            self.require_only_option('ABORT', options)
            set_migration_complete(domain)
        if options['blow_away']:
            delete_planning_db(domain)
            self.stdout.write('Removed file {}\n'.format(filepath))
        if options['prepare']:
            self.planning_db = prepare_planning_db(domain)
            self.stdout.write('Created and loaded file {}\n'.format(filepath))
        else:
            self.planning_db = get_planning_db(domain)

        if options['COMMIT']:
            self.require_only_option('COMMIT', options)
            commit_plan(domain, self.planning_db)
            set_migration_complete(domain)

        if options['prepare_case_json']:
            prepare_case_json(self.planning_db)
        if options['stats']:
            self.valiate_forms_and_cases(domain)
        if options['show_diffs']:
            self.show_diffs()
        if options['play']:
            from corehq.apps.tzmigration.planning import *
            session = self.planning_db.Session()  # noqa
            try:
                import ipdb as pdb
            except ImportError:
                import pdb

            pdb.set_trace()

    def valiate_forms_and_cases(self, domain):
        form_ids_in_couch = set(get_form_ids_by_type(domain, 'XFormInstance'))
        form_ids_in_sqlite = set(self.planning_db.get_all_form_ids())

        print 'Forms in Couch: {}'.format(len(form_ids_in_couch))
        print 'Forms in Sqlite: {}'.format(len(form_ids_in_sqlite))
        if form_ids_in_couch ^ form_ids_in_sqlite:
            print 'In Couch only: {}'.format(
                list(form_ids_in_couch - form_ids_in_sqlite))

        case_ids_in_couch = set(get_case_ids_in_domain(domain))
        case_ids_in_sqlite = set(self.planning_db.get_all_case_ids())

        print 'Cases in Couch: {}'.format(len(case_ids_in_couch))
        print 'Cases in Sqlite: {}'.format(len(case_ids_in_sqlite))
        if case_ids_in_couch ^ case_ids_in_sqlite:
            print 'In Couch only: {}'.format(
                list(case_ids_in_couch - case_ids_in_sqlite))
            print 'In Sqlite only: {}'.format(
                list(case_ids_in_sqlite - case_ids_in_couch))

    def show_diffs(self):
        for form_id, json_diff in self.planning_db.get_diffs():
            if json_diff.diff_type == 'diff':
                if _is_datetime(json_diff.old_value) and _is_datetime(json_diff.new_value):
                    continue
            if json_diff in (
                    FormJsonDiff(diff_type=u'type', path=[u'external_id'], old_value=u'', new_value=None),
                    FormJsonDiff(diff_type=u'type', path=[u'closed_by'], old_value=u'', new_value=None)):
                continue
            print '[{}] {}'.format(form_id, json_diff)
<commit_msg>fix tzmigrate commands<commit_after>from optparse import make_option
from django.core.management.base import BaseCommand
from corehq.apps.hqcase.dbaccessors import get_case_ids_in_domain
from corehq.apps.tzmigration import set_migration_started, \
    set_migration_complete
from corehq.apps.tzmigration.timezonemigration import prepare_planning_db, \
    get_planning_db, get_planning_db_filepath, delete_planning_db, \
    prepare_case_json, FormJsonDiff, commit_plan
from corehq.util.dates import iso_string_to_datetime
from couchforms.dbaccessors import get_form_ids_by_type

def _is_datetime(string):
    if not isinstance(string, basestring):
        return False
    try:
        iso_string_to_datetime(string)
    except (ValueError, OverflowError, TypeError):
        return False
    else:
        return True

class Command(BaseCommand):
    option_list = BaseCommand.option_list + (
        make_option('--BEGIN', action='store_true', default=False),
        make_option('--COMMIT', action='store_true', default=False),
        make_option('--ABORT', action='store_true', default=False),
        make_option('--prepare', action='store_true', default=False),
        make_option('--prepare-case-json', action='store_true', default=False),
        make_option('--blow-away', action='store_true', default=False),
        make_option('--stats', action='store_true', default=False),
        make_option('--show-diffs', action='store_true', default=False),
        make_option('--play', action='store_true', default=False),
    )

    @staticmethod
    def require_only_option(sole_option, options):
        base_options = {option.dest for option in BaseCommand.option_list}
        assert all(not value for key, value in options.items()
                   if key not in base_options and key != sole_option)

    def handle(self, domain, **options):
        filepath = get_planning_db_filepath(domain)
        self.stdout.write('Using file {}\n'.format(filepath))
        if options['BEGIN']:
            self.require_only_option('BEGIN', options)
            set_migration_started(domain)
        if options['ABORT']:
            self.require_only_option('ABORT', options)
            set_migration_complete(domain)
        if options['blow_away']:
            delete_planning_db(domain)
            self.stdout.write('Removed file {}\n'.format(filepath))
        if options['prepare']:
            self.planning_db = prepare_planning_db(domain)
            self.stdout.write('Created and loaded file {}\n'.format(filepath))
        else:
            self.planning_db = get_planning_db(domain)

        if options['COMMIT']:
            self.require_only_option('COMMIT', options)
            commit_plan(domain, self.planning_db)
            set_migration_complete(domain)

        if options['prepare_case_json']:
            prepare_case_json(self.planning_db)
        if options['stats']:
            self.valiate_forms_and_cases(domain)
        if options['show_diffs']:
            self.show_diffs()
        if options['play']:
            from corehq.apps.tzmigration.planning import *
            session = self.planning_db.Session()  # noqa
            try:
                import ipdb as pdb
            except ImportError:
                import pdb

            pdb.set_trace()

    def valiate_forms_and_cases(self, domain):
        form_ids_in_couch = set(get_form_ids_by_type(domain, 'XFormInstance'))
        form_ids_in_sqlite = set(self.planning_db.get_all_form_ids())

        print 'Forms in Couch: {}'.format(len(form_ids_in_couch))
        print 'Forms in Sqlite: {}'.format(len(form_ids_in_sqlite))
        if form_ids_in_couch ^ form_ids_in_sqlite:
            print 'In Couch only: {}'.format(
                list(form_ids_in_couch - form_ids_in_sqlite))

        case_ids_in_couch = set(get_case_ids_in_domain(domain))
        case_ids_in_sqlite = set(self.planning_db.get_all_case_ids())

        print 'Cases in Couch: {}'.format(len(case_ids_in_couch))
        print 'Cases in Sqlite: {}'.format(len(case_ids_in_sqlite))
        if case_ids_in_couch ^ case_ids_in_sqlite:
            print 'In Couch only: {}'.format(
                list(case_ids_in_couch - case_ids_in_sqlite))
            print 'In Sqlite only: {}'.format(
                list(case_ids_in_sqlite - case_ids_in_couch))

    def show_diffs(self):
        for form_id, json_diff in self.planning_db.get_diffs():
            if json_diff.diff_type == 'diff':
                if _is_datetime(json_diff.old_value) and _is_datetime(json_diff.new_value):
                    continue
            if json_diff in (
                    FormJsonDiff(diff_type=u'type', path=[u'external_id'], old_value=u'', new_value=None),
                    FormJsonDiff(diff_type=u'type', path=[u'closed_by'], old_value=u'', new_value=None)):
                continue
            print '[{}] {}'.format(form_id, json_diff)
<|endoftext|>


<commit_before>import os
import re
from waliki.signals import page_saved
from optparse import make_option
from django.core.management.base import BaseCommand  # CommandError
from waliki.settings import WALIKI_DATA_DIR
from waliki.models import Page
from django.utils.translation import ugettext_lazy as _
try:
    from waliki.attachments.models import Attachment
except ImportError:
    Attachment = None

def clean_meta(rst_content):
    """remove moinmoin metada from the top of the file"""

    rst = rst_content.split('\n')
    for i, line in enumerate(rst):
        if line.startswith('#'):
            continue
        break
    return '\n'.join(rst[i:])

def delete_relative_links(rst_content):
    """remove links relatives. Waliki point them correctly implicitly"""

    return re.sub(r'^(\.\. .*: \.\./.*)\n$', '', rst_content, flags=re.MULTILINE)

def attachments(rst_content, slug):

    def rep(matchobj):
        for filename in matchobj.groups(1):
            try:
                a = Attachment.objects.filter(file__endswith=filename, page__slug=slug)[0]
            except IndexError:
                print('Cant find %s in %s' % (filename, slug))
                return None
        return '`%s <%s>`_' % (filename, a.get_absolute_url())

    return re.sub(r'`attachment:(.*)`_', rep, rst_content, flags=re.MULTILINE)

def directives(rst_content):
    for directive in re.findall(r':(\w+):`.*`', rst_content, flags=re.MULTILINE):
        rst_content += """

.. role:: {directive}
   :class: {directive}

""".format(directive=directive)
    return rst_content

class Command(BaseCommand):
    help = 'Cleanups for a moin2git import'

    option_list = (
        make_option('--limit-to',
                    dest='slug',
                    default='',
                    help="optional slug namespace"),
    ) + BaseCommand.option_list

    def handle(self, *args, **options):
        slug = options['slug']
        if slug:
            pages = Page.objects.filter(slug__startswith=slug)
        else:
            pages = Page.objects.all()

        for page in pages:
            print('Cleaning up %s' % page.slug)
            raw = clean_meta(page.raw)
            raw = delete_relative_links(raw)
            raw = attachments(raw, page.slug)
            raw = directives(raw)
            page.raw = raw
            if not page.title:
                page.title = page._get_part('get_document_title')

            page.save()
            page_saved.send_robust(sender='moin',
                                   page=page,
                                   author=None,
                                   message=_("RestructuredText clean up"),
                                   form_extra_data={})<commit_msg>added emojis filters. more options for moin cleanup command<commit_after>import re
from waliki.signals import page_saved
from optparse import make_option
from django.core.management.base import BaseCommand, CommandError
from waliki.models import Page
from django.utils.translation import ugettext_lazy as _
from django.utils.text import get_text_list
try:
    from waliki.attachments.models import Attachment
except ImportError:
    Attachment = None

def clean_meta(rst_content):
    """remove moinmoin metada from the top of the file"""

    rst = rst_content.split('\n')
    for i, line in enumerate(rst):
        if line.startswith('#'):
            continue
        break
    return '\n'.join(rst[i:])

def delete_relative_links(rst_content):
    """remove links relatives. Waliki point them correctly implicitly"""

    return re.sub(r'^(\.\. .*: \.\./.*)\n$', '', rst_content, flags=re.MULTILINE)

def attachments(rst_content, slug):

    def rep(matchobj):
        for filename in matchobj.groups(1):
            try:
                a = Attachment.objects.filter(file__endswith=filename, page__slug=slug)[0]
            except IndexError:
                print('Cant find %s in %s' % (filename, slug))
                return None
        return '`%s <%s>`_' % (filename, a.get_absolute_url())

    return re.sub(r'`attachment:(.*)`_', rep, rst_content, flags=re.MULTILINE)

def directives(rst_content):
    for directive in re.findall(r':(\w+):`.*`', rst_content, flags=re.MULTILINE):
        rst_content += """

.. role:: {directive}
   :class: {directive}

""".format(directive=directive)
    return rst_content

def emojis(rst_content):
    # require
    emojis_map = {
        ':)': 'smile',
        ':-)': 'smile',
        ';)': 'wink',
        ';-)': 'wink',
        ':-?': 'smirk',
        ':?': 'smirk',
        ':(': 'confused',
        ':-(': 'confused',
        ':D': 'laughing',
        ':-D': 'laughing',
        ':-P': 'stuck_out_tongue_closed_eyes',
        ':P': 'stuck_out_tongue_closed_eyes',
        ":'(": 'cry',
        ":'-(": 'cry',
    }

    def replace_emoji(pattern):

        replacement = emojis_map.get(pattern.groups()[0], '')
        import ipdb; ipdb.set_trace()
        if replacement:
            return '|%s|' % replacement
        return ''
    result = re.sub(r'\|((?:\:|;).{1,3})\|', replace_emoji, rst_content, flags=re.MULTILINE)
    return result

class Command(BaseCommand):
    help = 'Cleanups for a moin2git import'

    option_list = (
        make_option('--limit-to',
                    dest='slug',
                    default='',
                    help="optional slug namespace"),
        make_option('--apply-filter',
                    dest='filters',
                    default='all',
                    help="comma separated list of filter functions to apply"),
        make_option('--message',
                    dest='message',
                    default=_("RestructuredText clean up"),
                    help="log message"),
    ) + BaseCommand.option_list

    def handle(self, *args, **options):
        valid_filters = ['clean_meta', 'delete_relative_links',
                         'attachments', 'directives',
                         'emojis', 'set_title']
        slug = options['slug']
        filters = options['filters']

        if filters == 'all':
            filters = valid_filters

        else:
            filters = [f.strip() for f in filters.split(',')]
            if not set(filters).issubset(valid_filters):
                valid = get_text_list(valid_filters, 'and')
                raise CommandError("At least one filter is unknown\n. Valid filters are %s" % valid)

        if slug:
            pages = Page.objects.filter(slug__startswith=slug)
        else:
            pages = Page.objects.all()

        for page in pages:
            title = None
            print('\nApplying filter/s %s to %s' % (get_text_list(filters, 'and'), page.slug))
            raw = page.raw
            if 'clean_meta' in filters:
                raw = clean_meta(raw)
            if 'delete_relative_links' in filters:
                raw = delete_relative_links(raw)
            if 'attachments' in filters:
                raw = attachments(raw, page.slug)

            if 'directives' in filters:
                raw = directives(raw)

            if 'emojis' in filters:
                raw = emojis(raw)

            if 'set_title' in filters and not page.title:
                title = page._get_part('get_document_title')

            if raw != page.raw or title:
                if title:
                    page.title = title
                if raw != page.raw:
                    page.raw = raw
                page.save()
                page_saved.send_robust(sender='moin',
                                       page=page,
                                       author=None,
                                       message=options['message'],
                                       form_extra_data={})
            else:
                print('Nothing changed. Ignoring update')
<|endoftext|>


<commit_before>r"""
>>> format(my_birthday, '')
u''
>>> format(my_birthday, 'a')
u'p.m.'
>>> format(my_birthday, 'A')
u'PM'
>>> format(my_birthday, 'd')
u'08'
>>> format(my_birthday, 'j')
u'8'
>>> format(my_birthday, 'l')
u'Sunday'
>>> format(my_birthday, 'L')
u'False'
>>> format(my_birthday, 'm')
u'07'
>>> format(my_birthday, 'M')
u'Jul'
>>> format(my_birthday, 'b')
u'jul'
>>> format(my_birthday, 'n')
u'7'
>>> format(my_birthday, 'N')
u'July'
>>> no_tz or format(my_birthday, 'O') == '+0100'
True
>>> format(my_birthday, 'P')
u'10 p.m.'
>>> no_tz or format(my_birthday, 'r') == 'Sun, 8 Jul 1979 22:00:00 +0100'
True
>>> format(my_birthday, 's')
u'00'
>>> format(my_birthday, 'S')
u'th'
>>> format(my_birthday, 't')
u'31'
>>> no_tz or format(my_birthday, 'T') == 'CET'
True
>>> no_tz or format(my_birthday, 'U') == '300315600'
True
>>> format(my_birthday, 'w')
u'0'
>>> format(my_birthday, 'W')
u'27'
>>> format(my_birthday, 'y')
u'79'
>>> format(my_birthday, 'Y')
u'1979'
>>> format(my_birthday, 'z')
u'189'
>>> no_tz or format(my_birthday, 'Z') == '3600'
True

>>> no_tz or format(summertime, 'I') == '1'
True
>>> no_tz or format(summertime, 'O') == '+0200'
True
>>> no_tz or format(wintertime, 'I') == '0'
True
>>> no_tz or format(wintertime, 'O') == '+0100'
True

>>> format(my_birthday, r'Y z \C\E\T')
u'1979 189 CET'

>>> format(my_birthday, r'jS o\f F')
u'8th of July'

>>> format(the_future, r'Y')
u'2100'
"""

from django.utils import dateformat, translation
import datetime, os, time

format = dateformat.format
os.environ['TZ'] = 'Europe/Copenhagen'
translation.activate('en-us')

try:
    time.tzset()
    no_tz = False
except AttributeError:
    no_tz = True

my_birthday = datetime.datetime(1979, 7, 8, 22, 00)
summertime = datetime.datetime(2005, 10, 30, 1, 00)
wintertime = datetime.datetime(2005, 10, 30, 4, 00)
the_future = datetime.datetime(2100, 10, 25, 0, 00)
<commit_msg>Fixed #12123 -- Refactored dateformat tests to use unittest. This is needed so that we have a reliable teardown that can reset the timezone at the end of the test.<commit_after>
from django.utils import dateformat, translation
from unittest import TestCase
import datetime, os, time

class DateFormatTests(TestCase):
    def setUp(self):
        self.old_TZ = os.environ['TZ']
        os.environ['TZ'] = 'Europe/Copenhagen'
        translation.activate('en-us')

    def tearDown(self):
        os.environ['TZ'] = self.old_TZ

    def test_empty_format(self):
        my_birthday = datetime.datetime(1979, 7, 8, 22, 00)

        self.assertEquals(dateformat.format(my_birthday, ''), u'')

    def test_am_pm(self):
        my_birthday = datetime.datetime(1979, 7, 8, 22, 00)

        self.assertEquals(dateformat.format(my_birthday, 'a'), u'p.m.')

    def test_date_formats(self):
        my_birthday = datetime.datetime(1979, 7, 8, 22, 00)

        self.assertEquals(dateformat.format(my_birthday, 'A'), u'PM')
        self.assertEquals(dateformat.format(my_birthday, 'd'), u'08')
        self.assertEquals(dateformat.format(my_birthday, 'j'), u'8')
        self.assertEquals(dateformat.format(my_birthday, 'l'), u'Sunday')
        self.assertEquals(dateformat.format(my_birthday, 'L'), u'False')
        self.assertEquals(dateformat.format(my_birthday, 'm'), u'07')
        self.assertEquals(dateformat.format(my_birthday, 'M'), u'Jul')
        self.assertEquals(dateformat.format(my_birthday, 'b'), u'jul')
        self.assertEquals(dateformat.format(my_birthday, 'n'), u'7')
        self.assertEquals(dateformat.format(my_birthday, 'N'), u'July')

    def test_time_formats(self):
        my_birthday = datetime.datetime(1979, 7, 8, 22, 00)

        self.assertEquals(dateformat.format(my_birthday, 'P'), u'10 p.m.')
        self.assertEquals(dateformat.format(my_birthday, 's'), u'00')
        self.assertEquals(dateformat.format(my_birthday, 'S'), u'th')
        self.assertEquals(dateformat.format(my_birthday, 't'), u'31')
        self.assertEquals(dateformat.format(my_birthday, 'w'), u'0')
        self.assertEquals(dateformat.format(my_birthday, 'W'), u'27')
        self.assertEquals(dateformat.format(my_birthday, 'y'), u'79')
        self.assertEquals(dateformat.format(my_birthday, 'Y'), u'1979')
        self.assertEquals(dateformat.format(my_birthday, 'z'), u'189')

    def test_dateformat(self):
        my_birthday = datetime.datetime(1979, 7, 8, 22, 00)

        self.assertEquals(dateformat.format(my_birthday, r'Y z \C\E\T'), u'1979 189 CET')

        self.assertEquals(dateformat.format(my_birthday, r'jS o\f F'), u'8th of July')

    def test_futuredates(self):
        the_future = datetime.datetime(2100, 10, 25, 0, 00)
        self.assertEquals(dateformat.format(the_future, r'Y'), u'2100')

    def test_timezones(self):
        my_birthday = datetime.datetime(1979, 7, 8, 22, 00)
        summertime = datetime.datetime(2005, 10, 30, 1, 00)
        wintertime = datetime.datetime(2005, 10, 30, 4, 00)

        try:
            # Check if a timezone has been set
            time.tzset()

            self.assertEquals(dateformat.format(my_birthday, 'O'), u'+0100')
            self.assertEquals(dateformat.format(my_birthday, 'r'), u'Sun, 8 Jul 1979 22:00:00 +0100')
            self.assertEquals(dateformat.format(my_birthday, 'T'), u'CET')
            self.assertEquals(dateformat.format(my_birthday, 'U'), u'300315600')
            self.assertEquals(dateformat.format(my_birthday, 'Z'), u'3600')
            self.assertEquals(dateformat.format(summertime, 'I'), u'1')
            self.assertEquals(dateformat.format(summertime, 'O'), u'+0200')
            self.assertEquals(dateformat.format(wintertime, 'I'), u'0')
            self.assertEquals(dateformat.format(wintertime, 'O'), u'+0100')
        except AttributeError:
            # No timezone available. Don't run the tests
            pass<|endoftext|>


<commit_before>"""Simple watcher that prints wikipedia changes as they occur."""

from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

import json
import pprint
import argparse
import os
import pywikibot
import requests
import sseclient
from googleapiclient import errors as google_api_errors
from antidox import clean
from antidox import perspective

# pylint: disable=fixme, too-many-locals
def log_event(apikey_data, toxicity, dlp, change):
  """Logs event by printing.
  Args:
    change: a json object with the wikimedia change record.
  """
  # print(
  #     u'user:{user} namespace:{namespace} bot:{bot} comment:{comment} title:{title}'
  #     .format(**change))
  # print('\n########## change:')
  from_id = (str(change['revision']['old']))
  to_id = (str(change['revision']['new']))
  page = ('https://en.wikipedia.org/w/api.php?action=compare&fromrev=' +
          from_id + '&torev=' + to_id + '&format=json')
  get_page = requests.get(page)
  response = json.loads(get_page.content.decode('utf-8'))
  revision = response['compare']['*']

  text = clean.content_clean(revision)

  # for line in text:
  #print(text)
  if not text:
    return
  dlp_response = perspective.dlp_request(dlp, apikey_data, text)
  try:
    perspective_response = perspective.perspective_request(toxicity, text)
  # Perspective can't handle language errors at this time
  except google_api_errors.HttpError as err:
    print('Error:', err)
    return
  has_pii_bool, pii_type = perspective.contains_pii(dlp_response)
  if has_pii_bool:
    header = '==Possible Doxxing Detected: Waiting for review=='
    result = (
        u'{'
        'user:{user}, namespace:{namespace}, bot:{bot}, comment:{comment}' +
        'title:{title},'.format(**change) + ', ' + 'comment_text:' + str(text) +
        ', ' + 'contains_pii:' + 'True' + ', ' + 'pii_type:' + str(pii_type) +
        '\n')
    wiki_write(result, header)
    print (result)

  if perspective.contains_toxicity(perspective_response):
    header = '==Possibly Toxic Detected: Waiting for review=='
    result = (
        u'{'
        'user:{user}, namespace:{namespace}, bot:{bot}, comment:{comment}' +
        'title:{title}'.format(**change) + ', ' + 'comment_text:' + str(text) +
        ', ' + 'contains_toxicity:' + 'True' + ', ' + 'toxic_score:' +
        str(perspective_response['attributeScores']['TOXICITY']['summaryScore']['value']) +
        '\n')
    wiki_write(result, header)
    print (result)

# def wiki_write(result, header):
#   site = pywikibot.Site()
#   repo = site.data_repository()
#   page = pywikibot.Page(site, u'User_talk:DoxDetective')

#   heading = (header)
#   content = (result)
#   message = '\n\n{}\n{} --~~~~'.format(heading, content)
#   page.save(
#       summary='Testing',
#       watch=None,
#       minor=False,
#       botflag=True,
#       force=False,
#       async=False,
#       callback=None,
#       apply_cosmetic_changes=None,
#       appendtext=message)
def wiki_write(result, header):
  """ Writes results to wikipedia/wikimedia bot page """
  pywikibot.config.register_family_file('doxwiki', os.path.join(os.path.dirname(__file__), 'doxwiki_family.py'))
  pywikibot.config.usernames['doxwiki']['en'] = u'Antidoxer'
  site = pywikibot.Site()
  repo = site.data_repository()
  page = pywikibot.Page(site, u"User_talk:Antidoxer")

  heading = (header)
  content = (result)
  message = "\n\n{}\n{} --~~~~".format(heading, content)
  page.save(summary="Testing", watch=None, minor=False, botflag=True,
            force=False, async=False, callback=None,
            apply_cosmetic_changes=None, appendtext=message)

def watcher(event_source, wiki_filter, namespaces_filter, callback):
  """Watcher captures and filters evens from mediawiki.
  Args:
    event_source: an interable source of streaming sse events.
    wiki_filter: string for filtering 'wiki' class.
    namespaces_filter: a set() of namespaces to keep.
    callback: A method to invoke with the JSON params for each filterd event.
  """
  for event in event_source:
    if event.event == 'message' and event.data:
      try:
        change = json.loads(event.data)
      except json.decoder.JSONDecodeError as err:
        print('Error:', err)
        pprint.pprint(event.data)
        continue
      if change['bot']:
        continue
      if change['wiki'] != wiki_filter:
        continue
      if change['namespace'] not in namespaces_filter:
        continue
      if 'revision' not in change:
        continue
      if 'old' not in change['revision']:
        continue
      callback(change)

if __name__ == '__main__':
  parser = argparse.ArgumentParser()
  parser.add_argument('--input_file', help='Name of file to process')
  parser.add_argument('--wikibot', help='Name of Wikipedia bot')
  parser.add_argument(
      '--wiki_filter', default='enwiki', help='Wiki to scan, default enwiki.')
  parser.add_argument(
      '--namespaces',
      default='1,3',
      help='Namespaces defined in http://phabricator.wikimedia.'+
      'org/source/mediawiki/browse/master/includes/Defines.php separated by commas.'
  )
  parser.add_argument(
      '--url',
      default='https://stream.wikimedia.org/v2/stream/recentchange',
      help='SSE client url')
  parser.add_argument('--mediawiki', help='pathway to fake wiki:http://sorenj02.nyc.corp.google.com/mediawiki')
  parser.add_argument('--wikipedia', help='pathway to wikipedia:https://en.wikipedia.org/w ')
  args = parser.parse_args()

  namespaces = set([int(ns) for ns in args.namespaces.split(',')])
  client = sseclient.SSEClient(args.url)

  apikey_data, toxicity, dlp = perspective.get_client()

  def log_change(change):
    return log_event(apikey_data, toxicity, dlp, change)

  watcher(client, args.wiki_filter, namespaces, log_change)<commit_msg>Revert "wikiwatcher is now writing results to wikimedia"<commit_after>"""Simple watcher that prints wikipedia changes as they occur."""

from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

import json
import pprint
import argparse
import pywikibot
import requests
import sseclient
from googleapiclient import errors as google_api_errors

from antidox import clean
from antidox import perspective

# pylint: disable=fixme, too-many-locals
def log_event(apikey_data, toxicity, dlp, change):
  """Logs event by printing.

  Args:
    change: a json object with the wikimedia change record.
  """
  # print(
  #     u'user:{user} namespace:{namespace} bot:{bot} comment:{comment} title:{title}'
  #     .format(**change))
  # print('\n########## change:')
  from_id = (str(change['revision']['old']))
  to_id = (str(change['revision']['new']))
  page = ("https://en.wikipedia.org/w/api.php?action=compare&fromrev="
          + from_id + "&torev=" + to_id + "&format=json")
  get_page = requests.get(page)
  response = json.loads(get_page.content.decode('utf-8'))
  revision = response['compare']['*']

  text = clean.content_clean(revision)

  # for line in text:
  print(text)
  if not text:
    return
  dlp_response = perspective.dlp_request(dlp, apikey_data, text)
  try:
    perspective_response = perspective.perspective_request(toxicity, text)
  # Perspective can't handle language errors at this time
  except google_api_errors.HttpError as err:
    print("Error:", err)
    return
  has_pii_bool, pii_type = perspective.contains_pii(dlp_response)
  if has_pii_bool:
    header = "==Possible Doxxing Detected: Waiting for review=="
    result = (u"{"'user:{user}, namespace:{namespace}, bot:{bot}, comment:{comment}'+
               'title:{title},'.format(**change)+", "+"comment_text:" + str(text)+", "+'contains_pii:'
               +"True"+", "+"pii_type:"
               +str(pii_type)+", ""}""\n")
    wiki_write(result, header)

  if perspective.contains_toxicity(perspective_response):
    header = "==Possibly Toxic Detected: Waiting for review=="
    result = (u"{"'user:{user}, namespace:{namespace}, bot:{bot}, comment:{comment}'+
               'title:{title}'.format(**change)+", "+"comment_text:" +str(text)+", "
               +"contains_toxicity:"+"True"+", "+ "toxic_score:"
               str(perspective_response['attributeScores']
                   ['TOXICITY']['summaryScore']['value'])+", ""}""\n")
    wiki_write(result, header)

def wiki_write(result, header):
    site = pywikibot.Site()
    repo = site.data_repository()
    page = pywikibot.Page(site, u"User_talk:DoxDetective")

    heading = (header)
    content = (result)
    message = "\n\n{}\n{} --~~~~".format(heading, content)
    page.save(summary="Testing", watch=None, minor=False, botflag=True,
              force=False, async=False, callback=None,
              apply_cosmetic_changes=None, appendtext=message)

def watcher(event_source, wiki_filter, namespaces_filter, callback):
  """Watcher captures and filters evens from mediawiki.

  Args:
    event_source: an interable source of streaming sse events.
    wiki_filter: string for filtering 'wiki' class.
    namespaces_filter: a set() of namespaces to keep.
    callback: A method to invoke with the JSON params for each filterd event.
  """
  for event in event_source:
    if event.event == 'message' and event.data:
      try:
        change = json.loads(event.data)
      except json.decoder.JSONDecodeError as err:
        print("Error:", err)
        pprint.pprint(event.data)
        continue
      if change['bot']:
        continue
      if change['wiki'] != wiki_filter:
        continue
      if change['namespace'] not in namespaces_filter:
        continue
      if "revision" not in change:
        continue
      if "old" not in change['revision']:
        continue
      callback(change)

if __name__ == '__main__':
  parser = argparse.ArgumentParser()
  parser.add_argument(
      '--wiki_filter', default='enwiki', help='Wiki to scan, default enwiki.')
  parser.add_argument(
      '--namespaces',
      default='1,3',
      help='Namespaces defined in http://phabricator.wikimedia.'+
      'org/source/mediawiki/browse/master/includes/Defines.php separated by commas.'
  )
  parser.add_argument(
      '--url',
      default='https://stream.wikimedia.org/v2/stream/recentchange',
      help='SSE client url')
  args = parser.parse_args()

  namespaces = set([int(ns) for ns in args.namespaces.split(',')])
  client = sseclient.SSEClient(args.url)

  apikey_data, toxicity, dlp = perspective.get_client()
  def log_change(change):
    return log_event(apikey_data, toxicity, dlp, change)
  watcher(client, args.wiki_filter, namespaces, log_change)
<|endoftext|>