<commit_before>import os
import shutil
import random

from nxdrive.tests.common_unit_test import UnitTestCase
from nxdrive.tests.common_unit_test import log

class TestLocalMoveFolders(UnitTestCase):

    NUMBER_OF_LOCAL_IMAGE_FILES = 10
    FILE_NAME_PATTERN = 'file%03d.%s'

    def setUp(self):
        """
        1. Create folder a1 in Nuxeo Drive Test Workspace sycn root
        2. Create folder a2 in Nuxeo Drive Test Workspace sycn root
        3. Add 10 image files in a1
        4. Add 10 image files in a2
        """
        self.engine_1.start()
        self.wait_sync(wait_for_async=True)
        self.engine_1.stop()

        # Create a1 and a2
        self.folder_path_1 = self.local_client_1.make_folder(u'/', u'a1')
        self.folder_path_2 = self.local_client_1.make_folder(u'/', u'a2')

        # Add image files to a1
        abs_folder_path_1 = self.local_client_1._abspath(self.folder_path_1)
        for file_num in range(1, self.NUMBER_OF_LOCAL_IMAGE_FILES + 1):
            file_name = self.FILE_NAME_PATTERN % (file_num, 'jpg')
            file_path = os.path.join(abs_folder_path_1, file_name)
            self.generate_random_jpg(file_path, random.randint(1000, 3000))
        log.debug('Local test files created in a1')

        # Add image files to a2
        abs_folder_path_2 = self.local_client_1._abspath(self.folder_path_2)
        for file_num in range(1, self.NUMBER_OF_LOCAL_IMAGE_FILES + 1):
            file_name = self.FILE_NAME_PATTERN % (file_num, 'jpg')
            file_path = os.path.join(abs_folder_path_2, file_name)
            self.generate_random_jpg(file_path, random.randint(1000, 3000))
        log.debug('Local test files created in a2')

        self.engine_1.start()
        self.wait_sync(fail_if_timeout=False, wait_win=True)

        # Check local files in a1
        self.assertTrue(self.local_client_1.exists('/a1'))
        children_1 = [child.name for child in self.local_client_1.get_children_info('/a1')]
        self.assertEqual(len(children_1), self.NUMBER_OF_LOCAL_IMAGE_FILES,
                         'Number of local files (%d) in a1 is different from original (%d)' %
                         (len(children_1), self.NUMBER_OF_LOCAL_IMAGE_FILES))
        self.assertEqual(set(children_1), set(['file%03d.jpg' % file_num
                                               for file_num in range(1, self.NUMBER_OF_LOCAL_IMAGE_FILES + 1)]))

        # Check local files in a2
        self.assertTrue(self.local_client_1.exists('/a2'))
        children_2 = [child.name for child in self.local_client_1.get_children_info('/a2')]
        self.assertEqual(len(children_2), self.NUMBER_OF_LOCAL_IMAGE_FILES,
                         'Number of local files (%d) in a2 is different from original (%d)' %
                         (len(children_2), self.NUMBER_OF_LOCAL_IMAGE_FILES))
        self.assertEqual(set(children_2), set(['file%03d.jpg' % file_num
                                               for file_num in range(1, self.NUMBER_OF_LOCAL_IMAGE_FILES + 1)]))

        # Check remote files in a1
        a1_remote_id = self.local_client_1.get_remote_id('/a1')
        self.assertIsNotNone(a1_remote_id)
        log.debug("Remote ref of a1: %s", a1_remote_id)
        self.assertTrue(self.remote_file_system_client_1.exists(a1_remote_id))

        remote_children_1 = [child.name for child in self.remote_file_system_client_1.get_children_info(a1_remote_id)]
        self.assertEqual(len(remote_children_1), self.NUMBER_OF_LOCAL_IMAGE_FILES,
                         'Number of remote files (%d) in a1 is different from original (%d)' %
                         (len(remote_children_1), self.NUMBER_OF_LOCAL_IMAGE_FILES))
        self.assertEqual(set(remote_children_1), set(['file%03d.jpg' % file_num
                                                      for file_num in range(1, self.NUMBER_OF_LOCAL_IMAGE_FILES + 1)]))

        # Check remote files in a2
        a2_remote_id = self.local_client_1.get_remote_id('/a2')
        self.assertIsNotNone(a2_remote_id)
        log.debug("Remote ref of a2: %s", a2_remote_id)
        self.assertTrue(self.remote_file_system_client_1.exists(a2_remote_id))

        remote_children_2 = [child.name for child in self.remote_file_system_client_1.get_children_info(a2_remote_id)]
        self.assertEqual(len(remote_children_2), self.NUMBER_OF_LOCAL_IMAGE_FILES,
                         'Number of remote files (%d) in a2 is different from original (%d)' %
                         (len(remote_children_2), self.NUMBER_OF_LOCAL_IMAGE_FILES))
        self.assertEqual(set(remote_children_2), set(['file%03d.jpg' % file_num
                                                      for file_num in range(1, self.NUMBER_OF_LOCAL_IMAGE_FILES + 1)]))

    def test_local_move_folder_with_files(self):
        src = self.local_client_1._abspath(self.folder_path_1)
        dst = self.local_client_1._abspath(self.folder_path_2)
        shutil.move(src, dst)
        self.wait_sync()

        # Check that a1 doesn't exist anymore locally
        self.assertFalse(self.local_client_1.exists('/a1'))

        # Check local files in a2
        self.assertTrue(self.local_client_1.exists('/a2'))
        children_2 = [child.name for child in self.local_client_1.get_children_info('/a2') if not child.folderish]
        self.assertEqual(len(children_2), self.NUMBER_OF_LOCAL_IMAGE_FILES,
                         'Number of local files (%d) in a2 is different from original (%d)' %
                         (len(children_2), self.NUMBER_OF_LOCAL_IMAGE_FILES))
        self.assertEqual(set(children_2), set(['file%03d.jpg' % file_num
                                               for file_num in range(1, self.NUMBER_OF_LOCAL_IMAGE_FILES + 1)]))

        # Check local files in a2/a1
        self.assertTrue(self.local_client_1.exists('/a2/a1'))
        children_1 = [child.name for child in self.local_client_1.get_children_info('/a2/a1')]
        self.assertEqual(len(children_1), self.NUMBER_OF_LOCAL_IMAGE_FILES,
                         'Number of local files (%d) in a1 is different from original (%d)' %
                         (len(children_1), self.NUMBER_OF_LOCAL_IMAGE_FILES))
        self.assertEqual(set(children_1), set(['file%03d.jpg' % file_num
                                               for file_num in range(1, self.NUMBER_OF_LOCAL_IMAGE_FILES + 1)]))

        # Check that a1 doesn't exist anymore remotely
        self.assertEqual(len(self.remote_document_client_1.get_children_info(self.workspace)), 1)

        # Check remote files in a2
        a2_remote_id = self.local_client_1.get_remote_id('/a2')
        self.assertIsNotNone(a2_remote_id)
        log.debug("Remote ref of a2: %s", a2_remote_id)
        self.assertTrue(self.remote_file_system_client_1.exists(a2_remote_id))

        remote_children_2 = [child.name for child in self.remote_file_system_client_1.get_children_info(a2_remote_id)
                             if not child.folderish]
        self.assertEqual(len(remote_children_2), self.NUMBER_OF_LOCAL_IMAGE_FILES,
                         'Number of remote files (%d) in a2 is different from original (%d)' %
                         (len(remote_children_2), self.NUMBER_OF_LOCAL_IMAGE_FILES))
        self.assertEqual(set(remote_children_2), set(['file%03d.jpg' % file_num
                                                      for file_num in range(1, self.NUMBER_OF_LOCAL_IMAGE_FILES + 1)]))
        # Check remote files in a2/a1
        a1_remote_id = self.local_client_1.get_remote_id('/a2/a1')
        self.assertIsNotNone(a1_remote_id)
        log.debug("Remote ref of a1: %s", a1_remote_id)
        self.assertTrue(self.remote_file_system_client_1.exists(a1_remote_id))

        remote_children_1 = [child.name for child in self.remote_file_system_client_1.get_children_info(a1_remote_id)]
        self.assertEqual(len(remote_children_1), self.NUMBER_OF_LOCAL_IMAGE_FILES,
                         'Number of remote files (%d) in a1 is different from original (%d)' %
                         (len(remote_children_1), self.NUMBER_OF_LOCAL_IMAGE_FILES))
        self.assertEqual(set(remote_children_1), set(['file%03d.jpg' % file_num
                                                      for file_num in range(1, self.NUMBER_OF_LOCAL_IMAGE_FILES + 1)]))
<commit_msg>NXDRIVE-286: Set fail on timeout if slave is slow, but increase the timeout value<commit_after>import os
import shutil
import random

from nxdrive.tests.common_unit_test import UnitTestCase
from nxdrive.tests.common_unit_test import log

class TestLocalMoveFolders(UnitTestCase):

    NUMBER_OF_LOCAL_IMAGE_FILES = 10
    FILE_NAME_PATTERN = 'file%03d.%s'

    def setUp(self):
        """
        1. Create folder a1 in Nuxeo Drive Test Workspace sycn root
        2. Create folder a2 in Nuxeo Drive Test Workspace sycn root
        3. Add 10 image files in a1
        4. Add 10 image files in a2
        """
        self.engine_1.start()
        self.wait_sync(wait_for_async=True)
        self.engine_1.stop()

        # Create a1 and a2
        self.folder_path_1 = self.local_client_1.make_folder(u'/', u'a1')
        self.folder_path_2 = self.local_client_1.make_folder(u'/', u'a2')

        # Add image files to a1
        abs_folder_path_1 = self.local_client_1._abspath(self.folder_path_1)
        for file_num in range(1, self.NUMBER_OF_LOCAL_IMAGE_FILES + 1):
            file_name = self.FILE_NAME_PATTERN % (file_num, 'jpg')
            file_path = os.path.join(abs_folder_path_1, file_name)
            self.generate_random_jpg(file_path, random.randint(1000, 3000))
        log.debug('Local test files created in a1')

        # Add image files to a2
        abs_folder_path_2 = self.local_client_1._abspath(self.folder_path_2)
        for file_num in range(1, self.NUMBER_OF_LOCAL_IMAGE_FILES + 1):
            file_name = self.FILE_NAME_PATTERN % (file_num, 'jpg')
            file_path = os.path.join(abs_folder_path_2, file_name)
            self.generate_random_jpg(file_path, random.randint(1000, 3000))
        log.debug('Local test files created in a2')

        self.engine_1.start()
        self.wait_sync(timeout=60, wait_win=True)

        # Check local files in a1
        self.assertTrue(self.local_client_1.exists('/a1'))
        children_1 = [child.name for child in self.local_client_1.get_children_info('/a1')]
        self.assertEqual(len(children_1), self.NUMBER_OF_LOCAL_IMAGE_FILES,
                         'Number of local files (%d) in a1 is different from original (%d)' %
                         (len(children_1), self.NUMBER_OF_LOCAL_IMAGE_FILES))
        self.assertEqual(set(children_1), set(['file%03d.jpg' % file_num
                                               for file_num in range(1, self.NUMBER_OF_LOCAL_IMAGE_FILES + 1)]))

        # Check local files in a2
        self.assertTrue(self.local_client_1.exists('/a2'))
        children_2 = [child.name for child in self.local_client_1.get_children_info('/a2')]
        self.assertEqual(len(children_2), self.NUMBER_OF_LOCAL_IMAGE_FILES,
                         'Number of local files (%d) in a2 is different from original (%d)' %
                         (len(children_2), self.NUMBER_OF_LOCAL_IMAGE_FILES))
        self.assertEqual(set(children_2), set(['file%03d.jpg' % file_num
                                               for file_num in range(1, self.NUMBER_OF_LOCAL_IMAGE_FILES + 1)]))

        # Check remote files in a1
        a1_remote_id = self.local_client_1.get_remote_id('/a1')
        self.assertIsNotNone(a1_remote_id)
        log.debug("Remote ref of a1: %s", a1_remote_id)
        self.assertTrue(self.remote_file_system_client_1.exists(a1_remote_id))

        remote_children_1 = [child.name for child in self.remote_file_system_client_1.get_children_info(a1_remote_id)]
        self.assertEqual(len(remote_children_1), self.NUMBER_OF_LOCAL_IMAGE_FILES,
                         'Number of remote files (%d) in a1 is different from original (%d)' %
                         (len(remote_children_1), self.NUMBER_OF_LOCAL_IMAGE_FILES))
        self.assertEqual(set(remote_children_1), set(['file%03d.jpg' % file_num
                                                      for file_num in range(1, self.NUMBER_OF_LOCAL_IMAGE_FILES + 1)]))

        # Check remote files in a2
        a2_remote_id = self.local_client_1.get_remote_id('/a2')
        self.assertIsNotNone(a2_remote_id)
        log.debug("Remote ref of a2: %s", a2_remote_id)
        self.assertTrue(self.remote_file_system_client_1.exists(a2_remote_id))

        remote_children_2 = [child.name for child in self.remote_file_system_client_1.get_children_info(a2_remote_id)]
        self.assertEqual(len(remote_children_2), self.NUMBER_OF_LOCAL_IMAGE_FILES,
                         'Number of remote files (%d) in a2 is different from original (%d)' %
                         (len(remote_children_2), self.NUMBER_OF_LOCAL_IMAGE_FILES))
        self.assertEqual(set(remote_children_2), set(['file%03d.jpg' % file_num
                                                      for file_num in range(1, self.NUMBER_OF_LOCAL_IMAGE_FILES + 1)]))

    def test_local_move_folder_with_files(self):
        src = self.local_client_1._abspath(self.folder_path_1)
        dst = self.local_client_1._abspath(self.folder_path_2)
        shutil.move(src, dst)
        self.wait_sync()

        # Check that a1 doesn't exist anymore locally
        self.assertFalse(self.local_client_1.exists('/a1'))

        # Check local files in a2
        self.assertTrue(self.local_client_1.exists('/a2'))
        children_2 = [child.name for child in self.local_client_1.get_children_info('/a2') if not child.folderish]
        self.assertEqual(len(children_2), self.NUMBER_OF_LOCAL_IMAGE_FILES,
                         'Number of local files (%d) in a2 is different from original (%d)' %
                         (len(children_2), self.NUMBER_OF_LOCAL_IMAGE_FILES))
        self.assertEqual(set(children_2), set(['file%03d.jpg' % file_num
                                               for file_num in range(1, self.NUMBER_OF_LOCAL_IMAGE_FILES + 1)]))

        # Check local files in a2/a1
        self.assertTrue(self.local_client_1.exists('/a2/a1'))
        children_1 = [child.name for child in self.local_client_1.get_children_info('/a2/a1')]
        self.assertEqual(len(children_1), self.NUMBER_OF_LOCAL_IMAGE_FILES,
                         'Number of local files (%d) in a1 is different from original (%d)' %
                         (len(children_1), self.NUMBER_OF_LOCAL_IMAGE_FILES))
        self.assertEqual(set(children_1), set(['file%03d.jpg' % file_num
                                               for file_num in range(1, self.NUMBER_OF_LOCAL_IMAGE_FILES + 1)]))

        # Check that a1 doesn't exist anymore remotely
        self.assertEqual(len(self.remote_document_client_1.get_children_info(self.workspace)), 1)

        # Check remote files in a2
        a2_remote_id = self.local_client_1.get_remote_id('/a2')
        self.assertIsNotNone(a2_remote_id)
        log.debug("Remote ref of a2: %s", a2_remote_id)
        self.assertTrue(self.remote_file_system_client_1.exists(a2_remote_id))

        remote_children_2 = [child.name for child in self.remote_file_system_client_1.get_children_info(a2_remote_id)
                             if not child.folderish]
        self.assertEqual(len(remote_children_2), self.NUMBER_OF_LOCAL_IMAGE_FILES,
                         'Number of remote files (%d) in a2 is different from original (%d)' %
                         (len(remote_children_2), self.NUMBER_OF_LOCAL_IMAGE_FILES))
        self.assertEqual(set(remote_children_2), set(['file%03d.jpg' % file_num
                                                      for file_num in range(1, self.NUMBER_OF_LOCAL_IMAGE_FILES + 1)]))
        # Check remote files in a2/a1
        a1_remote_id = self.local_client_1.get_remote_id('/a2/a1')
        self.assertIsNotNone(a1_remote_id)
        log.debug("Remote ref of a1: %s", a1_remote_id)
        self.assertTrue(self.remote_file_system_client_1.exists(a1_remote_id))

        remote_children_1 = [child.name for child in self.remote_file_system_client_1.get_children_info(a1_remote_id)]
        self.assertEqual(len(remote_children_1), self.NUMBER_OF_LOCAL_IMAGE_FILES,
                         'Number of remote files (%d) in a1 is different from original (%d)' %
                         (len(remote_children_1), self.NUMBER_OF_LOCAL_IMAGE_FILES))
        self.assertEqual(set(remote_children_1), set(['file%03d.jpg' % file_num
                                                      for file_num in range(1, self.NUMBER_OF_LOCAL_IMAGE_FILES + 1)]))
<|endoftext|>


<commit_before>from __future__ import division, print_function
import sys
import h5py
import glob
import numpy as np

theid = int(sys.argv[1])
hidenan = True

print('Abriendo archivo...')
with h5py.File('PM.h5') as f:
    print('Buscando ID')
    ids = f['data'][:,0]
    idx = ids == theid

    if np.sum(idx) == 0:
        print('ID no encontrado!')
        sys.exit(1)

    print('Buscando archivos de epocas')
    pms = glob.glob('PMs/*.dat')
    pms = np.sort([int(s.split('_')[-1].split('.')[0]) for s in pms])

    print('Extrayendo seeing')
    yep = np.genfromtxt('zinfo_img',unpack=True,usecols=(0,),dtype='string')
    hak = np.array(['k' in y for y in yep])
    yep = yep[hak]

    molde = yep[0].split('-')[0]
    tengo = np.array([molde+'-%03d.fits' % i for i in pms])
    effep = np.in1d(yep, tengo)

    zinfodat = np.genfromtxt('zinfo_img',usecols=(4,6))[hak][effep]
    see, jd  = zinfodat.T

    print('Extrayendo DX y DY')
    idx = np.where(idx)[0][0]
    row = f['data'][idx]
    dx  = row[8::6]
    dy  = row[9::6]
    ex  = row[11::6]
    ey  = row[12::6]

    output = np.vstack((pms, jd, dx, dy, ex, ey, see)).T
    if hidenan:
        print('No se guardaran filas donde la estrella no esta')
        output = output[np.isfinite(dx)]

    print('Guardando...')
    np.savetxt('%d.dat' % theid, output, fmt='%03d %.8f %.3f %.3f %.8f', header='EPOCH JD DX DY DX_ERR DY_ERR SEEING')
<commit_msg>extract 2<commit_after>from __future__ import division, print_function
import sys
import h5py
import glob
import numpy as np

theid = int(sys.argv[1])
hidenan = True

print('Abriendo archivo...')
with h5py.File('PM.h5') as f:
    print('Buscando ID')
    ids = f['data'][:,0]
    idx = ids == theid

    if np.sum(idx) == 0:
        print('ID no encontrado!')
        sys.exit(1)

    print('Buscando archivos de epocas')
    pms = glob.glob('PMs/*.dat')
    pms = np.sort([int(s.split('_')[-1].split('.')[0]) for s in pms])

    print('Extrayendo seeing')
    yep = np.genfromtxt('zinfo_img',unpack=True,usecols=(0,),dtype='string')
    hak = np.array(['k' in y for y in yep])
    yep = yep[hak]

    molde = yep[0].split('-')[0]
    tengo = np.array([molde+'-%03d.fits' % i for i in pms])
    effep = np.in1d(yep, tengo)

    zinfodat = np.genfromtxt('zinfo_img',usecols=(4,6))[hak][effep]
    see, jd  = zinfodat.T

    print('Extrayendo DX y DY')
    idx = np.where(idx)[0][0]
    row = f['data'][idx]
    dx  = row[8::6]
    dy  = row[9::6]
    ex  = row[11::6]
    ey  = row[12::6]

    output = np.vstack((pms, jd, dx, dy, ex, ey, see)).T
    if hidenan:
        print('No se guardaran filas donde la estrella no esta')
        output = output[np.isfinite(dx)]

    print('Guardando...')
    np.savetxt('%d.dat' % theid, output, fmt='%03d %.8f %.3f %.3f %.3f %.3f %.8f', header='EPOCH JD DX DY DX_ERR DY_ERR SEEING')
<|endoftext|>


<commit_before>
import requests
import base64

from bs4 import BeautifulSoup

requests.packages.urllib3.disable_warnings()

def all_images(html):
    soup = BeautifulSoup(html)
    imgs = soup.find_all('img')
    return list({t.attrs['src'] for t in imgs})

def is_http_url(img_path):
    img_path = img_path.lower()
    return (img_path.startswith("http://") or
            img_path.startswith("https://"))

def image_base64(imgpath):
    with open(imgpath, 'rb') as obj:
        return base64.b64encode(obj.read())

def image_content(img_path):
    """Get the image content according to its path
    Different case:
      - PNG,JPEG: convert it to base64
      - SVG: open it and read it
      - HTTP URL: get the content via requests.get
    """
    if is_http_url(img_path):
        content = requests.get(img_path, verify=False).content
        if img_path.endswith(".svg"):
            return content
        return base64.b64encode(content)
    if img_path.endswith(".svg"):
        return open(img_path).read()
    return image_base64(img_path)

def replace_imgpath_by_content(html, image_path):
    for path in image_path:
        content = image_content(path)
        if path.endswith(".svg"):
            html = html.replace('<img alt="alt text" src="{}" />'.format(path),
                                '{}'.format(content))
        else:
            html = html.replace('src="{}"'.format(path),
                                'src="data:image/png;base64,{}"'.format(content))
    return html

def embed_img(html):
    image_paths = all_images(html)
    output = replace_imgpath_by_content(html, image_paths)
    return output

    <commit_msg>add footer and embed img<commit_after><|endoftext|>


<commit_before>def title_word_feature(title, processed_text):
	""" List of values from 0 to 1 rating the number title words that appear in the sentence"""
	title_word_feature_values = []
	word_intersection = [set(filter(lambda title_word: title_word in title.bag_of_words, sublist)) for sublist in [sentence.bag_of_words for sentence in processed_text]]
	for word_list in word_intersection:
		title_word_feature_values.append(len(word_list) / len(title.bag_of_words))
	return title_word_feature_values

def sentence_length_feature(sentences):
	sentence_length_feature_values = []
	max_length_sentence = len(sentences[0].split(" "))
	for sentence in sentences[1:]:
		if(len(sentence.split(" ")) > max_length_sentence):
			max_length_sentence = len(sentence.split(" "))

	for sentence in sentences:
		sentence_length_feature_values.append(len(sentence.split(" "))/max_length_sentence)

	return sentence_length_feature_values

def sentence_location_feature(sentences):
	sentence_location_feature_values = []
	for sentence in sentences:
		sentence_location_feature_values.append(10/sentence.pos)
	return sentence_location_feature_values

def keyword_feature(sentences):
	total_number_of_sentences = len(sentences)

def pos_tag_feature(sentences, words, pos_tag):
	""" List of values from 0 to 1 rating the number of words with a certain part of speech tag that appear in the sentence"""
	pos_tag_words_count_list = []
	for sentence in sentences:
		pos_tag_words_count_list.append(len([word for word in sentence.bag_of_words if words[word].part_of_speech[1] == pos_tag]))
	return [pos_tag_words_sentence/max(pos_tag_words_count_list) for pos_tag_words_sentence in pos_tag_words_count_list] if max(pos_tag_words_count_list)!=0 else [0]*len(pos_tag_words_count_list)

<commit_msg>Features almost finished<commit_after>import math

def title_word_feature(title, processed_text):
	""" List of values from 0 to 1 rating the number title words that appear in the sentence"""
	title_word_feature_values = []
	word_intersection = [set(filter(lambda title_word: title_word in title.bag_of_words, sublist)) for sublist in [sentence.bag_of_words for sentence in processed_text]]
	for word_list in word_intersection:
		title_word_feature_values.append(len(word_list) / len(title.bag_of_words))
	return title_word_feature_values

def sentence_length_feature(sentences):
	sentence_length_feature_values = []
	max_length_sentence = len(sentences[0].original.split(" "))
	for sentence in sentences[1:]:
		if(len(sentence.original.split(" ")) > max_length_sentence):
			max_length_sentence = len(sentence.original.split(" "))

	for sentence in sentences:
		sentence_length_feature_values.append(len(sentence.original.split(" "))/max_length_sentence)

	return sentence_length_feature_values

def sentence_location_feature(sentences):
	sentence_location_feature_values = []
	for sentence in sentences:
		sentence_location_feature_values.append(1/sentence.position)
	return sentence_location_feature_values

def keyword_feature(sentences, words):
	keyword_feature_values = []
	total_number_of_sentences = len(sentences)
	for word in words:
		number_of_sentences = 0
		for sentence in sentences:
			if word in sentence.bag_of_words:
			 number_of_sentences += 1
		word = Word(word.stem, word.abs_frequency , word.abs_frequency * log10(total_number_of_sentences/number_of_sentences), word.part_of_speech, word.synonym_list)

	for sentence in sentences:
		sum_of_term_weights = 0
		for word in sentence.bag_of_words:
			sum_of_term_weights += word.term_weight
		keyword_feature_values.append(sum_of_term_weights)

	map(lambda x: x/max(keyword_feature_values), keyword_feature_values)
	return keyword_feature_values

def pos_tag_feature(sentences, words, pos_tag):
	""" List of values from 0 to 1 rating the number of words with a certain part of speech tag that appear in the sentence"""
	pos_tag_words_count_list = []
	for sentence in sentences:
		pos_tag_words_count_list.append(len([word for word in sentence.bag_of_words if words[word].part_of_speech[1] == pos_tag]))
	return [pos_tag_words_sentence/max(pos_tag_words_count_list) for pos_tag_words_sentence in pos_tag_words_count_list] if max(pos_tag_words_count_list)!=0 else [0]*len(pos_tag_words_count_list)

<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
import regex
import phonenumbers

class FindSpam:
    bad_keywords = ["baba ?ji", "fifa.*coins?", "fifabay", "Long Path Tool",
                    "fifaodell", "brianfo", "nike", "tosterone", "bajotz",
                    "vashi?k[ae]r[ae]n", "sumer", "kolcak"
                    "porn", "molvi", "judi bola", "ituBola.com", "lost lover'?s?",
                    "acai", "skin care", "rejuvenated skin",
                    "LifeForce", "swtor2credits", "me2.do", "black magic",
                    "bam2u", "Neuro(3X|flexyn|fuse)", "Nutra", "TesteroneXL",
                    "Bowtrol", "Slim ?Genix", "Cleanse EFX",
                    "Forskolin", "Blackline Elite", "TestCore Pro",
                    "Xtreme Antler", "Maxx Test 3000", "orvigomax",
                    "Cheap Wigs?", "jivam", "(Improve )?Brain Power", "Maximum ?Shred",
                    "aging skin", "acne( prone)? skin", "(skin )?eye serum",
                    "skin (serum|eye)", "bagprada", "6611165613" "Apowersoft",
                    "Service Solahart", "junisse",
                    "(fake|original|uk) (passports?|driver'?s? licen[cs]e|ID cards?)",
                    "(support|service|helpline)( phone)? number|1[ -]?[ -]?[ -]?866[ -]?978[ -]?(6819|6762)",
                    "(hotmail|gmail|outlook|yahoo|lexmark (printer)?) ?(password( recovery)?|tech)? ?((customer|technical) (support|service))? (support|contact|telephone|help(line)?|phone) number",
                    "kitchen for sale", "dolphin porn", "pdftoexcelconverter", "keepbrowsersafe", "SpyHunter",
                    "pcerror-fix", "filerepairtool", "combatpcviruses", "SkinCentric",
                    "JobsTribune", "join the illuminati", "Brorsoft", "Remo Recover"]
    bad_keywords_nwb = [u"ಌ", "babyliss", "garcinia", "acai ?berry"]  # "nwb" == "no word boundary"
    blacklisted_websites = ["online ?kelas", "careyourhealths", "wowtoes",
                            "ipubsoft", "orabank", "powerigfaustralia",
                            "cfpchampionship2015playofflive", "optimalstackfacts",
                            "maletestosteronebooster", "x4facts",
                            "tripleeffectseyeserum", "healthcaresup",
                            "garciniacambogiaprofacts", "filerepairforum",
                            "lxwpro-t", "casque-beatsbydre", "tenderpublish",
                            "elliskinantiaging", "funmac", "lovebiscuits",
                            "Eglobalfitness", "musclezx90site", "fifapal",
                            "hits4slim", "screenshot\\.net", "downloadavideo\\.net",
                            "strongmenmuscle", "sh\\.st/", "musclehealthfitness",
                            "svelmeblog", "preply\\.com", "hellofifa",
                            "fifa15online", "wearepropeople.com", "tagwitty",
                            "axsoccertours", "ragednablog", "ios8easyjailbreak",
                            "totalfitnesspoint", "trustessaywriting",
                            "trustmyessay", "faasoft", "besttvshows", "mytechlabs",
                            "giikers", "pagetube", "myenv\\.org",
                            "supertestoboostsfacts", "idolizeadvancedeyeserumfacts", "pelevoniface",
                            "skinphysiciantips", "fifa2coins", "xtrememusclerecoveryrev",
                            "diabacordoesitwork", "thehealthyadvise", "supracleanse350facts",
                            "premiumpureforskolinrev", "hyperglycemiaabout", "dietandhealthguide",
                            "health350", "sourceforge\\.net/projects/freepdftojpgconverter",
                            "pdftoexel\\.wordpress\\.com", "best7th\\.in", "any-data-recovery\\.com",
                            "recoverytoolbox\\.com", "mkmk9", "malwaretips", "intellipaat\\.com",
                            "webbuildersguide\\.com", "idealshare.net", "lankabpoacademy\\.com",
                            "evomailserver\\.com", "gameart\\.net", "any-data-recovery\\.com",
                            "recoverytoolbox\\.com", "sofotex\\.com",
                            "mybloggingmoney\\.com", "windows-techsupport\\.com",
                            "supplementsdeal\\.com", "drivethelife\\.com", "testolimitfacts\\.com",
                            "lafozi\\.com", "hipslimgarcinia\\.com", "open-swiss-bank\\.com",
                            "originalgarciniacambogiafacts", "healthy-weight-loss-tips\\.com",
                            "tenorshare\\.com", "advancedpdfconverter\\.com",
                            "androiddatarecoverypro\\.com", "card-data-recovery\\.com",
                            "excel-password-recovery\\.com", "fix-computer\\.net",
                            "lost-password\\.net", "lostwindowspassword\\.com",
                            "macvideoconverterpro\\.com", "password-master\\.net",
                            "photorecovery-formac\\.com", "powerpointpasswordrecovery\\.net",
                            "rarpasswordunlocker\\.net", "recover-windows-password\\.net",
                            "recoverlostpassword\\.com", "sqlpassword-recovery\\.com",
                            "windows7-password-reset\\.net", "windowspasswordcracker\\.com"
                            "windowspasswordreset\\.net", "windowspasswordsrecovery\\.com",
                            "wordpassword-recovery\\.com", "youtubedownloaderconverter\\.net",
                            "smartpcfixer\\.com", "1fix\\.org", "cardrecoverypro\\.com",
                            "drivertuner\\.com", "easyfix\\.org", "errorsfixer\\.org",
                            "faq800\\.com", "fix1\\.org", "guru4pc\\.net", "howto4pc\\.org",
                            "official-drivers\\.com", "pceasynow\\.com", "photorecoverypro\\.net",
                            "regeasypro\\.com", "registryware\\.org", "smartfixer\\.net",
                            "smartfixer\\.org", "wisefixer\\.com", "wisefixer\\.net",
                            "wiserecovery\\.com", "passwordunlocker\\.com",
                            "password-unlocker\\.com", "passwordtech\\.com", "goshareware\\.com",
                            "nemopdf\\.com", "apowersoft\\.com", "downloaddailymotion\\.com",
                            "free-download-youtube\\.com", "free-music-downloader\\.com",
                            "video-download-capture\\.com", "videograbber\\.net",
                            "recoverwindowspassword\\.com", "password-buster\\.com",
                            "remorecover\\.com", "remosoftware\\.com", "crazybulkreviewsz\\.com"]
    rules = [
        {'regex': u"(?i)\\b(%s)\\b|%s" % ("|".join(bad_keywords), "|".join(bad_keywords_nwb)), 'all': True,
         'sites': [], 'reason': "Bad keyword in {}", 'title': True, 'body': True, 'username': True, 'stripcodeblocks': False, 'body_summary': True},
        {'regex': u"(?i)\\bbaba\\b", 'all': True,
         'sites': [], 'reason': "Bad keyword in {}", 'title': True, 'body': True, 'username': False, 'stripcodeblocks': False, 'body_summary': True},
        {'regex': u"(?i)(>>>>|===>|==>>>)(?s).*http", 'all': True,
         'sites': [], 'reason': "Bad keyword in {}", 'title': True, 'body': True, 'username': True, 'stripcodeblocks': True, 'body_summary': False},
        {'regex': u"
[^\/]*
", 'all': True,
         'sites': [], 'reason': "Nested quote blocks in {}", 'title': False, 'body': True, 'username': False, 'stripcodeblocks': True, 'body_summary': False},
        {'regex': u"(?i)\\b(weight (body ?builder|loo?s[es]|reduction)|muscles?|anti aging|loo?s[es] weight|wrinkles?|diet ?plan)\\b", 'all': True,
         'sites': ["fitness.stackexchange.com", "biology.stackexchange.com"], 'reason': "Bad keyword in {}", 'title': True, 'body': False, 'username': True, 'stripcodeblocks': False, 'body_summary': False},
        {'regex': u"(?i)^(?:(?=.*?\\b(?:online|hd)\\b)(?=.*?(?:free|full|unlimited)).*?movies?\\b|(?=.*?\\b(?:acai|kisn)\\b)(?=.*?care).*products?\\b|(?=.*?packer).*mover)", 'all': True,
         'sites': [], 'reason': "Bad keywords in {}", 'title': True, 'body': False, 'username': True, 'stripcodeblocks': False, 'body_summary': False},
        {'regex': u"\\d(?:_*\\d){9}|\\+?\\d_*\\d[\\s\\-]?(?:_*\\d){8,11}|\\d[ -]?\\d{3}[ -]?\\d{3}[ -]?\\d{4}", 'all': True,
         'sites': ["patents.stackexchange.com"], 'reason': "Phone number detected", 'validation_method': 'check_phone_numbers', 'title': True, 'body': False, 'username': False, 'stripcodeblocks': False, 'body_summary': False},
        {'regex': u"(?i)\\b(nigg(a|er)|asshole|fag|fuck(ing?)?|shit|whore|cunt)s?\\b", 'all': True,
         'sites': [], 'reason': "Offensive {} detected", 'insensitive':True, 'title': True, 'body': True, 'username': False, 'stripcodeblocks': False, 'body_summary': True},
        {'regex': u"(?i)\\b(crap)\\b", 'all': True, 'sites': [], 'reason': "Offensive {} detected", 'insensitive': True, 'title': True, 'body': False, 'username': False, 'stripcodeblocks': False, 'body_summary': False},
        {'regex': ur"^(?=.*\p{upper})\P{lower}*$", 'all': True, 'sites': [], 'reason': "All-caps title", 'title': True, 'body': False, 'username': False, 'stripcodeblocks': False, 'body_summary': False},
        {'regex': u"^(?=.*[0-9])[^a-zA-Z]*$", 'all': True, 'sites': [], 'reason': "Numbers-only title", 'title': True, 'body': False, 'username': False, 'stripcodeblocks': False, 'body_summary': False},
        {'regex': u"https?://[a-zA-Z0-9_.-]+\\.[a-zA-Z]{2,4}(/[a-zA-Z0-9_/?=.-])?", 'all': True,
         'sites': ["stackoverflow.com", "superuser.com", "askubuntu.com"], 'reason': "URL in title", 'title': True, 'body': False, 'username': False, 'stripcodeblocks': False, 'body_summary': False},
        {'regex': u"(?i)(%s)" % "|".join(blacklisted_websites), 'all': True,
         'sites': [], 'reason': "Blacklisted website", 'title': True, 'body': True, 'username': True, 'stripcodeblocks': False, 'body_summary': True},
        {'regex': u"([^\\s_.?!=0-9-])\\1{10,}", 'all': True, 'sites': [], 'reason': "Repeating characters in {}", 'title': True, 'body': True, 'username': False, 'stripcodeblocks': True, 'body_summary': False},
        {'regex': u"(?i)(?P<word>[a-zA-Z]+).*((\\b| )+(?P=word)){5,}", 'all': True, 'sites': [], 'reason': "Repeating words in {}", 'title': True, 'body': True, 'username': False, 'stripcodeblocks': True, 'body_summary': False},
        {'regex': u"^(.)\\1+$", 'all': True, 'sites': [], 'reason': "{} has only one unique char", 'title': True, 'body': True, 'username': False, 'stripcodeblocks': True, 'body_summary': False},
        {'regex': u"[A-z0-9_.%+-]+@[A-z0-9_.%+-]+\.[A-z]{2,4}", 'all': True,
         'sites': ["stackoverflow.com", "superuser.com", "serverfault.com", "askubuntu.com", "webapps.stackexchange.com"], 'reason': "Email in {}", 'title': True, 'body': True, 'username': False, 'stripcodeblocks': True, 'body_summary': False},
        {'regex': u"(?i)(tejveer ?iq|ser?vice pemanas?)", 'all': True, 'sites': [], 'reason': "Blacklisted username", 'title': False, 'body': False, 'username': True, 'stripcodeblocks': False, 'body_summary': False},
        {'regex': u".*
.*", 'all': False, 'sites': ["puzzling.stackexchange.com"], 'reason': 'Code block', 'title': False, 'body': True, 'username': False, 'stripcodeblocks': False, 'report_everywhere': False, 'body_summary': False}
    ]

    @staticmethod
    def test_post(title, body, user_name, site, is_answer, body_is_summary):
        result = []
        for rule in FindSpam.rules:
            body_to_check = body
            if rule['stripcodeblocks']:
                body_to_check = regex.sub("
.*?
", "", body, flags=regex.DOTALL)
                body_to_check = regex.sub(".*?", "", body_to_check, flags=regex.DOTALL)
            if rule['all'] != (site in rule['sites']):
                matched_title = regex.compile(rule['regex'], regex.UNICODE).findall(title)
                matched_username = regex.compile(rule['regex'], regex.UNICODE).findall(user_name)
                matched_body = regex.compile(rule['regex'], regex.UNICODE).findall(body_to_check)
                if matched_title and rule['title']:
                    try:
                        if getattr(FindSpam, "%s" % rule['validation_method'])(matched_title):
                            result.append(rule['reason'])
                    except KeyError:  # There is no special logic for this rule
                        result.append(rule['reason'].replace("{}", "title"))
                if matched_username and rule['username']:
                    try:
                        if getattr(FindSpam, "%s" % rule['validation_method'])(matched_username):
                            result.append(rule['reason'])
                    except KeyError:  # There is no special logic for this rule
                        result.append(rule['reason'].replace("{}", "username"))
                if matched_body and rule['body'] and (not body_is_summary or rule['body_summary']):
                    type_of_post = "answer" if is_answer else "body"
                    try:
                        if getattr(FindSpam, "%s" % rule['validation_method'])(matched_body):
                            result.append(rule['reason'].replace("{}", type_of_post))
                    except KeyError:  # There is no special logic for this rule
                        result.append(rule['reason'].replace("{}", type_of_post))
        return result

    @staticmethod
    def check_phone_numbers(matched):
        test_formats = ["IN", "US", None]
        for phone_number in matched:
            for testf in test_formats:
                try:
                    z = phonenumbers.parse(phone_number, testf)
                    if phonenumbers.is_possible_number(z) and phonenumbers.is_valid_number(z):
                        print "Possible %s, Valid %s, Explain: %s" % (phonenumbers.is_possible_number(z), phonenumbers.is_valid_number(z), z)
                        return True
                except phonenumbers.phonenumberutil.NumberParseException:
                    pass
        return False
<commit_msg>+patch.com<commit_after># -*- coding: utf-8 -*-
import regex
import phonenumbers

class FindSpam:
    bad_keywords = ["baba ?ji", "fifa.*coins?", "fifabay", "Long Path Tool",
                    "fifaodell", "brianfo", "nike", "tosterone", "bajotz",
                    "vashi?k[ae]r[ae]n", "sumer", "kolcak"
                    "porn", "molvi", "judi bola", "ituBola.com", "lost lover'?s?",
                    "acai", "skin care", "rejuvenated skin",
                    "LifeForce", "swtor2credits", "me2.do", "black magic",
                    "bam2u", "Neuro(3X|flexyn|fuse)", "Nutra", "TesteroneXL",
                    "Bowtrol", "Slim ?Genix", "Cleanse EFX",
                    "Forskolin", "Blackline Elite", "TestCore Pro",
                    "Xtreme Antler", "Maxx Test 3000", "orvigomax",
                    "Cheap Wigs?", "jivam", "(Improve )?Brain Power", "Maximum ?Shred",
                    "aging skin", "acne( prone)? skin", "(skin )?eye serum",
                    "skin (serum|eye)", "bagprada", "6611165613" "Apowersoft",
                    "Service Solahart", "junisse",
                    "(fake|original|uk) (passports?|driver'?s? licen[cs]e|ID cards?)",
                    "(support|service|helpline)( phone)? number|1[ -]?[ -]?[ -]?866[ -]?978[ -]?(6819|6762)",
                    "(hotmail|gmail|outlook|yahoo|lexmark (printer)?) ?(password( recovery)?|tech)? ?((customer|technical) (support|service))? (support|contact|telephone|help(line)?|phone) number",
                    "kitchen for sale", "dolphin porn", "pdftoexcelconverter", "keepbrowsersafe", "SpyHunter",
                    "pcerror-fix", "filerepairtool", "combatpcviruses", "SkinCentric",
                    "JobsTribune", "join the illuminati", "Brorsoft", "Remo Recover"]
    bad_keywords_nwb = [u"ಌ", "babyliss", "garcinia", "acai ?berry"]  # "nwb" == "no word boundary"
    blacklisted_websites = ["online ?kelas", "careyourhealths", "wowtoes",
                            "ipubsoft", "orabank", "powerigfaustralia",
                            "cfpchampionship2015playofflive", "optimalstackfacts",
                            "maletestosteronebooster", "x4facts",
                            "tripleeffectseyeserum", "healthcaresup",
                            "garciniacambogiaprofacts", "filerepairforum",
                            "lxwpro-t", "casque-beatsbydre", "tenderpublish",
                            "elliskinantiaging", "funmac", "lovebiscuits",
                            "Eglobalfitness", "musclezx90site", "fifapal",
                            "hits4slim", "screenshot\\.net", "downloadavideo\\.net",
                            "strongmenmuscle", "sh\\.st/", "musclehealthfitness",
                            "svelmeblog", "preply\\.com", "hellofifa",
                            "fifa15online", "wearepropeople.com", "tagwitty",
                            "axsoccertours", "ragednablog", "ios8easyjailbreak",
                            "totalfitnesspoint", "trustessaywriting",
                            "trustmyessay", "faasoft", "besttvshows", "mytechlabs",
                            "giikers", "pagetube", "myenv\\.org",
                            "supertestoboostsfacts", "idolizeadvancedeyeserumfacts", "pelevoniface",
                            "skinphysiciantips", "fifa2coins", "xtrememusclerecoveryrev",
                            "diabacordoesitwork", "thehealthyadvise", "supracleanse350facts",
                            "premiumpureforskolinrev", "hyperglycemiaabout", "dietandhealthguide",
                            "health350", "sourceforge\\.net/projects/freepdftojpgconverter",
                            "pdftoexel\\.wordpress\\.com", "best7th\\.in", "any-data-recovery\\.com",
                            "recoverytoolbox\\.com", "mkmk9", "malwaretips", "intellipaat\\.com",
                            "webbuildersguide\\.com", "idealshare.net", "lankabpoacademy\\.com",
                            "evomailserver\\.com", "gameart\\.net", "any-data-recovery\\.com",
                            "recoverytoolbox\\.com", "sofotex\\.com",
                            "mybloggingmoney\\.com", "windows-techsupport\\.com",
                            "supplementsdeal\\.com", "drivethelife\\.com", "testolimitfacts\\.com",
                            "lafozi\\.com", "hipslimgarcinia\\.com", "open-swiss-bank\\.com",
                            "originalgarciniacambogiafacts", "healthy-weight-loss-tips\\.com",
                            "tenorshare\\.com", "advancedpdfconverter\\.com",
                            "androiddatarecoverypro\\.com", "card-data-recovery\\.com",
                            "excel-password-recovery\\.com", "fix-computer\\.net",
                            "lost-password\\.net", "lostwindowspassword\\.com",
                            "macvideoconverterpro\\.com", "password-master\\.net",
                            "photorecovery-formac\\.com", "powerpointpasswordrecovery\\.net",
                            "rarpasswordunlocker\\.net", "recover-windows-password\\.net",
                            "recoverlostpassword\\.com", "sqlpassword-recovery\\.com",
                            "windows7-password-reset\\.net", "windowspasswordcracker\\.com"
                            "windowspasswordreset\\.net", "windowspasswordsrecovery\\.com",
                            "wordpassword-recovery\\.com", "youtubedownloaderconverter\\.net",
                            "smartpcfixer\\.com", "1fix\\.org", "cardrecoverypro\\.com",
                            "drivertuner\\.com", "easyfix\\.org", "errorsfixer\\.org",
                            "faq800\\.com", "fix1\\.org", "guru4pc\\.net", "howto4pc\\.org",
                            "official-drivers\\.com", "pceasynow\\.com", "photorecoverypro\\.net",
                            "regeasypro\\.com", "registryware\\.org", "smartfixer\\.net",
                            "smartfixer\\.org", "wisefixer\\.com", "wisefixer\\.net",
                            "wiserecovery\\.com", "passwordunlocker\\.com",
                            "password-unlocker\\.com", "passwordtech\\.com", "goshareware\\.com",
                            "nemopdf\\.com", "apowersoft\\.com", "downloaddailymotion\\.com",
                            "free-download-youtube\\.com", "free-music-downloader\\.com",
                            "video-download-capture\\.com", "videograbber\\.net",
                            "recoverwindowspassword\\.com", "password-buster\\.com",
                            "remorecover\\.com", "remosoftware\\.com", "crazybulkreviewsz\\.com",
                            "patch\\.com"]
    rules = [
        {'regex': u"(?i)\\b(%s)\\b|%s" % ("|".join(bad_keywords), "|".join(bad_keywords_nwb)), 'all': True,
         'sites': [], 'reason': "Bad keyword in {}", 'title': True, 'body': True, 'username': True, 'stripcodeblocks': False, 'body_summary': True},
        {'regex': u"(?i)\\bbaba\\b", 'all': True,
         'sites': [], 'reason': "Bad keyword in {}", 'title': True, 'body': True, 'username': False, 'stripcodeblocks': False, 'body_summary': True},
        {'regex': u"(?i)(>>>>|===>|==>>>)(?s).*http", 'all': True,
         'sites': [], 'reason': "Bad keyword in {}", 'title': True, 'body': True, 'username': True, 'stripcodeblocks': True, 'body_summary': False},
        {'regex': u"
[^\/]*
", 'all': True,
         'sites': [], 'reason': "Nested quote blocks in {}", 'title': False, 'body': True, 'username': False, 'stripcodeblocks': True, 'body_summary': False},
        {'regex': u"(?i)\\b(weight (body ?builder|loo?s[es]|reduction)|muscles?|anti aging|loo?s[es] weight|wrinkles?|diet ?plan)\\b", 'all': True,
         'sites': ["fitness.stackexchange.com", "biology.stackexchange.com"], 'reason': "Bad keyword in {}", 'title': True, 'body': False, 'username': True, 'stripcodeblocks': False, 'body_summary': False},
        {'regex': u"(?i)^(?:(?=.*?\\b(?:online|hd)\\b)(?=.*?(?:free|full|unlimited)).*?movies?\\b|(?=.*?\\b(?:acai|kisn)\\b)(?=.*?care).*products?\\b|(?=.*?packer).*mover)", 'all': True,
         'sites': [], 'reason': "Bad keywords in {}", 'title': True, 'body': False, 'username': True, 'stripcodeblocks': False, 'body_summary': False},
        {'regex': u"\\d(?:_*\\d){9}|\\+?\\d_*\\d[\\s\\-]?(?:_*\\d){8,11}|\\d[ -]?\\d{3}[ -]?\\d{3}[ -]?\\d{4}", 'all': True,
         'sites': ["patents.stackexchange.com"], 'reason': "Phone number detected", 'validation_method': 'check_phone_numbers', 'title': True, 'body': False, 'username': False, 'stripcodeblocks': False, 'body_summary': False},
        {'regex': u"(?i)\\b(nigg(a|er)|asshole|fag|fuck(ing?)?|shit|whore|cunt)s?\\b", 'all': True,
         'sites': [], 'reason': "Offensive {} detected", 'insensitive':True, 'title': True, 'body': True, 'username': False, 'stripcodeblocks': False, 'body_summary': True},
        {'regex': u"(?i)\\b(crap)\\b", 'all': True, 'sites': [], 'reason': "Offensive {} detected", 'insensitive': True, 'title': True, 'body': False, 'username': False, 'stripcodeblocks': False, 'body_summary': False},
        {'regex': ur"^(?=.*\p{upper})\P{lower}*$", 'all': True, 'sites': [], 'reason': "All-caps title", 'title': True, 'body': False, 'username': False, 'stripcodeblocks': False, 'body_summary': False},
        {'regex': u"^(?=.*[0-9])[^a-zA-Z]*$", 'all': True, 'sites': [], 'reason': "Numbers-only title", 'title': True, 'body': False, 'username': False, 'stripcodeblocks': False, 'body_summary': False},
        {'regex': u"https?://[a-zA-Z0-9_.-]+\\.[a-zA-Z]{2,4}(/[a-zA-Z0-9_/?=.-])?", 'all': True,
         'sites': ["stackoverflow.com", "superuser.com", "askubuntu.com"], 'reason': "URL in title", 'title': True, 'body': False, 'username': False, 'stripcodeblocks': False, 'body_summary': False},
        {'regex': u"(?i)(%s)" % "|".join(blacklisted_websites), 'all': True,
         'sites': [], 'reason': "Blacklisted website", 'title': True, 'body': True, 'username': True, 'stripcodeblocks': False, 'body_summary': True},
        {'regex': u"([^\\s_.?!=0-9-])\\1{10,}", 'all': True, 'sites': [], 'reason': "Repeating characters in {}", 'title': True, 'body': True, 'username': False, 'stripcodeblocks': True, 'body_summary': False},
        {'regex': u"(?i)(?P<word>[a-zA-Z]+).*((\\b| )+(?P=word)){5,}", 'all': True, 'sites': [], 'reason': "Repeating words in {}", 'title': True, 'body': True, 'username': False, 'stripcodeblocks': True, 'body_summary': False},
        {'regex': u"^(.)\\1+$", 'all': True, 'sites': [], 'reason': "{} has only one unique char", 'title': True, 'body': True, 'username': False, 'stripcodeblocks': True, 'body_summary': False},
        {'regex': u"[A-z0-9_.%+-]+@[A-z0-9_.%+-]+\.[A-z]{2,4}", 'all': True,
         'sites': ["stackoverflow.com", "superuser.com", "serverfault.com", "askubuntu.com", "webapps.stackexchange.com"], 'reason': "Email in {}", 'title': True, 'body': True, 'username': False, 'stripcodeblocks': True, 'body_summary': False},
        {'regex': u"(?i)(tejveer ?iq|ser?vice pemanas?)", 'all': True, 'sites': [], 'reason': "Blacklisted username", 'title': False, 'body': False, 'username': True, 'stripcodeblocks': False, 'body_summary': False},
        {'regex': u".*
.*", 'all': False, 'sites': ["puzzling.stackexchange.com"], 'reason': 'Code block', 'title': False, 'body': True, 'username': False, 'stripcodeblocks': False, 'report_everywhere': False, 'body_summary': False}
    ]

    @staticmethod
    def test_post(title, body, user_name, site, is_answer, body_is_summary):
        result = []
        for rule in FindSpam.rules:
            body_to_check = body
            if rule['stripcodeblocks']:
                body_to_check = regex.sub("
.*?
", "", body, flags=regex.DOTALL)
                body_to_check = regex.sub(".*?", "", body_to_check, flags=regex.DOTALL)
            if rule['all'] != (site in rule['sites']):
                matched_title = regex.compile(rule['regex'], regex.UNICODE).findall(title)
                matched_username = regex.compile(rule['regex'], regex.UNICODE).findall(user_name)
                matched_body = regex.compile(rule['regex'], regex.UNICODE).findall(body_to_check)
                if matched_title and rule['title']:
                    try:
                        if getattr(FindSpam, "%s" % rule['validation_method'])(matched_title):
                            result.append(rule['reason'])
                    except KeyError:  # There is no special logic for this rule
                        result.append(rule['reason'].replace("{}", "title"))
                if matched_username and rule['username']:
                    try:
                        if getattr(FindSpam, "%s" % rule['validation_method'])(matched_username):
                            result.append(rule['reason'])
                    except KeyError:  # There is no special logic for this rule
                        result.append(rule['reason'].replace("{}", "username"))
                if matched_body and rule['body'] and (not body_is_summary or rule['body_summary']):
                    type_of_post = "answer" if is_answer else "body"
                    try:
                        if getattr(FindSpam, "%s" % rule['validation_method'])(matched_body):
                            result.append(rule['reason'].replace("{}", type_of_post))
                    except KeyError:  # There is no special logic for this rule
                        result.append(rule['reason'].replace("{}", type_of_post))
        return result

    @staticmethod
    def check_phone_numbers(matched):
        test_formats = ["IN", "US", None]
        for phone_number in matched:
            for testf in test_formats:
                try:
                    z = phonenumbers.parse(phone_number, testf)
                    if phonenumbers.is_possible_number(z) and phonenumbers.is_valid_number(z):
                        print "Possible %s, Valid %s, Explain: %s" % (phonenumbers.is_possible_number(z), phonenumbers.is_valid_number(z), z)
                        return True
                except phonenumbers.phonenumberutil.NumberParseException:
                    pass
        return False
<|endoftext|>


<commit_before>#!/usr/bin/env python

from main import PKT_DIR_INCOMING, PKT_DIR_OUTGOING
import socket, struct
from bisect import bisect_left
from datetime import datetime

# TODO: Feel free to import any Python standard modules as necessary.
# (http://docs.python.org/2/library/)
# You must NOT use any 3rd-party libraries, though.

class Firewall:
    def __init__(self, config, timer, iface_int, iface_ext):
        self.timer = timer
        self.iface_int = iface_int
        self.iface_ext = iface_ext

        parser = RulesParser(config['rule'])
        self.rules = parser.parse_rules()

        # TODO: Also do some initialization if needed.

    def handle_timer(self):
        # TODO: For the timer feature, refer to bypass.py
        pass

    # @pkt_dir: either PKT_DIR_INCOMING or PKT_DIR_OUTGOING
    # @pkt: the actual data of the IPv4 packet (including IP header)
    def handle_packet(self, pkt_dir, pkt):
        # TODO: Your main firewall code will be here.

        protocol, ext_IP_address, ext_port, is_dns_pkt = self.read_packet(pkt, pkt_dir)

        verdict = self.packet_lookup(protocol, ext_IP_address, ext_port, is_dns_pkt)

        if verdict == "pass":
          if pkt_dir == PKT_DIR_INCOMING:
            self.iface_int.send_ip_packet(pkt)
          else: # pkt_dir == PKT_DIR_OUTGOING
            self.iface_ext.send_ip_packet(pkt)

    # Acts as a parser for the packet
    # Returns the protocol, external IP address, and the external port associated with the packet
    # Also determines whether or not a packet is a DNS packet and returns that as well
    def read_packet(self, pkt, pkt_dir):
      # Need to retrieve the protocol that the packet follows

      if pkt_dir == PKT_DIR_INCOMING:
        pass
        # Retrieve the source IP address and source port of the packet
      else: # pkt_dir == PKT_DIR_OUTGOING
        pass
        # Retrieve the destination IP address and destination port of the packet

      # Determine whether or not the packet is a DNS query

      return protocol, ext_IP_address, ext_port, is_dns_pkt

    # Looks through the self.rules list and returns the verdict of the latest
    # rule in the list that matches the packet fields
    # Returns verdict==True if no rules in the list match the packet fields
    def packet_lookup(self, pkt_protocol, ext_IP_address, ext_port, is_dns_pkt):

    	# Set verdict initially to true so that if no rules match packet fields
    	# then the packet will be passed to the appropriate interface
    	verdict = Truehttp://us-mg6.mail.yahoo.com/neo/launch?.rand=f5d64sgps1s4i#

    	for rule in self.rules:
    		if rule.protocol == pkt_protocol:
    			# Examine rule further since protocol matches (TCP/UDP/ICMP)
    			if
    		elif rule.protocol == "dns" and is_dns_pkt == true:
    			# Examine rule further since it is a DNS rule and packet is dns query

    	return verdict

    # TODO: You can add more methods as you want.

class RulesParser:

  def __init__(self, filename):
    self.filename = filename

  def parse_rules(self):
    f = open(self.filename, 'r')
    rules = []
    for line in f:
      line = line.strip()
      #Ignore All lines that are blank or comment lines
      if line and not self._is_comment_line(line):
        rule = self.parse_line(line)
        if rule:
          rules.append(rule)
    return rules

  def parse_line(self, line):
    tokens = line.split()
    #Make all tokens lowercase so we ignore case sensitivity
    tokens = map(lambda token: token.lower(), tokens)

    #Rules that have 4 fields are normal rules
    if len(tokens) == 4:
      return Rule(*tokens)
    #DNS Rules
    elif len(tokens) == 3:
      return DNSRule(*tokens)
    else:
      return None

  def _is_comment_line(self, line):
    return line.startswith("%")

class GeoDBParser:

  def __init__(self, filename):
    self.filename = filename

  def parse_lines(self):
    f = open(self.filename, 'r')
    nodes = []
    for line in f:
      node = self.parse_line(line)
      nodes.append(node)
    return nodes

  def parse_line(self, line):
    tokens = line.split()
    #Make all tokens lowercase so we ignore case sensitivity
    tokens = map(lambda token: token.lower(), tokens)

    #GeoDB lines should have 3 fields
    if len(tokens) == 3:
      return GeoDBNode(*tokens)
    else:
      return None

class GeoDBNode:

  def __init__(self, start_ip, end_ip, country_code):
    self.start_ip = self.ip_to_int(start_ip)
    self.end_ip = self.ip_to_int(end_ip)
    self.country_code = country_code

  def __lt__(self, other):
    ip = self.ip_to_int(other)
    return self.end_ip < ip

  def __le__(self, other):
    return self.__lt__(other) or self.__eq__(other)

  def __ne__(self, other):
    return not self.__eq__(other)

  def __eq__(self, other):
    ip = self.ip_to_int(other)
    return self.start_ip <= ip and ip <= self.end_ip

  def __gt__(self, other):
    ip = self.ip_to_int(other)
    return self.start_ip > ip

  def __ge__(self, other):
    return self.__gt__(other) or self.__eq__(other)

  def ip_to_int(self, ip):
    #From http://gist/githib.com/cslarsen/1595135
    return reduce(lambda a,b: a<<8 | b, map(int, ip.split(".")))

class Rule:

  def __init__(self, verdict, protocol, ext_IP_address, ext_port):
    self.verdict = verdict
    self.protocol = protocol
    self.ext_IP_address = IPAddressField(ext_IP_address)
    self.ext_port = ExtPortField(ext_port)

class DNSRule(Rule):

  def __init__(self, verdict, protocol, domain_name):
    self.verdict = verdict
    self.protocol = protocol
    self.domain_name = DomainNameField(domain_name)

class IPAddressField:

  geoParser = GeoDBParser('geoipdb.txt')
  geo_nodes = geoParser.parse_lines()

  def __init__(self, ext_IP_address):
  	pass

  def __eq__(self, other):

  #Returns True if the given ip belongs to a certain country. False o/w
  def belongs_to_country(self, ip, country_code):
    geo_node = self._bin_search(ip)
    if geo_node:
      return geo_node.country_code == country_code
    else:
      return False

  #Finds the GeoDBNode within the range of the given IP or None if no node of appropriate range is found
  def _bin_search(self, ip):
    i = bisect_left(self.geo_nodes, ip)
    if i != len(self.geo_nodes):
      node = self.geo_nodes[i]
      if node == ip:
        return node
      else:
        return None
    else:
      return None

  def ip_to_int(self, ip):
    #From http://gist/githib.com/cslarsen/1595135
    return reduce(lambda a,b: a<<8 | b, map(int, ip.split(".")))

class ExtPortField:

  def __init__(self, ext_port):
    pass

  def __eq__(self, other):
    pass

  def _is_integer(self, ext_port):
    try:
      int(ext_port)
      return True
    except ValueError:
      return False

class DomainNameField:

  def __init__(self, domain_name):
    pass

  def __eq__(self, other):
    pass

# TODO: You may want to add more classes/functions as well.
<commit_msg>See diff for details - Missing packet parsing function<commit_after>#!/usr/bin/env python

from main import PKT_DIR_INCOMING, PKT_DIR_OUTGOING
import socket, struct
from bisect import bisect_left
from datetime import datetime

# TODO: Feel free to import any Python standard modules as necessary.
# (http://docs.python.org/2/library/)
# You must NOT use any 3rd-party libraries, though.

class Firewall:
    def __init__(self, config, timer, iface_int, iface_ext):
        self.timer = timer
        self.iface_int = iface_int
        self.iface_ext = iface_ext

        parser = RulesParser(config['rule'])
        self.rules = parser.parse_rules()

        # TODO: Also do some initialization if needed.

    def handle_timer(self):
        # TODO: For the timer feature, refer to bypass.py
        pass

    # @pkt_dir: either PKT_DIR_INCOMING or PKT_DIR_OUTGOING
    # @pkt: the actual data of the IPv4 packet (including IP header)
    def handle_packet(self, pkt_dir, pkt):
        # TODO: Your main firewall code will be here.

        protocol, ext_IP_address, ext_port, is_dns_pkt, domain_name = self.read_packet(pkt, pkt_dir)
        wrapped_packet = Wrap_Packet(protocol, ext_IP_address, ext_port, is_dns_pkt, domain_name)

        verdict = self.packet_lookup(wrapped_packet)

        if verdict == "pass":
          if pkt_dir == PKT_DIR_INCOMING:
            self.iface_int.send_ip_packet(pkt)
          else: # pkt_dir == PKT_DIR_OUTGOING
            self.iface_ext.send_ip_packet(pkt)

    # Acts as a parser for the packet
    # Returns the protocol, external IP address, and the external port associated with the packet
    # Also determines whether or not a packet is a DNS packet and returns that as well
    def read_packet(self, pkt, pkt_dir):
      # Need to retrieve the protocol that the packet follows

      if pkt_dir == PKT_DIR_INCOMING:
        pass
        # Retrieve the source IP address and source port of the packet
      else: # pkt_dir == PKT_DIR_OUTGOING
        pass
        # Retrieve the destination IP address and destination port of the packet

      # Determine whether or not the packet is a DNS query

      return protocol, ext_IP_address, ext_port, is_dns_pkt, domain_name

    # Looks through the self.rules list and returns the verdict of the latest
    # rule in the list that matches the packet fields
    # Returns verdict==True if no rules in the list match the packet fields
    def packet_lookup(self, wrapped_packet):

    	# Set verdict initially to true so that if no rules match packet fields
    	# then the packet will be passed to the appropriate interface
    	verdict = True

    	for rule in self.rules:
    		if rule.protocol == wrapped_packet.protocol:
    			# Examine rule further since rule protocol matches packet protocol (TCP/UDP/ICMP)
    			if wrapped_packet.ext_IP_address == rule.ext_IP_address and wrapped_packet.ext_port == rule.ext_port:
    				verdict = rule.verdict
    		elif rule.protocol == "dns" and wrapped_packet.is_dns_pkt == true:
    			# Examine rule further since it is a DNS rule and packet is dns query
    			# wrapped_packet only has a domain_name field if is_dns_pkt is true for wrapped_packet
    			if wrapped_packet.domain_name == rule.domain_name:
    				verdict = rule.verdict
    	return verdict

    # TODO: You can add more methods as you want.

class RulesParser:

  def __init__(self, filename):
    self.filename = filename

  def parse_rules(self):
    f = open(self.filename, 'r')
    rules = []
    for line in f:
      line = line.strip()
      #Ignore All lines that are blank or comment lines
      if line and not self._is_comment_line(line):
        rule = self.parse_line(line)
        if rule:
          rules.append(rule)
    return rules

  def parse_line(self, line):
    tokens = line.split()
    #Make all tokens lowercase so we ignore case sensitivity
    tokens = map(lambda token: token.lower(), tokens)

    #Rules that have 4 fields are normal rules
    if len(tokens) == 4:
      return Rule(*tokens)
    #DNS Rules
    elif len(tokens) == 3:
      return DNSRule(*tokens)
    else:
      return None

  def _is_comment_line(self, line):
    return line.startswith("%")

class GeoDBParser:

  def __init__(self, filename):
    self.filename = filename

  def parse_lines(self):
    f = open(self.filename, 'r')
    nodes = []
    for line in f:
      node = self.parse_line(line)
      nodes.append(node)
    return nodes

  def parse_line(self, line):
    tokens = line.split()
    #Make all tokens lowercase so we ignore case sensitivity
    tokens = map(lambda token: token.lower(), tokens)

    #GeoDB lines should have 3 fields
    if len(tokens) == 3:
      return GeoDBNode(*tokens)
    else:
      return None

class GeoDBNode:

  def __init__(self, start_ip, end_ip, country_code):
    self.start_ip = self.ip_to_int(start_ip)
    self.end_ip = self.ip_to_int(end_ip)
    self.country_code = country_code

  def __lt__(self, other):
    ip = self.ip_to_int(other)
    return self.end_ip < ip

  def __le__(self, other):
    return self.__lt__(other) or self.__eq__(other)

  def __ne__(self, other):
    return not self.__eq__(other)

  def __eq__(self, other):
    ip = self.ip_to_int(other)
    return self.start_ip <= ip and ip <= self.end_ip

  def __gt__(self, other):
    ip = self.ip_to_int(other)
    return self.start_ip > ip

  def __ge__(self, other):
    return self.__gt__(other) or self.__eq__(other)

  def ip_to_int(self, ip):
    #From http://gist/githib.com/cslarsen/1595135
    return reduce(lambda a,b: a<<8 | b, map(int, ip.split(".")))

class Rule:

  def __init__(self, verdict, protocol, ext_IP_address, ext_port):
    self.verdict = verdict
    self.protocol = protocol
    self.ext_IP_address = IPAddressField(ext_IP_address)
    self.ext_port = ExtPortField(ext_port)

class WrappedPacket:
	def __init__(self, protocol, ext_IP_address, ext_port, is_dns_pkt, domain_name):
		self.protocol = protocol
    self.ext_IP_address = IPAddressField(ext_IP_address)
    self.ext_port = ExtPortField(ext_port)
    self.is_dns_pkt = is_dns_pkt
    if self.is_dns_pkt == True:
    	self.domain_name = DomainNameField(domain_name)

class DNSRule(Rule):

  def __init__(self, verdict, protocol, domain_name):
    self.verdict = verdict
    self.protocol = protocol
    self.domain_name = DomainNameField(domain_name)

class IPAddressField:

  geoParser = GeoDBParser('geoipdb.txt')
  geo_nodes = geoParser.parse_lines()

  def __init__(self, ext_IP_address):
  	self.ext_IP_address = ext_IP_address

  	# Have this set up as a field so that way don't have to
  	# keep determining if IPAddressField obj wraps an IP prefix
  	self.is_IP_prefix = false
  	if "/" in self.ext_IP_address:
  		self.is_IP_prefix = true
  		# Element 0 is the IP, Element 1 is the slash #
  		self._decimal_ip_and_prefix = other.ext_IP_address.split("/")
  		# Ask: any reason to make ip_to_int an instance function?
  		self._decimal_ip = self.ip_to_int(_decimal_ip_and_prefix[0])
  		self.slash_num = _decimal_ip_and_prefix[1]
  		self.relevant_portion = self.relevant_ip_portion(self._decimal_ip, self.slash_num)

  # Assume that the lhs of "==" is always the external IP address
  # of the packet, while the rhs is always the external Ip addr of the
  # rule that you're trying to match up with the packet
  def __eq__(self, other):
  	if other.ext_IP_address == "any":
  		return True
  	elif len(other.ext_IP_address) == 2:
  		# other.ext_IP_address is a 2-byte country code
  		return belongs_to_country(self.ext_IP_address, other.ext_IP_address)
  	elif other.is_IP_prefix:
  		# TODO: Deal with IP prefix case
  		self.decimal_ip = ip_to_int(self.ext_IP_address)
  		return self.relevant_ip_portion(self.decimal_ip, other.slash_num) == other.relevant_portion
  	else:
  		# other.ext_IP_address is just an IP address
  		return self.ext_IP_address == other.ext_IP_address

  #Returns True if the given ip belongs to a certain country. False o/w
  def belongs_to_country(self, ip, country_code):
    geo_node = self._bin_search(ip)
    if geo_node:
      return geo_node.country_code == country_code
    else:
      return False

  #Finds the GeoDBNode within the range of the given IP or None if no node of appropriate range is found
  def _bin_search(self, ip):
    i = bisect_left(self.geo_nodes, ip)
    if i != len(self.geo_nodes):
      node = self.geo_nodes[i]
      if node == ip:
        return node
      else:
        return None
    else:
      return None

  def ip_to_int(self, ip):
    #From http://gist/githib.com/cslarsen/1595135
    return reduce(lambda a,b: a<<8 | b, map(int, ip.split(".")))

  # Return the network component of the inputted IP address zero-extended
  # at the end (for later comparison of network component portions)
  def relevant_ip_portion(self, ip, slash_num):
  	relevant_ip = 0xFFFFFFFF >> (32 - slash_num)
  	relevant_ip = 0xFFFFFFFF << (32 - slash_num)
  	return relevant_ip

class ExtPortField:

  def __init__(self, ext_port):
  	self.ext_port = ext_port
  	self.is_a_range = false
   	if "-" in self.ext_port:
  		self.is_a_range = true
  		self._temp_list = self.ext_port.split("-")
  		self.start_port = self._temp_list[0]
  		self.end_port = self._temp_list[1]

  # Assume that the lhs of "==" is always the external port
  # of the packet, while the rhs is always the external port of the
  # rule that you're trying to match up with the packet
  def __eq__(self, other):
  	if other.ext_port == "any":
  		return True
  	elif other.is_a_range == true:
  		if self.ext_port >= other.start_port && self.ext_port <= other.end_port:
  			return True
  		else:
  			return False
  	else:
  		# other.ext_port should be a single value
  		return self.ext_port == other.ext_port

  def _is_integer(self, ext_port):
    try:
      int(ext_port)
      return True
    except ValueError:
      return False

class DomainNameField:

  def __init__(self, domain_name):
  	self._domain_name = domain_name
  	self._intermediate_list = self._domain_name.split(".")
  	# self.rev_domain_name_list is the result of a string split into
  	# multiple elements by the . delimiter, followed by a reverse oper.
  	# that reverses the elements in the list
  	self.rev_domain_name_list = reversed(self._intermediate_list)

  # Assume that the lhs of "==" is always the domain name
  # of the packet, while the rhs is always the domain name of
  # the rule that you're trying to match up with the packet
  def __eq__(self, other):
  	int i = 0;
  	for partURL in other.rev_domain_name_list:
  		# Assume that DNS query rules only have good syntax
  		if partURL == "*":
  			# Since the current partURL is "*", do not care what the rest of self.rev_domain_name_list is
  			return True
  		else:
  			# partURL is a portion of a url like "gov" or "fda"
  			if partURL != self.rev_domain_name_list[i]:
  				return False
  			else:
  				# partURL portion matches self.self.rev_domain_name_list[i] (e.g. both domain names end in 'gov')
  				i++
  	# Should never reach this part
  	return False

# TODO: You may want to add more classes/functions as well.
<|endoftext|>


<commit_before>#!/usr/bin/env python

from itertools import compress
import os
import re
import shutil
import subprocess as sp
import sys
import multiprocessing as mp
import tempfile

def get_missing_programs(required_programs):
    """Gets a list of required programs that can't be found on the system."""

    # try to launch the programs, and add them to a list if they're not found
    missing = []
    for program in required_programs:
        try:
            sp.call(program, stdout=sp.PIPE, stderr=sp.STDOUT)
        except OSError, e:
            # if the binary couldn't be found, put it in the list
            if e.errno == 2:
                missing.append(program)
            else:
                # propogate other errors
                raise

    return missing

def ensure_directory(d, ignore_errors=False):
    """
    Given a directory, ensures that it exists by creating the directory tree if
    it's not already present. Returns True if the directory was created, False
    if it already exists.
    """

    try:
        os.makedirs(d)
        return True
    except OSError, e:
        # propogate the error if it DOESN'T indicate that the directory already
        # exists.
        if e.errno != 17 and not ignore_errors:
            raise e
        return False

def change_file_ext(fname, ext):
    """Transforms the given filename's extension to the given extension."""
    return os.path.splitext(fname)[0] + ext

def walk_dir(d, follow_links=False):
    """
    Yields all the file names in a given directory, including those in
    subdirectories.  If 'follow_links' is True, symbolic links will be followed.
    This option can lead to infinite looping since the function doesn't keep
    track of which directories have been visited.
    """

    # walk the directory and collect the full path of every file therein
    for root, dirs, files in os.walk(d, followlinks=follow_links):
        for name in files:
            # append the normalized file name
            yield os.path.abspath(os.path.join(root, name))

def get_filetype(fname):
    """Takes a file name and returns its MIME type."""

    # brief output, MIME version
    file_args = ["file", "-b"]
    if sys.platform == "darwin":
        file_args.append("-I")
    else:
        file_args.append("-i")
    file_args.append(fname)

    # return one item per line
    p_file = sp.Popen(file_args, stdout=sp.PIPE)
    return p_file.communicate()[0].strip()

def transcode(infile, outfile=None, skip_existing=False, bad_chars=""):
    """
    Transcodes a single flac file into a single mp3 file.  Preserves the file
    name but changes the extension.  Copies flac tag info from the original file
    to the transcoded file. If outfile is specified, the file is saved to that
    location, otherwise it's saved alongside the original file. If skip_existing
    is False (the default), overwrites existing files with the same name as
    outfile, otherwise skips the file completely. bad_chars is a collection of
    characters that should be removed from the output file name.  Returns the
    returncode of the lame process.
    """

    # get a new file name for the mp3 if no output name was specified
    outfile = outfile or change_file_ext(infile, ".mp3")

    # replace incompatible filename characters in output file
    for c in bad_chars:
        outfile = outfile.replace(c, "")

    # skip transcoding existing files if specified
    if skip_existing and os.path.exists(outfile):
        return

    # use a temp file to store the incremental in-flight transcode
    # this file will be moved to the final output filename when transcode is complete
    # this approach prevents partial or interrupted transcodes from getting in the way of
    # --skip-existing
    # create the temporary file in the same dir (and same filesystem) as the final target.
    # This allows us to use os.link rather than shutil.move later
    temp_outfile = tempfile.NamedTemporaryFile( dir=os.path.dirname(outfile), suffix=".tmp" )

    # get the tags from the input file
    flac_tags = get_tags(infile)

    # arguments for 'lame', including bitrate and tag values
    vbr_quality = 2 # ~190 kbps
    lame_args = ["lame", "-m", "s", "--vbr-new", "-V" + str(vbr_quality),
            "--add-id3v2", "--silent",
            "--tt", flac_tags["TITLE"],
            "--ta", flac_tags["ARTIST"],
            "--tl", flac_tags["ALBUM"],
            "--ty", flac_tags["DATE"],
            "--tc", flac_tags["COMMENT"],
            "--tn", flac_tags["TRACKNUMBER"] + "/" + flac_tags["TRACKTOTAL"],
            "--tg", flac_tags["GENRE"],
            "-", "-" ]

    # arguments for 'flac' decoding to be piped to 'lame'
    flac_args = ["flac", "--silent", "--stdout", "--decode", infile]

    # decode the 'flac' data and pass it to 'lame'
    # pass the lame encoding to our temp file
    p_flac = sp.Popen(flac_args, stdout=sp.PIPE)
    p_lame = sp.Popen(lame_args, stdin=p_flac.stdout, stdout = temp_outfile )

    # allow p_flac to receive a SIGPIPE if p_lame exits
    p_flac.stdout.close()

    # wait for the encoding to finish
    r = p_lame.wait()

    if r == 0:
        # transcode successful
        # link up our temp file to our final filename
        os.link(temp_outfile.name, outfile)
    temp_outfile.close() # deletes temp file reference
    return r

def get_tags(infile):
    """
    Gets the flac tags from the given file and returns them as a dict.  Ensures
    a minimun set of id3v2 tags is available, giving them default values if
    these tags aren't found in the orininal file.
    """

    # get tag info text using 'metaflac'
    metaflac_args = ["metaflac", "--list", "--block-type=VORBIS_COMMENT", infile]
    p_metaflac = sp.Popen(metaflac_args, stdout=sp.PIPE)
    metaflac_text = p_metaflac.communicate()[0]

    # ensure all possible id3v2 tags start off with a default value
    tag_dict = {
        "TITLE": "NONE",
        "ARTIST": "NONE",
        "ALBUM": "NONE",
        "DATE": "1",
        "COMMENT": "",
        "TRACKNUMBER": "00",
        "TRACKTOTAL": "00",
        "GENRE": "NONE"
    }

    # matches all lines like 'comment[0]: TITLE=Misery' and extracts them to
    # tuples like ('TITLE', 'Misery'), then stores them in a dict.
    pattern = "\s+comment\[\d+\]:\s+([^=]+)=([^\n]+)\n"

    # get the comment data from the obtained text
    for t in re.findall(pattern, metaflac_text):
        tag_dict[t[0].upper()] = t[1]

    return tag_dict

if __name__ == "__main__":
    import logging
    import time
    import argparse

    # parse arguments
    parser = argparse.ArgumentParser()
    parser.add_argument("files", metavar="FILES", type=str, nargs="+",
            help="Files and/or directories to transcode")

    # options and flags
    parser.add_argument("-o", "--output-dir", type=os.path.abspath,
            help="Directory to output transcoded files to")
    parser.add_argument("-s", "--skip-existing", action="store_true",
            help="Skip transcoding files if the output file already exists")
    parser.add_argument("-l", "--logfile", type=os.path.normpath, default=None,
            help="log output to a file as well as to the console.")
    parser.add_argument("-q", "--quiet", action="store_true",
            help="Disable console output.")
    parser.add_argument("-c", "--copy-pattern", type=re.compile,
            help="Copy files who's names match the given pattern into the " +
            "output directory. Only works if an output directory is specified.")
    parser.add_argument("-n", "--num-threads", type=int, default=mp.cpu_count(),
            help="The number of threads to use for transcoding. Defaults " +
            "to the number of CPUs on the machine.")
    args = parser.parse_args()

    # set log level and format
    log = logging.getLogger("flac2mp3")
    log.setLevel(logging.INFO)

    # prevent 'no loggers found' warning
    log.addHandler(logging.NullHandler())

    # custom log formatting
    formatter = logging.Formatter("[%(levelname)s] %(message)s")

    # log to stderr unless disabled
    if not args.quiet:
        sh = logging.StreamHandler()
        sh.setFormatter(formatter)
        log.addHandler(sh)

    # add a file handler if specified
    if args.logfile is not None:
        fh = logging.FileHandler(args.logfile)
        fh.setFormatter(formatter)
        log.addHandler(fh)

    # ensure we have all our required programs
    missing = get_missing_programs(["lame", "file", "flac", "metaflac"])
    if len(missing) > 0:
        log.critical("The following programs are required: " + ",".join(missing))
        sys.exit(1)

    # ensure the output directory exists
    if args.output_dir is not None:
        try:
            ensure_directory(args.output_dir)
        except OSError, e:
            log.error("Couldn't create directory '%s'" % args.output_dir)

    # add all the files/directories in the args recursively
    log.info("Enumerating files...")
    files = set()
    for f in args.files:
        if os.path.isdir(f):
            files.update(walk_dir(f))
        else:
            files.add(f)
    log.info("Found " + str(len(files)) + " files")

    # get the common prefix of all the files so we can preserve directory
    # structure when an output directory is specified.
    common_prefix = os.path.dirname(os.path.commonprefix(files))

    def transcode_with_logging(f):
        """Transcode the given file and print out progress statistics."""

        short_fname = os.path.basename(f)

        # copy any non-FLAC files to the output dir if they match a pattern
        if 'audio/x-flac' not in get_filetype(f):
            if args.output_dir is not None and args.copy_pattern is not None:
                match = args.copy_pattern.search(f)
                if match is not None:
                    dest = os.path.join(args.output_dir,
                            f.replace(common_prefix, '').strip('/'))
                    try:
                        ensure_directory(os.path.dirname(dest))
                        shutil.copy(f, dest)
                        log.info("Copied '%s' ('%s' matched)", short_fname,
                                match.group(0))
                    except Exception, e:
                        log.error("Failed to copy '%s' (%s)", short_fname,
                                e.message)

                    # we're done once we've attempted a copy
                    return

            log.info("Skipped '%s'", short_fname)

            # never proceed further if the file wasn't a FLAC file
            return

        # a more compact file name representation
        log.info("Transcoding '%s'..." % short_fname)

        # time the transcode
        start_time = time.time()

        # assign the output directory
        outfile = None
        if args.output_dir is not None:
            mp3file = change_file_ext(f, ".mp3")
            outfile = os.path.join(args.output_dir,
                    mp3file.replace(common_prefix, "").strip("/"))

            # make the directory to ensure it exists. ignore errors since
            # lame takes care of other error messages.
            ensure_directory(os.path.dirname(outfile), ignore_errors=True)

        # store the return code of the process so we can see if it errored
        retcode = transcode(f, outfile, args.skip_existing, ":")
        total_time = time.time() - start_time

        # log success or error
        if retcode == 0:
            log.info("Transcoded '%s' in %.2f seconds" % (short_fname,
                total_time))
        elif retcode == None:
            log.info("Skipped '%s'", short_fname)
        else:
            log.error("Failed to transcode '%s' after %.2f seconds" %
                    (short_fname, total_time))

    # log transcode status
    log.info("Beginning transcode of %d files..." % len(files))
    overall_start_time = time.time()

    # build a thread pool for transcoding
    pool = mp.Pool(processes=args.num_threads)

    # transcode all the found files
    terminated = False
    succeeded = False
    try:
        result = pool.map_async(transcode_with_logging, files)
        while 1:
            try:
                # wait for the result to come in, and mark success once it does
                result.get(0.1)
                succeeded = True
                break
            except mp.TimeoutError:
                continue
    except KeyboardInterrupt:
        terminated = True
        pool.terminate()
        pool.join()
    except Exception, e:
        # catch and log all other exceptions gracefully
        log.exception(e)

    # log our exit status/condition
    overall_time = time.time() - overall_start_time
    if succeeded:
        log.info("Completed transcode in %.2f seconds" % overall_time)
        sys.exit(0)
    elif terminated:
        log.warning("User terminated transcode after %.2f seconds" %
                overall_time)
        sys.exit(3)
    else:
        log.error("Transcode failed after %.2f seconds" % overall_time)
        sys.exit(4)
<commit_msg>Refactoring<commit_after>#!/usr/bin/env python

from itertools import compress
import multiprocessing as mp
import os
import re
import shutil
import subprocess as sp
import sys
import tempfile

def get_missing_programs(required_programs):
    '''Gets a list of required programs that can't be found on the system.'''

    # try to launch the programs, and add them to a list if they're not found
    missing = []
    for program in required_programs:
        try:
            sp.call(program, stdout=sp.PIPE, stderr=sp.STDOUT)
        except OSError, e:
            # if the binary couldn't be found, put it in the list
            if e.errno == 2:
                missing.append(program)
            else:
                # propogate other errors
                raise

    return missing

def ensure_directory(d, ignore_errors=False):
    '''
    Given a directory, ensures that it exists by creating the directory tree if
    it's not already present. Returns True if the directory was created, False
    if it already exists.
    '''

    try:
        os.makedirs(d)
        return True
    except OSError, e:
        # propogate the error if it DOESN'T indicate that the directory already
        # exists.
        if e.errno != 17 and not ignore_errors:
            raise e
        return False

def change_file_ext(fname, ext):
    '''Transforms the given filename's extension to the given extension.'''
    return os.path.splitext(fname)[0] + ext

def walk_dir(d, follow_links=False):
    '''
    Yields all the file names in a given directory, including those in
    subdirectories.  If 'follow_links' is True, symbolic links will be followed.
    This option can lead to infinite looping since the function doesn't keep
    track of which directories have been visited.
    '''

    # walk the directory and collect the full path of every file therein
    for root, dirs, files in os.walk(d, followlinks=follow_links):
        for name in files:
            # append the normalized file name
            yield os.path.abspath(os.path.join(root, name))

def get_filetype(fname):
    '''Takes a file name and returns its MIME type.'''

    # brief output, MIME version
    file_args = ['file', '-b']
    if sys.platform == 'darwin':
        file_args.append('-I')
    else:
        file_args.append('-i')
    file_args.append(fname)

    # return one item per line
    p_file = sp.Popen(file_args, stdout=sp.PIPE)
    return p_file.communicate()[0].strip()

def transcode(infile, outfile=None, skip_existing=False, bad_chars=''):
    '''
    Transcodes a single flac file into a single mp3 file.  Preserves the file
    name but changes the extension.  Copies flac tag info from the original file
    to the transcoded file. If outfile is specified, the file is saved to that
    location, otherwise it's saved alongside the original file. If skip_existing
    is False (the default), overwrites existing files with the same name as
    outfile, otherwise skips the file completely. bad_chars is a collection of
    characters that should be removed from the output file name.  Returns the
    returncode of the lame process.
    '''

    # get a new file name for the mp3 if no output name was specified
    outfile = outfile or change_file_ext(infile, '.mp3')

    # replace incompatible filename characters in output file
    for c in bad_chars:
        outfile = outfile.replace(c, '')

    # skip transcoding existing files if specified
    if skip_existing and os.path.exists(outfile):
        return

    # NOTE: we use a temp file to store the incremental in-flight transcode, and
    # move it to the final output filename when transcode is complete. this
    # approach prevents partial or interrupted transcodes from getting in the
    # way of --skip-existing.

    # create the file in the same dir (and same filesystem) as the final target,
    # allowing us to use os.link rather than shutil.move later.
    dirname = os.path.dirname(outfile)
    with tempfile.NamedTemporaryFile(dir=dirname, suffix='.tmp') as temp_outfile:
        # get the tags from the input file
        flac_tags = get_tags(infile)

        # arguments for 'lame', including bitrate and tag values
        vbr_quality = 2 # ~190 kbps
        lame_args = ['lame', '-m', 's', '--vbr-new', '-V' + str(vbr_quality),
                '--add-id3v2', '--silent',
                '--tt', flac_tags['TITLE'],
                '--ta', flac_tags['ARTIST'],
                '--tl', flac_tags['ALBUM'],
                '--ty', flac_tags['DATE'],
                '--tc', flac_tags['COMMENT'],
                '--tn', flac_tags['TRACKNUMBER'] + '/' + flac_tags['TRACKTOTAL'],
                '--tg', flac_tags['GENRE'],
                '-', '-' ]

        # arguments for 'flac' decoding to be piped to 'lame'
        flac_args = ['flac', '--silent', '--stdout', '--decode', infile]

        # decode the 'flac' data and pass it to 'lame'
        # pass the lame encoding to our temp file
        p_flac = sp.Popen(flac_args, stdout=sp.PIPE)
        p_lame = sp.Popen(lame_args, stdin=p_flac.stdout, stdout=temp_outfile)

        # allow p_flac to receive a SIGPIPE if p_lame exits
        p_flac.stdout.close()

        # wait for the encoding to finish
        retval = p_lame.wait()

        # if the transcode worked, link the temp file to the final filename
        if retval == 0:
            os.link(temp_outfile.name, outfile)

    return retval

def get_tags(infile):
    '''
    Gets the flac tags from the given file and returns them as a dict.  Ensures
    a minimun set of id3v2 tags is available, giving them default values if
    these tags aren't found in the orininal file.
    '''

    # get tag info text using 'metaflac'
    metaflac_args = ['metaflac', '--list', '--block-type=VORBIS_COMMENT', infile]
    p_metaflac = sp.Popen(metaflac_args, stdout=sp.PIPE)
    metaflac_text = p_metaflac.communicate()[0]

    # ensure all possible id3v2 tags start off with a default value
    tag_dict = {
        'TITLE': 'NONE',
        'ARTIST': 'NONE',
        'ALBUM': 'NONE',
        'DATE': '1',
        'COMMENT': '',
        'TRACKNUMBER': '00',
        'TRACKTOTAL': '00',
        'GENRE': 'NONE'
    }

    # matches all lines like 'comment[0]: TITLE=Misery' and extracts them to
    # tuples like ('TITLE', 'Misery'), then stores them in a dict.
    pattern = '\s+comment\[\d+\]:\s+([^=]+)=([^\n]+)\n'

    # get the comment data from the obtained text
    for name, value in re.findall(pattern, metaflac_text):
        tag_dict[name.upper()] = value

    return tag_dict

if __name__ == '__main__':
    import logging
    import time
    import argparse

    # parse arguments
    parser = argparse.ArgumentParser()
    parser.add_argument('files', metavar='FILES', type=str, nargs='+',
            help='Files and/or directories to transcode')

    # options and flags
    parser.add_argument('-o', '--output-dir', type=os.path.abspath,
            help='Directory to output transcoded files to')
    parser.add_argument('-s', '--skip-existing', action='store_true',
            help='Skip transcoding files if the output file already exists')
    parser.add_argument('-l', '--logfile', type=os.path.normpath, default=None,
            help='log output to a file as well as to the console.')
    parser.add_argument('-q', '--quiet', action='store_true',
            help='Disable console output.')
    parser.add_argument('-c', '--copy-pattern', type=re.compile,
            help="Copy files who's names match the given pattern into the " +
            'output directory. Only works if an output directory is specified.')
    parser.add_argument('-n', '--num-threads', type=int, default=mp.cpu_count(),
            help='The number of threads to use for transcoding. Defaults ' +
            'to the number of CPUs on the machine.')
    args = parser.parse_args()

    # set log level and format
    log = logging.getLogger('flac2mp3')
    log.setLevel(logging.INFO)

    # prevent 'no loggers found' warning
    log.addHandler(logging.NullHandler())

    # custom log formatting
    formatter = logging.Formatter('[%(levelname)s] %(message)s')

    # log to stderr unless disabled
    if not args.quiet:
        sh = logging.StreamHandler()
        sh.setFormatter(formatter)
        log.addHandler(sh)

    # add a file handler if specified
    if args.logfile is not None:
        fh = logging.FileHandler(args.logfile)
        fh.setFormatter(formatter)
        log.addHandler(fh)

    # ensure we have all our required programs
    missing = get_missing_programs(['lame', 'file', 'flac', 'metaflac'])
    if len(missing) > 0:
        log.critical('The following programs are required: ' + ','.join(missing))
        sys.exit(1)

    # ensure the output directory exists
    if args.output_dir is not None:
        try:
            ensure_directory(args.output_dir)
        except OSError, e:
            log.error("Couldn't create directory '%s'" % args.output_dir)

    # add all the files/directories in the args recursively
    log.info('Enumerating files...')
    files = set()
    for f in args.files:
        if os.path.isdir(f):
            files.update(walk_dir(f))
        else:
            files.add(f)
    log.info('Found ' + str(len(files)) + ' files')

    # get the common prefix of all the files so we can preserve directory
    # structure when an output directory is specified.
    common_prefix = os.path.dirname(os.path.commonprefix(files))

    def transcode_with_logging(f):
        '''Transcode the given file and print out progress statistics.'''

        short_fname = os.path.basename(f)

        # copy any non-FLAC files to the output dir if they match a pattern
        if 'audio/x-flac' not in get_filetype(f):
            if args.output_dir is not None and args.copy_pattern is not None:
                match = args.copy_pattern.search(f)
                if match is not None:
                    dest = os.path.join(args.output_dir,
                            f.replace(common_prefix, '').strip('/'))
                    try:
                        ensure_directory(os.path.dirname(dest))
                        shutil.copy(f, dest)
                        log.info("Copied '%s' ('%s' matched)", short_fname,
                                match.group(0))
                    except Exception, e:
                        log.error("Failed to copy '%s' (%s)", short_fname,
                                e.message)

                    # we're done once we've attempted a copy
                    return

            log.info("Skipped '%s'", short_fname)

            # never proceed further if the file wasn't a FLAC file
            return

        # a more compact file name representation
        log.info("Transcoding '%s'..." % short_fname)

        # time the transcode
        start_time = time.time()

        # assign the output directory
        outfile = None
        if args.output_dir is not None:
            mp3file = change_file_ext(f, '.mp3')
            outfile = os.path.join(args.output_dir,
                    mp3file.replace(common_prefix, '').strip('/'))

            # make the directory to ensure it exists. ignore errors since
            # lame takes care of other error messages.
            ensure_directory(os.path.dirname(outfile), ignore_errors=True)

        # store the return code of the process so we can see if it errored
        retcode = transcode(f, outfile, args.skip_existing, ':')
        total_time = time.time() - start_time

        # log success or error
        if retcode == 0:
            log.info("Transcoded '%s' in %.2f seconds" % (short_fname,
                total_time))
        elif retcode == None:
            log.info("Skipped '%s'", short_fname)
        else:
            log.error("Failed to transcode '%s' after %.2f seconds" %
                    (short_fname, total_time))

    # log transcode status
    log.info('Beginning transcode of %d files...' % len(files))
    overall_start_time = time.time()

    # build a thread pool for transcoding
    pool = mp.Pool(processes=args.num_threads)

    # transcode all the found files
    terminated = False
    succeeded = False
    try:
        result = pool.map_async(transcode_with_logging, files)
        while 1:
            try:
                # wait for the result to come in, and mark success once it does
                result.get(0.1)
                succeeded = True
                break
            except mp.TimeoutError:
                continue
    except KeyboardInterrupt:
        terminated = True
        pool.terminate()
        pool.join()
    except Exception, e:
        # catch and log all other exceptions gracefully
        log.exception(e)

    # log our exit status/condition
    overall_time = time.time() - overall_start_time
    if succeeded:
        log.info('Completed transcode in %.2f seconds' % overall_time)
        sys.exit(0)
    elif terminated:
        log.warning('User terminated transcode after %.2f seconds' %
                overall_time)
        sys.exit(3)
    else:
        log.error('Transcode failed after %.2f seconds' % overall_time)
        sys.exit(4)
<|endoftext|>