<commit_before># -*- coding: utf-8 -*-
#
# This file is part of Istex_Mental_Rotation.
# Copyright (C) 2016 3ST ERIC Laboratory.
#
# This is a free software; you can redistribute it and/or modify it
# under the terms of the Revised BSD License; see LICENSE file for
# more details.

# Load and transform ISTEX and wiki articles into bag_of_words decomposed by SVD.

# co-author : Lucie Martinet <lucie.martinet@univ-lorraine.fr>
# co-author : Hussein AL-NATSHEH <hussein.al-natsheh@ish-lyon.cnrs.fr.>
# Affiliation: University of Lyon, ERIC Laboratory, Lyon2, University of Lorraine

# Thanks to ISTEX project for the funding

import os, argparse, pickle, json
from sklearn.feature_extraction.text import CountVectorizer, TfidfVectorizer
from sklearn.decomposition import TruncatedSVD
from utils import Paragraphs, Lemmatizer, avg_inner_sim, n_neg_sampling_avg_inner_sim
import IPython

if __name__ == "__main__" :
	parser = argparse.ArgumentParser()
	parser.add_argument("--wiki_dir", default="sample_data/wiki", type=str) # contains wikipedia text files
	parser.add_argument("--istex_dir", default='sample_data/ISTEX/', type=str) # contains .json files
	parser.add_argument("--ucbl_file", default='sample_data/sportArticlesAsIstex_UniqID_182.json', type=str) # is a .json file
	parser.add_argument("--istex_mr_file", default='sample_data/MentalRotationInMetaDataIstexWithoutAnnotated.json', type=str) # is a .json file
	parser.add_argument("--max_nb_wiki", default=100000, type=int) # maximum number of Wikipedia paragraphs to use
	parser.add_argument("--paragraphs_per_article", default=2, type=int) # maximum number of paragraphs to load per article
	parser.add_argument("--vectorizer_type", default="tfidf", type=str) # possible values: "tfidf" and "count", futurework: "doc2vec"
	parser.add_argument("--lemmatizer", default=0, type=int) # for using lemmatization_tokenizer
	parser.add_argument("--mx_ngram", default=2, type=int) # the upper bound of the ngram range
	parser.add_argument("--mn_ngram", default=1, type=int) # the lower bound of the ngram range
	parser.add_argument("--stop_words", default=1, type=int) # filtering out English stop-words
	parser.add_argument("--vec_size", default=150, type=int) # the size of the vector in the semantics space
	parser.add_argument("--min_count", default=20, type=int) # minimum frequency of the token to be included in the vocabulary
	parser.add_argument("--max_df", default=0.95, type=float) # how much vocabulary percent to keep at max based on frequency
	parser.add_argument("--debug", default=0, type=int) # embed IPython to use the decomposed matrix while running
	parser.add_argument("--nb_neg_samplings", default=100, type=int) # number of negative samplings times used for evaluation
	parser.add_argument("--compress", default="pickle", type=str) # for dumping resulted files
	parser.add_argument("--out_dir", default="results", type=str) # name of the output directory

	args = parser.parse_args()
	istex = args.istex_dir
	if istex == "None":
		istex = None
	ucbl = args.ucbl_file
	istex_mr_file = args.istex_mr_file
	if istex_mr_file == "None":
		istex_mr_file = None
	wiki_dir = args.wiki_dir
	if wiki_dir == "None":
		wiki_dir = None
	max_nb_wiki = args.max_nb_wiki
	paragraphs_per_article = args.paragraphs_per_article
	vectorizer_type = args.vectorizer_type
	lemmatizer = args.lemmatizer
	if lemmatizer:
		lemmatizer = Lemmatizer()
	else:
		lemmatizer = None
	mx_ngram = args.mx_ngram
	mn_ngram =  args.mn_ngram
	stop_words = args.stop_words
	if stop_words:
		stop_words = 'english'
	else:
		stop_words = None
	n_components = args.vec_size
	min_count = args.min_count
	max_df = args.max_df
	debug = args.debug
	nb_neg_samplings = args.nb_neg_samplings
	compress = args.compress
	out_dir = args.out_dir

	paragraphs = Paragraphs(istex=istex, ucbl=ucbl, istex_mr=istex_mr_file, wiki=wiki_dir, max_nb_wiki_paragraphs=max_nb_wiki, paragraphs_per_article=paragraphs_per_article)

	if vectorizer_type == "count":
		vectorizer = CountVectorizer(input='content',
			         analyzer='word', stop_words=stop_words, tokenizer=lemmatizer,
			         min_df=min_count,  ngram_range=(mn_ngram, mx_ngram), max_df=max_df)
	elif vectorizer_type == "tfidf":
		vectorizer = TfidfVectorizer(input='content',
			         analyzer='word', stop_words=stop_words, tokenizer=lemmatizer,
			         min_df=min_count,  ngram_range=(mn_ngram, mx_ngram), max_df=max_df)
	else:
		raise NameError('Please check your vectorizer option. It must be either "tfidf" or "count"')

	bow = vectorizer.fit_transform(paragraphs)

	vocab = vectorizer.get_feature_names()
	print 'size of the vocabulary:', len(vocab)

	svd = TruncatedSVD(n_components=n_components, n_iter=5, random_state=42)
	decomposed_bow = svd.fit_transform(bow)
	print 'shape of the SVD decomposed bag_of_words', decomposed_bow.shape
	print 'explained variance ratio sum', svd.explained_variance_ratio_.sum()

	#Dump and clean-up the vectorized bow to free some memory
	if not os.path.exists(out_dir):
		os.makedirs(out_dir)
	try:
		pickle.dump(bow, open(out_dir+'/vectorized_bow.pickle','wb'))
		bow = []
	except:
		try:
			bow.dump(out_dir+'/vectorized_bow.spmat','wb')
			bow = []
		except:
			bow = []

	#Compute the average cosine similarity within ucbl vectors
	if istex_mr_file is not None:
		positives_count = paragraphs.ucbl_count + paragraphs.istex_mr_count
	else:
		positives_count = paragraphs.ucbl_count
	ucbl_lst = decomposed_bow[:positives_count]
	ucbl_inner_sim =  avg_inner_sim(ucbl_lst)
	print 'average cosine similarity within mental rotation articles', ucbl_inner_sim

	#Compute the average cosine similarity within articles randomly selected from ISTEX (other than ucbl)
	neg_inner_sim = n_neg_sampling_avg_inner_sim(decomposed_bow, positives_count, nb_neg_samplings)
	print 'average cosine similarity within articles randomly selected from ISTEX articles (same set size of mental rotation ones)', neg_inner_sim

	#Compute the estimated represeantation quality
	rep_eval = ucbl_inner_sim - neg_inner_sim
	print 'Representation quality based on inner_similarity approach', rep_eval

	print "Now dumping results. This might take several minutes depending on the corpus size..."
	if compress == "pickle":
		try:
			pickle.dump(decomposed_bow, open(out_dir+'/output_svd.pickle','wb'))
		except:
			print "we could not dump the decomposed_bow in pickle. Using pickle instead:"
			try:
				json.dump(decomposed_bow, open(out_dir+'/output_svd.json','wb'))
			except:
				print "could not dump it into json, using numpy dump instead..."
				decomposed_bow.dump(out_dir+'/output_svd.mat')

	elif compress == "json":
		try:
			json.dump(decomposed_bow, open(out_dir+'/output_svd.json','wb'))
		except:
			print "we could not dump the decomposed_bow in json. Using pickle instead:"
			try:
				pickle.dump(decomposed_bow, open(out_dir+'/output_svd.pickle','wb'))
			except:
				print "could not dump it into pickle, using numpy dump instead..."
				decomposed_bow.dump(out_dir+'/output_svd.mat')
	else:
		raise NameError('Unrecognized compress option. Must be either "pickle" or "json"')
	json.dump(paragraphs.index, open(out_dir+'/output_paragraph_index.json','wb'))
	json.dump(paragraphs.inversed_index, open(out_dir+'/output_paragraph_inversed_index.json','wb'))
	print 'size of index', len(paragraphs.index)
	print 'size of inversed_index', len(paragraphs.inversed_index)
	print "done successfully!"

	if debug:
		IPython.embed()
<commit_msg>documentation fix<commit_after># -*- coding: utf-8 -*-
#
# This file is part of Istex_Mental_Rotation.
# Copyright (C) 2017 SSbE ERIC Laboratory.
#
# This is a free software; you can redistribute it and/or modify it
# under the terms of the Revised BSD License; see LICENSE file for
# more details.

# Load and transform ISTEX and wiki articles into bag_of_words decomposed by SVD.

# co-author : Lucie Martinet <lucie.martinet@univ-lorraine.fr>
# co-author : Hussein AL-NATSHEH <hussein.al-natsheh@ish-lyon.cnrs.fr.>
# Affiliation: University of Lyon, ERIC Laboratory, Lyon2, University of Lorraine

# Thanks to ISTEX project for the funding

import os, argparse, pickle, json
from sklearn.feature_extraction.text import CountVectorizer, TfidfVectorizer
from sklearn.decomposition import TruncatedSVD
from utils import Paragraphs, Lemmatizer, avg_inner_sim, n_neg_sampling_avg_inner_sim
import IPython

if __name__ == "__main__" :
	parser = argparse.ArgumentParser()
	parser.add_argument("--wiki_dir", default="sample_data/wiki", type=str) # contains wikipedia text files
	parser.add_argument("--istex_dir", default='sample_data/ISTEX/', type=str) # contains .json files
	parser.add_argument("--ucbl_file", default='sample_data/sportArticlesAsIstex_UniqID_182.json', type=str) # is a .json file
	parser.add_argument("--istex_mr_file", default='sample_data/MentalRotationInMetaDataIstexWithoutAnnotated.json', type=str) # is a .json file
	parser.add_argument("--max_nb_wiki", default=100000, type=int) # maximum number of Wikipedia paragraphs to use
	parser.add_argument("--paragraphs_per_article", default=2, type=int) # maximum number of paragraphs to load per article
	parser.add_argument("--vectorizer_type", default="tfidf", type=str) # possible values: "tfidf" and "count", futurework: "doc2vec"
	parser.add_argument("--lemmatizer", default=0, type=int) # for using lemmatization_tokenizer
	parser.add_argument("--mx_ngram", default=2, type=int) # the upper bound of the ngram range
	parser.add_argument("--mn_ngram", default=1, type=int) # the lower bound of the ngram range
	parser.add_argument("--stop_words", default=1, type=int) # filtering out English stop-words
	parser.add_argument("--vec_size", default=150, type=int) # the size of the vector in the semantics space
	parser.add_argument("--min_count", default=20, type=int) # minimum frequency of the token to be included in the vocabulary
	parser.add_argument("--max_df", default=0.95, type=float) # how much vocabulary percent to keep at max based on frequency
	parser.add_argument("--debug", default=0, type=int) # embed IPython to use the decomposed matrix while running
	parser.add_argument("--nb_neg_samplings", default=100, type=int) # number of negative samplings times used for evaluation
	parser.add_argument("--compress", default="pickle", type=str) # for dumping resulted files
	parser.add_argument("--out_dir", default="results", type=str) # name of the output directory

	args = parser.parse_args()
	istex = args.istex_dir
	if istex == "None":
		istex = None
	ucbl = args.ucbl_file
	istex_mr_file = args.istex_mr_file
	if istex_mr_file == "None":
		istex_mr_file = None
	wiki_dir = args.wiki_dir
	if wiki_dir == "None":
		wiki_dir = None
	max_nb_wiki = args.max_nb_wiki
	paragraphs_per_article = args.paragraphs_per_article
	vectorizer_type = args.vectorizer_type
	lemmatizer = args.lemmatizer
	if lemmatizer:
		lemmatizer = Lemmatizer()
	else:
		lemmatizer = None
	mx_ngram = args.mx_ngram
	mn_ngram =  args.mn_ngram
	stop_words = args.stop_words
	if stop_words:
		stop_words = 'english'
	else:
		stop_words = None
	n_components = args.vec_size
	min_count = args.min_count
	max_df = args.max_df
	debug = args.debug
	nb_neg_samplings = args.nb_neg_samplings
	compress = args.compress
	out_dir = args.out_dir

	paragraphs = Paragraphs(istex=istex, ucbl=ucbl, istex_mr=istex_mr_file, wiki=wiki_dir, max_nb_wiki_paragraphs=max_nb_wiki, paragraphs_per_article=paragraphs_per_article)

	if vectorizer_type == "count":
		vectorizer = CountVectorizer(input='content',
			         analyzer='word', stop_words=stop_words, tokenizer=lemmatizer,
			         min_df=min_count,  ngram_range=(mn_ngram, mx_ngram), max_df=max_df)
	elif vectorizer_type == "tfidf":
		vectorizer = TfidfVectorizer(input='content',
			         analyzer='word', stop_words=stop_words, tokenizer=lemmatizer,
			         min_df=min_count,  ngram_range=(mn_ngram, mx_ngram), max_df=max_df)
	else:
		raise NameError('Please check your vectorizer option. It must be either "tfidf" or "count"')

	bow = vectorizer.fit_transform(paragraphs)

	vocab = vectorizer.get_feature_names()
	print 'size of the vocabulary:', len(vocab)

	svd = TruncatedSVD(n_components=n_components, n_iter=5, random_state=42)
	decomposed_bow = svd.fit_transform(bow)
	print 'shape of the SVD decomposed bag_of_words', decomposed_bow.shape
	print 'explained variance ratio sum', svd.explained_variance_ratio_.sum()

	#Dump and clean-up the vectorized bow to free some memory
	if not os.path.exists(out_dir):
		os.makedirs(out_dir)
	try:
		pickle.dump(bow, open(out_dir+'/vectorized_bow.pickle','wb'))
		bow = []
	except:
		try:
			bow.dump(out_dir+'/vectorized_bow.spmat','wb')
			bow = []
		except:
			bow = []

	#Compute the average cosine similarity within ucbl vectors
	if istex_mr_file is not None:
		positives_count = paragraphs.ucbl_count + paragraphs.istex_mr_count
	else:
		positives_count = paragraphs.ucbl_count
	ucbl_lst = decomposed_bow[:positives_count]
	ucbl_inner_sim =  avg_inner_sim(ucbl_lst)
	print 'average cosine similarity within mental rotation articles', ucbl_inner_sim

	#Compute the average cosine similarity within articles randomly selected from ISTEX (other than ucbl)
	neg_inner_sim = n_neg_sampling_avg_inner_sim(decomposed_bow, positives_count, nb_neg_samplings)
	print 'average cosine similarity within articles randomly selected from ISTEX articles (same set size of mental rotation ones)', neg_inner_sim

	#Compute the estimated represeantation quality
	rep_eval = ucbl_inner_sim - neg_inner_sim
	print 'Representation quality based on inner_similarity approach', rep_eval

	print "Now dumping results. This might take several minutes depending on the corpus size..."
	if compress == "pickle":
		try:
			pickle.dump(decomposed_bow, open(out_dir+'/output_svd.pickle','wb'))
		except:
			print "we could not dump the decomposed_bow in pickle. Using json instead:"
			try:
				json.dump(decomposed_bow, open(out_dir+'/output_svd.json','wb'))
			except:
				print "could not dump it into json, using numpy dump instead..."
				decomposed_bow.dump(out_dir+'/output_svd.mat')

	elif compress == "json":
		try:
			json.dump(decomposed_bow, open(out_dir+'/output_svd.json','wb'))
		except:
			print "we could not dump the decomposed_bow in json. Using pickle instead:"
			try:
				pickle.dump(decomposed_bow, open(out_dir+'/output_svd.pickle','wb'))
			except:
				print "could not dump it into pickle, using numpy dump instead..."
				decomposed_bow.dump(out_dir+'/output_svd.mat')
	else:
		raise NameError('Unrecognized compress option. Must be either "pickle" or "json"')
	json.dump(paragraphs.index, open(out_dir+'/output_paragraph_index.json','wb'))
	json.dump(paragraphs.inversed_index, open(out_dir+'/output_paragraph_inversed_index.json','wb'))
	print 'size of index', len(paragraphs.index)
	print 'size of inversed_index', len(paragraphs.inversed_index)
	print "done successfully!"

	if debug:
		IPython.embed()
<|endoftext|>


<commit_before>import unittest
from typing import List

import utils

class Solution:
    def summaryRanges(self, nums: List[int]) -> List[str]:
        result = []
        i = 0
        while i < len(nums):
            start = nums[i]
            base_val = start - i
            lo = i + 1
            hi = len(nums) - 1
            while lo <= hi:
                mid = (lo + hi) >> 1
                mid_val = nums[mid]
                expected = base_val + mid
                if mid_val > expected:
                    hi = mid - 1
                else:
                    lo = mid + 1
            if hi == i:
                result.append(str(start))
            else:
                result.append(str(start) + '->' + str(base_val + hi))
            i = lo
        return result

class Test(unittest.TestCase):
    def test(self):
        cases = utils.load_test_json(__file__).test_cases

        for case in cases:
            args = str(case.args)
            actual = Solution().summaryRanges(**case.args.__dict__)
            self.assertEqual(case.expected, actual, msg=args)

if __name__ == '__main__':
    unittest.main()
<commit_msg>228. Summary Ranges<commit_after>import unittest
from typing import List

import utils

# Best case: O(log(n)) time, worst case: O(nlog(n)) time. O(1) space. Binary search.
class Solution:
    def summaryRanges(self, nums: List[int]) -> List[str]:
        result = []
        i = 0
        while i < len(nums):
            start = nums[i]
            base_val = start - i
            lo = i + 1
            hi = len(nums) - 1
            while lo <= hi:
                mid = (lo + hi) >> 1
                mid_val = nums[mid]
                expected = base_val + mid
                if mid_val > expected:
                    hi = mid - 1
                else:
                    lo = mid + 1
            if hi == i:
                result.append(str(start))
            else:
                result.append(str(start) + '->' + str(base_val + hi))
            i = lo
        return result

class Test(unittest.TestCase):
    def test(self):
        cases = utils.load_test_json(__file__).test_cases

        for case in cases:
            args = str(case.args)
            actual = Solution().summaryRanges(**case.args.__dict__)
            self.assertEqual(case.expected, actual, msg=args)

if __name__ == '__main__':
    unittest.main()
<|endoftext|>


<commit_before>#!/usr/bin/env python2.7

import re
import struct
import argparse

from emu.cpu import CPU, labeltab, datatab
from emu.instruction import Instruction
from emu.memory import Memory

TYPE_TAB = {
    ".ascii":  None,
    ".asciiz": None,
    ".byte": 1,
    ".space": 1,
    ".halfword": 2,
    ".word": 4
}

def get_imm(imm):
    return int(imm, 16) if imm.startswith('0x') else int(imm)

def parse_data(data_segment):
    """Parses the .text segment source. Populates CPU datatab addresses of
    labels.

    Args:
        text_segment: list of lines, including directive

    Returns:
        bytearray representing data segment in memory, with variables declared
        later towards the beginning (lower in memory)
    """

    dseg = bytearray()
    offset = 0

    for each in data_segment[1:]:
        label, ty, decls = tuple(each.split(None, 2))

        label = label.split(':')[0]
        size = TYPE_TAB[ty]
        decls = map(lambda x: x.strip(), decls.split(','))

        datatab[label] = offset

        for decl in decls:
            if '.ascii' in ty:
                str = decl.split('"')[1]
                dseg += str
                offset += len(str)
                if ty.endswith('z'):
                    dseg += '\x00'
                    offset += 1
            else:
                if ty == '.byte':
                    dseg += struct.pack('<b', get_imm(decl))
                elif ty == '.space':
                    dseg += '\x00'*size
                elif ty == '.halfword':
                    # use big endian because we're going to reverse the whole
                    # bytearray at the end and flip it to little endian
                    dseg += struct.pack('<h', get_imm(decl))
                elif ty == '.word':
                    # re: big endian, see above
                    dseg += struct.pack('<i', get_imm(decl))
                else:
                    raise Exception('bad declaration type')
                offset += size

    # at this point offset contains the complete size of the data section,
    # and we can use that to transform the datatab values from data section
    # offsets to absolute addresses
    for each in datatab:
        datatab[each] = offset - datatab[each]

    dseg.reverse()
    return dseg

def parse_text(text_segment):
    """Parses the .text segment source

    Args:
        text_segment: list of lines, including directive

    Returns:
        list of Instructions
    """

    label_regex = re.compile('^\w+:$')
    processed_labels = 0
    imem = []
    for i, each in enumerate(text_segment[1:]):

        if label_regex.match(each):
            key = each.split(':')[0]
            if key in labeltab:
                raise Exception('label already used')
            # i here indexes into a list containing labels, however those
            # labels will not be present in the final list. to ensure that
            # the imem address stored in labeltab refers to the correct
            # index in the final list, subtract it by the number of labels
            # that occur above the current one which have been artifically
            # increasing the index
            labeltab[key] = i - processed_labels
            processed_labels += 1
        else:
            imem.append(Instruction(each))

    return imem

def generate_memory(source, dmem_size):
    directive_regex = re.compile('^\s*.(data|text)\s*$')
    dmem = Memory(dmem_size)
    imem = []

    # filter out empty lines and comments and strip everything else
    source = map(lambda x: x.strip(),
                 filter(lambda x: x.strip() and not x.strip().startswith('#'),
                        source))

    directs = filter(directive_regex.match, source)

    if '.text' not in directs:
        raise Exception('no .text directive found')
    elif len(directs) > 2 or len(directs) < 1:
        raise Exception('too many/too few segment directives')

    if len(directs) == 1:
        # it's text
        if source[0] != '.text':
            raise Exception('.text directive must be first line')
        imem = parse_text(source)
    else:
        # it's text and data
        first = source.index(directs[0])
        second = source.index(directs[1])
        # we've partitioned but we don't know which is which
        split = source[first:second], source[second:]
        split = split if split[0][0] == '.text' else split[::-1]

        imem = parse_text(split[0])
        dmem.memory = parse_data(split[1]) + dmem.memory

    return dmem, imem

def get_args():
    parser = argparse.ArgumentParser(description='Spym MIPS Interpreter. Starts in interactive shell mode, unless given MIPS source file as argument.')
    parser.add_argument('file', metavar='FILE', type=str,
                        help='MIPS source file', nargs='?')
    parser.add_argument('--memory', type=int, help='Data memory size',
                        default=64)
    return parser.parse_args()

def main():
    args = get_args()

    if args.file:
        with open(args.file) as f:
            source = f.readlines()
        cpu = CPU(*generate_memory(source, args.memory))
        cpu.start()
    else:
        cpu = CPU(Memory(args.memory))
        while True:
            inp = raw_input('spym > ').strip()
            if inp == 'exit':
                break
            elif inp == 'dump':
                cpu.dump()
                continue
            elif not inp:
                continue

            try:
                cpu.execute_single(Instruction(inp))
            except Exception, e:
                if e.message == 'exit syscall':
                    break
                raise e

    cpu.dump()

if __name__ == '__main__':
    main()
<commit_msg>Fix .data section generation<commit_after>#!/usr/bin/env python2.7

import re
import struct
import argparse

from emu.cpu import CPU, labeltab, datatab
from emu.instruction import Instruction
from emu.memory import Memory

TYPE_TAB = {
    ".ascii":  None,
    ".asciiz": None,
    ".byte": 1,
    ".space": 1,
    ".halfword": 2,
    ".word": 4
}

def get_imm(imm):
    return int(imm, 16) if imm.startswith('0x') else int(imm)

def parse_data(data_segment):
    """Parses the .text segment source. Populates CPU datatab addresses of
    labels.

    Args:
        text_segment: list of lines, including directive

    Returns:
        bytearray representing data segment in memory, with variables declared
        later towards the beginning (lower in memory)
    """

    dseg = bytearray()
    offset = 0

    for each in data_segment[1:]:
        label, ty, decls = tuple(each.split(None, 2))

        label = label.split(':')[0]
        size = TYPE_TAB[ty]
        decls = map(lambda x: x.strip(), decls.split(','))

        for decl in decls[::-1]:
            if '.ascii' in ty:
                if ty.endswith('z'):
                    dseg += '\x00'
                    offset += 1
                str = decl.split('"')[1][::-1]
                dseg += str
                offset += len(str)
            else:
                if ty == '.byte':
                    dseg += struct.pack('>b', get_imm(decl))
                elif ty == '.space':
                    dseg += '\x00'*size
                elif ty == '.halfword':
                    # use big endian because we're going to reverse the whole
                    # bytearray at the end and flip it to little endian
                    dseg += struct.pack('>h', get_imm(decl))
                elif ty == '.word':
                    # re: big endian, see above
                    dseg += struct.pack('>i', get_imm(decl))
                else:
                    raise Exception('bad declaration type')
                offset += size

        datatab[label] = offset

    # at this point offset contains the complete size of the data section,
    # and we can use that to transform the datatab values from data section
    # offsets to absolute addresses
    for each in datatab:
        datatab[each] = offset - datatab[each]

    dseg.reverse()
    return dseg

def parse_text(text_segment):
    """Parses the .text segment source

    Args:
        text_segment: list of lines, including directive

    Returns:
        list of Instructions
    """

    label_regex = re.compile('^\w+:$')
    processed_labels = 0
    imem = []
    for i, each in enumerate(text_segment[1:]):

        if label_regex.match(each):
            key = each.split(':')[0]
            if key in labeltab:
                raise Exception('label already used')
            # i here indexes into a list containing labels, however those
            # labels will not be present in the final list. to ensure that
            # the imem address stored in labeltab refers to the correct
            # index in the final list, subtract it by the number of labels
            # that occur above the current one which have been artifically
            # increasing the index
            labeltab[key] = i - processed_labels
            processed_labels += 1
        else:
            imem.append(Instruction(each))

    return imem

def generate_memory(source, dmem_size):
    directive_regex = re.compile('^\s*.(data|text)\s*$')
    dmem = Memory(dmem_size)
    imem = []

    # filter out empty lines and comments and strip everything else
    source = map(lambda x: x.strip(),
                 filter(lambda x: x.strip() and not x.strip().startswith('#'),
                        source))

    directs = filter(directive_regex.match, source)

    if '.text' not in directs:
        raise Exception('no .text directive found')
    elif len(directs) > 2 or len(directs) < 1:
        raise Exception('too many/too few segment directives')

    if len(directs) == 1:
        # it's text
        if source[0] != '.text':
            raise Exception('.text directive must be first line')
        imem = parse_text(source)
    else:
        # it's text and data
        first = source.index(directs[0])
        second = source.index(directs[1])
        # we've partitioned but we don't know which is which
        split = source[first:second], source[second:]
        split = split if split[0][0] == '.text' else split[::-1]

        imem = parse_text(split[0])
        dmem.memory = parse_data(split[1]) + dmem.memory

    return dmem, imem

def get_args():
    parser = argparse.ArgumentParser(description='Spym MIPS Interpreter. Starts in interactive shell mode, unless given MIPS source file as argument.')
    parser.add_argument('file', metavar='FILE', type=str,
                        help='MIPS source file', nargs='?')
    parser.add_argument('--stack', type=int, help='Stack memory size',
                        default=64)
    return parser.parse_args()

def main():
    args = get_args()

    if args.file:
        with open(args.file) as f:
            source = f.readlines()
        cpu = CPU(*generate_memory(source, args.stack))
        cpu.start()
    else:
        cpu = CPU(Memory(args.stack))
        while True:
            inp = raw_input('spym > ').strip()
            if inp == 'exit':
                break
            elif inp == 'dump':
                cpu.dump()
                continue
            elif not inp:
                continue

            try:
                cpu.execute_single(Instruction(inp))
            except Exception, e:
                if e.message == 'exit syscall':
                    break
                raise e

    cpu.dump()

if __name__ == '__main__':
    main()
<|endoftext|>


<commit_before>#!/usr/bin/env python3
import uinput
import time
import json
import threading
import socket
import time
import subprocess
import os
import logging
import argparse

l = None

def hex_to_rgb(value):
    value = value.lstrip('#')
    length = len(value)

    return tuple(int(value[i:i + length // 3], 16) for i in range(0, length, length // 3))

class ButtonEvent():
    UP = "Up"
    RIGHT = "Right"
    DOWN = "Down"
    LEFT = "Left"
    X = "X"
    A = "A"
    B = "B"
    Y = "Y"

    PRESSED = True
    RELEASED = False

    def __init__(self, button, state):
        self.button = button
        self.state = state

    def __repr__(self):
        return self.button + ": " + ("pressed" if self.state else "not pressed")

class C0ntroller():
    JOY_UP_MASK = 0x01
    JOY_RIGHT_MASK = 0x02
    JOY_DOWN_MASK = 0x04
    JOY_LEFT_MASK = 0x08
    BTN_X_MASK = 0x10
    BTN_A_MASK = 0x20
    BTN_B_MASK = 0x40
    BTN_Y_MASK = 0x80

    EVENTS = (
        uinput.BTN_0,
        uinput.BTN_1,
        uinput.BTN_2,
        uinput.BTN_3,
        uinput.ABS_X + (0, 255, 0, 0),
        uinput.ABS_Y + (0, 255, 0, 0))

    def __init__(self, ip, ns, ns_port, callback=None):
        self.ip = ip
        self.ns = ns
        self.ns_port = ns_port
        self.name = None
        self.dev = None
        self.callback = callback
        self.refresh_time = time.time()
        self.prev_event = 0x00

        if callback == None:
            self.init_new_device("{0}:{1}".format(self.ip, self.ip))

        self.name_thread = threading.Thread(target=self.listen_for_name, args=())
        self.name_thread.start()

    def init_new_device(self, name):
        self.dev = None
        self.dev = uinput.Device(self.EVENTS, name)

        # sync joystick to center
        self.dev.emit(uinput.ABS_X, 128, syn=False)
        self.dev.emit(uinput.ABS_Y, 128)

    def listen_for_name(self):
        while True:
            try:
                l.debug("Connecting to nameserver {0} on port {1} for ip {2}".format(self.ns, self.ns_port, self.ip))

                sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
                sock.connect((self.ns, self.ns_port))

                sock.send(bytes(self.ip, "utf-8"))

                while True:
                    self.name = str(sock.recv(1024), "utf-8")

                    # wait for a string
                    l.debug("The device {0} got a new name: {1}".format(self.ip, self.name))

                    if self.callback == None:
                        self.init_new_device("{0}:{1}".format(self.name, self.ip))

                    time.sleep(1)
            except:
                l.error("Lost connection to the nameserver for ip {0}, I will try to reconnect".format(self.ip))
                time.sleep(5)

    def refresh(self):
        self.refresh_time = time.time()

    def dead_time(self):
        return time.time() - self.refresh_time

    def fire(self, event):
        event_diff = event ^ self.prev_event

        if event_diff != 0:
            if self.callback != None:
                event_list = []

                if event_diff & self.JOY_UP_MASK != 0:
                    event_list.append(ButtonEvent(ButtonEvent.UP, event & self.JOY_UP_MASK != 0))

                if event_diff & self.JOY_RIGHT_MASK != 0:
                    event_list.append(ButtonEvent(ButtonEvent.RIGHT, event & self.JOY_RIGHT_MASK != 0))

                if event_diff & self.JOY_DOWN_MASK != 0:
                    event_list.append(ButtonEvent(ButtonEvent.DOWN, event & self.JOY_DOWN_MASK != 0))

                if event_diff & self.JOY_LEFT_MASK != 0:
                    event_list.append(ButtonEvent(ButtonEvent.LEFT, event & self.JOY_LEFT_MASK != 0))

                if event_diff & self.BTN_X_MASK != 0:
                    event_list.append(ButtonEvent(ButtonEvent.X, event & self.BTN_X_MASK != 0))

                if event_diff & self.BTN_A_MASK != 0:
                    event_list.append(ButtonEvent(ButtonEvent.A, event & self.BTN_A_MASK != 0))

                if event_diff & self.BTN_B_MASK != 0:
                    event_list.append(ButtonEvent(ButtonEvent.B, event & self.BTN_B_MASK != 0))

                if event_diff & self.BTN_Y_MASK != 0:
                    event_list.append(ButtonEvent(ButtonEvent.Y, event & self.BTN_Y_MASK != 0))

                self.callback(self.name, event_list)
            else:
                if event_diff & (self.JOY_UP_MASK | self.JOY_DOWN_MASK) != 0:
                    if event & self.JOY_UP_MASK != 0 and not event & self.JOY_DOWN_MASK != 0:
                        self.dev.emit(uinput.ABS_Y, 0)
                    elif event & self.JOY_DOWN_MASK != 0 and not event & self.JOY_UP_MASK != 0:
                        self.dev.emit(uinput.ABS_Y, 255)
                    else:
                        self.dev.emit(uinput.ABS_Y, 128)

                if event_diff & (self.JOY_LEFT_MASK | self.JOY_RIGHT_MASK) != 0:
                    if event & self.JOY_LEFT_MASK != 0 and not event & self.JOY_RIGHT_MASK != 0:
                        self.dev.emit(uinput.ABS_X, 0)
                    elif event & self.JOY_RIGHT_MASK != 0 and not event & self.JOY_LEFT_MASK != 0:
                        self.dev.emit(uinput.ABS_X, 255)
                    else:
                        self.dev.emit(uinput.ABS_X, 128)

                if event_diff & self.BTN_X_MASK != 0:
                    if event & self.BTN_X_MASK != 0:
                        self.dev.emit(uinput.BTN_0, 1)
                    else:
                        self.dev.emit(uinput.BTN_0, 0)

                if event_diff & self.BTN_A_MASK != 0:
                    if event & self.BTN_A_MASK != 0:
                        self.dev.emit(uinput.BTN_1, 1)
                    else:
                        self.dev.emit(uinput.BTN_1, 0)

                if event_diff & self.BTN_B_MASK != 0:
                    if event & self.BTN_B_MASK != 0:
                        self.dev.emit(uinput.BTN_2, 1)
                    else:
                        self.dev.emit(uinput.BTN_2, 0)

                if event_diff & self.BTN_Y_MASK != 0:
                    if event & self.BTN_Y_MASK != 0:
                        self.dev.emit(uinput.BTN_3, 1)
                    else:
                        self.dev.emit(uinput.BTN_3, 0)

                self.dev.syn()

        self.prev_event = event

class S3rver(object):
    HOST = "0.0.0.0"

    MSG_SIZE = 6

    CMD_KEEP_ALIVE = 0x10
    CMD_BUTTONS_CHANGED = 0x11

    CMD_SET_LED_LEFT = 0x20
    CMD_SET_LED_RIGHT = 0x21

    CMD_PROPAGATE_HOST = 0x30
    CMD_ASK_HOST = 0x31

    def __init__(self, port, config=None, callback=None):
        self.port = port
        self.controllers = {}

        no_config_present = False

        if config:
            try:
                with open(config) as config_file:
                    config_data = json.load(config_file)

                    l.info("Loaded config file {0}...".format(config))

                    self.color = hex_to_rgb(config_data["color"])
                    self.timeout = config_data["timeout"]
                    self.ns = config_data["nameserver"]["host"]
                    self.ns_port = config_data["nameserver"]["port"]
            except:
                no_config_present = True
                l.error("Error loading config file {0}...".format(config))
        else:
            no_config_present = True

        if no_config_present:
            self.color = (0xFF, 0xFF, 0xFF)
            self.timeout = 10
            self.ns = "localhost"
            self.ns_port = 1338

        self.callback = callback

    def propagate_host(self):
        l.debug("Propagating the new game server (me)...")
        set_host_msg = bytes([ 0x30, self.color[0], self.color[1], self.color[2], 0xFF, 0x30 ])
        self.sock.sendto(set_host_msg, ("<broadcast>", self.port))

    def serve(self):
        l.info("Starting server on port {0}...".format(self.port))

        self.sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
        self.sock.setsockopt(socket.SOL_SOCKET, socket.SO_BROADCAST, True)
        self.sock.settimeout(self.timeout)

        self.server_addr = (self.HOST, self.port)
        self.sock.bind(self.server_addr)

        self.propagate_host()

        while True:
            try:
                data, client_addr = self.sock.recvfrom(self.MSG_SIZE)

                l.debug("Received data from {0}: ".format(client_addr[0]) + str(data))

                if len(data) != self.MSG_SIZE:
                    l.warn("{0}: Not a valid command: wrong packet length!".format(client_addr))
                    continue

                if data[0] != data[self.MSG_SIZE - 1]:
                    l.warn("{0}: Not a valid command: first byte and last byte mismatching!".format(client_addr))
                    continue

                if data[0] == self.CMD_PROPAGATE_HOST:
                    continue

                if data[0] in [ self.CMD_KEEP_ALIVE, self.CMD_BUTTONS_CHANGED, self.CMD_ASK_HOST ]:
                    if client_addr in self.controllers:
                        self.controllers[client_addr].refresh()
                    else:
                        l.info("New client {0} connected!".format(client_addr[0]))
                        self.controllers[client_addr] = C0ntroller(client_addr[0], self.ns, self.ns_port, self.callback)

                if data[0] == self.CMD_KEEP_ALIVE:
                    self.controllers[client_addr].fire(data[1])
                elif data[0] == self.CMD_BUTTONS_CHANGED:
                    self.controllers[client_addr].fire(data[1])
                elif data[0] == self.CMD_ASK_HOST:
                    l.debug("Client asked for a game server.")
                    self.propagate_host()
            except socket.timeout:
                pass

            clients_to_kick = []

            for client_addr in self.controllers:
                if self.controllers[client_addr].dead_time() > self.timeout:
                    clients_to_kick.append(client_addr)

            for client_addr in clients_to_kick:
                l.info("Kicked client {0}! (timeout)".format(client_addr[0]))
                del self.controllers[client_addr]

def start_srvr(verbose=False, port=1337, config="srvr.cnfg", callback=None):
    global l

    formatter = logging.Formatter("%(asctime)s # %(levelname)-8s %(message)s")

    handler = logging.StreamHandler()
    handler.setFormatter(formatter)

    l = logging.getLogger()
    l.addHandler(handler)

    if verbose:
        l.setLevel(logging.DEBUG)
    else:
        l.setLevel(logging.INFO)

    l.info("Loading uinput kernel module, if it has not been loaded yet...")
    dev_null = open(os.devnull, "w")
    result = subprocess.call([ "modprobe", "uinput"], stdout=dev_null, stderr=dev_null)
    dev_null.close()

    if result == 0:
        server = S3rver(port, config, callback)
        thread = threading.Thread(target=server.serve, args=())
        thread.daemon = True
        thread.start()
    else:
        l.error("Could not load uinput kernel module! Am I root?")

def main():
    p = argparse.ArgumentParser()
    p.add_argument("-v", "--verbose", help="increase output verbosity", action="store_true")
    p.add_argument("-p", "--port", type=int, default=1337, help="use another port")
    p.add_argument("-c", "--config", type=str, default="srvr.cnfg", help="use another config file")

    args = p.parse_args()

    start_srvr(args.verbose, args.port, args.config)

    while True:
        time.sleep(1)

if __name__ == "__main__":
    main()
<commit_msg>Remove stupid output when there is no name server<commit_after>#!/usr/bin/env python3
import uinput
import time
import json
import threading
import socket
import time
import subprocess
import os
import logging
import argparse

l = None

def hex_to_rgb(value):
    value = value.lstrip('#')
    length = len(value)

    return tuple(int(value[i:i + length // 3], 16) for i in range(0, length, length // 3))

class ButtonEvent():
    UP = "Up"
    RIGHT = "Right"
    DOWN = "Down"
    LEFT = "Left"
    X = "X"
    A = "A"
    B = "B"
    Y = "Y"

    PRESSED = True
    RELEASED = False

    def __init__(self, button, state):
        self.button = button
        self.state = state

    def __repr__(self):
        return self.button + ": " + ("pressed" if self.state else "not pressed")

class C0ntroller():
    JOY_UP_MASK = 0x01
    JOY_RIGHT_MASK = 0x02
    JOY_DOWN_MASK = 0x04
    JOY_LEFT_MASK = 0x08
    BTN_X_MASK = 0x10
    BTN_A_MASK = 0x20
    BTN_B_MASK = 0x40
    BTN_Y_MASK = 0x80

    EVENTS = (
        uinput.BTN_0,
        uinput.BTN_1,
        uinput.BTN_2,
        uinput.BTN_3,
        uinput.ABS_X + (0, 255, 0, 0),
        uinput.ABS_Y + (0, 255, 0, 0))

    def __init__(self, ip, ns, ns_port, callback=None):
        self.ip = ip
        self.ns = ns
        self.ns_port = ns_port
        self.name = None
        self.dev = None
        self.callback = callback
        self.refresh_time = time.time()
        self.prev_event = 0x00

        if callback == None:
            self.init_new_device("{0}:{1}".format(self.ip, self.ip))

        self.name_thread = threading.Thread(target=self.listen_for_name, args=())
        self.name_thread.start()

    def init_new_device(self, name):
        self.dev = None
        self.dev = uinput.Device(self.EVENTS, name)

        # sync joystick to center
        self.dev.emit(uinput.ABS_X, 128, syn=False)
        self.dev.emit(uinput.ABS_Y, 128)

    def listen_for_name(self):
        while True:
            try:
                sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
                sock.connect((self.ns, self.ns_port))

                l.debug("Connected to nameserver {0} on port {1} for ip {2}".format(self.ns, self.ns_port, self.ip))

                sock.send(bytes(self.ip, "utf-8"))

                while True:
                    self.name = str(sock.recv(1024), "utf-8")

                    # wait for a string
                    l.debug("The device {0} got a new name: {1}".format(self.ip, self.name))

                    if self.callback == None:
                        self.init_new_device("{0}:{1}".format(self.name, self.ip))

                    time.sleep(1)
            except:
                time.sleep(5)

    def refresh(self):
        self.refresh_time = time.time()

    def dead_time(self):
        return time.time() - self.refresh_time

    def fire(self, event):
        event_diff = event ^ self.prev_event

        if event_diff != 0:
            if self.callback != None:
                event_list = []

                if event_diff & self.JOY_UP_MASK != 0:
                    event_list.append(ButtonEvent(ButtonEvent.UP, event & self.JOY_UP_MASK != 0))

                if event_diff & self.JOY_RIGHT_MASK != 0:
                    event_list.append(ButtonEvent(ButtonEvent.RIGHT, event & self.JOY_RIGHT_MASK != 0))

                if event_diff & self.JOY_DOWN_MASK != 0:
                    event_list.append(ButtonEvent(ButtonEvent.DOWN, event & self.JOY_DOWN_MASK != 0))

                if event_diff & self.JOY_LEFT_MASK != 0:
                    event_list.append(ButtonEvent(ButtonEvent.LEFT, event & self.JOY_LEFT_MASK != 0))

                if event_diff & self.BTN_X_MASK != 0:
                    event_list.append(ButtonEvent(ButtonEvent.X, event & self.BTN_X_MASK != 0))

                if event_diff & self.BTN_A_MASK != 0:
                    event_list.append(ButtonEvent(ButtonEvent.A, event & self.BTN_A_MASK != 0))

                if event_diff & self.BTN_B_MASK != 0:
                    event_list.append(ButtonEvent(ButtonEvent.B, event & self.BTN_B_MASK != 0))

                if event_diff & self.BTN_Y_MASK != 0:
                    event_list.append(ButtonEvent(ButtonEvent.Y, event & self.BTN_Y_MASK != 0))

                self.callback(self.name, event_list)
            else:
                if event_diff & (self.JOY_UP_MASK | self.JOY_DOWN_MASK) != 0:
                    if event & self.JOY_UP_MASK != 0 and not event & self.JOY_DOWN_MASK != 0:
                        self.dev.emit(uinput.ABS_Y, 0)
                    elif event & self.JOY_DOWN_MASK != 0 and not event & self.JOY_UP_MASK != 0:
                        self.dev.emit(uinput.ABS_Y, 255)
                    else:
                        self.dev.emit(uinput.ABS_Y, 128)

                if event_diff & (self.JOY_LEFT_MASK | self.JOY_RIGHT_MASK) != 0:
                    if event & self.JOY_LEFT_MASK != 0 and not event & self.JOY_RIGHT_MASK != 0:
                        self.dev.emit(uinput.ABS_X, 0)
                    elif event & self.JOY_RIGHT_MASK != 0 and not event & self.JOY_LEFT_MASK != 0:
                        self.dev.emit(uinput.ABS_X, 255)
                    else:
                        self.dev.emit(uinput.ABS_X, 128)

                if event_diff & self.BTN_X_MASK != 0:
                    if event & self.BTN_X_MASK != 0:
                        self.dev.emit(uinput.BTN_0, 1)
                    else:
                        self.dev.emit(uinput.BTN_0, 0)

                if event_diff & self.BTN_A_MASK != 0:
                    if event & self.BTN_A_MASK != 0:
                        self.dev.emit(uinput.BTN_1, 1)
                    else:
                        self.dev.emit(uinput.BTN_1, 0)

                if event_diff & self.BTN_B_MASK != 0:
                    if event & self.BTN_B_MASK != 0:
                        self.dev.emit(uinput.BTN_2, 1)
                    else:
                        self.dev.emit(uinput.BTN_2, 0)

                if event_diff & self.BTN_Y_MASK != 0:
                    if event & self.BTN_Y_MASK != 0:
                        self.dev.emit(uinput.BTN_3, 1)
                    else:
                        self.dev.emit(uinput.BTN_3, 0)

                self.dev.syn()

        self.prev_event = event

class S3rver(object):
    HOST = "0.0.0.0"

    MSG_SIZE = 6

    CMD_KEEP_ALIVE = 0x10
    CMD_BUTTONS_CHANGED = 0x11

    CMD_SET_LED_LEFT = 0x20
    CMD_SET_LED_RIGHT = 0x21

    CMD_PROPAGATE_HOST = 0x30
    CMD_ASK_HOST = 0x31

    def __init__(self, port, config=None, callback=None):
        self.port = port
        self.controllers = {}

        no_config_present = False

        if config:
            try:
                with open(config) as config_file:
                    config_data = json.load(config_file)

                    l.info("Loaded config file {0}...".format(config))

                    self.color = hex_to_rgb(config_data["color"])
                    self.timeout = config_data["timeout"]
                    self.ns = config_data["nameserver"]["host"]
                    self.ns_port = config_data["nameserver"]["port"]
            except:
                no_config_present = True
                l.error("Error loading config file {0}...".format(config))
        else:
            no_config_present = True

        if no_config_present:
            self.color = (0xFF, 0xFF, 0xFF)
            self.timeout = 10
            self.ns = "localhost"
            self.ns_port = 1338

        self.callback = callback

    def propagate_host(self):
        l.debug("Propagating the new game server (me)...")
        set_host_msg = bytes([ 0x30, self.color[0], self.color[1], self.color[2], 0xFF, 0x30 ])
        self.sock.sendto(set_host_msg, ("<broadcast>", self.port))

    def serve(self):
        l.info("Starting server on port {0}...".format(self.port))

        self.sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
        self.sock.setsockopt(socket.SOL_SOCKET, socket.SO_BROADCAST, True)
        self.sock.settimeout(self.timeout)

        self.server_addr = (self.HOST, self.port)
        self.sock.bind(self.server_addr)

        self.propagate_host()

        while True:
            try:
                data, client_addr = self.sock.recvfrom(self.MSG_SIZE)

                l.debug("Received data from {0}: ".format(client_addr[0]) + str(data))

                if len(data) != self.MSG_SIZE:
                    l.warn("{0}: Not a valid command: wrong packet length!".format(client_addr))
                    continue

                if data[0] != data[self.MSG_SIZE - 1]:
                    l.warn("{0}: Not a valid command: first byte and last byte mismatching!".format(client_addr))
                    continue

                if data[0] == self.CMD_PROPAGATE_HOST:
                    continue

                if data[0] in [ self.CMD_KEEP_ALIVE, self.CMD_BUTTONS_CHANGED, self.CMD_ASK_HOST ]:
                    if client_addr in self.controllers:
                        self.controllers[client_addr].refresh()
                    else:
                        l.info("New client {0} connected!".format(client_addr[0]))
                        self.controllers[client_addr] = C0ntroller(client_addr[0], self.ns, self.ns_port, self.callback)

                if data[0] == self.CMD_KEEP_ALIVE:
                    self.controllers[client_addr].fire(data[1])
                elif data[0] == self.CMD_BUTTONS_CHANGED:
                    self.controllers[client_addr].fire(data[1])
                elif data[0] == self.CMD_ASK_HOST:
                    l.debug("Client asked for a game server.")
                    self.propagate_host()
            except socket.timeout:
                pass

            clients_to_kick = []

            for client_addr in self.controllers:
                if self.controllers[client_addr].dead_time() > self.timeout:
                    clients_to_kick.append(client_addr)

            for client_addr in clients_to_kick:
                l.info("Kicked client {0}! (timeout)".format(client_addr[0]))
                del self.controllers[client_addr]

def start_srvr(verbose=False, port=1337, config="srvr.cnfg", callback=None):
    global l

    formatter = logging.Formatter("%(asctime)s # %(levelname)-8s %(message)s")

    handler = logging.StreamHandler()
    handler.setFormatter(formatter)

    l = logging.getLogger()
    l.addHandler(handler)

    if verbose:
        l.setLevel(logging.DEBUG)
    else:
        l.setLevel(logging.INFO)

    l.info("Loading uinput kernel module, if it has not been loaded yet...")
    dev_null = open(os.devnull, "w")
    result = subprocess.call([ "modprobe", "uinput"], stdout=dev_null, stderr=dev_null)
    dev_null.close()

    if result == 0:
        server = S3rver(port, config, callback)
        thread = threading.Thread(target=server.serve, args=())
        thread.daemon = True
        thread.start()
    else:
        l.error("Could not load uinput kernel module! Am I root?")

def main():
    p = argparse.ArgumentParser()
    p.add_argument("-v", "--verbose", help="increase output verbosity", action="store_true")
    p.add_argument("-p", "--port", type=int, default=1337, help="use another port")
    p.add_argument("-c", "--config", type=str, default="srvr.cnfg", help="use another config file")

    args = p.parse_args()

    start_srvr(args.verbose, args.port, args.config)

    while True:
        time.sleep(1)

if __name__ == "__main__":
    main()
<|endoftext|>


<commit_before>#!/usr/bin/env python

import pygame
from pygame.locals import *
from pygame.color import *
import pymunk as pm
import math, sys, random
X,Y,Z = 0,1,2 # Easy indexing

class Player(object):
    def __init__(self, thigh_muscle, calf_muscle, direction):
        self.thigh_muscle = thigh_muscle
        self.calf_muscle = calf_muscle
        self.direction = direction
        self.stretch = (math.pi/4)
        self.angle = 0
        self.score = 0

    def updateMuscles(self):
        self.thigh_muscle.rest_angle = (self.angle + self.stretch) * self.direction
        self.calf_muscle.rest_angle = (-self.stretch*2) * self.direction

def to_pygame(p):
    """Small hack to convert pymunk to pygame coordinates"""
    return int(p.x), int(-p.y+600)

def main():
    pygame.init()
    screen = pygame.display.set_mode((600, 600))
    pygame.display.set_caption("Fight!")
    clock = pygame.time.Clock()
    running = True

    ### Physics stuff
    pm.init_pymunk()
    space = pm.Space()
    space.gravity = (0.0, -900.0)

    space.resize_static_hash()
    space.resize_active_hash()

    ### static stuff
    ground_body = pm.Body(pm.inf, pm.inf)
    ground_body.position = (300,40)

    ground = pm.Segment(ground_body, (-1000,0), (1000,0), 5.0)
    ground.friction = 0.6
    ground.color = THECOLORS['black']
    mat = pm.Segment(ground_body, (-200,5), (200,5), 5.0)
    mat.friction = 0.6
    mat.color = THECOLORS['red']
    space.add(ground, mat)

    ### The moving L shape
    l_head = (25, (-50, 55))
    l_shoulder = (-50, 30)
    l_hip = (-50,0)

    r_head = (25, (50, 55))
    r_shoulder = (50, 30)
    r_hip = (50,0)
    r_knee = (30,-20)
    r_foot = (50,-40)

    body = pm.Body(10,10000)

    LEG_LEN = 35

    left_thigh = pm.Body(10, 10000)
    left_calf = pm.Body(10, 10000)

    right_thigh = pm.Body(10, 10000)
    right_calf = pm.Body(10, 10000)

    lines = [pm.Segment(body, l_shoulder, l_hip, 5.0)
            ,pm.Segment(body, r_shoulder, r_hip, 5.0)
            ,pm.Segment(body, r_shoulder, l_hip, 5.0)
            ,pm.Segment(body, l_shoulder, r_hip, 5.0)

            ,pm.Segment(left_thigh, (0,0), (0,-LEG_LEN), 5.0)
            ,pm.Segment(left_calf, (0,0), (0,-LEG_LEN), 5.0)
            ,pm.Segment(right_thigh, (0,0), (0,-LEG_LEN), 5.0)
            ,pm.Segment(right_calf, (0,0), (0,-LEG_LEN), 5.0)
    ]

    left_head = pm.Circle(body, *l_head)
    right_head = pm.Circle(body, *r_head)

    left_foot = pm.Circle(left_calf, 1, (0,-LEG_LEN))
    right_foot = pm.Circle(right_calf, 1, (0,-LEG_LEN))
    left_foot.friction = 0.6
    right_foot.friction = 0.6

    feet = [left_foot, right_foot]

    space.add(body, lines, left_head, right_head, feet, left_thigh, left_calf, right_thigh, right_calf)

    MUSCLE_STRENGTH = 4000000
    MUSCLE_STIFFNESS = 10000

    def reset():
        body.position = (300,300)
        left_thigh.position = 250,300
        left_calf.position = 250,300-LEG_LEN
        right_thigh.position = 350,300
        right_calf.position = 350,300-LEG_LEN
        for obj in (body, left_thigh, left_calf, right_thigh, right_calf):
            obj.angle = 0
            obj.velocity = (0,0)
            obj.angular_velocity = 0
            obj.reset_forces()

    def kill(player):
        if player is P1:
            P2.score += 1
        else:
            P1.score += 1
        print "Scores: Left %s, right %s" % (P1.score, P2.score)
        reset()

    def onCollision(space, arbiter):
        shapes = arbiter.shapes
        if left_head in shapes:
            kill(P1)
        elif right_head in shapes:
            kill(P2)
        elif ground in shapes:
            if left_head in shapes or left_foot in shapes:
                kill(P1)
            elif right_head in shapes or right_foot in shapes:
                kill(P2)

        return True
    space.add_collision_handler(0, 0, onCollision, None, None, None)

    reset()

    space.add(pm.PinJoint(body, left_thigh, l_hip, (0,0)))
    left_thigh_muscle = pm.DampedRotarySpring(body, left_thigh, -(math.pi/4), MUSCLE_STRENGTH, MUSCLE_STIFFNESS)
    left_calf_muscle = pm.DampedRotarySpring(left_thigh, left_calf, (math.pi/2), MUSCLE_STRENGTH, MUSCLE_STIFFNESS)
    space.add(pm.PinJoint(left_thigh, left_calf, (0,-LEG_LEN), (0,0)))
    space.add(left_thigh_muscle, left_calf_muscle)

    space.add(pm.PinJoint(body, right_thigh, r_hip, (0,0)))
    space.add(pm.PinJoint(right_thigh, right_calf, (0,-LEG_LEN), (0,0)))
    right_thigh_muscle = pm.DampedRotarySpring(body, right_thigh, (math.pi/4), MUSCLE_STRENGTH, MUSCLE_STIFFNESS)
    right_calf_muscle = pm.DampedRotarySpring(right_thigh, right_calf, -(math.pi/2), MUSCLE_STRENGTH, MUSCLE_STIFFNESS)
    space.add(right_thigh_muscle, right_calf_muscle)

    P1 = Player(left_thigh_muscle, left_calf_muscle, -1.0)
    P2 = Player(right_thigh_muscle, right_calf_muscle, 1.0)

    STRETCH_RANGE = math.pi/5
    ANGLE_RANGE = math.pi/5
    KEYS = {
        K_w: (P1, STRETCH_RANGE, 0),
        K_s: (P1, -STRETCH_RANGE, 0),
        K_d: (P1, 0, ANGLE_RANGE),
        K_a: (P1, 0, -ANGLE_RANGE),

        K_UP: (P2, STRETCH_RANGE, 0),
        K_DOWN: (P2, -STRETCH_RANGE, 0),
        K_LEFT: (P2, 0, ANGLE_RANGE),
        K_RIGHT: (P2, 0, -ANGLE_RANGE),
    }

    while running:
        for event in pygame.event.get():
            if event.type == QUIT:
                running = False
            elif event.type == KEYDOWN and event.key == K_ESCAPE:
                running = False

            elif event.type == KEYDOWN and event.key in KEYS:
                player, stretch, angle = KEYS[event.key]
                player.stretch += stretch
                player.angle += angle
                player.updateMuscles()

            elif event.type == KEYUP and event.key in KEYS:
                player, stretch, angle = KEYS[event.key]
                player.stretch -= stretch
                player.angle -= angle
                player.updateMuscles()

        ### Clear screen
        screen.fill(THECOLORS["white"])

        ### Draw stuff
        for line in lines:
            body = line.body
            pv1 = body.position + line.a.rotated(body.angle)
            pv2 = body.position + line.b.rotated(body.angle)
            pt1 = to_pygame(pv1)
            pt2 = to_pygame(pv2)
            pygame.draw.lines(screen, THECOLORS["black"], False, [pt1,pt2], 4)

        for head in (left_head, right_head):
            body = head.body
            cv = body.position + head.offset.rotated(body.angle)
            c = to_pygame(cv)
            pygame.draw.circle(screen, THECOLORS["black"], c, int(head.radius))

        for line in (ground, mat):
            offset = line.body.position
            a = to_pygame(offset+line.a)
            b = to_pygame(offset+line.b)
            pygame.draw.lines(screen, line.color, False, [a, b], 4)

        ### Update physics
        dt = 1.0/50.0/10.0
        for x in range(10):
            space.step(dt)

        ### Flip screen
        pygame.display.flip()
        clock.tick(50)

if __name__ == '__main__':
    sys.exit(main())
<commit_msg>Rewrite body-structure in Player.<commit_after>#!/usr/bin/env python

import pygame
from pygame.locals import *
from pygame.color import *
import pymunk as pm
import math, sys, random

LEG_LEN = 35
MUSCLE_STRENGTH = 4000000
MUSCLE_STIFFNESS = 20000

class Player(object):
    def __init__(self, world, shared_body, direction):
        self.world = world
        self.shared_body = shared_body
        self.direction = direction
        self.stretch = (math.pi/4)
        self.angle = 0
        self.score = 0

        self._setup()

    def _setup(self):
        direction = self.direction
        self.head = pm.Circle(self.shared_body, 25, (50*direction, 55))

        self.thigh = pm.Body(10, 10000)
        self.calf = pm.Body(10, 10000)

        self.segments = [
            pm.Segment(self.thigh, (0,0), (0,-LEG_LEN), 5.0),
            pm.Segment(self.calf, (0,0), (0,-LEG_LEN), 5.0),
        ]

        self.reset()

        hip_joint = pm.PinJoint(self.shared_body, self.thigh, (50*direction,0), (0,0))
        knee_joint = pm.PinJoint(self.thigh, self.calf, (0,-LEG_LEN), (0,0))

        self.thigh_muscle = pm.DampedRotarySpring(self.shared_body, self.thigh, 0, MUSCLE_STRENGTH, MUSCLE_STIFFNESS)
        self.calf_muscle = pm.DampedRotarySpring(self.thigh, self.calf, 0, MUSCLE_STRENGTH, MUSCLE_STIFFNESS)

        self.foot = pm.Circle(self.calf, 1, (0,-LEG_LEN))
        self.foot.friction = 0.6
        self.world.add(self.head, self.thigh, self.calf, self.segments, hip_joint, knee_joint, self.thigh_muscle, self.calf_muscle, self.foot)
        self.updateMuscles()

    def updateMuscles(self):
        self.thigh_muscle.rest_angle = (self.angle + self.stretch) * self.direction
        self.calf_muscle.rest_angle = (-self.stretch*2) * self.direction

    def reset(self):
        self.thigh.position = 300+(50*self.direction),300
        self.calf.position = 300+(50*self.direction),300-LEG_LEN
        for obj in (self.thigh, self.calf):
            obj.angle = 0
            obj.velocity = (0,0)
            obj.angular_velocity = 0
            obj.reset_forces()

    def draw(self, screen):
        head = self.head
        body = head.body
        cv = body.position + head.offset.rotated(body.angle)
        c = to_pygame(cv)
        pygame.draw.circle(screen, THECOLORS["black"], c, int(head.radius))

        for line in self.segments:
            body = line.body
            pv1 = body.position + line.a.rotated(body.angle)
            pv2 = body.position + line.b.rotated(body.angle)
            pt1 = to_pygame(pv1)
            pt2 = to_pygame(pv2)
            pygame.draw.lines(screen, THECOLORS["black"], False, [pt1,pt2], 4)

def to_pygame(p):
    """Small hack to convert pymunk to pygame coordinates"""
    return int(p.x), int(-p.y+600)

def main():
    pygame.init()
    screen = pygame.display.set_mode((600, 600))
    pygame.display.set_caption("Fight!")
    clock = pygame.time.Clock()
    running = True

    ### Physics stuff
    pm.init_pymunk()
    space = pm.Space()
    space.gravity = (0.0, -900.0)

    space.resize_static_hash()
    space.resize_active_hash()

    ### static stuff
    ground_body = pm.Body(pm.inf, pm.inf)
    ground_body.position = (300,40)

    ground = pm.Segment(ground_body, (-1000,0), (1000,0), 5.0)
    ground.friction = 0.6
    ground.color = THECOLORS['black']
    mat = pm.Segment(ground_body, (-200,5), (200,5), 5.0)
    mat.friction = 0.6
    mat.color = THECOLORS['red']
    space.add(ground, mat)

    ### The moving L shape
    l_shoulder = (-50, 30)
    l_hip = (-50,0)
    r_shoulder = (50, 30)
    r_hip = (50,0)

    body = pm.Body(10,10000)

    lines = [pm.Segment(body, l_shoulder, l_hip, 5.0)
            ,pm.Segment(body, r_shoulder, r_hip, 5.0)
            ,pm.Segment(body, r_shoulder, l_hip, 5.0)
            ,pm.Segment(body, l_shoulder, r_hip, 5.0)
    ]

    space.add(body, lines)
    body.position = 300,300

    P1 = Player(space, body, -1.0)
    P2 = Player(space, body, 1.0)

    def reset():
        body.position = (300,300)
        body.angle = 0
        body.velocity = (0,0)
        body.angular_velocity = 0
        body.reset_forces()
        P1.reset()
        P2.reset()

    def kill(player):
        if player is P1:
            P2.score += 1
        else:
            P1.score += 1
        print "Scores: Left %s, right %s" % (P1.score, P2.score)
        reset()

    def onCollision(space, arbiter):
        shapes = arbiter.shapes
        if P1.head in shapes:
            kill(P1)
        elif P2.head in shapes:
            kill(P2)
        elif ground in shapes:
            if P1.head in shapes or P1.foot in shapes:
                kill(P1)
            elif P2.head in shapes or P2.foot in shapes:
                kill(P2)

        return True
    space.add_collision_handler(0, 0, onCollision, None, None, None)

    reset()

    STRETCH_RANGE = math.pi/5
    ANGLE_RANGE = math.pi/5
    KEYS = {
        K_w: (P1, STRETCH_RANGE, 0),
        K_s: (P1, -STRETCH_RANGE, 0),
        K_d: (P1, 0, ANGLE_RANGE),
        K_a: (P1, 0, -ANGLE_RANGE),

        K_UP: (P2, STRETCH_RANGE, 0),
        K_DOWN: (P2, -STRETCH_RANGE, 0),
        K_LEFT: (P2, 0, ANGLE_RANGE),
        K_RIGHT: (P2, 0, -ANGLE_RANGE),
    }

    while running:
        for event in pygame.event.get():
            if event.type == QUIT:
                running = False
            elif event.type == KEYDOWN and event.key == K_ESCAPE:
                running = False

            elif event.type == KEYDOWN and event.key in KEYS:
                player, stretch, angle = KEYS[event.key]
                player.stretch += stretch
                player.angle += angle
                player.updateMuscles()

            elif event.type == KEYUP and event.key in KEYS:
                player, stretch, angle = KEYS[event.key]
                player.stretch -= stretch
                player.angle -= angle
                player.updateMuscles()

        ### Clear screen
        screen.fill(THECOLORS["white"])

        ### Draw stuff
        for line in lines:
            body = line.body
            pv1 = body.position + line.a.rotated(body.angle)
            pv2 = body.position + line.b.rotated(body.angle)
            pt1 = to_pygame(pv1)
            pt2 = to_pygame(pv2)
            pygame.draw.lines(screen, THECOLORS["black"], False, [pt1,pt2], 4)

        P1.draw(screen)
        P2.draw(screen)

        for line in (ground, mat):
            offset = line.body.position
            a = to_pygame(offset+line.a)
            b = to_pygame(offset+line.b)
            pygame.draw.lines(screen, line.color, False, [a, b], 4)

        ### Update physics
        dt = 1.0/50.0/10.0
        for x in range(10):
            space.step(dt)

        ### Flip screen
        pygame.display.flip()
        clock.tick(50)

if __name__ == '__main__':
    sys.exit(main())
<|endoftext|>


<commit_before>"""Syndicated feed with entries, based on feedparser library."""

import datetime
import logging
import re
import webbrowser
from contextlib import contextmanager
from functools import lru_cache
try:
    from http import HTTPStatus
except ImportError:
    from httpstatus import HTTPStatus  # HTTPStatus backported to 3.4.
from pathlib import Path

import common
from entry import Entry

import fpapi
from misctypes import Flag, TagDict

import util

log = logging.getLogger(__name__)
messager = util.Messager(__name__)

class Head(object):
    """Feed header."""
    def __init__(self, link, date_published, date_seen, author, title,
                 subtitle, summary, generator, language, publisher, rights,
                 image, tags):
        """Create new feed header."""
        self.link = link
        self.date_published = util.ParsedDatetime.parse(date_published)
        self.date_seen = util.ParsedDatetime.parse(date_seen)
        self.author = author
        self.title = title
        self.subtitle = subtitle
        self.summary = summary
        self.generator = generator
        self.language = language
        self.publisher = publisher
        self.rights = rights
        self.image = image
        self.tags = set(tags)

    def as_json(self):
        return dict(self.__dict__)

    @property
    def date(self):
        """Feed date."""
        return self.date_published or self.date_seen

class Feed(object):
    FEEDFILE = 'data.json'  # Feed information file.
    BAKEXT = '.bak'

    """Feed with entries."""
    def __init__(self, url, old_url=None, directory=None, parseinfo=None,
                 head=None, entries=None):
        """Create new feed."""
        self.url = url
        self.old_url = old_url
        self.directory = Path(directory) if directory else None
        self.parseinfo = parseinfo or {}
        self.head = Head(**head) if head else {}
        self.entries = [Entry(self, **x) for x in entries or []]
        self.modified = False
        self._nentries = None

    def __str__(self):
        return str(self.directory) or self.url

    def as_json(self):
        d = dict(
            url=self.url,
            old_url=self.old_url,
            directory=str(self.directory) if self.directory else None,
            parseinfo=self.parseinfo,
            head=self.head,
            entries=[x for x in self.entries],
            )
        return d

    def should_skip(self, gracetime=None):
        """Skip refresh or not? Gracetime given in hours."""
        tags = self.get_tags()
        if 'inactive' in tags:
            return True
        if 'gracetime' in tags:
            gracetime = float(tags['gracetime'])
        # if gracetime and self.head.date_seen and 'no_grace' not in tags:
        if gracetime and self.head.date_seen:
            delta = datetime.datetime.utcnow() - self.head.date_seen
            if delta.total_seconds() < gracetime * 60 * 60:
                return True
        return False

    def refresh(self, gracetime=None, force=False):
        """Retrieve and parse, if needed or forced. Return the number of
        updated entries, or None if feed skipped. Gracetime given in hours.
        """
        if not force:
            if self.should_skip(gracetime):
                log.debug('Skipping refresh: %s', self)
                return None

        if force:
            etag, modified = None, None
        else:
            etag, modified = self.parseinfo['etag'], self.parseinfo['modified']
        fp = fpapi.parse(self.url, etag=etag, modified=modified)
        if 'status' not in fp:
            log.error('Error retrieving feed: %s: %s', self,
                      fp.bozo_exception)
            return 0
        try:
            status = HTTPStatus(fp.status)
        except ValueError as e:
            log.error('Error connecting: %s: %s', self, e)
            return 0
        if status == HTTPStatus.MOVED_PERMANENTLY:
            log.warning('Permanent redirect: %s moved from %s to %s', self,
                        self.url, fp.href)
            self.old_url, self.url = self.url, fp.href
            return self.refresh(force=True)
        elif status == HTTPStatus.FOUND:
            log.warning('Feed temporarily redirected: %s', self)
        elif status == HTTPStatus.NOT_MODIFIED:
            log.debug('Skipping refresh: %s', self)
            return None  # No need to download. Don't change anything.
        elif status == HTTPStatus.NOT_FOUND:
            log.error('Feed not found: %s: %s', self.directory, self.url)
            return None
        elif status == HTTPStatus.UNAUTHORIZED:
            log.warning('Feed password-protected: %s', self)
        elif status == HTTPStatus.GONE:
            log.warning('Feed gone, should stop polling: %s', self)
        elif status not in [HTTPStatus.OK]:
            log.warning('%s: Weird HTTP status: %s', self, status.name)
        try:
            return self.update(fp)
        except (KeyError, AttributeError) as e:
            log.error('Error parsing feed: %s: %s', self, e)
            return 0

    def update(self, fp):
        """Update contents from a feedparser object. Return the number of
        updated entries.
        """
        log.debug('Updating feed: %s', self)
        self.parseinfo = fpapi.get_parseinfo(fp)
        self.head = Head(**(fpapi.get_head(fp.feed)))
        if not self.directory:
            self.directory = Path(self.head.title)
            log.warning('Got directory from title: %s', self.directory)
        entries = (Entry(**(fpapi.get_entry(x, self))) for x in fp.entries)
        n_updates = sum(self.add_entry(x) for x in entries)
        if n_updates:
            log.debug('Updated %i entries in feed: %s', n_updates, self)
            self.entries.sort()
        self.modified = True
        return n_updates

    def add_entry(self, entry):
        """Add another entry, if new or updated. Return True if changes done,
        False if not.
        """
        present = [x for x in self.entries if x.guid == entry.guid]
        assert len(present) < 2, len(present)
        if not present:
            # log.warning('Adding entry: %s: %s', self, entry)
            messager.msg('Adding entry: {}: {}'.format(self, entry),
                         truncate=True)
            # ##
            # import shutil
            # print('####', shutil.get_terminal_size().columns)
            # ##
            self.entries.append(entry)
            return True
        old = present[0]
        if entry.date_published and entry.date > old.date:
            # Replace old entry with new one, but keep flag info.
            # log.warning('Updating entry with newer: %s: %s: %s', self, entry,
            #             old.flag)
            s = 'Updating entry with newer: {}: {}: {}'
            messager.msg(s.format(self, entry, old.flag.name))
            entry.flag = old.flag
            self.entries.remove(old)
            self.entries.append(entry)
            return True
        if len(entry.enclosures) > len(old.enclosures):
            # However, if the entry got more enclosures, just replace all info.
            # Perhaps the publisher first forgot to add them.
            # log.warning('Updating entry with new enclosures: %s: %s: %s',
            #             self, entry, old.flag)
            s = 'Updating entry with new enclosures: {}: {}: {}'
            messager.msg(s.format(self, entry, old.flag.name))
            self.entries.remove(old)
            self.entries.append(entry)
            return True
        return False

    def customize_sortkey(self, sortkey):
        """Customize sortkey by tag contents."""
        if '=' not in sortkey:
            dateorder = self.get_tags().get('date')
            if dateorder == 'asc':
                sortkey = sortkey.replace('D', 'd')  # Ascending date.
            elif dateorder == 'desc':
                sortkey = sortkey.replace('d', 'D')  # Descending date.
            elif dateorder:
                log.error('Invalid date order: %s', self)
        return sortkey

    def list_entries(self, flags=None, sortkey=None, number=None):
        """List entries of certain criteria.

        Wildcards are '.' for any flag, -1 for infinite number.
        """
        entries = self.entries
        # if flags is not None and '.' not in flags:
        #     entries = [x for x in entries if x.flag in flags]
        if flags:
            flags = [Flag(x) for x in flags]
            entries = [x for x in entries if x.flag in flags]
        if sortkey is not None:
            sortkey = self.customize_sortkey(sortkey)
            sort_entries(entries, sortkey)
        self._nentries = len(entries)
        if number is not None and number != -1:
            entries = entries[:number]
        return entries

    def get_daystats(self, include_now=False, **kwargs):
        """Get stats on feed publishing frequency."""
        dates = sorted(x.date for x in self.list_entries(**kwargs))
        if include_now:
            dates.append(datetime.datetime.utcnow())
        if len(dates) < 3:
            raise ValueError('Not enough entries: {}'.format(self))
        deltas = [t2 - t1 for t1, t2 in zip(dates, dates[1:])]
        stats = util.timedelta_stats(deltas)
        daydeltas = map(util.timedelta_floatdays, deltas)
        daystats = map(util.timedelta_floatdays, stats.values())
        names = stats.keys()
        return list(daydeltas), tuple(daystats), tuple(names)

    def wait_to_refresh(self):
        """How much more should we wait until refresh?"""
        # TODO
        deltas, stats, names = self.get_daystats(include_now=False,
                                                 sortkey='D=', number=5)
        log.debug('%s %s %s', deltas, stats, names)
        return stats[0] / 2

    def check(self):
        """Feed sanity check."""
        status = HTTPStatus(self.parseinfo['status'])
        if status not in [HTTPStatus.OK, HTTPStatus.FOUND,
                          HTTPStatus.NOT_MODIFIED]:
            yield 'Weird HTTP status: {}'.format(status.name)
        if self.parseinfo['bozo']:
            yield 'Bozo: {}'.format(self.parseinfo['bozo'])
        if not self.entries:
            yield 'No entries'
        tags = self.get_tags()
        for tag in self.get_tags():
            if not tags.is_sane(tag):
                yield 'Weird tag: {}'.format(tag)
        for e in self.entries:
            for msg in e.check():
                yield msg
        orphans = self.get_orphans()
        if orphans:
            yield 'Orphan files: {}'.format(len(orphans))

    def get_orphans(self):
        """Return list of orphaned files in feed directory."""
        dirfiles = {x.name for x in self.directory.iterdir()}
        p = Feed.data_path()
        datafiles = {str(p), str(p.with_suffix(p.suffix + self.BAKEXT))}
        encfiles = {enc.filename for entry in self.entries
                    for enc in entry.encs()}
        orphans = dirfiles.difference(datafiles).difference(encfiles)
        return sorted(orphans)

    def open_link(self):
        """Open feed link in web browser."""
        webbrowser.open(self.head.link)

    @lru_cache()
    def get_tags(self):
        """Get feed tags."""
        tags = TagDict()
        for s in self.directory.parts:
            tags.parse(s)
        for s in self.head.tags:
            tags.parse(s)
        return tags

    @property
    @lru_cache()
    def priority(self):
        """Get feed priority (tag p=n). Greater number => greater interest."""
        try:
            return int(self.get_tags()['p'])
        except (KeyError, ValueError):
            return 0

    @classmethod
    def data_path(cls, directory=''):
        """Return feed data file path (relative)."""
        return Path(directory) / cls.FEEDFILE

    @staticmethod
    def read(directory):
        """Read data."""
        d = common.read_data(Feed.data_path(directory))
        d['directory'] = directory
        feed = Feed(**d)
        return feed

    def write(self, directory=None, force=False):
        """Write data."""
        if self.modified or force:
            if directory is None:
                directory = self.directory
            common.write_data(Feed.data_path(directory), self)
            self.modified = False

    @classmethod
    @contextmanager
    def open(cls, directory):
        """Context manager for data file."""
        f = cls.read(directory)
        yield f
        f.write()

# TODO: Make a class of sortkey stuff, also with util.general_sort.
# TODO: Or a view.

SORTKEYS = {
    '=': lambda _: 0,  # Indicates no per-feed alterations allowed.
    'd': lambda entry: entry.date,
    'e': lambda entry: len(entry.enclosures),
    'f': lambda entry: entry.flag.index(),
    'i': lambda entry: str(entry.feed.directory).lower(),
    'l': lambda entry: (entry.feed.head.language or 'zzz').lower(),
    'n': lambda entry: entry.feed._nentries,
    'p': lambda entry: entry.feed.priority,
    'r': lambda entry: entry.progress,
    's': lambda entry: entry.score,
    't': lambda entry: entry.title.lower(),
    'u': lambda entry: entry.duration(),
    'z': lambda entry: entry.size(),
    }

def sort_entries(entries, sortkey):
    """Sort entries by sortkey string."""
    keys = [SORTKEYS[x] for x in sortkey.lower()]
    reverses = [x.isupper() for x in sortkey]
    util.general_sort(entries, keys, reverses)

def search_entries(entries, patterns, flags=re.IGNORECASE, start=0):
    """Return index to first entry from start matching all RegExp patterns."""
    def get_strings(entry):
        """Return strings to match against."""
        return [str(entry.feed.directory), entry.feed.head.title,
                entry.feed.head.subtitle, entry.feed.head.summary, entry.title,
                entry.subtitle, entry.summary]

    def pattern_found(strings, pattern, flags):
        """Try matching."""
        return any(re.search(pattern, x, flags=flags) for x in strings)

    for i, entry in enumerate(entries[start:], start):
        if all(pattern_found(get_strings(entry), x, flags) for x in patterns):
            return i
<commit_msg>skipped sortkey<commit_after>"""Syndicated feed with entries, based on feedparser library."""

import datetime
import logging
import re
import webbrowser
from contextlib import contextmanager
from functools import lru_cache
try:
    from http import HTTPStatus
except ImportError:
    from httpstatus import HTTPStatus  # HTTPStatus backported to 3.4.
from pathlib import Path

import common
from entry import Entry

import fpapi
from misctypes import Flag, TagDict

import util

log = logging.getLogger(__name__)
messager = util.Messager(__name__)

class Head(object):
    """Feed header."""
    def __init__(self, link, date_published, date_seen, author, title,
                 subtitle, summary, generator, language, publisher, rights,
                 image, tags):
        """Create new feed header."""
        self.link = link
        self.date_published = util.ParsedDatetime.parse(date_published)
        self.date_seen = util.ParsedDatetime.parse(date_seen)
        self.author = author
        self.title = title
        self.subtitle = subtitle
        self.summary = summary
        self.generator = generator
        self.language = language
        self.publisher = publisher
        self.rights = rights
        self.image = image
        self.tags = set(tags)

    def as_json(self):
        return dict(self.__dict__)

    @property
    def date(self):
        """Feed date."""
        return self.date_published or self.date_seen

class Feed(object):
    FEEDFILE = 'data.json'  # Feed information file.
    BAKEXT = '.bak'

    """Feed with entries."""
    def __init__(self, url, old_url=None, directory=None, parseinfo=None,
                 head=None, entries=None):
        """Create new feed."""
        self.url = url
        self.old_url = old_url
        self.directory = Path(directory) if directory else None
        self.parseinfo = parseinfo or {}
        self.head = Head(**head) if head else {}
        self.entries = [Entry(self, **x) for x in entries or []]
        self.modified = False
        self._nentries = None

    def __str__(self):
        return str(self.directory) or self.url

    def as_json(self):
        d = dict(
            url=self.url,
            old_url=self.old_url,
            directory=str(self.directory) if self.directory else None,
            parseinfo=self.parseinfo,
            head=self.head,
            entries=[x for x in self.entries],
            )
        return d

    def should_skip(self, gracetime=None):
        """Skip refresh or not? Gracetime given in hours."""
        tags = self.get_tags()
        if 'inactive' in tags:
            return True
        if 'gracetime' in tags:
            gracetime = float(tags['gracetime'])
        # if gracetime and self.head.date_seen and 'no_grace' not in tags:
        if gracetime and self.head.date_seen:
            delta = datetime.datetime.utcnow() - self.head.date_seen
            if delta.total_seconds() < gracetime * 60 * 60:
                return True
        return False

    def refresh(self, gracetime=None, force=False):
        """Retrieve and parse, if needed or forced. Return the number of
        updated entries, or None if feed skipped. Gracetime given in hours.
        """
        if not force:
            if self.should_skip(gracetime):
                log.debug('Skipping refresh: %s', self)
                return None

        if force:
            etag, modified = None, None
        else:
            etag, modified = self.parseinfo['etag'], self.parseinfo['modified']
        fp = fpapi.parse(self.url, etag=etag, modified=modified)
        if 'status' not in fp:
            log.error('Error retrieving feed: %s: %s', self,
                      fp.bozo_exception)
            return 0
        try:
            status = HTTPStatus(fp.status)
        except ValueError as e:
            log.error('Error connecting: %s: %s', self, e)
            return 0
        if status == HTTPStatus.MOVED_PERMANENTLY:
            log.warning('Permanent redirect: %s moved from %s to %s', self,
                        self.url, fp.href)
            self.old_url, self.url = self.url, fp.href
            return self.refresh(force=True)
        elif status == HTTPStatus.FOUND:
            log.warning('Feed temporarily redirected: %s', self)
        elif status == HTTPStatus.NOT_MODIFIED:
            log.debug('Skipping refresh: %s', self)
            return None  # No need to download. Don't change anything.
        elif status == HTTPStatus.NOT_FOUND:
            log.error('Feed not found: %s: %s', self.directory, self.url)
            return None
        elif status == HTTPStatus.UNAUTHORIZED:
            log.warning('Feed password-protected: %s', self)
        elif status == HTTPStatus.GONE:
            log.warning('Feed gone, should stop polling: %s', self)
        elif status not in [HTTPStatus.OK]:
            log.warning('%s: Weird HTTP status: %s', self, status.name)
        try:
            return self.update(fp)
        except (KeyError, AttributeError) as e:
            log.error('Error parsing feed: %s: %s', self, e)
            return 0

    def update(self, fp):
        """Update contents from a feedparser object. Return the number of
        updated entries.
        """
        log.debug('Updating feed: %s', self)
        self.parseinfo = fpapi.get_parseinfo(fp)
        self.head = Head(**(fpapi.get_head(fp.feed)))
        if not self.directory:
            self.directory = Path(self.head.title)
            log.warning('Got directory from title: %s', self.directory)
        entries = (Entry(**(fpapi.get_entry(x, self))) for x in fp.entries)
        n_updates = sum(self.add_entry(x) for x in entries)
        if n_updates:
            log.debug('Updated %i entries in feed: %s', n_updates, self)
            self.entries.sort()
        self.modified = True
        return n_updates

    def add_entry(self, entry):
        """Add another entry, if new or updated. Return True if changes done,
        False if not.
        """
        present = [x for x in self.entries if x.guid == entry.guid]
        assert len(present) < 2, len(present)
        if not present:
            # log.warning('Adding entry: %s: %s', self, entry)
            messager.msg('Adding entry: {}: {}'.format(self, entry),
                         truncate=True)
            # ##
            # import shutil
            # print('####', shutil.get_terminal_size().columns)
            # ##
            self.entries.append(entry)
            return True
        old = present[0]
        if entry.date_published and entry.date > old.date:
            # Replace old entry with new one, but keep flag info.
            # log.warning('Updating entry with newer: %s: %s: %s', self, entry,
            #             old.flag)
            s = 'Updating entry with newer: {}: {}: {}'
            messager.msg(s.format(self, entry, old.flag.name))
            entry.flag = old.flag
            self.entries.remove(old)
            self.entries.append(entry)
            return True
        if len(entry.enclosures) > len(old.enclosures):
            # However, if the entry got more enclosures, just replace all info.
            # Perhaps the publisher first forgot to add them.
            # log.warning('Updating entry with new enclosures: %s: %s: %s',
            #             self, entry, old.flag)
            s = 'Updating entry with new enclosures: {}: {}: {}'
            messager.msg(s.format(self, entry, old.flag.name))
            self.entries.remove(old)
            self.entries.append(entry)
            return True
        return False

    def customize_sortkey(self, sortkey):
        """Customize sortkey by tag contents."""
        if '=' not in sortkey:
            dateorder = self.get_tags().get('date')
            if dateorder == 'asc':
                sortkey = sortkey.replace('D', 'd')  # Ascending date.
            elif dateorder == 'desc':
                sortkey = sortkey.replace('d', 'D')  # Descending date.
            elif dateorder:
                log.error('Invalid date order: %s', self)
        return sortkey

    def list_entries(self, flags=None, sortkey=None, number=None):
        """List entries of certain criteria.

        Wildcards are '.' for any flag, -1 for infinite number.
        """
        entries = self.entries
        # if flags is not None and '.' not in flags:
        #     entries = [x for x in entries if x.flag in flags]
        if flags:
            flags = [Flag(x) for x in flags]
            entries = [x for x in entries if x.flag in flags]
        if sortkey is not None:
            sortkey = self.customize_sortkey(sortkey)
            sort_entries(entries, sortkey)
        self._nentries = len(entries)
        if number is not None and number != -1:
            entries = entries[:number]
        return entries

    def get_daystats(self, include_now=False, **kwargs):
        """Get stats on feed publishing frequency."""
        dates = sorted(x.date for x in self.list_entries(**kwargs))
        if include_now:
            dates.append(datetime.datetime.utcnow())
        if len(dates) < 3:
            raise ValueError('Not enough entries: {}'.format(self))
        deltas = [t2 - t1 for t1, t2 in zip(dates, dates[1:])]
        stats = util.timedelta_stats(deltas)
        daydeltas = map(util.timedelta_floatdays, deltas)
        daystats = map(util.timedelta_floatdays, stats.values())
        names = stats.keys()
        return list(daydeltas), tuple(daystats), tuple(names)

    def wait_to_refresh(self):
        """How much more should we wait until refresh?"""
        # TODO
        deltas, stats, names = self.get_daystats(include_now=False,
                                                 sortkey='D=', number=5)
        log.debug('%s %s %s', deltas, stats, names)
        return stats[0] / 2

    def check(self):
        """Feed sanity check."""
        status = HTTPStatus(self.parseinfo['status'])
        if status not in [HTTPStatus.OK, HTTPStatus.FOUND,
                          HTTPStatus.NOT_MODIFIED]:
            yield 'Weird HTTP status: {}'.format(status.name)
        if self.parseinfo['bozo']:
            yield 'Bozo: {}'.format(self.parseinfo['bozo'])
        if not self.entries:
            yield 'No entries'
        tags = self.get_tags()
        for tag in self.get_tags():
            if not tags.is_sane(tag):
                yield 'Weird tag: {}'.format(tag)
        for e in self.entries:
            for msg in e.check():
                yield msg
        orphans = self.get_orphans()
        if orphans:
            yield 'Orphan files: {}'.format(len(orphans))

    def get_orphans(self):
        """Return list of orphaned files in feed directory."""
        dirfiles = {x.name for x in self.directory.iterdir()}
        p = Feed.data_path()
        datafiles = {str(p), str(p.with_suffix(p.suffix + self.BAKEXT))}
        encfiles = {enc.filename for entry in self.entries
                    for enc in entry.encs()}
        orphans = dirfiles.difference(datafiles).difference(encfiles)
        return sorted(orphans)

    def open_link(self):
        """Open feed link in web browser."""
        webbrowser.open(self.head.link)

    @lru_cache()
    def get_tags(self):
        """Get feed tags."""
        tags = TagDict()
        for s in self.directory.parts:
            tags.parse(s)
        for s in self.head.tags:
            tags.parse(s)
        return tags

    @property
    @lru_cache()
    def priority(self):
        """Get feed priority (tag p=n). Greater number => greater interest."""
        try:
            return int(self.get_tags()['p'])
        except (KeyError, ValueError):
            return 0

    @classmethod
    def data_path(cls, directory=''):
        """Return feed data file path (relative)."""
        return Path(directory) / cls.FEEDFILE

    @staticmethod
    def read(directory):
        """Read data."""
        d = common.read_data(Feed.data_path(directory))
        d['directory'] = directory
        feed = Feed(**d)
        return feed

    def write(self, directory=None, force=False):
        """Write data."""
        if self.modified or force:
            if directory is None:
                directory = self.directory
            common.write_data(Feed.data_path(directory), self)
            self.modified = False

    @classmethod
    @contextmanager
    def open(cls, directory):
        """Context manager for data file."""
        f = cls.read(directory)
        yield f
        f.write()

# TODO: Make a class of sortkey stuff, also with util.general_sort.
# TODO: Or a view.

SORTKEYS = {
    '=': lambda _: 0,  # Indicates no per-feed alterations allowed.
    'd': lambda entry: entry.date,
    'e': lambda entry: len(entry.enclosures),
    'f': lambda entry: entry.flag.index(),
    'i': lambda entry: str(entry.feed.directory).lower(),
    'l': lambda entry: (entry.feed.head.language or 'zzz').lower(),
    'n': lambda entry: entry.feed._nentries,
    'p': lambda entry: entry.feed.priority,
    'r': lambda entry: entry.progress,
    's': lambda entry: entry.score,
    't': lambda entry: entry.title.lower(),
    'u': lambda entry: entry.duration(),
    'x': lambda entry: entry.skipped(),
    'z': lambda entry: entry.size(),
    }

def sort_entries(entries, sortkey):
    """Sort entries by sortkey string."""
    keys = [SORTKEYS[x] for x in sortkey.lower()]
    reverses = [x.isupper() for x in sortkey]
    util.general_sort(entries, keys, reverses)

def search_entries(entries, patterns, flags=re.IGNORECASE, start=0):
    """Return index to first entry from start matching all RegExp patterns."""
    def get_strings(entry):
        """Return strings to match against."""
        return [str(entry.feed.directory), entry.feed.head.title,
                entry.feed.head.subtitle, entry.feed.head.summary, entry.title,
                entry.subtitle, entry.summary]

    def pattern_found(strings, pattern, flags):
        """Try matching."""
        return any(re.search(pattern, x, flags=flags) for x in strings)

    for i, entry in enumerate(entries[start:], start):
        if all(pattern_found(get_strings(entry), x, flags) for x in patterns):
            return i
<|endoftext|>


<commit_before>#!/usr/bin/env python3
# -*- coding: utf-8 -*-

import unittest

from mst_tree_builder import *
from framenetreader import *

if __name__ == '__main__':
    unittest.main()
<commit_msg>docstring for test module<commit_after>#!/usr/bin/env python3
# -*- coding: utf-8 -*-

"""
..module:: test
    :synopsys: Run all unit tests

"""

import unittest

from mst_tree_builder import *
from framenetreader import *

if __name__ == '__main__':
    unittest.main()
<|endoftext|>