<commit_before># Copyright 2015 0xc0170
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import os
import yaml
import logging
import bisect

from .project import FILES_EXTENSIONS

def _determine_tool(linker_ext):
        if "sct" in linker_ext or "lin" in linker_ext:
            return "uvision"
        elif "ld" in linker_ext:
            return "make_gcc_arm"
        elif "icf" in linker_ext:
            return "iar_arm"

def _scan(section, root, directory, extensions, list_files):
        if section == "sources":
            data_dict = {}
        else:
            data_dict = []
        for dirpath, dirnames, files in os.walk(directory):
            for filename in files:
                ext = filename.split('.')[-1]
                relpath = os.path.relpath(dirpath, root)
                if ext in extensions:
                    if section == "sources":
                        p = relpath
                        if list_files:
                            p = os.path.join(p,filename)
                        dir = directory.split(os.path.sep)[-1] if dirpath == directory else dirpath.replace(directory,'').split(os.path.sep)[1]
                        if dir in data_dict and relpath not in data_dict[dir]:
                            bisect.insort(data_dict[dir], p)
                        else:
                            data_dict[dir] = [(p)]
                    elif section == 'includes':
                        dirs = relpath.split(os.path.sep)
                        for i in range(1, len(dirs)+1):
                            data_dict.append(os.path.sep.join(dirs[:i]))
                    else:
                        data_dict.append(os.path.join(relpath, filename))
        if section == "sources":
            return data_dict
        l = list(set(data_dict))
        l.sort()
        return l

def _generate_file(filename,root,directory,data):
        logging.debug('Generating yaml file')
        overwrite = False
        if os.path.isfile(os.path.join(directory, filename)):
            print("Project file " +filename+  " already exists")
            while True:
                answer = raw_input('Should I overwrite it? (Y/n)')
                try:
                    overwrite = answer.lower() in ('y', 'yes')
                    if not overwrite:
                        logging.critical('Unable to save project file')
                        return -1
                    break
                except ValueError:
                    continue
        if overwrite:
            with open(os.path.join(root, filename), 'r+') as f:
                f.write(yaml.dump(data, default_flow_style=False))
        else:
            with open(os.path.join(root, filename), 'w+') as f:
                f.write(yaml.dump(data, default_flow_style=False))
        p = os.popen('attrib +h ' + filename)
        p.close()

def create_yaml(root, directory, project_name, board, files):
        common_section = {
            'linker_file': FILES_EXTENSIONS['linker_file'],
            'sources': FILES_EXTENSIONS['source_files_c'] + FILES_EXTENSIONS['source_files_cpp'] +
                        FILES_EXTENSIONS['source_files_s'] + FILES_EXTENSIONS['source_files_obj'],
            'includes': FILES_EXTENSIONS['include_files'],
            'target': [],
        }
        projects_yaml = {
            'projects': {
                project_name: ['project.yaml']
            }
        }

        project_yaml = {
            'common': {},
            'tool_specific': {}
        }

        for section in common_section:
            if len(common_section[section]) > 0:
                project_yaml['common'][section] = _scan(section, root, directory,common_section[section], files)

        project_yaml['common']['target'] = [board]
        tool = _determine_tool(str(project_yaml['common']['linker_file']).split('.')[-1])
        project_yaml['tool_specific'] = {
            tool: {
                'linker_file': project_yaml['common']['linker_file']
            }
        }
        _generate_file("projects.yaml", root, directory, projects_yaml)
        _generate_file("project.yaml", root, directory, project_yaml)
        return 0

<commit_msg>Init - generated yaml files are visible<commit_after># Copyright 2015 0xc0170
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import os
import yaml
import logging
import bisect

from .project import FILES_EXTENSIONS

def _determine_tool(linker_ext):
        if "sct" in linker_ext or "lin" in linker_ext:
            return "uvision"
        elif "ld" in linker_ext:
            return "make_gcc_arm"
        elif "icf" in linker_ext:
            return "iar_arm"

def _scan(section, root, directory, extensions, list_files):
        if section == "sources":
            data_dict = {}
        else:
            data_dict = []
        for dirpath, dirnames, files in os.walk(directory):
            for filename in files:
                ext = filename.split('.')[-1]
                relpath = os.path.relpath(dirpath, root)
                if ext in extensions:
                    if section == "sources":
                        p = relpath
                        if list_files:
                            p = os.path.join(p,filename)
                        dir = directory.split(os.path.sep)[-1] if dirpath == directory else dirpath.replace(directory,'').split(os.path.sep)[1]
                        if dir in data_dict and relpath not in data_dict[dir]:
                            bisect.insort(data_dict[dir], p)
                        else:
                            data_dict[dir] = [(p)]
                    elif section == 'includes':
                        dirs = relpath.split(os.path.sep)
                        for i in range(1, len(dirs)+1):
                            data_dict.append(os.path.sep.join(dirs[:i]))
                    else:
                        data_dict.append(os.path.join(relpath, filename))
        if section == "sources":
            return data_dict
        l = list(set(data_dict))
        l.sort()
        return l

def _generate_file(filename,root,directory,data):
        logging.debug('Generating yaml file')
        overwrite = False
        if os.path.isfile(os.path.join(directory, filename)):
            print("Project file " +filename+  " already exists")
            while True:
                answer = raw_input('Should I overwrite it? (Y/n)')
                try:
                    overwrite = answer.lower() in ('y', 'yes')
                    if not overwrite:
                        logging.critical('Unable to save project file')
                        return -1
                    break
                except ValueError:
                    continue
        if overwrite:
            with open(os.path.join(root, filename), 'r+') as f:
                f.write(yaml.dump(data, default_flow_style=False))
        else:
            with open(os.path.join(root, filename), 'w+') as f:
                f.write(yaml.dump(data, default_flow_style=False))

def create_yaml(root, directory, project_name, board, files):
        common_section = {
            'linker_file': FILES_EXTENSIONS['linker_file'],
            'sources': FILES_EXTENSIONS['source_files_c'] + FILES_EXTENSIONS['source_files_cpp'] +
                        FILES_EXTENSIONS['source_files_s'] + FILES_EXTENSIONS['source_files_obj'],
            'includes': FILES_EXTENSIONS['include_files'],
            'target': [],
        }
        projects_yaml = {
            'projects': {
                project_name: ['project.yaml']
            }
        }

        project_yaml = {
            'common': {},
            'tool_specific': {}
        }

        for section in common_section:
            if len(common_section[section]) > 0:
                project_yaml['common'][section] = _scan(section, root, directory,common_section[section], files)

        project_yaml['common']['target'] = [board]
        tool = _determine_tool(str(project_yaml['common']['linker_file']).split('.')[-1])
        project_yaml['tool_specific'] = {
            tool: {
                'linker_file': project_yaml['common']['linker_file']
            }
        }
        _generate_file("projects.yaml", root, directory, projects_yaml)
        _generate_file("project.yaml", root, directory, project_yaml)
        return 0

<|endoftext|>


<commit_before>import re
import sys
from .sandbox import ALLOW
from .syscalls import *

class CHROOTSecurity(dict):
    def __init__(self, filesystem, writable=(1, 2)):
        super(CHROOTSecurity, self).__init__()
        self.fs_jail = re.compile('|'.join(filesystem) if filesystem else '^')
        self.execve_count = 0
        self._writable = writable

        self.update({
            sys_execve: self.do_execve,
            sys_read: ALLOW,
            sys_write: self.do_write,
            sys_writev: self.do_write,
            sys_open: self.do_access,
            sys_access: self.do_access,
            sys_close: ALLOW,
            sys_stat: ALLOW,
            sys_dup: ALLOW,
            sys_fstat: ALLOW,
            sys_mmap: ALLOW,
            sys_mremap: ALLOW,
            sys_mprotect: ALLOW,
            sys_munmap: ALLOW,
            sys_brk: ALLOW,
            sys_fcntl: ALLOW,
            sys_arch_prctl: ALLOW,
            sys_set_tid_address: ALLOW,
            sys_set_robust_list: ALLOW,
            sys_futex: ALLOW,
            sys_rt_sigaction: ALLOW,
            sys_rt_sigprocmask: ALLOW,
            sys_getrlimit: ALLOW,
            sys_ioctl: ALLOW,
            sys_readlink: ALLOW,
            sys_getcwd: ALLOW,
            sys_geteuid: ALLOW,
            sys_getuid: ALLOW,
            sys_getegid: ALLOW,
            sys_getgid: ALLOW,
            sys_lstat: ALLOW,
            sys_openat: ALLOW,
            sys_getdents: ALLOW,
            sys_lseek: ALLOW,
            sys_getrusage: ALLOW,
            sys_sigaltstack: ALLOW,
            sys_pipe: ALLOW,
            sys_clock_gettime: ALLOW,
            sys_gettimeofday: ALLOW,

            sys_clone: ALLOW,
            sys_exit_group: ALLOW,
            sys_gettid: ALLOW,

            # x86 specific
            sys_mmap2: ALLOW,
            sys_fstat64: ALLOW,
            sys_set_thread_area: ALLOW,
        })

    def do_write(self, debugger):
        return debugger.arg0() in self._writable

    def do_execve(self, debugger):
        self.execve_count += 1
        return self.execve_count < 2

    def do_access(self, debugger):
        file = debugger.readstr(debugger.uarg0())
        if self.fs_jail.match(file) is None:
            print>>sys.stderr, 'Not allowed to access:', file
            return False
        return True
<commit_msg>x86 programs uses sys_ugetrlimit<commit_after>import re
import sys
from .sandbox import ALLOW
from .syscalls import *

class CHROOTSecurity(dict):
    def __init__(self, filesystem, writable=(1, 2)):
        super(CHROOTSecurity, self).__init__()
        self.fs_jail = re.compile('|'.join(filesystem) if filesystem else '^')
        self.execve_count = 0
        self._writable = writable

        self.update({
            sys_execve: self.do_execve,
            sys_read: ALLOW,
            sys_write: self.do_write,
            sys_writev: self.do_write,
            sys_open: self.do_access,
            sys_access: self.do_access,
            sys_close: ALLOW,
            sys_stat: ALLOW,
            sys_dup: ALLOW,
            sys_fstat: ALLOW,
            sys_mmap: ALLOW,
            sys_mremap: ALLOW,
            sys_mprotect: ALLOW,
            sys_munmap: ALLOW,
            sys_brk: ALLOW,
            sys_fcntl: ALLOW,
            sys_arch_prctl: ALLOW,
            sys_set_tid_address: ALLOW,
            sys_set_robust_list: ALLOW,
            sys_futex: ALLOW,
            sys_rt_sigaction: ALLOW,
            sys_rt_sigprocmask: ALLOW,
            sys_getrlimit: ALLOW,
            sys_ioctl: ALLOW,
            sys_readlink: ALLOW,
            sys_getcwd: ALLOW,
            sys_geteuid: ALLOW,
            sys_getuid: ALLOW,
            sys_getegid: ALLOW,
            sys_getgid: ALLOW,
            sys_lstat: ALLOW,
            sys_openat: ALLOW,
            sys_getdents: ALLOW,
            sys_lseek: ALLOW,
            sys_getrusage: ALLOW,
            sys_sigaltstack: ALLOW,
            sys_pipe: ALLOW,
            sys_clock_gettime: ALLOW,
            sys_gettimeofday: ALLOW,

            sys_clone: ALLOW,
            sys_exit_group: ALLOW,
            sys_gettid: ALLOW,

            # x86 specific
            sys_mmap2: ALLOW,
            sys_fstat64: ALLOW,
            sys_set_thread_area: ALLOW,
            sys_ugetrlimit: ALLOW,
        })

    def do_write(self, debugger):
        return debugger.arg0() in self._writable

    def do_execve(self, debugger):
        self.execve_count += 1
        return self.execve_count < 2

    def do_access(self, debugger):
        file = debugger.readstr(debugger.uarg0())
        if self.fs_jail.match(file) is None:
            print>>sys.stderr, 'Not allowed to access:', file
            return False
        return True
<|endoftext|>


<commit_before>#!/usr/bin/env python
# Copyright 2015 ARM Limited
#
# Licensed under the Apache License, Version 2.0
# See LICENSE file for details.

import copy

from .tool import Tool, Builder, Exporter

class gdb_definitions():

    SUPPORTED_MCUS = {
        'K64F': {
        }
    }

class GDB(Tool, Exporter, Builder):
    def __init__(self, workspace, env_settings):
        self.workspace = workspace
        self.env_settings = env_settings

    @staticmethod
    def get_toolnames():
        return ['gdb']

    @staticmethod
    def get_toolchain():
        return None

    def export_project(self):
        # for native debugging, no command files are necessary
        return None, []

    def supports_target(self, target):
        # !!! TODO: should be yes for native targets
        return False

    def is_supported_by_default(self, target):
        # does not require additional information
        return True

class ARMNoneEABIGDB(GDB):
    SUPPORTED = gdb_definitions.SUPPORTED_MCUS

    generated_project = {
        'path': '',
        'files': {
            'startupfile': '',
        }
    }

    def __init__(self, workspace, env_settings):
        super(ARMNoneEABIGDB, self).__init__(workspace, env_settings)

    @staticmethod
    def get_toolnames():
        return ['gdb']

    @staticmethod
    def get_toolchain():
        return None

    def export_project(self):
        generated_projects = copy.deepcopy(self.generated_project)
        expanded_dic = self.workspace.copy()

        # !!! TODO: store and read settings from gdb_definitions
        expanded_dic['gdb_server_port'] = 3333

        project_path, startupfile = self.gen_file_jinja(
            'gdb.tmpl', expanded_dic, '%s.gdbstartup' % expanded_dic['name'], expanded_dic['output_dir']['path'])
        generated_projects['path'] = project_path
        generated_projects['files']['startupfile'] = startupfile
        return generated_projects

    def get_generated_project_files(self):
        return {'path': self.workspace['path'], 'files': [self.workspace['files']['startupfile']]}

    def supports_target(self, target):
        return target in self.SUPPORTED

    @staticmethod
    def is_supported_by_default(target):
        # does not require additional information
        return True
<commit_msg>GDB - fix is supported by default - static method<commit_after>#!/usr/bin/env python
# Copyright 2015 ARM Limited
#
# Licensed under the Apache License, Version 2.0
# See LICENSE file for details.

import copy

from .tool import Tool, Builder, Exporter

class gdb_definitions():

    SUPPORTED_MCUS = {
        'K64F': {
        }
    }

class GDB(Tool, Exporter, Builder):
    def __init__(self, workspace, env_settings):
        self.workspace = workspace
        self.env_settings = env_settings

    @staticmethod
    def get_toolnames():
        return ['gdb']

    @staticmethod
    def get_toolchain():
        return None

    def export_project(self):
        # for native debugging, no command files are necessary
        return None, []

    def supports_target(self, target):
        # !!! TODO: should be yes for native targets
        return False

    @staticmethod
    def is_supported_by_default(target):
        # does not require additional information
        return True

class ARMNoneEABIGDB(GDB):
    SUPPORTED = gdb_definitions.SUPPORTED_MCUS

    generated_project = {
        'path': '',
        'files': {
            'startupfile': '',
        }
    }

    def __init__(self, workspace, env_settings):
        super(ARMNoneEABIGDB, self).__init__(workspace, env_settings)

    @staticmethod
    def get_toolnames():
        return ['gdb']

    @staticmethod
    def get_toolchain():
        return None

    def export_project(self):
        generated_projects = copy.deepcopy(self.generated_project)
        expanded_dic = self.workspace.copy()

        # !!! TODO: store and read settings from gdb_definitions
        expanded_dic['gdb_server_port'] = 3333

        project_path, startupfile = self.gen_file_jinja(
            'gdb.tmpl', expanded_dic, '%s.gdbstartup' % expanded_dic['name'], expanded_dic['output_dir']['path'])
        generated_projects['path'] = project_path
        generated_projects['files']['startupfile'] = startupfile
        return generated_projects

    def get_generated_project_files(self):
        return {'path': self.workspace['path'], 'files': [self.workspace['files']['startupfile']]}

    def supports_target(self, target):
        return target in self.SUPPORTED

    @staticmethod
    def is_supported_by_default(target):
        # does not require additional information
        return True
<|endoftext|>


<commit_before>#!/usr/bin/env python

# ============================================================================================================
# Created by: Lee Bergstrand & Matt McInnes
# Description: A next generation bioinformatics file format converter and sequence feature extractor.
# Requirements: This script requires the Biopython module: http://biopython.org/wiki/Download
# ============================================================================================================

from __future__ import print_function
from Bio import SeqIO
from Bio.Alphabet import IUPAC
from Bio import AlignIO
from Bio import Phylo
from BioID import BioID
from BioID import BioIDFormat
import ntpath
import os
import argparse
import sys
import yaml
import multiprocessing
import functools

# ==============================================================================
# BioMagickFormat: Defines the properties of a bioinformatic file format
#                  object with extra data specific for BioMagick's functionality
# ==============================================================================
class BioMagickFormat(BioIDFormat):
	def __init__(self, name, extension, bioclass):
		super(BioMagickFormat, self).__init__(name, None, None, [])
		self.extension = extension
		self.bioclass = bioclass

# ---------------------------------
# Command line interface controller
# ---------------------------------
def main(args):
	input_files = args.input
	out_fmt = args.outfmt
	out_dir = args.outdir[0] if args.outdir else "."

	if out_fmt is None or out_fmt == []:
		print("Error: at laest 1 output format is needed")
		sys.exit(1)

	if args.stdout is True:
		if len(input_files) > 1:
			print("Error: outputting to stdout is only possible with single-file conversions")
			sys.exit(1)
		else:
			out_dir = None  # Indicate to use stdout

	if sys.version_info[0] <= 2 and sys.version_info[1] <= 6:
		old_py = True

		if not args.stdout:
			print("Warning: disabling multicore processing for Python <2.7")
	else:
		old_py = False

	if args.alphabet is not None:
		alphabet = args.alphabet[0]
		if alphabet == 'ambigdna':
			alphabet = IUPAC.IUPACAmbiguousDNA()
		elif alphabet == "unambigdna":
			alphabet = IUPAC.IUPACUnambiguousDNA()
		elif alphabet == "exdna":
			alphabet = IUPAC.ExtendedIUPACDNA()
		elif alphabet == 'ambigrna':
			alphabet = IUPAC.IUPACAmbiguousRNA()
		elif alphabet == "unambigrna":
			alphabet = IUPAC.IUPACUnambiguousRNA()
		elif alphabet == 'prot':
			alphabet = IUPAC.IUPACProtein()
		elif alphabet == "exprot":
			alphabet = IUPAC.ExtendedIUPACProtein()
		else:
			print("Error: %s is not a valid alphabet" % alphabet)
			print("Valid alphabets: ambigdna, unambigdna, exdna, ambigrna, unambigrna, prot, exprot")
			sys.exit(1)
	else:
		alphabet = None

	# Load and parse YAML format export settings
	with open("BioMagickFormatInfo.yml", "rU") as settings_file:
		contents = settings_file.read()

	settings = {}
	for setting in yaml.safe_load(contents):
		settings[setting["name"]] = BioMagickFormat(setting["name"], setting["extension"], setting["bioclass"])

	if input_files is None or input_files == []:
		# Convert single file from stdin
		if sys.stdin.isatty():
			print("Error: you must either specify an input file or pipe some data to stdin")
			sys.exit(1)

		with open("converted.tmp", "w") as tmp_file:
			tmp_file.write(sys.stdin.read())

		id_results = BioID("./BioIDFormatInfo.yml").identify(["converted.tmp"])
		direct_convert(settings, id_results, out_dir, out_fmt, alphabet)
		os.remove("converted.tmp")
	elif len(input_files) == 1 or old_py:
		id_results = BioID("./BioIDFormatInfo.yml").identify(input_files)
		direct_convert(settings, id_results, out_dir, out_fmt, alphabet)
	else:
		if args.jobs is not None:
			process_count = args.jobs if args.jobs <= len(input_files) else len(input_files)
		else:
			process_count = multiprocessing.cpu_count() if multiprocessing.cpu_count() > len(input_files) else len(input_files)

		pool = multiprocessing.Pool(processes=process_count)
		pool.map(functools.partial(subprocess_controller, format_settings=settings, output_path=out_dir, output_formats=out_fmt,
		                           input_alphabet=alphabet), input_files)

# -----------------------------------------------------------
# Generates of dictionary of sequence record objects per file
# -----------------------------------------------------------
def generate_sequence_objects(id_results):
	seq_objects = {}

	for file_path, identity in id_results.items():
		with open(file_path, 'rU') as input_handle:
			new_seq_objects = SeqIO.parse(input_handle, identity)
			seq_objects[file_path] = new_seq_objects

	return seq_objects

# --------------------------------------------------------------------------
# A wrapper for the convert function which calls it from within a subprocess
# --------------------------------------------------------------------------
def subprocess_controller(input_file, format_settings, output_path, output_formats, input_alphabet):
	id_results = BioID("./BioIDFormatInfo.yml").identify([input_file])
	direct_convert(format_settings, id_results, output_path, output_formats, input_alphabet)

# --------------------------------------------------------------------------------------------
# Converts between bioinformatic formats using SeqIO's, AlignIO's and Phylo's convert function
# --------------------------------------------------------------------------------------------
def direct_convert(settings, id_results, out_path, out_formats, alphabet):
	if out_path is None:
		out_file = "./conv.tmp"
		in_path, in_format = list(id_results.items())[0]
		out_format = out_formats[0]

		if in_format == "unidentified":
			raise Exception("Failed to identify the file")

		try:
			format_setting = settings[in_format]
			if format_setting.bioclass == "seq":
				SeqIO.convert(in_path, in_format.lower(), out_file, out_format, alphabet)
			elif format_setting.bioclass == "phylo":
				Phylo.convert(in_path, in_format.lower(), out_file, out_format)
			elif format_setting.bioclass == "align":
				AlignIO.convert(in_path, in_format.lower(), out_file, out_format)
			else:
				print("Error: invalid BioPython conversion class: %s" % format_setting.bioclass)
				sys.exit(1)
		except ValueError as e:
			print("Error in conversion of " + in_path + " to " + out_format + ": " + str(e))
			sys.exit(1)

		with open(out_file, "r") as tmp_file:
			print(tmp_file.read())

		os.remove(out_file)  # Is this really necessary?
	else:
		for out_format in out_formats:
			for in_path, in_format in id_results.items():
				out_file = out_path
				if sys.platform == "win32":
					if out_file[-1] != "\\":
						out_file += "\\"

					out_file += ntpath.basename(in_path).split('.')[0]
				else:
					if out_file[-1] != "/":
						out_file += "/"

					out_file += os.path.basename(in_path).split('.')[0]

				out_extension = settings[out_format].extension
				out_file = out_file + "." + out_extension
				print("\nConverting %s file %s to %s file %s" % (in_format, in_path, out_format, out_file))

				try:
					format_setting = settings[in_format]
					if format_setting.bioclass == "seq":
						SeqIO.convert(in_path, in_format.lower(), out_file, out_format, alphabet)
					elif format_setting.bioclass == "phylo":
						Phylo.convert(in_path, in_format.lower(), out_file, out_format)
					elif format_setting.bioclass == "align":
						AlignIO.convert(in_path, in_format.lower(), out_file, out_format)
					else:
						print("Error: invalid BioPython conversion class: %s" % format_setting.bioclass)
						sys.exit(1)
				except ValueError as e:
					print("\nError in conversion of " + in_path + " to " + out_format + ": " + str(e))
					print("Skipping " + in_path + " ...\n")
					continue

if __name__ == '__main__':
	# ------------------------------
	# Command line interface options
	# ------------------------------
	parser = argparse.ArgumentParser()
	parser.add_argument('-i', '--input', metavar='INPATH', nargs='1', help='''
	A comma-separated list of input file paths. If not specified, input is read from stdin.''')

	parser.add_argument('-s', '--stdout', dest="stdout", action='store_true', help='''
	Output result of single-file conversion to stdout.''')

	parser.add_argument('-o', '--outdir', metavar='OUTPATH', nargs=1, help='''
	An output directory for output files. If not specified, the current working directory is used.''')

	parser.add_argument('-f', '--outfmt', metavar='OUTFORMAT', nargs='1', help='''
	A comma-separated list of output file formats.''')

	parser.add_argument('-a', '--alphabet', metavar='ALPHA', nargs=1, help='''
	The alphabet to use for conversion (ambigdna, unambigdna, exdna, ambigrna, unambigrna, prot, exprot).''')

	parser.add_argument('-j', '--jobs', metavar='JOBS', nargs=1, type=int, help='''
	The number of processes to use for multiple files (defaults to the number of processor cores).''')

	cli_args = parser.parse_args()

	try:
		if cli_args.input:
			cli_args.input = cli_args.input[0].split(",")
		cli_args.outfmt = cli_args.outfmt[0].split(",")
	except Exception as ex:
		parser.print_help()
		sys.exit(1)

	try:
		main(cli_args)
	except Exception as ex:
		print("\nError: %s" % ex)
		sys.exit(1)
<commit_msg>More argparse fixing<commit_after>#!/usr/bin/env python

# ============================================================================================================
# Created by: Lee Bergstrand & Matt McInnes
# Description: A next generation bioinformatics file format converter and sequence feature extractor.
# Requirements: This script requires the Biopython module: http://biopython.org/wiki/Download
# ============================================================================================================

from __future__ import print_function
from Bio import SeqIO
from Bio.Alphabet import IUPAC
from Bio import AlignIO
from Bio import Phylo
from BioID import BioID
from BioID import BioIDFormat
import ntpath
import os
import argparse
import sys
import yaml
import multiprocessing
import functools

# ==============================================================================
# BioMagickFormat: Defines the properties of a bioinformatic file format
#                  object with extra data specific for BioMagick's functionality
# ==============================================================================
class BioMagickFormat(BioIDFormat):
	def __init__(self, name, extension, bioclass):
		super(BioMagickFormat, self).__init__(name, None, None, [])
		self.extension = extension
		self.bioclass = bioclass

# ---------------------------------
# Command line interface controller
# ---------------------------------
def main(args):
	input_files = args.input
	out_fmt = args.outfmt
	out_dir = args.outdir[0] if args.outdir else "."

	if out_fmt is None or out_fmt == []:
		print("Error: at laest 1 output format is needed")
		sys.exit(1)

	if args.stdout is True:
		if len(input_files) > 1:
			print("Error: outputting to stdout is only possible with single-file conversions")
			sys.exit(1)
		else:
			out_dir = None  # Indicate to use stdout

	if sys.version_info[0] <= 2 and sys.version_info[1] <= 6:
		old_py = True

		if not args.stdout:
			print("Warning: disabling multicore processing for Python <2.7")
	else:
		old_py = False

	if args.alphabet is not None:
		alphabet = args.alphabet[0]
		if alphabet == 'ambigdna':
			alphabet = IUPAC.IUPACAmbiguousDNA()
		elif alphabet == "unambigdna":
			alphabet = IUPAC.IUPACUnambiguousDNA()
		elif alphabet == "exdna":
			alphabet = IUPAC.ExtendedIUPACDNA()
		elif alphabet == 'ambigrna':
			alphabet = IUPAC.IUPACAmbiguousRNA()
		elif alphabet == "unambigrna":
			alphabet = IUPAC.IUPACUnambiguousRNA()
		elif alphabet == 'prot':
			alphabet = IUPAC.IUPACProtein()
		elif alphabet == "exprot":
			alphabet = IUPAC.ExtendedIUPACProtein()
		else:
			print("Error: %s is not a valid alphabet" % alphabet)
			print("Valid alphabets: ambigdna, unambigdna, exdna, ambigrna, unambigrna, prot, exprot")
			sys.exit(1)
	else:
		alphabet = None

	# Load and parse YAML format export settings
	with open("BioMagickFormatInfo.yml", "rU") as settings_file:
		contents = settings_file.read()

	settings = {}
	for setting in yaml.safe_load(contents):
		settings[setting["name"]] = BioMagickFormat(setting["name"], setting["extension"], setting["bioclass"])

	if input_files is None or input_files == []:
		# Convert single file from stdin
		if sys.stdin.isatty():
			print("Error: you must either specify an input file or pipe some data to stdin")
			sys.exit(1)

		with open("converted.tmp", "w") as tmp_file:
			tmp_file.write(sys.stdin.read())

		id_results = BioID("./BioIDFormatInfo.yml").identify(["converted.tmp"])
		direct_convert(settings, id_results, out_dir, out_fmt, alphabet)
		os.remove("converted.tmp")
	elif len(input_files) == 1 or old_py:
		id_results = BioID("./BioIDFormatInfo.yml").identify(input_files)
		direct_convert(settings, id_results, out_dir, out_fmt, alphabet)
	else:
		if args.jobs is not None:
			process_count = args.jobs if args.jobs <= len(input_files) else len(input_files)
		else:
			process_count = multiprocessing.cpu_count() if multiprocessing.cpu_count() > len(input_files) else len(input_files)

		pool = multiprocessing.Pool(processes=process_count)
		pool.map(functools.partial(subprocess_controller, format_settings=settings, output_path=out_dir, output_formats=out_fmt,
		                           input_alphabet=alphabet), input_files)

# -----------------------------------------------------------
# Generates of dictionary of sequence record objects per file
# -----------------------------------------------------------
def generate_sequence_objects(id_results):
	seq_objects = {}

	for file_path, identity in id_results.items():
		with open(file_path, 'rU') as input_handle:
			new_seq_objects = SeqIO.parse(input_handle, identity)
			seq_objects[file_path] = new_seq_objects

	return seq_objects

# --------------------------------------------------------------------------
# A wrapper for the convert function which calls it from within a subprocess
# --------------------------------------------------------------------------
def subprocess_controller(input_file, format_settings, output_path, output_formats, input_alphabet):
	id_results = BioID("./BioIDFormatInfo.yml").identify([input_file])
	direct_convert(format_settings, id_results, output_path, output_formats, input_alphabet)

# --------------------------------------------------------------------------------------------
# Converts between bioinformatic formats using SeqIO's, AlignIO's and Phylo's convert function
# --------------------------------------------------------------------------------------------
def direct_convert(settings, id_results, out_path, out_formats, alphabet):
	if out_path is None:
		out_file = "./conv.tmp"
		in_path, in_format = list(id_results.items())[0]
		out_format = out_formats[0]

		if in_format == "unidentified":
			raise Exception("Failed to identify the file")

		try:
			format_setting = settings[in_format]
			if format_setting.bioclass == "seq":
				SeqIO.convert(in_path, in_format.lower(), out_file, out_format, alphabet)
			elif format_setting.bioclass == "phylo":
				Phylo.convert(in_path, in_format.lower(), out_file, out_format)
			elif format_setting.bioclass == "align":
				AlignIO.convert(in_path, in_format.lower(), out_file, out_format)
			else:
				print("Error: invalid BioPython conversion class: %s" % format_setting.bioclass)
				sys.exit(1)
		except ValueError as e:
			print("Error in conversion of " + in_path + " to " + out_format + ": " + str(e))
			sys.exit(1)

		with open(out_file, "r") as tmp_file:
			print(tmp_file.read())

		os.remove(out_file)  # Is this really necessary?
	else:
		for out_format in out_formats:
			for in_path, in_format in id_results.items():
				out_file = out_path
				if sys.platform == "win32":
					if out_file[-1] != "\\":
						out_file += "\\"

					out_file += ntpath.basename(in_path).split('.')[0]
				else:
					if out_file[-1] != "/":
						out_file += "/"

					out_file += os.path.basename(in_path).split('.')[0]

				out_extension = settings[out_format].extension
				out_file = out_file + "." + out_extension
				print("\nConverting %s file %s to %s file %s" % (in_format, in_path, out_format, out_file))

				try:
					format_setting = settings[in_format]
					if format_setting.bioclass == "seq":
						SeqIO.convert(in_path, in_format.lower(), out_file, out_format, alphabet)
					elif format_setting.bioclass == "phylo":
						Phylo.convert(in_path, in_format.lower(), out_file, out_format)
					elif format_setting.bioclass == "align":
						AlignIO.convert(in_path, in_format.lower(), out_file, out_format)
					else:
						print("Error: invalid BioPython conversion class: %s" % format_setting.bioclass)
						sys.exit(1)
				except ValueError as e:
					print("\nError in conversion of " + in_path + " to " + out_format + ": " + str(e))
					print("Skipping " + in_path + " ...\n")
					continue

if __name__ == '__main__':
	# ------------------------------
	# Command line interface options
	# ------------------------------
	parser = argparse.ArgumentParser()
	parser.add_argument('-i', '--input', metavar='INPATH', nargs=1, help='''
	A comma-separated list of input file paths. If not specified, input is read from stdin.''')

	parser.add_argument('-s', '--stdout', dest="stdout", action='store_true', help='''
	Output result of single-file conversion to stdout.''')

	parser.add_argument('-o', '--outdir', metavar='OUTPATH', nargs=1, help='''
	An output directory for output files. If not specified, the current working directory is used.''')

	parser.add_argument('-f', '--outfmt', metavar='OUTFORMAT', nargs=1, help='''
	A comma-separated list of output file formats.''')

	parser.add_argument('-a', '--alphabet', metavar='ALPHA', nargs=1, help='''
	The alphabet to use for conversion (ambigdna, unambigdna, exdna, ambigrna, unambigrna, prot, exprot).''')

	parser.add_argument('-j', '--jobs', metavar='JOBS', nargs=1, type=int, help='''
	The number of processes to use for multiple files (defaults to the number of processor cores).''')

	cli_args = parser.parse_args()

	try:
		if cli_args.input:
			cli_args.input = cli_args.input[0].split(",")
		cli_args.outfmt = cli_args.outfmt[0].split(",")
	except Exception as ex:
		parser.print_help()
		sys.exit(1)

	try:
		main(cli_args)
	except Exception as ex:
		print("\nError: %s" % ex)
		sys.exit(1)
<|endoftext|>


<commit_before>#!/usr/bin/env python
#-*- coding: utf-8 -*-
from __future__ import print_function

from importlib import import_module
import os
import sys

from pip.req import parse_requirements
from setuptools import setup
from setuptools.command.test import test as _TestCommand

######################################################################
# Temporary fix to https://github.com/globaleaks/GlobaLeaks/issues/572
#                  https://github.com/habnabit/txsocksx/issues/5
from distutils import version
version.StrictVersion = version.LooseVersion
######################################################################

import globaleaks

GLCLIENT_PATH = os.path.join(os.path.dirname(__file__),
                             '..', 'client', 'build')

if not sys.version_info[:2] == (2, 7):
    print('Error, GlobaLeaks is tested only with python 2.7')
    print('https://github.com/globaleaks/GlobaLeaks/wiki/Technical-requirements')
    raise AssertionError

install_requires = [str(r.req) for r in parse_requirements('requirements.txt')]

def list_files(path):
    """
    Return the list of files present in a directory.
    """
    result = []
    if not os.path.exists(path):
        print('Warning: {} does not exist.'.format(path))
        return []

    for f in os.listdir(path):
        f = os.path.join(path, f)
        if os.path.isfile(f):
            result.append(f)

    return result

class TestCommand(_TestCommand):
    def run_tests(self):
        from twisted.trial import runner, reporter

        testsuite_runner= runner.TrialRunner(reporter.TreeReporter)
        loader = runner.TestLoader()
        suite = loader.loadPackage(import_module(self.test_suite), True)
        test_result = testsuite_runner.run(suite)
        sys.exit(not test_result.wasSuccessful())

data_files = [
    ('/usr/share/globaleaks/glclient',
     list_files(os.path.join(GLCLIENT_PATH))),
    ('/usr/share/globaleaks/glclient/data',
     list_files(os.path.join(GLCLIENT_PATH, 'data'))),
    ('/usr/share/globaleaks/glclient/fonts',
     list_files(os.path.join(GLCLIENT_PATH, 'fonts'))),
    ('/usr/share/globaleaks/glclient/img',
     list_files(os.path.join(GLCLIENT_PATH, 'img'))),
    ('/usr/share/globaleaks/glclient/l10n',
     list_files(os.path.join(GLCLIENT_PATH, 'l10n'))),
    ('/usr/share/globaleaks/glbackend',
     ['requirements.txt'] + list_files('staticdata'))
]

setup(
    name='globaleaks',
    version=globaleaks.__version__,
    author=globaleaks.__author__,
    author_email=globaleaks.__email__,
    url='https://globaleaks.org/',
    cmdclass={'test': TestCommand},
    package_dir={'globaleaks': 'globaleaks'},
    test_suite='globaleaks.tests',
    package_data={'globaleaks': [
        'db/sqlite.sql',
    ]},
    packages=[
        'globaleaks',
        'globaleaks.db',
        'globaleaks.handlers',
        'globaleaks.jobs',
        'globaleaks.plugins',
        'globaleaks.rest',
        'globaleaks.third_party',
        'globaleaks.third_party.rstr',
        'globaleaks.utils',
    ],
    data_files=data_files,
    scripts=[
        'bin/globaleaks',
        'bin/gl-admin',
        'bin/gl-fix-permissions',
    ],
    install_requires=install_requires,
    tests_require=['mock'],
)
<commit_msg>Autodiscover globaleaks packages in setup.py.<commit_after>#!/usr/bin/env python
#-*- coding: utf-8 -*-
from __future__ import print_function

from importlib import import_module
import os
import sys

from pip.req import parse_requirements
from setuptools import find_packages, setup
from setuptools.command.test import test as _TestCommand

######################################################################
# Temporary fix to https://github.com/globaleaks/GlobaLeaks/issues/572
#                  https://github.com/habnabit/txsocksx/issues/5
from distutils import version
version.StrictVersion = version.LooseVersion
######################################################################

import globaleaks

GLCLIENT_PATH = os.path.join(os.path.dirname(__file__),
                             '..', 'client', 'build')

if not sys.version_info[:2] == (2, 7):
    print('Error, GlobaLeaks is tested only with python 2.7')
    print('https://github.com/globaleaks/GlobaLeaks/wiki/Technical-requirements')
    raise AssertionError

install_requires = [str(r.req) for r in parse_requirements('requirements.txt')]

def list_files(path):
    """
    Return the list of files present in a directory.
    """
    result = []
    if not os.path.exists(path):
        print('Warning: {} does not exist.'.format(path))
        return []

    for f in os.listdir(path):
        f = os.path.join(path, f)
        if os.path.isfile(f):
            result.append(f)

    return result

class TestCommand(_TestCommand):
    def run_tests(self):
        from twisted.trial import runner, reporter

        testsuite_runner= runner.TrialRunner(reporter.TreeReporter)
        loader = runner.TestLoader()
        suite = loader.loadPackage(import_module(self.test_suite), True)
        test_result = testsuite_runner.run(suite)
        sys.exit(not test_result.wasSuccessful())

data_files = [
    ('/usr/share/globaleaks/glclient',
     list_files(os.path.join(GLCLIENT_PATH))),
    ('/usr/share/globaleaks/glclient/data',
     list_files(os.path.join(GLCLIENT_PATH, 'data'))),
    ('/usr/share/globaleaks/glclient/fonts',
     list_files(os.path.join(GLCLIENT_PATH, 'fonts'))),
    ('/usr/share/globaleaks/glclient/img',
     list_files(os.path.join(GLCLIENT_PATH, 'img'))),
    ('/usr/share/globaleaks/glclient/l10n',
     list_files(os.path.join(GLCLIENT_PATH, 'l10n'))),
    ('/usr/share/globaleaks/glbackend',
     ['requirements.txt'] + list_files('staticdata'))
]

setup(
    name='globaleaks',
    version=globaleaks.__version__,
    author=globaleaks.__author__,
    author_email=globaleaks.__email__,
    url='https://globaleaks.org/',
    cmdclass={'test': TestCommand},
    package_dir={'globaleaks': 'globaleaks'},
    test_suite='globaleaks.tests',
    package_data={'globaleaks': [
        'db/sqlite.sql',
    ]},
    packages=find_packages(exclude=['*.tests', '*.tests.*']),
    data_files=data_files,
    scripts=[
        'bin/globaleaks',
        'bin/gl-admin',
        'bin/gl-fix-permissions',
    ],
    install_requires=install_requires,
    tests_require=['mock'],
)
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-

# Copyright(C) 2010-2011  Romain Bignon
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see <http://www.gnu.org/licenses/>.

import re

from weboob.tools.browser import BasePage

class Message(object):
    TIMESTAMP_REGEXP = re.compile(r'(\d{4})(\d{2})(\d{2})(\d{2})(\d{2})(\d{2})')
    def __init__(self, id, timestamp, login, message, is_me):
        self.id = id
        self.timestamp = timestamp
        self.login = login
        self.message = message
        self.is_me = is_me
        self.norloge = timestamp
        m = self.TIMESTAMP_REGEXP.match(timestamp)
        if m:
            self.norloge = '%02d:%02d:%02d' % (int(m.group(4)),
                                               int(m.group(5)),
                                               int(m.group(6)))
        else:
            warning('Unable to parse timestamp "%s"' % timestamp)

class BoardIndexPage(BasePage):
    def is_logged(self):
        return True

    def get_messages(self, last=None):
        msgs = []
        for post in self.parser.select(self.document.getroot(), 'post'):
            m = Message(int(post.attrib['id']),
                        post.attrib['time'],
                        post.find('login').text,
                        post.find('message').text,
                        post.find('login').text.lower() == self.browser.username.lower())
            if last is not None and last == m.id:
                break
            msgs.append(m)
        return msgs
<commit_msg>missing import<commit_after># -*- coding: utf-8 -*-

# Copyright(C) 2010-2011  Romain Bignon
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see <http://www.gnu.org/licenses/>.

import re
from logging import warning

from weboob.tools.browser import BasePage

class Message(object):
    TIMESTAMP_REGEXP = re.compile(r'(\d{4})(\d{2})(\d{2})(\d{2})(\d{2})(\d{2})')
    def __init__(self, id, timestamp, login, message, is_me):
        self.id = id
        self.timestamp = timestamp
        self.login = login
        self.message = message
        self.is_me = is_me
        self.norloge = timestamp
        m = self.TIMESTAMP_REGEXP.match(timestamp)
        if m:
            self.norloge = '%02d:%02d:%02d' % (int(m.group(4)),
                                               int(m.group(5)),
                                               int(m.group(6)))
        else:
            warning('Unable to parse timestamp "%s"' % timestamp)

class BoardIndexPage(BasePage):
    def is_logged(self):
        return True

    def get_messages(self, last=None):
        msgs = []
        for post in self.parser.select(self.document.getroot(), 'post'):
            m = Message(int(post.attrib['id']),
                        post.attrib['time'],
                        post.find('login').text,
                        post.find('message').text,
                        post.find('login').text.lower() == self.browser.username.lower())
            if last is not None and last == m.id:
                break
            msgs.append(m)
        return msgs
<|endoftext|>


<commit_before>import json

from collections import namedtuple

from six.moves.urllib.parse import urlencode

from .exceptions import ObjectDoesNotExist

all_ = object()
everything = object()
now = object()

class BaseQuery(object):

    def __init__(self, api, api_obj_class, namespace=None):
        self.api = api
        self.api_obj_class = api_obj_class
        self.namespace = namespace
        self.selector = everything

    def all(self):
        return self._clone()

    def filter(self, namespace=None, selector=None):
        clone = self._clone()
        if namespace is not None:
            clone.namespace = namespace
        if selector is not None:
            clone.selector = selector
        return clone

    def _clone(self):
        return self.__class__(self.api, self.api_obj_class, namespace=self.namespace)

    def _build_api_url(self, params=None):
        if params is None:
            params = {}
        if self.selector is not everything:
            params["labelSelector"] = as_selector(self.selector)
        query_string = urlencode(params)
        return "{}{}".format(self.api_obj_class.endpoint, "?{}".format(query_string) if query_string else "")

class Query(BaseQuery):

    def get_by_name(self, name):
        kwargs = {
            "url": "{}/{}".format(self.api_obj_class.endpoint, name),
            "namespace": self.namespace,
        }
        if self.api_obj_class.base:
            kwargs["base"] = self.api_obj_class.base
        if self.api_obj_class.version:
            kwargs["version"] = self.api_obj_class.version
        r = self.api.get(**kwargs)
        if not r.ok:
            if r.status_code == 404:
                raise ObjectDoesNotExist("{} does not exist.".format(name))
            r.raise_for_status()
        return self.api_obj_class(self.api, r.json())

    def get(self, *args, **kwargs):
        if "name" in kwargs:
            return self.get_by_name(kwargs["name"])
        clone = self.filter(*args, **kwargs)
        num = len(clone)
        if num == 1:
            return clone.query_cache[0]
        if not num:
            raise ObjectDoesNotExist("get() returned zero objects")
        raise ValueError("get() more than one object; use filter")

    def watch(self, since=None):
        kwargs = {"namespace": self.namespace}
        if since is now:
            kwargs["resource_version"] = self.response["metadata"]["resourceVersion"]
        elif since is not None:
            kwargs["resource_version"] = since
        return WatchQuery(self.api, self.api_obj_class, **kwargs)

    @property
    def query_cache(self):
        if not hasattr(self, "_query_cache"):
            cache = {"objects": []}
            kwargs = {"url": self._build_api_url()}
            if self.api_obj_class.base:
                kwargs["base"] = self.api_obj_class.base
            if self.api_obj_class.version:
                kwargs["version"] = self.api_obj_class.version
            if self.namespace is not None and self.namespace is not all_:
                kwargs["namespace"] = self.namespace
            r = self.api.get(**kwargs)
            r.raise_for_status()
            cache["response"] = r.json()
            for obj in cache["response"]["items"]:
                cache["objects"].append(self.api_obj_class(self.api, obj))
            self._query_cache = cache
        return self._query_cache

    def __len__(self):
        return len(self.query_cache["objects"])

    def __iter__(self):
        return iter(self.query_cache["objects"])

    @property
    def response(self):
        return self.query_cache["response"]

class WatchQuery(BaseQuery):

    def __init__(self, *args, resource_version=None, **kwargs):
        self.resource_version = resource_version
        super(WatchQuery, self).__init__(*args, **kwargs)

    def object_stream(self):
        params = {}
        if self.resource_version is not None:
            params["resourceVersion"] = self.resource_version
        url = self._build_api_url(params=params)
        if self.namespace is all_:
            url = "watch/" + url
        kwargs = {
            "url": url,
            "stream": True,
        }
        r = self.api.get(**kwargs)
        r.raise_for_status()
        WatchEvent = namedtuple("WatchEvent", "type object")
        for line in r.iter_lines():
            we = json.loads(line.decode("utf-8"))
            yield WatchEvent(type=we["type"], object=self.api_obj_class(self.api, we["object"]))

    def __iter__(self):
        return iter(self.object_stream())

class ObjectManager(object):

    def __init__(self, namespace=None):
        self.namespace = namespace

    def __call__(self, api):
        return Query(api, self.api_obj_class, namespace=self.namespace)

    def __get__(self, obj, api_obj_class):
        assert obj is None, "cannot invoke objects on resource object."
        self.api_obj_class = api_obj_class
        return self

def as_selector(value):
    if isinstance(value, str):
        return value
    s = []
    for k, v in value.items():
        bits = k.split("__")
        assert len(bits) <= 2, "too many __ in selector"
        if len(bits) == 1:
            label = bits[0]
            op = "eq"
        else:
            label = bits[0]
            op = bits[1]
        # map operator to selector
        if op == "eq":
            s.append("{} = {}".format(label, v))
        elif op == "neq":
            s.append("{} != {}".format(label, v))
        elif op == "in":
            s.append("{} in ({})".format(label, ",".join(v)))
        elif op == "notin":
            s.append("{} notin ({})".format(label, ",".join(v)))
        else:
            raise ValueError("{} is not a valid comparsion operator".format(op))
    return ",".join(s)
<commit_msg>Fixed syntax for Python 2<commit_after>import json

from collections import namedtuple

from six.moves.urllib.parse import urlencode

from .exceptions import ObjectDoesNotExist

all_ = object()
everything = object()
now = object()

class BaseQuery(object):

    def __init__(self, api, api_obj_class, namespace=None):
        self.api = api
        self.api_obj_class = api_obj_class
        self.namespace = namespace
        self.selector = everything

    def all(self):
        return self._clone()

    def filter(self, namespace=None, selector=None):
        clone = self._clone()
        if namespace is not None:
            clone.namespace = namespace
        if selector is not None:
            clone.selector = selector
        return clone

    def _clone(self):
        return self.__class__(self.api, self.api_obj_class, namespace=self.namespace)

    def _build_api_url(self, params=None):
        if params is None:
            params = {}
        if self.selector is not everything:
            params["labelSelector"] = as_selector(self.selector)
        query_string = urlencode(params)
        return "{}{}".format(self.api_obj_class.endpoint, "?{}".format(query_string) if query_string else "")

class Query(BaseQuery):

    def get_by_name(self, name):
        kwargs = {
            "url": "{}/{}".format(self.api_obj_class.endpoint, name),
            "namespace": self.namespace,
        }
        if self.api_obj_class.base:
            kwargs["base"] = self.api_obj_class.base
        if self.api_obj_class.version:
            kwargs["version"] = self.api_obj_class.version
        r = self.api.get(**kwargs)
        if not r.ok:
            if r.status_code == 404:
                raise ObjectDoesNotExist("{} does not exist.".format(name))
            r.raise_for_status()
        return self.api_obj_class(self.api, r.json())

    def get(self, *args, **kwargs):
        if "name" in kwargs:
            return self.get_by_name(kwargs["name"])
        clone = self.filter(*args, **kwargs)
        num = len(clone)
        if num == 1:
            return clone.query_cache[0]
        if not num:
            raise ObjectDoesNotExist("get() returned zero objects")
        raise ValueError("get() more than one object; use filter")

    def watch(self, since=None):
        kwargs = {"namespace": self.namespace}
        if since is now:
            kwargs["resource_version"] = self.response["metadata"]["resourceVersion"]
        elif since is not None:
            kwargs["resource_version"] = since
        return WatchQuery(self.api, self.api_obj_class, **kwargs)

    @property
    def query_cache(self):
        if not hasattr(self, "_query_cache"):
            cache = {"objects": []}
            kwargs = {"url": self._build_api_url()}
            if self.api_obj_class.base:
                kwargs["base"] = self.api_obj_class.base
            if self.api_obj_class.version:
                kwargs["version"] = self.api_obj_class.version
            if self.namespace is not None and self.namespace is not all_:
                kwargs["namespace"] = self.namespace
            r = self.api.get(**kwargs)
            r.raise_for_status()
            cache["response"] = r.json()
            for obj in cache["response"]["items"]:
                cache["objects"].append(self.api_obj_class(self.api, obj))
            self._query_cache = cache
        return self._query_cache

    def __len__(self):
        return len(self.query_cache["objects"])

    def __iter__(self):
        return iter(self.query_cache["objects"])

    @property
    def response(self):
        return self.query_cache["response"]

class WatchQuery(BaseQuery):

    def __init__(self, *args, **kwargs):
        self.resource_version = kwargs.pop("resource_version", None)
        super(WatchQuery, self).__init__(*args, **kwargs)

    def object_stream(self):
        params = {}
        if self.resource_version is not None:
            params["resourceVersion"] = self.resource_version
        url = self._build_api_url(params=params)
        if self.namespace is all_:
            url = "watch/" + url
        kwargs = {
            "url": url,
            "stream": True,
        }
        r = self.api.get(**kwargs)
        r.raise_for_status()
        WatchEvent = namedtuple("WatchEvent", "type object")
        for line in r.iter_lines():
            we = json.loads(line.decode("utf-8"))
            yield WatchEvent(type=we["type"], object=self.api_obj_class(self.api, we["object"]))

    def __iter__(self):
        return iter(self.object_stream())

class ObjectManager(object):

    def __init__(self, namespace=None):
        self.namespace = namespace

    def __call__(self, api):
        return Query(api, self.api_obj_class, namespace=self.namespace)

    def __get__(self, obj, api_obj_class):
        assert obj is None, "cannot invoke objects on resource object."
        self.api_obj_class = api_obj_class
        return self

def as_selector(value):
    if isinstance(value, str):
        return value
    s = []
    for k, v in value.items():
        bits = k.split("__")
        assert len(bits) <= 2, "too many __ in selector"
        if len(bits) == 1:
            label = bits[0]
            op = "eq"
        else:
            label = bits[0]
            op = bits[1]
        # map operator to selector
        if op == "eq":
            s.append("{} = {}".format(label, v))
        elif op == "neq":
            s.append("{} != {}".format(label, v))
        elif op == "in":
            s.append("{} in ({})".format(label, ",".join(v)))
        elif op == "notin":
            s.append("{} notin ({})".format(label, ",".join(v)))
        else:
            raise ValueError("{} is not a valid comparsion operator".format(op))
    return ",".join(s)
<|endoftext|>


<commit_before># -*- encoding: utf-8 -*-

# Dissemin: open access policy enforcement tool
# Copyright (C) 2014 Antonin Delpeuch
#
# This program is free software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License
# as published by the Free Software Foundation; either version 2
# of the License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
#

from __future__ import absolute_import, unicode_literals
import re

from celery import shared_task
from celery.utils.log import get_task_logger
from celery import current_task

from django.utils import timezone
from django.core.exceptions import ObjectDoesNotExist

from oaipmh.datestamp import tolerant_datestamp_to_datetime
from oaipmh.error import DatestampError, NoRecordsMatchError, BadArgumentError

from papers.models import *
from papers.doi import to_doi

from backend.globals import get_ccf
from backend.crossref import *
from backend.oai import *
from backend.core import CorePaperSource
from backend.base import BasePaperSource
from backend.orcid import *
from backend.name_cache import name_lookup_cache
from backend.extractors import * # to ensure that OaiSources are created
from backend.utils import run_only_once

logger = get_task_logger(__name__)

def update_researcher_task(r, task_name):
    """
    Update the task identifier associated to a given researcher.
    This also updates the 'last_harvest' field.
    """
    r.current_task = task_name
    r.last_harvest = timezone.now()
    r.save(update_fields=['current_task','last_harvest'])

@shared_task(name='init_profile_from_orcid')
@run_only_once('researcher', keys=['pk'])
def init_profile_from_orcid(pk):
    """
    Populates the profile from ORCID and Proaixy.
    Does not fetch DOIs from ORCID as it can be slow.

    This task is intended to be very quick, so that users
    can see their ORCID publications quickly.
    """

    ccf = get_ccf()
    r = Researcher.objects.get(pk=pk)
    update_task = lambda name: update_researcher_task(r, name)
    update_task('clustering')
    ccf.reclusterBatch(r)
    fetch_everything_for_researcher(pk)

@shared_task(name='fetch_everything_for_researcher')
@run_only_once('researcher', keys=['pk'], timeout=15*60)
def fetch_everything_for_researcher(pk):
    ccf = get_ccf()
    oai = OaiPaperSource(ccf, max_results=250)
    sources = [
        ('orcid',OrcidPaperSource(ccf, oai, max_results=1000)),
        ('crossref',CrossRefPaperSource(ccf, oai, max_results=500)),
        ('oai',oai),
        ('base',BasePaperSource(ccf, max_results=200)),
       # ('core',CorePaperSource(ccf, max_results=250)),
       ]
    try:
        r = Researcher.objects.get(pk=pk)
        update_task = lambda name: update_researcher_task(r, name)
        for key,source in sources:
            update_task(key)
            source.fetch(r, incremental=True)

        #fetch_papers_from_core_for_researcher(r)
        #fetch_papers_from_base_for_researcher(Researcher.objects.get(pk=pk))
    except MetadataSourceException as e:
        raise e
    finally:
        update_task('clustering')
        ccf.commitThemAll()
        ccf.unloadResearcher(pk)
        r = Researcher.objects.get(pk=pk)
        update_task('stats')
        r.update_stats()
        r.harvester = None
        update_task(None)

@shared_task(name='fetch_records_for_researcher')
def fetch_records_for_researcher(pk, signature=True):
    """
    Fetch OAI records from Proaixy for the given researcher.

    :param signature: Search by name signature (D. Knuth) instead
       of full name (Donald Knuth)
    """
    researcher = Researcher.objects.get(pk=pk)
    fetch_records_for_name(researcher.name, signature=signature)

@shared_task(name='recluster_researcher')
def recluster_researcher(pk):
    ccf = get_ccf()
    try:
        r = Researcher.objects.get(pk=pk)
        ccf.reclusterBatch(r)
    finally:
        r.update_stats()
        update_researcher_task(r, None)

@shared_task(name='fetch_dois_for_researcher')
def fetch_dois_for_researcher(pk):
    researcher = Researcher.objects.get(pk=pk)
    return fetch_publications(researcher)

@shared_task(name='change_publisher_oa_status')
def change_publisher_oa_status(pk, status):
    publisher = Publisher.objects.get(pk=pk)
    publisher.change_oa_status(status)
    publisher.update_stats()

@shared_task(name='consolidate_paper')
@run_only_once('consolidate_paper', keys=['pk'], timeout=1*60)
def consolidate_paper(pk):
    p = None
    try:
        p = Paper.objects.get(pk=pk)
        abstract = p.abstract or ''
        for pub in p.publication_set.all():
            pub = consolidate_publication(pub)
            if pub.abstract and len(pub.abstract) > len(abstract):
                abstract = pub.abstract
                break
    except Paper.DoesNotExist:
        print "consolidate_paper: unknown paper %d" % pk
    finally:
        if p is not None:
            p.task = None
            p.save(update_fields=['task'])

@shared_task(name='update_all_stats')
def update_all_stats():
    """
    Updates the stats for every model using them
    """
    AccessStatistics.update_all_stats(PaperWorld)
    AccessStatistics.update_all_stats(Publisher)
    AccessStatistics.update_all_stats(Journal)
    AccessStatistics.update_all_stats(Researcher)

@shared_task(name='update_all_stats_but_researchers')
def update_all_stats_but_researchers():
    """
    Updates the stats for every model using them
    """
    AccessStatistics.update_all_stats(PaperWorld)
    AccessStatistics.update_all_stats(Publisher)
    AccessStatistics.update_all_stats(Journal)

<commit_msg>Run stats update once at a time<commit_after># -*- encoding: utf-8 -*-

# Dissemin: open access policy enforcement tool
# Copyright (C) 2014 Antonin Delpeuch
#
# This program is free software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License
# as published by the Free Software Foundation; either version 2
# of the License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
#

from __future__ import absolute_import, unicode_literals
import re

from celery import shared_task
from celery.utils.log import get_task_logger
from celery import current_task

from django.utils import timezone
from django.core.exceptions import ObjectDoesNotExist

from oaipmh.datestamp import tolerant_datestamp_to_datetime
from oaipmh.error import DatestampError, NoRecordsMatchError, BadArgumentError

from papers.models import *
from papers.doi import to_doi

from backend.globals import get_ccf
from backend.crossref import *
from backend.oai import *
from backend.core import CorePaperSource
from backend.base import BasePaperSource
from backend.orcid import *
from backend.name_cache import name_lookup_cache
from backend.extractors import * # to ensure that OaiSources are created
from backend.utils import run_only_once

logger = get_task_logger(__name__)

def update_researcher_task(r, task_name):
    """
    Update the task identifier associated to a given researcher.
    This also updates the 'last_harvest' field.
    """
    r.current_task = task_name
    r.last_harvest = timezone.now()
    r.save(update_fields=['current_task','last_harvest'])

@shared_task(name='init_profile_from_orcid')
@run_only_once('researcher', keys=['pk'])
def init_profile_from_orcid(pk):
    """
    Populates the profile from ORCID and Proaixy.
    Does not fetch DOIs from ORCID as it can be slow.

    This task is intended to be very quick, so that users
    can see their ORCID publications quickly.
    """

    ccf = get_ccf()
    r = Researcher.objects.get(pk=pk)
    update_task = lambda name: update_researcher_task(r, name)
    update_task('clustering')
    ccf.reclusterBatch(r)
    fetch_everything_for_researcher(pk)

@shared_task(name='fetch_everything_for_researcher')
@run_only_once('researcher', keys=['pk'], timeout=15*60)
def fetch_everything_for_researcher(pk):
    ccf = get_ccf()
    oai = OaiPaperSource(ccf, max_results=250)
    sources = [
        ('orcid',OrcidPaperSource(ccf, oai, max_results=1000)),
        ('crossref',CrossRefPaperSource(ccf, oai, max_results=500)),
        ('oai',oai),
        ('base',BasePaperSource(ccf, max_results=200)),
       # ('core',CorePaperSource(ccf, max_results=250)),
       ]
    try:
        r = Researcher.objects.get(pk=pk)
        update_task = lambda name: update_researcher_task(r, name)
        for key,source in sources:
            update_task(key)
            source.fetch(r, incremental=True)
        update_task(None)

    except MetadataSourceException as e:
        raise e
    finally:
        update_task('clustering')
        ccf.commitThemAll()
        ccf.unloadResearcher(pk)
        r = Researcher.objects.get(pk=pk)
        update_task('stats')
        r.update_stats()
        r.harvester = None
        update_task(None)

@shared_task(name='fetch_records_for_researcher')
def fetch_records_for_researcher(pk, signature=True):
    """
    Fetch OAI records from Proaixy for the given researcher.

    :param signature: Search by name signature (D. Knuth) instead
       of full name (Donald Knuth)
    """
    researcher = Researcher.objects.get(pk=pk)
    fetch_records_for_name(researcher.name, signature=signature)

@shared_task(name='recluster_researcher')
def recluster_researcher(pk):
    ccf = get_ccf()
    try:
        r = Researcher.objects.get(pk=pk)
        ccf.reclusterBatch(r)
    finally:
        r.update_stats()
        update_researcher_task(r, None)

@shared_task(name='fetch_dois_for_researcher')
def fetch_dois_for_researcher(pk):
    researcher = Researcher.objects.get(pk=pk)
    return fetch_publications(researcher)

@shared_task(name='change_publisher_oa_status')
def change_publisher_oa_status(pk, status):
    publisher = Publisher.objects.get(pk=pk)
    publisher.change_oa_status(status)
    publisher.update_stats()

@shared_task(name='consolidate_paper')
@run_only_once('consolidate_paper', keys=['pk'], timeout=1*60)
def consolidate_paper(pk):
    p = None
    try:
        p = Paper.objects.get(pk=pk)
        abstract = p.abstract or ''
        for pub in p.publication_set.all():
            pub = consolidate_publication(pub)
            if pub.abstract and len(pub.abstract) > len(abstract):
                abstract = pub.abstract
                break
    except Paper.DoesNotExist:
        print "consolidate_paper: unknown paper %d" % pk
    finally:
        if p is not None:
            p.task = None
            p.save(update_fields=['task'])

@shared_task(name='update_all_stats')
@run_only_once('refresh_stats', timeout=3*60)
def update_all_stats():
    """
    Updates the stats for every model using them
    """
    AccessStatistics.update_all_stats(PaperWorld)
    AccessStatistics.update_all_stats(Publisher)
    AccessStatistics.update_all_stats(Journal)
    AccessStatistics.update_all_stats(Researcher)

@shared_task(name='update_all_stats_but_researchers')
@run_only_once('refresh_stats', timeout=3*60)
def update_all_stats_but_researchers():
    """
    Updates the stats for every model using them
    """
    AccessStatistics.update_all_stats(PaperWorld)
    AccessStatistics.update_all_stats(Publisher)
    AccessStatistics.update_all_stats(Journal)

<|endoftext|>


<commit_before>from Tkinter import *
import time
import sys
import random
import Queue
from random import shuffle

#establish deck
deck = [(i+1) for i in range(52)]
shuffle(deck)
print deck

#Global variables
bWidth = 800
bHeight = 400
HouseMoney = 1000000
p_Money = 100
temp = 0
betval = 0
Pic_file_path = 'C:\Python27\Jeff_home\Games\Pictures\Gif\\'
list_Comp = []
list_You = []

file_list = ['gif_CardBack.gif',
             'gif_d_ace.gif', 'gif_d_two.gif', 'gif_d_three.gif', 'gif_d_four.gif', 'gif_d_five.gif', 'gif_d_six.gif','gif_d_seven.gif',
             'gif_d_eight.gif','gif_d_nine.gif','gif_d_ten.gif','gif_d_jack.gif','gif_d_queen.gif','gif_d_king.gif',

             'gif_s_ace.gif', 'gif_s_two.gif', 'gif_s_three.gif', 'gif_s_four.gif', 'gif_s_five.gif', 'gif_s_six.gif','gif_s_seven.gif',
             'gif_s_eight.gif','gif_s_nine.gif','gif_s_ten.gif','gif_s_jack.gif','gif_s_queen.gif','gif_s_king.gif',

             'gif_h_ace.gif', 'gif_h_two.gif', 'gif_h_three.gif', 'gif_h_four.gif', 'gif_h_five.gif', 'gif_h_six.gif','gif_h_seven.gif',
             'gif_h_eight.gif','gif_h_nine.gif','gif_h_ten.gif','gif_h_jack.gif','gif_h_queen.gif','gif_h_king.gif',

             'gif_c_ace.gif', 'gif_c_two.gif', 'gif_c_three.gif', 'gif_c_four.gif', 'gif_c_five.gif', 'gif_c_six.gif','gif_c_seven.gif',
             'gif_c_eight.gif','gif_c_nine.gif','gif_c_ten.gif','gif_c_jack.gif','gif_c_queen.gif','gif_c_king.gif',
             ]

root = Tk()
board = Canvas(root, width=bWidth, height=bHeight)
board.create_line(0, bHeight/2, bWidth, bHeight/2)
board.configure(background = '#277714')
root.configure(background = 'blue')
betdisplay = board.create_text(55,300, text ="Place bet", font = ('Arial', 15))

ePlayerMoney = Entry(root)
ePlayerMoney.grid(row = 25, column = 20)
ePlayerMoney.insert(0, "Player: " + str(p_Money))

def Bet():
    global betval, betdisplay
    global p_Money
    global ePlayerMoney
    board.delete(betdisplay)
    betval = betval+10
    p_Money = p_Money - 10
    ePlayerMoney.delete(0, END)
    ePlayerMoney.insert(0, "Player: " + str(p_Money))
    if p_Money <= 0:
        BetBtn.config(state = 'disabled')
    betdisplay = board.create_text(55,300, text ="Bet: $" + str(betval), font = ('Arial',15))

def Deal():
    global list_Comp, list_You
    global t_Comp, t_You
    global you_image, comp_image
    global temp_You, temp_Comp
    global Pic_file_path, file_list

    card_x = 165

    House_secret = deck.pop(0)
    list_Comp.append(House_secret)
    comp_image = PhotoImage(file=(Pic_file_path + file_list[0]) )
    t_Comp = board.create_image(165,42, anchor = NW, image=comp_image)

    temp_Comp = deck.pop(0)
    list_Comp.append(temp_Comp)
    comp_image2 = PhotoImage(file=(Pic_file_path + file_list[temp_Comp]) )
    t_Comp = board.create_image(165+85,42, anchor = NW, image=comp_image2)
##
##    temp_You = deck.pop(0)
##    list_You.append(temp_You)
##    you_image = PhotoImage(file=(Pic_file_path + file_list[temp_You]) )
##    t_You = board.create_image(250,255, anchor = NW, image=you_image)
##
##
##    temp_You = deck.pop(0)
##    list_You.append(temp_You)
##    you_image2 = PhotoImage(file=(Pic_file_path + file_list[temp_You]) )
##    t_You = board.create_image(card_x + 85,255, anchor = NW, image=you_image)

    board.update()
    print "list_house_card", list_Comp
    print "list_you_card", list_You

#cards

eYouCards = Entry(root)
eCompCards = Entry(root)
eTurns = Entry(root)

you_image = PhotoImage(file=(Pic_file_path + file_list[0]) )
comp_image = PhotoImage(file=(Pic_file_path + file_list[26]) )
t_You = board.create_image(165,255, anchor = NW, image=comp_image)
t_Comp = board.create_image(165,42, anchor = NW, image=you_image)

#add button commands later
DealBtn = Button(root, text = "Deal", command = Deal)
HitBtn = Button(root, text = "Hit")
StayBtn = Button(root, text = "Stay")
BetBtn = Button(root, text = "Bet $10", command = Bet)

#words
board.create_text(400, 200, text='Beware, money will be lost here!',
font=('Helvetica', 30))

eHouseMoney = Entry(root)
eHouseMoney.grid(row = 25, column = 20)
eHouseMoney.pack()
eHouseMoney.insert(0, "House: " + str(HouseMoney))

board.pack()

HitBtn.pack()
StayBtn.pack()
BetBtn.pack()
DealBtn.place(bordermode=OUTSIDE, height = 40, width = 40, x= 600, y =300)
BetBtn.place(bordermode=OUTSIDE, height=40, width=50, x = 540, y = 300)
HitBtn.place(bordermode=OUTSIDE, height=40, width=40, x = 650, y = 300)
StayBtn.place(bordermode=OUTSIDE, height=40, width=40, x = 700, y = 300)
ePlayerMoney.pack()

root.mainloop()
<commit_msg>Delete BlackJack.py<commit_after><|endoftext|>


<commit_before>import csv
from datetime import datetime

import numpy as np
import requests
from django.core.exceptions import ValidationError
from lxml import html
from scipy.sparse import dok_matrix
from sklearn.metrics import pairwise_distances

from crawler.models import App, AppDescription, Category, CategoryDescription, Developer, AppCategory

DETAILS_URL = 'http://play.google.com/store/apps/details?id={}&hl={}'
DATE_MASK = {'en': '%B %d, %Y', 'pt_BR': '%d de %B de %Y'}

class Crawler:
    def __init__(self):
        pass

    def populate_category_description(self, content, category, loc):
        category_desc = CategoryDescription()
        category_desc.name = self.extract_category_desc(content)
        category_desc.locale = loc
        category_desc.category = category
        return category_desc

    def populate_category(self, content):
        category = Category()
        category.key = self.extract_category_key(content)
        return category

    def populate_app_description(self, content, app, loc):
        app_description = AppDescription()
        app_description.name = self.extract_name(content)
        app_description.description = self.extract_description(content)
        app_description.locale = loc
        app_description.app = app
        return app_description

    def populate_app(self, content, app_package, developer, loc):
        app = App()
        app.package_name = app_package
        app.icon_url = self.extract_icon_url(content)
        app.size = self.extract_file_size(content)
        app.publication_date = self.extract_update_date(content, loc)
        app.version = self.extract_version(content)
        app.rating = self.extract_rating(content)
        app.content_rating = self.extract_content_rating(content)
        app.developer = developer
        return app

    def populate_developer(self, content):
        developer = Developer()
        developer.name = self.extract_developer(content)
        return developer

    def populate_app_category(self, app, category):
        app_category = AppCategory()
        app_category.app = app
        app_category.category = category
        return app_category

    @staticmethod
    def extract_name(content):
        # return content.xpath('//div[@class="id-app-title"]')[0].text_content()
        name = ''
        names = content.xpath('//div[@class="id-app-title"]')
        if names:
            name = names[0].text_content().strip()
        return name.encode('utf-8')

    @staticmethod
    def extract_description(content):
        # return content.xpath('//div[@itemprop="description"]')[0].text_content().strip()
        description = ''
        descriptions = content.xpath('//div[@itemprop="description"]')
        if descriptions:
            description = descriptions[0].text_content().strip()
        return description.encode('utf-8')

    @staticmethod
    def extract_icon_url(content):
        return "http:{}".format(content.xpath('//img[@class="cover-image"]/@src')[0]).encode('utf-8')

    @staticmethod
    def extract_update_date(content, loc):
        # return datetime.strptime(content.xpath('//div[@itemprop="datePublished"]')[0].text_content(), DATE_MASK[loc])
        update_date = ''
        update_dates = content.xpath('//div[@itemprop="datePublished"]')
        if update_dates:
            update_date_str = update_dates[0].text_content().strip()
            update_date = datetime.strptime(update_date_str, DATE_MASK[loc])
        return update_date

    @staticmethod
    def extract_content_rating(content):
        # return content.xpath('//div[@itemprop="contentRating"]')[0].text_content()
        content_rating = ''
        content_ratings = content.xpath('//div[@itemprop="contentRating"]')
        if content_ratings:
            content_rating = content_ratings[0].text_content().strip()
        return content_rating

    @staticmethod
    def extract_file_size(content):
        # return content.xpath('//div[@itemprop="fileSize"]')[0].text_content().strip()
        size = 0
        sizes = content.xpath('//div[@itemprop="fileSize"]')
        if sizes:
            size = sizes[0].text_content().strip()
        return size

    @staticmethod
    def extract_version(content):
        # return content.xpath('//div[@itemprop="softwareVersion"]')[0].text_content().strip()
        version = 0
        versions = content.xpath('//div[@itemprop="softwareVersion"]')
        if versions:
            version = versions[0].text_content().strip()
        return version

    @staticmethod
    def extract_rating(content):
        # return content.xpath('//div[@class="score"]')[0].text_content()
        rating = ''
        ratings = content.xpath('//div[@class="score"]')
        if ratings:
            rating = ratings[0].text_content().strip()
        return rating

    @staticmethod
    def extract_developer(content):
        # return content.xpath('//a[@class="document-subtitle primary"]/span[@itemprop="name"]')[0].text_content()
        developer = ''
        developers = content.xpath('//a[@class="document-subtitle primary"]/span[@itemprop="name"]')
        if developers:
            developer = developers[0].text_content().strip()
        return developer.encode('utf-8')

    @staticmethod
    def extract_category_desc(content):
        # return content.xpath('//span[@itemprop="genre"]')[0].text_content()
        category = ''
        categories = content.xpath('//span[@itemprop="genre"]')
        if categories:
            category = categories[0].text_content().strip()
        return category.encode('utf-8')

    @staticmethod
    def extract_category_key(content):
        # return content.xpath('//a[@class="document-subtitle category"]/@href')[0].split('/')[-1]
        category_key = ''
        category_urls = content.xpath('//a[@class="document-subtitle category"]/@href')
        if category_urls:
            category_url = category_urls[0]
            category_key = category_url.split('/')[-1].strip()
        return category_key.encode('utf-8')

    @staticmethod
    def extract_similars(content):
        similars = content.xpath(
            '//div[@class="rec-cluster"]//div[@class="card no-rationale square-cover apps small"]/@data-docid')
        return similars

    def get_details(self, app_package, loc):
        url = DETAILS_URL.format(app_package, loc)

        try:
            response = requests.get(url, timeout=1.0)
            if response.status_code != 200:
                return
        except requests.exceptions.Timeout as e:
            print('Timeout Error\n' + str(e))
            return
        except requests.exceptions.ConnectionError as e:
            print ('Connection Err\n' + str(e))
            return

        content = html.fromstring(response.text)

        developer = self.populate_developer(content)

        developer, developer_created = Developer.objects.get_or_create(
            name=developer.name,
        )

        app = self.populate_app(content, app_package, developer, loc)

        app, app_created = App.objects.get_or_create(
            package_name=app.package_name,
            defaults={
                'icon_url': app.icon_url,
                'size': app.size,
                'publication_date': app.publication_date,
                'rating': app.rating,
                'version': app.version,
                'content_rating': app.content_rating,
                'developer': app.developer
            },
        )

        app_description = self.populate_app_description(content, app, loc)

        app_description = AppDescription.objects.get_or_create(
            app=app_description.app,
            locale=app_description.locale,
            defaults={
                'name': app_description.name,
                'description': app_description.description
            },
        )

        category = self.populate_category(content)

        category, category_created = Category.objects.get_or_create(
            key=category.key,
        )

        category_description = self.populate_category_description(content, category, loc)

        category_description, category_description_created = CategoryDescription.objects.get_or_create(
            category=category_description.category,
            locale=category_description.locale,
            defaults={
                'name': category_description.name
            }
        )

        app_category = self.populate_app_category(app, category)

        app_description, app_category = AppCategory.objects.get_or_create(
            app=app_category.app,
            category=app_category.category,
        )

        similars = self.extract_similars(content)

        return {'developer': developer,
                'app': app,
                'app_description': app_description,
                'category': category,
                'category_description': category_description,
                'similars': similars,
                }

    def crawl(self, app_packages, date):
        with open('error-apps-id-list-{}.txt'.format(date), 'w') as error_file, \
                open('similar-apps-{}.csv'.format(date), 'wb') as similar_file:
            crawled_count = 0
            total_count = 0
            for app_package in app_packages:
                package = app_package.rstrip()
                try:
                    total_count += 1
                    app_details_map = self.get_details(package, 'en')
                    if not app_details_map:
                        error_file.write(package + '\n')
                        print '{} Not Found'.format(package)
                        continue
                    if app_details_map.get('similars'):
                        for similar in app_details_map.get('similars'):
                            csv_writer = csv.writer(similar_file, delimiter=';', quotechar='"',
                                                    quoting=csv.QUOTE_MINIMAL)
                            csv_writer.writerow([package, similar])
                    crawled_count += 1
                except IOError as e:
                    print('Error on parsing')
                    error_file.write(package + '\n')
                    pass
                except ValidationError as ve:
                    error_file.write(package + '\n')
                    print('Error on validation')
                    pass

            return crawled_count, total_count

class AppClassifier:
    apps_list = []
    category_keys = dict()
    developer_list = dict()

    def __init__(self, apps, categories, developers):
        self.apps_list = apps
        self.category_keys = categories
        self.developer_list = developers

    def create_utility_matrix(self):
        app_count = len(self.apps_list)
        cat_count = len(self.category_keys)
        dev_count = len(self.developer_list)
        total_col = cat_count + dev_count
        mat_row = 0

        utility_matrix = dok_matrix((app_count, total_col), dtype=np.int)
        for app in self.apps_list:
            cat_key = app.category_key()
            dev_name = app.developer_name()

            mat_col1 = self.category_keys[cat_key]
            mat_col2 = self.developer_list[dev_name]

            utility_matrix[mat_row, mat_col1] = 1
            utility_matrix[mat_row, mat_col2] = 1

            mat_row += 1

        return utility_matrix

    def is_similar(self, u, v, similarity_boundary=0.6):
        cos_dist = pairwise_distances(u, v, 'cosine')
        return cos_dist < similarity_boundary
<commit_msg>Improving loop in method create_utility_matrix<commit_after>import csv
from datetime import datetime

import numpy as np
import requests
from django.core.exceptions import ValidationError
from lxml import html
from scipy.sparse import dok_matrix
from sklearn.metrics import pairwise_distances

from crawler.models import App, AppDescription, Category, CategoryDescription, Developer, AppCategory

DETAILS_URL = 'http://play.google.com/store/apps/details?id={}&hl={}'
DATE_MASK = {'en': '%B %d, %Y', 'pt_BR': '%d de %B de %Y'}

class Crawler:
    def __init__(self):
        pass

    def populate_category_description(self, content, category, loc):
        category_desc = CategoryDescription()
        category_desc.name = self.extract_category_desc(content)
        category_desc.locale = loc
        category_desc.category = category
        return category_desc

    def populate_category(self, content):
        category = Category()
        category.key = self.extract_category_key(content)
        return category

    def populate_app_description(self, content, app, loc):
        app_description = AppDescription()
        app_description.name = self.extract_name(content)
        app_description.description = self.extract_description(content)
        app_description.locale = loc
        app_description.app = app
        return app_description

    def populate_app(self, content, app_package, developer, loc):
        app = App()
        app.package_name = app_package
        app.icon_url = self.extract_icon_url(content)
        app.size = self.extract_file_size(content)
        app.publication_date = self.extract_update_date(content, loc)
        app.version = self.extract_version(content)
        app.rating = self.extract_rating(content)
        app.content_rating = self.extract_content_rating(content)
        app.developer = developer
        return app

    def populate_developer(self, content):
        developer = Developer()
        developer.name = self.extract_developer(content)
        return developer

    def populate_app_category(self, app, category):
        app_category = AppCategory()
        app_category.app = app
        app_category.category = category
        return app_category

    @staticmethod
    def extract_name(content):
        # return content.xpath('//div[@class="id-app-title"]')[0].text_content()
        name = ''
        names = content.xpath('//div[@class="id-app-title"]')
        if names:
            name = names[0].text_content().strip()
        return name.encode('utf-8')

    @staticmethod
    def extract_description(content):
        # return content.xpath('//div[@itemprop="description"]')[0].text_content().strip()
        description = ''
        descriptions = content.xpath('//div[@itemprop="description"]')
        if descriptions:
            description = descriptions[0].text_content().strip()
        return description.encode('utf-8')

    @staticmethod
    def extract_icon_url(content):
        return "http:{}".format(content.xpath('//img[@class="cover-image"]/@src')[0]).encode('utf-8')

    @staticmethod
    def extract_update_date(content, loc):
        # return datetime.strptime(content.xpath('//div[@itemprop="datePublished"]')[0].text_content(), DATE_MASK[loc])
        update_date = ''
        update_dates = content.xpath('//div[@itemprop="datePublished"]')
        if update_dates:
            update_date_str = update_dates[0].text_content().strip()
            update_date = datetime.strptime(update_date_str, DATE_MASK[loc])
        return update_date

    @staticmethod
    def extract_content_rating(content):
        # return content.xpath('//div[@itemprop="contentRating"]')[0].text_content()
        content_rating = ''
        content_ratings = content.xpath('//div[@itemprop="contentRating"]')
        if content_ratings:
            content_rating = content_ratings[0].text_content().strip()
        return content_rating

    @staticmethod
    def extract_file_size(content):
        # return content.xpath('//div[@itemprop="fileSize"]')[0].text_content().strip()
        size = 0
        sizes = content.xpath('//div[@itemprop="fileSize"]')
        if sizes:
            size = sizes[0].text_content().strip()
        return size

    @staticmethod
    def extract_version(content):
        # return content.xpath('//div[@itemprop="softwareVersion"]')[0].text_content().strip()
        version = 0
        versions = content.xpath('//div[@itemprop="softwareVersion"]')
        if versions:
            version = versions[0].text_content().strip()
        return version

    @staticmethod
    def extract_rating(content):
        # return content.xpath('//div[@class="score"]')[0].text_content()
        rating = ''
        ratings = content.xpath('//div[@class="score"]')
        if ratings:
            rating = ratings[0].text_content().strip()
        return rating

    @staticmethod
    def extract_developer(content):
        # return content.xpath('//a[@class="document-subtitle primary"]/span[@itemprop="name"]')[0].text_content()
        developer = ''
        developers = content.xpath('//a[@class="document-subtitle primary"]/span[@itemprop="name"]')
        if developers:
            developer = developers[0].text_content().strip()
        return developer.encode('utf-8')

    @staticmethod
    def extract_category_desc(content):
        # return content.xpath('//span[@itemprop="genre"]')[0].text_content()
        category = ''
        categories = content.xpath('//span[@itemprop="genre"]')
        if categories:
            category = categories[0].text_content().strip()
        return category.encode('utf-8')

    @staticmethod
    def extract_category_key(content):
        # return content.xpath('//a[@class="document-subtitle category"]/@href')[0].split('/')[-1]
        category_key = ''
        category_urls = content.xpath('//a[@class="document-subtitle category"]/@href')
        if category_urls:
            category_url = category_urls[0]
            category_key = category_url.split('/')[-1].strip()
        return category_key.encode('utf-8')

    @staticmethod
    def extract_similars(content):
        similars = content.xpath(
            '//div[@class="rec-cluster"]//div[@class="card no-rationale square-cover apps small"]/@data-docid')
        return similars

    def get_details(self, app_package, loc):
        url = DETAILS_URL.format(app_package, loc)

        try:
            response = requests.get(url, timeout=1.0)
            if response.status_code != 200:
                return
        except requests.exceptions.Timeout as e:
            print('Timeout Error\n' + str(e))
            return
        except requests.exceptions.ConnectionError as e:
            print ('Connection Err\n' + str(e))
            return

        content = html.fromstring(response.text)

        developer = self.populate_developer(content)

        developer, developer_created = Developer.objects.get_or_create(
            name=developer.name,
        )

        app = self.populate_app(content, app_package, developer, loc)

        app, app_created = App.objects.get_or_create(
            package_name=app.package_name,
            defaults={
                'icon_url': app.icon_url,
                'size': app.size,
                'publication_date': app.publication_date,
                'rating': app.rating,
                'version': app.version,
                'content_rating': app.content_rating,
                'developer': app.developer
            },
        )

        app_description = self.populate_app_description(content, app, loc)

        app_description = AppDescription.objects.get_or_create(
            app=app_description.app,
            locale=app_description.locale,
            defaults={
                'name': app_description.name,
                'description': app_description.description
            },
        )

        category = self.populate_category(content)

        category, category_created = Category.objects.get_or_create(
            key=category.key,
        )

        category_description = self.populate_category_description(content, category, loc)

        category_description, category_description_created = CategoryDescription.objects.get_or_create(
            category=category_description.category,
            locale=category_description.locale,
            defaults={
                'name': category_description.name
            }
        )

        app_category = self.populate_app_category(app, category)

        app_description, app_category = AppCategory.objects.get_or_create(
            app=app_category.app,
            category=app_category.category,
        )

        similars = self.extract_similars(content)

        return {'developer': developer,
                'app': app,
                'app_description': app_description,
                'category': category,
                'category_description': category_description,
                'similars': similars,
                }

    def crawl(self, app_packages, date):
        with open('error-apps-id-list-{}.txt'.format(date), 'w') as error_file, \
                open('similar-apps-{}.csv'.format(date), 'wb') as similar_file:
            crawled_count = 0
            total_count = 0
            for app_package in app_packages:
                package = app_package.rstrip()
                try:
                    total_count += 1
                    app_details_map = self.get_details(package, 'en')
                    if not app_details_map:
                        error_file.write(package + '\n')
                        print '{} Not Found'.format(package)
                        continue
                    if app_details_map.get('similars'):
                        for similar in app_details_map.get('similars'):
                            csv_writer = csv.writer(similar_file, delimiter=';', quotechar='"',
                                                    quoting=csv.QUOTE_MINIMAL)
                            csv_writer.writerow([package, similar])
                    crawled_count += 1
                except IOError as e:
                    print('Error on parsing')
                    error_file.write(package + '\n')
                    pass
                except ValidationError as ve:
                    error_file.write(package + '\n')
                    print('Error on validation')
                    pass

            return crawled_count, total_count

class AppClassifier:
    apps_list = []
    category_keys = dict()
    developer_list = dict()

    def __init__(self, apps, categories, developers):
        self.apps_list = apps
        self.category_keys = categories
        self.developer_list = developers

    def create_utility_matrix(self):
        app_count = len(self.apps_list)
        cat_count = len(self.category_keys)
        dev_count = len(self.developer_list)
        total_col = cat_count + dev_count

        utility_matrix = dok_matrix((app_count, total_col), dtype=np.int)
        for mat_row, app in enumerate(self.apps_list):
            cat_key = app.category_key()
            mat_col1 = self.category_keys[cat_key]
            utility_matrix[mat_row, mat_col1] = 1

            dev_name = app.developer_name()
            mat_col2 = self.developer_list[dev_name]
            utility_matrix[mat_row, mat_col2] = 1

        return utility_matrix

    def is_similar(self, u, v, similarity_boundary=0.6):
        cos_dist = pairwise_distances(u, v, 'cosine')
        return cos_dist < similarity_boundary
<|endoftext|>