<commit_before>'''
Resources needed by pkg providers
'''

# Import python libs
import os
import re
import yaml
import pprint
import logging

log = logging.getLogger(__name__)

def _parse_pkg_meta(path):
    '''
    Parse metadata from a binary package and return the package's name and
    version number.
    '''
    def parse_rpm(path):
        name = ''
        version = ''
        rel = ''
        result = __salt__['cmd.run_all']('rpm -qpi "{0}"'.format(path))
        if result['retcode'] == 0:
            for line in result['stdout'].splitlines():
                if not name:
                    m = re.match('^Name\s*:\s*(\S+)', line)
                    if m:
                        name = m.group(1)
                        continue
                if not version:
                    m = re.match('^Version\s*:\s*(\S+)', line)
                    if m:
                        version = m.group(1)
                        continue
                if not rel:
                    m = re.match('^Release\s*:\s*(\S+)', line)
                    if m:
                        version = m.group(1)
                        continue
        if rel:
            version += '-{0}'.format(rel)
        return name,version

    def parse_pacman(path):
        name = ''
        version = ''
        result = __salt__['cmd.run_all']('pacman -Qpi "{0}"'.format(path))
        if result['retcode'] == 0:
            for line in result['stdout'].splitlines():
                if not name:
                    m = re.match('^Name\s*:\s*(\S+)',line)
                    if m:
                        name = m.group(1)
                        continue
                if not version:
                    m = re.match('^Version\s*:\s*(\S+)',line)
                    if m:
                        version = m.group(1)
                        continue
        return name,version

    def parse_deb(path):
        name = ''
        version = ''
        result = __salt__['cmd.run_all']('dpkg-deb -I "{0}"'.format(path))
        if result['retcode'] == 0:
            for line in result['stdout'].splitlines():
                if not name:
                    m = re.match('^\s*Package\s*:\s*(\S+)',line)
                    if m:
                        name = m.group(1)
                        continue
                if not version:
                    m = re.match('^\s*Version\s*:\s*(\S+)',line)
                    if m:
                        version = m.group(1)
                        continue
        return name,version

    if __grains__['os_family'] in ('Suse','RedHat','Mandriva'):
        metaparser = parse_rpm
    elif __grains__['os_family'] in ('Arch'):
        metaparser = parse_pacman
    elif __grains__['os_family'] in ('Debian'):
        metaparser = parse_deb
    else:
        log.error('No metadata parser found for {0}'.format(path))
        return '',''

    return metaparser(path)

def _pack_pkgs(sources):
    '''
    Accepts list (or a string representing a list) and returns back either the
    list passed, or the list represenation of the string passed.

    Example: '["foo","bar","baz"]' would become ["foo","bar","baz"]
    '''
    if isinstance(sources, basestring):
        try:
            sources = yaml.load(sources)
        except yaml.parser.ParserError as e:
            log.error(e)
            return []
<<<<<<< HEAD
    if not isinstance(sources, list) \
    or [x for x in sources if not isinstance(x, basestring)]:
        log.error('Invalid input: {0}'.format(pprint.pformat(source)))
=======
    if not isinstance(sources,list) \
    or [x for x in sources if not isinstance(x, basestring)]:
        log.error('Invalid input: {0}'.format(pformat(source)))
>>>>>>> 868aa1b221528f9330b1a6e87b31a73fdcd8df11
        log.error('Input must be a list of strings')
        return []
    return sources

def _pack_sources(sources):
    '''
    Accepts list of dicts (or a string representing a list of dicts) and packs
    the key/value pairs into a single dict.

    Example: '[{"foo": "salt://foo.rpm"}, {"bar": "salt://bar.rpm"}]' would
    become {"foo": "salt://foo.rpm", "bar": "salt://bar.rpm"}
    '''
<<<<<<< HEAD
    if instance(sources, basestring):
=======
    if isinstance(sources, basestring):
>>>>>>> 868aa1b221528f9330b1a6e87b31a73fdcd8df11
        try:
            sources = yaml.load(sources)
        except yaml.parser.ParserError as e:
            log.error(e)
            return {}
    ret = {}
    for source in sources:
        if (not isinstance(source,dict)) or len(source) != 1:
            log.error('Invalid input: {0}'.format(pprint.pformat(sources)))
            log.error('Input must be a list of 1-element dicts')
            return {}
        else:
            ret.update(source)
    return ret

def _verify_binary_pkg(srcinfo):
    '''
    Compares package files (s) against the metadata to confirm that they match
    what is expected.
    '''
    problems = []
    for pkg_name,pkg_uri,pkg_path,pkg_type in srcinfo:
        pkgmeta_name,pkgmeta_version = _parse_pkg_meta(pkg_path)
        if not pkgmeta_name:
            if pkg_type == 'remote':
                problems.append('Failed to cache {0}. Are you sure this '
                                'path is correct?'.format(pkg_uri))
            elif pkg_type == 'local':
                if not os.path.isfile(pkg_path):
                    problems.append('Package file {0} not found. Are '
                                    'you sure this path is '
                                    'correct?'.format(pkg_path))
                else:
                    problems.append('Unable to parse package metadata for '
                                    '{0}.'.format(pkg_path))
        elif pkg_name != pkgmeta_name:
            problems.append('Package file {0} (Name: {1}) does not '
                            'match the specified package name '
                            '({2}).'.format(pkg_uri,pkgmeta_name,pkg_name))
    return problems

def parse_targets(name=None, pkgs=None, sources=None):
    '''
    Parses the input to pkg.install and returns back the package(s) to be
    installed. Returns a list of packages, as well as a string noting whether
    the packages are to come from a repository or a binary package.
    '''

    # For Solaris, there is no repository, and only the "sources" param can be
    # used. Warn if "name" or "pkgs" is provided, and require that "sources" is
    # present.
    if __grains__['os_family'] == 'Solaris':
        if name:
            log.warning('Parameter "name" ignored on Solaris hosts.')
        if pkgs:
            log.warning('Parameter "pkgs" ignored on Solaris hosts.')
        if not sources:
            log.error('"sources" option required with Solaris pkg installs')
            return None,None

    # "pkgs" is always ignored on Solaris.
    if pkgs and sources and __grains__['os_family'] != 'Solaris':
        log.error('Only one of "pkgs" and "sources" can be used.')
        return None,None

    elif pkgs and __grains__['os_family'] != 'Solaris':
        if name:
            log.warning('"name" parameter will be ignored in favor of "pkgs"')
        pkgs = _pack_pkgs(pkgs)
        if not pkgs:
            return None,None
        else:
            return pkgs,'repository'

    elif sources:
        # No need to warn for Solaris, warning taken care of above.
        if name and __grains__['os_family'] != 'Solaris':
            log.warning('"name" parameter will be ignored in favor of '
                        '"sources".')
        sources = _pack_sources(sources)
        if not sources:
            return None,None

        srcinfo = []
        for pkg_name,pkg_src in sources.iteritems():
            if __salt__['config.valid_fileproto'](pkg_src):
                # Cache package from remote source (salt master, http, ftp)
                srcinfo.append((pkg_name,
                                pkg_src,
                               __salt__['cp.cache_file'](pkg_src),
                               'remote'))
            else:
                # Package file local to the minion
                srcinfo.append((pkg_name,pkg_src,pkg_src,'local'))

        # Check metadata to make sure the name passed matches the source
        if __grains__['os_family'] not in ('Solaris',) \
        and __grains__['os'] not in ('Gentoo', 'OpenBSD',):
            problems = _verify_binary_pkg(srcinfo)
            # If any problems are found in the caching or metadata parsing done
            # in the above for loop, log each problem and return None,None,
            # which will keep package installation from proceeding.
            if problems:
                for problem in problems: log.error(problem)
                return None,None

        # srcinfo is a 4-tuple (pkg_name,pkg_uri,pkg_path,pkg_type), so grab
        # the package path (3rd element of tuple).
        return [x[2] for x in srcinfo],'file'

    elif name and __grains__['os_family'] != 'Solaris':
        return [name],'repository'

    else:
        log.error('No package sources passed to pkg.install.')
        return None,None

def add_pkg(pkgs, name, version):
    '''
    Add a package to a dict of installed packages.
    '''

    ''' multiple-version support (not yet implemented)
    cur = pkgs.get(name)
    if cur is None:
        pkgs[name] = version
    elif isinstance(cur, basestring):
        pkgs[name] = [cur, version]
    else:
        pkgs[name].append(version)
    '''
    pkgs[name] = version

def sort_pkglist(pkgs):
    '''
    Accepts a dict obtained from pkg.list_pkgs() and sorts in place the list of
    versions for any packages that have multiple versions installed, so that
    two package lists can be compared to one another.
    '''
    # It doesn't matter that ['4.9','4.10'] would be sorted to ['4.10','4.9'],
    # so long as the sorting is consistent.
    for key in pkgs.keys():
        if isinstance(pkgs[key],list):
            pkgs[key].sort()

def find_changes(old={}, new={}):
    '''
    Compare before and after results from pkg.list_pkgs() to determine what
    changes were made to the packages installed on the minion.
    '''
    pkgs = {}
    for npkg in new:
        if npkg in old:
            if old[npkg] == new[npkg]:
                # no change in the package
                continue
            else:
                # the package was here before and the version has changed
                pkgs[npkg] = {'old': old[npkg],
                              'new': new[npkg]}
        else:
            # the package is freshly installed
            pkgs[npkg] = {'old': '',
                          'new': new[npkg]}
    return pkgs
<commit_msg>Clean out some more merge conflicts<commit_after>'''
Resources needed by pkg providers
'''

# Import python libs
import os
import re
import yaml
import pprint
import logging

log = logging.getLogger(__name__)

def _parse_pkg_meta(path):
    '''
    Parse metadata from a binary package and return the package's name and
    version number.
    '''
    def parse_rpm(path):
        name = ''
        version = ''
        rel = ''
        result = __salt__['cmd.run_all']('rpm -qpi "{0}"'.format(path))
        if result['retcode'] == 0:
            for line in result['stdout'].splitlines():
                if not name:
                    m = re.match('^Name\s*:\s*(\S+)', line)
                    if m:
                        name = m.group(1)
                        continue
                if not version:
                    m = re.match('^Version\s*:\s*(\S+)', line)
                    if m:
                        version = m.group(1)
                        continue
                if not rel:
                    m = re.match('^Release\s*:\s*(\S+)', line)
                    if m:
                        version = m.group(1)
                        continue
        if rel:
            version += '-{0}'.format(rel)
        return name,version

    def parse_pacman(path):
        name = ''
        version = ''
        result = __salt__['cmd.run_all']('pacman -Qpi "{0}"'.format(path))
        if result['retcode'] == 0:
            for line in result['stdout'].splitlines():
                if not name:
                    m = re.match('^Name\s*:\s*(\S+)',line)
                    if m:
                        name = m.group(1)
                        continue
                if not version:
                    m = re.match('^Version\s*:\s*(\S+)',line)
                    if m:
                        version = m.group(1)
                        continue
        return name,version

    def parse_deb(path):
        name = ''
        version = ''
        result = __salt__['cmd.run_all']('dpkg-deb -I "{0}"'.format(path))
        if result['retcode'] == 0:
            for line in result['stdout'].splitlines():
                if not name:
                    m = re.match('^\s*Package\s*:\s*(\S+)',line)
                    if m:
                        name = m.group(1)
                        continue
                if not version:
                    m = re.match('^\s*Version\s*:\s*(\S+)',line)
                    if m:
                        version = m.group(1)
                        continue
        return name,version

    if __grains__['os_family'] in ('Suse','RedHat','Mandriva'):
        metaparser = parse_rpm
    elif __grains__['os_family'] in ('Arch'):
        metaparser = parse_pacman
    elif __grains__['os_family'] in ('Debian'):
        metaparser = parse_deb
    else:
        log.error('No metadata parser found for {0}'.format(path))
        return '',''

    return metaparser(path)

def _pack_pkgs(sources):
    '''
    Accepts list (or a string representing a list) and returns back either the
    list passed, or the list represenation of the string passed.

    Example: '["foo","bar","baz"]' would become ["foo","bar","baz"]
    '''
    if isinstance(sources, basestring):
        try:
            sources = yaml.load(sources)
        except yaml.parser.ParserError as e:
            log.error(e)
            return []
    if not isinstance(sources,list) \
    or [x for x in sources if not isinstance(x, basestring)]:
        log.error('Invalid input: {0}'.format(pformat(source)))
        return []
    return sources

def _pack_sources(sources):
    '''
    Accepts list of dicts (or a string representing a list of dicts) and packs
    the key/value pairs into a single dict.

    Example: '[{"foo": "salt://foo.rpm"}, {"bar": "salt://bar.rpm"}]' would
    become {"foo": "salt://foo.rpm", "bar": "salt://bar.rpm"}
    '''
    if isinstance(sources, basestring):
        try:
            sources = yaml.load(sources)
        except yaml.parser.ParserError as e:
            log.error(e)
            return {}
    ret = {}
    for source in sources:
        if (not isinstance(source,dict)) or len(source) != 1:
            log.error('Invalid input: {0}'.format(pprint.pformat(sources)))
            log.error('Input must be a list of 1-element dicts')
            return {}
        else:
            ret.update(source)
    return ret

def _verify_binary_pkg(srcinfo):
    '''
    Compares package files (s) against the metadata to confirm that they match
    what is expected.
    '''
    problems = []
    for pkg_name,pkg_uri,pkg_path,pkg_type in srcinfo:
        pkgmeta_name,pkgmeta_version = _parse_pkg_meta(pkg_path)
        if not pkgmeta_name:
            if pkg_type == 'remote':
                problems.append('Failed to cache {0}. Are you sure this '
                                'path is correct?'.format(pkg_uri))
            elif pkg_type == 'local':
                if not os.path.isfile(pkg_path):
                    problems.append('Package file {0} not found. Are '
                                    'you sure this path is '
                                    'correct?'.format(pkg_path))
                else:
                    problems.append('Unable to parse package metadata for '
                                    '{0}.'.format(pkg_path))
        elif pkg_name != pkgmeta_name:
            problems.append('Package file {0} (Name: {1}) does not '
                            'match the specified package name '
                            '({2}).'.format(pkg_uri,pkgmeta_name,pkg_name))
    return problems

def parse_targets(name=None, pkgs=None, sources=None):
    '''
    Parses the input to pkg.install and returns back the package(s) to be
    installed. Returns a list of packages, as well as a string noting whether
    the packages are to come from a repository or a binary package.
    '''

    # For Solaris, there is no repository, and only the "sources" param can be
    # used. Warn if "name" or "pkgs" is provided, and require that "sources" is
    # present.
    if __grains__['os_family'] == 'Solaris':
        if name:
            log.warning('Parameter "name" ignored on Solaris hosts.')
        if pkgs:
            log.warning('Parameter "pkgs" ignored on Solaris hosts.')
        if not sources:
            log.error('"sources" option required with Solaris pkg installs')
            return None,None

    # "pkgs" is always ignored on Solaris.
    if pkgs and sources and __grains__['os_family'] != 'Solaris':
        log.error('Only one of "pkgs" and "sources" can be used.')
        return None,None

    elif pkgs and __grains__['os_family'] != 'Solaris':
        if name:
            log.warning('"name" parameter will be ignored in favor of "pkgs"')
        pkgs = _pack_pkgs(pkgs)
        if not pkgs:
            return None,None
        else:
            return pkgs,'repository'

    elif sources:
        # No need to warn for Solaris, warning taken care of above.
        if name and __grains__['os_family'] != 'Solaris':
            log.warning('"name" parameter will be ignored in favor of '
                        '"sources".')
        sources = _pack_sources(sources)
        if not sources:
            return None,None

        srcinfo = []
        for pkg_name,pkg_src in sources.iteritems():
            if __salt__['config.valid_fileproto'](pkg_src):
                # Cache package from remote source (salt master, http, ftp)
                srcinfo.append((pkg_name,
                                pkg_src,
                               __salt__['cp.cache_file'](pkg_src),
                               'remote'))
            else:
                # Package file local to the minion
                srcinfo.append((pkg_name,pkg_src,pkg_src,'local'))

        # Check metadata to make sure the name passed matches the source
        if __grains__['os_family'] not in ('Solaris',) \
        and __grains__['os'] not in ('Gentoo', 'OpenBSD',):
            problems = _verify_binary_pkg(srcinfo)
            # If any problems are found in the caching or metadata parsing done
            # in the above for loop, log each problem and return None,None,
            # which will keep package installation from proceeding.
            if problems:
                for problem in problems: log.error(problem)
                return None,None

        # srcinfo is a 4-tuple (pkg_name,pkg_uri,pkg_path,pkg_type), so grab
        # the package path (3rd element of tuple).
        return [x[2] for x in srcinfo],'file'

    elif name and __grains__['os_family'] != 'Solaris':
        return [name],'repository'

    else:
        log.error('No package sources passed to pkg.install.')
        return None,None

def add_pkg(pkgs, name, version):
    '''
    Add a package to a dict of installed packages.
    '''

    ''' multiple-version support (not yet implemented)
    cur = pkgs.get(name)
    if cur is None:
        pkgs[name] = version
    elif isinstance(cur, basestring):
        pkgs[name] = [cur, version]
    else:
        pkgs[name].append(version)
    '''
    pkgs[name] = version

def sort_pkglist(pkgs):
    '''
    Accepts a dict obtained from pkg.list_pkgs() and sorts in place the list of
    versions for any packages that have multiple versions installed, so that
    two package lists can be compared to one another.
    '''
    # It doesn't matter that ['4.9','4.10'] would be sorted to ['4.10','4.9'],
    # so long as the sorting is consistent.
    for key in pkgs.keys():
        if isinstance(pkgs[key],list):
            pkgs[key].sort()

def find_changes(old={}, new={}):
    '''
    Compare before and after results from pkg.list_pkgs() to determine what
    changes were made to the packages installed on the minion.
    '''
    pkgs = {}
    for npkg in new:
        if npkg in old:
            if old[npkg] == new[npkg]:
                # no change in the package
                continue
            else:
                # the package was here before and the version has changed
                pkgs[npkg] = {'old': old[npkg],
                              'new': new[npkg]}
        else:
            # the package is freshly installed
            pkgs[npkg] = {'old': '',
                          'new': new[npkg]}
    return pkgs
<|endoftext|>


<commit_before>from __future__ import print_function
import optparse
import os
import codecs
import collections
import numpy as np
from gensim.models import Word2Vec
from pandas import read_json
from entity2vec import entity2vec
from entity2rel import entity2rel
import argparse
import time
from random import shuffle

class entity2rec(entity2vec, entity2rel):

	def __init__(self, is_directed, preprocessing, is_weighted, p, q, walk_length, num_walks, dimensions, window_size, workers, iterations, config, sparql, dataset, entities, default_graph, training, test, implicit):

		entity2vec.__init__(self, is_directed, preprocessing, is_weighted, p, q, walk_length, num_walks, dimensions, window_size, workers, iterations, config, sparql, dataset, entities, default_graph)

		entity2rel.__init__(self, True) #binary format embeddings

		self.training = training

		self.test = test

		self.implicit = implicit

		self._get_items_liked_by_user() #defines the dictionary of items liked by each user in the training set

		self._get_all_items() #define all the items that can be used as candidates for the recommandations

	def _get_embedding_files(self):

		for prop in self.properties:

			self.add_embedding(u'emb/%s/%s/num%s_p%d_q%d_l%s_d%s.emd' %(self.dataset, prop, self.num_walks, int(self.p), int(self.q), self.walk_length,self.dimensions))

	def _get_items_liked_by_user(self):

		self.all_train_items = []

		self.items_liked_by_user_dict = collections.defaultdict(list)

		self.items_ratings_by_user_test = {}

		self.items_rated_by_user_train = collections.defaultdict(list)

		with codecs.open(self.training,'r', encoding='utf-8') as train:

			for line in train:

				line = line.split(' ')

				u = line[0]

				item = line[1]

				relevance = int(line[2])

				self.items_rated_by_user_train[u].append(item)

				self.items_ratings_by_user_test[(u,item)] = relevance #independently from the rating

				if self.implicit == False and relevance >= 4: #only relevant items are used to compute the similarity, rel = 5 in a previous work

					self.items_liked_by_user_dict[u].append(item)

				elif self.implicit == True and relevance == 1:

					self.items_liked_by_user_dict[u].append(item)

				self.all_train_items.append(item)

		self.all_train_items = list(set(self.all_train_items)) #remove duplicates

	def _get_all_items(self):

		self.all_items = []

		if self.entities != "all": #if it has been provided a list of items as an external file, read from it

			del self.all_train_items #free memory space

			with codecs.open(self.entities, 'r', encoding='utf-8') as items:

				for item in items:

					item = item.strip('\n')

					self.all_items.append(item)

		else: #otherwise join the items from the train and test set

			with codecs.open(self.test,'r', encoding='utf-8') as test:

				test_items = []

				for line in test:

					line = line.split(' ')

					u = line[0]

					item = line[1]

					relevance = int(line[2])

					test_items.append(item)

					self.items_ratings_by_user_test[(u,item)] = relevance

				self.all_items = list(set(self.all_train_items+test_items)) #merge lists and remove duplicates

				del self.all_train_items

	def collab_similarity(self, user, item):

		#feedback embedding is always the last of the list

		return self.relatedness_score_by_position(user, item, -1)

	def content_similarities(self, user, item):

		#all other properties

		items_liked_by_user = self.items_liked_by_user_dict[user]

		sims = []

		for past_item in items_liked_by_user:

			sims.append(self.relatedness_scores(past_item,item, -1)) #append a list of property-specific scores, skip feedback

		if len(sims) == 0:
			sims = 0.5*np.ones(len(self.properties))
			return sims

		return np.mean(sims, axis = 0) #return a list of averages of property-specific scores

	@staticmethod
	def parse_user_id(user):

		return int(user.strip('user')) #29

	def parse_users_items_rel(self,line):

		line = line.split(' ')

		user = line[0] #user29

		user_id = entity2rec.parse_user_id(user) #29

		item = line[1] #http://dbpedia.org/resource/The_Golden_Child

		relevance = int(line[2]) #5

		#binarization of the relevance values
		if self.implicit == False:
			relevance = 1 if relevance >= 4 else 0

		return (user, user_id, item, relevance)

	def write_line(self,user, user_id, item, relevance, file):

		file.write('%d qid:%d' %(relevance,user_id))

		count = 1

		collab_score = self.collab_similarity(user, item)

		file.write(' %d:%f' %(count,collab_score))

		count += 1

		content_scores = self.content_similarities(user, item)

		l = len(content_scores)

		for content_score in content_scores:

			if count == l + 1: #last score, end of line

				file.write(' %d:%f # %s\n' %(count,content_score,item))

			else:

				file.write(' %d:%f' %(count,content_score))

				count += 1

	def get_candidates(self,user):

		#get candidates according to the all unrated items protocol
		#use as candidates all the the items that are not in the training set

		rated_items_train = self.items_rated_by_user_train[user] #both in the train and in the test

		candidate_items = [item for item in self.all_items if item not in rated_items_train] #all unrated items in the train

		return candidate_items

	def feature_generator(self):

		#write training set

		start_time = time.time()

		train_name = ((self.training).split('/')[-1]).split('.')[0]

		with codecs.open('features/%s/%s.svm' %(self.dataset,train_name),'w', encoding='utf-8') as train_write:

			with codecs.open(self.training,'r', encoding='utf-8') as training:

				for i, line in enumerate(training):

					user, user_id, item, relevance = self.parse_users_items_rel(line)

					print(user)

					self.write_line(user, user_id, item, relevance, train_write)

		print('finished writing training')

		print("--- %s seconds ---" % (time.time() - start_time))

		#write test set

		test_name = ((self.test).split('/')[-1]).split('.')[0]

		with codecs.open('features/%s/%s.svm' %(self.dataset,test_name),'w', encoding='utf-8') as test_write:

			for user in self.items_rated_by_user_train.keys():

				#write some candidate items

				print(user)

				user_id = entity2rec.parse_user_id(user)

				candidate_items = self.get_candidates(user)

				shuffle(candidate_items) #relevant and non relevant items are shuffled

				for item in candidate_items:

					try:
						rel = int(self.items_ratings_by_user_test[(user,item)]) #get the relevance score if it's in the test

						if self.implicit == False:
							rel = 1 if rel >= 4 else 0

					except KeyError:
						rel = 0. #unrated items are assumed to be negative

					self.write_line(user, user_id, item, rel, test_write)

		print('finished writing test')

		print("--- %s seconds ---" % (time.time() - start_time))

	def run(self, run_all):

		if run_all:
			self.entity2vec.run()
			self._get_embedding_files()

		else:
			self._get_embedding_files()
			self.feature_generator()

	@staticmethod
	def parse_args():

		'''
		Parses the entity2vec arguments.
		'''

		parser = argparse.ArgumentParser(description="Run entity2rec.")

		parser.add_argument('--walk_length', type=int, default=10,
		                    help='Length of walk per source. Default is 10.')

		parser.add_argument('--num_walks', type=int, default=500,
		                    help='Number of walks per source. Default is 40.')

		parser.add_argument('--p', type=float, default=1,
		                    help='Return hyperparameter. Default is 1.')

		parser.add_argument('--q', type=float, default=1,
		                    help='Inout hyperparameter. Default is 1.')

		parser.add_argument('--weighted', dest='weighted', action='store_true',
		                    help='Boolean specifying (un)weighted. Default is unweighted.')
		parser.add_argument('--unweighted', dest='unweighted', action='store_false')
		parser.set_defaults(weighted=False)

		parser.add_argument('--directed', dest='directed', action='store_true',
		                    help='Graph is (un)directed. Default is directed.')
		parser.set_defaults(directed=False)

		parser.add_argument('--no_preprocessing', dest = 'preprocessing', action='store_false',
		                    help='Whether preprocess all transition probabilities or compute on the fly')
		parser.set_defaults(preprocessing=True)

		parser.add_argument('--dimensions', type=int, default=500,
		                    help='Number of dimensions. Default is 128.')

		parser.add_argument('--window-size', type=int, default=10,
	                    	help='Context size for optimization. Default is 10.')

		parser.add_argument('--iter', default=5, type=int,
	                      help='Number of epochs in SGD')

		parser.add_argument('--workers', type=int, default=8,
		                    help='Number of parallel workers. Default is 8.')

		parser.add_argument('--config_file', nargs='?', default='config/properties.json',
		                    help='Path to configuration file')

		parser.add_argument('--dataset', nargs='?', default='movielens_1m',
		                    help='Dataset')

		parser.add_argument('--sparql', dest = 'sparql',
		                    help='Whether downloading the graphs from a sparql endpoint')
		parser.set_defaults(sparql=False)

		parser.add_argument('--entities', dest = 'entities', default = "all",
		                    help='A specific list of entities for which the embeddings have to be computed')

		parser.add_argument('--default_graph', dest = 'default_graph', default = False,
		                    help='Default graph to query when using a Sparql endpoint')

		parser.add_argument('--train', dest='train', help='train', default = False)

		parser.add_argument('--test', dest='test', help='test')

		parser.add_argument('--run_all', dest='run_all', default = False, help='If computing also the embeddings')

		parser.add_argument('--implicit', dest='implicit', default = False, help='Implicit feedback with boolean values')

		return parser.parse_args()

if __name__ == '__main__':

	start_time = time.time()

	args = entity2rec.parse_args()

	rec = entity2rec(args.directed, args.preprocessing, args.weighted, args.p, args.q, args.walk_length, args.num_walks, args.dimensions, args.window_size, args.workers, args.iter, args.config_file, args.sparql, args.dataset, args.entities, args.default_graph, args.train, args.test, args.implicit)

	rec.run(args.run_all)

	print("--- %s seconds ---" % (time.time() - start_time))<commit_msg>add naming p and q to the output features<commit_after>from __future__ import print_function
import optparse
import os
import codecs
import collections
import numpy as np
from gensim.models import Word2Vec
from pandas import read_json
from entity2vec import entity2vec
from entity2rel import entity2rel
import argparse
import time
from random import shuffle

class entity2rec(entity2vec, entity2rel):

	def __init__(self, is_directed, preprocessing, is_weighted, p, q, walk_length, num_walks, dimensions, window_size, workers, iterations, config, sparql, dataset, entities, default_graph, training, test, implicit):

		entity2vec.__init__(self, is_directed, preprocessing, is_weighted, p, q, walk_length, num_walks, dimensions, window_size, workers, iterations, config, sparql, dataset, entities, default_graph)

		entity2rel.__init__(self, True) #binary format embeddings

		self.training = training

		self.test = test

		self.implicit = implicit

		self._get_items_liked_by_user() #defines the dictionary of items liked by each user in the training set

		self._get_all_items() #define all the items that can be used as candidates for the recommandations

	def _get_embedding_files(self):

		for prop in self.properties:

			self.add_embedding(u'emb/%s/%s/num%s_p%d_q%d_l%s_d%s.emd' %(self.dataset, prop, self.num_walks, int(self.p), int(self.q), self.walk_length,self.dimensions))

	def _get_items_liked_by_user(self):

		self.all_train_items = []

		self.items_liked_by_user_dict = collections.defaultdict(list)

		self.items_ratings_by_user_test = {}

		self.items_rated_by_user_train = collections.defaultdict(list)

		with codecs.open(self.training,'r', encoding='utf-8') as train:

			for line in train:

				line = line.split(' ')

				u = line[0]

				item = line[1]

				relevance = int(line[2])

				self.items_rated_by_user_train[u].append(item)

				self.items_ratings_by_user_test[(u,item)] = relevance #independently from the rating

				if self.implicit == False and relevance >= 4: #only relevant items are used to compute the similarity, rel = 5 in a previous work

					self.items_liked_by_user_dict[u].append(item)

				elif self.implicit == True and relevance == 1:

					self.items_liked_by_user_dict[u].append(item)

				self.all_train_items.append(item)

		self.all_train_items = list(set(self.all_train_items)) #remove duplicates

	def _get_all_items(self):

		self.all_items = []

		if self.entities != "all": #if it has been provided a list of items as an external file, read from it

			del self.all_train_items #free memory space

			with codecs.open(self.entities, 'r', encoding='utf-8') as items:

				for item in items:

					item = item.strip('\n')

					self.all_items.append(item)

		else: #otherwise join the items from the train and test set

			with codecs.open(self.test,'r', encoding='utf-8') as test:

				test_items = []

				for line in test:

					line = line.split(' ')

					u = line[0]

					item = line[1]

					relevance = int(line[2])

					test_items.append(item)

					self.items_ratings_by_user_test[(u,item)] = relevance

				self.all_items = list(set(self.all_train_items+test_items)) #merge lists and remove duplicates

				del self.all_train_items

	def collab_similarity(self, user, item):

		#feedback embedding is always the last of the list

		return self.relatedness_score_by_position(user, item, -1)

	def content_similarities(self, user, item):

		#all other properties

		items_liked_by_user = self.items_liked_by_user_dict[user]

		sims = []

		for past_item in items_liked_by_user:

			sims.append(self.relatedness_scores(past_item,item, -1)) #append a list of property-specific scores, skip feedback

		if len(sims) == 0:
			sims = 0.5*np.ones(len(self.properties))
			return sims

		return np.mean(sims, axis = 0) #return a list of averages of property-specific scores

	@staticmethod
	def parse_user_id(user):

		return int(user.strip('user')) #29

	def parse_users_items_rel(self,line):

		line = line.split(' ')

		user = line[0] #user29

		user_id = entity2rec.parse_user_id(user) #29

		item = line[1] #http://dbpedia.org/resource/The_Golden_Child

		relevance = int(line[2]) #5

		#binarization of the relevance values
		if self.implicit == False:
			relevance = 1 if relevance >= 4 else 0

		return (user, user_id, item, relevance)

	def write_line(self,user, user_id, item, relevance, file):

		file.write('%d qid:%d' %(relevance,user_id))

		count = 1

		collab_score = self.collab_similarity(user, item)

		file.write(' %d:%f' %(count,collab_score))

		count += 1

		content_scores = self.content_similarities(user, item)

		l = len(content_scores)

		for content_score in content_scores:

			if count == l + 1: #last score, end of line

				file.write(' %d:%f # %s\n' %(count,content_score,item))

			else:

				file.write(' %d:%f' %(count,content_score))

				count += 1

	def get_candidates(self,user):

		#get candidates according to the all unrated items protocol
		#use as candidates all the the items that are not in the training set

		rated_items_train = self.items_rated_by_user_train[user] #both in the train and in the test

		candidate_items = [item for item in self.all_items if item not in rated_items_train] #all unrated items in the train

		return candidate_items

	def feature_generator(self):

		#write training set

		start_time = time.time()

		train_name = ((self.training).split('/')[-1]).split('.')[0]

		with codecs.open('features/%s/%s_p%d_q%d.svm' %(self.dataset,train_name, int(self.p), int(self.q)),'w', encoding='utf-8') as train_write:

			with codecs.open(self.training,'r', encoding='utf-8') as training:

				for i, line in enumerate(training):

					user, user_id, item, relevance = self.parse_users_items_rel(line)

					print(user)

					self.write_line(user, user_id, item, relevance, train_write)

		print('finished writing training')

		print("--- %s seconds ---" % (time.time() - start_time))

		#write test set

		test_name = ((self.test).split('/')[-1]).split('.')[0]

		with codecs.open('features/%s/%s_p%d_q%d.svm' %(self.dataset,test_name, int(self.p), int(self.q)),'w', encoding='utf-8') as test_write:

			for user in self.items_rated_by_user_train.keys():

				#write some candidate items

				print(user)

				user_id = entity2rec.parse_user_id(user)

				candidate_items = self.get_candidates(user)

				shuffle(candidate_items) #relevant and non relevant items are shuffled

				for item in candidate_items:

					try:
						rel = int(self.items_ratings_by_user_test[(user,item)]) #get the relevance score if it's in the test

						if self.implicit == False:
							rel = 1 if rel >= 4 else 0

					except KeyError:
						rel = 0. #unrated items are assumed to be negative

					self.write_line(user, user_id, item, rel, test_write)

		print('finished writing test')

		print("--- %s seconds ---" % (time.time() - start_time))

	def run(self, run_all):

		if run_all:
			self.entity2vec.run()
			self._get_embedding_files()

		else:
			self._get_embedding_files()
			self.feature_generator()

	@staticmethod
	def parse_args():

		'''
		Parses the entity2vec arguments.
		'''

		parser = argparse.ArgumentParser(description="Run entity2rec.")

		parser.add_argument('--walk_length', type=int, default=10,
		                    help='Length of walk per source. Default is 10.')

		parser.add_argument('--num_walks', type=int, default=500,
		                    help='Number of walks per source. Default is 40.')

		parser.add_argument('--p', type=float, default=1,
		                    help='Return hyperparameter. Default is 1.')

		parser.add_argument('--q', type=float, default=1,
		                    help='Inout hyperparameter. Default is 1.')

		parser.add_argument('--weighted', dest='weighted', action='store_true',
		                    help='Boolean specifying (un)weighted. Default is unweighted.')
		parser.add_argument('--unweighted', dest='unweighted', action='store_false')
		parser.set_defaults(weighted=False)

		parser.add_argument('--directed', dest='directed', action='store_true',
		                    help='Graph is (un)directed. Default is directed.')
		parser.set_defaults(directed=False)

		parser.add_argument('--no_preprocessing', dest = 'preprocessing', action='store_false',
		                    help='Whether preprocess all transition probabilities or compute on the fly')
		parser.set_defaults(preprocessing=True)

		parser.add_argument('--dimensions', type=int, default=500,
		                    help='Number of dimensions. Default is 128.')

		parser.add_argument('--window-size', type=int, default=10,
	                    	help='Context size for optimization. Default is 10.')

		parser.add_argument('--iter', default=5, type=int,
	                      help='Number of epochs in SGD')

		parser.add_argument('--workers', type=int, default=8,
		                    help='Number of parallel workers. Default is 8.')

		parser.add_argument('--config_file', nargs='?', default='config/properties.json',
		                    help='Path to configuration file')

		parser.add_argument('--dataset', nargs='?', default='movielens_1m',
		                    help='Dataset')

		parser.add_argument('--sparql', dest = 'sparql',
		                    help='Whether downloading the graphs from a sparql endpoint')
		parser.set_defaults(sparql=False)

		parser.add_argument('--entities', dest = 'entities', default = "all",
		                    help='A specific list of entities for which the embeddings have to be computed')

		parser.add_argument('--default_graph', dest = 'default_graph', default = False,
		                    help='Default graph to query when using a Sparql endpoint')

		parser.add_argument('--train', dest='train', help='train', default = False)

		parser.add_argument('--test', dest='test', help='test')

		parser.add_argument('--run_all', dest='run_all', default = False, help='If computing also the embeddings')

		parser.add_argument('--implicit', dest='implicit', default = False, help='Implicit feedback with boolean values')

		return parser.parse_args()

if __name__ == '__main__':

	start_time = time.time()

	args = entity2rec.parse_args()

	rec = entity2rec(args.directed, args.preprocessing, args.weighted, args.p, args.q, args.walk_length, args.num_walks, args.dimensions, args.window_size, args.workers, args.iter, args.config_file, args.sparql, args.dataset, args.entities, args.default_graph, args.train, args.test, args.implicit)

	rec.run(args.run_all)

	print("--- %s seconds ---" % (time.time() - start_time))<|endoftext|>


<commit_before>import requests

from modularodm import Q
from rest_framework import generics, permissions as drf_permissions
from rest_framework.exceptions import PermissionDenied, ValidationError, NotFound

from framework.auth.core import Auth
from framework.auth.oauth_scopes import CoreScopes
from api.base import permissions as base_permissions

from website.exceptions import NodeStateError
from website.models import Node, Pointer
from api.users.serializers import ContributorSerializer
from api.base.filters import ODMFilterMixin, ListFilterMixin
from api.base.utils import get_object_or_error
from .serializers import NodeSerializer, NodeLinksSerializer
from .serializers import NodeProviderSerializer
from .permissions import ContributorOrPublic, ReadOnlyIfRegistration, ContributorOrPublicForPointers

from website.files.models import FileNode
from website.files.models import OsfStorageFileNode
from website.util import waterbutler_api_url_for

from api.files.serializers import FileSerializer

class NodeMixin(object):
    """Mixin with convenience methods for retrieving the current node based on the
    current URL. By default, fetches the current node based on the node_id kwarg.
    """

    serializer_class = NodeSerializer
    node_lookup_url_kwarg = 'node_id'

    def get_node(self):
        node = get_object_or_error(
            Node,
            self.kwargs[self.node_lookup_url_kwarg],
            display_name='node'
        )
        # Nodes that are folders/collections are treated as a separate resource, so if the client
        # requests a collection through a node endpoint, we return a 404
        if node.is_folder:
            raise NotFound
        # May raise a permission denied
        self.check_object_permissions(self.request, node)
        return node

class NodeList(generics.ListCreateAPIView, ODMFilterMixin):
    """Projects and components.

    On the front end, nodes are considered 'projects' or 'components'. The difference between a project and a component
    is that a project is the top-level node, and components are children of the project. There is also a category field
    that includes the option of project. The categorization essentially determines which icon is displayed by the
    Node in the front-end UI and helps with search organization. Top-level Nodes may have a category other than
    project, and children nodes may have a category of project.

    By default, a GET will return a list of public nodes, sorted by date_modified. You can filter Nodes by their title,
    description, and public fields.
    """
    permission_classes = (
        drf_permissions.IsAuthenticatedOrReadOnly,
        base_permissions.TokenHasScope
    )
    serializer_class = NodeSerializer
    ordering = ('-date_modified', )  # default ordering

    required_read_scopes = [CoreScopes.NODE_BASE_READ]
    required_write_scopes = [CoreScopes.NODE_BASE_WRITE]

    # overrides ODMFilterMixin
    def get_default_odm_query(self):
        base_query = (
            Q('is_deleted', 'ne', True) &
            Q('is_folder', 'ne', True)
        )
        user = self.request.user
        permission_query = Q('is_public', 'eq', True)
        if not user.is_anonymous():
            permission_query = (Q('is_public', 'eq', True) | Q('contributors', 'icontains', user._id))

        query = base_query & permission_query
        return query

    # overrides ListCreateAPIView
    def get_queryset(self):
        query = self.get_query_from_request()
        return Node.find(query)

    # overrides ListCreateAPIView
    def perform_create(self, serializer):
        """Create a node.

        :param serializer:
        """
        # On creation, make sure that current user is the creator
        user = self.request.user
        serializer.save(creator=user)

class NodeDetail(generics.RetrieveUpdateDestroyAPIView, NodeMixin):
    """Projects and component details.

    On the front end, nodes are considered 'projects' or 'components'. The difference between a project and a component
    is that a project is the top-level node, and components are children of the project. There is also a category field
    that includes the option of project. The categorization essentially determines which icon is displayed by the
    Node in the front-end UI and helps with search organization. Top-level Nodes may have a category other than
    project, and children nodes may have a category of project.
    """
    permission_classes = (
        ContributorOrPublic,
        ReadOnlyIfRegistration,
        base_permissions.TokenHasScope
    )

    serializer_class = NodeSerializer

    required_read_scopes = [CoreScopes.NODE_BASE_READ]
    required_write_scopes = [CoreScopes.NODE_BASE_WRITE]

    # overrides RetrieveUpdateDestroyAPIView
    def get_object(self):
        return self.get_node()

    # overrides RetrieveUpdateDestroyAPIView
    def get_serializer_context(self):
        # Serializer needs the request in order to make an update to privacy
        # TODO: The method it overrides already returns request (plus more stuff). Why does this method exist?
        return {'request': self.request}

    # overrides RetrieveUpdateDestroyAPIView
    def perform_destroy(self, instance):
        user = self.request.user
        auth = Auth(user)
        node = self.get_object()
        try:
            node.remove_node(auth=auth)
        except NodeStateError as err:
            raise ValidationError(err.message)
        node.save()

class NodeContributorsList(generics.ListAPIView, ListFilterMixin, NodeMixin):
    """Contributors (users) for a node.

    Contributors are users who can make changes to the node or, in the case of private nodes,
    have read access to the node. Contributors are divided between 'bibliographic' and 'non-bibliographic'
    contributors. From a permissions standpoint, both are the same, but bibliographic contributors
    are included in citations, while non-bibliographic contributors are not included in citations.
    """

    permission_classes = (
        drf_permissions.IsAuthenticatedOrReadOnly,
        ContributorOrPublic,
        base_permissions.TokenHasScope
    )

    serializer_class = ContributorSerializer

    required_read_scopes = [CoreScopes.NODE_CONTRIBUTORS_READ]
    required_write_scopes = [CoreScopes.NODE_CONTRIBUTORS_WRITE]

    def get_default_queryset(self):
        node = self.get_node()
        visible_contributors = node.visible_contributor_ids
        contributors = []
        for contributor in node.contributors:
            contributor.bibliographic = contributor._id in visible_contributors
            contributors.append(contributor)
        return contributors

    # overrides ListAPIView
    def get_queryset(self):
        return self.get_queryset_from_request()

class NodeRegistrationsList(generics.ListAPIView, NodeMixin):
    """Registrations of the current node.

    Registrations are read-only snapshots of a project. This view lists all of the existing registrations
     created for the current node."""
    permission_classes = (
        ContributorOrPublic,
        drf_permissions.IsAuthenticatedOrReadOnly,
        base_permissions.TokenHasScope
    )

    serializer_class = NodeSerializer

    required_read_scopes = [CoreScopes.NODE_REGISTRATIONS_READ]
    required_write_scopes = [CoreScopes.NODE_REGISTRATIONS_WRITE]

    # overrides ListAPIView
    def get_queryset(self):
        nodes = self.get_node().node__registrations
        user = self.request.user
        if user.is_anonymous():
            auth = Auth(None)
        else:
            auth = Auth(user)
        registrations = [node for node in nodes if node.can_view(auth)]
        return registrations

class NodeChildrenList(generics.ListCreateAPIView, NodeMixin):
    """Children of the current node.

    This will get the next level of child nodes for the selected node if the current user has read access for those
    nodes. Currently, if there is a discrepancy between the children count and the number of children returned, it
    probably indicates private nodes that aren't being returned. That discrepancy should disappear before everything
    is finalized.
    """
    permission_classes = (
        ContributorOrPublic,
        drf_permissions.IsAuthenticatedOrReadOnly,
        base_permissions.TokenHasScope
    )

    serializer_class = NodeSerializer

    required_read_scopes = [CoreScopes.NODE_CHILDREN_READ]
    required_write_scopes = [CoreScopes.NODE_CHILDREN_WRITE]

    # overrides ListAPIView
    def get_queryset(self):
        nodes = self.get_node().nodes
        user = self.request.user
        if user.is_anonymous():
            auth = Auth(None)
        else:
            auth = Auth(user)
        children = [node for node in nodes if node.can_view(auth) and node.primary and not node.is_deleted]
        return children

    # overrides ListCreateAPIView
    def perform_create(self, serializer):
        user = self.request.user
        serializer.save(creator=user, parent=self.get_node())

class NodeLinksList(generics.ListCreateAPIView, NodeMixin):
    """Node Links to other nodes.

    Node Links act as pointers to other nodes. Unlike Forks, they are not copies of nodes;
    Node Links are a direct reference to the node that they point to.
    """
    permission_classes = (
        drf_permissions.IsAuthenticatedOrReadOnly,
        ContributorOrPublic,
        base_permissions.TokenHasScope
    )

    serializer_class = NodeLinksSerializer

    required_read_scopes = [CoreScopes.NODE_LINKS_READ]
    required_write_scopes = [CoreScopes.NODE_LINKS_WRITE]

    def get_queryset(self):
        pointers = self.get_node().nodes_pointer
        return pointers

class NodeLinksDetail(generics.RetrieveDestroyAPIView, NodeMixin):
    """Node Link details.

    Node Links act as pointers to other nodes. Unlike Forks, they are not copies of nodes;
    Node Links are a direct reference to the node that they point to.
    """
    permission_classes = (
        ContributorOrPublicForPointers,
        drf_permissions.IsAuthenticatedOrReadOnly,
        base_permissions.TokenHasScope
    )

    serializer_class = NodeLinksSerializer

    required_read_scopes = [CoreScopes.NODE_LINKS_READ]
    required_write_scopes = [CoreScopes.NODE_LINKS_WRITE]

    # overrides RetrieveAPIView
    def get_object(self):
        node_link_lookup_url_kwarg = 'node_link_id'
        node_link = get_object_or_error(
            Pointer,
            self.kwargs[node_link_lookup_url_kwarg],
            'node link'
        )
        # May raise a permission denied
        self.check_object_permissions(self.request, node_link)
        return node_link

    # overrides DestroyAPIView
    def perform_destroy(self, instance):
        user = self.request.user
        auth = Auth(user)
        node = self.get_node()
        pointer = self.get_object()
        try:
            node.rm_pointer(pointer, auth=auth)
        except ValueError as err:  # pointer doesn't belong to node
            raise ValidationError(err.message)
        node.save()

class NodeFilesList(generics.ListAPIView, NodeMixin):
    """Files attached to a node.

    This gives a list of all of the files that are on your project. Because this works with external services, some
    ours and some not, there is some extra data that you need for how to interact with those services.

    At the top level file list of your project you have a list of providers that are connected to this project. If you
    want to add more, you will need to do that in the Open Science Framework front end for now. For everything in the
    data.links dictionary, you'll have two types of fields: `self` and `related`. These are the same as everywhere else:
    self links are what you use to manipulate the object itself with GET, POST, DELETE, and PUT requests, while
    related links give you further data about that resource.

    So if you GET a self link for a file, it will return the file itself for downloading. If you GET a related link for
    a file, you'll get the metadata about the file. GETting a related link for a folder will get you the listing of
    what's in that folder. GETting a folder's self link won't work, because there's nothing to get.

    Which brings us to the other useful thing about the links here: there's a field called `self-methods`. This field
    will tell you what the valid methods are for the self links given the kind of thing they are (file vs folder) and
    given your permissions on the object.

    NOTE: Most of the API will be stable as far as how the links work because the things they are accessing are fairly
    stable and predictable, so if you felt the need, you could construct them in the normal REST way and they should
    be fine.
    The 'self' links from the NodeFilesList may have to change from time to time, so you are highly encouraged to use
    the links as we provide them before you use them, and not to reverse engineer the structure of the links as they
    are at any given time.
    """
    permission_classes = (
        drf_permissions.IsAuthenticatedOrReadOnly,
        ContributorOrPublic,
        base_permissions.TokenHasScope
    )

    serializer_class = FileSerializer

    required_read_scopes = [CoreScopes.NODE_FILE_READ]
    required_write_scopes = [CoreScopes.NODE_FILE_WRITE]

    def get_file_item(self, item):
        file_node = FileNode.resolve_class(
            item['provider'],
            FileNode.FOLDER if item['kind'] == 'folder'
            else FileNode.FILE
        ).get_or_create(self.get_node(), item['path'])

        file_node.update(None, item)

        return file_node

    def get_queryset(self):
        # Dont bother going to waterbutler for osfstorage
        if self.kwargs['provider'] == 'osfstorage':
            self.check_object_permissions(self.request, self.get_node())
            # Kinda like /me for a user
            # The one odd case where path is not really path
            if self.kwargs['path'] == '/':
                return list(self.get_node().get_addon('osfstorage').get_root().children)

            fobj = OsfStorageFileNode.find_one(
                Q('node', 'eq', self.get_node()._id) &
                Q('path', 'eq', self.kwargs['path'])
            )

            if fobj.is_file:
                return [fobj]

            return list(fobj.children)

        url = waterbutler_api_url_for(
            self.get_node()._id,
            self.kwargs['provider'],
            self.kwargs['path'],
            meta=True
        )

        waterbutler_request = requests.get(url)
        if waterbutler_request.status_code == 401:
            raise PermissionDenied
        try:
            files_list = waterbutler_request.json()['data']
        except KeyError:
            raise ValidationError(detail='detail: Could not retrieve files information.')

        if isinstance(files_list, dict):
            files_list = [files_list]

        return [self.get_file_item(file) for file in files_list]

class NodeProvider(object):

    def __init__(self, provider, node):
        self.path = '/'
        self.node = node
        self.kind = 'folder'
        self.name = provider
        self.provider = provider
        self.node_id = node._id
        self.pk = node._id

class NodeProvidersList(generics.ListAPIView, NodeMixin):
    serializer_class = NodeProviderSerializer

    permission_classes = (
        drf_permissions.IsAuthenticatedOrReadOnly,
        ContributorOrPublic,
    )

    def get_provider_item(self, provider):
        return NodeProvider(provider, self.get_node())

    def get_queryset(self):
        return [
            self.get_provider_item(addon.config.short_name)
            for addon
            in self.get_node().get_addons()
            if addon.config.has_hgrid_files
            and addon.complete
        ]
<commit_msg>Std. Scopes on Node views<commit_after>import requests

from modularodm import Q
from rest_framework import generics, permissions as drf_permissions
from rest_framework.exceptions import PermissionDenied, ValidationError, NotFound

from framework.auth.core import Auth
from framework.auth.oauth_scopes import CoreScopes
from api.base import permissions as base_permissions

from website.exceptions import NodeStateError
from website.models import Node, Pointer
from api.users.serializers import ContributorSerializer
from api.base.filters import ODMFilterMixin, ListFilterMixin
from api.base.utils import get_object_or_error
from .serializers import NodeSerializer, NodeLinksSerializer
from .serializers import NodeProviderSerializer
from .permissions import ContributorOrPublic, ReadOnlyIfRegistration, ContributorOrPublicForPointers

from website.files.models import FileNode
from website.files.models import OsfStorageFileNode
from website.util import waterbutler_api_url_for

from api.files.serializers import FileSerializer

class NodeMixin(object):
    """Mixin with convenience methods for retrieving the current node based on the
    current URL. By default, fetches the current node based on the node_id kwarg.
    """

    serializer_class = NodeSerializer
    node_lookup_url_kwarg = 'node_id'

    def get_node(self):
        node = get_object_or_error(
            Node,
            self.kwargs[self.node_lookup_url_kwarg],
            display_name='node'
        )
        # Nodes that are folders/collections are treated as a separate resource, so if the client
        # requests a collection through a node endpoint, we return a 404
        if node.is_folder:
            raise NotFound
        # May raise a permission denied
        self.check_object_permissions(self.request, node)
        return node

class NodeList(generics.ListCreateAPIView, ODMFilterMixin):
    """Projects and components.

    On the front end, nodes are considered 'projects' or 'components'. The difference between a project and a component
    is that a project is the top-level node, and components are children of the project. There is also a category field
    that includes the option of project. The categorization essentially determines which icon is displayed by the
    Node in the front-end UI and helps with search organization. Top-level Nodes may have a category other than
    project, and children nodes may have a category of project.

    By default, a GET will return a list of public nodes, sorted by date_modified. You can filter Nodes by their title,
    description, and public fields.
    """
    permission_classes = (
        drf_permissions.IsAuthenticatedOrReadOnly,
        base_permissions.TokenHasScope,
    )

    required_read_scopes = [CoreScopes.NODE_BASE_READ]
    required_write_scopes = [CoreScopes.NODE_BASE_WRITE]

    serializer_class = NodeSerializer

    ordering = ('-date_modified', )  # default ordering

    # overrides ODMFilterMixin
    def get_default_odm_query(self):
        base_query = (
            Q('is_deleted', 'ne', True) &
            Q('is_folder', 'ne', True)
        )
        user = self.request.user
        permission_query = Q('is_public', 'eq', True)
        if not user.is_anonymous():
            permission_query = (Q('is_public', 'eq', True) | Q('contributors', 'icontains', user._id))

        query = base_query & permission_query
        return query

    # overrides ListCreateAPIView
    def get_queryset(self):
        query = self.get_query_from_request()
        return Node.find(query)

    # overrides ListCreateAPIView
    def perform_create(self, serializer):
        """Create a node.

        :param serializer:
        """
        # On creation, make sure that current user is the creator
        user = self.request.user
        serializer.save(creator=user)

class NodeDetail(generics.RetrieveUpdateDestroyAPIView, NodeMixin):
    """Projects and component details.

    On the front end, nodes are considered 'projects' or 'components'. The difference between a project and a component
    is that a project is the top-level node, and components are children of the project. There is also a category field
    that includes the option of project. The categorization essentially determines which icon is displayed by the
    Node in the front-end UI and helps with search organization. Top-level Nodes may have a category other than
    project, and children nodes may have a category of project.
    """
    permission_classes = (
        ContributorOrPublic,
        ReadOnlyIfRegistration,
        base_permissions.TokenHasScope,
    )

    required_read_scopes = [CoreScopes.NODE_BASE_READ]
    required_write_scopes = [CoreScopes.NODE_BASE_WRITE]

    serializer_class = NodeSerializer

    # overrides RetrieveUpdateDestroyAPIView
    def get_object(self):
        return self.get_node()

    # overrides RetrieveUpdateDestroyAPIView
    def get_serializer_context(self):
        # Serializer needs the request in order to make an update to privacy
        # TODO: The method it overrides already returns request (plus more stuff). Why does this method exist?
        return {'request': self.request}

    # overrides RetrieveUpdateDestroyAPIView
    def perform_destroy(self, instance):
        user = self.request.user
        auth = Auth(user)
        node = self.get_object()
        try:
            node.remove_node(auth=auth)
        except NodeStateError as err:
            raise ValidationError(err.message)
        node.save()

class NodeContributorsList(generics.ListAPIView, ListFilterMixin, NodeMixin):
    """Contributors (users) for a node.

    Contributors are users who can make changes to the node or, in the case of private nodes,
    have read access to the node. Contributors are divided between 'bibliographic' and 'non-bibliographic'
    contributors. From a permissions standpoint, both are the same, but bibliographic contributors
    are included in citations, while non-bibliographic contributors are not included in citations.
    """
    permission_classes = (
        drf_permissions.IsAuthenticatedOrReadOnly,
        ContributorOrPublic,
        base_permissions.TokenHasScope,
    )

    required_read_scopes = [CoreScopes.NODE_CONTRIBUTORS_READ]
    required_write_scopes = [CoreScopes.NODE_CONTRIBUTORS_WRITE]

    serializer_class = ContributorSerializer

    def get_default_queryset(self):
        node = self.get_node()
        visible_contributors = node.visible_contributor_ids
        contributors = []
        for contributor in node.contributors:
            contributor.bibliographic = contributor._id in visible_contributors
            contributors.append(contributor)
        return contributors

    # overrides ListAPIView
    def get_queryset(self):
        return self.get_queryset_from_request()

class NodeRegistrationsList(generics.ListAPIView, NodeMixin):
    """Registrations of the current node.

    Registrations are read-only snapshots of a project. This view lists all of the existing registrations
     created for the current node."""
    permission_classes = (
        ContributorOrPublic,
        drf_permissions.IsAuthenticatedOrReadOnly,
        base_permissions.TokenHasScope,
    )

    required_read_scopes = [CoreScopes.NODE_REGISTRATIONS_READ]
    required_write_scopes = [CoreScopes.NODE_REGISTRATIONS_WRITE]

    serializer_class = NodeSerializer

    # overrides ListAPIView
    def get_queryset(self):
        nodes = self.get_node().node__registrations
        user = self.request.user
        if user.is_anonymous():
            auth = Auth(None)
        else:
            auth = Auth(user)
        registrations = [node for node in nodes if node.can_view(auth)]
        return registrations

class NodeChildrenList(generics.ListCreateAPIView, NodeMixin):
    """Children of the current node.

    This will get the next level of child nodes for the selected node if the current user has read access for those
    nodes. Currently, if there is a discrepancy between the children count and the number of children returned, it
    probably indicates private nodes that aren't being returned. That discrepancy should disappear before everything
    is finalized.
    """
    permission_classes = (
        ContributorOrPublic,
        drf_permissions.IsAuthenticatedOrReadOnly,
        base_permissions.TokenHasScope,
    )

    required_read_scopes = [CoreScopes.NODE_CHILDREN_READ]
    required_write_scopes = [CoreScopes.NODE_CHILDREN_WRITE]

    serializer_class = NodeSerializer

    # overrides ListAPIView
    def get_queryset(self):
        nodes = self.get_node().nodes
        user = self.request.user
        if user.is_anonymous():
            auth = Auth(None)
        else:
            auth = Auth(user)
        children = [node for node in nodes if node.can_view(auth) and node.primary and not node.is_deleted]
        return children

    # overrides ListCreateAPIView
    def perform_create(self, serializer):
        user = self.request.user
        serializer.save(creator=user, parent=self.get_node())

class NodeLinksList(generics.ListCreateAPIView, NodeMixin):
    """Node Links to other nodes.

    Node Links act as pointers to other nodes. Unlike Forks, they are not copies of nodes;
    Node Links are a direct reference to the node that they point to.
    """
    permission_classes = (
        drf_permissions.IsAuthenticatedOrReadOnly,
        ContributorOrPublic,
        base_permissions.TokenHasScope,
    )

    required_read_scopes = [CoreScopes.NODE_LINKS_READ]
    required_write_scopes = [CoreScopes.NODE_LINKS_WRITE]

    serializer_class = NodeLinksSerializer

    def get_queryset(self):
        pointers = self.get_node().nodes_pointer
        return pointers

class NodeLinksDetail(generics.RetrieveDestroyAPIView, NodeMixin):
    """Node Link details.

    Node Links act as pointers to other nodes. Unlike Forks, they are not copies of nodes;
    Node Links are a direct reference to the node that they point to.
    """
    permission_classes = (
        ContributorOrPublicForPointers,
        drf_permissions.IsAuthenticatedOrReadOnly,
        base_permissions.TokenHasScope,
    )

    required_read_scopes = [CoreScopes.NODE_LINKS_READ]
    required_write_scopes = [CoreScopes.NODE_LINKS_WRITE]

    serializer_class = NodeLinksSerializer

    # overrides RetrieveAPIView
    def get_object(self):
        node_link_lookup_url_kwarg = 'node_link_id'
        node_link = get_object_or_error(
            Pointer,
            self.kwargs[node_link_lookup_url_kwarg],
            'node link'
        )
        # May raise a permission denied
        self.check_object_permissions(self.request, node_link)
        return node_link

    # overrides DestroyAPIView
    def perform_destroy(self, instance):
        user = self.request.user
        auth = Auth(user)
        node = self.get_node()
        pointer = self.get_object()
        try:
            node.rm_pointer(pointer, auth=auth)
        except ValueError as err:  # pointer doesn't belong to node
            raise ValidationError(err.message)
        node.save()

class NodeFilesList(generics.ListAPIView, NodeMixin):
    """Files attached to a node.

    This gives a list of all of the files that are on your project. Because this works with external services, some
    ours and some not, there is some extra data that you need for how to interact with those services.

    At the top level file list of your project you have a list of providers that are connected to this project. If you
    want to add more, you will need to do that in the Open Science Framework front end for now. For everything in the
    data.links dictionary, you'll have two types of fields: `self` and `related`. These are the same as everywhere else:
    self links are what you use to manipulate the object itself with GET, POST, DELETE, and PUT requests, while
    related links give you further data about that resource.

    So if you GET a self link for a file, it will return the file itself for downloading. If you GET a related link for
    a file, you'll get the metadata about the file. GETting a related link for a folder will get you the listing of
    what's in that folder. GETting a folder's self link won't work, because there's nothing to get.

    Which brings us to the other useful thing about the links here: there's a field called `self-methods`. This field
    will tell you what the valid methods are for the self links given the kind of thing they are (file vs folder) and
    given your permissions on the object.

    NOTE: Most of the API will be stable as far as how the links work because the things they are accessing are fairly
    stable and predictable, so if you felt the need, you could construct them in the normal REST way and they should
    be fine.
    The 'self' links from the NodeFilesList may have to change from time to time, so you are highly encouraged to use
    the links as we provide them before you use them, and not to reverse engineer the structure of the links as they
    are at any given time.
    """
    permission_classes = (
        drf_permissions.IsAuthenticatedOrReadOnly,
        ContributorOrPublic,
        base_permissions.TokenHasScope,
    )

    required_read_scopes = [CoreScopes.NODE_FILE_READ]
    required_write_scopes = [CoreScopes.NODE_FILE_WRITE]

    serializer_class = FileSerializer

    def get_file_item(self, item):
        file_node = FileNode.resolve_class(
            item['provider'],
            FileNode.FOLDER if item['kind'] == 'folder'
            else FileNode.FILE
        ).get_or_create(self.get_node(), item['path'])

        file_node.update(None, item)

        return file_node

    def get_queryset(self):
        # Don't bother going to waterbutler for osfstorage
        if self.kwargs['provider'] == 'osfstorage':
            self.check_object_permissions(self.request, self.get_node())
            # Kinda like /me for a user
            # The one odd case where path is not really path
            if self.kwargs['path'] == '/':
                return list(self.get_node().get_addon('osfstorage').get_root().children)

            fobj = OsfStorageFileNode.find_one(
                Q('node', 'eq', self.get_node()._id) &
                Q('path', 'eq', self.kwargs['path'])
            )

            if fobj.is_file:
                return [fobj]

            return list(fobj.children)

        url = waterbutler_api_url_for(
            self.get_node()._id,
            self.kwargs['provider'],
            self.kwargs['path'],
            meta=True
        )

        waterbutler_request = requests.get(url)
        if waterbutler_request.status_code == 401:
            raise PermissionDenied
        try:
            files_list = waterbutler_request.json()['data']
        except KeyError:
            raise ValidationError(detail='detail: Could not retrieve files information.')

        if isinstance(files_list, dict):
            files_list = [files_list]

        return [self.get_file_item(file) for file in files_list]

class NodeProvider(object):

    def __init__(self, provider, node):
        self.path = '/'
        self.node = node
        self.kind = 'folder'
        self.name = provider
        self.provider = provider
        self.node_id = node._id
        self.pk = node._id

class NodeProvidersList(generics.ListAPIView, NodeMixin):
    permission_classes = (
        drf_permissions.IsAuthenticatedOrReadOnly,
        ContributorOrPublic,
        base_permissions.TokenHasScope,
    )

    required_read_scopes = [CoreScopes.NODE_FILE_READ]
    required_write_scopes = [CoreScopes.NODE_FILE_WRITE]

    serializer_class = NodeProviderSerializer

    def get_provider_item(self, provider):
        return NodeProvider(provider, self.get_node())

    def get_queryset(self):
        return [
            self.get_provider_item(addon.config.short_name)
            for addon
            in self.get_node().get_addons()
            if addon.config.has_hgrid_files
            and addon.complete
        ]
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
#
#   Debian Changes Bot
#   Copyright (C) 2008 Chris Lamb <chris@chris-lamb.co.uk>
#
#   This program is free software: you can redistribute it and/or modify
#   it under the terms of the GNU Affero General Public License as
#   published by the Free Software Foundation, either version 3 of the
#   License, or (at your option) any later version.
#
#   This program is distributed in the hope that it will be useful,
#   but WITHOUT ANY WARRANTY; without even the implied warranty of
#   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#   GNU Affero General Public License for more details.
#
#   You should have received a copy of the GNU Affero General Public License
#   along with this program.  If not, see <http://www.gnu.org/licenses/>.

import urllib2

import socket
socket.setdefaulttimeout(10)

# curl -qs https://bugs.debian.org/pseudopackages/pseudo-packages.description | sed -e 's@\([^\t ]*\)[\t ]*\(.*\)@    '\''\1'\'': "\2"\,@g'
pseudo_packages = {
    'base': 'Base system general bugs',
    'cdrom': 'Installation system',
    'spam': 'Spam (reassign spam to here so we can complain about it)',
    'press': 'Press release issues',
    'project': 'Problems related to project administration',
    'general': 'General problems (e.g. "many manpages are mode 755")',
    'nm.debian.org': 'New Member process and nm.debian.org webpages',
    'qa.debian.org': 'The Quality Assurance group',
    'ftp.debian.org': 'Problems with the FTP site',
    'www.debian.org': 'Problems with the WWW site',
    'bugs.debian.org': 'The bug tracking system, @bugs.debian.org',
    'lists.debian.org': 'The mailing lists, debian-*@lists.debian.org',
    'wnpp': 'Work-Needing and Prospective Packages list',
    'cdimage.debian.org': 'CD Image issues',
    'tech-ctte': 'The Debian Technical Committee (see the Constitution)',
    'mirrors': 'Problems with the official mirrors',
    'security.debian.org': 'The Debian Security Team',
    'installation-reports': 'Reports of installation problems with stable & testing',
    'upgrade-reports': 'Reports of upgrade problems for stable & testing',
    'release-notes': 'Problems with the Release Notes',
    'wiki.debian.org': 'Problems with the Debian wiki',
    'security-tracker': 'The Debian Security Bug Tracker',
    'release.debian.org': 'Requests regarding Debian releases and release team tools',
    'debian-i18n': 'Requests regarding Internationalization (i18n) of the distribution',
    'buildd.emdebian.org': 'Problems related to building packages for Emdebian',
    'buildd.debian.org': 'Problems and requests related to the Debian Buildds',
    'debian-maintainers': 'Problems and requests related to Debian Maintainers',
    'snapshot.debian.org': 'Issues with the snapshot.debian.org service ',
    'sponsorship-requests': 'Requests for package review and sponsorship',
    'debian-live': 'General problems with Debian Live systems',
    'pet.debian.net': 'The Debian Package Entropy Tracker',
    'cloud.debian.org': 'Issues involving Debian images for public/private clouds',
    'piuparts.debian.org': 'Issues with the piuparts.debian.org service',
    'sso.debian.org': 'Problems and requests related to the Debian Single Sign On system',
    'tracker.debian.org': 'Issues with the Debian Package Tracker and coordination of its maintenance',
}

class Datasource(object):
    class DataError(Exception):
        pass

    def update(self):
        fileobj = urllib2.urlopen(self.URL)
        return self.parse(fileobj)

class MailParser(object):
    pass

class Message(object):
    def __init__(self):
        if hasattr(self, 'FIELDS'):
            for field in self.FIELDS:
                setattr(self, field, None)
        if hasattr(self, 'OPTIONAL'):
            for field in self.OPTIONAL:
                setattr(self, field, None)

    def __nonzero__(self):
        if hasattr(self, 'FIELDS'):
            for field in self.FIELDS:
                if getattr(self, field) is None:
                    return False
        return True

    def for_irc(self):
        return self.format().encode('utf-8')

    def package_name(self):
        if self.package in pseudo_packages.keys():
            return '[pseudo-package]%s[reset]' % self.package
        else:
            return '[package]%s[reset]' % self.package

import utils
import messages
import datasources
import mailparsers

reload(utils)
reload(messages)
reload(datasources)
reload(mailparsers)
<commit_msg>Update pseudo packages<commit_after># -*- coding: utf-8 -*-
#
#   Debian Changes Bot
#   Copyright (C) 2008 Chris Lamb <chris@chris-lamb.co.uk>
#
#   This program is free software: you can redistribute it and/or modify
#   it under the terms of the GNU Affero General Public License as
#   published by the Free Software Foundation, either version 3 of the
#   License, or (at your option) any later version.
#
#   This program is distributed in the hope that it will be useful,
#   but WITHOUT ANY WARRANTY; without even the implied warranty of
#   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#   GNU Affero General Public License for more details.
#
#   You should have received a copy of the GNU Affero General Public License
#   along with this program.  If not, see <http://www.gnu.org/licenses/>.

import urllib2

import socket
socket.setdefaulttimeout(10)

# curl -qs https://bugs.debian.org/pseudopackages/pseudo-packages.description | sed -e 's@\([^\t ]*\)[\t ]*\(.*\)@    '\''\1'\'': "\2"\,@g'
pseudo_packages = {
    'base': 'Base system general bugs',
    'cdrom': 'Installation system',
    'spam': 'Spam (reassign spam to here so we can complain about it)',
    'press': 'Press release issues',
    'project': 'Problems related to project administration',
    'general': 'General problems (e.g. "many manpages are mode 755")',
    'nm.debian.org': 'New Member process and nm.debian.org webpages',
    'qa.debian.org': 'The Quality Assurance group',
    'ftp.debian.org': 'Problems with the FTP site',
    'www.debian.org': 'Problems with the WWW site',
    'bugs.debian.org': 'The bug tracking system, @bugs.debian.org',
    'lists.debian.org': 'The mailing lists, debian-*@lists.debian.org',
    'wnpp': 'Work-Needing and Prospective Packages list',
    'cdimage.debian.org': 'CD Image issues',
    'tech-ctte': 'The Debian Technical Committee (see the Constitution)',
    'mirrors': 'Problems with the official mirrors',
    'security.debian.org': 'The Debian Security Team',
    'installation-reports': 'Reports of installation problems with stable & testing',
    'upgrade-reports': 'Reports of upgrade problems for stable & testing',
    'release-notes': 'Problems with the Release Notes',
    'wiki.debian.org': 'Problems with the Debian wiki',
    'security-tracker': 'The Debian Security Bug Tracker',
    'release.debian.org': 'Requests regarding Debian releases and release team tools',
    'debian-i18n': 'Requests regarding Internationalization (i18n) of the distribution',
    'buildd.emdebian.org': 'Problems related to building packages for Emdebian',
    'buildd.debian.org': 'Problems and requests related to the Debian Buildds',
    'debian-maintainers': 'Problems and requests related to Debian Maintainers',
    'snapshot.debian.org': 'Issues with the snapshot.debian.org service ',
    'sponsorship-requests': 'Requests for package review and sponsorship',
    'debian-live': 'General problems with Debian Live systems',
    'pet.debian.net': 'The Debian Package Entropy Tracker',
    'cloud.debian.org': 'Issues involving Debian images for public/private clouds',
    'piuparts.debian.org': 'Issues with the piuparts.debian.org service',
    'sso.debian.org': 'Problems and requests related to the Debian Single Sign On system',
    'tracker.debian.org': 'Issues with the Debian Package Tracker and coordination of its maintenance',
    'd-i.debian.org': 'Issues regarding the d-i.debian.org service and general Debian Installer tasks',
    'summit.debconf.org': 'Problems and requests related to the DebConf Summit instance',
}

class Datasource(object):
    class DataError(Exception):
        pass

    def update(self):
        fileobj = urllib2.urlopen(self.URL)
        return self.parse(fileobj)

class MailParser(object):
    pass

class Message(object):
    def __init__(self):
        if hasattr(self, 'FIELDS'):
            for field in self.FIELDS:
                setattr(self, field, None)
        if hasattr(self, 'OPTIONAL'):
            for field in self.OPTIONAL:
                setattr(self, field, None)

    def __nonzero__(self):
        if hasattr(self, 'FIELDS'):
            for field in self.FIELDS:
                if getattr(self, field) is None:
                    return False
        return True

    def for_irc(self):
        return self.format().encode('utf-8')

    def package_name(self):
        if self.package in pseudo_packages.keys():
            return '[pseudo-package]%s[reset]' % self.package
        else:
            return '[package]%s[reset]' % self.package

import utils
import messages
import datasources
import mailparsers

reload(utils)
reload(messages)
reload(datasources)
reload(mailparsers)
<|endoftext|>


<commit_before>import threading
import requests
import Queue
import urlparse
import time
import sys
import traceback
import json

DEBUG=False

class CheckableQueue(Queue.Queue):
    """We only check to forestall doing double work. So we don't
    really care about the race condition..."""

    def __contains__(self, item):
        with self.mutex:
            return item in self.queue

class URLCallRunner(threading.Thread):
    def __init__(self, app, what):
        threading.Thread.__init__(self)
        self.daemon = True
        self.app = app
        self.queue = CheckableQueue()
        self.what = what
        self.stopping = False

    def run(self):
        while not self.stopping:
            tocall = self.queue.get()
            if tocall == None: continue # To implement stop
            method = tocall['method']
            url = tocall['url']
            token = tocall['token']
            data = tocall.get('data')
            headers = {}
            env = {}
            token.addToHeaders(headers)
            token.addToEnv(env)
            if 'mimetype' in tocall:
                headers['Content-type'] = tocall['mimetype']
            # xxxjack should also have credentials, etc
            if headers == {}: headers = None
            if not method:
                method = 'GET'
            try:
                resultStatus = ""
                resultData = ""
                errorMessage = ""
                datetime = time.strftime('%d/%b/%Y %H:%M:%S')
                parsedUrl = urlparse.urlparse(url)
                if DEBUG: print 'URLCaller.run calling', url, 'method', method, 'headers', headers, 'data', data
                if parsedUrl.scheme == '' and parsedUrl.netloc == '':
                    # Local. Call the app directly.
                    # xxxjack have to work out exceptions
                    rep = self.app.request(url, method=method, data=data, headers=headers, env=env)
                    resultStatus = rep.status
                    resultData = rep.data
                    if not resultData:
                        resultData = resultStatus
                else:
                    # Remote.
                    r = requests.request(method, url, data=data, headers=headers)
                    resultStatus = str(r.status_code)
                    resultData = r.text
            except requests.exceptions.RequestException as e:
                msg = traceback.format_exception_only(type(e), e.message)[0].strip()
                resultStatus = '502 URLCaller: %s' % msg
                errorMessage = msg
            except:
                resultStatus = '502 URLCaller: exception while calling URL %s' % url
                print resultStatus
                errorMessage = resultStatus
                traceback.print_exc(file=sys.stdout)
            print '- - - [%s] "- %s %s" - %s' % (datetime, method, url, resultStatus)
            alive = resultStatus[:3] == '200'
            if not alive or DEBUG:
                if resultData:
                    print 'Output:'
                    resultLines = resultData.splitlines()
                    for line in resultLines:
                        print '\t'+line
            representing = tocall.get('representing')
            if representing:
                if not alive and not resultData:
                    resultData = errorMessage
                args = dict(alive=alive, resultData=resultData)
                # xxxjack should we add the token here too?
                self.app.request('/internal/updateStatus/%s' % representing, method='POST', data=json.dumps(args), headers={'Content-type':'application/json'})

    def dump(self):
        rv = 'URLCaller %s (%s):\n' % (repr(self), self.what)
        for qel in self.queue.queue:
            rv += '\t' + repr(qel) + '\n'
        return rv

    def callURL(self, tocall):
        if DEBUG: print 'URLCaller.callURL(%s)' % repr(tocall)
        assert 'token' in tocall
        if tocall.get('aggregate'):
            # We should aggregate this action, so don't insert if already in the queue
            if tocall in self.queue:
                if DEBUG: print '(skipped because aggregate is true)'
                return
        self.queue.put(tocall)

    def stop(self):
        self.stopping = True
        self.queue.put(None)
        try:
            self.join()
        except RuntimeError:
            pass # This can happen if we are actually running via callUrl ourselves...

class URLCaller:
    def __init__(self, app):
        self.dataRunner = URLCallRunner(app, 'internal actions')
        self.extRunner = URLCallRunner(app, 'network actions')
        self.otherRunner = URLCallRunner(app, 'scripting and plugin actions')

    def start(self):
        self.dataRunner.start()
        self.extRunner.start()
        self.otherRunner.start()

    def stop(self):
        self.dataRunner.stop()
        self.extRunner.stop()
        self.otherRunner.stop()

    def dump(self):
        rv = (self.dataRunner.dump() + '\n' +
            self.extRunner.dump() + '\n' +
            self.otherRunner.dump())
        return rv

    def callURL(self, tocall):
        url = tocall['url']
        parsedUrl = urlparse.urlparse(url)
        if parsedUrl.scheme:
            self.extRunner.callURL(tocall)
        elif parsedUrl.path.startswith('/data/') or parsedUrl.path.startswith('/internal/') or parsedUrl.path.startswith('/action/'):
            self.dataRunner.callURL(tocall)
        else:
            self.otherRunner.callURL(tocall)

<commit_msg>Don't print request output if it is identical to the status<commit_after>import threading
import requests
import Queue
import urlparse
import time
import sys
import traceback
import json

DEBUG=False

class CheckableQueue(Queue.Queue):
    """We only check to forestall doing double work. So we don't
    really care about the race condition..."""

    def __contains__(self, item):
        with self.mutex:
            return item in self.queue

class URLCallRunner(threading.Thread):
    def __init__(self, app, what):
        threading.Thread.__init__(self)
        self.daemon = True
        self.app = app
        self.queue = CheckableQueue()
        self.what = what
        self.stopping = False

    def run(self):
        while not self.stopping:
            tocall = self.queue.get()
            if tocall == None: continue # To implement stop
            method = tocall['method']
            url = tocall['url']
            token = tocall['token']
            data = tocall.get('data')
            headers = {}
            env = {}
            token.addToHeaders(headers)
            token.addToEnv(env)
            if 'mimetype' in tocall:
                headers['Content-type'] = tocall['mimetype']
            # xxxjack should also have credentials, etc
            if headers == {}: headers = None
            if not method:
                method = 'GET'
            try:
                resultStatus = ""
                resultData = ""
                errorMessage = ""
                datetime = time.strftime('%d/%b/%Y %H:%M:%S')
                parsedUrl = urlparse.urlparse(url)
                if DEBUG: print 'URLCaller.run calling', url, 'method', method, 'headers', headers, 'data', data
                if parsedUrl.scheme == '' and parsedUrl.netloc == '':
                    # Local. Call the app directly.
                    # xxxjack have to work out exceptions
                    rep = self.app.request(url, method=method, data=data, headers=headers, env=env)
                    resultStatus = rep.status
                    resultData = rep.data
                    if not resultData:
                        resultData = resultStatus
                else:
                    # Remote.
                    r = requests.request(method, url, data=data, headers=headers)
                    resultStatus = str(r.status_code)
                    resultData = r.text
            except requests.exceptions.RequestException as e:
                msg = traceback.format_exception_only(type(e), e.message)[0].strip()
                resultStatus = '502 URLCaller: %s' % msg
                errorMessage = msg
            except:
                resultStatus = '502 URLCaller: exception while calling URL %s' % url
                print resultStatus
                errorMessage = resultStatus
                traceback.print_exc(file=sys.stdout)
            print '- - - [%s] "- %s %s" - %s' % (datetime, method, url, resultStatus)
            alive = resultStatus[:3] == '200'
            if not alive or DEBUG:
                if resultData and resultData.strip() != resultStatus.strip():
                    print 'Output:'
                    resultLines = resultData.splitlines()
                    for line in resultLines:
                        print '\t'+line
            representing = tocall.get('representing')
            if representing:
                if not alive and not resultData:
                    resultData = errorMessage
                args = dict(alive=alive, resultData=resultData)
                # xxxjack should we add the token here too?
                self.app.request('/internal/updateStatus/%s' % representing, method='POST', data=json.dumps(args), headers={'Content-type':'application/json'})

    def dump(self):
        rv = 'URLCaller %s (%s):\n' % (repr(self), self.what)
        for qel in self.queue.queue:
            rv += '\t' + repr(qel) + '\n'
        return rv

    def callURL(self, tocall):
        if DEBUG: print 'URLCaller.callURL(%s)' % repr(tocall)
        assert 'token' in tocall
        if tocall.get('aggregate'):
            # We should aggregate this action, so don't insert if already in the queue
            if tocall in self.queue:
                if DEBUG: print '(skipped because aggregate is true)'
                return
        self.queue.put(tocall)

    def stop(self):
        self.stopping = True
        self.queue.put(None)
        try:
            self.join()
        except RuntimeError:
            pass # This can happen if we are actually running via callUrl ourselves...

class URLCaller:
    def __init__(self, app):
        self.dataRunner = URLCallRunner(app, 'internal actions')
        self.extRunner = URLCallRunner(app, 'network actions')
        self.otherRunner = URLCallRunner(app, 'scripting and plugin actions')

    def start(self):
        self.dataRunner.start()
        self.extRunner.start()
        self.otherRunner.start()

    def stop(self):
        self.dataRunner.stop()
        self.extRunner.stop()
        self.otherRunner.stop()

    def dump(self):
        rv = (self.dataRunner.dump() + '\n' +
            self.extRunner.dump() + '\n' +
            self.otherRunner.dump())
        return rv

    def callURL(self, tocall):
        url = tocall['url']
        parsedUrl = urlparse.urlparse(url)
        if parsedUrl.scheme:
            self.extRunner.callURL(tocall)
        elif parsedUrl.path.startswith('/data/') or parsedUrl.path.startswith('/internal/') or parsedUrl.path.startswith('/action/'):
            self.dataRunner.callURL(tocall)
        else:
            self.otherRunner.callURL(tocall)

<|endoftext|>