<commit_before>from django import forms
from django.forms.models import modelformset_factory

import happyforms
from tower import ugettext_lazy as _lazy

from quieter_formset.formset import BaseModelFormSet

import amo
import reviews
from .models import ReviewFlag, Review

class ReviewReplyForm(forms.Form):
    title = forms.CharField(required=False)
    body = forms.CharField(widget=forms.Textarea(attrs={'rows': 3}))

class ReviewForm(ReviewReplyForm):
    rating = forms.ChoiceField(zip(range(1, 6), range(1, 6)))

class ReviewFlagForm(forms.ModelForm):

    class Meta:
        model = ReviewFlag

    def clean(self):
        data = super(ReviewFlagForm, self).clean()
        if 'note' in data and data['note'].strip():
            data['flag'] = ReviewFlag.OTHER
        return data

class BaseReviewFlagFormSet(BaseModelFormSet):

    def __init__(self, *args, **kwargs):
        self.form = ModerateReviewFlagForm
        super(BaseReviewFlagFormSet, self).__init__(*args, **kwargs)

    def save(self):
        for form in self.forms:
            if form.cleaned_data:
                action = int(form.cleaned_data['action'])

                is_flagged = (form.instance.reviewflag_set.count() > 0)

                if action != reviews.REVIEW_MODERATE_SKIP:  # Delete flags.
                    for flag in form.instance.reviewflag_set.all():
                        flag.delete()

                review = form.instance
                if action == reviews.REVIEW_MODERATE_DELETE:
                    review_addon = review.addon
                    review_id = review.id
                    review.delete()

                    addon = review.addon

                    amo.log(amo.LOG.DELETE_REVIEW, review_addon, review_id,
                            details=dict(title=unicode(review.title),
                                         body=unicode(review.body),
                                         addon_id=addon.id,
                                         addon_title=unicode(addon.name),
                                         is_flagged=is_flagged))
                elif action == reviews.REVIEW_MODERATE_KEEP:
                    review.editorreview = False
                    review.save()
                    amo.log(amo.LOG.APPROVE_REVIEW, review.addon, review
                            details=dict(title=unicode(review.title),
                                         body=unicode(review.body),
                                         addon_id=addon.id,
                                         addon_title=unicode(addon.name),
                                         is_flagged=is_flagged))

class ModerateReviewFlagForm(happyforms.ModelForm):

    action_choices = [(reviews.REVIEW_MODERATE_KEEP,
                       _lazy('Keep review; remove flags')),
                      (reviews.REVIEW_MODERATE_SKIP, _lazy('Skip for now')),
                      (reviews.REVIEW_MODERATE_DELETE, _lazy('Delete review'))]
    action = forms.ChoiceField(choices=action_choices, required=False,
                               initial=0, widget=forms.RadioSelect())

    class Meta:
        model = Review
        fields = ('action',)

ReviewFlagFormSet = modelformset_factory(Review, extra=0,
                                         form=ModerateReviewFlagForm,
                                         formset=BaseReviewFlagFormSet)
<commit_msg>Fixing preview/hudson/etc<commit_after>from django import forms
from django.forms.models import modelformset_factory

import happyforms
from tower import ugettext_lazy as _lazy

from quieter_formset.formset import BaseModelFormSet

import amo
import reviews
from .models import ReviewFlag, Review

class ReviewReplyForm(forms.Form):
    title = forms.CharField(required=False)
    body = forms.CharField(widget=forms.Textarea(attrs={'rows': 3}))

class ReviewForm(ReviewReplyForm):
    rating = forms.ChoiceField(zip(range(1, 6), range(1, 6)))

class ReviewFlagForm(forms.ModelForm):

    class Meta:
        model = ReviewFlag

    def clean(self):
        data = super(ReviewFlagForm, self).clean()
        if 'note' in data and data['note'].strip():
            data['flag'] = ReviewFlag.OTHER
        return data

class BaseReviewFlagFormSet(BaseModelFormSet):

    def __init__(self, *args, **kwargs):
        self.form = ModerateReviewFlagForm
        super(BaseReviewFlagFormSet, self).__init__(*args, **kwargs)

    def save(self):
        for form in self.forms:
            if form.cleaned_data:
                action = int(form.cleaned_data['action'])

                is_flagged = (form.instance.reviewflag_set.count() > 0)

                if action != reviews.REVIEW_MODERATE_SKIP:  # Delete flags.
                    for flag in form.instance.reviewflag_set.all():
                        flag.delete()

                review = form.instance
                if action == reviews.REVIEW_MODERATE_DELETE:
                    review_addon = review.addon
                    review_id = review.id
                    review.delete()

                    addon = review.addon

                    amo.log(amo.LOG.DELETE_REVIEW, review_addon, review_id,
                            details=dict(title=unicode(review.title),
                                         body=unicode(review.body),
                                         addon_id=addon.id,
                                         addon_title=unicode(addon.name),
                                         is_flagged=is_flagged))
                elif action == reviews.REVIEW_MODERATE_KEEP:
                    review.editorreview = False
                    review.save()
                    amo.log(amo.LOG.APPROVE_REVIEW, review.addon, review,
                            details=dict(title=unicode(review.title),
                                         body=unicode(review.body),
                                         addon_id=addon.id,
                                         addon_title=unicode(addon.name),
                                         is_flagged=is_flagged))

class ModerateReviewFlagForm(happyforms.ModelForm):

    action_choices = [(reviews.REVIEW_MODERATE_KEEP,
                       _lazy('Keep review; remove flags')),
                      (reviews.REVIEW_MODERATE_SKIP, _lazy('Skip for now')),
                      (reviews.REVIEW_MODERATE_DELETE, _lazy('Delete review'))]
    action = forms.ChoiceField(choices=action_choices, required=False,
                               initial=0, widget=forms.RadioSelect())

    class Meta:
        model = Review
        fields = ('action',)

ReviewFlagFormSet = modelformset_factory(Review, extra=0,
                                         form=ModerateReviewFlagForm,
                                         formset=BaseReviewFlagFormSet)
<|endoftext|>


<commit_before>from django import http
from django.shortcuts import redirect
from django.utils.encoding import smart_unicode as u

from amo.helpers import page_title

from . import get_service
from .forms import ShareForm

def share(request, obj, name, description):
    try:
        service = get_service(request.GET['service'])
    except KeyError:
        raise http.Http404()
    is_webapp = hasattr(obj, 'is_webapp') and obj.is_webapp()

    form = ShareForm({
        'title': page_title({'request': request}, name,
                            force_webapps=is_webapp),
        'url': u(obj.get_url_path()),
        'description': description,
    })
    form.full_clean()
    return redirect(service.url.format(**form.cleaned_data))
<commit_msg>fix our 404 test.  I'm not sure where all get_service() so adding this outside of it<commit_after>from django import http
from django.shortcuts import redirect
from django.utils.encoding import smart_unicode as u

from amo.helpers import page_title

from . import get_service
from .forms import ShareForm

def share(request, obj, name, description):
    try:
        service = get_service(request.GET['service'])
    except KeyError:
        raise http.Http404()

    if not service:
        raise http.Http404()

    is_webapp = hasattr(obj, 'is_webapp') and obj.is_webapp()

    form = ShareForm({
        'title': page_title({'request': request}, name,
                            force_webapps=is_webapp),
        'url': u(obj.get_url_path()),
        'description': description,
    })
    form.full_clean()
    return redirect(service.url.format(**form.cleaned_data))
<|endoftext|>


<commit_before>import VeloAPI
import os

def get_file(filename, user, username, password):
    velo_api = VeloAPI.Velo()
    if not velo_api.isJVMStarted():
        velo_api.start_jvm()
    rs = velo_api.init_velo(username, password)
    path = os.getcwd() + '/userdata/' + user
    if not os.path.isdir(path):
        os.makedirs(path)

    if velo_api.download_file(filepath, filename):
        content = open(path + file_to_get['filename']).read()
        print content
    else:
        return -1
<commit_msg>working with get_file<commit_after>import VeloAPI
import os

def get_file(filename, user, username, password):
    velo_api = VeloAPI.Velo()
    if not velo_api.isJVMStarted():
        velo_api.start_jvm()
    rs = velo_api.init_velo(username, password)
    path = os.getcwd() + '/userdata/' + user
    print 'path to file ', path
    if not os.path.isdir(path):
        os.makedirs(path)

    if velo_api.download_file(filename, path):
        content = open(path + file_to_get['filename']).read()
        print content
    else:
        return -1
<|endoftext|>


<commit_before># This code is heavily based on contrib.admin's autodiscover mechanism.

import logging
import sys

__name__ = 'appsearch'
__author__ = 'Pivotal Energy Solutions'
__version_info__ = (1, 1, 5)
__version__ = '.'.join(map(str, __version_info__))
__date__ = '2014/07/22 4:47:00 PM'
__credits__ = ['Tim Valenta', "Steven Klass"]
__license__ = 'See the file LICENSE.txt for licensing information.'

log = logging.getLogger(__name__)

def autodiscover():
    """
    Auto-discover INSTALLED_APPS search.py modules and fail silently when
    not present. This forces an import on them to register any appsearch bits
    they may want.
    """

    from importlib import import_module
    from django.apps import apps
    from django.utils.module_loading import module_has_submodule
    from appsearch.registry import search

    for config in apps.get_app_configs():
        if module_has_submodule(config.module, 'search'):
            try:
                mod = import_module(config.name + '.search')
            except:
                exc_type, e, traceback = sys.exc_info()
                raise exc_type, e.message, traceback
<commit_msg>Python 3.7 updates<commit_after># This code is heavily based on contrib.admin's autodiscover mechanism.

import logging
import sys

__name__ = 'appsearch'
__author__ = 'Pivotal Energy Solutions'
__version_info__ = (1, 1, 5)
__version__ = '.'.join(map(str, __version_info__))
__date__ = '2014/07/22 4:47:00 PM'
__credits__ = ['Tim Valenta', "Steven Klass"]
__license__ = 'See the file LICENSE.txt for licensing information.'

log = logging.getLogger(__name__)

def autodiscover():
    """
    Auto-discover INSTALLED_APPS search.py modules and fail silently when
    not present. This forces an import on them to register any appsearch bits
    they may want.
    """

    from importlib import import_module
    from django.apps import apps
    from django.utils.module_loading import module_has_submodule
    from appsearch.registry import search

    for config in apps.get_app_configs():
        if module_has_submodule(config.module, 'search'):
            try:
                mod = import_module(config.name + '.search')
            except:
                exc_type, e, traceback = sys.exc_info()
                raise exc_type(e.message).with_traceback(traceback)

<|endoftext|>


<commit_before>'''
The official executor API for validation test and full test scripts.
'''

import os
import sys
import importlib

from .compiler import call_compiler, call_make, call_configure, GCC
from .result import PassResult, FailResult
from .config import read_config
from . import server

import logging
logger = logging.getLogger('opensubmitexec')

class Job():
    # The current executor configuration.
    _config = None
    # Talk to the configured OpenSubmit server?
    _online = None
    # Download source for the student sub
    submission_url = None
    # Download source for the validator
    validator_url = None
    # The working directory for this job
    working_dir = None
    # The timeout for execution, as demanded by the server
    timeout = None
    # The OpenSubmit submission ID
    submission_id = None
    # The OpenSubmit submission file ID
    file_id = None

    # The base name of the validation / full test script
    # on disk, for importing.
    validator_import_name = 'validator'

    @property
    # The file name of the validation / full test script
    # on disk, after unpacking / renaming.
    def validator_script_name(self):
        return self.working_dir + self.validator_import_name + '.py'

    def __init__(self, config=None, online=True):
        if config:
            self._config = config
        else:
            self._config = read_config()
        self._online = online

    def __str__(self):
        '''
        Nicer logging of job objects.
        '''
        return str(vars(self))

    def run(self):
        '''
        Execute the validate() method in the test script belonging to this job.
        '''
        assert(os.path.exists(self.validator_script_name))
        old_path = sys.path
        sys.path = [self.working_dir] + old_path
        logger.debug('Python search path is now {0}.'.format(sys.path))
        module = importlib.import_module(self.validator_import_name)

        # Looped validator loading in the test suite demands this
        importlib.reload(module)

        # make the call
        module.validate(self)

        # roll back
        sys.path = old_path

    def send_result(self, result):
        '''
        Send test result to the OpenSubmit server.
        '''
        if result:
            post_data = [("SubmissionFileId", self.file_id),
                         ("Message", result.info_student),
                         ("MessageTutor", result.info_tutor),
                         ("ExecutorDir", self.working_dir),
                         ("ErrorCode", result.error_code),
                         ("Secret", self._config.get("Server", "secret")),
                         ("UUID", self._config.get("Server", "uuid"))
                         ]
            logger.debug(
                'Sending result to OpenSubmit Server: ' + str(post_data))
            if self._online:
                server.send(self._config, "/jobs/", post_data)
        else:
            logger.debug('Result is empty, nothing to send.')

    def delete_binaries(self):
        '''
        Scans the submission files in the self.working_dir for
        binaries and deletes them.
        Returns the list of deleted files.
        '''
        pass

    def run_configure(self, mandatory=True):
        '''
        Runs the configure tool configured for the machine in self.working_dir.

        Returns a Result object.
        '''
        result = call_configure(self.working_dir)
        if mandatory:
            return result
        else:
            return PassResult()

    def run_make(self, mandatory=True):
        '''
        Runs the make tool configured for the machine in self.working_dir.

        Returns a Result object.
        '''
        result = call_make(self.working_dir)
        if mandatory:
            return result
        else:
            return PassResult()

    def run_compiler(self, compiler=GCC, inputs=None, output=None):
        '''
        Runs the compiler in self.working_dir.

        Returns a Result object.
        '''
        logger.debug("Running compiler ...")
        return call_compiler(self.working_dir, compiler, output, inputs)

    def run_build(self, compiler=GCC, inputs=None, output=None):
        logger.debug("Running build (configure) ...")
        self.run_configure(mandatory=False)
        logger.debug("Running build (make) ...")
        self.run_make(mandatory=False)
        logger.debug("Running build (compiler) ...")
        return self.run_compiler(compiler=compiler, inputs=inputs, output=output)

    def run_binary(self, args, timeout, exclusive=False):
        '''
        Runs something from self.working_dir in a shell.
        The caller can demand exclusive execution on this machine.
        Returns a CompletedProcess object.
        '''
        return FailResult("Not implemented")

    def find_keywords(self, keywords, filepattern):
        '''
        Searches self.working_dir for files containing specific keywords.
        Expects a list of keywords to be searched for and the file pattern
        (*.c) as parameters.
        Returns the names of the files containing all of the keywords.
        '''
        return FailResult("Not implemented")

    def ensure_files(self, filenames):
        '''
        Searches the student submission for specific files.
        Expects a list of filenames.

        Returns a Result object.
        '''
        logger.debug("Testing {0} for the following files: {1}".format(
            self.working_dir, filenames))
        dircontent = os.listdir(self.working_dir)
        for fname in filenames:
            if fname not in dircontent:
                return FailResult("The file %s is missing." % fname)
        return PassResult("All files found: " + ','.join(filenames))
<commit_msg>Send result automatically, Timeout handling<commit_after>'''
The official executor API for validation test and full test scripts.
'''

import os
import sys
import importlib

from .compiler import call_compiler, call_make, call_configure, GCC
from .result import PassResult, FailResult
from .config import read_config
from .execution import shell_execution, kill_longrunning
from . import server

import logging
logger = logging.getLogger('opensubmitexec')

class Job():
    # The current executor configuration.
    _config = None
    # Talk to the configured OpenSubmit server?
    _online = None
    # Download source for the student sub
    submission_url = None
    # Download source for the validator
    validator_url = None
    # The working directory for this job
    working_dir = None
    # The timeout for execution, as demanded by the server
    timeout = None
    # The OpenSubmit submission ID
    submission_id = None
    # The OpenSubmit submission file ID
    file_id = None
    # Did the validator script sent a result to the server?
    result_sent = False
    # Action requested by the server (legacy)
    action = None

    # The base name of the validation / full test script
    # on disk, for importing.
    validator_import_name = 'validator'

    @property
    # The file name of the validation / full test script
    # on disk, after unpacking / renaming.
    def validator_script_name(self):
        return self.working_dir + self.validator_import_name + '.py'

    def __init__(self, config=None, online=True):
        if config:
            self._config = config
        else:
            self._config = read_config()
        self._online = online

    def __str__(self):
        '''
        Nicer logging of job objects.
        '''
        return str(vars(self))

    def run(self):
        '''
        Execute the validate() method in the test script belonging to this job.
        '''
        assert(os.path.exists(self.validator_script_name))
        old_path = sys.path
        sys.path = [self.working_dir] + old_path
        logger.debug('Python search path is now {0}.'.format(sys.path))
        module = importlib.import_module(self.validator_import_name)

        # Looped validator loading in the test suite demands this
        importlib.reload(module)

        # make the call
        try:
            module.validate(self)
        except Exception as e:
            error_msg = "Exception while running the validate() function: {0}".format(
                e)
            logger.debug(error_msg)
            if not self.result_sent:
                logger.debug(
                    "Sending negative result in the name of the validation script.")
                self.send_result(FailResult(error_msg))
                return
        if not self.result_sent:
            logger.debug(
                "Validation script forgot result sending, sending positive default text.")
            self.send_result(PassResult(
                "The validation was successful. Good job!"))

        # roll back
        sys.path = old_path

    def send_result(self, result):
        '''
        Send test result to the OpenSubmit server.
        '''
        if result:
            post_data = [("SubmissionFileId", self.file_id),
                         ("Message", result.info_student),
                         ("Action", self.action),
                         ("MessageTutor", result.info_tutor),
                         ("ExecutorDir", self.working_dir),
                         ("ErrorCode", result.error_code),
                         ("Secret", self._config.get("Server", "secret")),
                         ("UUID", self._config.get("Server", "uuid"))
                         ]
            logger.debug(
                'Sending result to OpenSubmit Server: ' + str(post_data))
            if self._online:
                server.send(self._config, "/jobs/", post_data)
            self.result_sent = True
        else:
            logger.debug('Result is empty, nothing to send.')

    def delete_binaries(self):
        '''
        Scans the submission files in the self.working_dir for
        binaries and deletes them.
        Returns the list of deleted files.
        '''
        pass

    def run_configure(self, mandatory=True):
        '''
        Runs the configure tool configured for the machine in self.working_dir.

        Returns a Result object.
        '''
        result = call_configure(self.working_dir)
        if mandatory:
            return result
        else:
            return PassResult()

    def run_make(self, mandatory=True):
        '''
        Runs the make tool configured for the machine in self.working_dir.

        Returns a Result object.
        '''
        result = call_make(self.working_dir)
        if mandatory:
            return result
        else:
            return PassResult()

    def run_compiler(self, compiler=GCC, inputs=None, output=None):
        '''
        Runs the compiler in self.working_dir.

        Returns a Result object.
        '''
        logger.debug("Running compiler ...")
        return call_compiler(self.working_dir, compiler, output, inputs)

    def run_build(self, compiler=GCC, inputs=None, output=None):
        logger.debug("Running build (configure) ...")
        self.run_configure(mandatory=False)
        logger.debug("Running build (make) ...")
        self.run_make(mandatory=False)
        logger.debug("Running build (compiler) ...")
        return self.run_compiler(compiler=compiler,
                                 inputs=inputs,
                                 output=output)

    def run_binary(self, name, args=None, timeout=None, exclusive=False):
        '''
        Runs something from self.working_dir in a shell.
        The caller can demand exclusive execution on this machine.
        Returns a CompletedProcess object.
        '''
        if type(name) is str:
            name = [name]
        assert(type(args is list))
        cmdline = name + args

        if exclusive:
            kill_longrunning(self.config)

        return shell_execution(cmdline, self.working_dir, timeout=timeout)

    def find_keywords(self, keywords, filepattern):
        '''
        Searches self.working_dir for files containing specific keywords.
        Expects a list of keywords to be searched for and the file pattern
        (*.c) as parameters.
        Returns the names of the files containing all of the keywords.
        '''
        return FailResult("Not implemented")

    def ensure_files(self, filenames):
        '''
        Searches the student submission for specific files.
        Expects a list of filenames.

        Returns a Result object.
        '''
        logger.debug("Testing {0} for the following files: {1}".format(
            self.working_dir, filenames))
        dircontent = os.listdir(self.working_dir)
        for fname in filenames:
            if fname not in dircontent:
                return FailResult("The file %s is missing." % fname)
        return PassResult("All files found: " + ','.join(filenames))
<|endoftext|>


<commit_before>list_of_list = []
with open("Sacramentorealestatetransactions.csv", 'r') as file:
    for line in file:
        list_of_list.append(line.split(","))
# take input
highest_price = int(input())
lowest_price = int(input())
# find info based on input
new_list = []
for item in list_of_list[1:]:
    if (int(item[9]) >= lowest_price) and (int(item[9]) <= highest_price):
        new_list.append(item)
# show homes
for h in new_list:
    print("\n${}\n{}, {}, {}, {}".format(h[9], h[0], h[1], h[2], h[3]))
<commit_msg>today's exercise<commit_after>with open("Sacramentorealestatetransactions.csv", 'r') as file:
    sret_list = file.read().splitlines()  # open file
highest_price = int(input("Highest price: "))  # take input
lowest_price = int(input("Lowest price: "))  # take info
new_list = []  # create list to append info to
for item in sret_list[1:]:  # find info based on input
    info = item.split(",")  # split lines by commas
    if (int(info[9]) >= lowest_price) and (int(info[9]) <= highest_price):
        new_list.append(info)  # add info to list created earlier
for h in new_list:  # show homes
    print("\n${}\n{}, {}, {}, {}".format(h[9], h[0], h[1], h[2], h[3]))
<|endoftext|>


<commit_before>
"""
Run some experiments using the Netflix dataset
"""
import os
import sys
import errno
import logging
import numpy
import argparse
from exp.recommendexp.RecommendExpHelper import RecommendExpHelper
from exp.recommendexp.NetflixDataset import NetflixDataset

#Uncomment this for the final run
if __debug__:
    raise RuntimeError("Must run python with -O flag")

numpy.random.seed(21)
logging.basicConfig(stream=sys.stdout, level=logging.DEBUG)
numpy.set_printoptions(suppress=True, linewidth=60)
numpy.seterr("raise", under="ignore")

# Arguments related to the dataset
dataArgs = argparse.Namespace()
dataArgs.maxIter = 20

# Arguments related to the algorithm
defaultAlgoArgs = argparse.Namespace()
defaultAlgoArgs.k = 100
defaultAlgoArgs.svdAlg = "arpack"

# init (reading/writting command line arguments)
# data args parser #
dataParser = argparse.ArgumentParser(description="", add_help=False)
dataParser.add_argument("-h", "--help", action="store_true", help="show this help message and exit")
devNull, remainingArgs = dataParser.parse_known_args(namespace=dataArgs)
if dataArgs.help:
    helpParser  = argparse.ArgumentParser(description="", add_help=False, parents=[dataParser, RecommendExpHelper.newAlgoParser(defaultAlgoArgs)])
    helpParser.print_help()
    exit()

dataArgs.extendedDirName = ""
dataArgs.extendedDirName += "NetflixDataset"

# print args #
logging.info("Running on NetflixDataset")
logging.info("Data params:")
keys = list(vars(dataArgs).keys())
keys.sort()
for key in keys:
    logging.info("    " + str(key) + ": " + str(dataArgs.__getattribute__(key)))

# data
generator = NetflixDataset(maxIter=dataArgs.maxIter)

# run
logging.info("Creating the exp-runner")
recommendExpHelper = RecommendExpHelper(generator.getTrainIteratorFunc, generator.getTestIteratorFunc, remainingArgs, defaultAlgoArgs, dataArgs.extendedDirName)
recommendExpHelper.printAlgoArgs()
#    os.makedirs(resultsDir, exist_ok=True) # for python 3.2
try:
    os.makedirs(recommendExpHelper.resultsDir)
except OSError as err:
    if err.errno != errno.EEXIST:
        raise

recommendExpHelper.runExperiment()
<commit_msg>Param tweak. <commit_after>
"""
Run some experiments using the Netflix dataset
"""
import os
import sys
import errno
import logging
import numpy
import argparse
from exp.recommendexp.RecommendExpHelper import RecommendExpHelper
from exp.recommendexp.NetflixDataset import NetflixDataset

#Uncomment this for the final run
if __debug__:
    raise RuntimeError("Must run python with -O flag")

numpy.random.seed(21)
logging.basicConfig(stream=sys.stdout, level=logging.DEBUG)
numpy.set_printoptions(suppress=True, linewidth=60)
numpy.seterr("raise", under="ignore")

# Arguments related to the dataset
dataArgs = argparse.Namespace()
dataArgs.maxIter = 40

# Arguments related to the algorithm
defaultAlgoArgs = argparse.Namespace()
defaultAlgoArgs.k = 100
defaultAlgoArgs.svdAlg = "arpack"

# init (reading/writting command line arguments)
# data args parser #
dataParser = argparse.ArgumentParser(description="", add_help=False)
dataParser.add_argument("-h", "--help", action="store_true", help="show this help message and exit")
devNull, remainingArgs = dataParser.parse_known_args(namespace=dataArgs)
if dataArgs.help:
    helpParser  = argparse.ArgumentParser(description="", add_help=False, parents=[dataParser, RecommendExpHelper.newAlgoParser(defaultAlgoArgs)])
    helpParser.print_help()
    exit()

dataArgs.extendedDirName = ""
dataArgs.extendedDirName += "NetflixDataset"

# print args #
logging.info("Running on NetflixDataset")
logging.info("Data params:")
keys = list(vars(dataArgs).keys())
keys.sort()
for key in keys:
    logging.info("    " + str(key) + ": " + str(dataArgs.__getattribute__(key)))

# data
generator = NetflixDataset(maxIter=dataArgs.maxIter)

# run
logging.info("Creating the exp-runner")
recommendExpHelper = RecommendExpHelper(generator.getTrainIteratorFunc, generator.getTestIteratorFunc, remainingArgs, defaultAlgoArgs, dataArgs.extendedDirName)
recommendExpHelper.printAlgoArgs()
#    os.makedirs(resultsDir, exist_ok=True) # for python 3.2
try:
    os.makedirs(recommendExpHelper.resultsDir)
except OSError as err:
    if err.errno != errno.EEXIST:
        raise

recommendExpHelper.runExperiment()
<|endoftext|>


<commit_before># This module should be kept compatible with Python 1.5.2.

__revision__ = "$Id$"

import sys, os, string
from types import IntType
from distutils.core import Command
from distutils.errors import DistutilsOptionError

class install_lib (Command):

    description = "install all Python modules (extensions and pure Python)"

    # The byte-compilation options are a tad confusing.  Here are the
    # possible scenarios:
    #   1) no compilation at all (--no-compile --no-optimize)
    #   2) compile .pyc only (--compile --no-optimize; default)
    #   3) compile .pyc and "level 1" .pyo (--compile --optimize)
    #   4) compile "level 1" .pyo only (--no-compile --optimize)
    #   5) compile .pyc and "level 2" .pyo (--compile --optimize-more)
    #   6) compile "level 2" .pyo only (--no-compile --optimize-more)
    #
    # The UI for this is two option, 'compile' and 'optimize'.
    # 'compile' is strictly boolean, and only decides whether to
    # generate .pyc files.  'optimize' is three-way (0, 1, or 2), and
    # decides both whether to generate .pyo files and what level of
    # optimization to use.

    user_options = [
        ('install-dir=', 'd', "directory to install to"),
        ('build-dir=','b', "build directory (where to install from)"),
        ('force', 'f', "force installation (overwrite existing files)"),
        ('compile', 'c', "compile .py to .pyc [default]"),
        ('no-compile', None, "don't compile .py files"),
        ('optimize=', 'O',
         "also compile with optimization: -O1 for \"python -O\", "
         "-O2 for \"python -OO\", and -O0 to disable [default: -O0]"),
        ('skip-build', None, "skip the build steps"),
        ]

    boolean_options = ['force', 'compile', 'skip-build']
    negative_opt = {'no-compile' : 'compile'}

    def initialize_options (self):
        # let the 'install' command dictate our installation directory
        self.install_dir = None
        self.build_dir = None
        self.force = 0
        self.compile = None
        self.optimize = None
        self.skip_build = None

    def finalize_options (self):

        # Get all the information we need to install pure Python modules
        # from the umbrella 'install' command -- build (source) directory,
        # install (target) directory, and whether to compile .py files.
        self.set_undefined_options('install',
                                   ('build_lib', 'build_dir'),
                                   ('install_lib', 'install_dir'),
                                   ('force', 'force'),
                                   ('compile', 'compile'),
                                   ('optimize', 'optimize'),
                                   ('skip_build', 'skip_build'),
                                  )

        if self.compile is None:
            self.compile = 1
        if self.optimize is None:
            self.optimize = 0

        if type(self.optimize) is not IntType:
            try:
                self.optimize = int(self.optimize)
                assert 0 <= self.optimize <= 2
            except (ValueError, AssertionError):
                raise DistutilsOptionError, "optimize must be 0, 1, or 2"

    def run (self):

        # Make sure we have built everything we need first
        self.build()

        # Install everything: simply dump the entire contents of the build
        # directory to the installation directory (that's the beauty of
        # having a build directory!)
        outfiles = self.install()

        # (Optionally) compile .py to .pyc
        if outfiles is not None and self.distribution.has_pure_modules():
            self.byte_compile(outfiles)

    # run ()

    # -- Top-level worker functions ------------------------------------
    # (called from 'run()')

    def build (self):
        if not self.skip_build:
            if self.distribution.has_pure_modules():
                self.run_command('build_py')
            if self.distribution.has_ext_modules():
                self.run_command('build_ext')

    def install (self):
        if os.path.isdir(self.build_dir):
            outfiles = self.copy_tree(self.build_dir, self.install_dir)
        else:
            self.warn("'%s' does not exist -- no Python modules to install" %
                      self.build_dir)
            return
        return outfiles

    def byte_compile (self, files):
        from distutils.util import byte_compile

        # Get the "--root" directory supplied to the "install" command,
        # and use it as a prefix to strip off the purported filename
        # encoded in bytecode files.  This is far from complete, but it
        # should at least generate usable bytecode in RPM distributions.
        install_root = self.get_finalized_command('install').root

        if self.compile:
            byte_compile(files, optimize=0,
                         force=self.force, prefix=install_root,
                         dry_run=self.dry_run)
        if self.optimize > 0:
            byte_compile(files, optimize=self.optimize,
                         force=self.force, prefix=install_root,
                         verbose=self.verbose, dry_run=self.dry_run)

    # -- Utility methods -----------------------------------------------

    def _mutate_outputs (self, has_any, build_cmd, cmd_option, output_dir):

        if not has_any:
            return []

        build_cmd = self.get_finalized_command(build_cmd)
        build_files = build_cmd.get_outputs()
        build_dir = getattr(build_cmd, cmd_option)

        prefix_len = len(build_dir) + len(os.sep)
        outputs = []
        for file in build_files:
            outputs.append(os.path.join(output_dir, file[prefix_len:]))

        return outputs

    # _mutate_outputs ()

    def _bytecode_filenames (self, py_filenames):
        bytecode_files = []
        for py_file in py_filenames:
            if self.compile:
                bytecode_files.append(py_file + "c")
            if self.optimize > 0:
                bytecode_files.append(py_file + "o")

        return bytecode_files

    # -- External interface --------------------------------------------
    # (called by outsiders)

    def get_outputs (self):
        """Return the list of files that would be installed if this command
        were actually run.  Not affected by the "dry-run" flag or whether
        modules have actually been built yet.
        """
        pure_outputs = \
            self._mutate_outputs(self.distribution.has_pure_modules(),
                                 'build_py', 'build_lib',
                                 self.install_dir)
        if self.compile:
            bytecode_outputs = self._bytecode_filenames(pure_outputs)
        else:
            bytecode_outputs = []

        ext_outputs = \
            self._mutate_outputs(self.distribution.has_ext_modules(),
                                 'build_ext', 'build_lib',
                                 self.install_dir)

        return pure_outputs + bytecode_outputs + ext_outputs

    # get_outputs ()

    def get_inputs (self):
        """Get the list of files that are input to this command, ie. the
        files that get installed as they are named in the build tree.
        The files in this list correspond one-to-one to the output
        filenames returned by 'get_outputs()'.
        """
        inputs = []

        if self.distribution.has_pure_modules():
            build_py = self.get_finalized_command('build_py')
            inputs.extend(build_py.get_outputs())

        if self.distribution.has_ext_modules():
            build_ext = self.get_finalized_command('build_ext')
            inputs.extend(build_ext.get_outputs())

        return inputs

# class install_lib
<commit_msg>Since build_py handles package data installation, the list of outputs can contain more than just .py files.  Make sure we only report bytecode files for the .py files.<commit_after># This module should be kept compatible with Python 1.5.2.

__revision__ = "$Id$"

import sys, os, string
from types import IntType
from distutils.core import Command
from distutils.errors import DistutilsOptionError

# Extension for Python source files.
PYTHON_SOURCE_EXTENSION = os.extsep + "py"

class install_lib (Command):

    description = "install all Python modules (extensions and pure Python)"

    # The byte-compilation options are a tad confusing.  Here are the
    # possible scenarios:
    #   1) no compilation at all (--no-compile --no-optimize)
    #   2) compile .pyc only (--compile --no-optimize; default)
    #   3) compile .pyc and "level 1" .pyo (--compile --optimize)
    #   4) compile "level 1" .pyo only (--no-compile --optimize)
    #   5) compile .pyc and "level 2" .pyo (--compile --optimize-more)
    #   6) compile "level 2" .pyo only (--no-compile --optimize-more)
    #
    # The UI for this is two option, 'compile' and 'optimize'.
    # 'compile' is strictly boolean, and only decides whether to
    # generate .pyc files.  'optimize' is three-way (0, 1, or 2), and
    # decides both whether to generate .pyo files and what level of
    # optimization to use.

    user_options = [
        ('install-dir=', 'd', "directory to install to"),
        ('build-dir=','b', "build directory (where to install from)"),
        ('force', 'f', "force installation (overwrite existing files)"),
        ('compile', 'c', "compile .py to .pyc [default]"),
        ('no-compile', None, "don't compile .py files"),
        ('optimize=', 'O',
         "also compile with optimization: -O1 for \"python -O\", "
         "-O2 for \"python -OO\", and -O0 to disable [default: -O0]"),
        ('skip-build', None, "skip the build steps"),
        ]

    boolean_options = ['force', 'compile', 'skip-build']
    negative_opt = {'no-compile' : 'compile'}

    def initialize_options (self):
        # let the 'install' command dictate our installation directory
        self.install_dir = None
        self.build_dir = None
        self.force = 0
        self.compile = None
        self.optimize = None
        self.skip_build = None

    def finalize_options (self):

        # Get all the information we need to install pure Python modules
        # from the umbrella 'install' command -- build (source) directory,
        # install (target) directory, and whether to compile .py files.
        self.set_undefined_options('install',
                                   ('build_lib', 'build_dir'),
                                   ('install_lib', 'install_dir'),
                                   ('force', 'force'),
                                   ('compile', 'compile'),
                                   ('optimize', 'optimize'),
                                   ('skip_build', 'skip_build'),
                                  )

        if self.compile is None:
            self.compile = 1
        if self.optimize is None:
            self.optimize = 0

        if type(self.optimize) is not IntType:
            try:
                self.optimize = int(self.optimize)
                assert 0 <= self.optimize <= 2
            except (ValueError, AssertionError):
                raise DistutilsOptionError, "optimize must be 0, 1, or 2"

    def run (self):

        # Make sure we have built everything we need first
        self.build()

        # Install everything: simply dump the entire contents of the build
        # directory to the installation directory (that's the beauty of
        # having a build directory!)
        outfiles = self.install()

        # (Optionally) compile .py to .pyc
        if outfiles is not None and self.distribution.has_pure_modules():
            self.byte_compile(outfiles)

    # run ()

    # -- Top-level worker functions ------------------------------------
    # (called from 'run()')

    def build (self):
        if not self.skip_build:
            if self.distribution.has_pure_modules():
                self.run_command('build_py')
            if self.distribution.has_ext_modules():
                self.run_command('build_ext')

    def install (self):
        if os.path.isdir(self.build_dir):
            outfiles = self.copy_tree(self.build_dir, self.install_dir)
        else:
            self.warn("'%s' does not exist -- no Python modules to install" %
                      self.build_dir)
            return
        return outfiles

    def byte_compile (self, files):
        from distutils.util import byte_compile

        # Get the "--root" directory supplied to the "install" command,
        # and use it as a prefix to strip off the purported filename
        # encoded in bytecode files.  This is far from complete, but it
        # should at least generate usable bytecode in RPM distributions.
        install_root = self.get_finalized_command('install').root

        if self.compile:
            byte_compile(files, optimize=0,
                         force=self.force, prefix=install_root,
                         dry_run=self.dry_run)
        if self.optimize > 0:
            byte_compile(files, optimize=self.optimize,
                         force=self.force, prefix=install_root,
                         verbose=self.verbose, dry_run=self.dry_run)

    # -- Utility methods -----------------------------------------------

    def _mutate_outputs (self, has_any, build_cmd, cmd_option, output_dir):

        if not has_any:
            return []

        build_cmd = self.get_finalized_command(build_cmd)
        build_files = build_cmd.get_outputs()
        build_dir = getattr(build_cmd, cmd_option)

        prefix_len = len(build_dir) + len(os.sep)
        outputs = []
        for file in build_files:
            outputs.append(os.path.join(output_dir, file[prefix_len:]))

        return outputs

    # _mutate_outputs ()

    def _bytecode_filenames (self, py_filenames):
        bytecode_files = []
        for py_file in py_filenames:
            # Since build_py handles package data installation, the
            # list of outputs can contain more than just .py files.
            # Make sure we only report bytecode for the .py files.
            ext = os.path.splitext(os.path.normcase(py_file))[1]
            if ext != PYTHON_SOURCE_EXTENSION:
                continue
            if self.compile:
                bytecode_files.append(py_file + "c")
            if self.optimize > 0:
                bytecode_files.append(py_file + "o")

        return bytecode_files

    # -- External interface --------------------------------------------
    # (called by outsiders)

    def get_outputs (self):
        """Return the list of files that would be installed if this command
        were actually run.  Not affected by the "dry-run" flag or whether
        modules have actually been built yet.
        """
        pure_outputs = \
            self._mutate_outputs(self.distribution.has_pure_modules(),
                                 'build_py', 'build_lib',
                                 self.install_dir)
        if self.compile:
            bytecode_outputs = self._bytecode_filenames(pure_outputs)
        else:
            bytecode_outputs = []

        ext_outputs = \
            self._mutate_outputs(self.distribution.has_ext_modules(),
                                 'build_ext', 'build_lib',
                                 self.install_dir)

        return pure_outputs + bytecode_outputs + ext_outputs

    # get_outputs ()

    def get_inputs (self):
        """Get the list of files that are input to this command, ie. the
        files that get installed as they are named in the build tree.
        The files in this list correspond one-to-one to the output
        filenames returned by 'get_outputs()'.
        """
        inputs = []

        if self.distribution.has_pure_modules():
            build_py = self.get_finalized_command('build_py')
            inputs.extend(build_py.get_outputs())

        if self.distribution.has_ext_modules():
            build_ext = self.get_finalized_command('build_ext')
            inputs.extend(build_ext.get_outputs())

        return inputs

# class install_lib
<|endoftext|>


<commit_before>from django.core.serializers import python
from StringIO import StringIO
from django.utils.encoding import smart_str, smart_unicode
from django.utils import datetime_safe

class Serializer(python.Serializer):
    """
    """
    def start_serialization(self, total):
        self._current = None
        self.objects = {
            self.meta['root']: [],
            self.meta['total']: total,
            self.meta['success']: True
        }

    def end_serialization(self):
        pass

    def start_object(self, obj):
        self._current = {}

    def end_object(self, obj):
        rec = self._current
        rec['id'] = smart_unicode(obj._get_pk_val(), strings_only=True)

        for extra in self.extras:
                rec[extra[0]] = extra[1](obj)

        self.objects[self.meta['root']].append(rec)
        self._current = None

    def handle_field(self, obj, field):
        self._current[field.name] = smart_unicode(getattr(obj, field.name), strings_only=True)

    def handle_fk_field(self, obj, field):
        related = getattr(obj, field.name)
        if related is not None:
            if field.rel.field_name == related._meta.pk.name:
                # Related to remote object via primary key
                self._current[field.name+'_id'] = smart_unicode(related._get_pk_val(), strings_only=True)
                self._current[field.name] = smart_unicode(related, strings_only=True)
            else:
                # Related to remote object via other field
                related = getattr(related, field.rel.field_name)
                self._current[field.name] = self._current[field.name+'_id'] = smart_unicode(getattr(related, field.rel.field_name), strings_only=True)

    def handle_m2m_field(self, obj, field):
        if field.creates_table:
            self._current[field.name+'_ids'] = [smart_unicode(related._get_pk_val(), strings_only=True)
                                                for related in getattr(obj, field.name).iterator()]

    def serialize(self, queryset, **options):
        """
        Serialize a queryset.
        """
        self.options = options

        self.stream = options.get("stream", StringIO())
        self.selected_fields = options.get("fields")

        self.local_fields = options.get("local")

        self.exclude_fields = options.get("exclude_fields")

        self.meta = options.get('meta', dict(root='records', total='total', success='success'))
        self.extras = options.get('extras', [])

        total = options.get("total", queryset.count())
        self.start_serialization(total)

        for obj in queryset:
            if self.local_fields:
                fields = obj._meta.local_fields
            else:
                fields = obj._meta.fields

            fields = [f for f in fields if f.name not in self.exclude_fields]
            self.start_object(obj)
            for field in fields:
                if field.serialize:
                    if field.rel is None:
                        if self.selected_fields is None or field.attname in self.selected_fields:
                            self.handle_field(obj, field)
                    else:
                        if self.selected_fields is None or field.attname[:-3] in self.selected_fields:
                            self.handle_fk_field(obj, field)
            for field in obj._meta.many_to_many:
                if field.serialize:
                    if self.selected_fields is None or field.attname in self.selected_fields:
                        self.handle_m2m_field(obj, field)
            self.end_object(obj)
        self.end_serialization()
        return self.getvalue()

<commit_msg>Change on the "extdirect" serializer because of changes in ManyToManyField in recent versions of Django.<commit_after>from django.core.serializers import python
from StringIO import StringIO
from django.utils.encoding import smart_str, smart_unicode
from django.utils import datetime_safe

class Serializer(python.Serializer):
    """
    """
    def start_serialization(self, total):
        self._current = None
        self.objects = {
            self.meta['root']: [],
            self.meta['total']: total,
            self.meta['success']: True
        }

    def end_serialization(self):
        pass

    def start_object(self, obj):
        self._current = {}

    def end_object(self, obj):
        rec = self._current
        rec['id'] = smart_unicode(obj._get_pk_val(), strings_only=True)

        for extra in self.extras:
                rec[extra[0]] = extra[1](obj)

        self.objects[self.meta['root']].append(rec)
        self._current = None

    def handle_field(self, obj, field):
        self._current[field.name] = smart_unicode(getattr(obj, field.name), strings_only=True)

    def handle_fk_field(self, obj, field):
        related = getattr(obj, field.name)
        if related is not None:
            if field.rel.field_name == related._meta.pk.name:
                # Related to remote object via primary key
                self._current[field.name+'_id'] = smart_unicode(related._get_pk_val(), strings_only=True)
                self._current[field.name] = smart_unicode(related, strings_only=True)
            else:
                # Related to remote object via other field
                related = getattr(related, field.rel.field_name)
                self._current[field.name] = self._current[field.name+'_id'] = smart_unicode(getattr(related, field.rel.field_name), strings_only=True)

    def handle_m2m_field(self, obj, field):
        if field.rel.through._meta.auto_created:
            if self.use_natural_keys and hasattr(field.rel.to, 'natural_key'):
                m2m_value = lambda value: value.natural_key()
            else:
                m2m_value = lambda value: smart_unicode(value._get_pk_val(), strings_only=True)
            self._current[field.name+'_ids'] = [m2m_value(related)
                               for related in getattr(obj, field.name).iterator()]

        # Seems that recent version of Django, lost the creates_table attribute.
        #if field.creates_table:
        #    self._current[field.name+'_ids'] = [smart_unicode(related._get_pk_val(), strings_only=True)
        #                                        for related in getattr(obj, field.name).iterator()]

    def serialize(self, queryset, **options):
        """
        Serialize a queryset.
        """
        self.options = options

        self.stream = options.get("stream", StringIO())
        self.selected_fields = options.get("fields")
        self.use_natural_keys = options.get("use_natural_keys", False)
        self.local_fields = options.get("local")

        self.exclude_fields = options.get("exclude_fields")

        self.meta = options.get('meta', dict(root='records', total='total', success='success'))
        self.extras = options.get('extras', [])

        total = options.get("total", queryset.count())
        self.start_serialization(total)

        for obj in queryset:
            if self.local_fields:
                fields = obj._meta.local_fields
            else:
                fields = obj._meta.fields

            fields = [f for f in fields if f.name not in self.exclude_fields]
            self.start_object(obj)
            for field in fields:
                if field.serialize:
                    if field.rel is None:
                        if self.selected_fields is None or field.attname in self.selected_fields:
                            self.handle_field(obj, field)
                    else:
                        if self.selected_fields is None or field.attname[:-3] in self.selected_fields:
                            self.handle_fk_field(obj, field)
            for field in obj._meta.many_to_many:
                if field.serialize:
                    if self.selected_fields is None or field.attname in self.selected_fields:
                        self.handle_m2m_field(obj, field)
            self.end_object(obj)
        self.end_serialization()
        return self.getvalue()

<|endoftext|>


<commit_before># YASS, Yet Another Subdomainer Software
# Copyright 2015-2019 Francesco Marano (@mrnfrancesco) and individual contributors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

from yass.models import PluginBase

class Aol(PluginBase):
    class Meta:
        search_url = 'https://search.aol.com/aol/search'
        subdomains_selector = 'span.fz-ms.fw-m.fc-12th.wr-bw.lh-17'

class Ask(PluginBase):
    class Meta:
        search_url = 'https://www.ask.com/web'
        subdomains_selector = 'p.PartialSearchResults-item-url'

class Baidu(PluginBase):
    class Meta:
        search_url = 'https://www.baidu.com/s'
        query_param = 'wd'
        subdomains_selector = 'span.g'

class Bing(PluginBase):
    class Meta:
        search_url = 'https://www.bing.com/search'
        subdomains_selector = '#b_results li.b_algo cite'

# Anti-bot tricks make yass fails with Google
#
# class Google(PluginBase):
#     class Meta:
#         search_url = 'https://www.google.com/search'
#         subdomains_selector = '#search div.g cite'
#         request_delay = 1

class StartPage(PluginBase):
    class Meta:
        search_url = 'https://startpage.com/do/search'
        subdomains_selector = 'span.search-item__url'

class WebCrawler(PluginBase):
    class Meta:
        search_url = 'https://www.webcrawler.com/serp'
        subdomains_selector = 'div.resultDisplayUrl'
        include_param = ''
        exclude_param = '-'

class Yahoo(PluginBase):
    class Meta:
        search_url = 'https://search.yahoo.com/search'
        query_param = 'p'
        subdomains_selector = 'span.fz-ms.fw-m.fc-12th.wr-bw.lh-17'
<commit_msg>Add Teoma and Exaled search-engines support<commit_after># YASS, Yet Another Subdomainer Software
# Copyright 2015-2019 Francesco Marano (@mrnfrancesco) and individual contributors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

from yass.models import PluginBase

class Aol(PluginBase):
    class Meta:
        search_url = 'https://search.aol.com/aol/search'
        subdomains_selector = 'span.fz-ms.fw-m.fc-12th.wr-bw.lh-17'

class Ask(PluginBase):
    class Meta:
        search_url = 'https://www.ask.com/web'
        subdomains_selector = 'p.PartialSearchResults-item-url'

class Baidu(PluginBase):
    class Meta:
        search_url = 'https://www.baidu.com/s'
        query_param = 'wd'
        subdomains_selector = 'span.g'

class Bing(PluginBase):
    class Meta:
        search_url = 'https://www.bing.com/search'
        subdomains_selector = '#b_results li.b_algo cite'

# Anti-bot tricks make yass fails with Google
#
# class Google(PluginBase):
#     class Meta:
#         search_url = 'https://www.google.com/search'
#         subdomains_selector = '#search div.g cite'
#         request_delay = 1

class StartPage(PluginBase):
    class Meta:
        search_url = 'https://startpage.com/do/search'
        subdomains_selector = 'span.search-item__url'

class WebCrawler(PluginBase):
    class Meta:
        search_url = 'https://www.webcrawler.com/serp'
        subdomains_selector = 'div.resultDisplayUrl'
        include_param = ''
        exclude_param = '-'

class Yahoo(PluginBase):
    class Meta:
        search_url = 'https://search.yahoo.com/search'
        query_param = 'p'
        subdomains_selector = 'span.fz-ms.fw-m.fc-12th.wr-bw.lh-17'

class Teoma(PluginBase):
    class Meta:
        search_url = 'https://www.teoma.com/web'
        subdomains_selector = '.algo-display-url'

class Exalead(PluginBase):
    class Meta:
        search_url = 'https://www.exalead.com/search/web/results'
        subdomains_selector = 'li.media a.ellipsis'
<|endoftext|>


<commit_before># YASS, Yet Another Subdomainer Software
# Copyright 2015 Francesco Marano (@mrnfrancesco) and individual contributors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

from models.plugin import PluginBase

class Aol(PluginBase):
    class Meta:
        search_url = 'search.aol.com/aol/search'
        subdomains_selector = 'span[property="f:durl"]'

class Ask(PluginBase):
    class Meta:
        search_url = 'http://www.ask.com/web'
        subdomains_selector = 'p.durl'

class Baidu(PluginBase):
    class Meta:
        search_url = 'https://www.baidu.com/s'
        query_param = 'wd'
        subdomains_selector = 'span.g'

class Bing(PluginBase):
    class Meta:
        search_url = 'http://www.bing.com/search'
        subdomains_selector = '#b_results li.b_algo cite'

class Google(PluginBase):
    class Meta:
        search_url = 'https://www.google.com/search'
        subdomains_selector = 'li.g cite'
        request_delay = 1

class StartPage(PluginBase):
    class Meta:
        search_url = 'https://startpage.com/do/search'
        subdomains_selector = 'span.url'

class WebCrawler(PluginBase):
    class Meta:
        search_url = 'http://www.webcrawler.com/search/web'
        subdomains_selector = 'div.resultDisplayUrl'

class Yahoo(PluginBase):
    class Meta:
        search_url = 'https://search.yahoo.com/search'
        query_param = 'p'
        subdomains_selector = 'div.compTitle div span'
<commit_msg>Fixed MissingSchema error on Aol plugin<commit_after># YASS, Yet Another Subdomainer Software
# Copyright 2015 Francesco Marano (@mrnfrancesco) and individual contributors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

from models.plugin import PluginBase

class Aol(PluginBase):
    class Meta:
        search_url = 'http://search.aol.com/aol/search'
        subdomains_selector = 'span[property="f:durl"]'

class Ask(PluginBase):
    class Meta:
        search_url = 'http://www.ask.com/web'
        subdomains_selector = 'p.durl'

class Baidu(PluginBase):
    class Meta:
        search_url = 'https://www.baidu.com/s'
        query_param = 'wd'
        subdomains_selector = 'span.g'

class Bing(PluginBase):
    class Meta:
        search_url = 'http://www.bing.com/search'
        subdomains_selector = '#b_results li.b_algo cite'

class Google(PluginBase):
    class Meta:
        search_url = 'https://www.google.com/search'
        subdomains_selector = 'li.g cite'
        request_delay = 1

class StartPage(PluginBase):
    class Meta:
        search_url = 'https://startpage.com/do/search'
        subdomains_selector = 'span.url'

class WebCrawler(PluginBase):
    class Meta:
        search_url = 'http://www.webcrawler.com/search/web'
        subdomains_selector = 'div.resultDisplayUrl'

class Yahoo(PluginBase):
    class Meta:
        search_url = 'https://search.yahoo.com/search'
        query_param = 'p'
        subdomains_selector = 'div.compTitle div span'
<|endoftext|>


<commit_before>import os

from fab_deploy2.base import gunicorn as base_gunicorn
from fab_deploy2.tasks import task_method
from fab_deploy2 import functions

from fabric.api import sudo, env
from fabric.contrib.files import append

class Gunicorn(base_gunicorn.Gunicorn):
    """
    Install gunicorn and set it up with supervisor.
    """

    user = 'nginx'
    group = 'nginx'
    daemonize = False

    @task_method
    def start(self):
        with settings(warn_only=True):
            result = sudo('start %s' % self.get_name())

        if result.failed:
            self.restart()

    @task_method
    def stop(self):
        sudo('stop %s' % self.get_name())

    def _setup_service(self, env_value=None):
        gunicorn_conf = os.path.join(env.configs_path,
            "gunicorn/supervisor_{0}.conf".format(
                self.gunicorn_name))
        # Copy instead of linking so upstart
        # picks up the changes
        sudo('cp %s /etc/init/' % gunicorn_conf)
        sudo('initctl reload-configuration')

    def upload_templates(self):
        context = super(Gunicorn, self).upload_templates()

        return context

    def _setup_rotate(self, path):
        text = [
        "%s {" % path,
        "    copytruncate",
        "    size 1M",
        "    rotate 5",
        "}"]
        sudo('touch /etc/logrotate.d/%s.conf' % self.gunicorn_name)
        append('/etc/logrotate.d/%s.conf' % self.gunicorn_name,
                                    text, use_sudo=True)

Gunicorn().as_tasks()
<commit_msg>gunicorn redhat mod<commit_after>import os

from fab_deploy2.base import gunicorn as base_gunicorn
from fab_deploy2.tasks import task_method
from fab_deploy2 import functions

from fabric.api import sudo, env
from fabric.contrib.files import append

class Gunicorn(base_gunicorn.Gunicorn):
    """
    Install gunicorn and set it up with supervisor.
    """

    user = 'nginx'
    group = 'nginx'
    daemonize = False

    @task_method
    def start(self):
        with settings(warn_only=True):
            result = sudo('start %s' % self.get_name())

        if result.failed:
            self.restart()

    @task_method
    def stop(self):
        sudo('stop %s' % self.get_name())

    def _setup_service(self, env_value=None):
        gunicorn_conf = os.path.join(env.configs_path,
            "gunicorn/redhat_{0}.conf".format(
                self.gunicorn_name))
        # Copy instead of linking so upstart
        # picks up the changes
        sudo('cp %s /etc/init/' % gunicorn_conf)
        sudo('initctl reload-configuration')

    def upload_templates(self):
        context = super(Gunicorn, self).upload_templates()
        functions.render_template("gunicorn/supervisor_gunicorn.conf",
                        os.path.join(
                            env.configs_path, "gunicorn/redhat_{0}.conf".format(
                                self.gunicorn_name)),
                        context=context)

        return context

    def _setup_rotate(self, path):
        text = [
        "%s {" % path,
        "    copytruncate",
        "    size 1M",
        "    rotate 5",
        "}"]
        sudo('touch /etc/logrotate.d/%s.conf' % self.gunicorn_name)
        append('/etc/logrotate.d/%s.conf' % self.gunicorn_name,
                                    text, use_sudo=True)

Gunicorn().as_tasks()
<|endoftext|>


<commit_before>import json
import uuid
from nga_districts.models import LGA
from facilities.models import Facility, Variable, KeyRename, \
    CalculatedVariable, Sector

class FacilityBuilder(object):
    """
    facility_builder.py is where we'll have code to convert a
    dictionary into a facility with data. This will allow us to load
    facilities from survey data or from csv files.
    """

    SURVEYS_WITH_FACILITIES = ['Health', 'Water', 'Education']

    @classmethod
    def create_facility_from_dict(cls, d):
        """
        Requires facility type and lga to be specified in d, all other
        key value pairs in d that are in the data dictionary will be
        added to the database.
        """
        KeyRename.rename_keys(d)

        # using gps as facility id is a slight hack to get a unique id
        kwargs = {
            'facility_id': d.get('gps', uuid.uuid4())
            }
        if '_lga_id' in d:
            kwargs['lga'] = LGA.objects.get(id=d['_lga_id'])
        if '_facility_type' in d:
             kwargs['sector'] = Sector.objects.get(
                slug=d['_facility_type'].lower())
        facility, created = Facility.objects.get_or_create(**kwargs)
        facility.add_data_from_dict(d)

        return facility

    @classmethod
    def print_dict(cls, d):
        print json.dumps(d, indent=4, sort_keys=True)

    @classmethod
    def create_facility_from_instance(cls, survey_instance):
        # When an instance is saved, first delete the parsed_instance
        # associated with it.
        idict = survey_instance.get_dict()

        def add_survey_type_from_xform_id_string(d):
            l = d['_xform_id_string'].split("_")
            d['_survey_type'] = l[0]
        add_survey_type_from_xform_id_string(idict)

        if idict['_survey_type'] not in cls.SURVEYS_WITH_FACILITIES:
            return

        def add_lga_id(d):
            try:
                zone = d['location/zone']
                state = d['location/state_in_%s' % zone]
                lga = d['location/lga_in_%s' % state]
                d['_lga_id'] = LGA.objects.get(slug=lga, state__slug=state).id
            except KeyError:
                raise Exception(d)
        add_lga_id(idict)

        def add_data_source(d):
            d['_data_source'] = d['_xform_id_string']
        add_data_source(idict)

        return cls.create_facility_from_dict(idict)
<commit_msg>catching LGA & Sector not found errors to stop failing test<commit_after>import json
import uuid
from nga_districts.models import LGA
from facilities.models import Facility, Variable, KeyRename, \
    CalculatedVariable, Sector

class FacilityBuilder(object):
    """
    facility_builder.py is where we'll have code to convert a
    dictionary into a facility with data. This will allow us to load
    facilities from survey data or from csv files.
    """

    SURVEYS_WITH_FACILITIES = ['Health', 'Water', 'Education']

    @classmethod
    def create_facility_from_dict(cls, d):
        """
        Requires facility type and lga to be specified in d, all other
        key value pairs in d that are in the data dictionary will be
        added to the database.
        """
        KeyRename.rename_keys(d)

        # using gps as facility id is a slight hack to get a unique id
        kwargs = {
            'facility_id': d.get('gps', uuid.uuid4())
            }
        if '_lga_id' in d:
            try:
                kwargs['lga'] = LGA.objects.get(id=d['_lga_id'])
            except LGA.DoesNotExist, e:
                pass
        if '_facility_type' in d:
            try:
                kwargs['sector'] = Sector.objects.get(
                   slug=d['_facility_type'].lower())
            except Sector.DoesNotExist, e:
                pass
        facility, created = Facility.objects.get_or_create(**kwargs)
        facility.add_data_from_dict(d)

        return facility

    @classmethod
    def print_dict(cls, d):
        print json.dumps(d, indent=4, sort_keys=True)

    @classmethod
    def create_facility_from_instance(cls, survey_instance):
        # When an instance is saved, first delete the parsed_instance
        # associated with it.
        idict = survey_instance.get_dict()

        def add_survey_type_from_xform_id_string(d):
            l = d['_xform_id_string'].split("_")
            d['_survey_type'] = l[0]
        add_survey_type_from_xform_id_string(idict)

        if idict['_survey_type'] not in cls.SURVEYS_WITH_FACILITIES:
            return

        def add_lga_id(d):
            try:
                zone = d['location/zone']
                state = d['location/state_in_%s' % zone]
                lga = d['location/lga_in_%s' % state]
                d['_lga_id'] = LGA.objects.get(slug=lga, state__slug=state).id
            except KeyError:
                raise Exception(d)
        add_lga_id(idict)

        def add_data_source(d):
            d['_data_source'] = d['_xform_id_string']
        add_data_source(idict)

        return cls.create_facility_from_dict(idict)
<|endoftext|>


<commit_before>#!/usr/bin/env python
# file: youtube-feed.py
# vim:fileencoding=utf-8:fdm=marker:ft=python
#
# Copyright © 2019 R.F. Smith <rsmith@xs4all.nl>.
# SPDX-License-Identifier: MIT
# Created: 2019-07-28T13:42:58+0200
# Last modified: 2021-03-12T23:15:09+0100
"""Get the latest video's from your favorite youtube channels.

    This script now generates commands for the mpv viewer that force it
    to use youtube-dl.  This was done so I could limit the quality of the
    downloaded clips to my screen resolution.
"""

import datetime
import html
import json
import os
import re
import urllib.request
import sys

base = "https://www.youtube.com/feeds/videos.xml?channel_id="
now = datetime.datetime.now(tz=datetime.timezone.utc)

# Read limit and channels from the configuration file.
# The file format is JSON, as shown below:
# {
#     "limit": 7,
#     "viewer": "mpv",
#     "channels": {
#         "first channel name": "UC7QoixBiFO2zstyEXkbVpVg",
#         "second channel name": "UCvY8pgX_3ksKvZBNFHVoUAQ",
#         "third channel name": "UCk3wqxZummX-HTzYGjHDBpw"
#     }
# }
# The “limit” is optional. If not present, it will be set to 7 days.
# The "viewer" is also optional. If not present, it will be set to 'mpv'.
# Note that the channelId's above are not real, they're randomly generated!
# You can find the “channelId” in the source of the channel's homepage.
rcpath = os.environ["HOME"] + os.sep + ".youtube-feedrc"
with open(rcpath) as rc:
    settings = json.load(rc)
    channels = settings["channels"]
    if "limit" in settings:
        limit = settings["limit"]
    else:
        limit = 7
    if "viewer" in settings:
        viewer = settings["viewer"]
    else:
        viewer = "mpv"

# Flush after every line.
sys.stdout.reconfigure(line_buffering=True)
# Retrieve and print video information
for channel_title, channel_id in channels.items():
    with urllib.request.urlopen(base + channel_id) as con:
        text = con.read().decode('utf-8')
        rv = con.code
    if not rv == 200:
        print(f"Could not retrieve data for “{channel_title}”, skipping it.\n")
        continue
    titles = re.findall("<title>(.*)</title>", text, re.IGNORECASE)
    links = re.findall('<link rel="alternate" href="(.*)"/>', text, re.IGNORECASE)
    published = [
        datetime.datetime.fromisoformat(pt)
        for pt in re.findall("<published>(.*)</published>", text, re.IGNORECASE)
    ]
    items = tuple(
        (html.unescape(title), link, date)
        for title, link, date in zip(titles, links, published)
        if "watch" in link and (now - date).days < limit
    )
    if not items:
        print(f"No video's less than {limit} days old found for “{channel_title}”.\n")
        continue
    print(f"Channel “{channel_title}”")
    for title, link, date in items:
        print(f"∙ title: “{title}”")
        print(f"   date: {date}")
        # Instruct mpv to use youtube-dl.
        if viewer == "mpv":
            link = re.sub("^https", "ytdl", link)
        print(f"   view: {viewer} '{link}' &")
    print()
<commit_msg>Better error reporting for youtube-feed.<commit_after>#!/usr/bin/env python
# file: youtube-feed.py
# vim:fileencoding=utf-8:fdm=marker:ft=python
#
# Copyright © 2019 R.F. Smith <rsmith@xs4all.nl>.
# SPDX-License-Identifier: MIT
# Created: 2019-07-28T13:42:58+0200
# Last modified: 2021-03-13T19:20:06+0100
"""Get the latest video's from your favorite youtube channels.

    This script now generates commands for the mpv viewer that force it
    to use youtube-dl.  This was done so I could limit the quality of the
    downloaded clips to my screen resolution.
"""

import datetime
import html
import json
import os
import re
import urllib.request
import sys

base = "https://www.youtube.com/feeds/videos.xml?channel_id="
now = datetime.datetime.now(tz=datetime.timezone.utc)

# Read limit and channels from the configuration file.
# The file format is JSON, as shown below:
# {
#     "limit": 7,
#     "viewer": "mpv",
#     "channels": {
#         "first channel name": "UC7QoixBiFO2zstyEXkbVpVg",
#         "second channel name": "UCvY8pgX_3ksKvZBNFHVoUAQ",
#         "third channel name": "UCk3wqxZummX-HTzYGjHDBpw"
#     }
# }
# The “limit” is optional. If not present, it will be set to 7 days.
# The "viewer" is also optional. If not present, it will be set to 'mpv'.
# Note that the channelId's above are not real, they're randomly generated!
# You can find the “channelId” in the source of the channel's homepage.
rcpath = os.environ["HOME"] + os.sep + ".youtube-feedrc"
with open(rcpath) as rc:
    settings = json.load(rc)
    channels = settings["channels"]
    if "limit" in settings:
        limit = settings["limit"]
    else:
        limit = 7
    if "viewer" in settings:
        viewer = settings["viewer"]
    else:
        viewer = "mpv"

# Flush after every line.
sys.stdout.reconfigure(line_buffering=True)
# Retrieve and print video information
for channel_title, channel_id in channels.items():
    with urllib.request.urlopen(base + channel_id) as con:
        text = con.read().decode('utf-8')
        rv = con.code
    if rv != 200:
        print(f"Could not retrieve data for “{channel_title}” (code: {rv}), skipping it.\n")
        continue
    titles = re.findall("<title>(.*)</title>", text, re.IGNORECASE)
    links = re.findall('<link rel="alternate" href="(.*)"/>', text, re.IGNORECASE)
    published = [
        datetime.datetime.fromisoformat(pt)
        for pt in re.findall("<published>(.*)</published>", text, re.IGNORECASE)
    ]
    items = tuple(
        (html.unescape(title), link, date)
        for title, link, date in zip(titles, links, published)
        if "watch" in link and (now - date).days < limit
    )
    if not items:
        print(f"No video's less than {limit} days old found for “{channel_title}”.\n")
        continue
    print(f"Channel “{channel_title}”")
    for title, link, date in items:
        print(f"∙ title: “{title}”")
        print(f"   date: {date}")
        # Instruct mpv to use youtube-dl.
        if viewer == "mpv":
            link = re.sub("^https", "ytdl", link)
        print(f"   view: {viewer} '{link}' &")
    print()
<|endoftext|>


<commit_before>"""
Baltrad to S3 porting
"""

import sys

from creds import URL, LOGIN, PASSWORD
import datamover as dm

def main():
    """Run data transfer from Baltrad to S3"""

    # ------------------
    # DATA TRANSFER
    # ------------------

    # Setup the connection of the Baltrad and S3
    btos = dm.BaltradToS3(URL, LOGIN, PASSWORD, "lw-enram")

    # Execute the transfer
    btos.transfer(name_match="_vp_", overwrite=False, limit=None)
    btos.report(reset_file=False, transfertype="Baltrad to S3")

    # ------------------
    # UPDATE COVERAGE
    # ------------------

    # Connect to S3 client
    s3client = dm.S3EnramHandler("lw-enram")

    # Rerun file list overview to extract the current coverage
    coverage_count = s3client.count_enram_coverage(level='day')
    with open("coverage.csv", 'w') as outfile:
        dm.coverage_to_csv(outfile, coverage_count)
    s3client.upload_file("coverage.csv", "coverage.csv")

    # ----------------------------
    # UPDATE ZIP FILE AVAILABILITY
    # ----------------------------
    # Rerun ZIP handling of S3 for the transferred files, given by report
    s3client.create_zip_version(btos.transferred)

if __name__ == "__main__":
    sys.exit(main())
<commit_msg>Extend data pipeline from baltrad<commit_after>"""
Baltrad to S3 porting
"""

import sys

from creds import URL, LOGIN, PASSWORD
import datamover as dm

def main():
    """Run data transfer from Baltrad to S3"""

    # ------------------
    # DATA TRANSFER
    # ------------------

    # Setup the connection of the Baltrad and S3
    btos = dm.BaltradToS3(URL, LOGIN, PASSWORD, "lw-enram")

    # Execute the transfer
    btos.transfer(name_match="_vp_", overwrite=False, limit=None)
    btos.report(reset_file=False, transfertype="Baltrad to S3")

    # ---------------------------------------------
    # UPDATE COVERAGE AND MOST RECENT FILE DATETIME
    # ---------------------------------------------

    # Connect to S3 client
    s3client = dm.S3EnramHandler("lw-enram")

    # Rerun file list overview to extract the current coverage
    coverage_count, most_recent = s3client.count_enram_coverage(level='day')

    # Save the coverage information on S3
    with open("coverage.csv", 'w') as outfile:
        dm.coverage_to_csv(outfile, coverage_count)
    s3client.upload_file("coverage.csv", "coverage.csv")

    # Save the last provided radar file information on S3
    with open("radars.csv", 'w') as outfile:
        dm.most_recent_to_csv(outfile, most_recent)
    s3client.upload_file("radars.csv", "radars.csv")

    # ----------------------------
    # UPDATE ZIP FILE AVAILABILITY
    # ----------------------------
    # Rerun ZIP handling of S3 for the transferred files, given by report
    s3client.create_zip_version(btos.transferred)

if __name__ == "__main__":
    sys.exit(main())
<|endoftext|>


<commit_before>"""
Excition Object
"""

import time
from math import e

import numpy as np

from src.utils import generate_random_points_in_sphere

class Exciton(object):
    def __init__(self, nanoparticle, num_exc, gen_exition, r_elec=0):
        """
        A exciton, which will be inside the nanoparticle.
        It can be generated at any place of this, with uniformal distribution,
        using the method laser_generated() or by using a chemical electrolysis
        electro_generated() method.

        Parameters
        ----------
        nanoparticle : object
            Nanoparticle objects.
        num_exc : float
            Numbers of exitation of the same nanoparticle.
        gen_exition : str
            Way to generate the exiton
        """
        self.nano_particle = nanoparticle
        self.num_exc = num_exc
        self.generation_exition = gen_exition
        self.r_electro = r_elec

        # Constante usada en el calculo de la probabilidad
        # la calculo una sola vez.
        self.cte_p_et = self.nano_particle.delta_t * self.nano_particle.r_forster**6/self.nano_particle.tau_d

    def laser_generated(self):
        """
        Generate the random position of the exciton in any
        part of the nanoparticle, pretending that this is
        bombarded by a laser. Because the diameter of the
        nanoparticle is too small, it is assumed that all
        are bombarded with the same intensity.
        """
        point = generate_random_points_in_sphere(1, self.nano_particle.radius)
        self.position = point[0]

    def electro_generated(self):
        """
        The random position is generated by a chemical electrolysis.
        This position is generated between the radius R of the
        nanoparticle and a radius r, where r depends electrolysis.
        """
        point = generate_random_points_in_sphere(1, self.nano_particle.radius,
                                                 self.r_electro)
        self.position = point[0]

    def walk(self):
        """
        Exciton make a random walk inside the nanoparticle.
        Like in this case the radius is fixed (epsilon),
        the point of the random walk can generate using spherical coordinates.
        Then we'll have to check that taking this step,
        the exciton does not leave the nanoparticle
        """
        check = 1

        while check == 1:
            new_r = generate_random_points_in_sphere(1, self.nano_particle.epsilon,
                                                     self.nano_particle.epsilon)[0]

            new_dist = np.sqrt(sum((new_r + self.position)**2))
            if new_dist <= self.nano_particle.radius:
                check = 0

        self.position += new_r

    def p_transfer(self):
        """
        Returns the sum of the probability that a exciton is
        transferred to the acceptor. Each probability has the form:
        (1 / tau_D) (R_0 / r [i]) ** 6
        where r [i] is the distance between the exciton and
        each acceptor and R_0 Forster radius.

        Returns
        -------
        prob : float
             Probability that a exciton is transferred to the acceptor.
        """

        diff = self.position - self.nano_particle.acceptors_positions
        component_square = diff*diff
        one_over_distance = 1/(component_square[:, 0] + component_square[:, 1] +
                               component_square[:, 2])
        distance_6 = one_over_distance*one_over_distance*one_over_distance

        prob = 1 - np.e**(-self.cte_p_et*sum(distance_6))

        return prob

    def quenching(self, each=True):
        """
        A random variable (VA) with the probability of decay is compared.
        If the VA is less, compared with the probability of transferirce.
        If less, the exiton makes a random wall.
        This is repeated n-times where n is the number of exitation (num_exc).

        Parameters:
        -----------
        each : bool, optinal
          If True, in each repetition, new positions of dopants is generated.
          Else, only generate one time the dopants
        """

        self.cant_decay = 0
        self.cant_transf = 0
        num_walk = 0

        time_ini = time.time()
        self.positions_init = np.zeros((self.num_exc, 3))
        self.positions_end = np.zeros((self.num_exc, 3))

        if not each:
            # Dopamiento
            if self.nano_particle.generation_acceptors == 'sup':
                self.nano_particle.deposit_superficial_acceptors()
            else:
                self.nano_particle.deposit_volumetrically_acceptors()

        for cont in range(self.num_exc):
            check = 0

            if each:
                # Dopamiento
                if self.nano_particle.generation_acceptors == 'sup':
                    self.nano_particle.deposit_superficial_acceptors()
                else:
                    self.nano_particle.deposit_volumetrically_acceptors()

            # Excition
            if self.generation_exition == 'elec':
                self.electro_generated()
            else:
                self.laser_generated()

            self.positions_init[cont] = self.position.copy()

            while check == 0:
                rand_num = np.random.random()
                if self.nano_particle.p_decay > rand_num:
                    self.cant_decay += 1
                    check = 1
                elif self.p_transfer() > rand_num:
                    self.cant_transf += 1
                    check = 1
                else:
                    self.walk()
                    num_walk += 1

            self.positions_end[cont] = self.position.copy()

        dist = np.zeros(self.num_exc)
        resta = (self.positions_init - self.positions_end)*(self.positions_init -
                                                            self.positions_end)

        dist[:] = np.sqrt(resta[:, 0] + resta[:, 1] + resta[:, 2])
        self.l_d_rms = np.sqrt(sum(dist**2)/(len(dist)*6))
        #self.l_d_rms = sum(dist)/len(dist)

        self.walk_mean = num_walk/self.num_exc
        self.total_time = time.time() - time_ini
        self.efficiency = self.cant_transf / self.num_exc

    def get_input(self):
        """Return a list with the imputs parametes"""
        return [self.nano_particle.r_mean, self.nano_particle.r_deviation,
                self.nano_particle.r_forster, self.nano_particle.mean_path,
                self.nano_particle.tau_d, self.nano_particle.n_acceptors,
                self.nano_particle.epsilon, self.num_exc,
                self.nano_particle.delta_t, self.nano_particle.p_decay]

    def get_output(self):
        """Return a list with the output parameters"""
        return np.array([self.cant_decay, self.cant_transf,
                         self.efficiency, self.total_time,
                         self.l_d_rms, self.walk_mean])

<commit_msg>Cambiado el orden, se ve si decae y luego en que forma<commit_after>"""
Excition Object
"""

import time
from math import e

import numpy as np

from src.utils import generate_random_points_in_sphere

class Exciton(object):
    def __init__(self, nanoparticle, num_exc, gen_exition, r_elec=0):
        """
        A exciton, which will be inside the nanoparticle.
        It can be generated at any place of this, with uniformal distribution,
        using the method laser_generated() or by using a chemical electrolysis
        electro_generated() method.

        Parameters
        ----------
        nanoparticle : object
            Nanoparticle objects.
        num_exc : float
            Numbers of exitation of the same nanoparticle.
        gen_exition : str
            Way to generate the exiton
        """
        self.nano_particle = nanoparticle
        self.num_exc = num_exc
        self.generation_exition = gen_exition
        self.r_electro = r_elec

        # Constante usada en el calculo de la probabilidad
        # la calculo una sola vez.
        self.cte_p_et = self.nano_particle.delta_t * self.nano_particle.r_forster**6/self.nano_particle.tau_d

    def laser_generated(self):
        """
        Generate the random position of the exciton in any
        part of the nanoparticle, pretending that this is
        bombarded by a laser. Because the diameter of the
        nanoparticle is too small, it is assumed that all
        are bombarded with the same intensity.
        """
        point = generate_random_points_in_sphere(1, self.nano_particle.radius)
        self.position = point[0]

    def electro_generated(self):
        """
        The random position is generated by a chemical electrolysis.
        This position is generated between the radius R of the
        nanoparticle and a radius r, where r depends electrolysis.
        """
        point = generate_random_points_in_sphere(1, self.nano_particle.radius,
                                                 self.r_electro)
        self.position = point[0]

    def walk(self):
        """
        Exciton make a random walk inside the nanoparticle.
        Like in this case the radius is fixed (epsilon),
        the point of the random walk can generate using spherical coordinates.
        Then we'll have to check that taking this step,
        the exciton does not leave the nanoparticle
        """
        check = 1

        while check == 1:
            new_r = generate_random_points_in_sphere(1, self.nano_particle.epsilon,
                                                     self.nano_particle.epsilon)[0]

            new_dist = np.sqrt(sum((new_r + self.position)**2))
            if new_dist <= self.nano_particle.radius:
                check = 0

        self.position += new_r

    def p_transfer(self):
        """
        Returns the sum of the probability that a exciton is
        transferred to the acceptor. Each probability has the form:
        (1 / tau_D) (R_0 / r [i]) ** 6
        where r [i] is the distance between the exciton and
        each acceptor and R_0 Forster radius.

        Returns
        -------
        prob : float
             Probability that a exciton is transferred to the acceptor.
        """

        diff = self.position - self.nano_particle.acceptors_positions
        component_square = diff*diff
        one_over_distance = 1/(component_square[:, 0] + component_square[:, 1] +
                               component_square[:, 2])
        distance_6 = one_over_distance*one_over_distance*one_over_distance

        prob = 1 - np.e**(-self.cte_p_et*sum(distance_6))

        return prob

    def quenching(self, each=True):
        """
        A random variable (VA) with the probability of decay is compared.
        If the VA is less, compared with the probability of transferirce.
        If less, the exiton makes a random wall.
        This is repeated n-times where n is the number of exitation (num_exc).

        Parameters:
        -----------
        each : bool, optinal
          If True, in each repetition, new positions of dopants is generated.
          Else, only generate one time the dopants
        """

        self.cant_decay = 0
        self.cant_transf = 0
        num_walk = 0

        time_ini = time.time()
        self.positions_init = np.zeros((self.num_exc, 3))
        self.positions_end = np.zeros((self.num_exc, 3))

        if not each:
            # Dopamiento
            if self.nano_particle.generation_acceptors == 'sup':
                self.nano_particle.deposit_superficial_acceptors()
            else:
                self.nano_particle.deposit_volumetrically_acceptors()

        for cont in range(self.num_exc):
            check = 0

            if each:
                # Dopamiento
                if self.nano_particle.generation_acceptors == 'sup':
                    self.nano_particle.deposit_superficial_acceptors()
                else:
                    self.nano_particle.deposit_volumetrically_acceptors()

            # Excition
            if self.generation_exition == 'elec':
                self.electro_generated()
            else:
                self.laser_generated()

            self.positions_init[cont] = self.position.copy()

            # Cambiado el orden, se ve si decae y luego en que forma
            while check == 0:
                rand_num = np.random.random()
                p_transfer = self.p_transfer()
                if self.nano_particle.p_decay + p_transfer > rand_num:
                    psi_et = p_transfer / (p_transfer + 1 / self.nano_particle.tau_d)
                    if psi_et < rand_num:
                        self.cant_transf += 1
                    else:
                        self.cant_decay += 1
                    check = 1
                else:
                    self.walk()
                    num_walk += 1

            self.positions_end[cont] = self.position.copy()

        dist = np.zeros(self.num_exc)
        resta = (self.positions_init - self.positions_end)*(self.positions_init -
                                                            self.positions_end)

        dist[:] = np.sqrt(resta[:, 0] + resta[:, 1] + resta[:, 2])
        self.l_d_rms = np.sqrt(sum(dist**2)/(len(dist)*6))
        #self.l_d_rms = sum(dist)/len(dist)

        self.walk_mean = num_walk/self.num_exc
        self.total_time = time.time() - time_ini
        self.efficiency = self.cant_transf / self.num_exc

    def get_input(self):
        """Return a list with the imputs parametes"""
        return [self.nano_particle.r_mean, self.nano_particle.r_deviation,
                self.nano_particle.r_forster, self.nano_particle.mean_path,
                self.nano_particle.tau_d, self.nano_particle.n_acceptors,
                self.nano_particle.epsilon, self.num_exc,
                self.nano_particle.delta_t, self.nano_particle.p_decay]

    def get_output(self):
        """Return a list with the output parameters"""
        return np.array([self.cant_decay, self.cant_transf,
                         self.efficiency, self.total_time,
                         self.l_d_rms, self.walk_mean])

<|endoftext|>


<commit_before>__version__ = "0.9.2"
<commit_msg>0.9.3<commit_after>__version__ = "0.9.3"
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
import logging

from webassets import Environment, Bundle

from website import settings

logger = logging.getLogger(__name__)

env = Environment(settings.STATIC_FOLDER, settings.STATIC_URL_PATH)

css = Bundle(
        # Vendorized libraries
         Bundle(
            "vendor/jquery-ui/css/jquery-ui.css",
            "vendor/jquery-tagit/css/jquery.tagit.css",
            "vendor/jquery-tagsinput/css/jquery.tagsinput.css",
            "vendor/jquery-tagit/css/tagit.ui-zendesk.css",
            "vendor/jquery-treeview/jquery.treeview.css",
            "vendor/jquery-fileupload/css/jquery.fileupload-ui.css",
            "vendor/pygments.css",
            "vendor/bootstrap3-editable/css/bootstrap-editable.css",
            "vendor/bootstrap3/css/bootstrap-theme.css",
            filters="cssmin"),
        # Site-specific CSS
        Bundle(
            "css/site.css",
            filters="cssmin"),
        output="public/css/common.css"
)

js = Bundle(
        # Vendorized libraries that are already minified
        Bundle(
                "vendor/jquery/jquery.min.js",
                "vendor/jquery-ui/js/jquery-ui.min.js",
                "vendor/bootstrap3/js/bootstrap.min.js",
                "vendor/bootstrap3-editable/js/bootstrap-editable.min.js",
                "vendor/bootbox/bootbox.min.js",
                "vendor/jquery-tagsinput/js/jquery.tagsinput.min.js",
                "vendor/jquery-tagcloud/jquery.tagcloud.js",
                "vendor/jquery-treeview/jquery.treeview.js",
                "vendor/jquery-tagit/js/tag-it.js",
                "vendor/knockout/knockout-min.js",
                "vendor/moment/moment.min.js",
                "vendor/jquery-blockui/jquery.blockui.js"),
        # Site-specific JS
        Bundle("js/site.js", "js/project.js", "js/app.js", filters="jsmin"),
        output="public/js/common.js"
)

logger.debug("Registering asset bundles")
env.register("js", js)
env.register("css", css)
# Don't bundle in debug mode
env.debug = settings.DEBUG_MODE
<commit_msg>remove treeview references<commit_after># -*- coding: utf-8 -*-
import logging

from webassets import Environment, Bundle

from website import settings

logger = logging.getLogger(__name__)

env = Environment(settings.STATIC_FOLDER, settings.STATIC_URL_PATH)

css = Bundle(
        # Vendorized libraries
         Bundle(
            "vendor/jquery-ui/css/jquery-ui.css",
            "vendor/jquery-tagit/css/jquery.tagit.css",
            "vendor/jquery-tagsinput/css/jquery.tagsinput.css",
            "vendor/jquery-tagit/css/tagit.ui-zendesk.css",
            "vendor/jquery-fileupload/css/jquery.fileupload-ui.css",
            "vendor/pygments.css",
            "vendor/bootstrap3-editable/css/bootstrap-editable.css",
            "vendor/bootstrap3/css/bootstrap-theme.css",
            filters="cssmin"),
        # Site-specific CSS
        Bundle(
            "css/site.css",
            filters="cssmin"),
        output="public/css/common.css"
)

js = Bundle(
        # Vendorized libraries that are already minified
        Bundle(
                "vendor/jquery/jquery.min.js",
                "vendor/jquery-ui/js/jquery-ui.min.js",
                "vendor/bootstrap3/js/bootstrap.min.js",
                "vendor/bootstrap3-editable/js/bootstrap-editable.min.js",
                "vendor/bootbox/bootbox.min.js",
                "vendor/jquery-tagsinput/js/jquery.tagsinput.min.js",
                "vendor/jquery-tagcloud/jquery.tagcloud.js",
                "vendor/jquery-tagit/js/tag-it.js",
                "vendor/knockout/knockout-min.js",
                "vendor/moment/moment.min.js",
                "vendor/jquery-blockui/jquery.blockui.js"),
        # Site-specific JS
        Bundle("js/site.js", "js/project.js", "js/app.js", filters="jsmin"),
        output="public/js/common.js"
)

logger.debug("Registering asset bundles")
env.register("js", js)
env.register("css", css)
# Don't bundle in debug mode
env.debug = settings.DEBUG_MODE
<|endoftext|>