<commit_before># Copyright 2019 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
"""NCF model input pipeline."""

from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

import functools

# pylint: disable=g-bad-import-order
import tensorflow.compat.v2 as tf
# pylint: enable=g-bad-import-order

from official.recommendation import constants as rconst
from official.recommendation import movielens
from official.recommendation import data_pipeline

NUM_SHARDS = 16

def create_dataset_from_tf_record_files(input_file_pattern,
                                        pre_batch_size,
                                        batch_size,
                                        is_training=True):
  """Creates dataset from (tf)records files for training/evaluation."""

  files = tf.data.Dataset.list_files(input_file_pattern, shuffle=is_training)

  def make_dataset(files_dataset, shard_index):
    """Returns dataset for sharded tf record files."""
    if pre_batch_size != batch_size:
      raise ValueError("Pre-batch ({}) size is not equal to batch "
                       "size ({})".format(pre_batch_size, batch_size))
    files_dataset = files_dataset.shard(NUM_SHARDS, shard_index)
    dataset = files_dataset.interleave(tf.data.TFRecordDataset)
    decode_fn = functools.partial(
        data_pipeline.DatasetManager.deserialize,
        batch_size=pre_batch_size,
        is_training=is_training)
    dataset = dataset.map(
        decode_fn, num_parallel_calls=tf.data.experimental.AUTOTUNE)
    return dataset

  dataset = tf.data.Dataset.range(NUM_SHARDS)
  map_fn = functools.partial(make_dataset, files)
  dataset = dataset.interleave(
      map_fn,
      cycle_length=NUM_SHARDS,
      num_parallel_calls=tf.data.experimental.AUTOTUNE)
  dataset = dataset.prefetch(tf.data.experimental.AUTOTUNE)
  return dataset

def create_dataset_from_data_producer(producer, params):
  """Return dataset online-generating data."""

  def preprocess_train_input(features, labels):
    """Pre-process the training data.

    This is needed because
    - The label needs to be extended to be used in the loss fn
    - We need the same inputs for training and eval so adding fake inputs
      for DUPLICATE_MASK in training data.

    Args:
      features: Dictionary of features for training.
      labels: Training labels.

    Returns:
      Processed training features.
    """
    fake_dup_mask = tf.zeros_like(features[movielens.USER_COLUMN])
    features[rconst.DUPLICATE_MASK] = fake_dup_mask
    features[rconst.TRAIN_LABEL_KEY] = labels
    return features

  train_input_fn = producer.make_input_fn(is_training=True)
  train_input_dataset = train_input_fn(params).map(preprocess_train_input)

  def preprocess_eval_input(features):
    """Pre-process the eval data.

    This is needed because:
    - The label needs to be extended to be used in the loss fn
    - We need the same inputs for training and eval so adding fake inputs
      for VALID_PT_MASK in eval data.

    Args:
      features: Dictionary of features for evaluation.

    Returns:
      Processed evaluation features.
    """
    labels = tf.cast(tf.zeros_like(features[movielens.USER_COLUMN]), tf.bool)
    fake_valid_pt_mask = tf.cast(
        tf.zeros_like(features[movielens.USER_COLUMN]), tf.bool)
    features[rconst.VALID_POINT_MASK] = fake_valid_pt_mask
    features[rconst.TRAIN_LABEL_KEY] = labels
    return features

  eval_input_fn = producer.make_input_fn(is_training=False)
  eval_input_dataset = eval_input_fn(params).map(preprocess_eval_input)

  return train_input_dataset, eval_input_dataset

def create_ncf_input_data(params,
                          producer=None,
                          input_meta_data=None,
                          strategy=None):
  """Creates NCF training/evaluation dataset.

  Args:
    params: Dictionary containing parameters for train/evaluation data.
    producer: Instance of BaseDataConstructor that generates data online. Must
      not be None when params['train_dataset_path'] or
      params['eval_dataset_path'] is not specified.
    input_meta_data: A dictionary of input metadata to be used when reading data
      from tf record files. Must be specified when params["train_input_dataset"]
      is specified.
    strategy: Distribution strategy used for distributed training. If specified,
      used to assert that evaluation batch size is correctly a multiple of
      total number of devices used.

  Returns:
    (training dataset, evaluation dataset, train steps per epoch,
    eval steps per epoch)

  Raises:
    ValueError: If data is being generated online for when using TPU's.
  """
  # NCF evaluation metric calculation logic assumes that evaluation data
  # sample size are in multiples of (1 + number of negative samples in
  # evaluation) for each device. As so, evaluation batch size must be a
  # multiple of (number of replicas * (1 + number of negative samples)).
  num_devices = strategy.num_replicas_in_sync if strategy else 1
  if (params["eval_batch_size"] % (num_devices *
                                   (1 + rconst.NUM_EVAL_NEGATIVES))):
    raise ValueError("Evaluation batch size must be divisible by {} "
                     "times {}".format(num_devices,
                                       (1 + rconst.NUM_EVAL_NEGATIVES)))

  if params["train_dataset_path"]:
    assert params["eval_dataset_path"]

    train_dataset = create_dataset_from_tf_record_files(
        params["train_dataset_path"],
        input_meta_data["train_prebatch_size"],
        params["batch_size"],
        is_training=True)
    eval_dataset = create_dataset_from_tf_record_files(
        params["eval_dataset_path"],
        input_meta_data["eval_prebatch_size"],
        params["eval_batch_size"],
        is_training=False)

    num_train_steps = int(input_meta_data["num_train_steps"])
    num_eval_steps = int(input_meta_data["num_eval_steps"])
  else:
    if params["use_tpu"]:
      raise ValueError("TPU training does not support data producer yet. "
                       "Use pre-processed data.")

    assert producer
    # Start retrieving data from producer.
    train_dataset, eval_dataset = create_dataset_from_data_producer(
        producer, params)
    num_train_steps = producer.train_batches_per_epoch
    num_eval_steps = producer.eval_batches_per_epoch

  return train_dataset, eval_dataset, num_train_steps, num_eval_steps
<commit_msg>Internal change<commit_after># Copyright 2019 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
"""NCF model input pipeline."""

from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

import functools

# pylint: disable=g-bad-import-order
import tensorflow.compat.v2 as tf
# pylint: enable=g-bad-import-order

from official.recommendation import constants as rconst
from official.recommendation import movielens
from official.recommendation import data_pipeline

NUM_SHARDS = 16

def create_dataset_from_tf_record_files(input_file_pattern,
                                        pre_batch_size,
                                        batch_size,
                                        is_training=True):
  """Creates dataset from (tf)records files for training/evaluation."""

  files = tf.data.Dataset.list_files(input_file_pattern, shuffle=is_training)

  def make_dataset(files_dataset, shard_index):
    """Returns dataset for sharded tf record files."""
    if pre_batch_size != batch_size:
      raise ValueError("Pre-batch ({}) size is not equal to batch "
                       "size ({})".format(pre_batch_size, batch_size))
    files_dataset = files_dataset.shard(NUM_SHARDS, shard_index)
    dataset = files_dataset.interleave(
        tf.data.TFRecordDataset,
        num_parallel_calls=tf.data.experimental.AUTOTUNE)
    decode_fn = functools.partial(
        data_pipeline.DatasetManager.deserialize,
        batch_size=pre_batch_size,
        is_training=is_training)
    dataset = dataset.map(
        decode_fn, num_parallel_calls=tf.data.experimental.AUTOTUNE)
    return dataset

  dataset = tf.data.Dataset.range(NUM_SHARDS)
  map_fn = functools.partial(make_dataset, files)
  dataset = dataset.interleave(
      map_fn,
      cycle_length=NUM_SHARDS,
      num_parallel_calls=tf.data.experimental.AUTOTUNE)
  dataset = dataset.prefetch(tf.data.experimental.AUTOTUNE)
  return dataset

def create_dataset_from_data_producer(producer, params):
  """Return dataset online-generating data."""

  def preprocess_train_input(features, labels):
    """Pre-process the training data.

    This is needed because
    - The label needs to be extended to be used in the loss fn
    - We need the same inputs for training and eval so adding fake inputs
      for DUPLICATE_MASK in training data.

    Args:
      features: Dictionary of features for training.
      labels: Training labels.

    Returns:
      Processed training features.
    """
    fake_dup_mask = tf.zeros_like(features[movielens.USER_COLUMN])
    features[rconst.DUPLICATE_MASK] = fake_dup_mask
    features[rconst.TRAIN_LABEL_KEY] = labels
    return features

  train_input_fn = producer.make_input_fn(is_training=True)
  train_input_dataset = train_input_fn(params).map(preprocess_train_input)

  def preprocess_eval_input(features):
    """Pre-process the eval data.

    This is needed because:
    - The label needs to be extended to be used in the loss fn
    - We need the same inputs for training and eval so adding fake inputs
      for VALID_PT_MASK in eval data.

    Args:
      features: Dictionary of features for evaluation.

    Returns:
      Processed evaluation features.
    """
    labels = tf.cast(tf.zeros_like(features[movielens.USER_COLUMN]), tf.bool)
    fake_valid_pt_mask = tf.cast(
        tf.zeros_like(features[movielens.USER_COLUMN]), tf.bool)
    features[rconst.VALID_POINT_MASK] = fake_valid_pt_mask
    features[rconst.TRAIN_LABEL_KEY] = labels
    return features

  eval_input_fn = producer.make_input_fn(is_training=False)
  eval_input_dataset = eval_input_fn(params).map(preprocess_eval_input)

  return train_input_dataset, eval_input_dataset

def create_ncf_input_data(params,
                          producer=None,
                          input_meta_data=None,
                          strategy=None):
  """Creates NCF training/evaluation dataset.

  Args:
    params: Dictionary containing parameters for train/evaluation data.
    producer: Instance of BaseDataConstructor that generates data online. Must
      not be None when params['train_dataset_path'] or
      params['eval_dataset_path'] is not specified.
    input_meta_data: A dictionary of input metadata to be used when reading data
      from tf record files. Must be specified when params["train_input_dataset"]
      is specified.
    strategy: Distribution strategy used for distributed training. If specified,
      used to assert that evaluation batch size is correctly a multiple of
      total number of devices used.

  Returns:
    (training dataset, evaluation dataset, train steps per epoch,
    eval steps per epoch)

  Raises:
    ValueError: If data is being generated online for when using TPU's.
  """
  # NCF evaluation metric calculation logic assumes that evaluation data
  # sample size are in multiples of (1 + number of negative samples in
  # evaluation) for each device. As so, evaluation batch size must be a
  # multiple of (number of replicas * (1 + number of negative samples)).
  num_devices = strategy.num_replicas_in_sync if strategy else 1
  if (params["eval_batch_size"] % (num_devices *
                                   (1 + rconst.NUM_EVAL_NEGATIVES))):
    raise ValueError("Evaluation batch size must be divisible by {} "
                     "times {}".format(num_devices,
                                       (1 + rconst.NUM_EVAL_NEGATIVES)))

  if params["train_dataset_path"]:
    assert params["eval_dataset_path"]

    train_dataset = create_dataset_from_tf_record_files(
        params["train_dataset_path"],
        input_meta_data["train_prebatch_size"],
        params["batch_size"],
        is_training=True)
    eval_dataset = create_dataset_from_tf_record_files(
        params["eval_dataset_path"],
        input_meta_data["eval_prebatch_size"],
        params["eval_batch_size"],
        is_training=False)

    num_train_steps = int(input_meta_data["num_train_steps"])
    num_eval_steps = int(input_meta_data["num_eval_steps"])
  else:
    if params["use_tpu"]:
      raise ValueError("TPU training does not support data producer yet. "
                       "Use pre-processed data.")

    assert producer
    # Start retrieving data from producer.
    train_dataset, eval_dataset = create_dataset_from_data_producer(
        producer, params)
    num_train_steps = producer.train_batches_per_epoch
    num_eval_steps = producer.eval_batches_per_epoch

  return train_dataset, eval_dataset, num_train_steps, num_eval_steps
<|endoftext|>


<commit_before>#!/usr/bin/python
#
# Copyright (c) 2006 rPath, Inc.
#
# This program is distributed under the terms of the Common Public License,
# version 1.0. A copy of this license should have been distributed with this
# source file in a file called LICENSE. If it is not present, the license
# is always available at http://www.opensource.org/licenses/cpl.php.
#
# This program is distributed in the hope that it will be useful, but
# without any waranty; without even the implied warranty of merchantability
# or fitness for a particular purpose. See the Common Public License for
# full details.
#

import cmd
import sys
from conary import dbstore
import sqlerrors

class DbSql(cmd.Cmd):
    def __init__(self, driver=None, path=None, db=None):
        cmd.Cmd.__init__(self)
        self.prompt = 'dbsql> '
        self.doc_header = "Documented commands (type .help <topic>):"
        self.intro = 'dbstore sql shell.  type ".quit" to exit'
        if driver and path:
            self.db = dbstore.connect(path, driver=driver)
        elif db:
            self.db = db
        else:
            raise RuntimeError, 'driver and path OR db must be given'
        self.cu = self.db.cursor()

    def onecmd(self, cmd):
        # remove any trailing whitespace
        cmd = cmd.strip()

        if cmd.startswith('.'):
            cmd = cmd[1:]
            cmd, arg, line = self.parseline(cmd)
            try:
                func = getattr(self, 'do_' + cmd)
            except AttributeError:
                return self.default(line)
            return func(arg)

        if not cmd:
            return
        if ';' in cmd:
            onecmd, rest = cmd.split(';')
            self.onecmd(onecmd)
            self.onecmd(rest)
            return
        try:
            self.cu.execute(cmd)
        except sqlerrors.DatabaseError, e:
            print 'Error:', str(e.args[0])
        fields = None
        for row in self.cu:
            if not fields:
                fields = self.cu.fields()
                print '|'.join(fields)
            print '|'.join(repr(x) for x in row)

    def do_quit(self, arg):
        sys.exit()

    def help_quit(self):
        print """quit
quit the shell"""

    def help_help(self):
        print """quit
display help"""
<commit_msg>1) add command line history 2) quit by having do_quit() return True, which will let you go    from epdb to dbsql and back again 3) add a shell() method to make debugging easier:    from conary.dbstore import dbsql    dbsql.shell(db)<commit_after>#!/usr/bin/python
#
# Copyright (c) 2006 rPath, Inc.
#
# This program is distributed under the terms of the Common Public License,
# version 1.0. A copy of this license should have been distributed with this
# source file in a file called LICENSE. If it is not present, the license
# is always available at http://www.opensource.org/licenses/cpl.php.
#
# This program is distributed in the hope that it will be useful, but
# without any waranty; without even the implied warranty of merchantability
# or fitness for a particular purpose. See the Common Public License for
# full details.
#

import cmd
import sys
from conary import dbstore
import sqlerrors
import os

try:
    import readline
except ImportError:
    hasReadline = False
else:
    hasReadline = True

class DbSql(cmd.Cmd):
    _historyPath = os.path.expanduser('~/.dbsqlhistory')

    def __init__(self, db = None, driver = None, path = None):
        cmd.Cmd.__init__(self)
        self.prompt = 'dbsql> '
        self.doc_header = "Documented commands (type .help <topic>):"
        self.intro = 'dbstore sql shell.  type ".quit" to exit'
        if driver and path:
            self.db = dbstore.connect(path, driver=driver)
        elif db:
            self.db = db
        else:
            raise RuntimeError, 'driver and path OR db must be given'
        self.cu = self.db.cursor()

    def onecmd(self, cmd):
        # remove any trailing whitespace
        cmd = cmd.strip()

        if cmd.startswith('.'):
            cmd = cmd[1:]
            cmd, arg, line = self.parseline(cmd)
            try:
                func = getattr(self, 'do_' + cmd)
            except AttributeError:
                return self.default(line)
            return func(arg)

        if not cmd:
            return
        if ';' in cmd:
            onecmd, rest = cmd.split(';')
            self.onecmd(onecmd)
            self.onecmd(rest)
            return
        try:
            self.cu.execute(cmd)
        except sqlerrors.DatabaseError, e:
            print 'Error:', str(e.args[0])
        fields = None
        for row in self.cu:
            if not fields:
                fields = self.cu.fields()
                print '|'.join(fields)
            print '|'.join(repr(x) for x in row)

    def cmdloop(self):
        self.read_history()
        rc = cmd.Cmd.cmdloop(self)
        self.save_history()
        return rc

    def read_history(self):
        if hasReadline and self._historyPath:
            try:
                readline.read_history_file(self._historyPath)
            except:
                pass

    def save_history(self):
        if hasReadline and self._historyPath:
            readline.set_history_length(1000)
            try:
                readline.write_history_file(self._historyPath)
            except:
                pass

    def do_quit(self, arg):
        # ask to stop
        return True

    def help_quit(self):
        print """quit
quit the shell"""

    def help_help(self):
        print """quit
display help"""

def shell(db):
    'invokes a dbstore sql shell on an existing db connection'
    shell = DbSql(db)
    return shell.cmdloop()
<|endoftext|>


<commit_before>#!/usr/bin/python2.4
# -*- mode: python -*-
#
# Copyright (c) 2004-2006 rPath, Inc.
#
# This program is distributed under the terms of the Common Public License,
# version 1.0. A copy of this license should have been distributed with this
# source file in a file called LICENSE. If it is not present, the license
# is always available at http://www.opensource.org/licenses/cpl.php.
#
# This program is distributed in the hope that it will be useful, but
# without any waranty; without even the implied warranty of merchantability
# or fitness for a particular purpose. See the Common Public License for
# full details.
#

import base64
import cgi
import errno
import os
import posixpath
import select
import sys
import tempfile
import traceback
import xmlrpclib
import urllib
import zlib
from BaseHTTPServer import HTTPServer
from SimpleHTTPServer import SimpleHTTPRequestHandler

thisFile = sys.modules[__name__].__file__
thisPath = os.path.dirname(thisFile)
if thisPath:
    mainPath = thisPath + "/../.."
else:
    mainPath = "../.."
mainPath = os.path.realpath(mainPath)
sys.path.insert(0, mainPath)

from conary import dbstore
from conary.conarycfg import CfgRepoMap
from conary.lib import options
from conary.lib import util
from conary.lib.cfg import ConfigFile,CfgPath,CfgInt,CfgBool
from conary.lib.tracelog import initLog, logMe
from conary.repository import changeset
from conary.repository import errors
from conary.repository.filecontainer import FileContainer
from conary.repository.netrepos import netauth
from conary.repository.netrepos import netserver
from conary.repository.netrepos.netserver import NetworkRepositoryServer
from conary.server import schema

class HttpRequests(SimpleHTTPRequestHandler):

    outFiles = {}
    inFiles = {}

    tmpDir = None

    def translate_path(self, path):
        """Translate a /-separated PATH to the local filename syntax.

        Components that mean special things to the local file system
        (e.g. drive or directory names) are ignored.  (XXX They should
        probably be diagnosed.)

        """
        path = posixpath.normpath(urllib.unquote(path))
	path = path.split("?", 1)[1]
        words = path.split('/')
        words = filter(None, words)
        path = self.tmpDir
        for word in words:
            drive, word = os.path.splitdrive(word)
            head, word = os.path.split(word)
            if word in (os.curdir, os.pardir): continue
            path = os.path.join(path, word)

	path += "-out"

	self.cleanup = path
        return path

    def do_GET(self):
        def _writeNestedFile(outF, name, tag, size, f, sizeCb):
            if changeset.ChangedFileTypes.refr[4:] == tag[2:]:
                path = f.read()
                size = os.stat(path).st_size
                f = open(path)
                tag = tag[0:2] + changeset.ChangedFileTypes.file[4:]

            sizeCb(size, tag)
            bytes = util.copyfileobj(f, outF)

        if self.path.endswith('/'):
            self.path = self.path[:-1]
        base = os.path.basename(self.path)
        if "?" in base:
            base, queryString = base.split("?")
        else:
            queryString = ""

        if base == 'changeset':
            urlPath = posixpath.normpath(urllib.unquote(self.path))
            localName = self.tmpDir + "/" + urlPath.split('?', 1)[1] + "-out"
            if os.path.realpath(localName) != localName:
                self.send_error(403, "File not found")
                return None

            if localName.endswith(".cf-out"):
                try:
                    f = open(localName, "r")
                except IOError:
                    self.send_error(404, "File not found")
                    return None

                os.unlink(localName)

                items = []
                totalSize = 0
                for l in f.readlines():
                    (path, size) = l.split()
                    size = int(size)
                    totalSize += size
                    items.append((path, size))
                f.close()
                del f
            else:
                size = os.stat(localName).st_size;
                items = [ (localName, size) ]
                totalSize = size

            self.send_response(200)
            self.send_header("Content-type", "application/octet-stream")
            self.send_header("Content-Length", str(totalSize))
            self.end_headers()

            for path, size in items:
                if path.endswith('.ccs-out'):
                    cs = FileContainer(open(path))
                    cs.dump(self.wfile.write,
                            lambda name, tag, size, f, sizeCb:
                                _writeNestedFile(self.wfile, name, tag, size, f,
                                                 sizeCb))

                    del cs
                else:
                    f = open(path)
                    util.copyfileobj(f, self.wfile)

                if path.startswith(self.tmpDir):
                    os.unlink(path)
        else:
            self.send_error(501, "Not Implemented")

    def do_POST(self):
        if self.headers.get('Content-Type', '') == 'text/xml':
            authToken = self.getAuth()
            if authToken is None:
                return

            return self.handleXml(authToken)
        else:
            self.send_error(501, "Not Implemented")

    def getAuth(self):
        info = self.headers.get('Authorization', None)
        if info is None:
            httpAuthToken = [ 'anonymous', 'anonymous' ]
        else:
            info = info.split()

            try:
                authString = base64.decodestring(info[1])
            except:
                self.send_error(400)
                return None

            if authString.count(":") != 1:
                self.send_error(400)
                return None

            httpAuthToken = authString.split(":")

        entitlement = self.headers.get('X-Conary-Entitlement', None)
        if entitlement is not None:
            try:
                entitlement = entitlement.split()
                entitlement[1] = base64.decodestring(entitlement[1])
            except:
                self.send_error(400)
                return None
        else:
            entitlement = [ None, None ]

        return httpAuthToken + entitlement

    def checkAuth(self):
 	if not self.headers.has_key('Authorization'):
            self.requestAuth()
            return None
	else:
            authToken = self.getAuth()
            if authToken is None:
                return

            # verify that the user/password actually exists in the database
            if not netRepos.auth.checkUserPass(authToken):
                self.send_error(403)
                return None

	return authToken

    def requestAuth(self):
        self.send_response(401)
        self.send_header("WWW-Authenticate", 'Basic realm="Conary Repository"')
        self.end_headers()
        return None

    def handleXml(self, authToken):
	contentLength = int(self.headers['Content-Length'])
        data = self.rfile.read(contentLength)

        encoding = self.headers.get('Content-Encoding', None)
        if encoding == 'deflate':
            data = zlib.decompress(data)

        (params, method) = xmlrpclib.loads(data)
        logMe(3, "decoded xml-rpc call %s from %d bytes request" %(method, contentLength))

	try:
	    result = netRepos.callWrapper(None, None, method, authToken, params)
	except errors.InsufficientPermission:
	    self.send_error(403)
	    return None
        logMe(3, "returned from", method)

        usedAnonymous = result[0]
        result = result[1:]

	resp = xmlrpclib.dumps((result,), methodresponse=1)
        logMe(3, "encoded xml-rpc response to %d bytes" % (len(resp),))

	self.send_response(200)
        encoding = self.headers.get('Accept-encoding', '')
        if len(resp) > 200 and 'deflate' in encoding:
            resp = zlib.compress(resp, 5)
            self.send_header('Content-encoding', 'deflate')
	self.send_header("Content-type", "text/xml")
	self.send_header("Content-length", str(len(resp)))
        if usedAnonymous:
            self.send_header("X-Conary-UsedAnonymous", '1')

	self.end_headers()
	self.wfile.write(resp)
        logMe(3, "sent response to client", len(resp), "bytes")
	return resp

    def do_PUT(self):
	path = self.path.split("?")[-1]

        if '/' in path:
	    self.send_error(403, "Forbidden")

	path = self.tmpDir + '/' + path + "-in"

	size = os.stat(path).st_size
	if size != 0:
	    self.send_error(410, "Gone")
	    return

	out = open(path, "w")

	contentLength = int(self.headers['Content-Length'])
	while contentLength:
	    s = self.rfile.read(contentLength)
	    contentLength -= len(s)
	    out.write(s)

	self.send_response(200, 'OK')

class ResetableNetworkRepositoryServer(NetworkRepositoryServer):
    def reset(self, authToken, clientVersion):
        import shutil
        try:
            shutil.rmtree(self.contentsDir)
        except OSError, e:
            if e.errno != errno.ENOENT:
                raise
        os.mkdir(self.contentsDir)

        logMe(1, "resetting NetworkRepositoryServer", self.repDB)
        # cheap trick. sqlite3 doesn't mind zero byte files; just replace
        # the file with a zero byte one (to change the inode) and reopen
        open(self.repDB[1] + '.new', "w")
        os.rename(self.repDB[1] + '.new', self.repDB[1])
        self.reopen()

        return 0

class ServerConfig(netserver.ServerConfig):

    port		= (CfgInt,  8000)

    def __init__(self, path="serverrc"):
	netserver.ServerConfig.__init__(self)
	self.read(path, exception=False)

    def check(self):
        if self.cacheDB:
            print >> sys.stderr, "warning: cacheDB config option is ignored "
                                 "by the standalone server"

        if self.closed:
            print >> sys.stderr, "warning: closed config option is ignored "
                                 "by the standalone server"

        if self.forceSSL:
            print >> sys.stderr, "warning: commitAction config option is "
                                 "ignored by the standalone server"

def usage():
    print "usage: %s" % sys.argv[0]
    print "       %s --add-user [--admin] [--mirror] <username>" % sys.argv[0]
    print ""
    print "server flags: --config-file <path>"
    print '              --db "driver <path>"'
    print '              --log-file <path>'
    print '              --map "<from> <to>"'
    print "              --server-name <host>"
    print "              --tmp-dir <path>"
    sys.exit(1)

def addUser(netRepos, userName, admin = False, mirror = False):
    if os.isatty(0):
        from getpass import getpass

        pw1 = getpass('Password:')
        pw2 = getpass('Reenter password:')

        if pw1 != pw2:
            print "Passwords do not match."
            return 1
    else:
        # chop off the trailing newline
        pw1 = sys.stdin.readline()[:-1]

    # never give anonymous write access by default
    write = userName != 'anonymous'
    netRepos.auth.addUser(userName, pw1)
    # user/group, trovePattern, label, write, capped, admin
    netRepos.auth.addAcl(userName, None, None, write, False, admin)
    netRepos.auth.setMirror(userName, mirror)

if __name__ == '__main__':
    argDef = {}
    cfgMap = {
        'contents-dir'  : 'contentsDir',
	'db'	        : 'repositoryDB',
	'log-file'	: 'logFile',
	'map'	        : 'repositoryMap',
	'port'	        : 'port',
	'tmp-dir'       : 'tmpDir',
        'require-sigs'  : 'requireSigs',
        'server-name'   : 'serverName'
    }

    cfg = ServerConfig()

    argDef["config"] = options.MULT_PARAM
    # magically handled by processArgs
    argDef["config-file"] = options.ONE_PARAM

    argDef['add-user'] = options.ONE_PARAM
    argDef['admin'] = options.NO_PARAM
    argDef['help'] = options.NO_PARAM
    argDef['migrate'] = options.NO_PARAM
    argDef['mirror'] = options.NO_PARAM

    try:
        argSet, otherArgs = options.processArgs(argDef, cfgMap, cfg, usage)
    except options.OptionError, msg:
        print >> sys.stderr, msg
        sys.exit(1)

    if 'migrate' not in argSet:
        cfg.check()

    if argSet.has_key('help'):
        usage()

    if not os.path.isdir(cfg.tmpDir):
	print cfg.tmpDir + " needs to be a directory"
	sys.exit(1)
    if not os.access(cfg.tmpDir, os.R_OK | os.W_OK | os.X_OK):
        print cfg.tmpDir + " needs to allow full read/write access"
        sys.exit(1)
    HttpRequests.tmpDir = cfg.tmpDir

    profile = 0
    if profile:
        import hotshot
        prof = hotshot.Profile('server.prof')
        prof.start()

    baseUrl="http://%s:%s/" % (os.uname()[1], cfg.port)

    # start the logging
    if 'add-user' not in argSet:
        initLog(level=3, trace=1)

    if not cfg.repositoryDB:
        cfg.repositoryDB = ("sqlite",  "%s/sqldb" % otherArgs[1])
        del otherArgs[1]

    if len(otherArgs) > 1:
	usage()

    if not cfg.contentsDir:
        assert(cfg.repositoryDB[0] == "sqlite")
        cfg.contentsDir = os.path.dirname(cfg.repositoryDB[1]) + '/contents'

    if cfg.repositoryDB[0] == 'sqlite':
        util.mkdirChain(os.path.dirname(cfg.repositoryDB[1]))

    (driver, database) = cfg.repositoryDB
    db= dbstore.connect(database, driver)
    schema.loadSchema(db)

    if 'migrate' in argSet:
        sys.exit(0)

    netRepos = NetworkRepositoryServer(cfg, baseUrl)

    if 'add-user' in argSet:
        admin = argSet.pop('admin', False)
        mirror = argSet.pop('mirror', False)
        userName = argSet.pop('add-user')
        if argSet:
            usage()
        sys.exit(addUser(netRepos, userName, admin = admin, mirror = mirror))

    if argSet:
        usage()

    httpServer = HTTPServer(("", cfg.port), HttpRequests)

    fds = {}
    fds[httpServer.fileno()] = httpServer

    p = select.poll()
    for fd in fds.iterkeys():
        p.register(fd, select.POLLIN)

    logMe(1, "Server ready for requests")

    while True:
        try:
            events = p.poll()
            for (fd, event) in events:
                fds[fd].handle_request()
        except select.error:
            pass
        except:
            if profile:
                prof.stop()
                print "exception happened, exiting"
                sys.exit(1)
            else:
                raise
<commit_msg>fix quoting on error messages<commit_after>#!/usr/bin/python2.4
# -*- mode: python -*-
#
# Copyright (c) 2004-2006 rPath, Inc.
#
# This program is distributed under the terms of the Common Public License,
# version 1.0. A copy of this license should have been distributed with this
# source file in a file called LICENSE. If it is not present, the license
# is always available at http://www.opensource.org/licenses/cpl.php.
#
# This program is distributed in the hope that it will be useful, but
# without any waranty; without even the implied warranty of merchantability
# or fitness for a particular purpose. See the Common Public License for
# full details.
#

import base64
import cgi
import errno
import os
import posixpath
import select
import sys
import tempfile
import traceback
import xmlrpclib
import urllib
import zlib
from BaseHTTPServer import HTTPServer
from SimpleHTTPServer import SimpleHTTPRequestHandler

thisFile = sys.modules[__name__].__file__
thisPath = os.path.dirname(thisFile)
if thisPath:
    mainPath = thisPath + "/../.."
else:
    mainPath = "../.."
mainPath = os.path.realpath(mainPath)
sys.path.insert(0, mainPath)

from conary import dbstore
from conary.conarycfg import CfgRepoMap
from conary.lib import options
from conary.lib import util
from conary.lib.cfg import ConfigFile,CfgPath,CfgInt,CfgBool
from conary.lib.tracelog import initLog, logMe
from conary.repository import changeset
from conary.repository import errors
from conary.repository.filecontainer import FileContainer
from conary.repository.netrepos import netauth
from conary.repository.netrepos import netserver
from conary.repository.netrepos.netserver import NetworkRepositoryServer
from conary.server import schema

class HttpRequests(SimpleHTTPRequestHandler):

    outFiles = {}
    inFiles = {}

    tmpDir = None

    def translate_path(self, path):
        """Translate a /-separated PATH to the local filename syntax.

        Components that mean special things to the local file system
        (e.g. drive or directory names) are ignored.  (XXX They should
        probably be diagnosed.)

        """
        path = posixpath.normpath(urllib.unquote(path))
	path = path.split("?", 1)[1]
        words = path.split('/')
        words = filter(None, words)
        path = self.tmpDir
        for word in words:
            drive, word = os.path.splitdrive(word)
            head, word = os.path.split(word)
            if word in (os.curdir, os.pardir): continue
            path = os.path.join(path, word)

	path += "-out"

	self.cleanup = path
        return path

    def do_GET(self):
        def _writeNestedFile(outF, name, tag, size, f, sizeCb):
            if changeset.ChangedFileTypes.refr[4:] == tag[2:]:
                path = f.read()
                size = os.stat(path).st_size
                f = open(path)
                tag = tag[0:2] + changeset.ChangedFileTypes.file[4:]

            sizeCb(size, tag)
            bytes = util.copyfileobj(f, outF)

        if self.path.endswith('/'):
            self.path = self.path[:-1]
        base = os.path.basename(self.path)
        if "?" in base:
            base, queryString = base.split("?")
        else:
            queryString = ""

        if base == 'changeset':
            urlPath = posixpath.normpath(urllib.unquote(self.path))
            localName = self.tmpDir + "/" + urlPath.split('?', 1)[1] + "-out"
            if os.path.realpath(localName) != localName:
                self.send_error(403, "File not found")
                return None

            if localName.endswith(".cf-out"):
                try:
                    f = open(localName, "r")
                except IOError:
                    self.send_error(404, "File not found")
                    return None

                os.unlink(localName)

                items = []
                totalSize = 0
                for l in f.readlines():
                    (path, size) = l.split()
                    size = int(size)
                    totalSize += size
                    items.append((path, size))
                f.close()
                del f
            else:
                size = os.stat(localName).st_size;
                items = [ (localName, size) ]
                totalSize = size

            self.send_response(200)
            self.send_header("Content-type", "application/octet-stream")
            self.send_header("Content-Length", str(totalSize))
            self.end_headers()

            for path, size in items:
                if path.endswith('.ccs-out'):
                    cs = FileContainer(open(path))
                    cs.dump(self.wfile.write,
                            lambda name, tag, size, f, sizeCb:
                                _writeNestedFile(self.wfile, name, tag, size, f,
                                                 sizeCb))

                    del cs
                else:
                    f = open(path)
                    util.copyfileobj(f, self.wfile)

                if path.startswith(self.tmpDir):
                    os.unlink(path)
        else:
            self.send_error(501, "Not Implemented")

    def do_POST(self):
        if self.headers.get('Content-Type', '') == 'text/xml':
            authToken = self.getAuth()
            if authToken is None:
                return

            return self.handleXml(authToken)
        else:
            self.send_error(501, "Not Implemented")

    def getAuth(self):
        info = self.headers.get('Authorization', None)
        if info is None:
            httpAuthToken = [ 'anonymous', 'anonymous' ]
        else:
            info = info.split()

            try:
                authString = base64.decodestring(info[1])
            except:
                self.send_error(400)
                return None

            if authString.count(":") != 1:
                self.send_error(400)
                return None

            httpAuthToken = authString.split(":")

        entitlement = self.headers.get('X-Conary-Entitlement', None)
        if entitlement is not None:
            try:
                entitlement = entitlement.split()
                entitlement[1] = base64.decodestring(entitlement[1])
            except:
                self.send_error(400)
                return None
        else:
            entitlement = [ None, None ]

        return httpAuthToken + entitlement

    def checkAuth(self):
 	if not self.headers.has_key('Authorization'):
            self.requestAuth()
            return None
	else:
            authToken = self.getAuth()
            if authToken is None:
                return

            # verify that the user/password actually exists in the database
            if not netRepos.auth.checkUserPass(authToken):
                self.send_error(403)
                return None

	return authToken

    def requestAuth(self):
        self.send_response(401)
        self.send_header("WWW-Authenticate", 'Basic realm="Conary Repository"')
        self.end_headers()
        return None

    def handleXml(self, authToken):
	contentLength = int(self.headers['Content-Length'])
        data = self.rfile.read(contentLength)

        encoding = self.headers.get('Content-Encoding', None)
        if encoding == 'deflate':
            data = zlib.decompress(data)

        (params, method) = xmlrpclib.loads(data)
        logMe(3, "decoded xml-rpc call %s from %d bytes request" %(method, contentLength))

	try:
	    result = netRepos.callWrapper(None, None, method, authToken, params)
	except errors.InsufficientPermission:
	    self.send_error(403)
	    return None
        logMe(3, "returned from", method)

        usedAnonymous = result[0]
        result = result[1:]

	resp = xmlrpclib.dumps((result,), methodresponse=1)
        logMe(3, "encoded xml-rpc response to %d bytes" % (len(resp),))

	self.send_response(200)
        encoding = self.headers.get('Accept-encoding', '')
        if len(resp) > 200 and 'deflate' in encoding:
            resp = zlib.compress(resp, 5)
            self.send_header('Content-encoding', 'deflate')
	self.send_header("Content-type", "text/xml")
	self.send_header("Content-length", str(len(resp)))
        if usedAnonymous:
            self.send_header("X-Conary-UsedAnonymous", '1')

	self.end_headers()
	self.wfile.write(resp)
        logMe(3, "sent response to client", len(resp), "bytes")
	return resp

    def do_PUT(self):
	path = self.path.split("?")[-1]

        if '/' in path:
	    self.send_error(403, "Forbidden")

	path = self.tmpDir + '/' + path + "-in"

	size = os.stat(path).st_size
	if size != 0:
	    self.send_error(410, "Gone")
	    return

	out = open(path, "w")

	contentLength = int(self.headers['Content-Length'])
	while contentLength:
	    s = self.rfile.read(contentLength)
	    contentLength -= len(s)
	    out.write(s)

	self.send_response(200, 'OK')

class ResetableNetworkRepositoryServer(NetworkRepositoryServer):
    def reset(self, authToken, clientVersion):
        import shutil
        try:
            shutil.rmtree(self.contentsDir)
        except OSError, e:
            if e.errno != errno.ENOENT:
                raise
        os.mkdir(self.contentsDir)

        logMe(1, "resetting NetworkRepositoryServer", self.repDB)
        # cheap trick. sqlite3 doesn't mind zero byte files; just replace
        # the file with a zero byte one (to change the inode) and reopen
        open(self.repDB[1] + '.new', "w")
        os.rename(self.repDB[1] + '.new', self.repDB[1])
        self.reopen()

        return 0

class ServerConfig(netserver.ServerConfig):

    port		= (CfgInt,  8000)

    def __init__(self, path="serverrc"):
	netserver.ServerConfig.__init__(self)
	self.read(path, exception=False)

    def check(self):
        if self.cacheDB:
            print >> sys.stderr, ("warning: cacheDB config option is ignored "
                                  "by the standalone server")

        if self.closed:
            print >> sys.stderr, ("warning: closed config option is ignored "
                                  "by the standalone server")

        if self.forceSSL:
            print >> sys.stderr, ("warning: commitAction config option is "
                                  "ignored by the standalone server")

def usage():
    print "usage: %s" % sys.argv[0]
    print "       %s --add-user [--admin] [--mirror] <username>" % sys.argv[0]
    print ""
    print "server flags: --config-file <path>"
    print '              --db "driver <path>"'
    print '              --log-file <path>'
    print '              --map "<from> <to>"'
    print "              --server-name <host>"
    print "              --tmp-dir <path>"
    sys.exit(1)

def addUser(netRepos, userName, admin = False, mirror = False):
    if os.isatty(0):
        from getpass import getpass

        pw1 = getpass('Password:')
        pw2 = getpass('Reenter password:')

        if pw1 != pw2:
            print "Passwords do not match."
            return 1
    else:
        # chop off the trailing newline
        pw1 = sys.stdin.readline()[:-1]

    # never give anonymous write access by default
    write = userName != 'anonymous'
    netRepos.auth.addUser(userName, pw1)
    # user/group, trovePattern, label, write, capped, admin
    netRepos.auth.addAcl(userName, None, None, write, False, admin)
    netRepos.auth.setMirror(userName, mirror)

if __name__ == '__main__':
    argDef = {}
    cfgMap = {
        'contents-dir'  : 'contentsDir',
	'db'	        : 'repositoryDB',
	'log-file'	: 'logFile',
	'map'	        : 'repositoryMap',
	'port'	        : 'port',
	'tmp-dir'       : 'tmpDir',
        'require-sigs'  : 'requireSigs',
        'server-name'   : 'serverName'
    }

    cfg = ServerConfig()

    argDef["config"] = options.MULT_PARAM
    # magically handled by processArgs
    argDef["config-file"] = options.ONE_PARAM

    argDef['add-user'] = options.ONE_PARAM
    argDef['admin'] = options.NO_PARAM
    argDef['help'] = options.NO_PARAM
    argDef['migrate'] = options.NO_PARAM
    argDef['mirror'] = options.NO_PARAM

    try:
        argSet, otherArgs = options.processArgs(argDef, cfgMap, cfg, usage)
    except options.OptionError, msg:
        print >> sys.stderr, msg
        sys.exit(1)

    if 'migrate' not in argSet:
        cfg.check()

    if argSet.has_key('help'):
        usage()

    if not os.path.isdir(cfg.tmpDir):
	print cfg.tmpDir + " needs to be a directory"
	sys.exit(1)
    if not os.access(cfg.tmpDir, os.R_OK | os.W_OK | os.X_OK):
        print cfg.tmpDir + " needs to allow full read/write access"
        sys.exit(1)
    HttpRequests.tmpDir = cfg.tmpDir

    profile = 0
    if profile:
        import hotshot
        prof = hotshot.Profile('server.prof')
        prof.start()

    baseUrl="http://%s:%s/" % (os.uname()[1], cfg.port)

    # start the logging
    if 'add-user' not in argSet:
        initLog(level=3, trace=1)

    if not cfg.repositoryDB:
        cfg.repositoryDB = ("sqlite",  "%s/sqldb" % otherArgs[1])
        del otherArgs[1]

    if len(otherArgs) > 1:
	usage()

    if not cfg.contentsDir:
        assert(cfg.repositoryDB[0] == "sqlite")
        cfg.contentsDir = os.path.dirname(cfg.repositoryDB[1]) + '/contents'

    if cfg.repositoryDB[0] == 'sqlite':
        util.mkdirChain(os.path.dirname(cfg.repositoryDB[1]))

    (driver, database) = cfg.repositoryDB
    db= dbstore.connect(database, driver)
    schema.loadSchema(db)

    if 'migrate' in argSet:
        sys.exit(0)

    netRepos = NetworkRepositoryServer(cfg, baseUrl)

    if 'add-user' in argSet:
        admin = argSet.pop('admin', False)
        mirror = argSet.pop('mirror', False)
        userName = argSet.pop('add-user')
        if argSet:
            usage()
        sys.exit(addUser(netRepos, userName, admin = admin, mirror = mirror))

    if argSet:
        usage()

    httpServer = HTTPServer(("", cfg.port), HttpRequests)

    fds = {}
    fds[httpServer.fileno()] = httpServer

    p = select.poll()
    for fd in fds.iterkeys():
        p.register(fd, select.POLLIN)

    logMe(1, "Server ready for requests")

    while True:
        try:
            events = p.poll()
            for (fd, event) in events:
                fds[fd].handle_request()
        except select.error:
            pass
        except:
            if profile:
                prof.stop()
                print "exception happened, exiting"
                sys.exit(1)
            else:
                raise
<|endoftext|>


<commit_before>#!/usr/bin/env python

import os
from bs4 import BeautifulSoup

import json
import requests
import ssl

import logging

# Remove SSL error
requests.packages.urllib3.disable_warnings()

try:
    _create_unverified_https_context = ssl._create_unverified_context
except AttributeError:
    # Legacy Python that doesn't verify HTTPS certificates by default
    pass
else:
    # Handle target environment that doesn't support HTTPS verification
    ssl._create_default_https_context = _create_unverified_https_context

headers = {'user-agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_6)\
           AppleWebKit/537.36 (KHTML, like Gecko)\
           Chrome/53.0.2785.143 Safari/537.36'}

class PageSpeedInsights(object):

    """docstring for PageSpeedInsights"""
    def __init__(self):
        super(PageSpeedInsights, self).__init__()

        pagespeed_url = 'https://developers.google.com/speed/pagespeed/insights/'
        source_code = requests.get(pagespeed_url, headers=headers, timeout=15)
        plain_text = source_code.text
        soup = BeautifulSoup(plain_text, "html.parser")
        scripts = soup.findAll('script')

        for x in scripts:
            if 'PAGESPEED_API_KEY' in x.text:
                script_tag = x.text
                break

        self.conf = {}

        # print(script_tag + '\n')
        if script_tag:
            variables = script_tag.split("var ")
            for variable in variables:
                if variable is not '':
                    variable = variable.replace('\'', '')
                    variable = variable.replace(';', '')
                    l_var = variable.split('=')
                    self.conf[l_var[0]] = l_var[1]

        for key in self.conf:
            print(key, " : ", self.conf[key])

        print("Got Page Speed API keys")

    def getPagespeedScore(self, siteurl, stategy='desktop'):

        if not siteurl:
            return {}

        url = self.conf['PAGESPEED_API_ROOT_URL'] + 'runPagespeed?key' + self.conf['PAGESPEED_API_KEY'] + \
                '&screenshot=false&snapshots=false&locale=en_US&filter_third_party_resources=false&strategy=' \
                + stategy + '&url=' + siteurl
        # print(url, end=', ')
        response = requests.get(url, headers=headers, timeout=25)
        json_data = json.loads(response.text)

        # print(response.text)
        # print(json_data['ruleGroups']['SPEED']['score'])
        return json_data['ruleGroups']['SPEED']['score']

tester = PageSpeedInsights()

import datetime

if __name__ == "__main__":

    print(datetime.datetime.now())
    count = 1
    with open('pageSpeed.list', 'r') as f:
        for line in f:
            line = line.strip('\n')
            if not line:
                continue
            print(count, end=', ')
            print(line, end=', ')
            m_score = tester.getPagespeedScore(line, 'mobile')
            print(m_score, end=', ')
            d_score = tester.getPagespeedScore(line)
            print(d_score)
            count += 1

    print(datetime.datetime.now())
<commit_msg>Work around for page speed captcha<commit_after>#!/usr/bin/env python

import os
from bs4 import BeautifulSoup

import json
import requests
import ssl

import logging

# Remove SSL error
requests.packages.urllib3.disable_warnings()

try:
    _create_unverified_https_context = ssl._create_unverified_context
except AttributeError:
    # Legacy Python that doesn't verify HTTPS certificates by default
    pass
else:
    # Handle target environment that doesn't support HTTPS verification
    ssl._create_default_https_context = _create_unverified_https_context

headers = {'user-agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_6)\
           AppleWebKit/537.36 (KHTML, like Gecko)\
           Chrome/53.0.2785.143 Safari/537.36'}

class PageSpeedInsights(object):

    """docstring for PageSpeedInsights"""
    def __init__(self):
        super(PageSpeedInsights, self).__init__()

        pagespeed_url = 'https://developers.google.com/speed/pagespeed/insights/'
        source_code = requests.get(pagespeed_url, headers=headers, timeout=15)
        plain_text = source_code.text
        soup = BeautifulSoup(plain_text, "html.parser")
        scripts = soup.findAll('script')

        for x in scripts:
            if 'PAGESPEED_API_KEY' in x.text:
                script_tag = x.text
                break

        self.conf = {}

        # print(script_tag + '\n')
        if script_tag:
            variables = script_tag.split("var ")
            for variable in variables:
                if variable is not '':
                    variable = variable.replace('\'', '')
                    variable = variable.replace(';', '')
                    l_var = variable.split('=')
                    self.conf[l_var[0]] = l_var[1]

        for key in self.conf:
            print(key, " : ", self.conf[key])

        print("Got Page Speed API keys")

    def getPagespeedScore(self, siteurl, stategy='desktop'):

        if not siteurl:
            return {}

        url = self.conf['PAGESPEED_API_ROOT_URL'] + 'runPagespeed?key' + self.conf['PAGESPEED_API_KEY'] + \
                '&screenshot=false&snapshots=false&locale=en_US&filter_third_party_resources=false&strategy=' \
                + stategy + '&url=' + siteurl
        # print(url, end=', ')
        response = requests.get(url, headers=headers, timeout=25)
        json_data = json.loads(response.text)

        # print(response.text)
        # print(json_data['ruleGroups']['SPEED']['score'])
        try:
            score = json_data['ruleGroups']['SPEED']['score']
        except:
            score = 0
            print(response.text)

        return score

import datetime
import time

if __name__ == "__main__":

    print(datetime.datetime.now())
    tester = PageSpeedInsights()
    count = 1
    with open('pageSpeed.list', 'r') as f:
        for line in f:
            line = line.strip('\n')
            if not line:
                continue
            print(count, end=', ')
            print(line, end=', ')
            m_score = tester.getPagespeedScore(line, 'mobile')
            print(m_score, end=', ')
            d_score = tester.getPagespeedScore(line)
            print(d_score)
            count += 1

            if count == 10:
                print("Waiting for 30 seconds. Otherwise captcha will be asked.")
                time.sleep(30)

    print(datetime.datetime.now())
<|endoftext|>


<commit_before>from .models import Discipline, Variant, Diagram, Technology, Semester, TrainingTerms
from courses.models import Course
from rest_framework import serializers
from courses.serializers import CourseSerializer

class DisciplineSerializer(serializers.HyperlinkedModelSerializer):

    results = serializers.HyperlinkedRelatedField(
        many=True,
        read_only=True,
        view_name='result-detail'
    )

    class Meta:
        model = Discipline
        fields = ("title", "labor", "period", "results", "results_text", "status", "archived", "created", "updated")

class DiagramSerializer(serializers.ModelSerializer):
    class Meta:
        model = Diagram
        fields = ("title", "diagram")

class TechnologySerializer(serializers.ModelSerializer):
    class Meta:
        model = Technology
        fields = ("title", "description", "contact_work_category", "color")

class TrainingTermsSerializer(serializers.ModelSerializer):
    class Meta:
        model = TrainingTerms
        fields = ("title", "limit")

class SemesterSerializer(serializers.ModelSerializer):
    discipline = DisciplineSerializer()
    term = TrainingTermsSerializer()

    class Meta:
        model = Semester
        fields = ("discipline", "year", "admission_semester", "training_semester", "term")

class VariantSerializer(serializers.ModelSerializer):
    discipline = DisciplineSerializer()
    diagram = DiagramSerializer()
    technology = TechnologySerializer()
    course = serializers.PrimaryKeyRelatedField(
        many=True,
        queryset=Course.objects.filter(status="p", archived=False)
    )

    class Meta:
        model = Variant
        fields = ("discipline", "diagram", "technology", "course", "semester", "parity", "link")
<commit_msg>create_variant#4<commit_after>from .models import Discipline, Variant, Diagram, Technology, Semester, TrainingTerms
from courses.models import Course
from rest_framework import serializers
from courses.serializers import CourseSerializer

class DisciplineSerializer(serializers.HyperlinkedModelSerializer):

    results = serializers.HyperlinkedRelatedField(
        many=True,
        read_only=True,
        view_name='result-detail'
    )

    class Meta:
        model = Discipline
        fields = ("title", "labor", "period", "results", "results_text", "status", "archived", "created", "updated")

class DiagramSerializer(serializers.ModelSerializer):
    class Meta:
        model = Diagram
        fields = ("title", "diagram")

class TechnologySerializer(serializers.ModelSerializer):
    class Meta:
        model = Technology
        fields = ("title", "description", "contact_work_category", "color")

class TrainingTermsSerializer(serializers.ModelSerializer):
    class Meta:
        model = TrainingTerms
        fields = ("title", "limit")

class SemesterSerializer(serializers.ModelSerializer):
    discipline = DisciplineSerializer()
    term = TrainingTermsSerializer()

    class Meta:
        model = Semester
        fields = ("discipline", "year", "admission_semester", "training_semester", "term")

class VariantSerializer(serializers.ModelSerializer):
    discipline = serializers.PrimaryKeyRelatedField(
        many=True,
        queryset=Discipline.objects.filter(status="p", archived=False)
    )
    diagram = serializers.PrimaryKeyRelatedField(
        many=True,
        queryset=Diagram.objects.filter(status="p", archived=False)
    )
    technology = serializers.PrimaryKeyRelatedField(
        many=True,
        queryset=Technology.objects.filter(status="p", archived=False)
    )
    course = serializers.PrimaryKeyRelatedField(
        many=True,
        queryset=Course.objects.filter(status="p", archived=False)
    )

    class Meta:
        model = Variant
        fields = ("discipline", "diagram", "technology", "course", "semester", "parity", "link")
<|endoftext|>


<commit_before>import core.job
import core.implant
import uuid

class ExecCmdJob(core.job.Job):
    def done(self):
        self.display()

    def display(self):
        self.shell.print_plain("Result for `%s`:" % self.options.get('CMD'))
        self.shell.print_plain(self.data)

class ExecCmdImplant(core.implant.Implant):

    NAME = "Execute Command"
    DESCRIPTION = "Executes a command on the target system."
    AUTHORS = ["RiskSense, Inc."]

    def load(self):
        self.options.register("CMD", "hostname", "command to run")
        self.options.register("OUTPUT", "true", "retrieve output?", enum=["true", "false"])
        self.options.register("DIRECTORY", "%TEMP%", "writeable directory for output", required=False)
        self.options.register("FILE", "", "random uuid for file name", hidden=True)

    def run(self):
        # generate new file every time this is run
        self.options.set("FILE", uuid.uuid4().hex)

        payloads = {}
        #payloads["vbs"] = self.load_script("data/implant/manage/exec_cmd.vbs", self.options)
        payloads["js"] = self.loader.load_script("data/implant/manage/exec_cmd.js", self.options)

        self.dispatch(payloads, ExecCmdJob)
<commit_msg>escape double quotes and backslashes<commit_after>import core.job
import core.implant
import uuid

class ExecCmdJob(core.job.Job):
    def done(self):
        self.display()

    def display(self):
        self.shell.print_plain("Result for `%s`:" % self.options.get('CMD').replace('\\"', '"').replace("\\\\", "\\"))
        self.shell.print_plain(self.data)

class ExecCmdImplant(core.implant.Implant):

    NAME = "Execute Command"
    DESCRIPTION = "Executes a command on the target system."
    AUTHORS = ["RiskSense, Inc."]

    def load(self):
        self.options.register("CMD", "hostname", "command to run")
        self.options.register("OUTPUT", "true", "retrieve output?", enum=["true", "false"])
        self.options.register("DIRECTORY", "%TEMP%", "writeable directory for output", required=False)
        self.options.register("FILE", "", "random uuid for file name", hidden=True)

    def run(self):
        # generate new file every time this is run
        self.options.set("FILE", uuid.uuid4().hex)
        self.options.set("CMD", self.options.get('CMD').replace("\\", "\\\\").replace('"', '\\"'))

        payloads = {}
        #payloads["vbs"] = self.load_script("data/implant/manage/exec_cmd.vbs", self.options)
        payloads["js"] = self.loader.load_script("data/implant/manage/exec_cmd.js", self.options)

        self.dispatch(payloads, ExecCmdJob)
<|endoftext|>


<commit_before>
from __future__ import print_function
import plac
from thinc.neural.vec2vec import ReLu, Softmax

from thinc.loss import categorical_crossentropy
from thinc.neural.optimizers import Adam
from thinc.extra import datasets
from thinc.neural.util import score_model
from thinc.neural.base import Model
from thinc.neural.ops import NumpyOps

try:
    import cPickle as pickle
except ImportError:
    import pickle

def main(depth=2, width=512, nb_epoch=10):
    model = Model(
              ReLu(512, 784, name='relu1'),
              Softmax(10, 512, name='softmax'),
              ops=NumpyOps())

    train_data, dev_data, test_data = datasets.mnist()
    train_X, train_Y = zip(*train_data)
    dev_X, dev_Y = zip(*dev_data)

    optimizer = Adam(model.ops, 0.001)
    with model.begin_training(train_data) as (trainer, _):
        for i in range(nb_epoch):
            for batch_X, batch_Y in trainer.iterate(
                    model, train_data, dev_data, nb_epoch=1):
                batch_X = model.ops.asarray(batch_X)
                guess, finish_update = model.begin_update(batch_X, dropout=0.0)
                gradient, loss = categorical_crossentropy(guess, batch_Y)
                finish_update(gradient, optimizer)
            print(i, score_model(model, dev_X, dev_Y))
    with open('out.pickle', 'wb') as file_:
        pickle.dump(model, file_, -1)

if __name__ == '__main__':
    plac.call(main)
<commit_msg>Tweak mnist example<commit_after>from __future__ import print_function
import plac
from thinc.neural.vec2vec import ReLu, Softmax

from thinc.loss import categorical_crossentropy
from thinc.neural.optimizers import Adam
from thinc.extra import datasets
from thinc.neural.util import score_model
from thinc.neural.base import Model
from thinc.neural.ops import NumpyOps

try:
    import cPickle as pickle
except ImportError:
    import pickle

def main(depth=2, width=512, nb_epoch=10):
    model = Model(
              ReLu(128, 784, name='relu1'),
              ReLu(128, 128, name='relu2'),
              ReLu(128, 128, name='relu3'),
              Softmax(10, 128, name='softmax'),
              ops=NumpyOps())

    train_data, dev_data, test_data = datasets.mnist()
    train_X, train_Y = zip(*train_data)
    dev_X, dev_Y = zip(*dev_data)

    with model.begin_training(train_data) as (trainer, optimizer):
        for i in range(nb_epoch):
            for batch_X, batch_Y in trainer.iterate(
                    model, train_data, dev_data, nb_epoch=1):
                batch_X = model.ops.asarray(batch_X)
                guess, finish_update = model.begin_update(batch_X, dropout=0.0)
                gradient, loss = categorical_crossentropy(guess, batch_Y)
                optimizer.set_loss(loss)
                finish_update(gradient, optimizer)
    with open('out.pickle', 'wb') as file_:
        pickle.dump(model, file_, -1)

if __name__ == '__main__':
    plac.call(main)
<|endoftext|>


<commit_before>import esn
import signals

SEED = 0
PATTERN_LENGTH = 1
PATTERN_PAUSE = 0.5
OUTPUT_PULSE_LENGTH = 0.2

class Signal :

    def __init__( self ) :
        self.value = 0
        self.time = 0
        self.front_edge = 0
        self.back_edge = 0
        self.pattern_noise = \
            signals.PerlinNoise( persistence=1, octave_count=7 )
        if SEED > 0 :
            self.pattern_noise.seed( SEED )
        self.pulse_noise = \
            signals.PerlinNoise( persistence=0.5, octave_count=1 )
        if SEED > 0 :
            self.pulse_noise.seed( SEED + 1 )
        self.prev_pulse_noise = self.pulse_noise( 0 )
        self.cur_pulse_noise = self.pulse_noise( 0 )

    def step( self, step ) :
        if self.time > ( self.front_edge + PATTERN_LENGTH + \
           PATTERN_PAUSE ) and self.is_front_edge() :
            self.front_edge = self.time
            self.back_edge = self.front_edge + PATTERN_LENGTH

        if self.front_edge <= self.time and \
           self.time <= self.back_edge :
            self.value = self.pattern_noise( self.time - \
                    self.front_edge ) * 0.15
        else :
            self.value = 0

        self.prev_pulse_noise = \
            self.pulse_noise( self.time )
        self.time += step

    def is_front_edge( self ) :
        if self.prev_pulse_noise <= 0 and \
            self.pulse_noise( self.time ) > 0 :
            return True
        else :
            return False

class Model :

    def __init__( self, neuron_count ) :
        self.network = esn.CreateNetworkNSLI(
                inputCount=1,
                neuronCount=neuron_count,
                outputCount=1
            )
        self.noise = signals.PerlinNoise( persistence=0.5, octave_count=8 )
        if SEED > 0 :
            self.noise.seed( SEED + 2 )
        self.pattern = Signal()
        self.train_pulse = signals.GaussianPulse(
            amplitude=0.5, width=OUTPUT_PULSE_LENGTH )
        self.time = 0

    def step( self, step ) :
        self.pattern.step( step )
        self.noise_value = self.noise( self.time ) * 0.1
        self.input = self.noise_value + self.pattern.value
        self.network.set_inputs( [ self.input ] )
        self.network.step( step )
        self.output = self.network.capture_output( 1 )[ 0 ]
        self.train_output = self.train_pulse( self.time - \
            self.pattern.back_edge )

        print( "%10s %10s %10s %10s %10s" %
                (
                    str( "%0.3f" % self.time ),
                    str( "%0.5f" % self.input ),
                    str( "%0.5f" % self.pattern.value ),
                    str( "%0.5f" % self.train_output ),
                    str( "%0.5f" % self.output )
                )
            )

        self.time += step
<commit_msg>samples : python : pattern : initialize Signal.back_edge<commit_after>import esn
import signals

SEED = 0
PATTERN_LENGTH = 1
PATTERN_PAUSE = 0.5
OUTPUT_PULSE_LENGTH = 0.2

class Signal :

    def __init__( self ) :
        self.value = 0
        self.time = 0
        self.front_edge = 0
        self.back_edge = -OUTPUT_PULSE_LENGTH
        self.pattern_noise = \
            signals.PerlinNoise( persistence=1, octave_count=7 )
        if SEED > 0 :
            self.pattern_noise.seed( SEED )
        self.pulse_noise = \
            signals.PerlinNoise( persistence=0.5, octave_count=1 )
        if SEED > 0 :
            self.pulse_noise.seed( SEED + 1 )
        self.prev_pulse_noise = self.pulse_noise( 0 )
        self.cur_pulse_noise = self.pulse_noise( 0 )

    def step( self, step ) :
        if self.time > ( self.front_edge + PATTERN_LENGTH + \
           PATTERN_PAUSE ) and self.is_front_edge() :
            self.front_edge = self.time
            self.back_edge = self.front_edge + PATTERN_LENGTH

        if self.front_edge <= self.time and \
           self.time <= self.back_edge :
            self.value = self.pattern_noise( self.time - \
                    self.front_edge ) * 0.15
        else :
            self.value = 0

        self.prev_pulse_noise = \
            self.pulse_noise( self.time )
        self.time += step

    def is_front_edge( self ) :
        if self.prev_pulse_noise <= 0 and \
            self.pulse_noise( self.time ) > 0 :
            return True
        else :
            return False

class Model :

    def __init__( self, neuron_count ) :
        self.network = esn.CreateNetworkNSLI(
                inputCount=1,
                neuronCount=neuron_count,
                outputCount=1
            )
        self.noise = signals.PerlinNoise( persistence=0.5, octave_count=8 )
        if SEED > 0 :
            self.noise.seed( SEED + 2 )
        self.pattern = Signal()
        self.train_pulse = signals.GaussianPulse(
            amplitude=0.5, width=OUTPUT_PULSE_LENGTH )
        self.time = 0

    def step( self, step ) :
        self.pattern.step( step )
        self.noise_value = self.noise( self.time ) * 0.1
        self.input = self.noise_value + self.pattern.value
        self.network.set_inputs( [ self.input ] )
        self.network.step( step )
        self.output = self.network.capture_output( 1 )[ 0 ]
        self.train_output = self.train_pulse( self.time - \
            self.pattern.back_edge )

        print( "%10s %10s %10s %10s %10s" %
                (
                    str( "%0.3f" % self.time ),
                    str( "%0.5f" % self.input ),
                    str( "%0.5f" % self.pattern.value ),
                    str( "%0.5f" % self.train_output ),
                    str( "%0.5f" % self.output )
                )
            )

        self.time += step
<|endoftext|>


<commit_before># pylint: disable=unicode-format-string
# -*- coding: utf-8 -*-
"""
Tests for js_utils.py
"""
from __future__ import absolute_import
import six.moves.html_parser  # pylint: disable=import-error
import json
from unittest import TestCase

from mako.template import Template

from openedx.core.djangolib.js_utils import dump_js_escaped_json, js_escaped_string
import six  # pylint: disable=ungrouped-imports
import re

class TestJSUtils(TestCase):
    """
    Test JS utils
    """

    class NoDefaultEncoding(object):
        """
        Helper class that has no default JSON encoding
        """
        def __init__(self, value):
            self.value = value

    class SampleJSONEncoder(json.JSONEncoder):
        """
        A test encoder that is used to prove that the encoder does its job before the escaping.
        """
        # pylint: disable=method-hidden
        def default(self, noDefaultEncodingObj):
            return noDefaultEncodingObj.value.replace("": "</script>"}
        expected_escaped_json = (
            r'''{"\u003c/script\u003e\u003cscript\u003ealert('hello, ');\u003c/script\u003e": '''
            r'''"\u003c/script\u003e\u003cscript\u003ealert('\u0026world!');\u003c/script\u003e"}'''
        )

        escaped_json = dump_js_escaped_json(malicious_dict)
        self.assertEquals(expected_escaped_json, escaped_json)

    def test_dump_js_escaped_json_with_custom_encoder_escapes_unsafe_html(self):
        """
        Test dump_js_escaped_json first encodes with custom JSNOEncoder before escaping &, <, and >

        The test encoder class should first perform the replacement of "":
            self.NoDefaultEncoding("</script>")
        }
        expected_custom_escaped_json = (
            r'''{"\u003c/script\u003e\u003cscript\u003ealert('hello, ');\u003c/script\u003e": '''
            r'''"\u003c/script\u003esample-encoder-was-herealert('\u0026world!');\u003c/script\u003e"}'''
        )

        escaped_json = dump_js_escaped_json(malicious_dict, cls=self.SampleJSONEncoder)
        self.assertEquals(expected_custom_escaped_json, escaped_json)

    def test_js_escaped_string_escapes_unsafe_html(self):
        """
        Test js_escaped_string escapes &, <, and >, as well as returns a unicode type
        """
        malicious_js_string = "</script>"

        expected_escaped_string_for_js = six.text_type(
            r"\u003C/script\u003E\u003Cscript\u003Ealert(\u0027hello, \u0027)\u003B\u003C/script\u003E"
        )
        escaped_string_for_js = js_escaped_string(malicious_js_string)
        self.assertEquals(expected_escaped_string_for_js, escaped_string_for_js)

    def test_js_escaped_string_with_none(self):
        """
        Test js_escaped_string returns empty string for None
        """
        escaped_string_for_js = js_escaped_string(None)
        self.assertEquals(u"", escaped_string_for_js)

    def test_mako(self):
        """
        Tests the full suite of Mako best practices by running all of the
        combinations of types of data and types of escaping through a Mako
        template.

        Additionally, validates the best practices themselves by validating
        the expectations to ensure they can properly be unescaped and/or
        parsed from json where applicable.
        """
        test_dict = {
            'test_string': u'test-=&\\;\'"<>☃',
            'test_tuple': (1, 2, 3),
            'test_number': 3.5,
            'test_bool': False,
        }

        template = Template(
            u"""
                <%!
                import json
                from openedx.core.djangolib.js_utils import (
                    dump_js_escaped_json, js_escaped_string
                )
                %>

            """,
            default_filters=['decode.utf8', 'h'],
        )
        out = template.render(test_dict=test_dict)

        expected_json_for_html = (
            r"{"test_bool": false, "test_number": 3.5, "
            r""test_tuple": [1, 2, 3], "test_string": "
            r""test-=&\\;'\"<>\u2603"}"
        )
        expected_attr_json_for_html = "data-test-dict='" + expected_json_for_html + "'"
        self._validate_expectation_of_json_for_html(test_dict, expected_json_for_html)
        self.assertIn(""test_tuple": [1, 2, 3]", out)
        self.assertIn(""test_number": 3.5", out)
        self.assertIn(""test_bool": false", out)
        self.assertIn(""test_string": "test-=&\\\\;'\\"<>\\u2603&#34", out)
        self.assertIn(u"data-test-string='test-=&\\;'"<>☃'", out)
        self.assertIn("data-test-tuple='[1, 2, 3]'", out)
        self.assertIn("data-test-number='3.5'", out)
        self.assertIn("data-test-bool='false'", out)

        expected_string_for_js_in_dict = r'''test-=\u0026\\;'\"\u003c\u003e\u2603'''
        self._validate_expectation_of_string_for_js(test_dict['test_string'], expected_string_for_js_in_dict)
        location_of_dict_in_out = re.search("var test_dict.*}", out)
        var_dict_in_out = out[location_of_dict_in_out.span()[0]:location_of_dict_in_out.span()[1]]
        self.assertIn('"test_number": 3.5', var_dict_in_out)
        self.assertIn('"test_string": "test-=\\u0026\\\\;\'\\"\\u003c\\u003e\\u2603"', var_dict_in_out)
        self.assertIn('"test_tuple": [1, 2, 3]', var_dict_in_out)
        self.assertIn('"test_bool": false', var_dict_in_out)

        expected_string_for_js = u"test\\u002D\\u003D\\u0026\\u005C\\u003B\\u0027\\u0022\\u003C\\u003E☃"
        self._validate_expectation_of_string_for_js(test_dict['test_string'], expected_string_for_js)
        self.assertIn("var test_string = '" + expected_string_for_js + "'", out)
        self.assertIn("var test_none_string = ''", out)
        self.assertIn("var test_tuple = [1, 2, 3]", out)
        self.assertIn("var test_number = 3.5", out)
        self.assertIn("var test_bool = false", out)
        self.assertIn("var test_none_json = null", out)

    def _validate_expectation_of_json_for_html(self, test_dict, expected_json_for_html_string):
        """
        Proves that the expectation string is a reasonable one, since it is
        not very human readable with all of the escaping.

        Ensures that after unescaping (html) the string can be parsed to a
        (nearly) equivalent dict.

        Assertions will fail if the expectation is invalid.

        Arguments:
            test_dict: The original dict to be tested in the Mako template.
            expected_json_for_html_string: An html escaped json string that
                should be parseable into a near equivalent to test_dict.

        """
        html_parser = six.moves.html_parser.HTMLParser()

        expected_json = html_parser.unescape(expected_json_for_html_string)
        parsed_expected_dict = json.loads(expected_json)
        # tuples become arrays in json, so it is parsed to a list that is
        # switched back to a tuple before comparing
        parsed_expected_dict['test_tuple'] = tuple(parsed_expected_dict['test_tuple'])
        self.assertEqual(test_dict['test_string'], parsed_expected_dict['test_string'])
        self.assertEqual(test_dict['test_tuple'], parsed_expected_dict['test_tuple'])
        self.assertEqual(test_dict['test_number'], parsed_expected_dict['test_number'])
        self.assertEqual(test_dict['test_bool'], parsed_expected_dict['test_bool'])

    def _validate_expectation_of_string_for_js(self, test_string, expected_string_for_js):
        """
        Proves that the expectation string is a reasonable one, since it is
        not very human readable with all of the escaping.

        Ensures that after parsing the string is equal to the original.

        Assertions will fail if the expectation is invalid.

        Arguments:
            test_string: The original string to be tested in the Mako template.
            expected_string_for_js: An escaped for js string that should be
                parseable into the same string as test_string.

        """
        parsed_expected_string = json.loads('"' + expected_string_for_js + '"')
        self.assertEqual(test_string, parsed_expected_string)
<commit_msg>quality fixes<commit_after># pylint: disable=unicode-format-string
# -*- coding: utf-8 -*-
"""
Tests for js_utils.py
"""
from __future__ import absolute_import
import re
import six.moves.html_parser  # pylint: disable=import-error
import json
from unittest import TestCase

from mako.template import Template

from openedx.core.djangolib.js_utils import dump_js_escaped_json, js_escaped_string
import six  # pylint: disable=ungrouped-imports

class TestJSUtils(TestCase):
    """
    Test JS utils
    """

    class NoDefaultEncoding(object):
        """
        Helper class that has no default JSON encoding
        """
        def __init__(self, value):
            self.value = value

    class SampleJSONEncoder(json.JSONEncoder):
        """
        A test encoder that is used to prove that the encoder does its job before the escaping.
        """
        # pylint: disable=method-hidden
        def default(self, noDefaultEncodingObj):
            return noDefaultEncodingObj.value.replace("": "</script>"}
        expected_escaped_json = (
            r'''{"\u003c/script\u003e\u003cscript\u003ealert('hello, ');\u003c/script\u003e": '''
            r'''"\u003c/script\u003e\u003cscript\u003ealert('\u0026world!');\u003c/script\u003e"}'''
        )

        escaped_json = dump_js_escaped_json(malicious_dict)
        self.assertEquals(expected_escaped_json, escaped_json)

    def test_dump_js_escaped_json_with_custom_encoder_escapes_unsafe_html(self):
        """
        Test dump_js_escaped_json first encodes with custom JSNOEncoder before escaping &, <, and >

        The test encoder class should first perform the replacement of "":
            self.NoDefaultEncoding("</script>")
        }
        expected_custom_escaped_json = (
            r'''{"\u003c/script\u003e\u003cscript\u003ealert('hello, ');\u003c/script\u003e": '''
            r'''"\u003c/script\u003esample-encoder-was-herealert('\u0026world!');\u003c/script\u003e"}'''
        )

        escaped_json = dump_js_escaped_json(malicious_dict, cls=self.SampleJSONEncoder)
        self.assertEquals(expected_custom_escaped_json, escaped_json)

    def test_js_escaped_string_escapes_unsafe_html(self):
        """
        Test js_escaped_string escapes &, <, and >, as well as returns a unicode type
        """
        malicious_js_string = "</script>"

        expected_escaped_string_for_js = six.text_type(
            r"\u003C/script\u003E\u003Cscript\u003Ealert(\u0027hello, \u0027)\u003B\u003C/script\u003E"
        )
        escaped_string_for_js = js_escaped_string(malicious_js_string)
        self.assertEquals(expected_escaped_string_for_js, escaped_string_for_js)

    def test_js_escaped_string_with_none(self):
        """
        Test js_escaped_string returns empty string for None
        """
        escaped_string_for_js = js_escaped_string(None)
        self.assertEquals(u"", escaped_string_for_js)

    def test_mako(self):
        """
        Tests the full suite of Mako best practices by running all of the
        combinations of types of data and types of escaping through a Mako
        template.

        Additionally, validates the best practices themselves by validating
        the expectations to ensure they can properly be unescaped and/or
        parsed from json where applicable.
        """
        test_dict = {
            'test_string': u'test-=&\\;\'"<>☃',
            'test_tuple': (1, 2, 3),
            'test_number': 3.5,
            'test_bool': False,
        }

        template = Template(
            u"""
                <%!
                import json
                from openedx.core.djangolib.js_utils import (
                    dump_js_escaped_json, js_escaped_string
                )
                %>

            """,
            default_filters=['decode.utf8', 'h'],
        )
        out = template.render(test_dict=test_dict)

        expected_json_for_html = (
            r"{"test_bool": false, "test_number": 3.5, "
            r""test_tuple": [1, 2, 3], "test_string": "
            r""test-=&\\;'\"<>\u2603"}"
        )
        self._validate_expectation_of_json_for_html(test_dict, expected_json_for_html)
        self.assertIn(""test_tuple": [1, 2, 3]", out)
        self.assertIn(""test_number": 3.5", out)
        self.assertIn(""test_bool": false", out)
        self.assertIn(""test_string": "test-=&\\\\;'\\"<>\\u2603&#34", out)
        self.assertIn(u"data-test-string='test-=&\\;'"<>☃'", out)
        self.assertIn("data-test-tuple='[1, 2, 3]'", out)
        self.assertIn("data-test-number='3.5'", out)
        self.assertIn("data-test-bool='false'", out)

        expected_string_for_js_in_dict = r'''test-=\u0026\\;'\"\u003c\u003e\u2603'''
        self._validate_expectation_of_string_for_js(test_dict['test_string'], expected_string_for_js_in_dict)
        location_of_dict_in_out = re.search("var test_dict.*}", out)
        var_dict_in_out = out[location_of_dict_in_out.span()[0]:location_of_dict_in_out.span()[1]]
        self.assertIn('"test_number": 3.5', var_dict_in_out)
        self.assertIn('"test_string": "test-=\\u0026\\\\;\'\\"\\u003c\\u003e\\u2603"', var_dict_in_out)
        self.assertIn('"test_tuple": [1, 2, 3]', var_dict_in_out)
        self.assertIn('"test_bool": false', var_dict_in_out)

        expected_string_for_js = u"test\\u002D\\u003D\\u0026\\u005C\\u003B\\u0027\\u0022\\u003C\\u003E☃"
        self._validate_expectation_of_string_for_js(test_dict['test_string'], expected_string_for_js)
        self.assertIn("var test_string = '" + expected_string_for_js + "'", out)
        self.assertIn("var test_none_string = ''", out)
        self.assertIn("var test_tuple = [1, 2, 3]", out)
        self.assertIn("var test_number = 3.5", out)
        self.assertIn("var test_bool = false", out)
        self.assertIn("var test_none_json = null", out)

    def _validate_expectation_of_json_for_html(self, test_dict, expected_json_for_html_string):
        """
        Proves that the expectation string is a reasonable one, since it is
        not very human readable with all of the escaping.

        Ensures that after unescaping (html) the string can be parsed to a
        (nearly) equivalent dict.

        Assertions will fail if the expectation is invalid.

        Arguments:
            test_dict: The original dict to be tested in the Mako template.
            expected_json_for_html_string: An html escaped json string that
                should be parseable into a near equivalent to test_dict.

        """
        html_parser = six.moves.html_parser.HTMLParser()

        expected_json = html_parser.unescape(expected_json_for_html_string)
        parsed_expected_dict = json.loads(expected_json)
        # tuples become arrays in json, so it is parsed to a list that is
        # switched back to a tuple before comparing
        parsed_expected_dict['test_tuple'] = tuple(parsed_expected_dict['test_tuple'])
        self.assertEqual(test_dict['test_string'], parsed_expected_dict['test_string'])
        self.assertEqual(test_dict['test_tuple'], parsed_expected_dict['test_tuple'])
        self.assertEqual(test_dict['test_number'], parsed_expected_dict['test_number'])
        self.assertEqual(test_dict['test_bool'], parsed_expected_dict['test_bool'])

    def _validate_expectation_of_string_for_js(self, test_string, expected_string_for_js):
        """
        Proves that the expectation string is a reasonable one, since it is
        not very human readable with all of the escaping.

        Ensures that after parsing the string is equal to the original.

        Assertions will fail if the expectation is invalid.

        Arguments:
            test_string: The original string to be tested in the Mako template.
            expected_string_for_js: An escaped for js string that should be
                parseable into the same string as test_string.

        """
        parsed_expected_string = json.loads('"' + expected_string_for_js + '"')
        self.assertEqual(test_string, parsed_expected_string)
<|endoftext|>


<commit_before>import os, glob
from imp import find_module, load_module

EXTENSIONS_PATH = os.path.dirname(os.path.abspath(__file__))
EXTENSIONS_DIRECTORIES = glob.glob(os.path.join(EXTENSIONS_PATH, '*/'))

extensions_parameters = []

def import_extension(extension_directory):
	extension_name = os.path.basename(os.path.normpath(extension_directory))
	py_files = glob.glob(os.path.join(extension_directory, "*.py"))
	module_names = [
		os.path.basename(f)[:-3]
		for f in py_files
		if not os.path.basename(f).startswith('_')
		]
	for module_name in module_names:
		module = find_module(module_name, [extension_directory])
		load_module(module_name, *module)
	extensions_parameters.append(os.path.join(extension_directory, extension_name + '.xml'))

for extension_dir in EXTENSIONS_DIRECTORIES:
	import_extension(extension_dir)
<commit_msg>parameter file is optional<commit_after>import os, glob
from imp import find_module, load_module

EXTENSIONS_PATH = os.path.dirname(os.path.abspath(__file__))
EXTENSIONS_DIRECTORIES = glob.glob(os.path.join(EXTENSIONS_PATH, '*/'))

extensions_parameters = []

def import_extension(extension_directory):
	extension_name = os.path.basename(os.path.normpath(extension_directory))
	py_files = glob.glob(os.path.join(extension_directory, "*.py"))
	module_names = [
		os.path.basename(f)[:-3]
		for f in py_files
		if not os.path.basename(f).startswith('_')
		]
	for module_name in module_names:
		module = find_module(module_name, [extension_directory])
		load_module(module_name, *module)
	param_file = os.path.join(extension_directory, extension_name + '.xml')
	if os.path.isfile(param_file):
		extensions_parameters.append(param_file)

for extension_dir in EXTENSIONS_DIRECTORIES:
	import_extension(extension_dir)
<|endoftext|>