<commit_before># -*- coding: utf-8 -*-

# Copyright 2015 Telefonica Investigación y Desarrollo, S.A.U
#
# This file is part of FIWARE project.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
#
# You may obtain a copy of the License at:
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#
# See the License for the specific language governing permissions and
# limitations under the License.
#
# For those usages not covered by the Apache version 2.0 License please
# contact with opensource@tid.es

__author__ = 'gjp'

from tests.fiware_region_base_tests import FiwareRegionsBaseTests
from commons.constants import TEST_CONTAINER_PREFIX, TEST_TEXT_OBJECT_PREFIX, TEST_TEXT_FILE_EXTENSION, \
    SWIFT_RESOURCES_PATH, TEST_BIG_OBJECT_PREFIX, TEST_BIG_FILE_EXTENSION, PROPERTIES_CONFIG_SWIFT,\
    PROPERTIES_CONFIG_SWIFT_BIG_FILE_1, PROPERTIES_CONFIG_SWIFT_BIG_FILE_2, PROPERTIES_CONFIG_TEST,\
    TEST_BIG_OBJECT_REMOTE_PREFIX
from swiftclient.exceptions import ClientException as SwiftClientException
from datetime import datetime
import hashlib
import urllib2

class FiwareRegionsObjectStorageTests(FiwareRegionsBaseTests):

    with_storage = True

    def test_create_container(self):
        """
        Test whether it is possible to create a new container into the object storage.
        """

        suffix = datetime.utcnow().strftime('%Y%m%d%H%M%S')
        containerName = TEST_CONTAINER_PREFIX + suffix

        response = self.swift_operations.create_container(containerName)

        self.assertIsNone(response, "Container could not be created")
        self.test_world['containers'].append(containerName)
        self.logger.debug("Created %s container was created", containerName)

        response = self.swift_operations.get_container(containerName)
        self.assertEquals('x-container-object-count' in response[0], True, "There is no container header in response")
        self.assertEquals(len(response[-1]), 0, "The container is not empty")  # The list of items should be 0.
        self.logger.debug("Getting %s container details from the object storage", containerName)

    def test_delete_container(self):
        """
        Test whether it is possible to delete a container.
        """
        suffix = datetime.utcnow().strftime('%Y%m%d%H%M%S')
        containerName = TEST_CONTAINER_PREFIX + suffix

        response = self.swift_operations.create_container(containerName)
        self.assertIsNone(response, "Container could not be created")
        self.test_world['containers'].append(containerName)

        response = self.swift_operations.delete_container(containerName)
        self.assertIsNone(response, "Container could not be deleted")
        self.test_world['containers'].remove(containerName)

        try:
            self.swift_operations.get_container(containerName)
        except SwiftClientException as e:
            self.assertRaises(e)
            self.logger.debug("%s container was successfully removed from the object storage", containerName)

    def test_create_text_object_and_download_it_from_container(self):
        """
        Test whether it is possible to upload a text file and download it.
        """

        suffix = datetime.utcnow().strftime('%Y%m%d%H%M%S%m')
        containerName = TEST_CONTAINER_PREFIX + suffix
        textObjectName = self.region_name + TEST_TEXT_OBJECT_PREFIX + suffix + TEST_TEXT_FILE_EXTENSION

        response = self.swift_operations.create_container(containerName)
        self.assertIsNone(response, "Container could not be created")
        self.test_world['containers'].append(containerName)
        self.logger.debug("Created %s container was created", containerName)

        # Uploading the object
        response = self.swift_operations.create_object(containerName, SWIFT_RESOURCES_PATH
                                                            + TEST_TEXT_OBJECT_PREFIX + TEST_TEXT_FILE_EXTENSION,
                                                            textObjectName)
        origin = hashlib.md5(open(SWIFT_RESOURCES_PATH + TEST_TEXT_OBJECT_PREFIX + TEST_TEXT_FILE_EXTENSION, 'rb')
                          .read()).hexdigest()

        self.assertIsNotNone(response, "Object could not be created")
        self.test_world['swift_objects'].append(containerName + "/" + textObjectName)
        self.logger.debug("Created %s object was created", textObjectName)

        # Downloading the object
        response = self.swift_operations.get_object(containerName, textObjectName,
                                                         SWIFT_RESOURCES_PATH)
        self.assertTrue(response, "Object could not be downloaded")
        self.test_world['local_objects'].append(textObjectName)

        remote = hashlib.md5(open(SWIFT_RESOURCES_PATH + textObjectName, 'rb')
                          .read()).hexdigest()

        # Checking original file with remote file
        try:
            self.assertEqual(origin, remote, "The original file and the downloaded file are different")
        except AssertionError as e:
            self.logger.error("Object uploaded and object downloaded are different (%s)", textObjectName)
            self.fail(e)

    def test_delete_an_object_from_a_container(self):
        """
        Test whether it is possible to delete an object from a container.
        """

        suffix = datetime.utcnow().strftime('%Y%m%d%H%M%S%m')
        containerName = TEST_CONTAINER_PREFIX + suffix
        textObjectName = self.region_name + TEST_TEXT_OBJECT_PREFIX + suffix + TEST_TEXT_FILE_EXTENSION

        response = self.swift_operations.create_container(containerName)
        self.assertIsNone(response, "Container could not be created")
        self.test_world['containers'].append(containerName)
        self.logger.debug("Created %s container was created", containerName)

        # Uploading the object

        response = self.swift_operations.create_object(containerName, SWIFT_RESOURCES_PATH
                                                            + TEST_TEXT_OBJECT_PREFIX + TEST_TEXT_FILE_EXTENSION,
                                                            textObjectName)
        self.assertIsNotNone(response, "Object could not be created")
        self.test_world['swift_objects'].append(containerName + "/" + textObjectName)
        self.logger.debug("Created %s object was created", textObjectName)

        # Cleaning the object
        response = self.swift_operations.delete_object(containerName, textObjectName)
        self.assertIsNone(response, "Container could not be deleted")
        self.test_world['swift_objects'].remove(containerName + "/" + textObjectName)

        try:
            self.swift_operations.get_object(containerName, textObjectName,
                                                             SWIFT_RESOURCES_PATH)
        except SwiftClientException as e:
            self.assertRaises(e)
            self.logger.debug("%s object was successfully removed from the object storage", textObjectName)

    def test_create_big_object_and_download_it_from_container(self):
        """
        Test whether it is possible to upload a big file and download it (More than 5Mb).
        """

        suffix = datetime.utcnow().strftime('%Y%m%d%H%M%S%m')
        containerName = self.region_name + TEST_CONTAINER_PREFIX + suffix
        big_object_name = self.region_name + TEST_BIG_OBJECT_PREFIX + suffix + TEST_BIG_FILE_EXTENSION
        remote_big_object_name = self.region_name + TEST_BIG_OBJECT_REMOTE_PREFIX + TEST_BIG_OBJECT_PREFIX + suffix \
                                 + TEST_BIG_FILE_EXTENSION

        response = self.swift_operations.create_container(containerName)
        self.assertIsNone(response, "Container could not be created")
        self.test_world['containers'].append(containerName)
        self.logger.debug("Created %s container was created", containerName)

        # Creating a new big object
        big_file_url_1 = self.conf[PROPERTIES_CONFIG_TEST][PROPERTIES_CONFIG_SWIFT][PROPERTIES_CONFIG_SWIFT_BIG_FILE_1]
        f = urllib2.urlopen(big_file_url_1)
        data = f.read()
        try:
            with open(SWIFT_RESOURCES_PATH + big_object_name, "wb") as code:
                code.write(data)
        except IOError:
            self.logger.error("Error while writing %s file", SWIFT_RESOURCES_PATH + big_object_name)

        self.test_world['local_objects'].append(big_object_name)

        # Uploading the object
        response = self.swift_operations.create_object(containerName, SWIFT_RESOURCES_PATH
                                                            + big_object_name,
                                                            remote_big_object_name)
        originHash = hashlib.md5(open(SWIFT_RESOURCES_PATH + big_object_name, 'rb')
                          .read()).hexdigest()

        self.assertIsNotNone(response, "Object could not be created")
        self.test_world['swift_objects'].append(containerName + "/" + big_object_name)
        self.logger.debug("Created %s object was created", big_object_name)

        # Downloading the object
        response = self.swift_operations.get_object(containerName, remote_big_object_name,
                                                         SWIFT_RESOURCES_PATH)
        self.assertTrue(response, "Object could not be downloaded")
        self.test_world['local_objects'].append(remote_big_object_name)

        remoteHash = hashlib.md5(open(SWIFT_RESOURCES_PATH + remote_big_object_name, 'rb').read()).hexdigest()

        # Checking original file with remote file
        self.assertEqual(originHash, remoteHash, "The original file and the downloaded file are different")

        # Downloading a second big file to check that is different
        suffix2 = datetime.utcnow().strftime('%Y%m%d%H%M%S%m')
        big_file_url_2 = self.conf[PROPERTIES_CONFIG_TEST][PROPERTIES_CONFIG_SWIFT][PROPERTIES_CONFIG_SWIFT_BIG_FILE_2]
        f2 = urllib2.urlopen(big_file_url_2)
        data = f2.read()
        try:
            with open(SWIFT_RESOURCES_PATH + TEST_BIG_OBJECT_PREFIX + suffix2 + TEST_BIG_FILE_EXTENSION, "wb") as code2:
                code2.write(data)
        except IOError:
            self.logger.error("Error while writing %s file", SWIFT_RESOURCES_PATH + TEST_BIG_OBJECT_PREFIX +
                              suffix2 + TEST_BIG_FILE_EXTENSION)

        self.test_world['local_objects'].append(TEST_BIG_OBJECT_PREFIX + suffix2 + TEST_BIG_FILE_EXTENSION)

        # Checking that two different files generates are different
        origin2 = hashlib.md5(open(SWIFT_RESOURCES_PATH + TEST_BIG_OBJECT_PREFIX + suffix2 + TEST_BIG_FILE_EXTENSION,
                                   'rb')
                              .read()).hexdigest()

        self.assertNotEqual(origin2, remoteHash, "The second file and the downloaded file are the same, "
                                             "it cannot be possible")
<commit_msg>fix: test: big files are downloaded just in the first execution to prevent others could remove it in future executions.<commit_after># -*- coding: utf-8 -*-

# Copyright 2015 Telefonica Investigación y Desarrollo, S.A.U
#
# This file is part of FIWARE project.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
#
# You may obtain a copy of the License at:
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#
# See the License for the specific language governing permissions and
# limitations under the License.
#
# For those usages not covered by the Apache version 2.0 License please
# contact with opensource@tid.es

__author__ = 'gjp'

from tests.fiware_region_base_tests import FiwareRegionsBaseTests
from commons.constants import TEST_CONTAINER_PREFIX, TEST_TEXT_OBJECT_PREFIX, TEST_TEXT_FILE_EXTENSION, \
    SWIFT_RESOURCES_PATH, TEST_BIG_OBJECT_PREFIX, TEST_BIG_FILE_EXTENSION, PROPERTIES_CONFIG_SWIFT,\
    PROPERTIES_CONFIG_SWIFT_BIG_FILE_1, PROPERTIES_CONFIG_SWIFT_BIG_FILE_2, PROPERTIES_CONFIG_TEST,\
    TEST_BIG_OBJECT_REMOTE_PREFIX
from swiftclient.exceptions import ClientException as SwiftClientException
from datetime import datetime
import hashlib
import urllib2
import os

class FiwareRegionsObjectStorageTests(FiwareRegionsBaseTests):

    with_storage = True

    def test_create_container(self):
        """
        Test whether it is possible to create a new container into the object storage.
        """

        suffix = datetime.utcnow().strftime('%Y%m%d%H%M%S')
        containerName = TEST_CONTAINER_PREFIX + suffix

        response = self.swift_operations.create_container(containerName)

        self.assertIsNone(response, "Container could not be created")
        self.test_world['containers'].append(containerName)
        self.logger.debug("Created %s container was created", containerName)

        response = self.swift_operations.get_container(containerName)
        self.assertEquals('x-container-object-count' in response[0], True, "There is no container header in response")
        self.assertEquals(len(response[-1]), 0, "The container is not empty")  # The list of items should be 0.
        self.logger.debug("Getting %s container details from the object storage", containerName)

    def test_delete_container(self):
        """
        Test whether it is possible to delete a container.
        """
        suffix = datetime.utcnow().strftime('%Y%m%d%H%M%S')
        containerName = TEST_CONTAINER_PREFIX + suffix

        response = self.swift_operations.create_container(containerName)
        self.assertIsNone(response, "Container could not be created")
        self.test_world['containers'].append(containerName)

        response = self.swift_operations.delete_container(containerName)
        self.assertIsNone(response, "Container could not be deleted")
        self.test_world['containers'].remove(containerName)

        try:
            self.swift_operations.get_container(containerName)
        except SwiftClientException as e:
            self.assertRaises(e)
            self.logger.debug("%s container was successfully removed from the object storage", containerName)

    def test_create_text_object_and_download_it_from_container(self):
        """
        Test whether it is possible to upload a text file and download it.
        """

        suffix = datetime.utcnow().strftime('%Y%m%d%H%M%S%m')
        containerName = TEST_CONTAINER_PREFIX + suffix
        textObjectName = self.region_name + TEST_TEXT_OBJECT_PREFIX + suffix + TEST_TEXT_FILE_EXTENSION

        response = self.swift_operations.create_container(containerName)
        self.assertIsNone(response, "Container could not be created")
        self.test_world['containers'].append(containerName)
        self.logger.debug("Created %s container was created", containerName)

        # Uploading the object
        response = self.swift_operations.create_object(containerName, SWIFT_RESOURCES_PATH
                                                            + TEST_TEXT_OBJECT_PREFIX + TEST_TEXT_FILE_EXTENSION,
                                                            textObjectName)
        origin = hashlib.md5(open(SWIFT_RESOURCES_PATH + TEST_TEXT_OBJECT_PREFIX + TEST_TEXT_FILE_EXTENSION, 'rb')
                          .read()).hexdigest()

        self.assertIsNotNone(response, "Object could not be created")
        self.test_world['swift_objects'].append(containerName + "/" + textObjectName)
        self.logger.debug("Created %s object was created", textObjectName)

        # Downloading the object
        response = self.swift_operations.get_object(containerName, textObjectName,
                                                         SWIFT_RESOURCES_PATH)
        self.assertTrue(response, "Object could not be downloaded")
        self.test_world['local_objects'].append(textObjectName)

        remote = hashlib.md5(open(SWIFT_RESOURCES_PATH + textObjectName, 'rb')
                          .read()).hexdigest()

        # Checking original file with remote file
        try:
            self.assertEqual(origin, remote, "The original file and the downloaded file are different")
        except AssertionError as e:
            self.logger.error("Object uploaded and object downloaded are different (%s)", textObjectName)
            self.fail(e)

    def test_delete_an_object_from_a_container(self):
        """
        Test whether it is possible to delete an object from a container.
        """

        suffix = datetime.utcnow().strftime('%Y%m%d%H%M%S%m')
        containerName = TEST_CONTAINER_PREFIX + suffix
        textObjectName = self.region_name + TEST_TEXT_OBJECT_PREFIX + suffix + TEST_TEXT_FILE_EXTENSION

        response = self.swift_operations.create_container(containerName)
        self.assertIsNone(response, "Container could not be created")
        self.test_world['containers'].append(containerName)
        self.logger.debug("Created %s container was created", containerName)

        # Uploading the object

        response = self.swift_operations.create_object(containerName, SWIFT_RESOURCES_PATH
                                                            + TEST_TEXT_OBJECT_PREFIX + TEST_TEXT_FILE_EXTENSION,
                                                            textObjectName)
        self.assertIsNotNone(response, "Object could not be created")
        self.test_world['swift_objects'].append(containerName + "/" + textObjectName)
        self.logger.debug("Created %s object was created", textObjectName)

        # Cleaning the object
        response = self.swift_operations.delete_object(containerName, textObjectName)
        self.assertIsNone(response, "Container could not be deleted")
        self.test_world['swift_objects'].remove(containerName + "/" + textObjectName)

        try:
            self.swift_operations.get_object(containerName, textObjectName,
                                                             SWIFT_RESOURCES_PATH)
        except SwiftClientException as e:
            self.assertRaises(e)
            self.logger.debug("%s object was successfully removed from the object storage", textObjectName)

    def test_create_big_object_and_download_it_from_container(self):
        """
        Test whether it is possible to upload a big file and download it (More than 5Mb).
        """

        suffix = datetime.utcnow().strftime('%Y%m%d%H%M%S%m')
        containerName = self.region_name + TEST_CONTAINER_PREFIX + suffix
        big_object_name = TEST_BIG_OBJECT_PREFIX + TEST_BIG_FILE_EXTENSION
        remote_big_object_name = self.region_name + TEST_BIG_OBJECT_REMOTE_PREFIX + TEST_BIG_OBJECT_PREFIX + suffix \
                                 + TEST_BIG_FILE_EXTENSION

        response = self.swift_operations.create_container(containerName)
        self.assertIsNone(response, "Container could not be created")
        self.test_world['containers'].append(containerName)
        self.logger.debug("Created %s container was created", containerName)

        # Creating a new big object if do not exits
        if not os.path.isfile(SWIFT_RESOURCES_PATH + big_object_name):
            big_file_url_1 = \
                self.conf[PROPERTIES_CONFIG_TEST][PROPERTIES_CONFIG_SWIFT][PROPERTIES_CONFIG_SWIFT_BIG_FILE_1]
            f = urllib2.urlopen(big_file_url_1)
            data = f.read()

            try:
                with open(SWIFT_RESOURCES_PATH + big_object_name, "wb") as code:
                        code.write(data)
            except IOError:
                self.logger.error("Error while writing %s file", SWIFT_RESOURCES_PATH + big_object_name)

        # Uploading the object
        response = self.swift_operations.create_object(containerName, SWIFT_RESOURCES_PATH
                                                            + big_object_name,
                                                            remote_big_object_name)
        originHash = hashlib.md5(open(SWIFT_RESOURCES_PATH + big_object_name, 'rb')
                          .read()).hexdigest()

        self.assertIsNotNone(response, "Object could not be created")
        self.test_world['swift_objects'].append(containerName + "/" + remote_big_object_name)
        self.logger.debug("Created %s object with name %s", big_object_name, remote_big_object_name)

        # Downloading the object
        response = self.swift_operations.get_object(containerName, remote_big_object_name,
                                                         SWIFT_RESOURCES_PATH)
        self.assertTrue(response, "Object could not be downloaded")
        self.test_world['local_objects'].append(remote_big_object_name)

        remoteHash = hashlib.md5(open(SWIFT_RESOURCES_PATH + remote_big_object_name, 'rb').read()).hexdigest()

        # Checking original file with remote file
        self.assertEqual(originHash, remoteHash, "The original file and the downloaded file are different")

        # Downloading a second big file to check that is different
        if not os.path.isfile(SWIFT_RESOURCES_PATH + big_object_name):
            suffix2 = "second_file"
            big_file_url_2 = \
                self.conf[PROPERTIES_CONFIG_TEST][PROPERTIES_CONFIG_SWIFT][PROPERTIES_CONFIG_SWIFT_BIG_FILE_2]
            f2 = urllib2.urlopen(big_file_url_2)
            data = f2.read()
            try:
                with open(SWIFT_RESOURCES_PATH + TEST_BIG_OBJECT_PREFIX + suffix2 +
                                  TEST_BIG_FILE_EXTENSION, "wb") as code2:
                    code2.write(data)
            except IOError:
                self.logger.error("Error while writing %s file", SWIFT_RESOURCES_PATH + TEST_BIG_OBJECT_PREFIX +
                                  suffix2 + TEST_BIG_FILE_EXTENSION)

        # Checking that two different files generates are different
        origin2 = hashlib.md5(open(SWIFT_RESOURCES_PATH + TEST_BIG_OBJECT_PREFIX + suffix2 + TEST_BIG_FILE_EXTENSION,
                                   'rb')
                              .read()).hexdigest()

        self.assertNotEqual(origin2, remoteHash, "The second file and the downloaded file are the same, "
                                             "it cannot be possible")
<|endoftext|>


<commit_before>from flask import Flask
from flask import request
app = Flask(__name__)

@app.route("/")
def hello():
    return "Hello World!"

@app.route('/listfiles')
def listfiles():
    path = "./uploads/"
    dirs = os.listdir( path )
    f_str=""
    for name in dirs:
        f_str +=(str)(name)+"\n"
    return f_str

@app.route('/upload', methods=['GET', 'POST'])
def upload_file():
    if request.method == 'POST':
        f = request.files['file']
        f.save('./uploads/'+f.filename)
    return '',201
if __name__ == "__main__":
    app.run(host="0.0.0.0", debug=True)
<commit_msg>Update app.py<commit_after>from flask import Flask
from flask import request
app = Flask(__name__)

@app.route("/")
def hello():
    return "Hello World!"

@app.route('/listfiles')
def listfiles():
    path = "./upload/"
    dirs = os.listdir( path )
    f_str=""
    for name in dirs:
        f_str +=(str)(name)+"\n"
    return f_str

@app.route('/upload', methods=['GET', 'POST'])
def upload_file():
    if request.method == 'POST':
        f = request.files['file']
        f.save('./uploads/'+f.filename)
    return '',201
if __name__ == "__main__":
    app.run(host="0.0.0.0", debug=True)
<|endoftext|>


<commit_before>#!/usr/bin/env python
# Copyright Datacratic 2016
# Author: Jean Raby <jean@datacratic.com>

# Wrapper around tee -a
# Read from stdin and write to logfile and to stdout

import os
tee = "/usr/bin/tee"
os.execv(tee,[tee, '-a', '{{MLDB_LOGFILE}}'])

<commit_msg>First stab at a tornado based logger<commit_after>#!/usr/bin/env python
# Copyright Datacratic 2016
# Author: Jean Raby <jean@datacratic.com>

# Wrapper around tee -a
# Read from stdin and write to logfile and to stdout

import functools
import os
import fcntl
import sys
import time

import tornado.web

from tornado.ioloop import IOLoop
from datetime import datetime
from collections import namedtuple, deque

RINGBUFSIZE = 1024

LogLine = namedtuple('LogLine', ['dt', 'data', ])

logline_cnt = 0

def stdin_ready(f, ringbuf, fd, events):
  global logline_cnt
  if events & IOLoop.READ:
    try:
      for line in f:
        logline = LogLine(dt=datetime.utcnow(), data=line)
        ringbuf.append(logline)
        logline_cnt += 1
        sys.stdout.write(line)
    except IOError:
      # If we get a EWOULDBLOCK, continue
      # EOF handled below
      pass
  if events & IOLoop.ERROR:
      exit(0)

class LogsMldbHandler(tornado.web.RequestHandler):
  def get(self):
    self.write('
\n')
    for l in log_lines_ringbuf:
      self.write("%s %s" % (l.dt.isoformat(), l.data))
    self.write('
\n')

if __name__ == "__main__":

  log_lines_ringbuf = deque(maxlen=RINGBUFSIZE)
  io_loop = IOLoop.current()

  # set stdin to non blocking mode for use with tornado
  fl = fcntl.fcntl(sys.stdin.fileno(), fcntl.F_GETFL)
  fcntl.fcntl(sys.stdin.fileno(), fcntl.F_SETFL, fl | os.O_NONBLOCK)

  callback = functools.partial(stdin_ready, sys.stdin, log_lines_ringbuf)
  io_loop.add_handler(sys.stdin.fileno(), callback,
                      io_loop.READ | io_loop.ERROR)

  app = tornado.web.Application([ ("/logs/mldb", LogsMldbHandler) ])
  app.listen(12346, '0.0.0.0')

  try:
    t1 = time.time()
    io_loop.start()
  except KeyboardInterrupt:
    total_time = time.time() - t1
    sys.stderr.write("Got %d lines in %d sec: %f lines/s\n" % (logline_cnt, total_time, logline_cnt/total_time))
    raise
<|endoftext|>


<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-

from setuptools import setup, find_packages

with open("README.md", "r") as fh:
    long_description = fh.read()

requirements = ['python-jose', 'everett', 'boto3', 'requests']

setup_requirements = ['pytest-runner']

test_requirements = ['pytest', 'pytest-watch', 'pytest-cov', 'pytest-mock']

setup(
    name="cis_crypto",
    version="0.0.1",
    author="Andrew Krug",
    author_email="akrug@mozilla.com",
    description="Per attribute signature system for jwks sign-verify in mozill-iam.",
    long_description=long_description,
    url="https://github.com/mozilla-iam/cis",
    classifiers=[
        "Programming Language :: Python :: 3",
        "License :: OSI Approved :: Mozilla Public License",
        "Operating System :: OS Independent",
    ],
    install_requires=requirements,
    license="Mozilla Public License 2.0",
    include_package_data=True,
    packages=find_packages(include=['cis_crypto', 'bin']),
    scripts=['bin/cis_crypto'],
    setup_requires=setup_requirements,
    test_suite='tests',
    tests_require=test_requirements,
    zip_safe=False
)
<commit_msg>Add missing requirements Note: this still is missing `moto` but there's a conflict between it and `botocore` (req by `boto3`) so I didn't add it<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-

from setuptools import setup, find_packages

with open("README.md", "r") as fh:
    long_description = fh.read()

requirements = ['python-jose', 'everett', 'boto3', 'requests', 'pyaml']

setup_requirements = ['pytest-runner']

test_requirements = ['pytest', 'pytest-watch', 'pytest-cov', 'pytest-mock', 'mock']

setup(
    name="cis_crypto",
    version="0.0.1",
    author="Andrew Krug",
    author_email="akrug@mozilla.com",
    description="Per attribute signature system for jwks sign-verify in mozill-iam.",
    long_description=long_description,
    url="https://github.com/mozilla-iam/cis",
    classifiers=[
        "Programming Language :: Python :: 3",
        "License :: OSI Approved :: Mozilla Public License",
        "Operating System :: OS Independent",
    ],
    install_requires=requirements,
    license="Mozilla Public License 2.0",
    include_package_data=True,
    packages=find_packages(include=['cis_crypto', 'bin']),
    scripts=['bin/cis_crypto'],
    setup_requires=setup_requirements,
    test_suite='tests',
    tests_require=test_requirements,
    zip_safe=False
)
<|endoftext|>


<commit_before>import json
import tweepy
from tweepy import Stream
from tweepy.streaming import StreamListener
from tweepy import OAuthHandler
import pandas as pd
import pickle
import sys

from elasticsearch import Elasticsearch

es = Elasticsearch([{'host': 'localhost', 'port': 9200}])

#Variables para poner las llaves y tokens para acceso a la API
consumer_key = ''
consumer_secret = ''
access_token = ''
access_secret = ''

auth = OAuthHandler(consumer_key, consumer_secret)
auth.set_access_token(access_token, access_secret)

api = tweepy.API(auth)
datos_tweets={}
datos_tweets.setdefault('date',[])
datos_tweets.setdefault('texts',[])
datos_tweets.setdefault('user_id',[])
datos_tweets.setdefault('retweet_count',[])

args=sys.argv

#print(place_id)
for tweet in tweepy.Cursor(api.search, q=args[1], lang="es", since=args[2], until=args[3]).items():
     if not hasattr(tweet,'retweeted_status'):
         datos_tweets['date'].append(tweet.created_at.isoformat())
         datos_tweets['texts'].append(tweet.text)
         datos_tweets['user_id'].append(tweet.user.id)
         datos_tweets['retweet_count'].append(tweet.retweet_count)

es.index(index='skynet_beeva', doc_type=args[1], id=args[2], body= datos_tweets)
es.get(index='skynet_beeva', doc_type=args[1], id=args[2])

ela_bancomer = es.get(index='skynet_beeva', doc_type=args[1], id=args[2])
#Añadir la ruta absoluta a datoJ para ejecutar con el cron
with open('datoJ/'+args[1]+args[2]+'.json', 'w') as fp:
         json.dump(ela_bancomer, fp)
<commit_msg>añadir credenciales<commit_after>import json
import tweepy
from tweepy import Stream
from tweepy.streaming import StreamListener
from tweepy import OAuthHandler
import pandas as pd
import pickle
import sys

from elasticsearch import Elasticsearch

es = Elasticsearch([{'host': 'localhost', 'port': 9200}])

#Variables para poner las credenciales para la API
consumer_key = ''
consumer_secret = ''
access_token = ''
access_secret = ''

auth = OAuthHandler(consumer_key, consumer_secret)
auth.set_access_token(access_token, access_secret)

api = tweepy.API(auth)
datos_tweets={}
datos_tweets.setdefault('date',[])
datos_tweets.setdefault('texts',[])
datos_tweets.setdefault('user_id',[])
datos_tweets.setdefault('retweet_count',[])

args=sys.argv

#print(place_id)
for tweet in tweepy.Cursor(api.search, q=args[1], lang="es", since=args[2], until=args[3]).items():
     if not hasattr(tweet,'retweeted_status'):
         datos_tweets['date'].append(tweet.created_at.isoformat())
         datos_tweets['texts'].append(tweet.text)
         datos_tweets['user_id'].append(tweet.user.id)
         datos_tweets['retweet_count'].append(tweet.retweet_count)

es.index(index='skynet_beeva', doc_type=args[1], id=args[2], body= datos_tweets)
es.get(index='skynet_beeva', doc_type=args[1], id=args[2])

ela_bancomer = es.get(index='skynet_beeva', doc_type=args[1], id=args[2])
#Añadir la ruta absoluta a datoJ para ejecutar con el cron
with open('datoJ/'+args[1]+args[2]+'.json', 'w') as fp:
         json.dump(ela_bancomer, fp)
<|endoftext|>


<commit_before>"""Developer tools for MooseDocs."""
import os
import collections
import logging

import anytree

import MooseDocs
from MooseDocs import common
from MooseDocs.tree import syntax
from MooseDocs.common import exceptions

LOG = logging.getLogger(__name__)
STUB_HEADER = '<!-- MOOSE Documentation Stub: Remove this when content is added. -->'

def command_line_options(subparser, parent):
    """Define the 'check' command."""

    parser = subparser.add_parser('check',
                                  parents=[parent],
                                  help='Syntax checking tools for MooseDocs.')

    parser.add_argument('--config', default='config.yml', help="The configuration file.")
    parser.add_argument('--generate', action='store_true',
                        help="When checking the application for complete documentation generate "
                             "any missing markdown documentation files.")
    parser.add_argument('--update', action='store_true',
                        help="When checking the application for complete documentation generate "
                             "any missing markdown documentation files and update the stubs for "
                             "files that have not been modified.")
    parser.add_argument('--dump', action='store_true',
                        help="Dump the complete MooseDocs syntax tree to the screen.")
    parser.add_argument('--object_prefix',
                        default=os.path.join('doc', 'content', 'source'),
                        help="The folder install prefix for MooseObject documentations pages.")
    parser.add_argument('--syntax_prefix',
                        default=os.path.join('doc', 'content', 'syntax'),
                        help="The folder install prefix for MooseObject documentations pages.")

def main(options):
    """./moosedocs check"""

    translator, _ = common.load_config(options.config)
    check(translator,
          dump=options.dump,
          update=options.update,
          generate=options.generate,
          object_prefix=options.object_prefix,
          syntax_prefix=options.syntax_prefix)

def check(translator,
          dump=False,
          update=False,
          generate=False,
          object_prefix=os.path.join('doc', 'content', 'source'),
          syntax_prefix=os.path.join('doc', 'content', 'syntax')):
    """Helper to all both main and build.py:main to perform check."""

    # Extract the syntax root node
    app_syntax = None
    extension = None
    for ext in translator.extensions:
        extension = ext
        if isinstance(ext, MooseDocs.extensions.appsyntax.AppSyntaxExtension):
            app_syntax = ext.syntax
            break

    if extension['disable']:
        LOG.info("Syntax is disabled, skipping the check.")
        return

    if app_syntax is None:
        msg = "Failed to locate AppSyntaxExtension for the given configuration."
        raise exceptions.MooseDocsException(msg)

    # Dump the complete syntax for the application
    if dump:
        print app_syntax

    # Perform check for all the nodes
    for node in anytree.PreOrderIter(app_syntax):
        if node.is_root or node.removed:# or node.hidden:# or node.test:
            continue
        elif isinstance(node, syntax.ObjectNode):
            _check_object_node(node, generate, update, object_prefix)
        elif isinstance(node, syntax.SyntaxNode):
            _check_syntax_node(node, generate, update, syntax_prefix)
        else:
            LOG.critical("Unexpected object type of %s, only %s and %s based types are supported",
                         type(node).__name__,
                         syntax.ObjectNode.__name__,
                         syntax.SyntaxNode.__name__)

def _check_object_node(node, generate, update, prefix):
    """
    Check that required pages for supplied ObjectNode (i.e., MooseObject/Action).
    """
    idx = node.source().find('/src/')
    filename = os.path.join(node.source()[:idx], prefix, node.markdown())

    not_exist = not os.path.isfile(filename)
    if not_exist and (not generate) and (not node.hidden):
        msg = "No documentation for %s.\n"
        msg += "    - The page should be located at %s.\n"
        msg += "    - It is possible to generate stub pages using " \
               "'./moosedocs.py check --generate'."
        LOG.error(msg, node.fullpath, filename)

    elif not_exist and generate:
        if not os.path.exists(os.path.dirname(filename)):
            os.makedirs(os.path.dirname(filename))
        LOG.info('Creating stub page for %s at %s.', node.fullpath, filename)
        with open(filename, 'w') as fid:
            content = _default_content(node)
            fid.write(content)

    elif not not_exist:
        _check_page_for_stub(node, filename, update)

def _check_syntax_node(node, generate, update, prefix):
    """
    Check that required pages for syntax exists (e.g., Adaptivity/index.md).
    """

    # Tuple for storing filename and existence
    FileInfo = collections.namedtuple('FileInfo', 'name exists') #pylint: disable=invalid-name

    # Build a set if information tuples to consider
    filenames = set()
    actions = anytree.search.PreOrderIter(node, filter_=lambda n: isinstance(n, syntax.ActionNode))
    for action in actions:
        idx = action.source().find('/src/')
        name = os.path.join(action.source()[:idx], prefix,
                            os.path.dirname(node.markdown()), 'index.md')
        filenames.add(FileInfo(name=name, exists=os.path.isfile(name)))

    # Case when no file exists
    not_exist = all(not f.exists for f in filenames)
    if not_exist and (not generate) and (not node.hidden):
        msg = "No documentation for {}.\n".format(node.fullpath)
        if len(filenames) == 1:
            msg += "    - The page should be located at {}.\n".format(filenames.pop().name)
            msg += "    - It is possible to generate stub pages using " \
                   "'./moosedocs.py check --generate'."
        else:
            msg += "    - The page should be located at one of the following locations:\n"
            for info in filenames:
                msg += "      {}\n".format(info.name)

        LOG.error(msg)

    # Case when when no file exists but --generate was provided
    elif not_exist and generate:
        if len(filenames) == 1:
            fname = filenames.pop().name
            LOG.info("Creating stub page for %s at %s.", node.fullpath, fname)
            if not os.path.exists(os.path.dirname(fname)):
                os.makedirs(os.path.dirname(fname))
            with open(fname, 'w') as fid:
                content = _default_content(node)
                fid.write(content)
        else:
            msg = "A stub page for {} cannot be created because the syntax definition location " \
                  "exists in multiple applications, the file can be located at any of the " \
                  "following locations.\n".format(node.fullpath)
            for info in filenames:
                msg += "      {}\n".format(info.name)
            LOG.error(msg)

    else:
        for info in filenames:
            if info.exists:
                _check_page_for_stub(node, info.name, update)

def _check_page_for_stub(node, filename, update):
    """
    Helper method to check if a page is a stub.
    """
    with open(filename, 'r') as fid:
        lines = fid.readlines()

    if lines and STUB_HEADER in lines[0]:
        if update:
            LOG.info("Updating stub page for %s in file %s.", node.fullpath, filename)
            with open(filename, 'w') as fid:
                content = _default_content(node)
                fid.write(content)
        elif not node.hidden:
            msg = "A MOOSE generated stub page for %s exists, but no content was " \
                  "added. Add documentation content to %s."
            LOG.error(msg, node.fullpath, filename)

def _default_content(node):
    """
    Markdown stub content.
    """
    stub = STUB_HEADER + '\n\n'
    if isinstance(node, syntax.SyntaxNode):
        stub += '# {} System\n\n'.format(node.name)
        stub += '!syntax list {} objects=True actions=False subsystems=False\n\n' \
                .format(node.fullpath)
        stub += '!syntax list {} objects=False actions=False subsystems=True\n\n' \
                .format(node.fullpath)
        stub += '!syntax list {} objects=False actions=True subsystems=False\n\n' \
                .format(node.fullpath)

    elif isinstance(node, syntax.MooseObjectNode):
        template_filename = os.path.join(MooseDocs.MOOSE_DIR, 'framework', 'doc', 'templates',
                                         'moose_object.md.template')
        with open(template_filename, 'r') as fid:
            template_content = fid.read()

        template_content = template_content.replace('FullPathCodeClassName',
                                                    '{}'.format(node.fullpath))
        template_content = template_content.replace('CodeClassName', '{}'.format(node.name))
        stub += template_content

    elif isinstance(node, syntax.ActionNode):
        template_filename = os.path.join(MooseDocs.MOOSE_DIR, 'framework', 'doc', 'templates',
                                         'action_object.md.template')
        with open(template_filename, 'r') as fid:
            template_content = fid.read()

        template_content = template_content.replace('FullPathCodeActionName',
                                                    '{}'.format(node.fullpath))
        template_content = template_content.replace('CodeActionName', '{}'.format(node.name))
        stub += template_content

    return stub
<commit_msg>Handle error in syntax node markdown method when running check<commit_after>"""Developer tools for MooseDocs."""
import os
import collections
import logging

import anytree

import MooseDocs
from MooseDocs import common
from MooseDocs.tree import syntax
from MooseDocs.common import exceptions

LOG = logging.getLogger(__name__)
STUB_HEADER = '<!-- MOOSE Documentation Stub: Remove this when content is added. -->'

def command_line_options(subparser, parent):
    """Define the 'check' command."""

    parser = subparser.add_parser('check',
                                  parents=[parent],
                                  help='Syntax checking tools for MooseDocs.')

    parser.add_argument('--config', default='config.yml', help="The configuration file.")
    parser.add_argument('--generate', action='store_true',
                        help="When checking the application for complete documentation generate "
                             "any missing markdown documentation files.")
    parser.add_argument('--update', action='store_true',
                        help="When checking the application for complete documentation generate "
                             "any missing markdown documentation files and update the stubs for "
                             "files that have not been modified.")
    parser.add_argument('--dump', action='store_true',
                        help="Dump the complete MooseDocs syntax tree to the screen.")
    parser.add_argument('--object_prefix',
                        default=os.path.join('doc', 'content', 'source'),
                        help="The folder install prefix for MooseObject documentations pages.")
    parser.add_argument('--syntax_prefix',
                        default=os.path.join('doc', 'content', 'syntax'),
                        help="The folder install prefix for MooseObject documentations pages.")

def main(options):
    """./moosedocs check"""

    translator, _ = common.load_config(options.config)
    check(translator,
          dump=options.dump,
          update=options.update,
          generate=options.generate,
          object_prefix=options.object_prefix,
          syntax_prefix=options.syntax_prefix)

def check(translator,
          dump=False,
          update=False,
          generate=False,
          object_prefix=os.path.join('doc', 'content', 'source'),
          syntax_prefix=os.path.join('doc', 'content', 'syntax')):
    """Helper to all both main and build.py:main to perform check."""

    # Extract the syntax root node
    app_syntax = None
    extension = None
    for ext in translator.extensions:
        extension = ext
        if isinstance(ext, MooseDocs.extensions.appsyntax.AppSyntaxExtension):
            app_syntax = ext.syntax
            break

    if extension['disable']:
        LOG.info("Syntax is disabled, skipping the check.")
        return

    if app_syntax is None:
        msg = "Failed to locate AppSyntaxExtension for the given configuration."
        raise exceptions.MooseDocsException(msg)

    # Dump the complete syntax for the application
    if dump:
        print app_syntax

    # Perform check for all the nodes
    for node in anytree.PreOrderIter(app_syntax):
        if node.is_root or node.removed:# or node.hidden:# or node.test:
            continue
        elif isinstance(node, syntax.ObjectNode):
            _check_object_node(node, generate, update, object_prefix)
        elif isinstance(node, syntax.SyntaxNode):
            _check_syntax_node(node, generate, update, syntax_prefix)
        else:
            LOG.critical("Unexpected object type of %s, only %s and %s based types are supported",
                         type(node).__name__,
                         syntax.ObjectNode.__name__,
                         syntax.SyntaxNode.__name__)

def _check_object_node(node, generate, update, prefix):
    """
    Check that required pages for supplied ObjectNode (i.e., MooseObject/Action).
    """
    idx = node.source().find('/src/')

    # If the markdown method returns None, it failed to locate the file, which throws an error
    md = node.markdown()
    if md is None:
        return
    filename = os.path.join(node.source()[:idx], prefix, md)

    not_exist = not os.path.isfile(filename)
    if not_exist and (not generate) and (not node.hidden):
        msg = "No documentation for %s.\n"
        msg += "    - The page should be located at %s.\n"
        msg += "    - It is possible to generate stub pages using " \
               "'./moosedocs.py check --generate'."
        LOG.error(msg, node.fullpath, filename)

    elif not_exist and generate:
        if not os.path.exists(os.path.dirname(filename)):
            os.makedirs(os.path.dirname(filename))
        LOG.info('Creating stub page for %s at %s.', node.fullpath, filename)
        with open(filename, 'w') as fid:
            content = _default_content(node)
            fid.write(content)

    elif not not_exist:
        _check_page_for_stub(node, filename, update)

def _check_syntax_node(node, generate, update, prefix):
    """
    Check that required pages for syntax exists (e.g., Adaptivity/index.md).
    """

    # Tuple for storing filename and existence
    FileInfo = collections.namedtuple('FileInfo', 'name exists') #pylint: disable=invalid-name

    # Build a set if information tuples to consider
    filenames = set()
    actions = anytree.search.PreOrderIter(node, filter_=lambda n: isinstance(n, syntax.ActionNode))
    for action in actions:
        idx = action.source().find('/src/')
        name = os.path.join(action.source()[:idx], prefix,
                            os.path.dirname(node.markdown()), 'index.md')
        filenames.add(FileInfo(name=name, exists=os.path.isfile(name)))

    # Case when no file exists
    not_exist = all(not f.exists for f in filenames)
    if not_exist and (not generate) and (not node.hidden):
        msg = "No documentation for {}.\n".format(node.fullpath)
        if len(filenames) == 1:
            msg += "    - The page should be located at {}.\n".format(filenames.pop().name)
            msg += "    - It is possible to generate stub pages using " \
                   "'./moosedocs.py check --generate'."
        else:
            msg += "    - The page should be located at one of the following locations:\n"
            for info in filenames:
                msg += "      {}\n".format(info.name)

        LOG.error(msg)

    # Case when when no file exists but --generate was provided
    elif not_exist and generate:
        if len(filenames) == 1:
            fname = filenames.pop().name
            LOG.info("Creating stub page for %s at %s.", node.fullpath, fname)
            if not os.path.exists(os.path.dirname(fname)):
                os.makedirs(os.path.dirname(fname))
            with open(fname, 'w') as fid:
                content = _default_content(node)
                fid.write(content)
        else:
            msg = "A stub page for {} cannot be created because the syntax definition location " \
                  "exists in multiple applications, the file can be located at any of the " \
                  "following locations.\n".format(node.fullpath)
            for info in filenames:
                msg += "      {}\n".format(info.name)
            LOG.error(msg)

    else:
        for info in filenames:
            if info.exists:
                _check_page_for_stub(node, info.name, update)

def _check_page_for_stub(node, filename, update):
    """
    Helper method to check if a page is a stub.
    """
    with open(filename, 'r') as fid:
        lines = fid.readlines()

    if lines and STUB_HEADER in lines[0]:
        if update:
            LOG.info("Updating stub page for %s in file %s.", node.fullpath, filename)
            with open(filename, 'w') as fid:
                content = _default_content(node)
                fid.write(content)
        elif not node.hidden:
            msg = "A MOOSE generated stub page for %s exists, but no content was " \
                  "added. Add documentation content to %s."
            LOG.error(msg, node.fullpath, filename)

def _default_content(node):
    """
    Markdown stub content.
    """
    stub = STUB_HEADER + '\n\n'
    if isinstance(node, syntax.SyntaxNode):
        stub += '# {} System\n\n'.format(node.name)
        stub += '!syntax list {} objects=True actions=False subsystems=False\n\n' \
                .format(node.fullpath)
        stub += '!syntax list {} objects=False actions=False subsystems=True\n\n' \
                .format(node.fullpath)
        stub += '!syntax list {} objects=False actions=True subsystems=False\n\n' \
                .format(node.fullpath)

    elif isinstance(node, syntax.MooseObjectNode):
        template_filename = os.path.join(MooseDocs.MOOSE_DIR, 'framework', 'doc', 'templates',
                                         'moose_object.md.template')
        with open(template_filename, 'r') as fid:
            template_content = fid.read()

        template_content = template_content.replace('FullPathCodeClassName',
                                                    '{}'.format(node.fullpath))
        template_content = template_content.replace('CodeClassName', '{}'.format(node.name))
        stub += template_content

    elif isinstance(node, syntax.ActionNode):
        template_filename = os.path.join(MooseDocs.MOOSE_DIR, 'framework', 'doc', 'templates',
                                         'action_object.md.template')
        with open(template_filename, 'r') as fid:
            template_content = fid.read()

        template_content = template_content.replace('FullPathCodeActionName',
                                                    '{}'.format(node.fullpath))
        template_content = template_content.replace('CodeActionName', '{}'.format(node.name))
        stub += template_content

    return stub
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
# Copyright (c) 2012 Michael Babich
# See LICENSE.txt or http://opensource.org/licenses/MIT

'''Acts as an interface between what Flask serves and what goes on in
the rest of the application.
'''
from avenue import app, api
from flask import render_template, make_response, redirect
from copy import copy
import yaml
from os import path

def read_data(filename):
    '''Reads in data from a given YML file and returns it in a form
    usable by Python.
    '''
    filename = '%s.yml' % filename

    data_file = open(path.join(path.dirname(__file__), 'data', filename))
    data = yaml.load(data_file)
    data_file.close()

    return data

def forum_generator(site, forum):
    '''Reads in data files representing static forum states. Returns a
    function that accesses forum pages.
    '''
    navbar = read_data('navbar')
    tags = read_data('tags')
    threads = read_data('threads')

    def set_tags():
        '''Turns strings containing tag names into tag objects that
        can be used to generate HTML/CSS renderings of the tag.
        '''
        for thread in threads:
            for post in threads[thread]['posts']:
                if 'tags' in post:
                    for i in range(len(post['tags'])):
                        post['tags'][i] = tags[post['tags'][i]]

    def render_forum(thread_title='', main_title='', html_title='',
                     posts=[], threaded=False, content=''):
        '''Renders the forum.html template in a particular pattern
        that's used by all forum pages.
        '''
        return render_template('forum.html',
                               style='night',
                               sidebar=navbar,
                               thread_title=thread_title,
                               main_title=main_title,
                               html_title=html_title,
                               posts=posts,
                               threaded=threaded,
                               content=content)

    def forum_page(name):
        '''Makes a forum page of the given thread name.
        '''
        thread = threads[name]

        html_title = '%s :: %s :: %s' % (thread['title'], forum, site)
        main_title = '%s -- %s' % (site, forum)

        return render_forum(main_title=main_title,
                            thread_title=thread['title'],
                            html_title=html_title,
                            posts=thread['posts'],
                            threaded=thread['threaded'],
                            content=thread['content_type'])

    set_tags()

    return forum_page

def url_generator():
    def setup_url_rule(urls, action):
        '''Sets up URL rules, given a dictionary of urls and a function
        that they will act on.
        '''
        for url in urls:
            app.add_url_rule(url, url, action(urls[url]))

    urls = read_data('urls')

    def set_redirect(destination):
        '''Returns a function that redirects to a given URL.
        '''
        return lambda: redirect(destination)

    make_page = forum_generator('Zombie Raptor', 'Main Forum')

    def serve_forum_page(page):
        '''Returns a function that serves a given forum page.
        '''
        return lambda: make_page(page)

    setup_url_rule(urls['redirect'], set_redirect)
    setup_url_rule(urls['forum_urls'], serve_forum_page)

url_generator()

@app.route('/night.css')
def night():
    style = read_data('style')

    response = make_response(render_template('main.css',
                                             text=style['text'],
                                             background=style['background'],
                                             post=style['post']))
    response.mimetype = 'text/css'
    return response
<commit_msg>Use more lambdas.<commit_after># -*- coding: utf-8 -*-
# Copyright (c) 2012 Michael Babich
# See LICENSE.txt or http://opensource.org/licenses/MIT

'''Acts as an interface between what Flask serves and what goes on in
the rest of the application.
'''
from avenue import app, api
from flask import render_template, make_response, redirect
from copy import copy
import yaml
from os import path

def read_data(filename):
    '''Reads in data from a given YML file and returns it in a form
    usable by Python.
    '''
    filename = '%s.yml' % filename

    data_file = open(path.join(path.dirname(__file__), 'data', filename))
    data = yaml.load(data_file)
    data_file.close()

    return data

def forum_generator(site, forum):
    '''Reads in data files representing static forum states. Returns a
    function that accesses forum pages.
    '''
    navbar = read_data('navbar')
    tags = read_data('tags')
    threads = read_data('threads')

    def set_tags():
        '''Turns strings containing tag names into tag objects that
        can be used to generate HTML/CSS renderings of the tag.
        '''
        for thread in threads:
            for post in threads[thread]['posts']:
                if 'tags' in post:
                    for i in range(len(post['tags'])):
                        post['tags'][i] = tags[post['tags'][i]]

    def render_forum(thread_title='', main_title='', html_title='',
                     posts=[], threaded=False, content=''):
        '''Renders the forum.html template in a particular pattern
        that's used by all forum pages.
        '''
        return render_template('forum.html',
                               style='night',
                               sidebar=navbar,
                               thread_title=thread_title,
                               main_title=main_title,
                               html_title=html_title,
                               posts=posts,
                               threaded=threaded,
                               content=content)

    def forum_page(name):
        '''Makes a forum page of the given thread name.
        '''
        thread = threads[name]

        html_title = '%s :: %s :: %s' % (thread['title'], forum, site)
        main_title = '%s -- %s' % (site, forum)

        return render_forum(main_title=main_title,
                            thread_title=thread['title'],
                            html_title=html_title,
                            posts=thread['posts'],
                            threaded=thread['threaded'],
                            content=thread['content_type'])

    set_tags()

    return forum_page

def url_generator():
    def setup_url_rule(urls, action):
        '''Sets up URL rules, given a dictionary of urls and a function
        that they will act on.
        '''
        for url in urls:
            app.add_url_rule(url, url, action(urls[url]))

    urls = read_data('urls')

    make_page = forum_generator('Zombie Raptor', 'Main Forum')

    setup_url_rule(urls['redirect'], lambda x: lambda: redirect(x))
    setup_url_rule(urls['forum_urls'],lambda x: lambda: make_page(x))

url_generator()

@app.route('/night.css')
def night():
    style = read_data('style')

    response = make_response(render_template('main.css',
                                             text=style['text'],
                                             background=style['background'],
                                             post=style['post']))
    response.mimetype = 'text/css'
    return response
<|endoftext|>


<commit_before>#!/usr/bin/env python
import argparse
import getpass
import os
import subprocess
import sys
from email.mime.text import MIMEText

REST_ENDPOINT = "http://postgres.ci-connect.net/freesurfer"
VERSION = '0.2'

EMAIL_TEMPLATE = '''
This email is being sent to inform you that your freesurfer workflow {0} has
completed {2}.  You can download the output by running `fsurf --output {0}`
or download the Freesurfer log files by running `fsurf --log {0} .`

Please contact support@osgconnect.net if you have any questions.
'''

def email_user(success=True):
    """
    Email user informing them that a workflow has completed

    :param success: True if workflow completed successfully
    :return: None
    """
    if 'PEGASUS_SUBMIT_DIR' in os.environ:
        workflow = os.path.basename(os.environ['PEGASUS_SUBMIT_DIR'])
    else:
        workflow = ''
    if success:
        status = 'succesfully'
    else:
        status = 'with errors'

    msg = MIMEText(EMAIL_TEMPLATE.format(workflow,
                                         status))
    msg['Subject'] = 'Freesurfer workflow {0} completed'.format(workflow)
    sender = 'fsurf@login.osgconnect.net'
    dest = getpass.getuser()
    msg['From'] = sender
    msg['To'] = dest
    try:
        sendmail = subprocess.Popen(['/usr/sbin/sendmail', '-t'], stdin=subprocess.PIPE)
        sendmail.communicate(msg.as_string())
    except subprocess.CalledProcessError:
        pass

def main():
    """
    Main function that parses arguments and generates the pegasus
    workflow

    :return: True if any errors occurred during DAX generaton
    """
    parser = argparse.ArgumentParser(description="Process freesurfer information")
    # version info
    parser.add_argument('--version', action='version', version='%(prog)s ' + VERSION)
    # Arguments for workflow outcome
    parser.add_argument('--success', dest='success',
                        action='store_true',
                        help='Workflow completed successfully')
    parser.add_argument('--failure', dest='success',
                        action='store_false',
                        help='Workflow completed with errors')

    args = parser.parse_args(sys.argv[1:])
    if args.success:
        email_user(success=True)
    else:
        email_user(success=False)

    sys.exit(0)

if __name__ == '__main__':
    main()
<commit_msg>Fix spacing for PEP 8<commit_after>#!/usr/bin/env python
import argparse
import getpass
import os
import subprocess
import sys
from email.mime.text import MIMEText

REST_ENDPOINT = "http://postgres.ci-connect.net/freesurfer"
VERSION = '0.2'

EMAIL_TEMPLATE = '''
This email is being sent to inform you that your freesurfer workflow {0} has
completed {2}.  You can download the output by running `fsurf --output {0}`
or download the Freesurfer log files by running `fsurf --log {0} .`

Please contact support@osgconnect.net if you have any questions.
'''

def email_user(success=True):
    """
    Email user informing them that a workflow has completed

    :param success: True if workflow completed successfully
    :return: None
    """
    if 'PEGASUS_SUBMIT_DIR' in os.environ:
        workflow = os.path.basename(os.environ['PEGASUS_SUBMIT_DIR'])
    else:
        workflow = ''
    if success:
        status = 'succesfully'
    else:
        status = 'with errors'

    msg = MIMEText(EMAIL_TEMPLATE.format(workflow,
                                         status))
    msg['Subject'] = 'Freesurfer workflow {0} completed'.format(workflow)
    sender = 'fsurf@login.osgconnect.net'
    dest = getpass.getuser()
    msg['From'] = sender
    msg['To'] = dest
    try:
        sendmail = subprocess.Popen(['/usr/sbin/sendmail', '-t'], stdin=subprocess.PIPE)
        sendmail.communicate(msg.as_string())
    except subprocess.CalledProcessError:
        pass

def main():
    """
    Main function that parses arguments and generates the pegasus
    workflow

    :return: True if any errors occurred during DAX generaton
    """
    parser = argparse.ArgumentParser(description="Process freesurfer information")
    # version info
    parser.add_argument('--version', action='version', version='%(prog)s ' + VERSION)
    # Arguments for workflow outcome
    parser.add_argument('--success', dest='success',
                        action='store_true',
                        help='Workflow completed successfully')
    parser.add_argument('--failure', dest='success',
                        action='store_false',
                        help='Workflow completed with errors')

    args = parser.parse_args(sys.argv[1:])
    if args.success:
        email_user(success=True)
    else:
        email_user(success=False)

    sys.exit(0)

if __name__ == '__main__':
    main()
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
"""
fabfile template to deploy project to server

- Requirements:
    fabric==2.0.1
    asn1crypto==0.24.0
    bcrypt==3.1.4
    cffi==1.11.5
    cryptography==2.2.2
    invoke==1.0.0
    paramiko==2.4.1
    pyasn1==0.4.3
    pycparser==2.18
    pynacl==1.2.1

    slackclient==1.2.1
    websocket-client==0.47.0

- Usage:
    fab -H my-server deploy

"""

import traceback
import logging
from slackclient import SlackClient
from invoke import task

logging.basicConfig(level=logging.INFO)
logger = logging.getLogger(__name__)

WORK_DIR = '/deploy'

# file storage last git commit id
LAST_CID_FILE = "last_commit_id.txt"

# Config slack for notification
SLACK_API_KEY = "xxx-xxx-xxx"
CHANNEL_NAME = "#random"

class FabSlack:
    sc = SlackClient(SLACK_API_KEY)

    def send(self, **kargs):
        try:
            self.sc.api_call(
                "chat.postMessage",
                channel=CHANNEL_NAME,
                username='Deployment',
                # as_user=True,
                icon_emoji=":gear:",
                **kargs
            )
        except Exception:
            traceback.print_exc()

sc = FabSlack()

@task
def deploy(c):
    logger.info("Deploying on {}".format(c.host))
    with c.cd(WORK_DIR):
        if c.run('test -f {}'.format(LAST_CID_FILE), warn=True).failed:
            logger.info("Create {} file".format(LAST_CID_FILE))
            save_last_commit(c)
        c.run('git pull')
        notify_commit_applied(c)
        save_last_commit(c)

# Utils functions
# -----------------------------------------
def run(c, command, capture=False):
    result = c.run(command, echo=True)
    if capture:
        return result.stdout.strip()
    return result

def get_current_commit(c):
    return run(c, "git rev-parse HEAD", True)

def save_last_commit(c):
    run(c, "git rev-parse HEAD > {}".format(LAST_CID_FILE))

def get_last_commit(c):
    return run(c, "cat {}".format(LAST_CID_FILE), True)

def get_git_logs(c, last_commit_id, current_commit_id):
    return run(c, "git log {}...{} --oneline --pretty=format:'%s'".format(last_commit_id, current_commit_id), True)

def notify_commit_applied(c):
    last_commit_id = get_last_commit(c)
    current_commit_id = get_current_commit(c)
    commit_applied = get_git_logs(c, last_commit_id, current_commit_id)
    if commit_applied:
        commit_applied = "••• " + commit_applied
        commit_applied = commit_applied.replace("\n", "\n••• ")
    else:
        commit_applied = 'No commit applied!'
    attachments = [
        {
            "color": "good",
            "title": "Commit applied:",
            "text": commit_applied,
        },
    ]
    sc.send(attachments=attachments, text="Deploy to *{}* success".format(c.host))
<commit_msg>Update fabfile_standalon.py<commit_after># -*- coding: utf-8 -*-
"""
fabfile template to deploy project to server

- Requirements:
    fabric==2.5.0

    slackclient==1.2.1
    websocket-client==0.47.0

- Usage:
    fab -H my-server deploy

"""

import traceback
import logging
from slackclient import SlackClient
from invoke import task

logging.basicConfig(level=logging.INFO)
logger = logging.getLogger(__name__)

WORK_DIR = '/deploy'

# file storage last git commit id
LAST_CID_FILE = "last_commit_id.txt"

# Config slack for notification
SLACK_API_KEY = "xxx-xxx-xxx"
CHANNEL_NAME = "#random"

class FabSlack:
    sc = SlackClient(SLACK_API_KEY)

    def send(self, **kargs):
        try:
            self.sc.api_call(
                "chat.postMessage",
                channel=CHANNEL_NAME,
                username='Deployment',
                # as_user=True,
                icon_emoji=":gear:",
                **kargs
            )
        except Exception:
            traceback.print_exc()

sc = FabSlack()

@task
def deploy(c):
    logger.info("Deploying on {}".format(c.host))
    with c.cd(WORK_DIR):
        if c.run('test -f {}'.format(LAST_CID_FILE), warn=True).failed:
            logger.info("Create {} file".format(LAST_CID_FILE))
            save_last_commit(c)
        c.run('git pull')
        notify_commit_applied(c)
        save_last_commit(c)

# Utils functions
# -----------------------------------------
def run(c, command, capture=False):
    result = c.run(command, echo=True)
    if capture:
        return result.stdout.strip()
    return result

def get_current_commit(c):
    return run(c, "git rev-parse HEAD", True)

def save_last_commit(c):
    run(c, "git rev-parse HEAD > {}".format(LAST_CID_FILE))

def get_last_commit(c):
    return run(c, "cat {}".format(LAST_CID_FILE), True)

def get_git_logs(c, last_commit_id, current_commit_id):
    return run(c, "git log {}...{} --oneline --pretty=format:'%s'".format(last_commit_id, current_commit_id), True)

def notify_commit_applied(c):
    last_commit_id = get_last_commit(c)
    current_commit_id = get_current_commit(c)
    commit_applied = get_git_logs(c, last_commit_id, current_commit_id)
    if commit_applied:
        commit_applied = "••• " + commit_applied
        commit_applied = commit_applied.replace("\n", "\n••• ")
    else:
        commit_applied = 'No commit applied!'
    attachments = [
        {
            "color": "good",
            "title": "Commit applied:",
            "text": commit_applied,
        },
    ]
    sc.send(attachments=attachments, text="Deploy to *{}* success".format(c.host))
<|endoftext|>


<commit_before>#   Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import framework
import numpy as np
import contextlib

__all__ = [
    'ConstantInitializer', 'UniformInitializer', 'NormalInitializer', 'XavierInitializer', 'Constant', 'Uniform', 'Normal', 'Xavier', 'force_init_on_cpu','init_on_cpu'
]

_force_init_on_cpu_ = False

def force_init_on_cpu():
    return _force_init_on_cpu_

@contextlib.contextmanager
def init_on_cpu():
    """
    Switch program with `with` statement

    Examples:
        >>> with init_on_cpu():
        >>>   step = layers.create_global_var()

    """
    global _force_init_on_cpu_

    pre_state = force_init_on_cpu()
    _force_init_on_cpu_ = True
    yield
    _force_init_on_cpu_ = pre_state

class Initializer(object):
    """Base class for variable initializers

    Defines the common interface of variable initializers.
    They add operations to the init program that are used
    to initialize variables. Users should not use this class
    directly, but need to use one of its implementations.
    """

    def __init_(self):
        pass

    def __call__(self, param, block):
        """Add corresponding initialization operations to the network
        """
        raise NotImplementedError()

    def _compute_fans(self, var):
        """Compute the fan_in and the fan_out for layers

        This method computes the fan_in and the fan_out
        for neural network layers, if not specified. It is
        not possible to perfectly estimate fan_in and fan_out.
        This method will estimate it correctly for matrix multiply and
        convolutions.

        Args:
            var: variable for which fan_in and fan_out have to be computed

        Returns:
            tuple of two integers (fan_in, fan_out)
        """
        shape = var.shape
        if not shape or len(shape) == 0:
            fan_in = fan_out = 1
        elif len(shape) == 1:
            fan_in = fan_out = shape[0]
        elif len(shape) == 2:
            # This is the case for simple matrix multiply
            fan_in = shape[0]
            fan_out = shape[1]
        else:
            # Assume this to be a convolutional kernel
            # In PaddlePaddle, the shape of the kernel is like:
            # [num_filters, num_filter_channels, ...] where the remaining
            # dimensions are the filter_size
            receptive_field_size = np.prod(shape[2:])
            fan_in = shape[1] * receptive_field_size
            fan_out = shape[0] * receptive_field_size

        return (fan_in, fan_out)

class ConstantInitializer(Initializer):
    """Implements the constant initializer
    """

    def __init__(self, value=0.0, force_cpu=False):
        """Constructor for ConstantInitializer

        Args:
            value: constant value to initialize the variable
        """
        assert value is not None
        super(ConstantInitializer, self).__init__()
        self._value = value
        self._force_cpu = force_cpu

    def __call__(self, var, block):
        """Add constant initialization ops for a variable

        Args:
            var: Variable that needs to be initialized
            block: The block in which initialization ops
                   should be added

        Returns:
            the initialization op
        """
        assert isinstance(var, framework.Variable)
        assert isinstance(block, framework.Block)
        # Initialization Ops should be prepended and not appended
        op = block.prepend_op(
            type="fill_constant",
            outputs={"Out": var},
            attrs={
                "shape": var.shape,
                "dtype": int(var.dtype),
                "value": float(self._value),
                'force_cpu': self._force_cpu or force_init_on_cpu()
            })
        var.op = op
        return op

class UniformInitializer(Initializer):
    """Implements the random uniform distribution initializer
    """

    def __init__(self, low=-1.0, high=1.0, seed=0):
        """Constructor for UniformInitializer

        Args:
            low: lower boundary of the uniform distribution
            high: upper boundary of the uniform distribution
            seed: random seed
        """
        assert low is not None
        assert high is not None
        assert high >= low
        assert seed is not None
        super(UniformInitializer, self).__init__()
        self._low = low
        self._high = high
        self._seed = seed

    def __call__(self, var, block):
        """Add uniform distribution initialization ops for a variable

        Args:
            var: Variable that needs to be initialized
            block: The block in which initialization ops
                   should be added

        Returns:
            the initialization op
        """
        assert isinstance(var, framework.Variable)
        assert isinstance(block, framework.Block)
        # Initialization Ops should be prepended and not appended
        if self._seed == 0:
            self._seed = block.program.random_seed
        op = block.prepend_op(
            type="uniform_random",
            outputs={"Out": var},
            attrs={
                "shape": var.shape,
                "dtype": int(var.dtype),
                "min": self._low,
                "max": self._high,
                "seed": self._seed
            })
        var.op = op
        return op

class NormalInitializer(Initializer):
    """Implements the  random Normal(Gaussian) distribution initializer
    """

    def __init__(self, loc=0.0, scale=1.0, seed=0):
        """Constructor for NormalInitializer

        Args:
            loc: mean of the normal distribution
            scale: standard deviation of the normal distribution
            seed: random seed
        """
        assert loc is not None
        assert scale is not None
        assert seed is not None
        super(NormalInitializer, self).__init__()
        self._mean = loc
        self._std_dev = scale
        self._seed = seed

    def __call__(self, var, block):
        """Add normal distribution initialization ops for a variable

        Args:
            var: Variable that needs to be initialized
            block: The block in which initialization ops
                   should be added

        Returns:
            the initialization op
        """
        assert isinstance(var, framework.Variable)
        assert isinstance(block, framework.Block)
        # Initialization Ops should be prepended and not appended
        if self._seed == 0:
            self._seed = block.program.random_seed
        op = block.prepend_op(
            type="gaussian_random",
            outputs={"Out": var},
            attrs={
                "shape": var.shape,
                "dtype": int(var.dtype),
                "mean": self._mean,
                "std": self._std_dev,
                "seed": self._seed
            })
        var.op = op
        return op

class XavierInitializer(Initializer):
    """Implements the Xavier initializer

    This class implements the Xavier weight initializer from the paper
    Understanding the difficulty of training deep feedforward neural
    networks[1] by Xavier Glorot and Yoshua Bengio.

    This initializer is designed to keep the scale of the gradients
    approximately same in all the layers. In case of Uniform distribution,
    the range is [-x, x], where x = sqrt(6 / (fan_in + fan_out)).
    In case of Normal distribution, the mean is 0 and the standard deviation
    is sqrt(2/ (fan_in + fan_out)).

    References:
        [1] Understanding the difficulty of training deep feedforward neural
            networks. International conference on artificial intelligence and
            statistics.
            (http://proceedings.mlr.press/v9/glorot10a.html)
    """

    def __init__(self, uniform=True, fan_in=None, fan_out=None, seed=0):
        """Constructor for XavierInitializer

        Args:
            uniform: whether to use uniform or normal distribution
            fan_in: fan_in for Xavier initialization. If None, it is
                    inferred from the variable.
            fan_out: fan_out for Xavier initialization. If None, it is
                     inferred from the variable.
            seed: random seed

        Note: It is recommended to set fan_in and fan_out to None for
              most cases.
        """
        assert uniform is not None
        assert seed is not None
        super(XavierInitializer, self).__init__()
        self._uniform = uniform
        self._fan_in = fan_in
        self._fan_out = fan_out
        self._seed = seed

    def __call__(self, var, block):
        """Add xavier initialization ops for a variable

        Args:
            var: Variable that needs to be initialized
            block: The block in which initialization ops
                   should be added

        Returns:
            the initialization op
        """
        assert isinstance(var, framework.Variable)
        assert isinstance(block, framework.Block)
        f_in, f_out = self._compute_fans(var)

        # If fan_in and fan_out are passed, use them
        fan_in = f_in if self._fan_in is None else self._fan_in
        fan_out = f_out if self._fan_out is None else self._fan_out

        if self._seed == 0:
            self._seed = block.program.random_seed

        if self._uniform:
            limit = np.sqrt(6.0 / float(fan_in + fan_out))
            op = block.prepend_op(
                type="uniform_random",
                outputs={"Out": var},
                attrs={
                    "shape": var.shape,
                    "dtype": int(var.dtype),
                    "min": -limit,
                    "max": limit,
                    "seed": self._seed
                })

        else:
            std = np.sqrt(2.0 / float(fan_in + fan_out))
            op = block.prepend_op(
                type="gaussian_random",
                outputs={"Out": var},
                attrs={
                    "shape": var.shape,
                    "dtype": int(var.dtype),
                    "mean": 0.0,
                    "std": std,
                    "seed": self._seed
                })
        var.op = op
        return op

class MSRAInitializer(Initializer):
    """Implements the MSRA initializer a.k.a. Kaiming Initializer

    This class implements the weight initialization from the paper
    Delving Deep into Rectifiers: Surpassing Human-Level Performance on
    ImageNet Classification[1] by Kaiming He, Xiangyu Zhang, Shaoqing Ren
    and Jian Sun. This is a robust initialization method that particularly
    considers the rectifier nonlinearities. In case of Uniform distribution,
    the range is [-x, x], where x = sqrt(6 / fan_in). In case of Normal
    distribution, the mean is 0 and the standard deviation
    is sqrt(2/ fan_in).

    References:
        [1] Delving Deep into Rectifiers: Surpassing Human-Level Performance
            on ImageNet Classification
            (https://arxiv.org/abs/1502.01852)
    """

    def __init__(self, uniform=True, fan_in=None, seed=0):
        """Constructor for MSRAInitializer

        Args:
            uniform: whether to use uniform or normal distribution
            fan_in: fan_in for MSRAInitializer. If None, it is
                    inferred from the variable.
            seed: random seed

        Note: It is recommended to set fan_in to None for most cases.
        """
        assert uniform is not None
        assert seed is not None
        super(MSRAInitializer, self).__init__()
        self._uniform = uniform
        self._fan_in = fan_in
        self._seed = seed

    def __call__(self, var, block):
        """Add MSRA initialization ops for a variable

        Args:
            var: Variable that needs to be initialized
            block: The block in which initialization ops
                   should be added

        Returns:
            the initialization op
        """
        assert isinstance(var, framework.Variable)
        assert isinstance(block, framework.Block)
        f_in, f_out = self._compute_fans(var)

        # If fan_in is passed, use it
        fan_in = f_in if self._fan_in is None else self._fan_in

        if self._seed == 0:
            self._seed = block.program.random_seed

        if self._uniform:
            limit = np.sqrt(6.0 / float(fan_in))
            op = block.prepend_op(
                type="uniform_random",
                outputs={"Out": var},
                attrs={
                    "shape": var.shape,
                    "dtype": int(var.dtype),
                    "min": -limit,
                    "max": limit,
                    "seed": self._seed
                })

        else:
            std = np.sqrt(2.0 / float(fan_in))
            op = block.prepend_op(
                type="gaussian_random",
                outputs={"Out": var},
                attrs={
                    "shape": var.shape,
                    "dtype": int(var.dtype),
                    "mean": 0.0,
                    "std": std,
                    "seed": self._seed
                })
        var.op = op
        return op

# We short the class name, since users will use the initializer with the package
# name. The sample code:
#
# import paddle.fluid as fluid
#
# hidden = fluid.layers.fc(...,
#                          param_attr=ParamAttr(fluid.initializer.Xavier()))
#
# It is no need to add an `Initializer` as the class suffix
Constant = ConstantInitializer
Uniform = UniformInitializer
Normal = NormalInitializer
Xavier = XavierInitializer
MSRA = MSRAInitializer
<commit_msg>Adjust<commit_after>#   Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import framework
import numpy as np
import contextlib

__all__ = [
    'Constant', 'Uniform', 'Normal', 'Xavier', 'force_init_on_cpu',
    'init_on_cpu', 'ConstantInitializer', 'UniformInitializer', 'NormalInitializer', 'XavierInitializer'
]

_force_init_on_cpu_ = False

def force_init_on_cpu():
    return _force_init_on_cpu_

@contextlib.contextmanager
def init_on_cpu():
    """
    Switch program with `with` statement

    Examples:
        >>> with init_on_cpu():
        >>>   step = layers.create_global_var()

    """
    global _force_init_on_cpu_

    pre_state = force_init_on_cpu()
    _force_init_on_cpu_ = True
    yield
    _force_init_on_cpu_ = pre_state

class Initializer(object):
    """Base class for variable initializers

    Defines the common interface of variable initializers.
    They add operations to the init program that are used
    to initialize variables. Users should not use this class
    directly, but need to use one of its implementations.
    """

    def __init_(self):
        pass

    def __call__(self, param, block):
        """Add corresponding initialization operations to the network
        """
        raise NotImplementedError()

    def _compute_fans(self, var):
        """Compute the fan_in and the fan_out for layers

        This method computes the fan_in and the fan_out
        for neural network layers, if not specified. It is
        not possible to perfectly estimate fan_in and fan_out.
        This method will estimate it correctly for matrix multiply and
        convolutions.

        Args:
            var: variable for which fan_in and fan_out have to be computed

        Returns:
            tuple of two integers (fan_in, fan_out)
        """
        shape = var.shape
        if not shape or len(shape) == 0:
            fan_in = fan_out = 1
        elif len(shape) == 1:
            fan_in = fan_out = shape[0]
        elif len(shape) == 2:
            # This is the case for simple matrix multiply
            fan_in = shape[0]
            fan_out = shape[1]
        else:
            # Assume this to be a convolutional kernel
            # In PaddlePaddle, the shape of the kernel is like:
            # [num_filters, num_filter_channels, ...] where the remaining
            # dimensions are the filter_size
            receptive_field_size = np.prod(shape[2:])
            fan_in = shape[1] * receptive_field_size
            fan_out = shape[0] * receptive_field_size

        return (fan_in, fan_out)

class ConstantInitializer(Initializer):
    """Implements the constant initializer
    """

    def __init__(self, value=0.0, force_cpu=False):
        """Constructor for ConstantInitializer

        Args:
            value: constant value to initialize the variable
        """
        assert value is not None
        super(ConstantInitializer, self).__init__()
        self._value = value
        self._force_cpu = force_cpu

    def __call__(self, var, block):
        """Add constant initialization ops for a variable

        Args:
            var: Variable that needs to be initialized
            block: The block in which initialization ops
                   should be added

        Returns:
            the initialization op
        """
        assert isinstance(var, framework.Variable)
        assert isinstance(block, framework.Block)
        # Initialization Ops should be prepended and not appended
        op = block.prepend_op(
            type="fill_constant",
            outputs={"Out": var},
            attrs={
                "shape": var.shape,
                "dtype": int(var.dtype),
                "value": float(self._value),
                'force_cpu': self._force_cpu or force_init_on_cpu()
            })
        var.op = op
        return op

class UniformInitializer(Initializer):
    """Implements the random uniform distribution initializer
    """

    def __init__(self, low=-1.0, high=1.0, seed=0):
        """Constructor for UniformInitializer

        Args:
            low: lower boundary of the uniform distribution
            high: upper boundary of the uniform distribution
            seed: random seed
        """
        assert low is not None
        assert high is not None
        assert high >= low
        assert seed is not None
        super(UniformInitializer, self).__init__()
        self._low = low
        self._high = high
        self._seed = seed

    def __call__(self, var, block):
        """Add uniform distribution initialization ops for a variable

        Args:
            var: Variable that needs to be initialized
            block: The block in which initialization ops
                   should be added

        Returns:
            the initialization op
        """
        assert isinstance(var, framework.Variable)
        assert isinstance(block, framework.Block)
        # Initialization Ops should be prepended and not appended
        if self._seed == 0:
            self._seed = block.program.random_seed
        op = block.prepend_op(
            type="uniform_random",
            outputs={"Out": var},
            attrs={
                "shape": var.shape,
                "dtype": int(var.dtype),
                "min": self._low,
                "max": self._high,
                "seed": self._seed
            })
        var.op = op
        return op

class NormalInitializer(Initializer):
    """Implements the  random Normal(Gaussian) distribution initializer
    """

    def __init__(self, loc=0.0, scale=1.0, seed=0):
        """Constructor for NormalInitializer

        Args:
            loc: mean of the normal distribution
            scale: standard deviation of the normal distribution
            seed: random seed
        """
        assert loc is not None
        assert scale is not None
        assert seed is not None
        super(NormalInitializer, self).__init__()
        self._mean = loc
        self._std_dev = scale
        self._seed = seed

    def __call__(self, var, block):
        """Add normal distribution initialization ops for a variable

        Args:
            var: Variable that needs to be initialized
            block: The block in which initialization ops
                   should be added

        Returns:
            the initialization op
        """
        assert isinstance(var, framework.Variable)
        assert isinstance(block, framework.Block)
        # Initialization Ops should be prepended and not appended
        if self._seed == 0:
            self._seed = block.program.random_seed
        op = block.prepend_op(
            type="gaussian_random",
            outputs={"Out": var},
            attrs={
                "shape": var.shape,
                "dtype": int(var.dtype),
                "mean": self._mean,
                "std": self._std_dev,
                "seed": self._seed
            })
        var.op = op
        return op

class XavierInitializer(Initializer):
    """Implements the Xavier initializer

    This class implements the Xavier weight initializer from the paper
    Understanding the difficulty of training deep feedforward neural
    networks[1] by Xavier Glorot and Yoshua Bengio.

    This initializer is designed to keep the scale of the gradients
    approximately same in all the layers. In case of Uniform distribution,
    the range is [-x, x], where x = sqrt(6 / (fan_in + fan_out)).
    In case of Normal distribution, the mean is 0 and the standard deviation
    is sqrt(2/ (fan_in + fan_out)).

    References:
        [1] Understanding the difficulty of training deep feedforward neural
            networks. International conference on artificial intelligence and
            statistics.
            (http://proceedings.mlr.press/v9/glorot10a.html)
    """

    def __init__(self, uniform=True, fan_in=None, fan_out=None, seed=0):
        """Constructor for XavierInitializer

        Args:
            uniform: whether to use uniform or normal distribution
            fan_in: fan_in for Xavier initialization. If None, it is
                    inferred from the variable.
            fan_out: fan_out for Xavier initialization. If None, it is
                     inferred from the variable.
            seed: random seed

        Note: It is recommended to set fan_in and fan_out to None for
              most cases.
        """
        assert uniform is not None
        assert seed is not None
        super(XavierInitializer, self).__init__()
        self._uniform = uniform
        self._fan_in = fan_in
        self._fan_out = fan_out
        self._seed = seed

    def __call__(self, var, block):
        """Add xavier initialization ops for a variable

        Args:
            var: Variable that needs to be initialized
            block: The block in which initialization ops
                   should be added

        Returns:
            the initialization op
        """
        assert isinstance(var, framework.Variable)
        assert isinstance(block, framework.Block)
        f_in, f_out = self._compute_fans(var)

        # If fan_in and fan_out are passed, use them
        fan_in = f_in if self._fan_in is None else self._fan_in
        fan_out = f_out if self._fan_out is None else self._fan_out

        if self._seed == 0:
            self._seed = block.program.random_seed

        if self._uniform:
            limit = np.sqrt(6.0 / float(fan_in + fan_out))
            op = block.prepend_op(
                type="uniform_random",
                outputs={"Out": var},
                attrs={
                    "shape": var.shape,
                    "dtype": int(var.dtype),
                    "min": -limit,
                    "max": limit,
                    "seed": self._seed
                })

        else:
            std = np.sqrt(2.0 / float(fan_in + fan_out))
            op = block.prepend_op(
                type="gaussian_random",
                outputs={"Out": var},
                attrs={
                    "shape": var.shape,
                    "dtype": int(var.dtype),
                    "mean": 0.0,
                    "std": std,
                    "seed": self._seed
                })
        var.op = op
        return op

class MSRAInitializer(Initializer):
    """Implements the MSRA initializer a.k.a. Kaiming Initializer

    This class implements the weight initialization from the paper
    Delving Deep into Rectifiers: Surpassing Human-Level Performance on
    ImageNet Classification[1] by Kaiming He, Xiangyu Zhang, Shaoqing Ren
    and Jian Sun. This is a robust initialization method that particularly
    considers the rectifier nonlinearities. In case of Uniform distribution,
    the range is [-x, x], where x = sqrt(6 / fan_in). In case of Normal
    distribution, the mean is 0 and the standard deviation
    is sqrt(2/ fan_in).

    References:
        [1] Delving Deep into Rectifiers: Surpassing Human-Level Performance
            on ImageNet Classification
            (https://arxiv.org/abs/1502.01852)
    """

    def __init__(self, uniform=True, fan_in=None, seed=0):
        """Constructor for MSRAInitializer

        Args:
            uniform: whether to use uniform or normal distribution
            fan_in: fan_in for MSRAInitializer. If None, it is
                    inferred from the variable.
            seed: random seed

        Note: It is recommended to set fan_in to None for most cases.
        """
        assert uniform is not None
        assert seed is not None
        super(MSRAInitializer, self).__init__()
        self._uniform = uniform
        self._fan_in = fan_in
        self._seed = seed

    def __call__(self, var, block):
        """Add MSRA initialization ops for a variable

        Args:
            var: Variable that needs to be initialized
            block: The block in which initialization ops
                   should be added

        Returns:
            the initialization op
        """
        assert isinstance(var, framework.Variable)
        assert isinstance(block, framework.Block)
        f_in, f_out = self._compute_fans(var)

        # If fan_in is passed, use it
        fan_in = f_in if self._fan_in is None else self._fan_in

        if self._seed == 0:
            self._seed = block.program.random_seed

        if self._uniform:
            limit = np.sqrt(6.0 / float(fan_in))
            op = block.prepend_op(
                type="uniform_random",
                outputs={"Out": var},
                attrs={
                    "shape": var.shape,
                    "dtype": int(var.dtype),
                    "min": -limit,
                    "max": limit,
                    "seed": self._seed
                })

        else:
            std = np.sqrt(2.0 / float(fan_in))
            op = block.prepend_op(
                type="gaussian_random",
                outputs={"Out": var},
                attrs={
                    "shape": var.shape,
                    "dtype": int(var.dtype),
                    "mean": 0.0,
                    "std": std,
                    "seed": self._seed
                })
        var.op = op
        return op

# We short the class name, since users will use the initializer with the package
# name. The sample code:
#
# import paddle.fluid as fluid
#
# hidden = fluid.layers.fc(...,
#                          param_attr=ParamAttr(fluid.initializer.Xavier()))
#
# It is no need to add an `Initializer` as the class suffix
Constant = ConstantInitializer
Uniform = UniformInitializer
Normal = NormalInitializer
Xavier = XavierInitializer
MSRA = MSRAInitializer
<|endoftext|>