<commit_before># Test cases can be run with either of the following:
# python -m unittest discover
# nosetests -v --rednose --nologcapture

import unittest
import logging
import json
from flask_api import status    # HTTP Status Codes
import server
from customers import Customer

# Status Codes
HTTP_200_OK = 200
HTTP_201_CREATED = 201
HTTP_204_NO_CONTENT = 204
HTTP_400_BAD_REQUEST = 400
HTTP_404_NOT_FOUND = 404
HTTP_409_CONFLICT = 409

######################################################################
#  T E S T   C A S E S
######################################################################
class TestCustomerServer(unittest.TestCase):

    def setUp(self):
        server.app.debug = True
        server.app.logger.addHandler(logging.StreamHandler())
        server.app.logger.setLevel(logging.CRITICAL)

        self.app = server.app.test_client()
        server.inititalize_redis()
        server.data_reset()
        #server.data_load({"name": "fido", "category": "dog"})

        server.data_load({"first_name": "Andrea", "last_name": "Pirlo", "gender": "M",
        "age": "35", "email" : "a@p.com", "address_line1": "Milan",
        "address_line2": "Italy", "phonenumber": "123"})

        server.data_load({"first_name": "Theirry", "last_name": "Henry", "gender": "M",
        "age": "30", "email" : "t@h.com", "address_line1": "London",
        "address_line2": "England", "phonenumber": "444"})

        server.data_load({"first_name": "Ronaldinho", "last_name": "Gaucho", "gender": "M",
        "age": "40", "email" : "r@g.br", "address_line1": "Barcelona",
        "address_line2": "Spain", "phonenumber": "999"})

        #server.data_load({"name": "kitty", "category": "cat"})

    def test_get_customer_list(self):
        resp = self.app.get('/customers')
        self.assertEqual( resp.status_code, status.HTTP_200_OK )
        data = json.loads(resp.data)
        self.assertTrue( len(data) > 0 )

    def test_index(self):
        resp = self.app.get('/')
        self.assertEqual( resp.status_code, HTTP_200_OK )
        self.assertTrue ('Customer REST API Service' in resp.data)

    def test_get_customer(self):
        resp = self.app.get('/customers/1')
        #print 'resp_data: ' + resp.data
        self.assertEqual( resp.status_code, HTTP_200_OK )
        data = json.loads(resp.data)
        self.assertEqual (data['first_name'], 'Andrea')

    def test_get_customer_not_found(self):
        resp = self.app.get('/customers/0')
        self.assertEqual( resp.status_code, HTTP_404_NOT_FOUND )

    def test_delete_customer(self):
        # save the current number of customers for later comparison
        customer_count = self.get_customer_count()
        # delete a customer
        resp = self.app.delete('/customers/2', content_type='application/json')
        self.assertEqual( resp.status_code, HTTP_204_NO_CONTENT )
        self.assertEqual( len(resp.data), 0 )
        new_count = self.get_customer_count()
        self.assertEqual( new_count, customer_count - 1)

    def test_repeated_delete_customer(self):
        # save the current number of customers for later comparison
        customer_count = self.get_customer_count()
        # delete a customer
        resp = self.app.delete('/customers/2', content_type='application/json')
        self.assertEqual( resp.status_code, HTTP_204_NO_CONTENT )
        self.assertEqual( len(resp.data), 0 )
        new_count = self.get_customer_count()
        self.assertEqual( new_count, customer_count - 1)

        #delete same customer again
        resp = self.app.delete('/customers/2', content_type='application/json')
        self.assertEqual( resp.status_code, HTTP_204_NO_CONTENT )
        self.assertEqual( len(resp.data), 0 )
        new_count = self.get_customer_count()
        self.assertEqual( new_count, customer_count - 1)

    def test_update_customer_not_found(self):
        new_customer = {"first_name": "Andrea", "last_name": "Pirlo", "gender": "M",
        "age": "35", "email" : "a@p.com", "address_line1": "Milan",
        "address_line2": "Italy", "phonenumber": "123456789"}
        data = json.dumps(new_customer)
        resp = self.app.put('/customers/0', data=data, content_type='application/json')
        self.assertEquals( resp.status_code, HTTP_404_NOT_FOUND )

    def test_search_by_keyword(self):
        resp = self.app.get('/customers/search-keyword/com', content_type='application/json')
        self.assertEqual( resp.status_code, HTTP_200_OK )
        self.assertTrue ('Andrea' in resp.data)
        self.assertTrue ('Theirry' in resp.data)
        data = json.loads(resp.data)
        self.assertEqual( len(data), 2 )

    def test_get_customer_list_by_email(self):
        resp = self.app.get('/customers?email=a@p.com')
        self.assertEqual( resp.status_code, status.HTTP_200_OK )
        data = json.loads(resp.data)
        self.assertEqual( len(data), 1 )
        self.assertTrue('Andrea' in resp.data)

    def test_get_customer_list_by_name(self):
        resp = self.app.get('/customers?first-name=Andrea')
        self.assertEqual( resp.status_code, status.HTTP_200_OK )
        data = json.loads(resp.data)
        self.assertEqual( len(data), 1 )
        self.assertTrue('Andrea' in resp.data)
        resp = self.app.get('/customers?last-name=Pirlo')
        self.assertEqual( resp.status_code, status.HTTP_200_OK )
        data = json.loads(resp.data)
        self.assertEqual( len(data), 1 )
        self.assertTrue('Andrea' in resp.data)
        resp = self.app.get('/customers?first-name=Andrea&last-name=Pirlo')
        self.assertEqual( resp.status_code, status.HTTP_200_OK )
        data = json.loads(resp.data)
        self.assertEqual( len(data), 1 )
        self.assertTrue('Andrea' in resp.data)

    def test_create_customer(self):

         # save the current number of customers for later comparrison
         customer_count = self.get_customer_count()
         # add a new customer
         new_customer =  {"first_name": "Lionel", "last_name": "Messi", "gender": "M",
        "age": "29", "email" : "messi@barca.com", "address_line1": "Camp Nou",
        "address_line2": "Barcelona", "phonenumber": "666"}
         data = json.dumps(new_customer)
         resp = self.app.post('/customers', data=data, content_type='application/json')
         self.assertEqual( resp.status_code, status.HTTP_201_CREATED )
         # Make sure location header is set
         location = resp.headers.get('Location', None)
         self.assertTrue( location != None)
         # Check the data is correct
         new_json = json.loads(resp.data)
         self.assertEqual (new_json['first_name'], 'Lionel')
         # check that count has gone up and includes Lionel
         resp = self.app.get('/customers')
         # print 'resp_data(2): ' + resp.data
         data = json.loads(resp.data)
         self.assertEqual( resp.status_code, status.HTTP_200_OK )
         self.assertEqual( len(data), customer_count + 1 )
         self.assertIn(new_json,data)
         # checking the error response when adding a faulty customer
         faulty_customer = {"fname":"Dinosaur"}
         data = json.dumps(faulty_customer)
         resp = self.app.post('/customers', data=data, content_type='application/json')
         self.assertEqual( resp.status_code, status.HTTP_400_BAD_REQUEST )
         new_faulty_json = json.loads(resp.data)
         # make sure the correct error message is put out
         self.assertEqual (new_faulty_json['error'], 'Data is not valid')

    def test_get_customer_list_by_age(self):
        resp = self.app.get('/customers?age=40')
        self.assertEqual( resp.status_code, status.HTTP_200_OK )
        data = json.loads(resp.data)
        self.assertEqual( len(data), 1 )
        self.assertTrue('Ronaldinho' in resp.data)

    def test_get_customer_list_by_gender(self):
        resp = self.app.get('/customers?gender=M')
        self.assertEqual( resp.status_code, status.HTTP_200_OK )
        data = json.loads(resp.data)
        self.assertEqual( len(data), 3 )
        self.assertTrue('Ronaldinho' in resp.data)
        resp = self.app.get('/customers?gender=F')
        self.assertEqual( resp.status_code, status.HTTP_200_OK )
        data = json.loads(resp.data)
        self.assertEqual( len(data), 0 )

    def test_get_customer_list_by_address(self):
        resp = self.app.get('/customers?address-line1=London')
        self.assertEqual( resp.status_code, status.HTTP_200_OK )
        data = json.loads(resp.data)
        self.assertEqual( len(data), 1 )
        self.assertTrue('Henry' in resp.data)
        resp = self.app.get('/customers?address-line2=Italy')
        self.assertEqual( resp.status_code, status.HTTP_200_OK )
        data = json.loads(resp.data)
        self.assertEqual( len(data), 1 )
        self.assertTrue('Pirlo' in resp.data)

    def test_get_customer_list_by_phone(self):
        resp = self.app.get('/customers?phonenumber=123')
        self.assertEqual( resp.status_code, status.HTTP_200_OK )
        data = json.loads(resp.data)
        self.assertEqual( len(data), 1 )
        self.assertTrue('Andrea' in resp.data)
        resp = self.app.get('/customers?phonenumber=321')
        self.assertEqual( resp.status_code, status.HTTP_200_OK )
        data = json.loads(resp.data)
        self.assertEqual( len(data), 0 )

    def test_get_customer_list_by_activity(self):
        resp = self.app.get('/customers?active=True')
        self.assertEqual( resp.status_code, status.HTTP_200_OK )
        data = json.loads(resp.data)
        self.assertEqual( len(data), 3 )
        self.assertTrue('Andrea' in resp.data)
        resp = self.app.get('/customers?active=False')
        self.assertEqual( resp.status_code, status.HTTP_200_OK )
        data = json.loads(resp.data)
        self.assertEqual( len(data), 0 )

    def test_deactivate_valid_customer(self):
        resp = self.app.put('/customers/deactivate/1')
        self.assertEqual( resp.status_code, HTTP_200_OK )
        data = json.loads(resp.data)
        self.assertEqual (data['first_name'], 'Andrea')
        self.assertEqual (data['active'], False)

    def test_deactivate_invalid_customer(self):
        resp = self.app.put('/customers/deactivate/99')
        self.assertEqual( resp.status_code, HTTP_404_NOT_FOUND )
        data = json.loads(resp.data)
        self.assertTrue('Customer 99 was not found' in resp.data)

######################################################################
# Utility functions
######################################################################

    def get_customer_count(self):
         # save the current number of customers
         resp = self.app.get('/customers')
         self.assertEqual( resp.status_code, status.HTTP_200_OK )
         data = json.loads(resp.data)
         return len(data)

######################################################################
#   M A I N
######################################################################
if __name__ == '__main__':
    unittest.main()
<commit_msg>fixed #127<commit_after># Test cases can be run with either of the following:
# python -m unittest discover
# nosetests -v --rednose --nologcapture

import unittest
import logging
import json
from flask_api import status    # HTTP Status Codes
import server
from customers import Customer

# Status Codes
HTTP_200_OK = 200
HTTP_201_CREATED = 201
HTTP_204_NO_CONTENT = 204
HTTP_400_BAD_REQUEST = 400
HTTP_404_NOT_FOUND = 404
HTTP_409_CONFLICT = 409

######################################################################
#  T E S T   C A S E S
######################################################################
class TestCustomerServer(unittest.TestCase):

    def setUp(self):
        server.app.debug = True
        server.app.logger.addHandler(logging.StreamHandler())
        server.app.logger.setLevel(logging.CRITICAL)

        self.app = server.app.test_client()
        server.inititalize_redis()
        server.data_reset()
        #server.data_load({"name": "fido", "category": "dog"})

        server.data_load({"first_name": "Andrea", "last_name": "Pirlo", "gender": "M",
        "age": "35", "email" : "a@p.com", "address_line1": "Milan",
        "address_line2": "Italy", "phonenumber": "123"})

        server.data_load({"first_name": "Theirry", "last_name": "Henry", "gender": "M",
        "age": "30", "email" : "t@h.com", "address_line1": "London",
        "address_line2": "England", "phonenumber": "444"})

        server.data_load({"first_name": "Ronaldinho", "last_name": "Gaucho", "gender": "M",
        "age": "40", "email" : "r@g.br", "address_line1": "Barcelona",
        "address_line2": "Spain", "phonenumber": "999"})

        #server.data_load({"name": "kitty", "category": "cat"})

    def test_get_customer_list(self):
        resp = self.app.get('/customers')
        self.assertEqual( resp.status_code, status.HTTP_200_OK )
        data = json.loads(resp.data)
        self.assertTrue( len(data) > 0 )

    def test_index(self):
        resp = self.app.get('/')
        self.assertEqual( resp.status_code, HTTP_200_OK )
        self.assertTrue ('Customer REST API Service' in resp.data)

    def test_get_customer(self):
        resp = self.app.get('/customers/1')
        #print 'resp_data: ' + resp.data
        self.assertEqual( resp.status_code, HTTP_200_OK )
        data = json.loads(resp.data)
        self.assertEqual (data['first_name'], 'Andrea')

    def test_get_customer_not_found(self):
        resp = self.app.get('/customers/0')
        self.assertEqual( resp.status_code, HTTP_404_NOT_FOUND )

    def test_delete_customer(self):
        # save the current number of customers for later comparison
        customer_count = self.get_customer_count()
        # delete a customer
        resp = self.app.delete('/customers/2', content_type='application/json')
        self.assertEqual( resp.status_code, HTTP_204_NO_CONTENT )
        self.assertEqual( len(resp.data), 0 )
        new_count = self.get_customer_count()
        self.assertEqual( new_count, customer_count - 1)

    def test_repeated_delete_customer(self):
        # save the current number of customers for later comparison
        customer_count = self.get_customer_count()
        # delete a customer
        resp = self.app.delete('/customers/2', content_type='application/json')
        self.assertEqual( resp.status_code, HTTP_204_NO_CONTENT )
        self.assertEqual( len(resp.data), 0 )
        new_count = self.get_customer_count()
        self.assertEqual( new_count, customer_count - 1)

        #delete same customer again
        resp = self.app.delete('/customers/2', content_type='application/json')
        self.assertEqual( resp.status_code, HTTP_204_NO_CONTENT )
        self.assertEqual( len(resp.data), 0 )
        new_count = self.get_customer_count()
        self.assertEqual( new_count, customer_count - 1)

    def test_update_customer_not_found(self):
        new_customer = {"first_name": "Andrea", "last_name": "Pirlo", "gender": "M",
        "age": "35", "email" : "a@p.com", "address_line1": "Milan",
        "address_line2": "Italy", "phonenumber": "123456789"}
        data = json.dumps(new_customer)
        resp = self.app.put('/customers/0', data=data, content_type='application/json')
        self.assertEquals( resp.status_code, HTTP_404_NOT_FOUND )

    def test_search_by_keyword(self):
        resp = self.app.get('/customers/search-keyword/com', content_type='application/json')
        self.assertEqual( resp.status_code, HTTP_200_OK )
        self.assertTrue ('Andrea' in resp.data)
        self.assertTrue ('Theirry' in resp.data)
        data = json.loads(resp.data)
        self.assertEqual( len(data), 2 )

    def test_get_customer_list_by_email(self):
        resp = self.app.get('/customers?email=a@p.com')
        self.assertEqual( resp.status_code, status.HTTP_200_OK )
        data = json.loads(resp.data)
        self.assertEqual( len(data), 1 )
        self.assertTrue('Andrea' in resp.data)

    def test_get_customer_list_by_name(self):
        resp = self.app.get('/customers?first-name=Andrea')
        self.assertEqual( resp.status_code, status.HTTP_200_OK )
        data = json.loads(resp.data)
        self.assertEqual( len(data), 1 )
        self.assertTrue('Andrea' in resp.data)
        resp = self.app.get('/customers?last-name=Pirlo')
        self.assertEqual( resp.status_code, status.HTTP_200_OK )
        data = json.loads(resp.data)
        self.assertEqual( len(data), 1 )
        self.assertTrue('Andrea' in resp.data)
        resp = self.app.get('/customers?first-name=Andrea&last-name=Pirlo')
        self.assertEqual( resp.status_code, status.HTTP_200_OK )
        data = json.loads(resp.data)
        self.assertEqual( len(data), 1 )
        self.assertTrue('Andrea' in resp.data)

    def test_create_customer(self):

         # save the current number of customers for later comparrison
         customer_count = self.get_customer_count()
         # add a new customer
         new_customer =  {"first_name": "Lionel", "last_name": "Messi", "gender": "M",
        "age": "29", "email" : "messi@barca.com", "address_line1": "Camp Nou",
        "address_line2": "Barcelona", "phonenumber": "666"}
         data = json.dumps(new_customer)
         resp = self.app.post('/customers', data=data, content_type='application/json')
         self.assertEqual( resp.status_code, status.HTTP_201_CREATED )
         # Make sure location header is set
         location = resp.headers.get('Location', None)
         self.assertTrue( location != None)
         # Check the data is correct
         new_json = json.loads(resp.data)
         self.assertEqual (new_json['first_name'], 'Lionel')
         # check that count has gone up and includes Lionel
         resp = self.app.get('/customers')
         # print 'resp_data(2): ' + resp.data
         data = json.loads(resp.data)
         self.assertEqual( resp.status_code, status.HTTP_200_OK )
         self.assertEqual( len(data), customer_count + 1 )
         self.assertIn(new_json,data)
         # checking the error response when adding a faulty customer
         faulty_customer = {"fname":"Dinosaur"}
         data = json.dumps(faulty_customer)
         resp = self.app.post('/customers', data=data, content_type='application/json')
         self.assertEqual( resp.status_code, status.HTTP_400_BAD_REQUEST )
         new_faulty_json = json.loads(resp.data)
         # make sure the correct error message is put out
         self.assertEqual (new_faulty_json['error'], 'Data is not valid')

    def test_get_customer_list_by_age(self):
        resp = self.app.get('/customers?age=40')
        self.assertEqual( resp.status_code, status.HTTP_200_OK )
        data = json.loads(resp.data)
        self.assertEqual( len(data), 1 )
        self.assertTrue('Ronaldinho' in resp.data)

    def test_get_customer_list_by_gender(self):
        resp = self.app.get('/customers?gender=M')
        self.assertEqual( resp.status_code, status.HTTP_200_OK )
        data = json.loads(resp.data)
        self.assertEqual( len(data), 3 )
        self.assertTrue('Ronaldinho' in resp.data)
        resp = self.app.get('/customers?gender=F')
        self.assertEqual( resp.status_code, status.HTTP_200_OK )
        data = json.loads(resp.data)
        self.assertEqual( len(data), 0 )

    def test_get_customer_list_by_address(self):
        resp = self.app.get('/customers?address-line1=London')
        self.assertEqual( resp.status_code, status.HTTP_200_OK )
        data = json.loads(resp.data)
        self.assertEqual( len(data), 1 )
        self.assertTrue('Henry' in resp.data)
        resp = self.app.get('/customers?address-line2=Italy')
        self.assertEqual( resp.status_code, status.HTTP_200_OK )
        data = json.loads(resp.data)
        self.assertEqual( len(data), 1 )
        self.assertTrue('Pirlo' in resp.data)

    def test_get_customer_list_by_phone(self):
        resp = self.app.get('/customers?phonenumber=123')
        self.assertEqual( resp.status_code, status.HTTP_200_OK )
        data = json.loads(resp.data)
        self.assertEqual( len(data), 1 )
        self.assertTrue('Andrea' in resp.data)
        resp = self.app.get('/customers?phonenumber=321')
        self.assertEqual( resp.status_code, status.HTTP_200_OK )
        data = json.loads(resp.data)
        self.assertEqual( len(data), 0 )

    def test_get_customer_list_by_activity(self):
        resp = self.app.get('/customers?active=True')
        self.assertEqual( resp.status_code, status.HTTP_200_OK )
        data = json.loads(resp.data)
        self.assertEqual( len(data), 3 )
        self.assertTrue('Andrea' in resp.data)
        resp = self.app.get('/customers?active=False')
        self.assertEqual( resp.status_code, status.HTTP_200_OK )
        data = json.loads(resp.data)
        self.assertEqual( len(data), 0 )

    def test_deactivate_valid_customer(self):
        resp = self.app.put('/customers/deactivate/1')
        self.assertEqual( resp.status_code, HTTP_200_OK )
        data = json.loads(resp.data)
        self.assertEqual (data['first_name'], 'Andrea')
        self.assertEqual (data['active'], False)

    def test_deactivate_invalid_customer(self):
        resp = self.app.put('/customers/deactivate/99')
        self.assertEqual( resp.status_code, HTTP_404_NOT_FOUND )
        data = json.loads(resp.data)
        self.assertTrue('Customer 99 was not found' in resp.data)

    def test_activate_valid_customer(self):
        resp = self.app.put('/customers/deactivate/1')
        data = json.loads(resp.data)
        self.assertEqual (data['active'], False)
        resp = self.app.put('/customers/activate/1')
        self.assertEqual( resp.status_code, HTTP_200_OK )
        data = json.loads(resp.data)
        self.assertEqual (data['first_name'], 'Andrea')
        self.assertEqual (data['active'], True)

######################################################################
# Utility functions
######################################################################

    def get_customer_count(self):
         # save the current number of customers
         resp = self.app.get('/customers')
         self.assertEqual( resp.status_code, status.HTTP_200_OK )
         data = json.loads(resp.data)
         return len(data)

######################################################################
#   M A I N
######################################################################
if __name__ == '__main__':
    unittest.main()
<|endoftext|>


<commit_before>"""
Loads synthetic reaction datasets from USPTO.

This file contains loaders for synthetic reaction datasets from the US Patent Office. http://nextmovesoftware.com/blog/2014/02/27/unleashing-over-a-million-reactions-into-the-wild/.
"""
import os
import logging
import deepchem
import numpy as np
from deepchem.data import Dataset
from deepchem.molnet.load_function.molnet_loader import _MolnetLoader
from typing import List, Optional, Tuple, Union
import deepchem as dc

logger = logging.getLogger(__name__)

DEFAULT_DIR = deepchem.utils.data_utils.get_data_dir()

USPTO_MIT_URL = "https://deepchemdata.s3.us-west-1.amazonaws.com/datasets/USPTO_MIT.csv"
USPTO_STEREO_URL = "https://deepchemdata.s3.us-west-1.amazonaws.com/datasets/USPTO_STEREO.csv"
USPTO_50K_URL = "https://deepchemdata.s3.us-west-1.amazonaws.com/datasets/USPTO_50K.csv"
USPTO_FULL_URL = "https://deepchemdata.s3.us-west-1.amazonaws.com/datasets/USPTO_FULL.csv"

class _USPTOLoader(_MolnetLoader):

  def __init__(self, *args, subset: str, sep_reagent: bool, **kwargs):
    super(_USPTOLoader, self).__init__(*args, **kwargs)
    self.subset = subset
    self.sep_reagent = sep_reagent
    self.name = 'USPTO_' + subset

  def create_dataset(self) -> Dataset:
    if self.subset not in ['MIT', 'STEREO', '50K', 'FULL']:
      raise ValueError("Valid Subset names are MIT, STEREO and 50K.")

    if self.subset == 'MIT':
      dataset_url = USPTO_MIT_URL

    if self.subset == 'STEREO':
      dataset_url = USPTO_STEREO_URL

    if self.subset == '50K':
      dataset_url = USPTO_50K_URL

    if self.subset == 'FULL':
      dataset_url = USPTO_FULL_URL

    dataset_file = os.path.join(self.data_dir, self.name + '.csv')

    if not os.path.exists(dataset_file):
      logger.info("Downloading dataset...")
      dc.utils.data_utils.download_url(url=dataset_url, dest_dir=self.data_dir)
      logger.info("Dataset download complete.")

    loader = dc.data.CSVLoader(tasks=[], featurizer=self.featurizer)

    return loader.create_dataset(dataset_file, shard_size=8192)

def load_uspto(
    featurizer=None,  # should I remove this?
    splitter: Union[dc.splits.Splitter, str, None] = 'SpecifiedSplitter',
    transformers=None,
    reload: bool = True,
    data_dir: Optional[str] = None,
    save_dir: Optional[str] = None,
    subset: str = "MIT",
    sep_reagent: bool = True,
    **kwargs
) -> Tuple[List[str], Tuple[Dataset, ...], List[dc.trans.Transformer]]:
  """Load USPTO Datasets.

  USPTO is a dataset of over 1.8 Million organic chemical reactions extracted
  from US patents and patent applications. The reactions present in the dataset
  are split into the source(src) and target(tgt) columns. The src column
  contains the SMILES of the reactant and reagent molecules in the form
  reactant>reagent. The tgt contains the SMILES of the product molecules.

  Molnet provides ability to load subsets of the USPTO dataset namely MIT,
  STEREO and 50K. The MIT dataset contains around 480k reactions
  The STEREO dataset contains around 1 Million Reactions.
  The 50K dataset contatins 50,000 reactions with an additional label
  indicating the class of reaction to which it belongs. The loader uses the
  specified splitter to use the same splits as used by Schwaller and Coley.
  Custom splitters could also be used. There is also a toggle to load the
  dataset with the reagents separated or mixed.

  Parameters
  ----------
  featurizer: Featurizer or str
    the featurizer to use for processing the data.  Alternatively you can pass
    one of the names from dc.molnet.featurizers as a shortcut.
  splitter: Splitter or str
    the splitter to use for splitting the data into training, validation, and
    test sets.  Alternatively you can pass one of the names from
    dc.molnet.splitters as a shortcut. If this is None, all the data
    will be included in a single dataset.
  transformers: list of TransformerGenerators or strings
    the Transformers to apply to the data. Each one is specified by a
    TransformerGenerator or, as a shortcut, one of the names from
    dc.molnet.transformers.
  reload: bool
    if True, the first call for a particular featurizer and splitter will cache
    the datasets to disk, and subsequent calls will reload the cached datasets.
  data_dir: str
    a directory to save the raw data in
  save_dir: str
    a directory to save the dataset in
  subset : str (default 'MIT')
    Subset of dataset to download. 'FULL', 'MIT', 'STEREO', and '50K' are supported.

  Returns
  -------
  tasks, datasets, transformers : tuple
    tasks : list
      Column names corresponding to machine learning target variables.
    datasets : tuple
      train, validation, test splits of data as
      ``deepchem.data.datasets.Dataset`` instances.
    transformers : list
      ``deepchem.trans.transformers.Transformer`` instances applied
      to dataset.
  ----------
  .. [1] Lowe, D.. (2017). Chemical reactions from US patents (1976-Sep2016)
        (Version 1). figshare. https://doi.org/10.6084/m9.figshare.5104873.v1
  .. [2] Schwaller, P., Laino, T., Gaudin, T., Bolgar, P., Hunter, C. A., Bekas,
         C., & Lee, A. A. (2019). Molecular transformer: a model for
         uncertainty-calibrated chemical reaction prediction.
         ACS central science, 5(9), 1572-1583.
  .. [3] Somnath, V. R., Bunne, C., Coley, C. W., Krause, A., & Barzilay, R.
         (2020). Learning Graph Models for Template-Free Retrosynthesis.
         arXiv preprint arXiv:2006.07038.
  .. [4] Dai, H., Li, C., Coley, C. W., Dai, B., & Song, L. (2020).
         Retrosynthesis prediction with conditional graph logic network.
         arXiv preprint arXiv:2001.01408.
  """
  # get test and valid lists if subset is MIT, 50K, STEREO and splitter = specified.
  # if subset is Full use splitter passed by the user.
  # splitter = dc.splits.SpecifiedSplitter(valid_indices=,test_indices=)
  featurizer = dc.feat.UserDefinedFeaturizer([])
  loader = _USPTOLoader(
      featurizer,
      splitter,
      transformers,
      data_dir,
      save_dir,
      subset=subset,
      sep_reagent=sep_reagent,
      **kwargs)
  return loader.load_dataset(loader.name, reload)
<commit_msg>Fix params in doc<commit_after>"""
Loads synthetic reaction datasets from USPTO.

This file contains loaders for synthetic reaction datasets from the US Patent Office. http://nextmovesoftware.com/blog/2014/02/27/unleashing-over-a-million-reactions-into-the-wild/.
"""
import os
import logging
import deepchem
import numpy as np
from deepchem.data import Dataset
from deepchem.molnet.load_function.molnet_loader import _MolnetLoader
from typing import List, Optional, Tuple, Union
import deepchem as dc

logger = logging.getLogger(__name__)

DEFAULT_DIR = deepchem.utils.data_utils.get_data_dir()

USPTO_MIT_URL = "https://deepchemdata.s3.us-west-1.amazonaws.com/datasets/USPTO_MIT.csv"
USPTO_STEREO_URL = "https://deepchemdata.s3.us-west-1.amazonaws.com/datasets/USPTO_STEREO.csv"
USPTO_50K_URL = "https://deepchemdata.s3.us-west-1.amazonaws.com/datasets/USPTO_50K.csv"
USPTO_FULL_URL = "https://deepchemdata.s3.us-west-1.amazonaws.com/datasets/USPTO_FULL.csv"

class _USPTOLoader(_MolnetLoader):

  def __init__(self, *args, subset: str, sep_reagent: bool, **kwargs):
    super(_USPTOLoader, self).__init__(*args, **kwargs)
    self.subset = subset
    self.sep_reagent = sep_reagent
    self.name = 'USPTO_' + subset

  def create_dataset(self) -> Dataset:
    if self.subset not in ['MIT', 'STEREO', '50K', 'FULL']:
      raise ValueError("Valid Subset names are MIT, STEREO and 50K.")

    if self.subset == 'MIT':
      dataset_url = USPTO_MIT_URL

    if self.subset == 'STEREO':
      dataset_url = USPTO_STEREO_URL

    if self.subset == '50K':
      dataset_url = USPTO_50K_URL

    if self.subset == 'FULL':
      dataset_url = USPTO_FULL_URL

    dataset_file = os.path.join(self.data_dir, self.name + '.csv')

    if not os.path.exists(dataset_file):
      logger.info("Downloading dataset...")
      dc.utils.data_utils.download_url(url=dataset_url, dest_dir=self.data_dir)
      logger.info("Dataset download complete.")

    loader = dc.data.CSVLoader(tasks=[], featurizer=self.featurizer)

    return loader.create_dataset(dataset_file, shard_size=8192)

def load_uspto(
    featurizer=None,  # should I remove this?
    splitter: Union[dc.splits.Splitter, str, None] = 'SpecifiedSplitter',
    transformers=None,
    reload: bool = True,
    data_dir: Optional[str] = None,
    save_dir: Optional[str] = None,
    subset: str = "MIT",
    sep_reagent: bool = True,
    **kwargs
) -> Tuple[List[str], Tuple[Dataset, ...], List[dc.trans.Transformer]]:
  """Load USPTO Datasets.

  USPTO is a dataset of over 1.8 Million organic chemical reactions extracted
  from US patents and patent applications. The reactions present in the dataset
  are split into the source(src) and target(tgt) columns. The src column
  contains the SMILES of the reactant and reagent molecules in the form
  reactant>reagent. The tgt contains the SMILES of the product molecules.

  Molnet provides ability to load subsets of the USPTO dataset namely MIT,
  STEREO and 50K. The MIT dataset contains around 479K reactions, curated by
  jin et al. The STEREO dataset contains around 1 Million Reactions, it does
  not have duplicates and the reactions include stereochemical information.
  The 50K dataset contatins 50,000 reactions and is the benchmark for
  retrosynthesis predictions. The reactions are additionally classified into 10
  reaction classes. The canonicalized version of the dataset used by the loader
  is the same as that used by somnath et. al.

  The loader uses the SpecifiedSplitter to use the same splits as specified
  by Schwaller and Coley. Custom splitters could also be used. There is also a
  toggle to load the dataset with the reagents separated or mixed. This alters
  the entries in src by replacing the '>' with '.', effectively loading them as
  a unified SMILES string.

  Parameters
  ----------
  featurizer: Featurizer or str
    the featurizer to use for processing the data.  Alternatively you can pass
    one of the names from dc.molnet.featurizers as a shortcut.
  splitter: Splitter or str
    the splitter to use for splitting the data into training, validation, and
    test sets.  Alternatively you can pass one of the names from
    dc.molnet.splitters as a shortcut. If this is None, all the data
    will be included in a single dataset.
  transformers: list of TransformerGenerators or strings
    the Transformers to apply to the data. Each one is specified by a
    TransformerGenerator or, as a shortcut, one of the names from
    dc.molnet.transformers.
  reload: bool
    if True, the first call for a particular featurizer and splitter will cache
    the datasets to disk, and subsequent calls will reload the cached datasets.
  data_dir: str
    a directory to save the raw data in
  save_dir: str
    a directory to save the dataset in
  subset : str (default 'MIT')
    Subset of dataset to download. 'FULL', 'MIT', 'STEREO', and '50K' are supported.
  sep_reagent : bool (default True)
    Toggle to load dataset with reactants and reagents separated or mixed.

  Returns
  -------
  tasks, datasets, transformers : tuple
    tasks : list
      Column names corresponding to machine learning target variables.
    datasets : tuple
      train, validation, test splits of data as
      ``deepchem.data.datasets.Dataset`` instances.
    transformers : list
      ``deepchem.trans.transformers.Transformer`` instances applied
      to dataset.
  ----------
  .. [1] Lowe, D.. (2017). Chemical reactions from US patents (1976-Sep2016)
        (Version 1). figshare. https://doi.org/10.6084/m9.figshare.5104873.v1
  .. [2] Schwaller, P., Laino, T., Gaudin, T., Bolgar, P., Hunter, C. A., Bekas,
         C., & Lee, A. A. (2019). Molecular transformer: a model for
         uncertainty-calibrated chemical reaction prediction.
         ACS central science, 5(9), 1572-1583.
  .. [3] Somnath, V. R., Bunne, C., Coley, C. W., Krause, A., & Barzilay, R.
         (2020). Learning Graph Models for Template-Free Retrosynthesis.
         arXiv preprint arXiv:2006.07038.
  .. [4] Dai, H., Li, C., Coley, C. W., Dai, B., & Song, L. (2020).
         Retrosynthesis prediction with conditional graph logic network.
         arXiv preprint arXiv:2001.01408.
  """
  # get test and valid lists if subset is MIT, 50K, STEREO and splitter = specified.
  # if subset is Full use splitter passed by the user.
  # splitter = dc.splits.SpecifiedSplitter(valid_indices=,test_indices=)
  featurizer = dc.feat.UserDefinedFeaturizer([])
  loader = _USPTOLoader(
      featurizer,
      splitter,
      transformers,
      data_dir,
      save_dir,
      subset=subset,
      sep_reagent=sep_reagent,
      **kwargs)
  return loader.load_dataset(loader.name, reload)
<|endoftext|>


<commit_before>#! /usr/bin/env python
# -*- coding: UTF-8 -*-
#
# @author: Jason Wu (Jasonwbw@yahoo.com)

import inverted_index
import pmi
import topkheap
import class_tfidf

__all__ = ['inverted_index', 'pmi', 'topkheap', 'class_tfidf']<commit_msg>edit init.py<commit_after>#! /usr/bin/env python
# -*- coding: UTF-8 -*-
#
# @author: Jason Wu (Jasonwbw@yahoo.com)

import inverted_index
import pmi
import topkheap
import class_tfidf
import tfidf

__all__ = ['inverted_index', 'pmi', 'topkheap', 'class_tfidf', 'tfidf']<|endoftext|>


<commit_before>from __future__ import division, absolute_import, print_function, unicode_literals

from .version import __version__
from struct import Struct as _Struct
import warnings

from . import wrapper

try:
    buffer
except NameError:
    buffer = memoryview  # py3

nanoconfig_started = False

#Import constants into module with NN_ prefix stripped
for name, value in wrapper.nn_symbols():
    if name.startswith('NN_'):
        name = name[3:]
    globals()[name] = value

if hasattr(wrapper, 'create_writable_buffer'):
    create_writable_buffer = wrapper.create_writable_buffer
else:
    def create_writable_buffer(size):
        """Returns a writable buffer"""
        return bytearray(size)

def create_message_buffer(size, type):
    """Create a message buffer"""
    rtn = wrapper.nn_allocmsg(size, type)
    if rtn is None:
        raise NanoMsgAPIError()
    return rtn

class NanoMsgError(Exception):
    """Base Exception for all errors in the nanomsg python package

    """
    pass

class NanoMsgAPIError(NanoMsgError):
    """Exception for all errors reported by the C API.

    msg and errno are from nanomsg C library.

    """
    __slots__ = ('msg', 'errno')

    def __init__(self):
        errno = wrapper.nn_errno()
        msg = wrapper.nn_strerror(errno)
        NanoMsgError.__init__(self, msg)
        self.errno, self.msg = errno, msg

def _nn_check_positive_rtn(rtn):
    if rtn < 0:
        raise NanoMsgAPIError()
    return rtn

class Device(object):
    """Create a nanomsg device to relay messages between sockets.

    If only one socket is supplied the device loops messages on that socket.
    """
    def __init__(self, socket1, socket2=None):
        self._fd1 = socket1.fd
        self._fd2 = -1 if socket2 is None else socket2.fd

    def start(self):
        """Run the device in the current thread.

        This will not return until the device stops due to error or
        termination.
        """
        _nn_check_positive_rtn(wrapper.nn_device(self._fd1, self._fd2))

def terminate_all():
    """Close all sockets and devices"""
    global nanoconfig_started
    if nanoconfig_started:
        wrapper.nc_term()
        nanoconfig_started = False
    wrapper.nn_term()

def poll(in_sockets, out_sockets, timeout=-1):
    """
    Poll a list of sockets
    :param in_sockets: sockets for reading
    :param out_sockets: sockets for writing
    :param timeout: poll timeout in seconds, -1 is infinite wait
    :return: tuple (read socket list, write socket list)
    """
    sockets = {}
    # reverse map fd => socket
    fd_sockets = {}
    for s in in_sockets:
        sockets[s.fd] = POLLIN
        fd_sockets[s.fd] = s
    for s in out_sockets:
        modes = sockets.get(s.fd, 0)
        sockets[s.fd] = modes | POLLOUT
        fd_sockets[s.fd] = s

    # convert to milliseconds or -1
    if timeout >= 0:
        timeout_ms = int(timeout*1000)
    else:
        timeout_ms = -1
    res, sockets = wrapper.nn_poll(sockets, timeout_ms)
    _nn_check_positive_rtn(res)
    read_list, write_list = [], []
    for fd, result in sockets.items():
        if (result & POLLIN) != 0:
            read_list.append(fd_sockets[fd])
        if (result & POLLOUT) != 0:
            write_list.append(fd_sockets[fd])

    return read_list, write_list

class Socket(object):
    """Class wrapping nanomsg socket.

    protocol should be a nanomsg protocol constant e.g. nanomsg.PAIR

    This class supports being used as a context manager which should guarantee
    it is closed.

    e.g.:
        import time
        from nanomsg import PUB, Socket

        with Socket(PUB) as pub_socket:
            pub_socket.bind('tcp://127.0.0.1:49234')
            for i in range(100):
                pub_socket.send(b'hello all')
                time.sleep(0.5)
        #pub_socket is closed

    Socket.bind and Socket.connect return subclass of Endpoint which allow
    you to shutdown selected endpoints.

    The constructor also allows you to wrap existing sockets by passing in the
    socket fd instead of the protocol e.g.:
        from nanomsg import AF_SP, PAIR, Socket
        from nanomsg import wrapper as nn

        socket_fd = nn.nn_socket(AF_SP, PAIR)
        socket = Socket(socket_fd=socket_fd)

    """

    _INT_PACKER = _Struct(str('i'))

    class _Endpoint(object):
        def __init__(self, socket, endpoint_id, address):
            self._endpoint_id = endpoint_id
            self._fdocket = socket
            self._address = address

        @property
        def address(self):
            return self._address

        def shutdown(self):
            self._fdocket._endpoints.remove(self)
            _nn_check_positive_rtn(wrapper.nn_shutdown(self._fdocket._fd,
                                               self._endpoint_id))

        def __repr__(self):
            return '<%s socket %r, id %r, address %r>' % (
                self.__class__.__name__,
                self._fdocket,
                self._endpoint_id,
                self._address
            )

    class BindEndpoint(_Endpoint):
        pass

    class ConnectEndpoint(_Endpoint):
        pass

    class NanoconfigEndpoint(_Endpoint):

        def shutdown(self):
            raise NotImplementedError(
                "Shutdown of nanoconfig endpoint is not supported")

    def __init__(self, protocol=None, socket_fd=None, domain=AF_SP):
        if protocol is not None and socket_fd is not None:
            raise NanoMsgError('Only one of protocol or socket_fd should be '
                               'passed to the Socket constructor')
        if protocol is not None:
            self._fd = _nn_check_positive_rtn(
                wrapper.nn_socket(domain, protocol)
            )
        else:
            self._fd = socket_fd
        self._endpoints = []

    def _get_send_fd(self):
        return self.get_int_option(SOL_SOCKET, SNDFD)

    def _get_recv_fd(self):
        return self.get_int_option(SOL_SOCKET, RCVFD)

    def _get_linger(self):
        return self.get_int_option(SOL_SOCKET, LINGER)

    def _set_linger(self, value):
        return self.set_int_option(SOL_SOCKET, LINGER, value)

    def _get_send_buffer_size(self):
        return self.get_int_option(SOL_SOCKET, SNDBUF)

    def _set_send_buffer_size(self, value):
        return self.set_int_option(SOL_SOCKET, SNDBUF, value)

    def _get_recv_buffer_size(self):
        return self.get_int_option(SOL_SOCKET, RCVBUF)

    def _set_recv_buffer_size(self, value):
        return self.set_int_option(SOL_SOCKET, RCVBUF, value)

    def _get_send_timeout(self):
        return self.get_int_option(SOL_SOCKET, SNDTIMEO)

    def _set_send_timeout(self, value):
        return self.set_int_option(SOL_SOCKET, SNDTIMEO, value)

    def _get_recv_timeout(self):
        return self.get_int_option(SOL_SOCKET, RCVTIMEO)

    def _set_recv_timeout(self, value):
        return self.set_int_option(SOL_SOCKET, RCVTIMEO, value)

    def _get_reconnect_interval(self):
        return self.get_int_option(SOL_SOCKET, RECONNECT_IVL)

    def _set_reconnect_interval(self, value):
        return self.set_int_option(SOL_SOCKET, RECONNECT_IVL, value)

    def _get_reconnect_interval_max(self):
        return self.get_int_option(SOL_SOCKET, RECONNECT_IVL_MAX)

    def _set_reconnect_interval_max(self, value):
        return self.set_int_option(SOL_SOCKET, RECONNECT_IVL_MAX, value)

    send_fd = property(_get_send_fd, doc='Send file descripter')
    recv_fd = property(_get_recv_fd, doc='Receive file descripter')
    linger  = property(_get_linger, _set_linger, doc='Socket linger in '
                       'milliseconds (0.001 seconds)')
    recv_buffer_size = property(_get_recv_buffer_size, _set_recv_buffer_size,
                                doc='Receive buffer size in bytes')
    send_buffer_size = property(_get_send_buffer_size, _set_send_buffer_size,
                                doc='Send buffer size in bytes')
    send_timeout = property(_get_send_timeout, _set_send_timeout,
                            doc='Send timeout in milliseconds (0.001 seconds)')
    recv_timeout = property(_get_recv_timeout, _set_recv_timeout,
                            doc='Receive timeout in milliseconds (0.001 '
                            'seconds)')
    reconnect_interval = property(
        _get_reconnect_interval,
        _set_reconnect_interval,
        doc='Base interval between connection failure and reconnect'
        ' attempt in milliseconds (0.001 seconds).'
    )
    reconnect_interval_max = property(
        _get_reconnect_interval_max,
        _set_reconnect_interval_max,
        doc='Max reconnect interval - see C API documentation.'
    )

    @property
    def fd(self):
        """Socket file descripter.

        Note this is not an OS file descripter (see .send_fd, .recv_fd).
        """
        return self._fd

    @property
    def endpoints(self):
        """Endpoints list

        """
        return list(self._endpoints)

    @property
    def uses_nanoconfig(self):
        return (self._endpoints and
            isinstance(self._endpoints[0], Socket.NanoconfigEndpoint))

    def bind(self, address):
        """Add a local endpoint to the socket"""
        if self.uses_nanoconfig:
            raise ValueError("Nanoconfig address must be sole endpoint")
        endpoint_id = _nn_check_positive_rtn(
            wrapper.nn_bind(self._fd, address)
        )
        ep = Socket.BindEndpoint(self, endpoint_id, address)
        self._endpoints.append(ep)
        return ep

    def connect(self, address):
        """Add a remote endpoint to the socket"""
        if self.uses_nanoconfig:
            raise ValueError("Nanoconfig address must be sole endpoint")
        endpoint_id = _nn_check_positive_rtn(
            wrapper.nn_connect(self.fd, address)
        )
        ep = Socket.ConnectEndpoint(self, endpoint_id, address)
        self._endpoints.append(ep)
        return ep

    def configure(self, address):
        """Configure socket's addresses with nanoconfig"""
        global nanoconfig_started
        if len(self._endpoints):
            raise ValueError("Nanoconfig address must be sole endpoint")
        endpoint_id = _nn_check_positive_rtn(
            wrapper.nc_configure(self.fd, address)
        )
        if not nanoconfig_started:
            nanoconfig_started = True
        ep = Socket.NanoconfigEndpoint(self, endpoint_id, address)
        self._endpoints.append(ep)
        return ep

    def close(self):
        """Close the socket"""
        if self.is_open():
            fd = self._fd
            self._fd = -1
            if self.uses_nanoconfig:
                wrapper.nc_close(fd)
            else:
                _nn_check_positive_rtn(wrapper.nn_close(fd))

    def is_open(self):
        """Returns true if the socket has a valid socket id.

        If the underlying socket is closed by some other means than
        Socket.close this method may return True when the socket is actually
        closed.
        """
        return self.fd >= 0

    def recv(self, buf=None, flags=0):
        """Recieve a message."""
        if buf is None:
            rtn, out_buf = wrapper.nn_recv(self.fd, flags)
        else:
            rtn, out_buf = wrapper.nn_recv(self.fd, buf, flags)
        _nn_check_positive_rtn(rtn)
        return bytes(buffer(out_buf))[:rtn]

    def set_string_option(self, level, option, value):
        _nn_check_positive_rtn(wrapper.nn_setsockopt(self.fd, level, option,
                               value))

    def set_int_option(self, level, option, value):
        buf = create_writable_buffer(Socket._INT_PACKER.size)
        Socket._INT_PACKER.pack_into(buf, 0, value)
        _nn_check_positive_rtn(wrapper.nn_setsockopt(self.fd, level, option,
                                                     buf))

    def get_int_option(self, level, option):
        size = Socket._INT_PACKER.size
        buf = create_writable_buffer(size)
        rtn, length = wrapper.nn_getsockopt(self._fd, level, option, buf)
        _nn_check_positive_rtn(rtn)
        if length != size:
            raise NanoMsgError(('Returned option size (%r) should be the same'
                                ' as size of int (%r)') % (rtn, size))
        return Socket._INT_PACKER.unpack_from(buffer(buf))[0]

    def get_string_option(self, level, option, max_len=16*1024):
        buf = create_writable_buffer(max_len)
        rtn, length = wrapper.nn_getsockopt(self._fd, level, option, buf)
        _nn_check_positive_rtn(rtn)
        return bytes(buffer(buf))[:length]

    def send(self, msg, flags=0):
        """Send a message"""
        _nn_check_positive_rtn(wrapper.nn_send(self.fd, msg, flags))

    def __enter__(self):
        return self

    def __exit__(self, *args):
        self.close()

    def __repr__(self):
        return '<%s fd %r, connected to %r, bound to %r>' % (
            self.__class__.__name__,
            self.fd,
            [i.address for i in self.endpoints if type(i) is
             Socket.ConnectEndpoint],
            [i.address for i in self.endpoints if type(i) is
             Socket.BindEndpoint],
        )

    def __del__(self):
        try:
            self.close()
        except NanoMsgError:
            pass
<commit_msg>Fix Issue #64 <commit_after>from __future__ import division, absolute_import, print_function, unicode_literals

from .version import __version__
from struct import Struct as _Struct
import warnings

from . import wrapper

try:
    buffer
except NameError:
    buffer = memoryview  # py3

nanoconfig_started = False

#Import constants into module with NN_ prefix stripped
for name, value in wrapper.nn_symbols():
    if name.startswith('NN_'):
        name = name[3:]
    globals()[name] = value

if hasattr(wrapper, 'create_writable_buffer'):
    create_writable_buffer = wrapper.create_writable_buffer
else:
    def create_writable_buffer(size):
        """Returns a writable buffer"""
        return bytearray(size)

def create_message_buffer(size, type):
    """Create a message buffer"""
    rtn = wrapper.nn_allocmsg(size, type)
    if rtn is None:
        raise NanoMsgAPIError()
    return rtn

class NanoMsgError(Exception):
    """Base Exception for all errors in the nanomsg python package

    """
    pass

class NanoMsgAPIError(NanoMsgError):
    """Exception for all errors reported by the C API.

    msg and errno are from nanomsg C library.

    """
    __slots__ = ('msg', 'errno')

    def __init__(self):
        errno = wrapper.nn_errno()
        msg = wrapper.nn_strerror(errno)
        NanoMsgError.__init__(self, msg)
        self.errno, self.msg = errno, msg

def _nn_check_positive_rtn(rtn):
    if rtn < 0:
        raise NanoMsgAPIError()
    return rtn

class Device(object):
    """Create a nanomsg device to relay messages between sockets.

    If only one socket is supplied the device loops messages on that socket.
    """
    def __init__(self, socket1, socket2=None):
        self._fd1 = socket1.fd
        self._fd2 = -1 if socket2 is None else socket2.fd

    def start(self):
        """Run the device in the current thread.

        This will not return until the device stops due to error or
        termination.
        """
        _nn_check_positive_rtn(wrapper.nn_device(self._fd1, self._fd2))

def terminate_all():
    """Close all sockets and devices"""
    global nanoconfig_started
    if nanoconfig_started:
        wrapper.nc_term()
        nanoconfig_started = False
    wrapper.nn_term()

def poll(in_sockets, out_sockets, timeout=-1):
    """
    Poll a list of sockets
    :param in_sockets: sockets for reading
    :param out_sockets: sockets for writing
    :param timeout: poll timeout in seconds, -1 is infinite wait
    :return: tuple (read socket list, write socket list)
    """
    sockets = {}
    # reverse map fd => socket
    fd_sockets = {}
    for s in in_sockets:
        sockets[s.fd] = POLLIN
        fd_sockets[s.fd] = s
    for s in out_sockets:
        modes = sockets.get(s.fd, 0)
        sockets[s.fd] = modes | POLLOUT
        fd_sockets[s.fd] = s

    # convert to milliseconds or -1
    if timeout >= 0:
        timeout_ms = int(timeout*1000)
    else:
        timeout_ms = -1
    res, sockets = wrapper.nn_poll(sockets, timeout_ms)
    _nn_check_positive_rtn(res)
    read_list, write_list = [], []
    for fd, result in sockets.items():
        if (result & POLLIN) != 0:
            read_list.append(fd_sockets[fd])
        if (result & POLLOUT) != 0:
            write_list.append(fd_sockets[fd])

    return read_list, write_list

class Socket(object):
    """Class wrapping nanomsg socket.

    protocol should be a nanomsg protocol constant e.g. nanomsg.PAIR

    This class supports being used as a context manager which should guarantee
    it is closed.

    e.g.:
        import time
        from nanomsg import PUB, Socket

        with Socket(PUB) as pub_socket:
            pub_socket.bind('tcp://127.0.0.1:49234')
            for i in range(100):
                pub_socket.send(b'hello all')
                time.sleep(0.5)
        #pub_socket is closed

    Socket.bind and Socket.connect return subclass of Endpoint which allow
    you to shutdown selected endpoints.

    The constructor also allows you to wrap existing sockets by passing in the
    socket fd instead of the protocol e.g.:
        from nanomsg import AF_SP, PAIR, Socket
        from nanomsg import wrapper as nn

        socket_fd = nn.nn_socket(AF_SP, PAIR)
        socket = Socket(socket_fd=socket_fd)

    """

    _INT_PACKER = _Struct(str('l'))

    class _Endpoint(object):
        def __init__(self, socket, endpoint_id, address):
            self._endpoint_id = endpoint_id
            self._fdocket = socket
            self._address = address

        @property
        def address(self):
            return self._address

        def shutdown(self):
            self._fdocket._endpoints.remove(self)
            _nn_check_positive_rtn(wrapper.nn_shutdown(self._fdocket._fd,
                                               self._endpoint_id))

        def __repr__(self):
            return '<%s socket %r, id %r, address %r>' % (
                self.__class__.__name__,
                self._fdocket,
                self._endpoint_id,
                self._address
            )

    class BindEndpoint(_Endpoint):
        pass

    class ConnectEndpoint(_Endpoint):
        pass

    class NanoconfigEndpoint(_Endpoint):

        def shutdown(self):
            raise NotImplementedError(
                "Shutdown of nanoconfig endpoint is not supported")

    def __init__(self, protocol=None, socket_fd=None, domain=AF_SP):
        if protocol is not None and socket_fd is not None:
            raise NanoMsgError('Only one of protocol or socket_fd should be '
                               'passed to the Socket constructor')
        if protocol is not None:
            self._fd = _nn_check_positive_rtn(
                wrapper.nn_socket(domain, protocol)
            )
        else:
            self._fd = socket_fd
        self._endpoints = []

    def _get_send_fd(self):
        return self.get_int_option(SOL_SOCKET, SNDFD)

    def _get_recv_fd(self):
        return self.get_int_option(SOL_SOCKET, RCVFD)

    def _get_linger(self):
        return self.get_int_option(SOL_SOCKET, LINGER)

    def _set_linger(self, value):
        return self.set_int_option(SOL_SOCKET, LINGER, value)

    def _get_send_buffer_size(self):
        return self.get_int_option(SOL_SOCKET, SNDBUF)

    def _set_send_buffer_size(self, value):
        return self.set_int_option(SOL_SOCKET, SNDBUF, value)

    def _get_recv_buffer_size(self):
        return self.get_int_option(SOL_SOCKET, RCVBUF)

    def _set_recv_buffer_size(self, value):
        return self.set_int_option(SOL_SOCKET, RCVBUF, value)

    def _get_send_timeout(self):
        return self.get_int_option(SOL_SOCKET, SNDTIMEO)

    def _set_send_timeout(self, value):
        return self.set_int_option(SOL_SOCKET, SNDTIMEO, value)

    def _get_recv_timeout(self):
        return self.get_int_option(SOL_SOCKET, RCVTIMEO)

    def _set_recv_timeout(self, value):
        return self.set_int_option(SOL_SOCKET, RCVTIMEO, value)

    def _get_reconnect_interval(self):
        return self.get_int_option(SOL_SOCKET, RECONNECT_IVL)

    def _set_reconnect_interval(self, value):
        return self.set_int_option(SOL_SOCKET, RECONNECT_IVL, value)

    def _get_reconnect_interval_max(self):
        return self.get_int_option(SOL_SOCKET, RECONNECT_IVL_MAX)

    def _set_reconnect_interval_max(self, value):
        return self.set_int_option(SOL_SOCKET, RECONNECT_IVL_MAX, value)

    send_fd = property(_get_send_fd, doc='Send file descripter')
    recv_fd = property(_get_recv_fd, doc='Receive file descripter')
    linger  = property(_get_linger, _set_linger, doc='Socket linger in '
                       'milliseconds (0.001 seconds)')
    recv_buffer_size = property(_get_recv_buffer_size, _set_recv_buffer_size,
                                doc='Receive buffer size in bytes')
    send_buffer_size = property(_get_send_buffer_size, _set_send_buffer_size,
                                doc='Send buffer size in bytes')
    send_timeout = property(_get_send_timeout, _set_send_timeout,
                            doc='Send timeout in milliseconds (0.001 seconds)')
    recv_timeout = property(_get_recv_timeout, _set_recv_timeout,
                            doc='Receive timeout in milliseconds (0.001 '
                            'seconds)')
    reconnect_interval = property(
        _get_reconnect_interval,
        _set_reconnect_interval,
        doc='Base interval between connection failure and reconnect'
        ' attempt in milliseconds (0.001 seconds).'
    )
    reconnect_interval_max = property(
        _get_reconnect_interval_max,
        _set_reconnect_interval_max,
        doc='Max reconnect interval - see C API documentation.'
    )

    @property
    def fd(self):
        """Socket file descripter.

        Note this is not an OS file descripter (see .send_fd, .recv_fd).
        """
        return self._fd

    @property
    def endpoints(self):
        """Endpoints list

        """
        return list(self._endpoints)

    @property
    def uses_nanoconfig(self):
        return (self._endpoints and
            isinstance(self._endpoints[0], Socket.NanoconfigEndpoint))

    def bind(self, address):
        """Add a local endpoint to the socket"""
        if self.uses_nanoconfig:
            raise ValueError("Nanoconfig address must be sole endpoint")
        endpoint_id = _nn_check_positive_rtn(
            wrapper.nn_bind(self._fd, address)
        )
        ep = Socket.BindEndpoint(self, endpoint_id, address)
        self._endpoints.append(ep)
        return ep

    def connect(self, address):
        """Add a remote endpoint to the socket"""
        if self.uses_nanoconfig:
            raise ValueError("Nanoconfig address must be sole endpoint")
        endpoint_id = _nn_check_positive_rtn(
            wrapper.nn_connect(self.fd, address)
        )
        ep = Socket.ConnectEndpoint(self, endpoint_id, address)
        self._endpoints.append(ep)
        return ep

    def configure(self, address):
        """Configure socket's addresses with nanoconfig"""
        global nanoconfig_started
        if len(self._endpoints):
            raise ValueError("Nanoconfig address must be sole endpoint")
        endpoint_id = _nn_check_positive_rtn(
            wrapper.nc_configure(self.fd, address)
        )
        if not nanoconfig_started:
            nanoconfig_started = True
        ep = Socket.NanoconfigEndpoint(self, endpoint_id, address)
        self._endpoints.append(ep)
        return ep

    def close(self):
        """Close the socket"""
        if self.is_open():
            fd = self._fd
            self._fd = -1
            if self.uses_nanoconfig:
                wrapper.nc_close(fd)
            else:
                _nn_check_positive_rtn(wrapper.nn_close(fd))

    def is_open(self):
        """Returns true if the socket has a valid socket id.

        If the underlying socket is closed by some other means than
        Socket.close this method may return True when the socket is actually
        closed.
        """
        return self.fd >= 0

    def recv(self, buf=None, flags=0):
        """Recieve a message."""
        if buf is None:
            rtn, out_buf = wrapper.nn_recv(self.fd, flags)
        else:
            rtn, out_buf = wrapper.nn_recv(self.fd, buf, flags)
        _nn_check_positive_rtn(rtn)
        return bytes(buffer(out_buf))[:rtn]

    def set_string_option(self, level, option, value):
        _nn_check_positive_rtn(wrapper.nn_setsockopt(self.fd, level, option,
                               value))

    def set_int_option(self, level, option, value):
        buf = create_writable_buffer(Socket._INT_PACKER.size)
        Socket._INT_PACKER.pack_into(buf, 0, value)
        _nn_check_positive_rtn(wrapper.nn_setsockopt(self.fd, level, option,
                                                     buf))

    def get_int_option(self, level, option):
        size = Socket._INT_PACKER.size
        buf = create_writable_buffer(size)
        rtn, length = wrapper.nn_getsockopt(self._fd, level, option, buf)
        _nn_check_positive_rtn(rtn)
        if length != size:
            raise NanoMsgError(('Returned option size (%r) should be the same'
                                ' as size of int (%r)') % (rtn, size))
        return Socket._INT_PACKER.unpack_from(buffer(buf))[0]

    def get_string_option(self, level, option, max_len=16*1024):
        buf = create_writable_buffer(max_len)
        rtn, length = wrapper.nn_getsockopt(self._fd, level, option, buf)
        _nn_check_positive_rtn(rtn)
        return bytes(buffer(buf))[:length]

    def send(self, msg, flags=0):
        """Send a message"""
        _nn_check_positive_rtn(wrapper.nn_send(self.fd, msg, flags))

    def __enter__(self):
        return self

    def __exit__(self, *args):
        self.close()

    def __repr__(self):
        return '<%s fd %r, connected to %r, bound to %r>' % (
            self.__class__.__name__,
            self.fd,
            [i.address for i in self.endpoints if type(i) is
             Socket.ConnectEndpoint],
            [i.address for i in self.endpoints if type(i) is
             Socket.BindEndpoint],
        )

    def __del__(self):
        try:
            self.close()
        except NanoMsgError:
            pass
<|endoftext|>


<commit_before># ica <master> <inputFile> <outputFile_Components> <outputFile_Weights>
#
# Perform ICA on a data matrix.
# Input is a local text file or a file in HDFS.
# Format should be rows of ' ' separated values.
# - Example: space (rows) x time (cols).
# - Rows should be whichever dim is larger.
# Returns independent components and unmixing matrix in separate files

# k is number of PCs to use in initial dimensionality reduction
# c is the number of ICA components to return

import sys
from numpy import *
from scipy.linalg import *
from pyspark import SparkContext

if len(sys.argv) < 7:
  print >> sys.stderr, \
  "(ica) usage: ica <master> <inputFile> <k> <c> <outputFile_Components> <outputFile_Weights>"
  exit(-1)

def parseVector(line):
    return array([float(x) for x in line.split(' ')])

# parse inputs
sc = SparkContext(sys.argv[1], "ica")
lines = sc.textFile(sys.argv[2])
k = int(sys.argv[3])
c = int(sys.argv[4])
fileOutComp = str(sys.argv[5])
fileOutWeight = str(sys.argv[6])

# compute covariance matrix
data = lines.map(parseVector).cache()
n = data.count()
m = len(data.first())
meanVec = data.reduce(lambda x,y : x+y) / n
sub = data.map(lambda x : x - meanVec)
cov = sub.map(lambda x : outer(x,x)).reduce(lambda x,y : (x + y)) / n

# do eigenvector decomposition
w, v = eig(cov)
inds = argsort(w)[::1]
kEigVecs = v[:,inds[0:k]]
kEigVals = w[inds[0:k]]

# whiten data
whtMat = real(dot(inv(sqrtm(diag(kEigVals))),transpose(kEigVecs)))
unwhtMat = dot(kEigVecs,sqrtm(diag(kEigVals)))
wht = sub.map(lambda x : dot(whtMat,x))

# do multiple independent component extraction
B = orth(random.randn(k,c))
Bold = zeros((k,c))
iterNum = 0
minAbsCos = 0
termTol = 0.0001
iterMax = 1000
errVec = zeros(iterMax)

while (iterNum < iterMax) & ((1 - minAbsCos)>termTol):
	iterNum += 1
	# update rule for pow3 nonlinearity (add other nonlins)
	B = wht.map(lambda x : outer(x,dot(x,B) ** 3)).reduce(lambda x,y : x + y) / n - 3 * B
	# orthognalize
	B = dot(B,real(sqrtm(inv(dot(transpose(B),B)))))
	# evaluate error
	minAbsCos = min(abs(diag(dot(transpose(B),Bold))))
	# store results
	Bold = B
	errVec[iterNum-1] = (1 - minAbsCos)

W = dot(transpose(B),whtMat)

print "W:" + str(W)

# need to save output files

<commit_msg>Generates results files<commit_after># ica <master> <inputFile> <outputFile_Components> <outputFile_Weights>
#
# Perform ICA on a data matrix.
# Input is a local text file or a file in HDFS.
# Format should be rows of ' ' separated values.
# - Example: space (rows) x time (cols).
# - Rows should be whichever dim is larger.
# Returns independent components and unmixing matrix in separate files

# k is number of PCs to use in initial dimensionality reduction
# c is the number of ICA components to return

import sys
from numpy import *
from scipy.linalg import *
from pyspark import SparkContext

if len(sys.argv) < 6:
  print >> sys.stderr, \
  "(ica) usage: ica <master> <inputFile> <outputFile> <k> <c>"
  exit(-1)

def parseVector(line):
    return array([float(x) for x in line.split(' ')])

# parse inputs
sc = SparkContext(sys.argv[1], "ica")
lines = sc.textFile(sys.argv[2])
outputFile = str(sys.argv[3])
k = int(sys.argv[4])
c = int(sys.argv[5])

# compute covariance matrix
data = lines.map(parseVector).cache()
n = data.count()
m = len(data.first())
meanVec = data.reduce(lambda x,y : x+y) / n
sub = data.map(lambda x : x - meanVec)
cov = sub.map(lambda x : outer(x,x)).reduce(lambda x,y : (x + y)) / n

# do eigenvector decomposition
w, v = eig(cov)
inds = argsort(w)[::1]
kEigVecs = v[:,inds[0:k]]
kEigVals = w[inds[0:k]]

# whiten data
whtMat = real(dot(inv(sqrtm(diag(kEigVals))),transpose(kEigVecs)))
unwhtMat = dot(kEigVecs,sqrtm(diag(kEigVals)))
wht = sub.map(lambda x : dot(whtMat,x))

# do multiple independent component extraction
B = orth(random.randn(k,c))
Bold = zeros((k,c))
iterNum = 0
minAbsCos = 0
termTol = 0.0001
iterMax = 1000
errVec = zeros(iterMax)

while (iterNum < iterMax) & ((1 - minAbsCos) > termTol):
	iterNum += 1
	print "(ica) starting iteration " + str(iterNum)
	# update rule for pow3 nonlinearity (todo: add other nonlins)
	B = wht.map(lambda x : outer(x,dot(x,B) ** 3)).reduce(lambda x,y : x + y) / n - 3 * B
	# orthognalize
	B = dot(B,real(sqrtm(inv(dot(transpose(B),B)))))
	# evaluate error
	minAbsCos = min(abs(diag(dot(transpose(B),Bold))))
	# store results
	Bold = B
	errVec[iterNum-1] = (1 - minAbsCos)

# get unmixing matrix
W = dot(transpose(B),whtMat)

# get unmixed signals
sigs = sub.map(lambda x : dot(W,x)).collect()

# save output files
print("(ica) writing output...")
savetxt("out-W-"+outputFile+".txt",W,fmt='%.8f')
savetxt("out-sigs-"+outputFile+".txt",sigs,fmt='%.8f')

<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
#
# This file is part of Invenio.
# Copyright (C) 2013, 2014, 2015 CERN.
#
# Invenio is free software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License as
# published by the Free Software Foundation; either version 2 of the
# License, or (at your option) any later version.
#
# Invenio is distributed in the hope that it will be useful, but
# WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
# General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Invenio; if not, write to the Free Software Foundation, Inc.,
# 59 Temple Place, Suite 330, Boston, MA 02111-1307, USA.

"""Initialize and configure Flask-Script extension.

Configuration
^^^^^^^^^^^^^
The following configuration variables are provided:

===================== =======================================================
`bind address`        Preferred binding address of the server. Can be used to
                      select a specific interface or to bind to all via
                      `0.0.0.0`.
`bind port`           Preferred binding port of the server. Can differ from
                      the one stated in `CFG_SITE_URL` so it can be accessed
                      via reverse proxy.
===================== =======================================================

They are assigned by the following parameters, in decreasing priority:

1. Command line arguments of `inveniomanage runserver`
2. `SERVER_BIND_ADDRESS` and `SERVER_BIND_PORT` configuration
3. Values guessed from `CFG_SITE_URL`
4. Defaults (`127.0.0.1:80`)
"""

from __future__ import print_function

import functools

import re

from types import FunctionType

from flask import current_app, flash

from flask_registry import ModuleAutoDiscoveryRegistry, RegistryProxy

from flask_script import Manager as FlaskExtManager
from flask_script.commands import Clean, Server, Shell, ShowUrls

from invenio.base.signals import post_command, pre_command

from six.moves import urllib

from werkzeug.utils import find_modules, import_string

def change_command_name(method=None, new_name=None):
    """Change command name to `new_name` or replace '_' by '-'."""
    if method is None:
        return functools.partial(change_command_name, new_name=new_name)

    if new_name is None:
        new_name = method.__name__.replace('_', '-')
    method.__name__ = new_name

    return method

def generate_secret_key():
    """Generate secret key."""
    import string
    import random
    return ''.join([random.choice(string.ascii_letters + string.digits)
                    for dummy in range(0, 256)])

def print_progress(p, L=40, prefix='', suffix=''):
    """Print textual progress bar."""
    bricks = int(p * L)
    print('\r', prefix, end=' ')
    print('[{0}{1}] {2}%'.format('#' * bricks, ' ' * (L - bricks),
                                 int(p * 100)), end=' ')
    print(suffix, end=' ')

def check_for_software_updates(flash_message=False):
    """Check for a new release of Invenio.

    :return: True if you have latest version, else False if you need to upgrade
             or None if server was not reachable.
    """
    from invenio.config import CFG_VERSION
    from invenio.base.i18n import _
    try:
        find = re.compile('Invenio v[0-9]+.[0-9]+.[0-9]+(\-rc[0-9])?'
                          ' is released')

        release_notes = 'https://raw.githubusercontent.com/' \
            'inveniosoftware/invenio/master/RELEASE-NOTES'

        webFile = urllib.request.urlopen(release_notes)

        temp = ""
        version = ""
        version1 = ""
        while 1:
            temp = webFile.readline()
            match1 = find.match(temp)
            try:
                version = match1.group()
                break
            except Exception:
                pass
            if not temp:
                break

        webFile.close()
        submatch = re.compile('[0-9]+.[0-9]+.[0-9]+(\-rc[0-9])?')
        version1 = submatch.search(version)
        web_version = version1.group().split(".")

        local_version = CFG_VERSION.split(".")

        if (web_version[0] > local_version[0] or
                web_version[0] == local_version[0] and
                web_version[1] > local_version[1] or
                web_version[0] == local_version[0] and
                web_version[1] == local_version[1] and
                web_version[2] > local_version[2]):
            if flash_message:
                flash(_('A newer version of Invenio is available for '
                        'download. You may want to visit '
                        '%()s',
                        wiki='<a href=\"http://invenio-software.org/wiki/'
                             '/Installation/Download'), 'warning')

            return False
    except Exception as e:
        print(e)
        if flash_message:
            flash(_('Cannot download or parse release notes '
                    'from %(release_notes)s', release_notes=release_notes),
                  'error')
        return None
    return True

class Manager(FlaskExtManager):

    """Custom manager implementation with signaling support."""

    def add_command(self, name, command):
        """Wrap default ``add_command`` method."""
        sender = command.run if type(command.run) is FunctionType \
            else command.__class__

        class SignalingCommand(command.__class__):
            def __call__(self, *args, **kwargs):
                app = self.app if not len(args) else args[0]
                with app.test_request_context():
                    pre_command.send(sender, args=args, **kwargs)
                res = super(SignalingCommand, self).__call__(*args, **kwargs)
                with app.test_request_context():
                    post_command.send(sender, args=args, **kwargs)
                return res

        command.__class__ = SignalingCommand
        return super(Manager, self).add_command(name, command)

def set_serve_static_files(sender, *args, **kwargs):
    """Enable serving of static files for `runserver` command.

    Normally Apache serves static files, but during development and if you are
    using the Werkzeug standalone development server, you can set this flag to
    `True`, to enable static file serving.
    """
    current_app.config.setdefault('CFG_FLASK_SERVE_STATIC_FILES', True)

pre_command.connect(set_serve_static_files, sender=Server)

def register_manager(manager):
    """Register all manager plugins and default commands with the manager."""
    from six.moves.urllib.parse import urlparse
    managers = RegistryProxy('managers', ModuleAutoDiscoveryRegistry, 'manage')

    with manager.app.app_context():
        for script in find_modules('invenio.base.scripts'):
            manager.add_command(script.split('.')[-1],
                                import_string(script + ':manager'))
        for script in managers:
            if script.__name__ == 'invenio.base.manage':
                continue
            manager.add_command(script.__name__.split('.')[-2],
                                getattr(script, 'manager'))

    manager.add_command("clean", Clean())
    manager.add_command("show-urls", ShowUrls())
    manager.add_command("shell", Shell())
    parsed_url = urlparse(manager.app.config.get('CFG_SITE_URL'))
    host = manager.app.config.get(
        'SERVER_BIND_ADDRESS',
        parsed_url.hostname or '127.0.0.1'
    )
    port = manager.app.config.get(
        'SERVER_BIND_PORT',
        parsed_url.port or 80
    )
    runserver = Server(host=host, port=port)
    manager.add_command("runserver", runserver)

    # FIXME separation of concerns is violated here.
    from invenio.ext.collect import collect
    collect.init_script(manager)

    from invenio.ext.assets import command, bower
    manager.add_command("assets", command)
    manager.add_command("bower", bower)
<commit_msg>script: secure RNG for secret key generation<commit_after># -*- coding: utf-8 -*-
#
# This file is part of Invenio.
# Copyright (C) 2013, 2014, 2015 CERN.
#
# Invenio is free software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License as
# published by the Free Software Foundation; either version 2 of the
# License, or (at your option) any later version.
#
# Invenio is distributed in the hope that it will be useful, but
# WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
# General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Invenio; if not, write to the Free Software Foundation, Inc.,
# 59 Temple Place, Suite 330, Boston, MA 02111-1307, USA.

"""Initialize and configure Flask-Script extension.

Configuration
^^^^^^^^^^^^^
The following configuration variables are provided:

===================== =======================================================
`bind address`        Preferred binding address of the server. Can be used to
                      select a specific interface or to bind to all via
                      `0.0.0.0`.
`bind port`           Preferred binding port of the server. Can differ from
                      the one stated in `CFG_SITE_URL` so it can be accessed
                      via reverse proxy.
===================== =======================================================

They are assigned by the following parameters, in decreasing priority:

1. Command line arguments of `inveniomanage runserver`
2. `SERVER_BIND_ADDRESS` and `SERVER_BIND_PORT` configuration
3. Values guessed from `CFG_SITE_URL`
4. Defaults (`127.0.0.1:80`)
"""

from __future__ import print_function

import functools

import re

from types import FunctionType

from flask import current_app, flash

from flask_registry import ModuleAutoDiscoveryRegistry, RegistryProxy

from flask_script import Manager as FlaskExtManager
from flask_script.commands import Clean, Server, Shell, ShowUrls

from invenio.base.signals import post_command, pre_command

from six.moves import urllib

from werkzeug.utils import find_modules, import_string

def change_command_name(method=None, new_name=None):
    """Change command name to `new_name` or replace '_' by '-'."""
    if method is None:
        return functools.partial(change_command_name, new_name=new_name)

    if new_name is None:
        new_name = method.__name__.replace('_', '-')
    method.__name__ = new_name

    return method

def generate_secret_key():
    """Generate secret key."""
    import string
    import random

    rng = random.SystemRandom()
    return ''.join(
        rng.choice(string.ascii_letters + string.digits)
        for dummy in range(0, 256)
    )

def print_progress(p, L=40, prefix='', suffix=''):
    """Print textual progress bar."""
    bricks = int(p * L)
    print('\r', prefix, end=' ')
    print('[{0}{1}] {2}%'.format('#' * bricks, ' ' * (L - bricks),
                                 int(p * 100)), end=' ')
    print(suffix, end=' ')

def check_for_software_updates(flash_message=False):
    """Check for a new release of Invenio.

    :return: True if you have latest version, else False if you need to upgrade
             or None if server was not reachable.
    """
    from invenio.config import CFG_VERSION
    from invenio.base.i18n import _
    try:
        find = re.compile('Invenio v[0-9]+.[0-9]+.[0-9]+(\-rc[0-9])?'
                          ' is released')

        release_notes = 'https://raw.githubusercontent.com/' \
            'inveniosoftware/invenio/master/RELEASE-NOTES'

        webFile = urllib.request.urlopen(release_notes)

        temp = ""
        version = ""
        version1 = ""
        while 1:
            temp = webFile.readline()
            match1 = find.match(temp)
            try:
                version = match1.group()
                break
            except Exception:
                pass
            if not temp:
                break

        webFile.close()
        submatch = re.compile('[0-9]+.[0-9]+.[0-9]+(\-rc[0-9])?')
        version1 = submatch.search(version)
        web_version = version1.group().split(".")

        local_version = CFG_VERSION.split(".")

        if (web_version[0] > local_version[0] or
                web_version[0] == local_version[0] and
                web_version[1] > local_version[1] or
                web_version[0] == local_version[0] and
                web_version[1] == local_version[1] and
                web_version[2] > local_version[2]):
            if flash_message:
                flash(_('A newer version of Invenio is available for '
                        'download. You may want to visit '
                        '%()s',
                        wiki='<a href=\"http://invenio-software.org/wiki/'
                             '/Installation/Download'), 'warning')

            return False
    except Exception as e:
        print(e)
        if flash_message:
            flash(_('Cannot download or parse release notes '
                    'from %(release_notes)s', release_notes=release_notes),
                  'error')
        return None
    return True

class Manager(FlaskExtManager):

    """Custom manager implementation with signaling support."""

    def add_command(self, name, command):
        """Wrap default ``add_command`` method."""
        sender = command.run if type(command.run) is FunctionType \
            else command.__class__

        class SignalingCommand(command.__class__):
            def __call__(self, *args, **kwargs):
                app = self.app if not len(args) else args[0]
                with app.test_request_context():
                    pre_command.send(sender, args=args, **kwargs)
                res = super(SignalingCommand, self).__call__(*args, **kwargs)
                with app.test_request_context():
                    post_command.send(sender, args=args, **kwargs)
                return res

        command.__class__ = SignalingCommand
        return super(Manager, self).add_command(name, command)

def set_serve_static_files(sender, *args, **kwargs):
    """Enable serving of static files for `runserver` command.

    Normally Apache serves static files, but during development and if you are
    using the Werkzeug standalone development server, you can set this flag to
    `True`, to enable static file serving.
    """
    current_app.config.setdefault('CFG_FLASK_SERVE_STATIC_FILES', True)

pre_command.connect(set_serve_static_files, sender=Server)

def register_manager(manager):
    """Register all manager plugins and default commands with the manager."""
    from six.moves.urllib.parse import urlparse
    managers = RegistryProxy('managers', ModuleAutoDiscoveryRegistry, 'manage')

    with manager.app.app_context():
        for script in find_modules('invenio.base.scripts'):
            manager.add_command(script.split('.')[-1],
                                import_string(script + ':manager'))
        for script in managers:
            if script.__name__ == 'invenio.base.manage':
                continue
            manager.add_command(script.__name__.split('.')[-2],
                                getattr(script, 'manager'))

    manager.add_command("clean", Clean())
    manager.add_command("show-urls", ShowUrls())
    manager.add_command("shell", Shell())
    parsed_url = urlparse(manager.app.config.get('CFG_SITE_URL'))
    host = manager.app.config.get(
        'SERVER_BIND_ADDRESS',
        parsed_url.hostname or '127.0.0.1'
    )
    port = manager.app.config.get(
        'SERVER_BIND_PORT',
        parsed_url.port or 80
    )
    runserver = Server(host=host, port=port)
    manager.add_command("runserver", runserver)

    # FIXME separation of concerns is violated here.
    from invenio.ext.collect import collect
    collect.init_script(manager)

    from invenio.ext.assets import command, bower
    manager.add_command("assets", command)
    manager.add_command("bower", bower)
<|endoftext|>


<commit_before>import os
import json
import twarc
import dotenv
import logging
import threading

dotenv.load_dotenv()
logging.basicConfig(filename="test.log", level=logging.INFO)

T = None
BEARER_TOKEN = os.environ.get("BEARER_TOKEN")

def test_bearer_token():
    assert BEARER_TOKEN

def test_constructor():
    global T
    T = twarc.Twarc2(bearer_token=BEARER_TOKEN)

def test_sample():
    count = 0

    # event to tell the filter stream to close
    event = threading.Event()

    for result in T.sample(event=event):
        assert int(result["data"]["id"])

        # users are passed by reference an dincluded as includes
        user_id = result["data"]["author_id"]
        assert len(pick_id(user_id, result["includes"]["users"])) == 1

        count += 1
        if count > 10:
            # close the sample
            event.set()

    assert count == 11

def test_sample_flattened():
    """
    This test uses the sample stream with the flatten option to move the
    expansions into place into each tweet. It will look at each tweet to find
    evidence that all the expansions have worked. Once it finds them all it
    stops. If it has listened to 5000 tweets and not found them it stops and
    assumes that something is not right. But it could be that it needed to wait
    longer. This 5000 cutoff may need to be adjusted based on experience.
    """
    found_geo = False
    found_in_reply_to_user = False
    found_attachments_media = False
    found_attachments_polls = False
    found_entities_mentions = False
    found_referenced_tweets = False

    count = 0
    event = threading.Event()
    for result in T.sample(flatten=True, event=event):
        tweet = result["data"]
        assert "id" in tweet
        logging.info("got sample tweet #%s %s", count, tweet["id"])

        author_id = tweet["author_id"]
        assert "author" in tweet
        assert result["data"]["author"]["id"] == author_id

        if "in_reply_to_user_id" in tweet:
            assert "in_reply_to_user" in tweet
            found_in_reply_to_user = True

        if "attachments" in tweet:
            if "media_keys" in tweet["attachments"]:
                assert "media" in tweet["attachments"]
                found_attachments_media = True
            if "poll_ids" in tweet["attachments"]:
                assert "polls" in tweet["attachments"]
                found_attachments_polls = True

        if "geo" in tweet:
            assert tweet["geo"]["place_id"]
            assert tweet["geo"]["place_id"] == tweet["geo"]["place"]["id"]
            found_geo = True

        if "entities" in tweet and "mentions" in tweet["entities"]:
            assert tweet["entities"]["mentions"][0]["username"]
            found_entities_mentions = True

        if "referenced_tweets" in tweet:
            assert tweet["referenced_tweets"][0]["id"]
            found_referenced_tweets = True

        count += 1
        if found_geo and found_in_reply_to_user and found_attachments_media \
                and found_attachments_polls and found_entities_mentions \
                and found_referenced_tweets:
            logging.info("found all expansions!")
            event.set()
        elif count > 5000:
            logging.info("didn't find all expansions in 5000 tweets")
            event.set()

    assert found_geo, "found geo"
    assert found_in_reply_to_user, "found in_reply_to_user"
    assert found_attachments_media, "found media"
    assert found_attachments_polls, "found polls"
    assert found_entities_mentions, "found mentions"
    assert found_referenced_tweets, "found referenced tweets"

def pick_id(id, objects):
    """pick an object out of a list of objects using its id
    """
    return list(filter(lambda o: o["id"] == id, objects))
<commit_msg>Move client constructor to the top level<commit_after>import os
import json
import twarc
import dotenv
import logging
import threading

dotenv.load_dotenv()
logging.basicConfig(filename="test.log", level=logging.INFO)

BEARER_TOKEN = os.environ.get("BEARER_TOKEN")

T = twarc.Twarc2(bearer_token=BEARER_TOKEN)

def test_bearer_token():
    assert BEARER_TOKEN

def test_sample():
    count = 0

    # event to tell the filter stream to close
    event = threading.Event()

    for result in T.sample(event=event):
        assert int(result["data"]["id"])

        # users are passed by reference an dincluded as includes
        user_id = result["data"]["author_id"]
        assert len(pick_id(user_id, result["includes"]["users"])) == 1

        count += 1
        if count > 10:
            # close the sample
            event.set()

    assert count == 11

def test_sample_flattened():
    """
    This test uses the sample stream with the flatten option to move the
    expansions into place into each tweet. It will look at each tweet to find
    evidence that all the expansions have worked. Once it finds them all it
    stops. If it has listened to 5000 tweets and not found them it stops and
    assumes that something is not right. But it could be that it needed to wait
    longer. This 5000 cutoff may need to be adjusted based on experience.
    """
    found_geo = False
    found_in_reply_to_user = False
    found_attachments_media = False
    found_attachments_polls = False
    found_entities_mentions = False
    found_referenced_tweets = False

    count = 0
    event = threading.Event()
    for result in T.sample(flatten=True, event=event):
        tweet = result["data"]
        assert "id" in tweet
        logging.info("got sample tweet #%s %s", count, tweet["id"])

        author_id = tweet["author_id"]
        assert "author" in tweet
        assert result["data"]["author"]["id"] == author_id

        if "in_reply_to_user_id" in tweet:
            assert "in_reply_to_user" in tweet
            found_in_reply_to_user = True

        if "attachments" in tweet:
            if "media_keys" in tweet["attachments"]:
                assert "media" in tweet["attachments"]
                found_attachments_media = True
            if "poll_ids" in tweet["attachments"]:
                assert "polls" in tweet["attachments"]
                found_attachments_polls = True

        if "geo" in tweet:
            assert tweet["geo"]["place_id"]
            assert tweet["geo"]["place_id"] == tweet["geo"]["place"]["id"]
            found_geo = True

        if "entities" in tweet and "mentions" in tweet["entities"]:
            assert tweet["entities"]["mentions"][0]["username"]
            found_entities_mentions = True

        if "referenced_tweets" in tweet:
            assert tweet["referenced_tweets"][0]["id"]
            found_referenced_tweets = True

        count += 1
        if found_geo and found_in_reply_to_user and found_attachments_media \
                and found_attachments_polls and found_entities_mentions \
                and found_referenced_tweets:
            logging.info("found all expansions!")
            event.set()
        elif count > 5000:
            logging.info("didn't find all expansions in 5000 tweets")
            event.set()

    assert found_geo, "found geo"
    assert found_in_reply_to_user, "found in_reply_to_user"
    assert found_attachments_media, "found media"
    assert found_attachments_polls, "found polls"
    assert found_entities_mentions, "found mentions"
    assert found_referenced_tweets, "found referenced tweets"

def pick_id(id, objects):
    """pick an object out of a list of objects using its id
    """
    return list(filter(lambda o: o["id"] == id, objects))
<|endoftext|>