<commit_before>import json
import os
import logging

import numpy as np
from flask import jsonify, send_file, current_app, request
from flask.ext.jwt import jwt_required
from flask.ext.jwt import current_identity

from tmlib.workflow.description import WorkflowDescription
from tmlib.workflow.submission import SubmissionManager
from tmlib.workflow.tmaps.api import WorkflowManager
from tmlib.models import (
    Experiment, ChannelLayer, Plate, Acquisition, Feature, PyramidTileFile
)
from tmlib.image import PyramidTile
from tmserver.util import (
    extract_model_from_path,
    extract_model_from_body
)
from tmserver.model import decode_pk
from tmserver.extensions import db
from tmserver.extensions import gc3pie
from tmserver.api import api
from tmserver.error import (
    MalformedRequestError,
    MissingGETParameterError,
    MissingPOSTParameterError,
    ResourceNotFoundError,
    NotAuthorizedError
)

logger = logging.getLogger(__name__)

@api.route('/channel_layers/<channel_layer_id>/tiles', methods=['GET'])
@extract_model_from_path(ChannelLayer)
def get_image_tile(channel_layer):
    """Sends a tile image for a specific layer.
    This route is accessed by openlayers."""
    x = request.args.get('x')
    y = request.args.get('y')
    z = request.args.get('z')
    if not x or not y or not z:
        raise MalformedRequestError()
    else:
        x = int(x)
        y = int(y)
        z = int(z)

    logger.debug(
        'get image tile: x=%d, y=%d, z=%d, zplane=%d, tpoint=%d',
        x, y, z, channel_layer.zplane, channel_layer.tpoint
    )

    tile_file = db.session.query(PyramidTileFile).filter_by(
        column=x, row=y, level=z, channel_layer_id=channel_layer.id
    ).one_or_none()
    if tile_file is None:
        raise ResourceNotFoundError(
            'Tile not found: column=%d, row=%d, level=%d', x, y, z
        )
    return send_file(tile_file.location)
    #     logger.warn('file does not exist - send empty image')
    #     from cStringIO import StringIO
    #     f = StringIO()
    #     img = PyramidTile.create_as_background()
    #     buf = img.array.get_buffer()
    #     f.write(buf.tostring)
    #     f.seek(0)
    #     send_file(f, mimetype='image/jpeg')

@api.route('/features', methods=['GET'])
@jwt_required()
@extract_model_from_path(Experiment, check_ownership=True)
def get_features(experiment):
    """Sends a list of feature objects.

    Request
    -------

    Required GET parameters:
        - experiment_id

    Response
    --------

    {
        "data": {
            mapobject_type_name: [
                {
                    "name": string
                },
                ...
            ],
            ...
        }
    }

    """
    experiment_id = request.args.get('experiment_id')
    if not experiment_id:
        raise MalformedRequestError(
            'The GET parameter "experiment_id" is required.')

    features = db.session.query(Feature).filter_by(experiment_id=experiment_id).all()

    return jsonify({
        'data': features
    })

@api.route('/experiments', methods=['GET'])
@jwt_required()
def get_experiments():
    """Gets all experiments for the current user.

    Response
    --------
    {
        "data": list of experiment objects,
    }

    """
    return jsonify({
        'data': current_identity.experiments
    })

@api.route('/experiments/<experiment_id>', methods=['GET'])
@jwt_required()
@extract_model_from_path(Experiment)
def get_experiment(experiment):
    """Gets an experiment by id.

    Response
    --------
    {
        "data": an experiment object serialized to json
    }

    """
    return jsonify({
        'data': experiment
    })

@api.route('/experiments/<experiment_id>/workflow/submit', methods=['POST'])
@jwt_required()
@extract_model_from_path(Experiment)
def submit_workflow(experiment):
    logger.info('submit workflow')
    data = json.loads(request.data)
    workflow_description = WorkflowDescription(**data['description'])
    experiment.persist_workflow_description(workflow_description)
    workflow_manager = WorkflowManager(experiment.id, 1)
    submission_manager = SubmissionManager(experiment.id, 'workflow')
    submission_id, user_name = submission_manager.register_submission()
    workflow = workflow_manager.create_workflow(
        submission_id, user_name, workflow_description
    )
    gc3pie.store_jobs(experiment, workflow)
    gc3pie.submit_jobs(workflow)

    return jsonify({
        'message': 'ok',
        'submission_id': workflow.submission_id
    })

@api.route('/experiments/<experiment_id>/workflow/resubmit', methods=['POST'])
@jwt_required()
@extract_model_from_path(Experiment)
def resubmit_workflow(experiment):
    logger.info('resubmit workflow')
    data = json.loads(request.data)
    index = data.get('index', 0)
    workflow_description = WorkflowDescription(**data['description'])
    workflow = gc3pie.retrieve_jobs(experiment, 'workflow')
    workflow.update_description(workflow_description)
    workflow.update_stage(index)
    gc3pie.resubmit_jobs(workflow, index)

    return jsonify({
        'message': 'ok',
        'submission_id': workflow.submission_id
    })

@api.route('/experiments/<experiment_id>/workflow/status', methods=['GET'])
@jwt_required()
@extract_model_from_path(Experiment)
def get_workflow_status(experiment):
    logger.info('get workflow status')
    workflow = gc3pie.retrieve_jobs(experiment, 'workflow')
    status = gc3pie.get_status_of_submitted_jobs(workflow)
    return jsonify({
        'data': status
    })

@api.route('/experiments/<experiment_id>/workflow/load', methods=['GET'])
@jwt_required()
@extract_model_from_path(Experiment)
def get_workflow_description(experiment):
    logger.info('get workflow description')
    description = experiment.workflow_description
    return jsonify({
        'data': description.as_dict()
    })

@api.route('/experiments/<experiment_id>/workflow/save', methods=['POST'])
@jwt_required()
@extract_model_from_path(Experiment)
def save_workflow_description(experiment):
    data = json.loads(request.data)
    workflow_description = WorkflowDescription(**data['description'])
    experiment.persist_workflow_description(workflow_description)
    return jsonify({
        'message': 'ok'
    })

@api.route('/experiments/<experiment_id>/workflow/kill', methods=['POST'])
@jwt_required()
@extract_model_from_path(Experiment)
def kill_workflow(experiment):
    data = json.loads(request.data)
    logger.info('kill workflow')
    workflow = gc3pie.retrieve_jobs(experiment, 'workflow')
    gc3pie.kill_jobs(workflow)
    return jsonify({
        'message': 'ok'
    })

@api.route('/experiments/<experiment_id>/workflow/log', methods=['POST'])
@jwt_required()
@extract_model_from_path(Experiment)
def get_job_log_output(experiment):
    data = json.loads(request.data)
    logger.info('get job log output')
    job = gc3pie.retrieve_single_job(data['id'])
    stdout_file = os.path.join(job.output_dir, job.stdout)
    with open(stdout_file, 'r') as f:
        out = f.read()
    stderr_file = os.path.join(job.output_dir, job.stderr)
    with open(stderr_file, 'r') as f:
        err = f.read()
    return jsonify({
        'message': 'ok',
        'stdout': out,
        'stderr': err
    })

@api.route('/experiments', methods=['POST'])
@jwt_required()
def create_experiment():
    data = json.loads(request.data)

    name = data.get('name')
    description = data.get('description', '')
    microscope_type = data.get('microscope_type')
    plate_format = data.get('plate_format')
    plate_acquisition_mode = data.get('plate_acquisition_mode')

    if any([var is None for var in [name, microscope_type, plate_format,
                                    plate_acquisition_mode]]):
        raise MalformedRequestError()

    e = Experiment(
        name=name,
        description=description,
        user_id=current_identity.id,
        microscope_type=microscope_type,
        plate_format=plate_format,
        root_directory=current_app.config['TMAPS_STORAGE_HOME'],
        plate_acquisition_mode=plate_acquisition_mode
    )
    db.session.add(e)
    db.session.commit()

    return jsonify({
        'data': e
    })

@api.route('/experiments/<experiment_id>', methods=['DELETE'])
@jwt_required()
@extract_model_from_path(Experiment, check_ownership=True)
def delete_experiment(experiment):
    """Delete an experiment for the current user.

    Response
    --------
    {
        "message": 'Deletion ok'
    }

    """
    db.session.delete(experiment)
    db.session.commit()

    return jsonify(message='Deletion ok')

@api.route('/plates/<plate_id>', methods=['GET'])
@jwt_required()
@extract_model_from_path(Plate, check_ownership=True)
def get_plate(plate):
    return jsonify(data=plate)

@api.route('/plates', methods=['GET'])
@jwt_required()
def get_plates():
    """Get all plates for a specific experiment.

    Request
    -------

    Required GET parameters:
        - experiment_id

    Response
    --------

    {
        "data": [
            {
                "id": string,
                "name": string,
                "description": string,
                "experiment_id": number,
                "acquisition": Object
            },
            ...
        ]
    }

    """
    experiment_id = request.args.get('experiment_id')
    if not experiment_id:
        raise MissingGETParameterError('experiment_id')
    experiment = db.session.query(Experiment).get_with_hash(experiment_id)
    if not experiment:
        raise ResourceNotFoundError('Experiment')
    if not experiment.belongs_to(current_identity):
        raise NotAuthorizedError()

    return jsonify(data=experiment.plates)

@api.route('/plates/<plate_id>', methods=['DELETE'])
@jwt_required()
@extract_model_from_path(Plate, check_ownership=True)
def delete_plate(plate):
    db.session.delete(plate)
    db.session.commit()

    return jsonify(message='Deletion ok')

@api.route('/plates', methods=['POST'])
@jwt_required()
@extract_model_from_body(Experiment, check_ownership=True)
def create_plate(experiment):
    """
    Create a new plate for the experiment with id `experiment_id`.

    Request
    -------

    {
        name: string,
        description: string,
        experiment_id: string
    }

    Response
    --------

    Plate object

    """
    data = json.loads(request.data)
    name = data.get('name')
    desc = data.get('description', '')

    if not name:
        raise MissingPOSTParameterError('name')

    pl = Plate(
        name=name, description=desc,
        experiment_id=experiment.id
    )
    db.session.add(pl)
    db.session.commit()

    return jsonify(data=pl)

@api.route('/acquisitions', methods=['POST'])
@jwt_required()
@extract_model_from_body(Plate, check_ownership=True)
def create_acquisition(plate):
    """
    Create a new acquisition for the plate with id `plate_id`.

    Request
    {
        name: string,
        description: string,
        plate_id: string
    }

    Response
    --------

    Acquisition object

    """
    data = json.loads(request.data)
    name = data.get('name')
    desc = data.get('description', '')

    if not name:
        raise MissingPOSTParameterError('name')

    aq = Acquisition(
        name=name, description=desc,
        plate_id=plate.id
    )
    db.session.add(aq)
    db.session.commit()

    return jsonify(data=aq)

@api.route('/acquisitions/<acquisition_id>', methods=['DELETE'])
@jwt_required()
@extract_model_from_path(Acquisition, check_ownership=True)
def delete_acquisition(acquisition):
    db.session.delete(acquisition)
    db.session.commit()
    return jsonify(message='Deletion ok')

@api.route('/acquisitions/<acquisition_id>', methods=['GET'])
@jwt_required()
@extract_model_from_path(Acquisition, check_ownership=True)
def get_acquisition(acquisition):
    return jsonify(data=acquisition)

@api.route('/acquisitions/<acquisition_id>/image_files', methods=['GET'])
@jwt_required()
@extract_model_from_path(Acquisition, check_ownership=True)
def get_acquisition_image_files(acquisition):
    return jsonify(
        data=[
            {'name': f.name, 'upload_status': f.upload_status}
            for f in acquisition.microscope_image_files
        ]
    )

@api.route('/acquisitions/<acquisition_id>/metadata_files', methods=['GET'])
@jwt_required()
@extract_model_from_path(Acquisition, check_ownership=True)
def get_acquisition_metadata_files(acquisition):
    return jsonify(
        data=[
            {'name': f.name, 'upload_status': f.upload_status}
            for f in acquisition.microscope_metadata_files
        ]
    )
<commit_msg>Restful api for requesting supported microscope types and acquisition modes<commit_after>import json
import os
import logging

import numpy as np
from flask import jsonify, send_file, current_app, request
from flask.ext.jwt import jwt_required
from flask.ext.jwt import current_identity

from tmlib.workflow.description import WorkflowDescription
from tmlib.workflow.submission import SubmissionManager
from tmlib.workflow.tmaps.api import WorkflowManager
from tmlib.models import (
    Experiment, ChannelLayer, Plate, Acquisition, Feature, PyramidTileFile
)
from tmlib.image import PyramidTile
from tmlib.workflow.metaconfig import SUPPORTED_MICROSCOPE_TYPES
from tmlib.models.plate import SUPPORTED_PLATE_AQUISITION_MODES
from tmserver.util import (
    extract_model_from_path,
    extract_model_from_body
)
from tmserver.model import decode_pk
from tmserver.extensions import db
from tmserver.extensions import gc3pie
from tmserver.api import api
from tmserver.error import (
    MalformedRequestError,
    MissingGETParameterError,
    MissingPOSTParameterError,
    ResourceNotFoundError,
    NotAuthorizedError
)

logger = logging.getLogger(__name__)

@api.route('/channel_layers/<channel_layer_id>/tiles', methods=['GET'])
@extract_model_from_path(ChannelLayer)
def get_image_tile(channel_layer):
    """Sends a tile image for a specific layer.
    This route is accessed by openlayers."""
    x = request.args.get('x')
    y = request.args.get('y')
    z = request.args.get('z')
    if not x or not y or not z:
        raise MalformedRequestError()
    else:
        x = int(x)
        y = int(y)
        z = int(z)

    logger.debug(
        'get image tile: x=%d, y=%d, z=%d, zplane=%d, tpoint=%d',
        x, y, z, channel_layer.zplane, channel_layer.tpoint
    )

    tile_file = db.session.query(PyramidTileFile).filter_by(
        column=x, row=y, level=z, channel_layer_id=channel_layer.id
    ).one_or_none()
    if tile_file is None:
        raise ResourceNotFoundError(
            'Tile not found: column=%d, row=%d, level=%d', x, y, z
        )
    return send_file(tile_file.location)
    #     logger.warn('file does not exist - send empty image')
    #     from cStringIO import StringIO
    #     f = StringIO()
    #     img = PyramidTile.create_as_background()
    #     buf = img.array.get_buffer()
    #     f.write(buf.tostring)
    #     f.seek(0)
    #     send_file(f, mimetype='image/jpeg')

@api.route('/features', methods=['GET'])
@jwt_required()
@extract_model_from_path(Experiment, check_ownership=True)
def get_features(experiment):
    """Sends a list of feature objects.

    Request
    -------

    Required GET parameters:
        - experiment_id

    Response
    --------

    {
        "data": {
            mapobject_type_name: [
                {
                    "name": string
                },
                ...
            ],
            ...
        }
    }

    """
    experiment_id = request.args.get('experiment_id')
    if not experiment_id:
        raise MalformedRequestError(
            'The GET parameter "experiment_id" is required.')

    features = db.session.query(Feature).filter_by(experiment_id=experiment_id).all()

    return jsonify({
        'data': features
    })

@api.route('/microscope_types', methods=['GET'])
@jwt_required()
def get_microscope_types():
    """Gets all implemented microscope types.

    Response
    --------
    {
        "data": list of microscope types,
    }

    See also
    --------
    :py:class:`tmlib.workflow.metaconfig.SUPPORTED_MICROSCOPE_TYPES`
    """
    return jsonify({
        'data': list(SUPPORTED_MICROSCOPE_TYPES)
    })

@api.route('/acquisition_modes', methods=['GET'])
@jwt_required()
def get_acquisition_modes():
    """Gets all implemented plate acquisition modes.

    Response
    --------
    {
        "data": list of plate acquisition modes,
    }

    See also
    --------
    :py:class:`tmlib.models.plate.SUPPORTED_PLATE_AQUISITION_MODES`
    """
    return jsonify({
        'data': list(SUPPORTED_PLATE_AQUISITION_MODES)
    })

@api.route('/experiments', methods=['GET'])
@jwt_required()
def get_experiments():
    """Gets all experiments for the current user.

    Response
    --------
    {
        "data": list of experiment objects,
    }

    """
    return jsonify({
        'data': current_identity.experiments
    })

@api.route('/experiments/<experiment_id>', methods=['GET'])
@jwt_required()
@extract_model_from_path(Experiment)
def get_experiment(experiment):
    """Gets an experiment by id.

    Response
    --------
    {
        "data": an experiment object serialized to json
    }

    """
    return jsonify({
        'data': experiment
    })

@api.route('/experiments/<experiment_id>/workflow/submit', methods=['POST'])
@jwt_required()
@extract_model_from_path(Experiment)
def submit_workflow(experiment):
    logger.info('submit workflow')
    data = json.loads(request.data)
    workflow_description = WorkflowDescription(**data['description'])
    experiment.persist_workflow_description(workflow_description)
    workflow_manager = WorkflowManager(experiment.id, 1)
    submission_manager = SubmissionManager(experiment.id, 'workflow')
    submission_id, user_name = submission_manager.register_submission()
    workflow = workflow_manager.create_workflow(
        submission_id, user_name, workflow_description
    )
    gc3pie.store_jobs(experiment, workflow)
    gc3pie.submit_jobs(workflow)

    return jsonify({
        'message': 'ok',
        'submission_id': workflow.submission_id
    })

@api.route('/experiments/<experiment_id>/workflow/resubmit', methods=['POST'])
@jwt_required()
@extract_model_from_path(Experiment)
def resubmit_workflow(experiment):
    logger.info('resubmit workflow')
    data = json.loads(request.data)
    index = data.get('index', 0)
    workflow_description = WorkflowDescription(**data['description'])
    workflow = gc3pie.retrieve_jobs(experiment, 'workflow')
    workflow.update_description(workflow_description)
    workflow.update_stage(index)
    gc3pie.resubmit_jobs(workflow, index)

    return jsonify({
        'message': 'ok',
        'submission_id': workflow.submission_id
    })

@api.route('/experiments/<experiment_id>/workflow/status', methods=['GET'])
@jwt_required()
@extract_model_from_path(Experiment)
def get_workflow_status(experiment):
    logger.info('get workflow status')
    workflow = gc3pie.retrieve_jobs(experiment, 'workflow')
    status = gc3pie.get_status_of_submitted_jobs(workflow)
    return jsonify({
        'data': status
    })

@api.route('/experiments/<experiment_id>/workflow/load', methods=['GET'])
@jwt_required()
@extract_model_from_path(Experiment)
def get_workflow_description(experiment):
    logger.info('get workflow description')
    description = experiment.workflow_description
    return jsonify({
        'data': description.as_dict()
    })

@api.route('/experiments/<experiment_id>/workflow/save', methods=['POST'])
@jwt_required()
@extract_model_from_path(Experiment)
def save_workflow_description(experiment):
    data = json.loads(request.data)
    workflow_description = WorkflowDescription(**data['description'])
    experiment.persist_workflow_description(workflow_description)
    return jsonify({
        'message': 'ok'
    })

@api.route('/experiments/<experiment_id>/workflow/kill', methods=['POST'])
@jwt_required()
@extract_model_from_path(Experiment)
def kill_workflow(experiment):
    data = json.loads(request.data)
    logger.info('kill workflow')
    workflow = gc3pie.retrieve_jobs(experiment, 'workflow')
    gc3pie.kill_jobs(workflow)
    return jsonify({
        'message': 'ok'
    })

@api.route('/experiments/<experiment_id>/workflow/log', methods=['POST'])
@jwt_required()
@extract_model_from_path(Experiment)
def get_job_log_output(experiment):
    data = json.loads(request.data)
    logger.info('get job log output')
    job = gc3pie.retrieve_single_job(data['id'])
    stdout_file = os.path.join(job.output_dir, job.stdout)
    with open(stdout_file, 'r') as f:
        out = f.read()
    stderr_file = os.path.join(job.output_dir, job.stderr)
    with open(stderr_file, 'r') as f:
        err = f.read()
    return jsonify({
        'message': 'ok',
        'stdout': out,
        'stderr': err
    })

@api.route('/experiments', methods=['POST'])
@jwt_required()
def create_experiment():
    data = json.loads(request.data)

    name = data.get('name')
    description = data.get('description', '')
    microscope_type = data.get('microscope_type')
    plate_format = data.get('plate_format')
    plate_acquisition_mode = data.get('plate_acquisition_mode')

    if any([var is None for var in [name, microscope_type, plate_format,
                                    plate_acquisition_mode]]):
        raise MalformedRequestError()

    e = Experiment(
        name=name,
        description=description,
        user_id=current_identity.id,
        microscope_type=microscope_type,
        plate_format=plate_format,
        root_directory=current_app.config['TMAPS_STORAGE_HOME'],
        plate_acquisition_mode=plate_acquisition_mode
    )
    db.session.add(e)
    db.session.commit()

    return jsonify({
        'data': e
    })

@api.route('/experiments/<experiment_id>', methods=['DELETE'])
@jwt_required()
@extract_model_from_path(Experiment, check_ownership=True)
def delete_experiment(experiment):
    """Delete an experiment for the current user.

    Response
    --------
    {
        "message": 'Deletion ok'
    }

    """
    db.session.delete(experiment)
    db.session.commit()

    return jsonify(message='Deletion ok')

@api.route('/plates/<plate_id>', methods=['GET'])
@jwt_required()
@extract_model_from_path(Plate, check_ownership=True)
def get_plate(plate):
    return jsonify(data=plate)

@api.route('/plates', methods=['GET'])
@jwt_required()
def get_plates():
    """Get all plates for a specific experiment.

    Request
    -------

    Required GET parameters:
        - experiment_id

    Response
    --------

    {
        "data": [
            {
                "id": string,
                "name": string,
                "description": string,
                "experiment_id": number,
                "acquisition": Object
            },
            ...
        ]
    }

    """
    experiment_id = request.args.get('experiment_id')
    if not experiment_id:
        raise MissingGETParameterError('experiment_id')
    experiment = db.session.query(Experiment).get_with_hash(experiment_id)
    if not experiment:
        raise ResourceNotFoundError('Experiment')
    if not experiment.belongs_to(current_identity):
        raise NotAuthorizedError()

    return jsonify(data=experiment.plates)

@api.route('/plates/<plate_id>', methods=['DELETE'])
@jwt_required()
@extract_model_from_path(Plate, check_ownership=True)
def delete_plate(plate):
    db.session.delete(plate)
    db.session.commit()

    return jsonify(message='Deletion ok')

@api.route('/plates', methods=['POST'])
@jwt_required()
@extract_model_from_body(Experiment, check_ownership=True)
def create_plate(experiment):
    """
    Create a new plate for the experiment with id `experiment_id`.

    Request
    -------

    {
        name: string,
        description: string,
        experiment_id: string
    }

    Response
    --------

    Plate object

    """
    data = json.loads(request.data)
    name = data.get('name')
    desc = data.get('description', '')

    if not name:
        raise MissingPOSTParameterError('name')

    pl = Plate(
        name=name, description=desc,
        experiment_id=experiment.id
    )
    db.session.add(pl)
    db.session.commit()

    return jsonify(data=pl)

@api.route('/acquisitions', methods=['POST'])
@jwt_required()
@extract_model_from_body(Plate, check_ownership=True)
def create_acquisition(plate):
    """
    Create a new acquisition for the plate with id `plate_id`.

    Request
    {
        name: string,
        description: string,
        plate_id: string
    }

    Response
    --------

    Acquisition object

    """
    data = json.loads(request.data)
    name = data.get('name')
    desc = data.get('description', '')

    if not name:
        raise MissingPOSTParameterError('name')

    aq = Acquisition(
        name=name, description=desc,
        plate_id=plate.id
    )
    db.session.add(aq)
    db.session.commit()

    return jsonify(data=aq)

@api.route('/acquisitions/<acquisition_id>', methods=['DELETE'])
@jwt_required()
@extract_model_from_path(Acquisition, check_ownership=True)
def delete_acquisition(acquisition):
    db.session.delete(acquisition)
    db.session.commit()
    return jsonify(message='Deletion ok')

@api.route('/acquisitions/<acquisition_id>', methods=['GET'])
@jwt_required()
@extract_model_from_path(Acquisition, check_ownership=True)
def get_acquisition(acquisition):
    return jsonify(data=acquisition)

@api.route('/acquisitions/<acquisition_id>/image_files', methods=['GET'])
@jwt_required()
@extract_model_from_path(Acquisition, check_ownership=True)
def get_acquisition_image_files(acquisition):
    return jsonify(
        data=[
            {'name': f.name, 'upload_status': f.upload_status}
            for f in acquisition.microscope_image_files
        ]
    )

@api.route('/acquisitions/<acquisition_id>/metadata_files', methods=['GET'])
@jwt_required()
@extract_model_from_path(Acquisition, check_ownership=True)
def get_acquisition_metadata_files(acquisition):
    return jsonify(
        data=[
            {'name': f.name, 'upload_status': f.upload_status}
            for f in acquisition.microscope_metadata_files
        ]
    )
<|endoftext|>


<commit_before>## Automatically adapted for numpy.oldnumeric Jun 27, 2008 by -c

# $Id$
#
#  Copyright (C) 2006-2008  greg Landrum and Rational Discovery LLC
#
#   @@ All Rights Reserved  @@
#
""" Import all RDKit chemistry modules

"""
import rdBase
import RDConfig
import DataStructs
from Geometry import rdGeometry
from Chem import *
from rdPartialCharges import *
from rdDepictor import *
from rdForceFieldHelpers import *
from Chem.ChemicalFeatures import *
from rdDistGeom import *
from rdMolAlign import *
from rdMolTransforms import *
from rdShapeHelpers import *
from rdChemReactions import *
import ForceField
Mol.Compute2DCoords = Compute2DCoords
Mol.ComputeGasteigerCharges = ComputeGasteigerCharges
import numpy

def TransformMol(mol,tform):
  """  Applies the transformation to a molecule and sets it up with
  a single conformer

  """
  newConf = Conformer()
  newConf.SetId(0)
  refConf = mol.GetConformer()
  for i in range(refConf.GetNumAtoms()):
    pos = list(refConf.GetAtomPosition(i))
    pos.append(1.0)
    newPos = numpy.dot(tform,numpy.array(pos))
    newConf.SetAtomPosition(i,list(newPos)[:3])
  mol.RemoveAllConformers()
  mol.AddConformer(newConf)

def ComputeMolShape(mol,confId=-1,boxDim=(20,20,20),spacing=0.5,**kwargs):
  res = rdGeometry.UniformGrid3D(boxDim[0],boxDim[1],boxDim[2],spacing=spacing)
  apply(EncodeShape,(mol,res,confId),kwargs)
  return res

def ComputeMolVolume(mol,confId=-1,gridSpacing=0.1,boxMargin=2.0):
  import copy
  mol = copy.deepcopy(mol)
  conf = mol.GetConformer(confId)
  CanonicalizeConformer(conf)
  box = ComputeConfBox(conf)
  sideLen = ( box[1].x-box[0].x + 2*boxMargin, \
              box[1].y-box[0].y + 2*boxMargin, \
              box[1].z-box[0].z + 2*boxMargin )
  shape = rdGeometry.UniformGrid3D(sideLen[0],sideLen[1],sideLen[2],
                                   spacing=gridSpacing)
  EncodeShape(mol,shape,confId,ignoreHs=False,vdwScale=1.0)
  voxelVol = gridSpacing**3
  vol = 0.0
  occVect = shape.GetOccupancyVect()
  for i in range(len(occVect)):
    if occVect[i]==3:
      vol+= voxelVol
  return vol

def GenerateDepictionMatching3DStructure(mol,reference,confId=-1,
                                         **kwargs):
  nAts = mol.GetNumAtoms()
  dm = []
  conf = mol.GetConformer(confId)
  for i in range(nAts):
    pi = conf.GetAtomPosition(i)
    for j in range(i+1,nAts):
      pj = conf.GetAtomPosition(j)
      dm.append((pi-pj).Length())
  dm = numpy.array(dm)
  apply(Compute2DCoordsMimicDistmat,(mol,dm),kwargs)

def GetBestRMS(ref,probe,refConfId=-1,probeConfId=-1,maps=None):
  if not maps:
    query = RemoveHs(probe)
    matches = ref.GetSubstructMatches(query)
    if not matches:
      raise ValueError,'mol %s does not match mol %s'%(ref.GetProp('_Name'),
                                                       probe.GetProp('_Name'))
    maps = []
    for match in matches:
      t=[]
      for j,idx in enumerate(match):
        t.append((j,idx))
    maps.append(t)
  bestRMS=1000.
  for amap in maps:
    rms=AlignMol(probe,ref,probeConfId,refConfId,atomMap=amap)
    if rms<bestRMS:
      bestRMS=rms
  return bestRMS

def EnumerateLibraryFromReaction(reaction,sidechainSets) :
  """ Returns a generator for the virtual library defined by
   a reaction and a sequence of sidechain sets

  >>> import Chem
  >>> from Chem import AllChem
  >>> s1=[Chem.MolFromSmiles(x) for x in ('NC','NCC')]
  >>> s2=[Chem.MolFromSmiles(x) for x in ('OC=O','OC(=O)C')]
  >>> rxn = AllChem.ReactionFromSmarts('[O:2]=[C:1][OH].[N:3]>>[O:2]=[C:1][N:3]')
  >>> r = AllChem.EnumerateLibraryFromReaction(rxn,[s2,s1])
  >>> [Chem.MolToSmiles(x[0]) for x in list(r)]
  ['CNC=O', 'CCNC=O', 'CNC(=O)C', 'CCNC(=O)C']

  Note that this is all done in a lazy manner, so "infinitely" large libraries can
  be done without worrying about running out of memory. Your patience will run out first:

  Define a set of 10000 amines:
  >>> amines = (Chem.MolFromSmiles('N'+'C'*x) for x in range(10000))

  ... a set of 10000 acids
  >>> acids = (Chem.MolFromSmiles('OC(=O)'+'C'*x) for x in range(10000))

  ... now the virtual library (1e8 compounds in principle):
  >>> r = AllChem.EnumerateLibraryFromReaction(rxn,[acids,amines])

  ... look at the first 4 compounds:
  >>> [Chem.MolToSmiles(r.next()[0]) for x in range(4)]
  ['NC=O', 'CNC=O', 'CCNC=O', 'CCCNC=O']

  """
  if len(sidechainSets) != reaction.GetNumReactantTemplates():
    raise ValueError,'%d sidechains provided, %d required'%(len(sidechainSets),reaction.getNumReactantTemplates())

  def _combiEnumerator(items,depth=0):
    for item in items[depth]:
      if depth+1 < len(items):
        v = _combiEnumerator(items,depth+1)
        for entry in v:
          l=[item]
          l.extend(entry)
          yield l
      else:
        yield [item]
  for chains in _combiEnumerator(sidechainSets):
    prodSets = reaction.RunReactants(chains)
    for prods in prodSets:
      yield prods

def ConstrainedEmbed(mol,core,useTethers,randomseed=2342):
  match = mol.GetSubstructMatch(core)
  if not match:
    raise ValueError,"molecule doesn't match the core"
  coordMap={}
  coreConf = core.GetConformer()
  for i,idxI in enumerate(match):
    corePtI = coreConf.GetAtomPosition(i)
    coordMap[idxI]=corePtI

  ci = EmbedMolecule(mol,coordMap=coordMap,randomSeed=randomseed)
  if ci<0:
    logger.error('could not embed molecule %s, no coordinates generated.'%mol.GetProp('_Name'))

  algMap=[]
  for i,itm in enumerate(match):
    algMap.append((itm,i))

  if not useTethers:
    # clean up the conformation
    ff = UFFGetMoleculeForceField(mol,confId=0)
    for i,idxI in enumerate(match):
      for j in range(i+1,len(match)):
        idxJ = match[j]
        d = coordMap[idxI].Distance(coordMap[idxJ])
        ff.AddDistanceConstraint(idxI,idxJ,d,d,100.)
    ff.Initialize()
    n=4
    more=ff.Minimize()
    while more and n:
      more=ff.Minimize()
      n-=1
    # rotate the embedded conformation onto the core:
    rms =AlignMol(mol,core,atomMap=algMap)
  else:
    # rotate the embedded conformation onto the core:
    rms = AlignMol(mol,core,atomMap=algMap)
    ff =  UFFGetMoleculeForceField(mol,confId=0)
    conf = core.GetConformer()
    for i in range(core.GetNumAtoms()):
      p =conf.GetAtomPosition(i)
      pIdx=ff.AddExtraPoint(p.x,p.y,p.z,fixed=True)-1
      ff.AddDistanceConstraint(pIdx,match[i],0,0,100.)
    ff.Initialize()
    n=4
    more=ff.Minimize(energyTol=1e-4,forceTol=1e-3)
    while more and n:
      more=ff.Minimize(energyTol=1e-4,forceTol=1e-3)
      n-=1
    # realign
    rms = AlignMol(mol,core,atomMap=algMap)
  print rms
  return mol

#------------------------------------
#
#  doctest boilerplate
#
def _test():
  import doctest,sys
  return doctest.testmod(sys.modules["__main__"])

if __name__ == '__main__':
  import sys
  failed,tried = _test()
  sys.exit(failed)
<commit_msg>add sln import<commit_after>## Automatically adapted for numpy.oldnumeric Jun 27, 2008 by -c

# $Id$
#
#  Copyright (C) 2006-2008  greg Landrum and Rational Discovery LLC
#
#   @@ All Rights Reserved  @@
#
""" Import all RDKit chemistry modules

"""
import rdBase
import RDConfig
import DataStructs
from Geometry import rdGeometry
from Chem import *
from rdPartialCharges import *
from rdDepictor import *
from rdForceFieldHelpers import *
from Chem.ChemicalFeatures import *
from rdDistGeom import *
from rdMolAlign import *
from rdMolTransforms import *
from rdShapeHelpers import *
from rdChemReactions import *
from rdSLNParse import *
import ForceField
Mol.Compute2DCoords = Compute2DCoords
Mol.ComputeGasteigerCharges = ComputeGasteigerCharges
import numpy

def TransformMol(mol,tform):
  """  Applies the transformation to a molecule and sets it up with
  a single conformer

  """
  newConf = Conformer()
  newConf.SetId(0)
  refConf = mol.GetConformer()
  for i in range(refConf.GetNumAtoms()):
    pos = list(refConf.GetAtomPosition(i))
    pos.append(1.0)
    newPos = numpy.dot(tform,numpy.array(pos))
    newConf.SetAtomPosition(i,list(newPos)[:3])
  mol.RemoveAllConformers()
  mol.AddConformer(newConf)

def ComputeMolShape(mol,confId=-1,boxDim=(20,20,20),spacing=0.5,**kwargs):
  res = rdGeometry.UniformGrid3D(boxDim[0],boxDim[1],boxDim[2],spacing=spacing)
  apply(EncodeShape,(mol,res,confId),kwargs)
  return res

def ComputeMolVolume(mol,confId=-1,gridSpacing=0.1,boxMargin=2.0):
  import copy
  mol = copy.deepcopy(mol)
  conf = mol.GetConformer(confId)
  CanonicalizeConformer(conf)
  box = ComputeConfBox(conf)
  sideLen = ( box[1].x-box[0].x + 2*boxMargin, \
              box[1].y-box[0].y + 2*boxMargin, \
              box[1].z-box[0].z + 2*boxMargin )
  shape = rdGeometry.UniformGrid3D(sideLen[0],sideLen[1],sideLen[2],
                                   spacing=gridSpacing)
  EncodeShape(mol,shape,confId,ignoreHs=False,vdwScale=1.0)
  voxelVol = gridSpacing**3
  vol = 0.0
  occVect = shape.GetOccupancyVect()
  for i in range(len(occVect)):
    if occVect[i]==3:
      vol+= voxelVol
  return vol

def GenerateDepictionMatching3DStructure(mol,reference,confId=-1,
                                         **kwargs):
  nAts = mol.GetNumAtoms()
  dm = []
  conf = mol.GetConformer(confId)
  for i in range(nAts):
    pi = conf.GetAtomPosition(i)
    for j in range(i+1,nAts):
      pj = conf.GetAtomPosition(j)
      dm.append((pi-pj).Length())
  dm = numpy.array(dm)
  apply(Compute2DCoordsMimicDistmat,(mol,dm),kwargs)

def GetBestRMS(ref,probe,refConfId=-1,probeConfId=-1,maps=None):
  if not maps:
    query = RemoveHs(probe)
    matches = ref.GetSubstructMatches(query)
    if not matches:
      raise ValueError,'mol %s does not match mol %s'%(ref.GetProp('_Name'),
                                                       probe.GetProp('_Name'))
    maps = []
    for match in matches:
      t=[]
      for j,idx in enumerate(match):
        t.append((j,idx))
    maps.append(t)
  bestRMS=1000.
  for amap in maps:
    rms=AlignMol(probe,ref,probeConfId,refConfId,atomMap=amap)
    if rms<bestRMS:
      bestRMS=rms
  return bestRMS

def EnumerateLibraryFromReaction(reaction,sidechainSets) :
  """ Returns a generator for the virtual library defined by
   a reaction and a sequence of sidechain sets

  >>> import Chem
  >>> from Chem import AllChem
  >>> s1=[Chem.MolFromSmiles(x) for x in ('NC','NCC')]
  >>> s2=[Chem.MolFromSmiles(x) for x in ('OC=O','OC(=O)C')]
  >>> rxn = AllChem.ReactionFromSmarts('[O:2]=[C:1][OH].[N:3]>>[O:2]=[C:1][N:3]')
  >>> r = AllChem.EnumerateLibraryFromReaction(rxn,[s2,s1])
  >>> [Chem.MolToSmiles(x[0]) for x in list(r)]
  ['CNC=O', 'CCNC=O', 'CNC(=O)C', 'CCNC(=O)C']

  Note that this is all done in a lazy manner, so "infinitely" large libraries can
  be done without worrying about running out of memory. Your patience will run out first:

  Define a set of 10000 amines:
  >>> amines = (Chem.MolFromSmiles('N'+'C'*x) for x in range(10000))

  ... a set of 10000 acids
  >>> acids = (Chem.MolFromSmiles('OC(=O)'+'C'*x) for x in range(10000))

  ... now the virtual library (1e8 compounds in principle):
  >>> r = AllChem.EnumerateLibraryFromReaction(rxn,[acids,amines])

  ... look at the first 4 compounds:
  >>> [Chem.MolToSmiles(r.next()[0]) for x in range(4)]
  ['NC=O', 'CNC=O', 'CCNC=O', 'CCCNC=O']

  """
  if len(sidechainSets) != reaction.GetNumReactantTemplates():
    raise ValueError,'%d sidechains provided, %d required'%(len(sidechainSets),reaction.getNumReactantTemplates())

  def _combiEnumerator(items,depth=0):
    for item in items[depth]:
      if depth+1 < len(items):
        v = _combiEnumerator(items,depth+1)
        for entry in v:
          l=[item]
          l.extend(entry)
          yield l
      else:
        yield [item]
  for chains in _combiEnumerator(sidechainSets):
    prodSets = reaction.RunReactants(chains)
    for prods in prodSets:
      yield prods

def ConstrainedEmbed(mol,core,useTethers,randomseed=2342):
  match = mol.GetSubstructMatch(core)
  if not match:
    raise ValueError,"molecule doesn't match the core"
  coordMap={}
  coreConf = core.GetConformer()
  for i,idxI in enumerate(match):
    corePtI = coreConf.GetAtomPosition(i)
    coordMap[idxI]=corePtI

  ci = EmbedMolecule(mol,coordMap=coordMap,randomSeed=randomseed)
  if ci<0:
    logger.error('could not embed molecule %s, no coordinates generated.'%mol.GetProp('_Name'))

  algMap=[]
  for i,itm in enumerate(match):
    algMap.append((itm,i))

  if not useTethers:
    # clean up the conformation
    ff = UFFGetMoleculeForceField(mol,confId=0)
    for i,idxI in enumerate(match):
      for j in range(i+1,len(match)):
        idxJ = match[j]
        d = coordMap[idxI].Distance(coordMap[idxJ])
        ff.AddDistanceConstraint(idxI,idxJ,d,d,100.)
    ff.Initialize()
    n=4
    more=ff.Minimize()
    while more and n:
      more=ff.Minimize()
      n-=1
    # rotate the embedded conformation onto the core:
    rms =AlignMol(mol,core,atomMap=algMap)
  else:
    # rotate the embedded conformation onto the core:
    rms = AlignMol(mol,core,atomMap=algMap)
    ff =  UFFGetMoleculeForceField(mol,confId=0)
    conf = core.GetConformer()
    for i in range(core.GetNumAtoms()):
      p =conf.GetAtomPosition(i)
      pIdx=ff.AddExtraPoint(p.x,p.y,p.z,fixed=True)-1
      ff.AddDistanceConstraint(pIdx,match[i],0,0,100.)
    ff.Initialize()
    n=4
    more=ff.Minimize(energyTol=1e-4,forceTol=1e-3)
    while more and n:
      more=ff.Minimize(energyTol=1e-4,forceTol=1e-3)
      n-=1
    # realign
    rms = AlignMol(mol,core,atomMap=algMap)
  print rms
  return mol

#------------------------------------
#
#  doctest boilerplate
#
def _test():
  import doctest,sys
  return doctest.testmod(sys.modules["__main__"])

if __name__ == '__main__':
  import sys
  failed,tried = _test()
  sys.exit(failed)
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
import os

class PsicrawlerPipeline(object):

    def write_item(self, item):

        if len(item['topics']) > 0:
            targetDir = os.path.dirname(os.path.realpath(__file__)) + '/../xmlfiles'

            firstLetter = item['title'][0].lower()
            targetDir += '/' + firstLetter

            if os.path.exists(targetDir) == False:
                os.makedirs(targetDir)

            fn = item['title'].replace(' ', '_') + '.xml'

            with open(targetDir + '/' + fn, 'w') as f:
                f.write(item.asXml())

    def process_item(self, item, spider):
        self.write_item(item)
        return item
<commit_msg>moved getting root path to its own method<commit_after># -*- coding: utf-8 -*-
import os

class PsicrawlerPipeline(object):

    def get_base_path(self):
        return os.path.dirname(os.path.realpath(__file__)) + "/../xmlfiles"

    def write_item(self, item):

        if len(item['topics']) > 0:
            targetDir = self.get_base_path()

            firstLetter = item['title'][0].lower()
            targetDir += '/' + firstLetter

            if os.path.exists(targetDir) == False:
                os.makedirs(targetDir)

            fn = item['title'].replace(' ', '_') + '.xml'

            with open(targetDir + '/' + fn, 'w') as f:
                f.write(item.asXml())

    def process_item(self, item, spider):
        self.write_item(item)
        return item
<|endoftext|>


<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-

import sys
import re
import json
import datetime
import collections
import textwrap
try:
    import urllib2
except ImportError:
    import urllib.request as urllib2

# List of people to ping when the status of a tool or a book changed.
MAINTAINERS = {
    'miri': '@oli-obk @RalfJung @eddyb',
    'clippy-driver': '@Manishearth @llogiq @mcarton @oli-obk @phansch',
    'rls': '@nrc @Xanewok',
    'rustfmt': '@nrc @topecongiro',
    'book': '@carols10cents @steveklabnik',
    'nomicon': '@frewsxcv @Gankro',
    'reference': '@steveklabnik @Havvy @matthewjasper @ehuss',
    'rust-by-example': '@steveklabnik @marioidival @projektir',
    'embedded-book': (
        '@adamgreig @andre-richter @jamesmunns @korken89 '
        '@ryankurte @thejpster @therealprof'
    ),
    'edition-guide': '@ehuss @Centril @steveklabnik',
}

REPOS = {
    'miri': 'https://github.com/rust-lang/miri',
    'clippy-driver': 'https://github.com/rust-lang/rust-clippy',
    'rls': 'https://github.com/rust-lang/rls',
    'rustfmt': 'https://github.com/rust-lang/rustfmt',
    'book': 'https://github.com/rust-lang/book',
    'nomicon': 'https://github.com/rust-lang-nursery/nomicon',
    'reference': 'https://github.com/rust-lang-nursery/reference',
    'rust-by-example': 'https://github.com/rust-lang/rust-by-example',
    'embedded-book': 'https://github.com/rust-embedded/book',
    'edition-guide': 'https://github.com/rust-lang-nursery/edition-guide',
}

def read_current_status(current_commit, path):
    '''Reads build status of `current_commit` from content of `history/*.tsv`
    '''
    with open(path, 'rU') as f:
        for line in f:
            (commit, status) = line.split('\t', 1)
            if commit == current_commit:
                return json.loads(status)
    return {}

def issue(
    tool,
    maintainers,
    relevant_pr_number,
    relevant_pr_user,
    pr_reviewer,
):
    # Open an issue about the toolstate failure.
    gh_url = 'https://api.github.com/repos/rust-lang/rust/issues'
    assignees = [x.strip() for x in maintainers.split('@') if x != '']
    assignees.append(relevant_pr_user)
    response = urllib2.urlopen(urllib2.Request(
        gh_url,
        json.dumps({
            'body': textwrap.dedent('''\
            Hello, this is your friendly neighborhood mergebot.
            After merging PR {}, I observed that the tool {} no longer builds.
            A follow-up PR to the repository {} is needed to fix the fallout.

            cc @{}, do you think you would have time to do the follow-up work?
            If so, that would be great!

            cc @{}, the PR reviewer, and @rust-lang/compiler -- nominating for prioritization.

            ''').format(relevant_pr_number, tool, REPOS.get(tool), relevant_pr_user, pr_reviewer),
            'title': '`{}` no longer builds after {}'.format(tool, relevant_pr_number),
            'assignees': assignees,
            'labels': ['T-compiler', 'I-nominated'],
        }),
        {
            'Authorization': 'token ' + github_token,
            'Content-Type': 'application/json',
        }
    ))
    response.read()

def update_latest(
    current_commit,
    relevant_pr_number,
    relevant_pr_url,
    relevant_pr_user,
    pr_reviewer,
    current_datetime
):
    '''Updates `_data/latest.json` to match build result of the given commit.
    '''
    with open('_data/latest.json', 'rb+') as f:
        latest = json.load(f, object_pairs_hook=collections.OrderedDict)

        current_status = {
            os: read_current_status(current_commit, 'history/' + os + '.tsv')
            for os in ['windows', 'linux']
        }

        slug = 'rust-lang/rust'
        message = textwrap.dedent('''\
            📣 Toolstate changed by {}!

            Tested on commit {}@{}.
            Direct link to PR: <{}>

        ''').format(relevant_pr_number, slug, current_commit, relevant_pr_url)
        anything_changed = False
        for status in latest:
            tool = status['tool']
            changed = False
            build_failed = False

            for os, s in current_status.items():
                old = status[os]
                new = s.get(tool, old)
                status[os] = new
                if new > old:
                    # things got fixed or at least the status quo improved
                    changed = True
                    message += '🎉 {} on {}: {} → {} (cc {}, @rust-lang/infra).\n' \
                        .format(tool, os, old, new, MAINTAINERS.get(tool))
                elif new < old:
                    # tests or builds are failing and were not failing before
                    changed = True
                    title = '💔 {} on {}: {} → {}' \
                        .format(tool, os, old, new)
                    message += '{} (cc {}, @rust-lang/infra).\n' \
                        .format(title, MAINTAINERS.get(tool))
                    # only create issues for build failures. Other failures can be spurious
                    if new == 'build-fail':
                        build_failed = True

            if build_failed:
                try:
                    issue(
                        tool, MAINTAINERS.get(tool, ''),
                        relevant_pr_number, relevant_pr_user, pr_reviewer,
                    )
                except IOError as e:
                    # network errors will simply end up not creating an issue, but that's better
                    # than failing the entire build job
                    print("I/O error: {0}".format(e))
                except:
                    print("Unexpected error: {0}".format(sys.exc_info()[0]))
                    raise

            if changed:
                status['commit'] = current_commit
                status['datetime'] = current_datetime
                anything_changed = True

        if not anything_changed:
            return ''

        f.seek(0)
        f.truncate(0)
        json.dump(latest, f, indent=4, separators=(',', ': '))
        return message

if __name__ == '__main__':
    cur_commit = sys.argv[1]
    cur_datetime = datetime.datetime.utcnow().strftime('%Y-%m-%dT%H:%M:%SZ')
    cur_commit_msg = sys.argv[2]
    save_message_to_path = sys.argv[3]
    github_token = sys.argv[4]

    # assume that PR authors are also owners of the repo where the branch lives
    relevant_pr_match = re.search(
        r'Auto merge of #([0-9]+) - ([^:]+):[^,]+, r=(\S+)',
        cur_commit_msg,
    )
    if relevant_pr_match:
        number = relevant_pr_match.group(1)
        relevant_pr_user = relevant_pr_match.group(2)
        relevant_pr_number = 'rust-lang/rust#' + number
        relevant_pr_url = 'https://github.com/rust-lang/rust/pull/' + number
        pr_reviewer = relevant_pr_match.group(3)
    else:
        number = '-1'
        relevant_pr_user = 'ghost'
        relevant_pr_number = '<unknown PR>'
        relevant_pr_url = '<unknown>'
        pr_reviewer = 'ghost'

    message = update_latest(
        cur_commit,
        relevant_pr_number,
        relevant_pr_url,
        relevant_pr_user,
        pr_reviewer,
        cur_datetime
    )
    if not message:
        print('<Nothing changed>')
        sys.exit(0)

    print(message)

    if not github_token:
        print('Dry run only, not committing anything')
        sys.exit(0)

    with open(save_message_to_path, 'w') as f:
        f.write(message)

    # Write the toolstate comment on the PR as well.
    gh_url = 'https://api.github.com/repos/rust-lang/rust/issues/{}/comments' \
        .format(number)
    response = urllib2.urlopen(urllib2.Request(
        gh_url,
        json.dumps({'body': message}),
        {
            'Authorization': 'token ' + github_token,
            'Content-Type': 'application/json',
        }
    ))
    response.read()
<commit_msg>Rollup merge of #60036 - rust-lang:nrc-patch-1, r=Xanewok<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-

import sys
import re
import json
import datetime
import collections
import textwrap
try:
    import urllib2
except ImportError:
    import urllib.request as urllib2

# List of people to ping when the status of a tool or a book changed.
MAINTAINERS = {
    'miri': '@oli-obk @RalfJung @eddyb',
    'clippy-driver': '@Manishearth @llogiq @mcarton @oli-obk @phansch',
    'rls': '@Xanewok',
    'rustfmt': '@topecongiro',
    'book': '@carols10cents @steveklabnik',
    'nomicon': '@frewsxcv @Gankro',
    'reference': '@steveklabnik @Havvy @matthewjasper @ehuss',
    'rust-by-example': '@steveklabnik @marioidival @projektir',
    'embedded-book': (
        '@adamgreig @andre-richter @jamesmunns @korken89 '
        '@ryankurte @thejpster @therealprof'
    ),
    'edition-guide': '@ehuss @Centril @steveklabnik',
}

REPOS = {
    'miri': 'https://github.com/rust-lang/miri',
    'clippy-driver': 'https://github.com/rust-lang/rust-clippy',
    'rls': 'https://github.com/rust-lang/rls',
    'rustfmt': 'https://github.com/rust-lang/rustfmt',
    'book': 'https://github.com/rust-lang/book',
    'nomicon': 'https://github.com/rust-lang-nursery/nomicon',
    'reference': 'https://github.com/rust-lang-nursery/reference',
    'rust-by-example': 'https://github.com/rust-lang/rust-by-example',
    'embedded-book': 'https://github.com/rust-embedded/book',
    'edition-guide': 'https://github.com/rust-lang-nursery/edition-guide',
}

def read_current_status(current_commit, path):
    '''Reads build status of `current_commit` from content of `history/*.tsv`
    '''
    with open(path, 'rU') as f:
        for line in f:
            (commit, status) = line.split('\t', 1)
            if commit == current_commit:
                return json.loads(status)
    return {}

def issue(
    tool,
    maintainers,
    relevant_pr_number,
    relevant_pr_user,
    pr_reviewer,
):
    # Open an issue about the toolstate failure.
    gh_url = 'https://api.github.com/repos/rust-lang/rust/issues'
    assignees = [x.strip() for x in maintainers.split('@') if x != '']
    assignees.append(relevant_pr_user)
    response = urllib2.urlopen(urllib2.Request(
        gh_url,
        json.dumps({
            'body': textwrap.dedent('''\
            Hello, this is your friendly neighborhood mergebot.
            After merging PR {}, I observed that the tool {} no longer builds.
            A follow-up PR to the repository {} is needed to fix the fallout.

            cc @{}, do you think you would have time to do the follow-up work?
            If so, that would be great!

            cc @{}, the PR reviewer, and @rust-lang/compiler -- nominating for prioritization.

            ''').format(relevant_pr_number, tool, REPOS.get(tool), relevant_pr_user, pr_reviewer),
            'title': '`{}` no longer builds after {}'.format(tool, relevant_pr_number),
            'assignees': assignees,
            'labels': ['T-compiler', 'I-nominated'],
        }),
        {
            'Authorization': 'token ' + github_token,
            'Content-Type': 'application/json',
        }
    ))
    response.read()

def update_latest(
    current_commit,
    relevant_pr_number,
    relevant_pr_url,
    relevant_pr_user,
    pr_reviewer,
    current_datetime
):
    '''Updates `_data/latest.json` to match build result of the given commit.
    '''
    with open('_data/latest.json', 'rb+') as f:
        latest = json.load(f, object_pairs_hook=collections.OrderedDict)

        current_status = {
            os: read_current_status(current_commit, 'history/' + os + '.tsv')
            for os in ['windows', 'linux']
        }

        slug = 'rust-lang/rust'
        message = textwrap.dedent('''\
            📣 Toolstate changed by {}!

            Tested on commit {}@{}.
            Direct link to PR: <{}>

        ''').format(relevant_pr_number, slug, current_commit, relevant_pr_url)
        anything_changed = False
        for status in latest:
            tool = status['tool']
            changed = False
            build_failed = False

            for os, s in current_status.items():
                old = status[os]
                new = s.get(tool, old)
                status[os] = new
                if new > old:
                    # things got fixed or at least the status quo improved
                    changed = True
                    message += '🎉 {} on {}: {} → {} (cc {}, @rust-lang/infra).\n' \
                        .format(tool, os, old, new, MAINTAINERS.get(tool))
                elif new < old:
                    # tests or builds are failing and were not failing before
                    changed = True
                    title = '💔 {} on {}: {} → {}' \
                        .format(tool, os, old, new)
                    message += '{} (cc {}, @rust-lang/infra).\n' \
                        .format(title, MAINTAINERS.get(tool))
                    # only create issues for build failures. Other failures can be spurious
                    if new == 'build-fail':
                        build_failed = True

            if build_failed:
                try:
                    issue(
                        tool, MAINTAINERS.get(tool, ''),
                        relevant_pr_number, relevant_pr_user, pr_reviewer,
                    )
                except IOError as e:
                    # network errors will simply end up not creating an issue, but that's better
                    # than failing the entire build job
                    print("I/O error: {0}".format(e))
                except:
                    print("Unexpected error: {0}".format(sys.exc_info()[0]))
                    raise

            if changed:
                status['commit'] = current_commit
                status['datetime'] = current_datetime
                anything_changed = True

        if not anything_changed:
            return ''

        f.seek(0)
        f.truncate(0)
        json.dump(latest, f, indent=4, separators=(',', ': '))
        return message

if __name__ == '__main__':
    cur_commit = sys.argv[1]
    cur_datetime = datetime.datetime.utcnow().strftime('%Y-%m-%dT%H:%M:%SZ')
    cur_commit_msg = sys.argv[2]
    save_message_to_path = sys.argv[3]
    github_token = sys.argv[4]

    # assume that PR authors are also owners of the repo where the branch lives
    relevant_pr_match = re.search(
        r'Auto merge of #([0-9]+) - ([^:]+):[^,]+, r=(\S+)',
        cur_commit_msg,
    )
    if relevant_pr_match:
        number = relevant_pr_match.group(1)
        relevant_pr_user = relevant_pr_match.group(2)
        relevant_pr_number = 'rust-lang/rust#' + number
        relevant_pr_url = 'https://github.com/rust-lang/rust/pull/' + number
        pr_reviewer = relevant_pr_match.group(3)
    else:
        number = '-1'
        relevant_pr_user = 'ghost'
        relevant_pr_number = '<unknown PR>'
        relevant_pr_url = '<unknown>'
        pr_reviewer = 'ghost'

    message = update_latest(
        cur_commit,
        relevant_pr_number,
        relevant_pr_url,
        relevant_pr_user,
        pr_reviewer,
        cur_datetime
    )
    if not message:
        print('<Nothing changed>')
        sys.exit(0)

    print(message)

    if not github_token:
        print('Dry run only, not committing anything')
        sys.exit(0)

    with open(save_message_to_path, 'w') as f:
        f.write(message)

    # Write the toolstate comment on the PR as well.
    gh_url = 'https://api.github.com/repos/rust-lang/rust/issues/{}/comments' \
        .format(number)
    response = urllib2.urlopen(urllib2.Request(
        gh_url,
        json.dumps({'body': message}),
        {
            'Authorization': 'token ' + github_token,
            'Content-Type': 'application/json',
        }
    ))
    response.read()
<|endoftext|>


<commit_before>import numpy as np
import time

def sigmoid(input_comb):
    return 1.0 / (1 + np.exp(-input_comb))

def rand_bern(length):
    # Return a random vector of -1s and 1s.
    rand_vec = np.random.randint(0, 2, length, dtype=int)  # Begin with a random 0-1 draw.
    return (rand_vec - .5) * 2  # Convert to -1, +1 state.

class BoltzmannMachine(object):
    def __init__(self, input_size, hidden_size, output_size):
        self.total_nodes = input_size + hidden_size + output_size
        self.input_size = input_size
        self.hidden_size = hidden_size
        self.output_size = output_size
        self.hidden_ind = input_size
        self.out_ind = input_size + output_size

        self.state = rand_bern(self.total_nodes)
        self.weights = self.create_random_weights()
        self.correct_weights()

        self.batch_size = 100
        self.inc = 80
        self.learning_rate = 0.5
        self.rate = self.learning_rate

        self.history = self.state
        self.sweeps = 1000
        self.stabilization = np.zeros((self.sweeps, self.total_nodes))
        self.threshold = .01
        self.energy_history = np.zeros(200)

    def print_current_state(self):
        print self.state

    def state_energy(self):
        agreement_matrix = np.outer(self.state, self.state)  # The (i,j) entry is 1 if i,j agree, else -1
        energy_contributions = agreement_matrix * self.weights  # Element-wise product.
        energy = np.sum(energy_contributions) / 2  # Leaving off bias for now.
        return energy

    def state_prob(self):
        """
        The (non-normalized) probability of this state. Does the whole calculation rather than just over the
        affected subsets.
        :return: conditional probability of this
        """
        return np.exp(-self.state_energy())

    def conditional_prob(self, node):
        lin_sum_neighbors = np.dot(self.weights[node,:], self.state)
        return sigmoid(lin_sum_neighbors)

    def update(self, node):
        plus_prob = self.conditional_prob(node)    # P( x_j = 1 |  all other node states)
        coin_flip = np.random.binomial(1, plus_prob, 1)
        result = 2*(coin_flip - .5)  # Convert biased coin flip to -1 or 1.
        # print result
        self.state[node] = result

    def mle_update(self,node):
        if self.conditional_prob(node) > .5 :
            self.state[node] = 1
        else:
            self.state[node] = -1

    def run_machine(self, sweep_num, stabilized=0):
        visit_list = np.arange(self.total_nodes)  # The array [0 1 ... n-1].
        for sweep in range(sweep_num):
            np.random.shuffle(visit_list)  # Shuffle the array [0 1 ... n-1].
            for node_num in range(self.total_nodes):
                node_to_update = visit_list[node_num]
                self.update(node_to_update)
            if stabilized == 0:
                if self.stabilization_check(sweep) == 1:
                    break
            if stabilized == 1:
                self.history = np.vstack((self.history, self.state))
                self.energy_history[sweep] = self.state_energy()

    def stabilization_check(self, sweep):
        prev_mean = self.empirical_mean()
        self.history = np.vstack((self.history, self.state))
        current_mean = self.empirical_mean()
        difference = np.abs(current_mean - prev_mean)
        self.stabilization[sweep, :] = np.less(difference, self.threshold)
        if (np.sum(self.stabilization[sweep, :]) > 27) & (sweep > 100):
            print sweep
            print self.stabilization[sweep, :]
            return 1
        else:
            return 0

    def create_random_weights(self):
        weights = np.random.uniform(-1, 1, size=(self.total_nodes, self.total_nodes))  # Random weights ~ U([-1,1])
        weights = np.triu(weights, k=1)  # discard lower diagonal terms (and the diagonal to avoid self-connections)
        weights = weights + weights.T  # make the weights symmetric
        return weights

    def correct_weights(self):
        self.weights[:self.hidden_ind, :self.hidden_ind] = 0
        self.weights[-self.output_size:,-self.output_size:] = 0
        self.weights[self.hidden_ind:,:self.hidden_ind] = 0
        self.weights[:self.hidden_ind, self.hidden_ind:] = 0

    def empirical_mean(self):
        return np.mean(self.history, axis=0)

    def clamped_run(self, in_state, out_state, sweep_num=1):
        # Runs the machine while forcing the input nodes and output nodes to stay the same.
        # Updates the hidden states.
        self.state[:self.hidden_ind] = in_state
        self.state[self.out_ind:] = out_state
        visit_list = np.arange(self.hidden_ind, self.out_ind)
        for sweep in range(sweep_num):
            np.random.shuffle(visit_list)
            for node_num in range(self.hidden_size):
                node_to_update = visit_list[node_num]
                self.update(node_to_update)

    def unclamped_run(self, in_state, sweep_num=1):
        # Runs the machine while forcing the input nodes to stay the same.
        # Updates the hidden states and output states.
        self.state[:self.hidden_ind] = in_state
        visit_list = np.arange(self.hidden_ind,self.total_nodes)
        for sweep in range(sweep_num):
            np.random.shuffle(visit_list)
            for node_num in range(self.hidden_size + self.output_size):
                node_to_update = visit_list[node_num]
                self.update(node_to_update)

    def clamped_run_mle(self, in_state, out_state):
        # Update the machine using the maximum likelihood states.
        self.state[:self.hidden_ind] = in_state
        self.state[self.out_ind:] = out_state
        for node_num in range(self.hidden_size):
            self.mle_update(node_num + self.input_size)

    def unclamped_run_mle(self, in_state):
        self.state[:self.hidden_ind] = in_state
        for node_num in range(self.hidden_size + self.output_size):
            self.mle_update(node_num + self.input_size)

    def training(self, example_set, num_batches, iterations):
        # Compute how to go through the batches.
        batch_size = self.batchSize
        inc = self.inc  # This is to allow overlap between batches, let it be about 80% of the batch size.
        set_size = example_set.shape[0]
        batches_per_iteration = int(set_size / batch_size)

        for it in range(self.iterations):
            print "Iteration: " + str(it)
            for batch_num, b in enumerate(range(0, set_size - inc, inc)):
                batch = example_set[b:b + batch_size, :]
                num_batches_seen = batch_num + batches_per_iteration * i
                # learning rate decay change by batch number
                # self.rate = self.learning_rate / (num + 1)
                self.batch_process(batch)

            # Manually calculate last batch. It includes some of the first and some of the last examples.
            batch = np.vstack((example_set[:inc, :], example_set[:(batch_size - inc), :]))
            self.batch_process(batch)
            self.rate = self.learning_rate / (1.0 + it)

    def batch_process(self, batch):
        batch_size = self.batch_size
        batch_coactivity_clampled = np.zeros((self.total_nodes,self.total_nodes))
        batch_coactivity_unclampled = np.zeros((self.total_nodes,self.total_nodes))
        for ex in range(batch_size):
            # First clamp down the input nodes and output nodes and compute coactivity.
            self.state = rand_bern(self.total_nodes)
            self.clamped_run_mle(batch[ex, :self.hidden_ind], batch[ex, self.out_ind:])
            batch_coactivity_clamped = batch_coactivity_clamped + self.coactivity()
            # Next clamp down just the input nodes and compute coactivity.
            self.state = rand_bern(self.total_nodes)
            self.unclamped_run(batch[ex, :self.hidden_ind])
            batch_coactivity_unclamped = batch_coactivity_unclamped + self.coactivity()
        dW = (batch_coactivity_clamped - batch_coactivity_unclamped) / batch_size

    def coactivity(self):
        return np.outer(self.state, self.state)

    def read_output(self, input_state):
        # Need to fix an input and then run the machine till it has stabilized.
        # Once stabilized, we can return both the maximizer state as well as the
        # averages for 100 or so states.
        visit_list = np.arange(self.total_nodes)  # The array [0 1 ... n-1].
        for sweep in range(sweep_num):
            np.random.shuffle(visit_list)  # Shuffle the array [0 1 ... n-1].
            for node_num in range(self.total_nodes):
                node_to_update = visit_list[node_num]
                self.update(node_to_update)
            if stabilized == 0:
                if self.stabilization_check(sweep) == 1:
                    break
            if stabilized == 1:
                self.history = np.vstack((self.history, self.state))
                self.energy_history[sweep] = self.state_energy()
        return rand_bern(self.output_size)

start_time = time.time()

BM = BoltzmannMachine(0, 30, 0)

# BM.print_current_state()

BM.run_machine(BM.sweeps)

# print BM.history

# print BM.empirical_mean()

print BM.stabilization

end_time = time.time()

print end_time - start_time

print BM.stabilization[-1,:]

BM.run_machine(200,1)
print BM.energy_history
<commit_msg>Debug fast_RBM, comment up the methods, and get a function version of the Boltzmann Machine Autoencoder.<commit_after>import numpy as np
import time

def sigmoid(input_comb):
    return 1.0 / (1 + np.exp(-input_comb))

def rand_bern(length):
    # Return a random vector of -1s and 1s.
    rand_vec = np.random.randint(0, 2, length, dtype=int)  # Begin with a random 0-1 draw.
    return (rand_vec - .5) * 2  # Convert to -1, +1 state.

def convert_binary_to_pm1(matrix):
    """
    Convert a 0/1 matrix to a -1/1 matrix.
    :param matrix: A binary matrix.
    :return: Converted matrix.
    """
    converted_matrix = (matrix - 0.5) * 2
    return converted_matrix

class BoltzmannMachine(object):
    def __init__(self, input_size, hidden_size, output_size):
        self.total_nodes = input_size + hidden_size + output_size
        self.input_size = input_size
        self.hidden_size = hidden_size
        self.output_size = output_size
        self.hidden_ind = input_size
        self.out_ind = input_size + hidden_size

        self.state = rand_bern(self.total_nodes)
        self.weights = self.create_random_weights()
        self.correct_weights()

        self.batch_size = 100
        self.inc = 80
        self.learning_rate = 0.5
        self.rate = self.learning_rate

        self.history = self.state
        self.sweeps = 1000
        self.stabilization = np.zeros((self.sweeps, self.total_nodes))
        self.threshold = .01
        self.energy_history = np.zeros(200)

    def print_current_state(self):
        print self.state

    def state_energy(self):
        """
        Computes the current energy of the system.
        :return: Current system energy.
        """
        agreement_matrix = np.outer(self.state, self.state)  # The (i,j) entry is 1 if i,j agree, else -1
        energy_contributions = agreement_matrix * self.weights  # Element-wise product.
        energy = np.sum(energy_contributions)  # Leaving off bias. TODO:Check if this needs a factor of 2 or 1/2.
        return energy

    def state_prob(self):
        """
        The (non-normalized) probability of this state. Does the whole calculation rather than just over some
        affected subsets.
        :return: conditional probability of this
        """
        return np.exp(-self.state_energy())

    def conditional_prob(self, node):
        lin_sum_neighbors = np.dot(self.weights[node, :], self.state)
        return sigmoid(lin_sum_neighbors)

    def update(self, node):
        """
        Probabilistically update a single node fixing all others.
        :param node: The number of the node to update.
        :return: Change the state of the node according to the probabilities of the two alternatives.
        """
        plus_prob = self.conditional_prob(node)    # P( x_j = 1 |  all other node states)
        coin_flip = np.random.binomial(1, plus_prob, 1)
        result = 2*(coin_flip - .5)  # Convert biased coin flip to -1 or 1.
        self.state[node] = result

    def simultaneous_update(self, mle=0):
        """
        Update ALL nodes at once. This currently will update everything, including the input/output nodes.
        :param mle: Whether to pick the most likely state of each node (1), or to use the probabilistic update (0).
        :return: Updates the state vector.
        """
        new_state = self.state
        if mle == 1:
            for node in range(0, self.total_nodes):
                new_state[node] = self.mle_update(node)
            self.state = new_state
        elif mle == 0:
            for node in range(0, self.total_nodes):
                plus_prob = self.conditional_prob(node)  # P( x_j = 1 |  all other node states)
                if plus_prob > .5:
                    new_state[node] = 1
                else:
                    new_state[node] = -1
            self.state = new_state

    def mle_update(self, node, alter=1):
        """
        Update a node to it's highest probability state.
        :param node: Which node to update.
        :param alter: Actually change the node (1), or just return the most likely state (0).
        :return: The most likely state for that given node.
        """
        if alter == 1:
            if self.conditional_prob(node) > .5:
                self.state[node] = 1
            else:
                self.state[node] = -1
        elif alter == 0:
            if self.conditional_prob(node) > .5:
                return 1
            else:
                return -1

    def run_machine(self, sweep_num, stabilized=0):
        """
        For sweep_num passes or until it stabilizes, update each of the nodes.
        :param sweep_num: A maximum number of times to update each node.
        :param stabilized: Update the machine until it stabilizes (0), or just run the machine the given amount (1).
        :return:
        """
        visit_list = np.arange(self.total_nodes)  # The array [0 1 ... n-1].
        for sweep in range(sweep_num):
            np.random.shuffle(visit_list)  # Shuffle the array [0 1 ... n-1].
            for node_num in range(self.total_nodes):
                node_to_update = visit_list[node_num]
                self.update(node_to_update)
            if stabilized == 0:
                if self.stabilization_check(sweep) == 1:
                    break
            # if stabilized == 1:
            #     self.history = np.vstack((self.history, self.state))
            #     self.energy_history[sweep] = self.state_energy()

    def stabilization_check(self, sweep):
        """
        Check to see if the machine has stabilized by this sweep.
        :param sweep: Which sweep of updates this is checking after.
        :return: Whether or not stabilization of the mean activation has occurred for 90% or more of the nodes.
        """
        prev_mean = self.empirical_mean()
        self.history = np.vstack((self.history, self.state))
        current_mean = self.empirical_mean()
        difference = np.abs(current_mean - prev_mean)
        self.stabilization[sweep, :] = np.less(difference, self.threshold)
        minimum_stabilized = np.floor(self.total_nodes * .9)
        if (np.sum(self.stabilization[sweep, :]) > minimum_stabilized) & (sweep > 100):
            # print sweep
            # print self.stabilization[sweep, :]
            return 1
        else:
            return 0

    def create_random_weights(self):
        weights = np.random.uniform(-1, 1, size=(self.total_nodes, self.total_nodes))  # Random weights ~ U([-1,1])
        weights = np.triu(weights, k=1)  # discard lower diagonal terms (and the diagonal to avoid self-connections)
        weights = weights + weights.T  # make the weights symmetric
        return weights

    def correct_weights(self):
        self.weights[:self.hidden_ind, :self.hidden_ind] = 0
        self.weights[-self.output_size:, -self.output_size:] = 0
        self.weights[self.hidden_ind:, :self.hidden_ind] = 0
        self.weights[:self.hidden_ind, self.hidden_ind:] = 0

    def empirical_mean(self, history=0):
        """
        Return the average state of each node for a given history.
        :param history: A record of the state of the machine over a given number of sweeps, defaults to the object
        history.
        :return: The mean activation of each node.
        """
        if history == 0:
            history = self.history
        return np.mean(history, axis=0)

    def clamped_run(self, in_state, out_state, sweep_num=1):
        """
        Runs the machine while forcing the input nodes and output nodes to stay the same.
        :param in_state: What the input node states are.
        :param out_state: What the output node states are.
        :param sweep_num: How many times to update the non-clamped nodes.
        :return: Updates the hidden states.
        """
        self.state[:self.hidden_ind] = in_state
        self.state[self.out_ind:] = out_state
        visit_list = np.arange(self.hidden_ind, self.out_ind)
        for sweep in range(sweep_num):
            np.random.shuffle(visit_list)
            for node_num in range(self.hidden_size):
                node_to_update = visit_list[node_num]
                self.update(node_to_update)

    def unclamped_run(self, in_state, sweep_num=1):
        """
        Runs the machine while forcing the input nodes to stay the same.
        :param in_state: What the input node states are.
        :param sweep_num: How many times to update the non-clamped nodes.
        :return: Updates the hidden and output states.
        """
        self.state[:self.hidden_ind] = in_state
        visit_list = np.arange(self.hidden_ind, self.total_nodes)
        for sweep in range(sweep_num):
            np.random.shuffle(visit_list)
            for node_num in range(self.hidden_size + self.output_size):
                node_to_update = visit_list[node_num]
                self.update(node_to_update)

    def clamped_run_mle(self, in_state, out_state):
        # Update the machine using the maximum likelihood states.
        self.state[:self.hidden_ind] = in_state
        self.state[self.out_ind:] = out_state
        for node_num in range(self.hidden_size):
            self.mle_update(node_num + self.input_size)

    def unclamped_run_mle(self, in_state):
        self.state[:self.hidden_ind] = in_state
        for node_num in range(self.hidden_size + self.output_size):
            self.mle_update(node_num + self.input_size)

    def training(self, example_set, iterations):
        """
        Go through the entire example set for the given number of iterations. For each iteration, you divide the set
        into batches and then pass each batch to the batch_process method to update the weights. After each iteration
        you update the rate at which the weights are changed.
        :param example_set: A set of unlabelled examples.
        :param iterations: How many times to go through the set for the learning.
        :return: The machine's weights are updated.
        """
        # Compute how to go through the batches.
        batch_size = self.batch_size
        inc = self.inc  # This is to allow overlap between batches, let it be about 80% of the batch size.
        set_size = example_set.shape[0]  # How many examples.
        # batches_per_iteration = int(set_size / batch_size)  # How many batches will be needed.

        for it in range(iterations):
            print "Iteration: " + str(it)
            for batch_num, b in enumerate(range(0, set_size - inc, inc)):
                batch = example_set[b:b + batch_size, :]
                # To do learning rate decay change by batch number:
                # num_batches_seen = batch_num + batches_per_iteration * it
                # self.rate = self.learning_rate / (num_batches_seen + 1)
                self.batch_process(batch)

            # Manually calculate last batch. It includes some of the first and some of the last examples.
            batch = np.vstack((example_set[:inc, :], example_set[:(batch_size - inc), :]))
            self.batch_process(batch)
            self.rate = self.learning_rate / (1.0 + it)

    def batch_process(self, batch):
        """
        Take in a batch of examples without class labels to be fed into autoencoder training.
        Go through each example in the batch, compute coactivity for clamped and unclamped runs adding to a running
        total that you then average to change the weights.
        :param batch: a matrix whose rows are examples.
        :return: Changes the weights of the machine.
        """
        batch_size = self.batch_size
        batch_coactivity_clamped = np.zeros((self.total_nodes, self.total_nodes))
        batch_coactivity_unclamped = np.zeros((self.total_nodes, self.total_nodes))
        for ex in range(batch_size):
            # First clamp down the input nodes and output nodes and compute coactivity.
            self.state = rand_bern(self.total_nodes)
            self.clamped_run_mle(batch[ex, :], batch[ex, :])
            batch_coactivity_clamped = batch_coactivity_clamped + self.coactivity()
            # Next clamp down just the input nodes and compute coactivity.
            self.state = rand_bern(self.total_nodes)
            self.unclamped_run(batch[ex, :])
            batch_coactivity_unclamped = batch_coactivity_unclamped + self.coactivity()
        dw = (batch_coactivity_clamped - batch_coactivity_unclamped) / batch_size
        self.weights += dw  # Not sure if this should be minus. TODO: find the correct rule.

    def coactivity(self, clamped=1, sweeps=1):
        """
        Computed the coactivity of each node pair averaged over a given number of sweeps.
        :param clamped: If 0, then do unclamped updating. Otherwise, clamp output nodes.
        :param sweeps: how many times to do a full update and compute the coactivities.
        :return: A matrix of coactivity.
        """
        coactivity_matrix = np.zeros((self.total_nodes, self.total_nodes))
        for s in range(0, sweeps):
            coactivity_matrix += np.outer(self.state, self.state)
            if clamped == 0 & sweeps > 1:
                self.unclamped_run(self.state[:self.hidden_ind])
            elif clamped == 1 & sweeps > 1:
                self.clamped_run(self.state[:self.hidden_ind], self.state[self.out_ind:])

        return coactivity_matrix  # TODO: Check how to compute coactivity during training.

    def read_output(self, input_state):
        # Need to fix an input and then run the machine till it has stabilized.
        # Once stabilized, we can return both the maximizer state as well as the
        # averages for 100 or so states.
        sweep_num = self.sweeps
        self.unclamped_run(input_state, sweep_num)
        return self.state[self.out_ind:]  # TODO: Decide on the exact rule for reading off the state.

def main():
    start_time = time.time()

    examples = np.zeros((100, 10))
    examples = convert_binary_to_pm1(examples)
    input_size = examples.shape[1]

    BM = BoltzmannMachine(input_size, 30, input_size)

    BM.run_machine(BM.sweeps)
    BM.training(examples, 10)

    end_time = time.time()

    print end_time - start_time

if __name__ == "__main__":
    main()
<|endoftext|>