<commit_before>#!/usr/bin/env python

# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.

#
# Documentation:
#
# OpenBMC cheatsheet
# https://github.com/openbmc/docs/blob/master/cheatsheet.md
#
# OpenBMC REST API
# https://github.com/openbmc/docs/blob/master/rest-api.md
#
# OpenBMC DBUS API
# https://github.com/openbmc/docs/blob/master/dbus-interfaces.md
#

import sys
import pdb
import requests
import json

# Sadly a way to fit the line into 78 characters mainly
JSON_HEADERS = {"Content-Type": "application/json"}

class HTTPError(Exception):
    def __init__(self, url, status_code, data = None):
        self.url = url
        self.status_code = status_code
        self.data = data

    def __str__(self):
        if self.data is not None:
            return "HTTP Error %s: %s %s" % (self.status_code,
                                             self.url,
                                             self.data, )
        else:
            return "HTTP Error %s: %s" % (self.status_code,
                                          self.url, )

    def __repr__(self):
        return self.__str__()

    def get_status_code(self):
        return self.status_code

class OpenBMC(object):
    def __init__(self,
                 hostname,
                 user,
                 password):
        self.session = None

        # Create a http session
        session = requests.Session()

        # Log in with a special URL and JSON data structure
        url = "https://%s/login" % (hostname, )
        login_data = json.dumps({"data": [ user, password ]})
        response = session.post (url,
                                 data=login_data,
                                 verify=False,
                                 headers=JSON_HEADERS)

        if response.status_code != 200:
            err_str = ("Error: Response code to login is not 200!"
                       " (%d)" % (response.status_code, ))
            print >> sys.stderr, err_str

            raise HTTPError(url, response.status_code, data=login_data)

        self.session = session
        self.hostname = hostname
        self.verbose = False

    def set_verbose(self, value):
        self.verbose = value

    def enumerate(self, key):
        if key.startswith("/"):
            path = key[1:]
        else:
            path = key

        if path.endswith("/"):
            path = path + "enumerate"
        else:
            path = path + "/enumerate"

        return self.get(path)

    def get(self, key):
        if key.startswith("/"):
            path = key[1:]
        else:
            path = key

        url = "https://%s/%s" % (self.hostname, path, )

        if self.verbose:
            print "GET %s" % (url, )

        response = self.session.get (url,
                                     verify=False,
                                     headers=JSON_HEADERS)

        if response.status_code != 200:
            err_str = ("Error: Response code to get %s enumerate is not 200!"
                       " (%d)" % (key, response.status_code, ))
            print >> sys.stderr, err_str

            raise HTTPError(url, response.status_code)

        return response.json()["data"]

    def get_power_chassis_mappings(self):
        # Enumerate the inventory of the system's control hardware
        mappings = {}
        filter_list = ["/power", "/chassis"]

        try:
            items = self.enumerate("/org/openbmc/control/").items()
        except HTTPError as e:
            if e.status_code == 404:
                # @BUG
                # There is no /org/openbmc/control entry?!
                entries = self.get("/org/openbmc/")
                msg = "Error: no /org/openbmc/control in %s" % (entries, )
                raise Exception(msg)
            else:
                raise

        # Loop through the returned map items
        for (item_key, item_value) in items:
            # We only care about filter entries
            if not any(x in item_key for x in filter_list):
                continue

            if self.verbose:
                print "Found:"
                print item_key
                print item_value

            # Add the entry into our mappings
            for fltr in filter_list:
                idx = item_key.find(fltr)
                if idx > -1:
                    # Get the identity (the rest of the string)
                    ident = item_key[idx+len(fltr):]
                    # Create a new map for the first time
                    if not mappings.has_key(ident):
                        mappings[ident] = {}
                    # Save both the full filename and map contents
                    mappings[ident][fltr] = (item_key, item_value)

        return mappings

    def power_on(self):
        # Query /org/openbmc/control for power and chassis entries
        mappings = self.get_power_chassis_mappings()
        if mappings is None:
            return False

        # Loop through the found power & chassis entries
        for (ident, ident_mappings) in mappings.items():
            # { '/power':
            #     ( u'/org/openbmc/control/power0',
            #       {u'pgood': 1,
            #        u'poll_interval': 3000,
            #        u'pgood_timeout': 10,
            #        u'heatbeat': 0,
            #        u'state': 1
            #       }
            #     ),
            #   '/chassis':
            #     ( u'/org/openbmc/control/chassis0',
            #       {u'reboot': 0,
            #        u'uuid': u'24340d83aa784d858468993286b390a5'
            #       }
            #     )
            # }

            # Grab our information back out of the mappings
            (power_url, power_mapping) = ident_mappings["/power"]
            (chassis_url, chassis_mapping) = ident_mappings["/chassis"]

            if self.verbose:
                msg = "Current state of %s is %s" % (power_url,
                                                     power_mapping["state"], )
                print msg

            url = None
            jdata = None

            if power_mapping["state"] == 0:
                # power_on called and machine is off
                url = "https://%s%s/action/powerOn" % (self.hostname,
                                                       chassis_url, )
                jdata = json.dumps({"data": []})
            elif power_mapping["state"] == 1:
                # power_on called and machine is on
                return False

            if url is not None:
                if self.verbose:
                    print "POST %s with %s" % (url, jdata, )

                response = self.session.post (url,
                                              data=jdata,
                                              verify=False,
                                              headers=JSON_HEADERS)

                if response.status_code != 200:
                    err_str = ("Error: Response code to PUT is not 200!"
                               " (%d)" % (response.status_code, ))
                    print >> sys.stderr, err_str

                    raise HTTPError(url, response.status_code, data=jdata)

        return True

    def power_off(self):
        # Query /org/openbmc/control for power and chassis entries
        mappings = self.get_power_chassis_mappings()
        if mappings is None:
            return False

        # Loop through the found power & chassis entries
        for (ident, ident_mappings) in mappings.items():
            # { '/power':
            #     ( u'/org/openbmc/control/power0',
            #       {u'pgood': 1,
            #        u'poll_interval': 3000,
            #        u'pgood_timeout': 10,
            #        u'heatbeat': 0,
            #        u'state': 1
            #       }
            #     ),
            #   '/chassis':
            #     ( u'/org/openbmc/control/chassis0',
            #       {u'reboot': 0,
            #        u'uuid': u'24340d83aa784d858468993286b390a5'
            #       }
            #     )
            # }

            # Grab our information back out of the mappings
            (power_url, power_mapping) = ident_mappings["/power"]
            (chassis_url, chassis_mapping) = ident_mappings["/chassis"]

            if self.verbose:
                msg = "Current state of %s is %s" % (power_url,
                                                     power_mapping["state"], )
                print msg

            url = None
            jdata = None

            if power_mapping["state"] == 0:
                # power_off called and machine is off
                return False
            elif power_mapping["state"] == 1:
                # power_off called and machine is on
                url = "https://%s%s/action/powerOff" % (self.hostname,
                                                        chassis_url, )
                jdata = json.dumps({"data": []})

            if url is not None:
                if self.verbose:
                    print "POST %s with %s" % (url, jdata, )

                response = self.session.post (url,
                                              data=jdata,
                                              verify=False,
                                              headers=JSON_HEADERS)

                if response.status_code != 200:
                    err_str = ("Error: Response code to PUT is not 200!"
                               " (%d)" % (response.status_code, ))
                    print >> sys.stderr, err_str

                    raise HTTPError(url, response.status_code, data=jdata)

        return True
<commit_msg>Move simple initializations to the beginning<commit_after>#!/usr/bin/env python

# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.

#
# Documentation:
#
# OpenBMC cheatsheet
# https://github.com/openbmc/docs/blob/master/cheatsheet.md
#
# OpenBMC REST API
# https://github.com/openbmc/docs/blob/master/rest-api.md
#
# OpenBMC DBUS API
# https://github.com/openbmc/docs/blob/master/dbus-interfaces.md
#

import sys
import pdb
import requests
import json

# Sadly a way to fit the line into 78 characters mainly
JSON_HEADERS = {"Content-Type": "application/json"}

class HTTPError(Exception):
    def __init__(self, url, status_code, data = None):
        self.url = url
        self.status_code = status_code
        self.data = data

    def __str__(self):
        if self.data is not None:
            return "HTTP Error %s: %s %s" % (self.status_code,
                                             self.url,
                                             self.data, )
        else:
            return "HTTP Error %s: %s" % (self.status_code,
                                          self.url, )

    def __repr__(self):
        return self.__str__()

    def get_status_code(self):
        return self.status_code

class OpenBMC(object):
    def __init__(self,
                 hostname,
                 user,
                 password):
        self.session = None
        self.hostname = hostname
        self.verbose = False

        # Create a http session
        session = requests.Session()

        # Log in with a special URL and JSON data structure
        url = "https://%s/login" % (hostname, )
        login_data = json.dumps({"data": [ user, password ]})
        response = session.post (url,
                                 data=login_data,
                                 verify=False,
                                 headers=JSON_HEADERS)

        if response.status_code != 200:
            err_str = ("Error: Response code to login is not 200!"
                       " (%d)" % (response.status_code, ))
            print >> sys.stderr, err_str

            raise HTTPError(url,
                            response.status_code,
                            data=login_data)

        self.session = session

    def set_verbose(self, value):
        self.verbose = value

    def enumerate(self, key):
        if key.startswith("/"):
            path = key[1:]
        else:
            path = key

        if path.endswith("/"):
            path = path + "enumerate"
        else:
            path = path + "/enumerate"

        return self.get(path)

    def get(self, key):
        if key.startswith("/"):
            path = key[1:]
        else:
            path = key

        url = "https://%s/%s" % (self.hostname, path, )

        if self.verbose:
            print "GET %s" % (url, )

        response = self.session.get (url,
                                     verify=False,
                                     headers=JSON_HEADERS)

        if response.status_code != 200:
            err_str = ("Error: Response code to get %s enumerate is not 200!"
                       " (%d)" % (key, response.status_code, ))
            print >> sys.stderr, err_str

            raise HTTPError(url, response.status_code)

        return response.json()["data"]

    def get_power_chassis_mappings(self):
        # Enumerate the inventory of the system's control hardware
        mappings = {}
        filter_list = ["/power", "/chassis"]

        try:
            items = self.enumerate("/org/openbmc/control/").items()
        except HTTPError as e:
            if e.status_code == 404:
                # @BUG
                # There is no /org/openbmc/control entry?!
                entries = self.get("/org/openbmc/")
                msg = "Error: no /org/openbmc/control in %s" % (entries, )
                raise Exception(msg)
            else:
                raise

        # Loop through the returned map items
        for (item_key, item_value) in items:
            # We only care about filter entries
            if not any(x in item_key for x in filter_list):
                continue

            if self.verbose:
                print "Found:"
                print item_key
                print item_value

            # Add the entry into our mappings
            for fltr in filter_list:
                idx = item_key.find(fltr)
                if idx > -1:
                    # Get the identity (the rest of the string)
                    ident = item_key[idx+len(fltr):]
                    # Create a new map for the first time
                    if not mappings.has_key(ident):
                        mappings[ident] = {}
                    # Save both the full filename and map contents
                    mappings[ident][fltr] = (item_key, item_value)

        return mappings

    def power_on(self):
        # Query /org/openbmc/control for power and chassis entries
        mappings = self.get_power_chassis_mappings()
        if mappings is None:
            return False

        # Loop through the found power & chassis entries
        for (ident, ident_mappings) in mappings.items():
            # { '/power':
            #     ( u'/org/openbmc/control/power0',
            #       {u'pgood': 1,
            #        u'poll_interval': 3000,
            #        u'pgood_timeout': 10,
            #        u'heatbeat': 0,
            #        u'state': 1
            #       }
            #     ),
            #   '/chassis':
            #     ( u'/org/openbmc/control/chassis0',
            #       {u'reboot': 0,
            #        u'uuid': u'24340d83aa784d858468993286b390a5'
            #       }
            #     )
            # }

            # Grab our information back out of the mappings
            (power_url, power_mapping) = ident_mappings["/power"]
            (chassis_url, chassis_mapping) = ident_mappings["/chassis"]

            if self.verbose:
                msg = "Current state of %s is %s" % (power_url,
                                                     power_mapping["state"], )
                print msg

            url = None
            jdata = None

            if power_mapping["state"] == 0:
                # power_on called and machine is off
                url = "https://%s%s/action/powerOn" % (self.hostname,
                                                       chassis_url, )
                jdata = json.dumps({"data": []})
            elif power_mapping["state"] == 1:
                # power_on called and machine is on
                return False

            if url is not None:
                if self.verbose:
                    print "POST %s with %s" % (url, jdata, )

                response = self.session.post (url,
                                              data=jdata,
                                              verify=False,
                                              headers=JSON_HEADERS)

                if response.status_code != 200:
                    err_str = ("Error: Response code to PUT is not 200!"
                               " (%d)" % (response.status_code, ))
                    print >> sys.stderr, err_str

                    raise HTTPError(url, response.status_code, data=jdata)

        return True

    def power_off(self):
        # Query /org/openbmc/control for power and chassis entries
        mappings = self.get_power_chassis_mappings()
        if mappings is None:
            return False

        # Loop through the found power & chassis entries
        for (ident, ident_mappings) in mappings.items():
            # { '/power':
            #     ( u'/org/openbmc/control/power0',
            #       {u'pgood': 1,
            #        u'poll_interval': 3000,
            #        u'pgood_timeout': 10,
            #        u'heatbeat': 0,
            #        u'state': 1
            #       }
            #     ),
            #   '/chassis':
            #     ( u'/org/openbmc/control/chassis0',
            #       {u'reboot': 0,
            #        u'uuid': u'24340d83aa784d858468993286b390a5'
            #       }
            #     )
            # }

            # Grab our information back out of the mappings
            (power_url, power_mapping) = ident_mappings["/power"]
            (chassis_url, chassis_mapping) = ident_mappings["/chassis"]

            if self.verbose:
                msg = "Current state of %s is %s" % (power_url,
                                                     power_mapping["state"], )
                print msg

            url = None
            jdata = None

            if power_mapping["state"] == 0:
                # power_off called and machine is off
                return False
            elif power_mapping["state"] == 1:
                # power_off called and machine is on
                url = "https://%s%s/action/powerOff" % (self.hostname,
                                                        chassis_url, )
                jdata = json.dumps({"data": []})

            if url is not None:
                if self.verbose:
                    print "POST %s with %s" % (url, jdata, )

                response = self.session.post (url,
                                              data=jdata,
                                              verify=False,
                                              headers=JSON_HEADERS)

                if response.status_code != 200:
                    err_str = ("Error: Response code to PUT is not 200!"
                               " (%d)" % (response.status_code, ))
                    print >> sys.stderr, err_str

                    raise HTTPError(url, response.status_code, data=jdata)

        return True
<|endoftext|>


<commit_before>import os
import logging
from urllib import quote, unquote

from pylons import c, g, request, response
from webob import exc
from tg import redirect, expose, override_template, flash, url, validate
from tg.decorators import with_trailing_slash, without_trailing_slash
from formencode import validators

from ming.orm import ThreadLocalORMSession, session

from allura.lib import patience
from allura.lib import security
from allura.lib import helpers as h
from allura.lib import widgets as w
from allura.controllers import AppDiscussionController
from allura.lib.widgets.repo import SCMLogWidget, SCMRevisionWidget, SCMTreeWidget
from allura.lib.widgets.repo import SCMMergeRequestWidget, SCMMergeRequestFilterWidget
from allura.lib.widgets.repo import SCMMergeRequestDisposeWidget
from allura import model as M

from .base import BaseController

log = logging.getLogger(__name__)

def on_import():
    BranchBrowser.CommitBrowserClass = CommitBrowser
    CommitBrowser.TreeBrowserClass = TreeBrowser
    TreeBrowser.FileBrowserClass = FileBrowser

class RepoRootController(BaseController):
    _discuss = AppDiscussionController()

    def _check_security(self):
        security.require(security.has_artifact_access('read'))

    @expose()
    def index(self, offset=0, branch=None, **kw):
        if branch is None:
            branch=c.app.default_branch_name
        redirect(url(quote('%s%s/' % (
                        branch, c.app.END_OF_REF_ESCAPE))))

    @expose()
    def refresh(self):
        g.publish('audit', 'repo.refresh')
        return '%r refresh queued.\n' % c.app.repo

    @with_trailing_slash
    @expose('jinja:repo/fork.html')
    def fork(self, to_name=None, project_name=None):
        security.require_authenticated()
        if not c.app.forkable: raise exc.HTTPNotFound
        from_repo = c.app.repo
        if project_name:
            to_project_name = project_name
        else:
            to_project_name = 'u/' + c.user.username
        ThreadLocalORMSession.flush_all()
        ThreadLocalORMSession.close_all()
        from_project = c.project
        to_project = M.Project.query.get(shortname=to_project_name)
        with h.push_config(c, project=to_project):
            if request.method!='POST' or to_name is None:
                if to_project is None:
                    in_use = []
                    to_project_name = ''
                else:
                    prefix_len = len(to_project_name+'/')
                    in_use = [sp.shortname[prefix_len:] for sp in to_project.direct_subprojects]
                    in_use += [ac.options['mount_point'] for ac in to_project.app_configs]
                return dict(from_repo=from_repo,
                            to_project_name=to_project_name,
                            in_use=in_use,
                            to_name=to_name or '')
            else:
                if not to_project.database_configured:
                    to_project.configure_project(is_user_project=True)
                security.require(security.has_project_access('tool', to_project))
                try:
                    to_project.install_app(
                        from_repo.tool_name, to_name,
                        cloned_from_project_id=from_project._id,
                        cloned_from_repo_id=from_repo._id)
                    redirect(to_project.url()+to_name+'/')
                except exc.HTTPRedirection:
                    raise
                except Exception, ex:
                    flash(str(ex), 'error')
                    redirect(request.referer)

    @property
    def mr_widget(self):
        source_branches = [
            b.name
            for b in c.app.repo.branches + c.app.repo.tags]
        with c.app.repo.push_upstream_context():
            target_branches = [
                b.name
                for b in c.app.repo.branches + c.app.repo.tags]
        return SCMMergeRequestWidget(
            source_branches=source_branches,
            target_branches=target_branches)

    @without_trailing_slash
    @expose('jinja:repo/request_merge.html')
    def request_merge(self, branch=None):
        c.form = self.mr_widget
        if branch is None:
            source_branch=c.app.repo.branches[0].name
        return dict(source_branch=source_branch)

    @expose()
    def do_request_merge(self, **kw):
        kw = self.mr_widget.to_python(kw)
        downstream=dict(
            project_id=c.project._id,
            mount_point=c.app.config.options.mount_point,
            commit_id=c.app.repo.commit(kw['source_branch']).object_id)
        with c.app.repo.push_upstream_context():
            mr = M.MergeRequest.upsert(
                downstream=downstream,
                target_branch=kw['target_branch'],
                summary=kw['summary'],
                description=kw['description'])
            t = M.Thread(
                discussion_id=c.app.config.discussion_id,
                artifact_reference=mr.dump_ref(),
                subject='Discussion for Merge Request #:%s: %s' % (
                    mr.request_number, mr.summary))
            session(t).flush()
            redirect(mr.url())

    @without_trailing_slash
    @expose()
    @validate(dict(
            since=h.DateTimeConverter(if_empty=None, if_invalid=None),
            until=h.DateTimeConverter(if_empty=None, if_invalid=None),
            offset=validators.Int(if_empty=None),
            limit=validators.Int(if_empty=None)))
    def feed(self, since=None, until=None, offset=None, limit=None):
        if request.environ['PATH_INFO'].endswith('.atom'):
            feed_type = 'atom'
        else:
            feed_type = 'rss'
        title = 'Recent changes to %s' % c.app.config.options.mount_point
        feed = M.Feed.feed(
            {'artifact_reference.mount_point':c.app.config.options.mount_point,
             'artifact_reference.project_id':c.project._id},
            feed_type,
            title,
            c.app.url,
            title,
            since, until, offset, limit)
        response.headers['Content-Type'] = ''
        response.content_type = 'application/xml'
        return feed.writeString('utf-8')

class MergeRequestsController(object):
    mr_filter=SCMMergeRequestFilterWidget()

    @expose('jinja:repo/merge_requests.html')
    @validate(mr_filter)
    def index(self, status=None):
        status = status or ['open']
        requests = c.app.repo.merge_requests_by_statuses(*status)
        c.mr_filter = self.mr_filter
        return dict(
            status=status,
            requests=requests)

    @expose()
    def _lookup(self, num, *remainder):
        return MergeRequestController(num), remainder

class MergeRequestController(object):
    log_widget=SCMLogWidget()
    thread_widget=w.Thread(
        page=None, limit=None, page_size=None, count=None,
        style='linear')
    mr_dispose_form=SCMMergeRequestDisposeWidget()

    def __init__(self, num):
        self.req = M.MergeRequest.query.get(
            request_number=int(num))
        if self.req is None: raise exc.HTTPNotFound

    @expose('jinja:repo/merge_request.html')
    def index(self, page=0, limit=250):
        c.thread = self.thread_widget
        c.log_widget = self.log_widget
        c.mr_dispose_form = self.mr_dispose_form
        return dict(
            req=self.req,
            page=page,
            limit=limit,
            count=self.req.discussion_thread.post_count)

    @expose()
    @validate(mr_dispose_form)
    def save(self, status=None):
        security.require(
            security.has_artifact_access('write', self.req), 'Write access required')
        self.req.status = status
        redirect('.')

class RefsController(object):

    def __init__(self, BranchBrowserClass):
        self.BranchBrowserClass = BranchBrowserClass

    @expose()
    def _lookup(self, *parts):
        parts = map(unquote, parts)
        ref = []
        while parts:
            part = parts.pop(0)
            ref.append(part)
            if part.endswith(c.app.END_OF_REF_ESCAPE):
                break
        ref = '/'.join(ref)[:-1]
        return self.BranchBrowserClass(ref), parts

class CommitsController(object):

    @expose()
    def _lookup(self, ci, *remainder):
        return CommitBrowser(ci), remainder

class BranchBrowser(BaseController):
    CommitBrowserClass=None

    def __init__(self, branch):
        self._branch = branch

    def _check_security(self):
        security.require(security.has_artifact_access('read', c.app.repo))

    @expose('jinja:repo/tags.html')
    @with_trailing_slash
    def tags(self, **kw):
        return dict(tags=c.app.repo.repo_tags)

    @expose()
    @with_trailing_slash
    def log(self, **kw):
        ci = c.app.repo.commit(self._branch)
        redirect(ci.url() + 'log/')

class CommitBrowser(BaseController):
    TreeBrowserClass=None
    revision_widget = SCMRevisionWidget()
    log_widget=SCMLogWidget()

    def __init__(self, revision):
        self._revision = revision
        self._commit = c.app.repo.commit(revision)
        if self._commit is None:
            raise exc.HTTPNotFound
        self.tree = self.TreeBrowserClass(self._commit, tree=self._commit.tree)

    @expose('jinja:repo/commit.html')
    def index(self):
        c.revision_widget = self.revision_widget
        result = dict(commit=self._commit)
        if self._commit:
            result.update(self._commit.context())
        return result

    @expose('jinja:repo/log.html')
    @with_trailing_slash
    def log(self, limit=None, page=0, count=0, **kw):
        limit, page, start = g.handle_paging(limit, page)
        revisions = c.app.repo.log(
                branch=self._commit.object_id,
                offset=start,
                limit=limit)
        c.log_widget = self.log_widget
        count = 0
        return dict(
            username=c.user._id and c.user.username,
            branch=None,
            log=revisions,
            page=page,
            limit=limit,
            count=count,
            **kw)

class TreeBrowser(BaseController):
    tree_widget = SCMTreeWidget()
    FileBrowserClass=None

    def __init__(self, commit, tree, path='', parent=None):
        self._commit = commit
        self._tree = tree
        self._path = path
        self._parent = parent

    @expose('jinja:repo/tree.html')
    @with_trailing_slash
    def index(self, **kw):
        c.tree_widget = self.tree_widget
        return dict(
            repo=c.app.repo,
            commit=self._commit,
            tree=self._tree,
            path=self._path,
            parent=self._parent)

    @expose()
    def _lookup(self, next, *rest):
        next=unquote(next)
        if not rest:
            # Might be a file rather than a dir
            filename = h.really_unicode(
                unquote(
                    request.environ['PATH_INFO'].rsplit('/')[-1]))
            if filename and self._tree.is_blob(filename):
                return self.FileBrowserClass(
                    self._commit,
                    self._tree,
                    filename), rest
        elif rest == ('index', ):
            rest = (request.environ['PATH_INFO'].rsplit('/')[-1],)
        tree = self._tree.get_tree(next)
        if tree is None:
            raise exc.HTTPNotFound
        return self.__class__(
            self._commit,
            tree,
            self._path + '/' + next,
            self), rest

class FileBrowser(BaseController):

    def __init__(self, commit, tree, filename):
        self._commit = commit
        self._tree = tree
        self._filename = filename
        self._blob = self._tree.get_blob(filename)

    @expose('jinja:repo/file.html')
    def index(self, **kw):
        if kw.pop('format', 'html') == 'raw':
            return self.raw()
        elif 'diff' in kw:
            override_template(self.index, 'jinja:repo/diff.html')
            return self.diff(kw['diff'])
        else:
            force_display = 'force' in kw
            context = self._blob.context()
            return dict(
                blob=self._blob,
                prev=context.get('prev', None),
                next=context.get('next', None),
                force_display=force_display
                )

    @expose()
    def raw(self):
        content_type = self._blob.content_type.encode('utf-8')
        filename = self._blob.name.encode('utf-8')
        response.headers['Content-Type'] = ''
        response.content_type = content_type
        if self._blob.content_encoding is not None:
            content_encoding = self._blob.content_encoding.encode('utf-8')
            response.headers['Content-Encoding'] = ''
            response.content_encoding = content_encoding
        response.headers.add(
            'Content-Disposition', 'attachment;filename=' + filename)
        return iter(self._blob)

    def diff(self, commit):
        try:
            path, filename = os.path.split(self._blob.path())
            a_ci = c.app.repo.commit(commit)
            a = a_ci.get_path(self._blob.path())
            apath = a.path()
        except:
            a = []
            apath = ''
        b = self._blob
        la = list(a)
        lb = list(b)
        diff = ''.join(patience.unified_diff(
                la, lb,
                ('a' + apath).encode('utf-8'),
                ('b' + b.path()).encode('utf-8')))
        return dict(
            a=a, b=b,
            diff=diff)

on_import()
<commit_msg>[#1423] fix 404 when repo path did not have a trailing slash<commit_after>import os
import logging
from urllib import quote, unquote

from pylons import c, g, request, response
from webob import exc
from tg import redirect, expose, override_template, flash, url, validate
from tg.decorators import with_trailing_slash, without_trailing_slash
from formencode import validators

from ming.orm import ThreadLocalORMSession, session

from allura.lib import patience
from allura.lib import security
from allura.lib import helpers as h
from allura.lib import widgets as w
from allura.controllers import AppDiscussionController
from allura.lib.widgets.repo import SCMLogWidget, SCMRevisionWidget, SCMTreeWidget
from allura.lib.widgets.repo import SCMMergeRequestWidget, SCMMergeRequestFilterWidget
from allura.lib.widgets.repo import SCMMergeRequestDisposeWidget
from allura import model as M

from .base import BaseController

log = logging.getLogger(__name__)

def on_import():
    BranchBrowser.CommitBrowserClass = CommitBrowser
    CommitBrowser.TreeBrowserClass = TreeBrowser
    TreeBrowser.FileBrowserClass = FileBrowser

class RepoRootController(BaseController):
    _discuss = AppDiscussionController()

    def _check_security(self):
        security.require(security.has_artifact_access('read'))

    @with_trailing_slash
    @expose()
    def index(self, offset=0, branch=None, **kw):
        if branch is None:
            branch=c.app.default_branch_name
        redirect(url(quote('%s%s/' % (
                        branch, c.app.END_OF_REF_ESCAPE))))

    @expose()
    def refresh(self):
        g.publish('audit', 'repo.refresh')
        return '%r refresh queued.\n' % c.app.repo

    @with_trailing_slash
    @expose('jinja:repo/fork.html')
    def fork(self, to_name=None, project_name=None):
        security.require_authenticated()
        if not c.app.forkable: raise exc.HTTPNotFound
        from_repo = c.app.repo
        if project_name:
            to_project_name = project_name
        else:
            to_project_name = 'u/' + c.user.username
        ThreadLocalORMSession.flush_all()
        ThreadLocalORMSession.close_all()
        from_project = c.project
        to_project = M.Project.query.get(shortname=to_project_name)
        with h.push_config(c, project=to_project):
            if request.method!='POST' or to_name is None:
                if to_project is None:
                    in_use = []
                    to_project_name = ''
                else:
                    prefix_len = len(to_project_name+'/')
                    in_use = [sp.shortname[prefix_len:] for sp in to_project.direct_subprojects]
                    in_use += [ac.options['mount_point'] for ac in to_project.app_configs]
                return dict(from_repo=from_repo,
                            to_project_name=to_project_name,
                            in_use=in_use,
                            to_name=to_name or '')
            else:
                if not to_project.database_configured:
                    to_project.configure_project(is_user_project=True)
                security.require(security.has_project_access('tool', to_project))
                try:
                    to_project.install_app(
                        from_repo.tool_name, to_name,
                        cloned_from_project_id=from_project._id,
                        cloned_from_repo_id=from_repo._id)
                    redirect(to_project.url()+to_name+'/')
                except exc.HTTPRedirection:
                    raise
                except Exception, ex:
                    flash(str(ex), 'error')
                    redirect(request.referer)

    @property
    def mr_widget(self):
        source_branches = [
            b.name
            for b in c.app.repo.branches + c.app.repo.tags]
        with c.app.repo.push_upstream_context():
            target_branches = [
                b.name
                for b in c.app.repo.branches + c.app.repo.tags]
        return SCMMergeRequestWidget(
            source_branches=source_branches,
            target_branches=target_branches)

    @without_trailing_slash
    @expose('jinja:repo/request_merge.html')
    def request_merge(self, branch=None):
        c.form = self.mr_widget
        if branch is None:
            source_branch=c.app.repo.branches[0].name
        return dict(source_branch=source_branch)

    @expose()
    def do_request_merge(self, **kw):
        kw = self.mr_widget.to_python(kw)
        downstream=dict(
            project_id=c.project._id,
            mount_point=c.app.config.options.mount_point,
            commit_id=c.app.repo.commit(kw['source_branch']).object_id)
        with c.app.repo.push_upstream_context():
            mr = M.MergeRequest.upsert(
                downstream=downstream,
                target_branch=kw['target_branch'],
                summary=kw['summary'],
                description=kw['description'])
            t = M.Thread(
                discussion_id=c.app.config.discussion_id,
                artifact_reference=mr.dump_ref(),
                subject='Discussion for Merge Request #:%s: %s' % (
                    mr.request_number, mr.summary))
            session(t).flush()
            redirect(mr.url())

    @without_trailing_slash
    @expose()
    @validate(dict(
            since=h.DateTimeConverter(if_empty=None, if_invalid=None),
            until=h.DateTimeConverter(if_empty=None, if_invalid=None),
            offset=validators.Int(if_empty=None),
            limit=validators.Int(if_empty=None)))
    def feed(self, since=None, until=None, offset=None, limit=None):
        if request.environ['PATH_INFO'].endswith('.atom'):
            feed_type = 'atom'
        else:
            feed_type = 'rss'
        title = 'Recent changes to %s' % c.app.config.options.mount_point
        feed = M.Feed.feed(
            {'artifact_reference.mount_point':c.app.config.options.mount_point,
             'artifact_reference.project_id':c.project._id},
            feed_type,
            title,
            c.app.url,
            title,
            since, until, offset, limit)
        response.headers['Content-Type'] = ''
        response.content_type = 'application/xml'
        return feed.writeString('utf-8')

class MergeRequestsController(object):
    mr_filter=SCMMergeRequestFilterWidget()

    @expose('jinja:repo/merge_requests.html')
    @validate(mr_filter)
    def index(self, status=None):
        status = status or ['open']
        requests = c.app.repo.merge_requests_by_statuses(*status)
        c.mr_filter = self.mr_filter
        return dict(
            status=status,
            requests=requests)

    @expose()
    def _lookup(self, num, *remainder):
        return MergeRequestController(num), remainder

class MergeRequestController(object):
    log_widget=SCMLogWidget()
    thread_widget=w.Thread(
        page=None, limit=None, page_size=None, count=None,
        style='linear')
    mr_dispose_form=SCMMergeRequestDisposeWidget()

    def __init__(self, num):
        self.req = M.MergeRequest.query.get(
            request_number=int(num))
        if self.req is None: raise exc.HTTPNotFound

    @expose('jinja:repo/merge_request.html')
    def index(self, page=0, limit=250):
        c.thread = self.thread_widget
        c.log_widget = self.log_widget
        c.mr_dispose_form = self.mr_dispose_form
        return dict(
            req=self.req,
            page=page,
            limit=limit,
            count=self.req.discussion_thread.post_count)

    @expose()
    @validate(mr_dispose_form)
    def save(self, status=None):
        security.require(
            security.has_artifact_access('write', self.req), 'Write access required')
        self.req.status = status
        redirect('.')

class RefsController(object):

    def __init__(self, BranchBrowserClass):
        self.BranchBrowserClass = BranchBrowserClass

    @expose()
    def _lookup(self, *parts):
        parts = map(unquote, parts)
        ref = []
        while parts:
            part = parts.pop(0)
            ref.append(part)
            if part.endswith(c.app.END_OF_REF_ESCAPE):
                break
        ref = '/'.join(ref)[:-1]
        return self.BranchBrowserClass(ref), parts

class CommitsController(object):

    @expose()
    def _lookup(self, ci, *remainder):
        return CommitBrowser(ci), remainder

class BranchBrowser(BaseController):
    CommitBrowserClass=None

    def __init__(self, branch):
        self._branch = branch

    def _check_security(self):
        security.require(security.has_artifact_access('read', c.app.repo))

    @expose('jinja:repo/tags.html')
    @with_trailing_slash
    def tags(self, **kw):
        return dict(tags=c.app.repo.repo_tags)

    @expose()
    @with_trailing_slash
    def log(self, **kw):
        ci = c.app.repo.commit(self._branch)
        redirect(ci.url() + 'log/')

class CommitBrowser(BaseController):
    TreeBrowserClass=None
    revision_widget = SCMRevisionWidget()
    log_widget=SCMLogWidget()

    def __init__(self, revision):
        self._revision = revision
        self._commit = c.app.repo.commit(revision)
        if self._commit is None:
            raise exc.HTTPNotFound
        self.tree = self.TreeBrowserClass(self._commit, tree=self._commit.tree)

    @expose('jinja:repo/commit.html')
    def index(self):
        c.revision_widget = self.revision_widget
        result = dict(commit=self._commit)
        if self._commit:
            result.update(self._commit.context())
        return result

    @expose('jinja:repo/log.html')
    @with_trailing_slash
    def log(self, limit=None, page=0, count=0, **kw):
        limit, page, start = g.handle_paging(limit, page)
        revisions = c.app.repo.log(
                branch=self._commit.object_id,
                offset=start,
                limit=limit)
        c.log_widget = self.log_widget
        count = 0
        return dict(
            username=c.user._id and c.user.username,
            branch=None,
            log=revisions,
            page=page,
            limit=limit,
            count=count,
            **kw)

class TreeBrowser(BaseController):
    tree_widget = SCMTreeWidget()
    FileBrowserClass=None

    def __init__(self, commit, tree, path='', parent=None):
        self._commit = commit
        self._tree = tree
        self._path = path
        self._parent = parent

    @expose('jinja:repo/tree.html')
    @with_trailing_slash
    def index(self, **kw):
        c.tree_widget = self.tree_widget
        return dict(
            repo=c.app.repo,
            commit=self._commit,
            tree=self._tree,
            path=self._path,
            parent=self._parent)

    @expose()
    def _lookup(self, next, *rest):
        next=unquote(next)
        if not rest:
            # Might be a file rather than a dir
            filename = h.really_unicode(
                unquote(
                    request.environ['PATH_INFO'].rsplit('/')[-1]))
            if filename and self._tree.is_blob(filename):
                return self.FileBrowserClass(
                    self._commit,
                    self._tree,
                    filename), rest
        elif rest == ('index', ):
            rest = (request.environ['PATH_INFO'].rsplit('/')[-1],)
        tree = self._tree.get_tree(next)
        if tree is None:
            raise exc.HTTPNotFound
        return self.__class__(
            self._commit,
            tree,
            self._path + '/' + next,
            self), rest

class FileBrowser(BaseController):

    def __init__(self, commit, tree, filename):
        self._commit = commit
        self._tree = tree
        self._filename = filename
        self._blob = self._tree.get_blob(filename)

    @expose('jinja:repo/file.html')
    def index(self, **kw):
        if kw.pop('format', 'html') == 'raw':
            return self.raw()
        elif 'diff' in kw:
            override_template(self.index, 'jinja:repo/diff.html')
            return self.diff(kw['diff'])
        else:
            force_display = 'force' in kw
            context = self._blob.context()
            return dict(
                blob=self._blob,
                prev=context.get('prev', None),
                next=context.get('next', None),
                force_display=force_display
                )

    @expose()
    def raw(self):
        content_type = self._blob.content_type.encode('utf-8')
        filename = self._blob.name.encode('utf-8')
        response.headers['Content-Type'] = ''
        response.content_type = content_type
        if self._blob.content_encoding is not None:
            content_encoding = self._blob.content_encoding.encode('utf-8')
            response.headers['Content-Encoding'] = ''
            response.content_encoding = content_encoding
        response.headers.add(
            'Content-Disposition', 'attachment;filename=' + filename)
        return iter(self._blob)

    def diff(self, commit):
        try:
            path, filename = os.path.split(self._blob.path())
            a_ci = c.app.repo.commit(commit)
            a = a_ci.get_path(self._blob.path())
            apath = a.path()
        except:
            a = []
            apath = ''
        b = self._blob
        la = list(a)
        lb = list(b)
        diff = ''.join(patience.unified_diff(
                la, lb,
                ('a' + apath).encode('utf-8'),
                ('b' + b.path()).encode('utf-8')))
        return dict(
            a=a, b=b,
            diff=diff)

on_import()
<|endoftext|>


<commit_before>from __future__ import print_function

import argparse
import sys
from opencc import OpenCC

def main():
    parser = argparse.ArgumentParser(
        formatter_class=argparse.ArgumentDefaultsHelpFormatter)
    parser.add_argument('-i', '--input', metavar='<file>',
                        help='Read original text from <file>.')
    parser.add_argument('-o', '--output', metavar='<file>',
                        help='Write converted text to <file>.')
    parser.add_argument('-c', '--config', metavar='<file>',
                        help='Configuration file')
    parser.add_argument('--in-enc', metavar='<encoding>', default='UTF-8',
                        help='Encoding for input')
    parser.add_argument('--out-enc', metavar='<encoding>', default='UTF-8',
                        help='Encoding for output')
    args = parser.parse_args()

    if args.config is None:
        print("Please specify a configuration file.", file=sys.stderr)
        return 1

    cc = OpenCC(args.config)

    with open(args.input if args.input else 0, encoding=args.in_enc) as f:
        input_str = f.read()
    output_str = cc.convert(input_str)
    with open(args.output if args.output else 1, 'w',
              encoding=args.out_enc) as f:
        f.write(output_str)

    return 0

if __name__ == '__main__':
    sys.exit(main())
<commit_msg>Add support for Python 2.6 and 2.7<commit_after>from __future__ import print_function

import argparse
import sys
import io
from opencc import OpenCC

def main():
    parser = argparse.ArgumentParser(
        formatter_class=argparse.ArgumentDefaultsHelpFormatter)
    parser.add_argument('-i', '--input', metavar='<file>',
                        help='Read original text from <file>.')
    parser.add_argument('-o', '--output', metavar='<file>',
                        help='Write converted text to <file>.')
    parser.add_argument('-c', '--config', metavar='<conversion>',
                        help='Conversion')
    parser.add_argument('--in-enc', metavar='<encoding>', default='UTF-8',
                        help='Encoding for input')
    parser.add_argument('--out-enc', metavar='<encoding>', default='UTF-8',
                        help='Encoding for output')
    args = parser.parse_args()

    if args.config is None:
        print("Please specify a conversion.", file=sys.stderr)
        return 1

    cc = OpenCC(args.config)

    with io.open(args.input if args.input else 0, encoding=args.in_enc) as f:
        input_str = f.read()
    output_str = cc.convert(input_str)
    with io.open(args.output if args.output else 1, 'w',
              encoding=args.out_enc) as f:
        f.write(output_str)

    return 0

if __name__ == '__main__':
    sys.exit(main())
<|endoftext|>


<commit_before># Find sources in the data and produce spectra for each suitable source.

# Author James Dempsey
# Date 28 Aug 2016

from __future__ import print_function, division

import magmo
import os
import sys
import time
import csv

from astropy.io import fits
from astropy.io import votable
from astropy.coordinates import SkyCoord, Angle
from astropy.wcs import WCS
from astropy.table import Table, Column
from astropy.io.votable.tree import Param,Info
from astropy.io.votable import from_table, writeto
from astropy import units as u
import matplotlib.pyplot as plt
import numpy as np
import numpy.core.records as rec

from string import Template

sn_min = 1.3
num_chan = 627

def get_high_signal_fields(day_dir_name):
    """
    Retrieve a list of fields observed in a particular day that have sufficient
    signal to noise to search for background sources.
    :param day_dir_name: The name of the day's directory.
    :return: A list of high signal fields.
    """
    field_list = []
    print ("Fields of interest:")
    with open(day_dir_name + '/stats.csv', 'rb') as stats:
        reader = csv.reader(stats)
        first = True
        for row in reader:
            if first:
                first = False
            else:
                if float(row[3]) > sn_min:
                    print (row)
                    field_list.append(row[0])

    return field_list

def find_sources(day_dir_name, field_name):
    """
    Search a contoinuum file for sources using the Aegean source finder. A
    VOTable file containing the list of discovered sources will be written out
    for the field. This function will use the Aegean source finder
    ( https://github.com/PaulHancock/Aegean ) to identify the sources.

    :param day_dir_name: The name of the day's directory.
    :param field_name:  The name fo the field to be searched for sources.
    :return: A list of error messages, if any
    """
    error_list = []
    cont_file = day_dir_name + "/1757/magmo-" + field_name + "_1757_restor.fits"
    table_file = day_dir_name + "/" + field_name + '_src.vot'
    try:
        print ("##--## Searching continuum image " + cont_file + " ##--##")
        magmo.run_os_cmd('bane ' + cont_file)
        aegean_cmd = 'aegean ' + cont_file + ' --autoload --telescope=ATCA ' \
                     '--cores=1 --table=' + table_file
        magmo.run_os_cmd(aegean_cmd)
    except magmo.CommandFailedError as e:
        error_list.append(str(e))
    return error_list

def read_sources(filename):
    print ("Extracting sources from " + filename)
    sources = []
    src_votable = votable.parse(filename, pedantic=False)
    results = src_votable.get_first_table().array
    for row in results:
        id = str(row['island']) + "-" + str(row['source'])
        ra = row['ra']
        dec = row['dec']
        rms = row['local_rms']
        flux = row['peak_flux']
        sn = flux / rms
        print ("Found source %s at %.4f, %.4f with flux %.4f and rms of %.4f "
               "giving S/N of %.4f" % (id, ra, dec, flux, rms, sn))
        if sn > 10 and flux > 0.02:
            sources.append([ra, dec, id, flux])
        else:
            print ("Ignoring source at %.4f, %.4f due to low S/N of %.4f or "
                   "flux of %.4f" % (ra, dec, sn, flux))

    return sources

def read_continuum_ranges():
    continuum_ranges = []
    with open('magmo-continuum.csv', 'rb') as con_def:
        reader = csv.reader(con_def)
        first = True
        for row in reader:
            if first:
                first = False
            else:
                continuum_ranges.append(
                    [int(row[0]), int(row[1]), int(row[2]), int(row[3])])

    print (continuum_ranges)
    return continuum_ranges

def find_edges(fluxes, num_edge_chan):
    """
    Seek from the edges to find where the data starts for this set of fluxes.
    This accounts for an optional number of channels in the data which have no
    data recorded.
    :param fluxes: The array of fluxes to be checked.
    :param num_edge_chan: The number of edge channels with data to be skipped
    :return: The index of the first and last cell to have data.
    """

    l_edge = 0
    r_edge = len(fluxes)-1

    while fluxes[l_edge] == 0 and l_edge < len(fluxes):
        l_edge += 1

    while fluxes[r_edge] == 0 and r_edge > 0:
        r_edge -= 1

    return l_edge + num_edge_chan, r_edge - num_edge_chan

def extract_spectra(daydirname, field):
    num_edge_chan = 10  # todo: make this a velocity relative value
    fits_filename = "{0}/1420/magmo-{1}_1420_sl_restor.fits".format(daydirname,
                                                                    field)
    src_filename = "{0}/{1}_src_comp.vot".format(daydirname, field)

    spectra = dict()
    source_ids = dict()
    if not os.path.exists(fits_filename):
        print ("Warning: File %s does not exist, skipping extraction." % \
              fits_filename)
        return spectra, source_ids

    sources = read_sources(src_filename)
    hdulist = fits.open(fits_filename)
    image = hdulist[0].data
    header = hdulist[0].header
    w = WCS(header)
    index = np.arange(header['NAXIS3'])
    velocities = w.wcs_pix2world(10,10,index[:],0,0)[2]
    for src in sources:
        pix = w.wcs_world2pix(src[0], src[1], 0, 0, 1)
        x_coord = int(round(pix[0])) - 1  # 266
        y_coord = int(round(pix[1])) - 1  # 197
        print ("Translated %.4f, %.4f to %d, %d" % (
            src[0], src[1], x_coord, y_coord))

        slice = image[0, :, y_coord, x_coord]
        l_edge, r_edge = find_edges(slice, num_edge_chan)
        print("Using data range %d - %d out of %d channels." % (
            l_edge, r_edge, len(slice)))

        # plotSpectrum(np.arange(slice.size), slice)
        spectrum_array = rec.fromarrays(
            [np.arange(slice.size)[l_edge:r_edge],
             velocities[l_edge:r_edge],
             slice[l_edge:r_edge]],
            names='plane,velocity,flux')
        c = SkyCoord(src[0], src[1], frame='icrs', unit="deg")
        spectra[c.galactic.l] = spectrum_array
        source_ids[c.galactic.l] = [src[2], src[3]]
    del image
    del header
    hdulist.close()

    return spectra, source_ids

def get_mean_continuum(spectrum, longitude, continuum_ranges):
    """
    Calculate the mean of the continuum values. Will divide the spectrum
    into continuum and line parts based on those values with a 1 sigma
    deviation from the mean of the first 5% of values.  This assumes that
    there is a leading continuum block in the spectrum.
    :param spectrum: The spectrum to be analysed, should be a numpy array of
        plane, velocity and flux values.
    :param longitude: The galactic longitude of the target object
    :param continuum_ranges: The predefined continuum blocks by longitude range
    :return: A single float which is the mean continuum flux.
    """
    continuum_start_vel, continuum_end_vel = magmo.lookup_continuum_range(
        continuum_ranges, int(longitude.degree))

    print(
        "Looking for velocity range %d to %d in data of %d to %d at longitude %.3f" %
        (continuum_start_vel, continuum_end_vel,
         np.min(spectrum.velocity) / 1000.0,
         np.max(spectrum.velocity) / 1000.0, longitude.degree))

    continuum_range = np.where(
        continuum_start_vel*1000 < spectrum.velocity)
    bin_start = continuum_range[0][0]
    continuum_range = np.where(
        spectrum.velocity < continuum_end_vel*1000)
    bin_end = continuum_range[0][-1]

    print("Using bins %d to %d (velocity range %d to %d) out of %d" % (
        bin_start, bin_end, continuum_start_vel, continuum_end_vel, len(spectrum.velocity)))
    continuum_sample = spectrum.flux[bin_start:bin_end]
    # print ("...gave sample of", continuum_sample)
    mean_cont = np.mean(continuum_sample)
    return mean_cont, continuum_start_vel, continuum_end_vel

def get_opacity(spectrum, mean):
    """
    Calculate the opacity profile for the spectrum. This simply divides the
    spectrum's flux by the mean.

    :param spectrum: The spectrum to be processed
    :param mean: The mean background flux, representing what the backlighting sources average flux.
    :return: The opacity (e^(-tau)) at each velocity step.
    """
    # print spectrum.flux
    # print spectrum.flux/mean
    return spectrum.flux/mean

def plot_spectrum(x, y, filename, title, con_start_vel, con_end_vel):
    """
    Output a plot of opacity vs LSR velocity to a specified file.

    :param x: The velocity data
    :param y: The opacity values for each velocity step
    :param filename: The file the plot should be written to. Should be
         an .eps or .pdf file.
    :param title: The title for the plot
    :param con_start_vel: The minimum velocity that the continuum was measured at.
    :param con_end_vel: The maximum velocity that the continuum was measured at.
    """
    fig = plt.figure()
    plt.plot(x/1000, y)

    plt.axhline(1, color='r')
    plt.axvline(con_start_vel, color='g', linestyle='dashed')
    plt.axvline(con_end_vel, color='g', linestyle='dashed')

    plt.xlabel(r'Velocity relative to LSR (km/s)')
    plt.ylabel(r'$e^{(-\tau)}$')
    plt.title(title)
    plt.grid(True)
    plt.savefig(filename)
    #plt.show()
    plt.close()
    return

def output_spectra(spectrum, opacity, filename, longitude, latitude):
    """
    Write the spectrum (velocity, flux and opacity) to a votable format file.

    :param spectrum: The spectrum to be output.
    :param opacity:  The opacity to be output.
    :param filename:  The filename to be created
    :param longitude: The galactic longitude of the target object
    :param latitude: The galactic latitude of the target object
    """
    table = Table([spectrum.plane, spectrum.velocity, opacity, spectrum.flux],
                  names=('plane', 'velocity', 'opacity', 'flux'),
                  meta={'name': 'Opacity', 'id' : 'opacity'})
    votable = from_table(table)
    votable.infos.append(Info('longitude', 'longitude', longitude.value))
    votable.infos.append(Info('latitude', 'latitude', latitude.value))
    writeto(votable, filename)

def produce_spectra(day_dir_name, day, field_list, continuum_ranges):
    with open(day_dir_name + '/spectra.html', 'w') as spectra_idx:
        t = Template(
            '\n<title>D$day Spectra</title>\n'
            + '\n
Spectra previews for day $day
\n
\n')
        spectra_idx.write(t.substitute(day=day))

        for field in field_list:
            spectra, source_ids = extract_spectra(day_dir_name, field)
            t = Template('
Field: ${field}
\n' +
                         '
Image Name	 Source Longitude 	' +
                         'Peak Flux	Spectra
\n')
            spectra_idx.write(t.substitute(field=field))

            idx = 0
            for longitude in sorted(spectra.keys()):
                spectrum = spectra.get(longitude)
                src_data = source_ids.get(longitude)
                name_prefix = field + '_src' + src_data[0]
                idx += 1
                mean, min_con_vel, max_con_vel = get_mean_continuum(spectrum,
                                                                    longitude,
                                                                    continuum_ranges)
                if mean < 0:
                    print(("WARNING: Skipped spectrum %s with negative " +
                          "mean: %.5f") % (name_prefix, mean))
                    continue;

                print ('Continuum mean of %s is %.5f Jy' % (name_prefix, mean))
                opacity = get_opacity(spectrum, mean)
                # print opacity
                dir_prefix = day_dir_name + "/"
                img_name = name_prefix + "_plot.png"
                plot_spectrum(spectrum.velocity, opacity, dir_prefix + img_name,
                              "Spectra for source {0} in field {1}".format(
                                  src_data[0], field), min_con_vel, max_con_vel)
                filename = dir_prefix + name_prefix + '_opacity.votable.xml'
                latitude = Angle(0 * u.deg) # todo: Read this from the spectra
                output_spectra(spectrum, opacity, filename, longitude, latitude)

                t = Template('
${img}	${longitude}	' +
                             '${peak_flux}	' +
                             '
\n')
                spectra_idx.write(t.substitute(img=img_name, peak_flux=src_data[1],
                                               longitude=longitude))

        spectra_idx.write('
\n')

def main():
    """
    Main script for analyse_data
    :return: The exit code
    """
    # Read day parameter
    if len(sys.argv) != 2:
        print("Incorrect number of parameters.")
        print("Usage: python analyse_data.py day")
        return 1
    day = sys.argv[1]
    start = time.time()

    # Check metadata against file system
    day_dir_name = "day" + day
    if not os.path.isdir(day_dir_name):
        print ("Directory %s could not be found." % day_dir_name)
        return 1

    print ("#### Started source finding on MAGMO day %s at %s ####" % \
          (day, time.strftime('%Y-%m-%d %H:%M:%S', time.localtime(start))))
    error_list = []

    # Read list of fields, filter for ones to be processed
    field_list = get_high_signal_fields(day_dir_name)

    # For each file, find the sources
    for field in field_list:
        error_list.extend(find_sources(day_dir_name, field))

    # For each file, extract spectra
    continuum_ranges = magmo.get_continuum_ranges()
    produce_spectra(day_dir_name, day, field_list, continuum_ranges)

    # Report
    end = time.time()
    print ('#### Processing completed at %s ####' \
          % (time.strftime('%Y-%m-%d %H:%M:%S', time.localtime(end))))
    print ('Searched %d images in %.02f s' % (len(field_list),
                                               end - start))
    if len(error_list) == 0:
        print ("Hooray! No errors found.")
    else:
        print ("%d errors were encountered:" % (len(error_list)))
        for err in error_list:
            print (err)
    return 0

# Run the script if it is called from the command line
if __name__ == "__main__":
    exit(main())<commit_msg>Skip missing source files - if Aegean exited abnormally<commit_after># Find sources in the data and produce spectra for each suitable source.

# Author James Dempsey
# Date 28 Aug 2016

from __future__ import print_function, division

import magmo
import os
import sys
import time
import csv

from astropy.io import fits
from astropy.io import votable
from astropy.coordinates import SkyCoord, Angle
from astropy.wcs import WCS
from astropy.table import Table, Column
from astropy.io.votable.tree import Param,Info
from astropy.io.votable import from_table, writeto
from astropy import units as u
import matplotlib.pyplot as plt
import numpy as np
import numpy.core.records as rec

from string import Template

sn_min = 1.3
num_chan = 627

def get_high_signal_fields(day_dir_name):
    """
    Retrieve a list of fields observed in a particular day that have sufficient
    signal to noise to search for background sources.
    :param day_dir_name: The name of the day's directory.
    :return: A list of high signal fields.
    """
    field_list = []
    print ("Fields of interest:")
    with open(day_dir_name + '/stats.csv', 'rb') as stats:
        reader = csv.reader(stats)
        first = True
        for row in reader:
            if first:
                first = False
            else:
                if float(row[3]) > sn_min:
                    print (row)
                    field_list.append(row[0])

    return field_list

def find_sources(day_dir_name, field_name):
    """
    Search a contoinuum file for sources using the Aegean source finder. A
    VOTable file containing the list of discovered sources will be written out
    for the field. This function will use the Aegean source finder
    ( https://github.com/PaulHancock/Aegean ) to identify the sources.

    :param day_dir_name: The name of the day's directory.
    :param field_name:  The name fo the field to be searched for sources.
    :return: A list of error messages, if any
    """
    error_list = []
    cont_file = day_dir_name + "/1757/magmo-" + field_name + "_1757_restor.fits"
    table_file = day_dir_name + "/" + field_name + '_src.vot'
    try:
        print ("##--## Searching continuum image " + cont_file + " ##--##")
        magmo.run_os_cmd('bane ' + cont_file)
        aegean_cmd = 'aegean ' + cont_file + ' --autoload --telescope=ATCA ' \
                     '--cores=1 --table=' + table_file
        magmo.run_os_cmd(aegean_cmd)
    except magmo.CommandFailedError as e:
        error_list.append(str(e))
    return error_list

def read_sources(filename):
    print ("Extracting sources from " + filename)
    sources = []

    if not os.path.exists(filename):
        print ("Warning: File %s does not exist, skipping source read." % \
               filename)
        return sources

    src_votable = votable.parse(filename, pedantic=False)
    results = src_votable.get_first_table().array
    for row in results:
        id = str(row['island']) + "-" + str(row['source'])
        ra = row['ra']
        dec = row['dec']
        rms = row['local_rms']
        flux = row['peak_flux']
        sn = flux / rms
        print ("Found source %s at %.4f, %.4f with flux %.4f and rms of %.4f "
               "giving S/N of %.4f" % (id, ra, dec, flux, rms, sn))
        if sn > 10 and flux > 0.02:
            sources.append([ra, dec, id, flux])
        else:
            print ("Ignoring source at %.4f, %.4f due to low S/N of %.4f or "
                   "flux of %.4f" % (ra, dec, sn, flux))

    return sources

def read_continuum_ranges():
    continuum_ranges = []
    with open('magmo-continuum.csv', 'rb') as con_def:
        reader = csv.reader(con_def)
        first = True
        for row in reader:
            if first:
                first = False
            else:
                continuum_ranges.append(
                    [int(row[0]), int(row[1]), int(row[2]), int(row[3])])

    print (continuum_ranges)
    return continuum_ranges

def find_edges(fluxes, num_edge_chan):
    """
    Seek from the edges to find where the data starts for this set of fluxes.
    This accounts for an optional number of channels in the data which have no
    data recorded.
    :param fluxes: The array of fluxes to be checked.
    :param num_edge_chan: The number of edge channels with data to be skipped
    :return: The index of the first and last cell to have data.
    """

    l_edge = 0
    r_edge = len(fluxes)-1

    while fluxes[l_edge] == 0 and l_edge < len(fluxes):
        l_edge += 1

    while fluxes[r_edge] == 0 and r_edge > 0:
        r_edge -= 1

    return l_edge + num_edge_chan, r_edge - num_edge_chan

def extract_spectra(daydirname, field):
    num_edge_chan = 10  # todo: make this a velocity relative value
    fits_filename = "{0}/1420/magmo-{1}_1420_sl_restor.fits".format(daydirname,
                                                                    field)
    src_filename = "{0}/{1}_src_comp.vot".format(daydirname, field)

    spectra = dict()
    source_ids = dict()
    if not os.path.exists(fits_filename):
        print ("Warning: File %s does not exist, skipping extraction." % \
              fits_filename)
        return spectra, source_ids

    sources = read_sources(src_filename)
    hdulist = fits.open(fits_filename)
    image = hdulist[0].data
    header = hdulist[0].header
    w = WCS(header)
    index = np.arange(header['NAXIS3'])
    velocities = w.wcs_pix2world(10,10,index[:],0,0)[2]
    for src in sources:
        pix = w.wcs_world2pix(src[0], src[1], 0, 0, 1)
        x_coord = int(round(pix[0])) - 1  # 266
        y_coord = int(round(pix[1])) - 1  # 197
        print ("Translated %.4f, %.4f to %d, %d" % (
            src[0], src[1], x_coord, y_coord))

        slice = image[0, :, y_coord, x_coord]
        l_edge, r_edge = find_edges(slice, num_edge_chan)
        print("Using data range %d - %d out of %d channels." % (
            l_edge, r_edge, len(slice)))

        # plotSpectrum(np.arange(slice.size), slice)
        spectrum_array = rec.fromarrays(
            [np.arange(slice.size)[l_edge:r_edge],
             velocities[l_edge:r_edge],
             slice[l_edge:r_edge]],
            names='plane,velocity,flux')
        c = SkyCoord(src[0], src[1], frame='icrs', unit="deg")
        spectra[c.galactic.l] = spectrum_array
        source_ids[c.galactic.l] = [src[2], src[3]]
    del image
    del header
    hdulist.close()

    return spectra, source_ids

def get_mean_continuum(spectrum, longitude, continuum_ranges):
    """
    Calculate the mean of the continuum values. Will divide the spectrum
    into continuum and line parts based on those values with a 1 sigma
    deviation from the mean of the first 5% of values.  This assumes that
    there is a leading continuum block in the spectrum.
    :param spectrum: The spectrum to be analysed, should be a numpy array of
        plane, velocity and flux values.
    :param longitude: The galactic longitude of the target object
    :param continuum_ranges: The predefined continuum blocks by longitude range
    :return: A single float which is the mean continuum flux.
    """
    continuum_start_vel, continuum_end_vel = magmo.lookup_continuum_range(
        continuum_ranges, int(longitude.degree))

    print(
        "Looking for velocity range %d to %d in data of %d to %d at longitude %.3f" %
        (continuum_start_vel, continuum_end_vel,
         np.min(spectrum.velocity) / 1000.0,
         np.max(spectrum.velocity) / 1000.0, longitude.degree))

    continuum_range = np.where(
        continuum_start_vel*1000 < spectrum.velocity)
    bin_start = continuum_range[0][0]
    continuum_range = np.where(
        spectrum.velocity < continuum_end_vel*1000)
    bin_end = continuum_range[0][-1]

    print("Using bins %d to %d (velocity range %d to %d) out of %d" % (
        bin_start, bin_end, continuum_start_vel, continuum_end_vel, len(spectrum.velocity)))
    continuum_sample = spectrum.flux[bin_start:bin_end]
    # print ("...gave sample of", continuum_sample)
    mean_cont = np.mean(continuum_sample)
    return mean_cont, continuum_start_vel, continuum_end_vel

def get_opacity(spectrum, mean):
    """
    Calculate the opacity profile for the spectrum. This simply divides the
    spectrum's flux by the mean.

    :param spectrum: The spectrum to be processed
    :param mean: The mean background flux, representing what the backlighting sources average flux.
    :return: The opacity (e^(-tau)) at each velocity step.
    """
    # print spectrum.flux
    # print spectrum.flux/mean
    return spectrum.flux/mean

def plot_spectrum(x, y, filename, title, con_start_vel, con_end_vel):
    """
    Output a plot of opacity vs LSR velocity to a specified file.

    :param x: The velocity data
    :param y: The opacity values for each velocity step
    :param filename: The file the plot should be written to. Should be
         an .eps or .pdf file.
    :param title: The title for the plot
    :param con_start_vel: The minimum velocity that the continuum was measured at.
    :param con_end_vel: The maximum velocity that the continuum was measured at.
    """
    fig = plt.figure()
    plt.plot(x/1000, y)

    plt.axhline(1, color='r')
    plt.axvline(con_start_vel, color='g', linestyle='dashed')
    plt.axvline(con_end_vel, color='g', linestyle='dashed')

    plt.xlabel(r'Velocity relative to LSR (km/s)')
    plt.ylabel(r'$e^{(-\tau)}$')
    plt.title(title)
    plt.grid(True)
    plt.savefig(filename)
    #plt.show()
    plt.close()
    return

def output_spectra(spectrum, opacity, filename, longitude, latitude):
    """
    Write the spectrum (velocity, flux and opacity) to a votable format file.

    :param spectrum: The spectrum to be output.
    :param opacity:  The opacity to be output.
    :param filename:  The filename to be created
    :param longitude: The galactic longitude of the target object
    :param latitude: The galactic latitude of the target object
    """
    table = Table([spectrum.plane, spectrum.velocity, opacity, spectrum.flux],
                  names=('plane', 'velocity', 'opacity', 'flux'),
                  meta={'name': 'Opacity', 'id' : 'opacity'})
    votable = from_table(table)
    votable.infos.append(Info('longitude', 'longitude', longitude.value))
    votable.infos.append(Info('latitude', 'latitude', latitude.value))
    writeto(votable, filename)

def produce_spectra(day_dir_name, day, field_list, continuum_ranges):
    with open(day_dir_name + '/spectra.html', 'w') as spectra_idx:
        t = Template(
            '\n<title>D$day Spectra</title>\n'
            + '\n
Spectra previews for day $day
\n
\n')
        spectra_idx.write(t.substitute(day=day))

        for field in field_list:
            spectra, source_ids = extract_spectra(day_dir_name, field)
            t = Template('
Field: ${field}
\n' +
                         '
Image Name	 Source Longitude 	' +
                         'Peak Flux	Spectra
\n')
            spectra_idx.write(t.substitute(field=field))

            idx = 0
            for longitude in sorted(spectra.keys()):
                spectrum = spectra.get(longitude)
                src_data = source_ids.get(longitude)
                name_prefix = field + '_src' + src_data[0]
                idx += 1
                mean, min_con_vel, max_con_vel = get_mean_continuum(spectrum,
                                                                    longitude,
                                                                    continuum_ranges)
                if mean < 0:
                    print(("WARNING: Skipped spectrum %s with negative " +
                          "mean: %.5f") % (name_prefix, mean))
                    continue;

                print ('Continuum mean of %s is %.5f Jy' % (name_prefix, mean))
                opacity = get_opacity(spectrum, mean)
                # print opacity
                dir_prefix = day_dir_name + "/"
                img_name = name_prefix + "_plot.png"
                plot_spectrum(spectrum.velocity, opacity, dir_prefix + img_name,
                              "Spectra for source {0} in field {1}".format(
                                  src_data[0], field), min_con_vel, max_con_vel)
                filename = dir_prefix + name_prefix + '_opacity.votable.xml'
                latitude = Angle(0 * u.deg) # todo: Read this from the spectra
                output_spectra(spectrum, opacity, filename, longitude, latitude)

                t = Template('
${img}	${longitude}	' +
                             '${peak_flux}	' +
                             '
\n')
                spectra_idx.write(t.substitute(img=img_name, peak_flux=src_data[1],
                                               longitude=longitude))

        spectra_idx.write('
\n')

def main():
    """
    Main script for analyse_data
    :return: The exit code
    """
    # Read day parameter
    if len(sys.argv) != 2:
        print("Incorrect number of parameters.")
        print("Usage: python analyse_data.py day")
        return 1
    day = sys.argv[1]
    start = time.time()

    # Check metadata against file system
    day_dir_name = "day" + day
    if not os.path.isdir(day_dir_name):
        print ("Directory %s could not be found." % day_dir_name)
        return 1

    print ("#### Started source finding on MAGMO day %s at %s ####" % \
          (day, time.strftime('%Y-%m-%d %H:%M:%S', time.localtime(start))))
    error_list = []

    # Read list of fields, filter for ones to be processed
    field_list = get_high_signal_fields(day_dir_name)

    # For each file, find the sources
    for field in field_list:
        error_list.extend(find_sources(day_dir_name, field))

    # For each file, extract spectra
    continuum_ranges = magmo.get_continuum_ranges()
    produce_spectra(day_dir_name, day, field_list, continuum_ranges)

    # Report
    end = time.time()
    print ('#### Processing completed at %s ####' \
          % (time.strftime('%Y-%m-%d %H:%M:%S', time.localtime(end))))
    print ('Searched %d images in %.02f s' % (len(field_list),
                                               end - start))
    if len(error_list) == 0:
        print ("Hooray! No errors found.")
    else:
        print ("%d errors were encountered:" % (len(error_list)))
        for err in error_list:
            print (err)
    return 0

# Run the script if it is called from the command line
if __name__ == "__main__":
    exit(main())<|endoftext|>


<commit_before>import graphAttack as ga
import numpy as np
import pickle
import sys
"""Control script"""
simulationIndex = 0
# simulationIndex = int(sys.argv[1])

pickleFilename = "dataSet/trump_campaign.pkl"
# pickleFilename = "dataSet/singleSentence.pkl"
with open(pickleFilename, "rb") as fp:
    x, index_to_word, word_to_index = pickle.load(fp)

seriesLength, nFeatures = x.shape
nExamples = simulationIndex
exampleLength = 20
nHidden = 80
nHidden2 = 100

# nExamples = 2
# exampleLength = 15
# nHidden = 35
# nHidden2 = 40

mainGraph = ga.Graph(False)
dummyX = np.zeros((nExamples, exampleLength, nFeatures))
feed = mainGraph.addOperation(ga.Variable(dummyX), feederOperation=True)

hactivations0 = ga.addInitialRNNLayer(mainGraph,
                                      inputOperation=feed,
                                      activation=ga.TanhActivation,
                                      nHidden=nHidden)

hactivations = ga.appendRNNLayer(mainGraph,
                                 previousActivations=hactivations0,
                                 activation=ga.TanhActivation,
                                 nHidden=nHidden2)

finalCost, costOperationsList = ga.addRNNCost(mainGraph,
                                              hactivations,
                                              costActivation=ga.SoftmaxActivation,
                                              costOperation=ga.CrossEntropyCostSoftmax,
                                              nHidden=nHidden2,
                                              labelsShape=feed.shape,
                                              labels=None)

def fprime(p, data, labels, costOperationsList=costOperationsList, mainGraph=mainGraph):
    mainGraph.feederOperation.assignData(data)
    mainGraph.resetAll()
    for index, cop in enumerate(costOperationsList):
        cop.assignLabels(labels[:, index, :])
    mainGraph.attachParameters(p)
    c = mainGraph.feedForward()
    mainGraph.feedBackward()
    g = mainGraph.unrollGradients()
    return c, g

param0 = mainGraph.unrollGradientParameters()
print("Number of parameters to train:", len(param0))
1/0
adaGrad = ga.adaptiveSGDrecurrent(trainingData=x,
                                  param0=param0,
                                  epochs=1e3,
                                  miniBatchSize=nExamples,
                                  exampleLength=exampleLength,
                                  initialLearningRate=1e-3,
                                  beta1=0.9,
                                  beta2=0.999,
                                  epsilon=1e-8,
                                  testFrequency=1e2,
                                  function=fprime)

params = adaGrad.minimize(printTrainigCost=True, printUpdateRate=False,
                          dumpParameters="paramsRNN_" + str(simulationIndex) + ".pkl")
# pickleFilename = "paramsRNN_" + str(simulationIndex) + ".pkl"
# with open(pickleFilename, "rb") as fp:
#     params = pickle.load(fp)

mainGraph.attachParameters(params)

def array2char(array):
    return index_to_word[np.argmax(array)]

def sampleSingle(previousX, previousH,
                 hactivations=hactivations,
                 costOperationsList=costOperationsList,
                 mainGraph=mainGraph):
    N, T, D = mainGraph.feederOperation.shape
    preiousData = np.zeros((1, T, D))
    preiousData[:, 0, :] = previousX

    mainGraph.resetAll()
    mainGraph.feederOperation.assignData(preiousData)
    hactivations[0].assignData(previousH)

    newH = hactivations[1].getValue()
    # newX = costOperationsList[0].makePredictions()
    newX = costOperationsList[0].inputA.getValue()

    return newX, newH

def sampleMany(n, hactivations=hactivations,
               costOperationsList=costOperationsList,
               mainGraph=mainGraph):
    string = ""
    nextH = np.zeros_like(hactivations[0].getValue())
    nextX = np.zeros_like(x[0])
    nextX[int(np.random.random() * nFeatures)] = 1

    for index in range(n):
        nextX, nextH = sampleSingle(nextX, nextH)
        idx = np.random.choice(nextX.size, p=nextX[0])
        nextX[:] = 0
        nextX[0, idx] = 1
        string += array2char(nextX) + " "
    return string

print(sampleMany(100))
<commit_msg>Prepping cx1 sim v2<commit_after>import graphAttack as ga
import numpy as np
import pickle
import sys
"""Control script"""
simulationIndex = 0
# simulationIndex = int(sys.argv[1])

pickleFilename = "dataSet/trump_campaign.pkl"
# pickleFilename = "dataSet/singleSentence.pkl"
with open(pickleFilename, "rb") as fp:
    x, index_to_word, word_to_index = pickle.load(fp)

seriesLength, nFeatures = x.shape
nExamples = simulationIndex
exampleLength = 20
nHidden = 100
nHidden2 = 100

# nExamples = 2
# exampleLength = 15
# nHidden = 35
# nHidden2 = 40

mainGraph = ga.Graph(False)
dummyX = np.zeros((nExamples, exampleLength, nFeatures))
feed = mainGraph.addOperation(ga.Variable(dummyX), feederOperation=True)

hactivations0 = ga.addInitialRNNLayer(mainGraph,
                                      inputOperation=feed,
                                      activation=ga.TanhActivation,
                                      nHidden=nHidden)

hactivations = ga.appendRNNLayer(mainGraph,
                                 previousActivations=hactivations0,
                                 activation=ga.TanhActivation,
                                 nHidden=nHidden2)

finalCost, costOperationsList = ga.addRNNCost(mainGraph,
                                              hactivations,
                                              costActivation=ga.SoftmaxActivation,
                                              costOperation=ga.CrossEntropyCostSoftmax,
                                              nHidden=nHidden2,
                                              labelsShape=feed.shape,
                                              labels=None)

def fprime(p, data, labels, costOperationsList=costOperationsList, mainGraph=mainGraph):
    mainGraph.feederOperation.assignData(data)
    mainGraph.resetAll()
    for index, cop in enumerate(costOperationsList):
        cop.assignLabels(labels[:, index, :])
    mainGraph.attachParameters(p)
    c = mainGraph.feedForward()
    mainGraph.feedBackward()
    g = mainGraph.unrollGradients()
    return c, g

param0 = mainGraph.unrollGradientParameters()
print("Number of parameters to train:", len(param0))
adaGrad = ga.adaptiveSGDrecurrent(trainingData=x,
                                  param0=param0,
                                  epochs=1e3,
                                  miniBatchSize=nExamples,
                                  exampleLength=exampleLength,
                                  initialLearningRate=1e-3,
                                  beta1=0.9,
                                  beta2=0.999,
                                  epsilon=1e-8,
                                  testFrequency=1e2,
                                  function=fprime)

params = adaGrad.minimize(printTrainigCost=True, printUpdateRate=False,
                          dumpParameters="paramsRNN_" + str(simulationIndex) + ".pkl")
# pickleFilename = "paramsRNN_" + str(simulationIndex) + ".pkl"
# with open(pickleFilename, "rb") as fp:
#     params = pickle.load(fp)

mainGraph.attachParameters(params)

def array2char(array):
    return index_to_word[np.argmax(array)]

def sampleSingle(previousX, previousH,
                 hactivations=hactivations,
                 costOperationsList=costOperationsList,
                 mainGraph=mainGraph):
    N, T, D = mainGraph.feederOperation.shape
    preiousData = np.zeros((1, T, D))
    preiousData[:, 0, :] = previousX

    mainGraph.resetAll()
    mainGraph.feederOperation.assignData(preiousData)
    hactivations[0].assignData(previousH)

    newH = hactivations[1].getValue()
    # newX = costOperationsList[0].makePredictions()
    newX = costOperationsList[0].inputA.getValue()

    return newX, newH

def sampleMany(n, hactivations=hactivations,
               costOperationsList=costOperationsList,
               mainGraph=mainGraph):
    string = ""
    nextH = np.zeros_like(hactivations[0].getValue())
    nextX = np.zeros_like(x[0])
    nextX[int(np.random.random() * nFeatures)] = 1

    for index in range(n):
        nextX, nextH = sampleSingle(nextX, nextH)
        idx = np.random.choice(nextX.size, p=nextX[0])
        nextX[:] = 0
        nextX[0, idx] = 1
        string += array2char(nextX) + " "
    return string

print(sampleMany(100))
<|endoftext|>


<commit_before>######################################
# File Name : web_version.py
# Author : Daniel Cuneo
# Creation Date : 11-21-2015
######################################
import pandas as pd
from flask import Flask
from flask import request, render_template, url_for
import indeed_scrape
import jinja2
from bokeh.embed import components
from bokeh.plotting import figure, output_file
from bokeh.util.string import encode_utf8
from bokeh.charts import Bar

input_template = jinja2.Template('''
<!DOCTYPE html>

    <title>indeed skill scraper</title>
    <meta charset="UTF-8">

INDEED.COM JOB OPENINGS SKILL SCRAPER

    <form action="/main/" method="POST">
        Enter keywords you normally use to search for openings on indeed.com

        <input type="text" name="kw">

        Enter zipcodes

        <input type="text" name="zipcodes">

        <input type="submit" value="Submit" name="submit">
    </form>

''')

output_template = jinja2.Template("""
<!DOCTYPE html>

    <title>indeed skill scraper results</title>
    <meta charset="UTF-8">

<link
    href="http://cdn.pydata.org/bokeh/release/bokeh-0.9.0.min.css"
    rel="stylesheet" type="text/css"
>

INDEED.COM JOB OPENINGS SKILL SCRAPER RESULTS

    {{ script }}

    {{ script2 }}

""")

app = Flask(__name__)

def plot_fig(df, num):

    title_string = "Analysis of %i Postings" % num

    p = Bar(df, 'kw',
            values='count',
            title=title_string,
            title_text_font_size='15',
            color='blue',
            xlabel="keywords",
            ylabel="Count")

    return p

@app.route('/')
def get_keywords():
    return input_template.render()

@app.route('/main/', methods=['POST'])
def main():
    kws = request.form['kw']
    zips = request.form['zipcodes']
    return output_template.render(script=kws, script2=zips)
#    kw, count, num, cities = run_analysis(kws, zips)
#
#    df = pd.DataFrame(columns=['keywords','counts', 'cities'])
#
#    df['kw'] = kw
#    df['count'] = count
#    df['cities'] = cities
#
#    p = plot_fig(df, num)
#    script, div = components(p)
#
#    html = output_template.render(script=script, div=div)
#
#    return encode_utf8(html)

def run_analysis(keywords, zipcodes):

    ind = indeed_scrape.Indeed()
    ind.query = keywords
    ind.stop_words = "and"
    ind.add_loc = zipcodes

    ind.main()
    df = ind.df
    df = df.drop_duplicates(['url']).dropna(how='any')

    count, kw = ind.vectorizer(df['summary_stem'])
    #convert from sparse matrix to single dim np array
    count = count.toarray().sum(axis=0)
    num = df['url'].count()

    return kw, count, num, df['city']

if __name__ == "__main__":
    app.debug = True
    app.run()
<commit_msg>added logging<commit_after>######################################
# File Name : web_version.py
# Author : Daniel Cuneo
# Creation Date : 11-21-2015
######################################
import logging
import pandas as pd
from flask import Flask
from flask import request, render_template, url_for
import indeed_scrape
import jinja2
from bokeh.embed import components
from bokeh.plotting import figure, output_file
from bokeh.util.string import encode_utf8
from bokeh.charts import Bar
import os

data_dir = os.getenv('OPENSHIFT_DATA_DIR')
logfile = os.path.join(data_dir, 'logfile.log')
logging.basicConfig(filename=logfile, level=logging.info)

input_template = jinja2.Template('''
<!DOCTYPE html>

    <title>indeed skill scraper</title>
    <meta charset="UTF-8">

INDEED.COM JOB OPENINGS SKILL SCRAPER

    <form action="/main/" method="POST">
        Enter keywords you normally use to search for openings on indeed.com

        <input type="text" name="kw">

        Enter zipcodes

        <input type="text" name="zipcodes">

        <input type="submit" value="Submit" name="submit">
    </form>

''')

output_template = jinja2.Template("""
<!DOCTYPE html>

    <title>indeed skill scraper results</title>
    <meta charset="UTF-8">

<link
    href="http://cdn.pydata.org/bokeh/release/bokeh-0.9.0.min.css"
    rel="stylesheet" type="text/css"
>

INDEED.COM JOB OPENINGS SKILL SCRAPER RESULTS

    {{ script }}

    {{ div }}

""")

app = Flask(__name__)

def plot_fig(df, num):

    title_string = "Analysis of %i Postings" % num

    p = Bar(df, 'kw',
            values='count',
            title=title_string,
            title_text_font_size='15',
            color='blue',
            xlabel="keywords",
            ylabel="Count")

    return p

@app.route('/')
def get_keywords():
    return input_template.render()

@app.route('/main/', methods=['POST'])
def main():
    try:
        kws = request.form['kw']
        zips = request.form['zipcodes']

        kw, count, num, cities = run_analysis(kws, zips)

        df = pd.DataFrame(columns=['keywords','counts', 'cities'])

        df['kw'] = kw
        df['count'] = count
        df['cities'] = cities

        p = plot_fig(df, num)
        script, div = components(p)

        html = output_template.render(script=script, div=div)

        return encode_utf8(html)

    except Exception, err:
        logging.error(err)
        raise

def run_analysis(keywords, zipcodes):

    ind = indeed_scrape.Indeed()
    ind.query = keywords
    ind.stop_words = "and"
    ind.add_loc = zipcodes

    ind.main()
    df = ind.df
    df = df.drop_duplicates(['url']).dropna(how='any')

    count, kw = ind.vectorizer(df['summary_stem'])
    #convert from sparse matrix to single dim np array
    count = count.toarray().sum(axis=0)
    num = df['url'].count()

    return kw, count, num, df['city']

if __name__ == "__main__":
    app.debug = True
    app.run()
<|endoftext|>