<commit_before>import logging
from collections import OrderedDict

import pytest
import gevent
from gevent import Greenlet
from geventwebsocket import (
    WebSocketApplication, WebSocketServer, Resource,
)
from mock import ANY
from mock import call, patch

from wampy.errors import ConnectionError
from wampy.peers.clients import Client
from wampy.testing.helpers import wait_for_session
from wampy.transports.websocket.connection import WebSocket
from wampy.transports.websocket.frames import Close, Ping

logger = logging.getLogger(__name__)

class TestApplication(WebSocketApplication):
    pass

@pytest.fixture
def server():
    s = WebSocketServer(
        ('0.0.0.0', 8001),
        Resource(OrderedDict([('/', TestApplication)]))
    )
    s.start()
    thread = Greenlet.spawn(s.serve_forever)
    yield s
    s.stop()
    thread.kill()

def test_send_ping(server):
    websocket = WebSocket(server_url='ws://0.0.0.0:8001')
    with patch.object(websocket, 'handle_ping') as mock_handle:
        assert websocket.connected is False

        websocket.connect(upgrade=False)

        def connection_handler():
            while True:
                try:
                    message = websocket.receive()
                except Exception:
                    logger.exception('connection handler exploded')
                    raise
                if message:
                    logger.info('got message: %s', message)

        assert websocket.connected is True

        # the first bytes sent down the connection are the response bytes
        # to the TCP connection and upgrade. we receieve in this thread
        # because it will block all execution
        Greenlet.spawn(connection_handler)
        gevent.sleep(0.01)  # enough for the upgrade to happen

        clients = server.clients
        assert len(clients) == 1

        client_handler = list(clients.values())[0]
        socket = client_handler.ws

        ping_frame = Ping()
        socket.send(ping_frame.frame)

        with gevent.Timeout(5):
            while mock_handle.call_count != 1:
                gevent.sleep(0.01)

        assert mock_handle.call_count == 1
        assert mock_handle.call_args == call(ping_frame=ANY)

        call_param = mock_handle.call_args[1]['ping_frame']
        assert isinstance(call_param, Ping)

@pytest.fixture(scope="function")
def config_path():
    return './wampy/testing/configs/crossbar.timeout.json'

def test_respond_to_ping_with_pong(config_path, router):
    # This test shows proper handling of ping/pong keep-alives
    # by connecting to a pong-demanding server (crossbar.timeout.json)
    # and keeping the connection open for longer than the server's timeout.
    # Failure would be an exception being thrown because of the server
    # closing the connection.

    class MyClient(Client):
        pass

    exceptionless = True

    try:
        client = MyClient(url=router.url)
        client.start()
        wait_for_session(client)

        gevent.sleep(5)

        # this is purely to demonstrate we can make calls while sending
        # pongs
        client.publish(topic="test", message="test")
        client.stop()
    except Exception as e:
        print(e)
        exceptionless = False

    assert exceptionless

def test_server_closess(server):
    websocket = WebSocket(server_url='ws://0.0.0.0:8001')
    with patch.object(websocket, 'handle_close') as mock_handle:
        websocket.connect(upgrade=False)

        def connection_handler():
            while True:
                try:
                    message = websocket.receive()
                except Exception:
                    logger.exception('connection handler exploded')
                    raise
                if message:
                    logger.info('got message: %s', message)

        Greenlet.spawn(connection_handler)
        gevent.sleep(0.01)  # enough for the upgrade to happen

        clients = server.clients
        client_handler = list(clients.values())[0]
        socket = client_handler.ws
        Greenlet.spawn(socket.close)

        with gevent.Timeout(1):
            while mock_handle.call_count != 1:
                gevent.sleep(0.01)

        assert mock_handle.call_count == 1
        assert mock_handle.call_args == call(close_frame=ANY)

        call_param = mock_handle.call_args[1]['close_frame']
        assert isinstance(call_param, Close)

def test_close_message_payload(server):
    websocket = WebSocket(server_url='ws://0.0.0.0:8001')
    close_frame = Close(payload="explosion")

    with pytest.raises(ConnectionError) as exc:
        websocket.handle_close(close_frame=close_frame)

    assert "explosion" in str(exc)
<commit_msg>skip gevent based test cases when eventlet is the async backend<commit_after>import os
import logging
from collections import OrderedDict

import pytest
import gevent
from gevent import Greenlet
from geventwebsocket import (
    WebSocketApplication, WebSocketServer, Resource,
)
from mock import ANY
from mock import call, patch

from wampy.async import async_adapter
from wampy.constants import GEVENT
from wampy.errors import ConnectionError
from wampy.peers.clients import Client
from wampy.testing.helpers import wait_for_session
from wampy.transports.websocket.connection import WebSocket
from wampy.transports.websocket.frames import Close, Ping

logger = logging.getLogger(__name__)

gevent_only = pytest.mark.skipif(
    os.environ['WAMPY_ASYNC_NAME'] != GEVENT,
    reason="requires a Greenlet WebSocket server and you're using eventlet"
)

class TestApplication(WebSocketApplication):
    pass

@pytest.fixture
def server():
    s = WebSocketServer(
        ('0.0.0.0', 8001),
        Resource(OrderedDict([('/', TestApplication)]))
    )
    s.start()
    thread = Greenlet.spawn(s.serve_forever)
    yield s
    s.stop()
    thread.kill()

@gevent_only
def test_send_ping(server):
    websocket = WebSocket(server_url='ws://0.0.0.0:8001')
    with patch.object(websocket, 'handle_ping') as mock_handle:
        assert websocket.connected is False

        websocket.connect(upgrade=False)

        def connection_handler():
            while True:
                try:
                    message = websocket.receive()
                except Exception:
                    logger.exception('connection handler exploded')
                    raise
                if message:
                    logger.info('got message: %s', message)

        assert websocket.connected is True

        # the first bytes sent down the connection are the response bytes
        # to the TCP connection and upgrade. we receieve in this thread
        # because it will block all execution
        Greenlet.spawn(connection_handler)
        gevent.sleep(0.01)  # enough for the upgrade to happen

        clients = server.clients
        assert len(clients) == 1

        client_handler = list(clients.values())[0]
        socket = client_handler.ws

        ping_frame = Ping()
        socket.send(ping_frame.frame)

        with gevent.Timeout(5):
            while mock_handle.call_count != 1:
                gevent.sleep(0.01)

        assert mock_handle.call_count == 1
        assert mock_handle.call_args == call(ping_frame=ANY)

        call_param = mock_handle.call_args[1]['ping_frame']
        assert isinstance(call_param, Ping)

@pytest.fixture(scope="function")
def config_path():
    return './wampy/testing/configs/crossbar.timeout.json'

def test_respond_to_ping_with_pong(config_path, router):
    # This test shows proper handling of ping/pong keep-alives
    # by connecting to a pong-demanding server (crossbar.timeout.json)
    # and keeping the connection open for longer than the server's timeout.
    # Failure would be an exception being thrown because of the server
    # closing the connection.

    class MyClient(Client):
        pass

    exceptionless = True

    try:
        client = MyClient(url=router.url)
        client.start()
        wait_for_session(client)

        async_adapter.sleep(5)

        # this is purely to demonstrate we can make calls while sending
        # pongs
        client.publish(topic="test", message="test")
        client.stop()
    except Exception as e:
        print(e)
        exceptionless = False

    assert exceptionless

@gevent_only
def test_server_closess(server):
    websocket = WebSocket(server_url='ws://0.0.0.0:8001')
    with patch.object(websocket, 'handle_close') as mock_handle:
        websocket.connect(upgrade=False)

        def connection_handler():
            while True:
                try:
                    message = websocket.receive()
                except Exception:
                    logger.exception('connection handler exploded')
                    raise
                if message:
                    logger.info('got message: %s', message)

        Greenlet.spawn(connection_handler)
        gevent.sleep(0.01)  # enough for the upgrade to happen

        clients = server.clients
        client_handler = list(clients.values())[0]
        socket = client_handler.ws
        Greenlet.spawn(socket.close)

        with gevent.Timeout(1):
            while mock_handle.call_count != 1:
                gevent.sleep(0.01)

        assert mock_handle.call_count == 1
        assert mock_handle.call_args == call(close_frame=ANY)

        call_param = mock_handle.call_args[1]['close_frame']
        assert isinstance(call_param, Close)

@gevent_only
def test_close_message_payload(server):
    websocket = WebSocket(server_url='ws://0.0.0.0:8001')
    close_frame = Close(payload="explosion")

    with pytest.raises(ConnectionError) as exc:
        websocket.handle_close(close_frame=close_frame)

    assert "explosion" in str(exc)
<|endoftext|>


<commit_before>'''Arsenal physical_elevations DB Model'''
#  Copyright 2015 CityGrid Media, LLC
#
#  Licensed under the Apache License, Version 2.0 (the "License");
#  you may not use this file except in compliance with the License.
#  You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
#  Unless required by applicable law or agreed to in writing, software
#  distributed under the License is distributed on an "AS IS" BASIS,
#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#  See the License for the specific language governing permissions and
#  limitations under the License.
#
import logging
from sqlalchemy import (
    Column,
    ForeignKey,
    Integer,
    TIMESTAMP,
    Text,
)
from sqlalchemy.orm import relationship
from sqlalchemy.orm import backref
from arsenalweb.models.common import (
    Base,
    BaseAudit,
    get_name_id_dict,
    jsonify,
)

LOG = logging.getLogger(__name__)

class PhysicalElevation(Base):
    '''Arsenal PhysicalElevation object.'''

    __tablename__ = 'physical_elevations'
    id = Column(Integer, primary_key=True, nullable=False)
    elevation = Column(Integer, nullable=False)
    physical_rack_id = Column(Integer, ForeignKey('physical_racks.id'), nullable=False)
    created = Column(TIMESTAMP, nullable=False)
    updated = Column(TIMESTAMP, nullable=False)
    updated_by = Column(Text, nullable=False)

    physical_rack = relationship('PhysicalRack', backref='physical_elevations', lazy='joined')

    def __json__(self, request):
        try:
            # FIXME: Not sure if this constraint is going to cause
            # other problems.
            if request.path_info.startswith('/api/physical_elevations'):
                fields = request.params['fields']

            if fields == 'all':
                # Everything.
                all_fields = dict(
                    id=self.id,
                    elevation=self.elevation,
#                    physical_device=self.physical_device,
                    physical_rack=get_name_id_dict([self.physical_rack],
                                                   default_keys=['id',
                                                                 'name',
                                                                 'physical_location']),
                    created=self.created,
                    updated=self.updated,
                    updated_by=self.updated_by,
                    )

                return jsonify(all_fields)

            else:
                # Always return id and name, then return whatever additional fields
                # are asked for.
                resp = get_name_id_dict([self])

                my_fields = fields.split(',')

                resp.update((key, getattr(self, key)) for key in my_fields if
                            key in self.__dict__)

                return jsonify(resp)

        # Default to returning only these fields.
        except (KeyError, UnboundLocalError):
            resp = get_name_id_dict([self], default_keys=['id',
                                                          'elevation',])

            return resp

class PhysicalElevationAudit(BaseAudit):
    '''Arsenal PhysicalElevationAudit object.'''

    __tablename__ = 'physical_elevations_audit'
<commit_msg>INFRA-14811: major<commit_after>'''Arsenal physical_elevations DB Model'''
#  Copyright 2015 CityGrid Media, LLC
#
#  Licensed under the Apache License, Version 2.0 (the "License");
#  you may not use this file except in compliance with the License.
#  You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
#  Unless required by applicable law or agreed to in writing, software
#  distributed under the License is distributed on an "AS IS" BASIS,
#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#  See the License for the specific language governing permissions and
#  limitations under the License.
#
import logging
from sqlalchemy import (
    Column,
    ForeignKey,
    Integer,
    TIMESTAMP,
    Text,
)
from sqlalchemy.orm import relationship
from sqlalchemy.orm import backref
from arsenalweb.models.common import (
    Base,
    BaseAudit,
    get_name_id_dict,
    jsonify,
)

LOG = logging.getLogger(__name__)

class PhysicalElevation(Base):
    '''Arsenal PhysicalElevation object.'''

    __tablename__ = 'physical_elevations'
    id = Column(Integer, primary_key=True, nullable=False)
    elevation = Column(Integer, nullable=False)
    physical_rack_id = Column(Integer, ForeignKey('physical_racks.id'), nullable=False)
    created = Column(TIMESTAMP, nullable=False)
    updated = Column(TIMESTAMP, nullable=False)
    updated_by = Column(Text, nullable=False)

    physical_rack = relationship('PhysicalRack', backref='physical_elevations', lazy='joined')

    def __json__(self, request):
        try:
            # FIXME: Not sure if this constraint is going to cause
            # other problems.
            if request.path_info.startswith('/api/physical_elevations'):
                fields = request.params['fields']

            if fields == 'all':
                # Everything.
                all_fields = dict(
                    id=self.id,
                    elevation=self.elevation,
                    physical_rack=get_name_id_dict([self.physical_rack],
                                                   default_keys=['id',
                                                                 'name',
                                                                 'physical_location']),
                    created=self.created,
                    updated=self.updated,
                    updated_by=self.updated_by,
                    )

                return jsonify(all_fields)

            else:
                # Always return id and name, then return whatever additional fields
                # are asked for.
                resp = get_name_id_dict([self])

                my_fields = fields.split(',')

                resp.update((key, getattr(self, key)) for key in my_fields if
                            key in self.__dict__)

                return jsonify(resp)

        # Default to returning only these fields.
        except (KeyError, UnboundLocalError):
            resp = get_name_id_dict([self], default_keys=['id',
                                                          'elevation',])

            return resp

class PhysicalElevationAudit(BaseAudit):
    '''Arsenal PhysicalElevationAudit object.'''

    __tablename__ = 'physical_elevations_audit'
<|endoftext|>


<commit_before>from .config import _cfg, _cfgi
from .database import r, _k

from flask import request

def rate_limit_exceeded():
    consumed = int(r.get(_k("rate_limit.%s") % request.remote_addr))
    if not consumed: return False

    return consumed >= _cfgi("bytes_per_hour")

def rate_limit_update(f):
    key = _k("rate_limit.%s" % request.remote_addr)
    f.seek(0, 2)
    b = f.tell()
    f.seek(0)

    if not r.exists(key):
        r.setex(key, 3600 * 60 * 60, b)
    else: r.incrby(key, b)
<commit_msg>Accidentally set the rate limit expiration date to 3600 hours.<commit_after>from .config import _cfg, _cfgi
from .database import r, _k

from flask import request

def rate_limit_exceeded():
    consumed = int(r.get(_k("rate_limit.%s") % request.remote_addr))
    if not consumed: return False

    return consumed >= _cfgi("bytes_per_hour")

def rate_limit_update(f):
    key = _k("rate_limit.%s" % request.remote_addr)
    f.seek(0, 2)
    b = f.tell()
    f.seek(0)

    if not r.exists(key):
        r.setex(key, 3600, b)
    else: r.incrby(key, b)
<|endoftext|>


<commit_before>from easyquery import Query, QueryMaker

_list_by_pgc = {
    # fmt: off
    'hostlist_v1': (
        1952, 2052, 2081, 2437, 2778, 3089, 3743, 4948, 5139, 5619, 6110, 6656,
        6993, 8160, 8631, 9057, 9747, 9788, 9843, 9988, 10330, 10965, 771919,
        11819, 12007, 12431, 12626, 13130, 13166, 13602, 13881, 13999, 14084,
        16716, 16906, 16949, 17436, 18394, 18880, 20458, 22270, 23028, 23701,
        24464, 25069, 25248, 25886, 25955, 25999, 26246, 26483, 26649, 26939,
        27004, 27635, 27723, 27982, 28805, 28876, 29009, 30176, 31166, 31428,
        31636, 32533, 32854, 33371, 33623, 33860, 34561, 34731, 34908, 35088,
        35097, 35285, 35294, 35539, 35869, 36138, 36266, 36304, 36737, 36827,
        36914, 37178, 37366, 37483, 37845, 38031, 38049, 38555, 38588, 38802,
        38851, 39414, 40055, 40265, 40284, 41083, 41150, 42139, 44025, 44145,
        44600, 45269, 45795, 46304, 46938, 47394, 47721, 48258, 48307, 48767,
        48815, 48816, 49342, 49563, 49604, 49618, 50031, 50670, 50732, 51055,
        51340, 51471, 51564, 51620, 51787, 51932, 52116, 52266, 52365, 52411,
        52735, 52825, 53037, 53043, 53499, 53630, 53724, 53755, 54119, 54234,
        54348, 54364, 54429, 54761, 54849, 55081, 55281, 55588, 56337, 57345,
        57924, 58470, 59426, 61742, 61833, 61972, 63797, 64427, 64446, 64772,
        65436, 66318, 66566, 66648, 66812, 66934, 67018, 67146, 67546, 67663,
        67817, 67839, 67892, 67932, 68128, 68389, 68469, 68511, 68712, 68743,
        68826, 69094, 69253, 69349, 69521, 69661, 70025, 70094, 70128, 70419,
        70505, 70676, 70795, 71548, 71729, 71883, 72001, 73163,
    ),
    'paper1_complete': (9988, 23028, 52735, 55588, 58470, 68743, 70795, 71883),
    'paper1_incomplete': (279, 4948, 28805, 31166, 37845, 38031, 38802, 53499),
    'paper2_complete': (
        4948, 9988, 12626, 13646, 18880, 23028, 26246, 27635, 27723,
        35294, 37483, 38802, 40284, 41083, 48815, 49342, 50031, 51340,
        51471, 51620, 52273, 52735, 53499, 54119, 55588, 58470, 59426,
        64725, 66318, 66934, 67782, 67817, 68743, 69349, 70795, 71883,
    ),
    "paper3": (
        279, 1952, 2081, 4948, 5139, 6110, 8279, 8974, 9747,
        9988, 10060, 10330, 11527, 11638, 12342, 12626, 13108, 13166,
        13646, 16044, 18880, 23028, 24111, 25237, 25955, 26034, 26100,
        26246, 27074, 27635,
        27723, 28805, 30176, 31636, 32605, 34561, 35294, 36138, 36304,
        37483, 38031, 38802, 40284, 40490, 41083, 44017, 44025, 45643,
        48767, 48815, 49112, 49342, 49604, 49618, 49820, 50031, 50670,
        51055, 51275, 51340, 51471, 51620, 51953, 52116, 52266, 52273,
        52735, 53043, 53499, 53630, 54111, 54119, 54234, 55588, 56337,
        57345, 58470, 59426, 64725, 66318, 66566, 66669, 66934, 67146,
        67532, 67663, 67782, 67817, 68312, 68743, 69349, 69677, 70025,
        70419, 70795, 71883, 72001, 72009, 72060, 73163,
    )
    # fmt: on
}

has_decals_image = Query("COVERAGE_DECALS_DR9 >= 0.95")
has_des_image = Query("COVERAGE_DES_DR1 >= 0.95")
has_sdss_image = Query("COVERAGE_SDSS >= 0.95")
has_decals_dr8_image = Query("COVERAGE_DECALS_DR8 >= 0.95")
has_decals_dr9_image = Query("COVERAGE_DECALS_DR9 >= 0.95")
has_decals_dr9_image_99 = Query("COVERAGE_DECALS_DR9 >= 0.99")

has_image = Query("HAS_IMAGE > 0")
has_better_image = Query("HAS_IMAGE > 1")

potential_hosts = Query("HOST_SCORE > 0")
good_hosts = Query("HOST_SCORE >= 4")
good = good_hosts & has_image
build_default = good
cos_saga = QueryMaker.in1d(
    "HOSTID",
    (
        "nsa85746",
        "nsa85746",
        "nsa85746",
        "nsa140594",
        "nsa13927",
        "nsa144953",
        "nsa165082",
        "nsa165153",
        "nsa165707",
        "nsa147606",
    ),
)

hostlist_v1 = QueryMaker.in1d("PGC", _list_by_pgc["hostlist_v1"])
paper1_complete = QueryMaker.in1d("PGC", _list_by_pgc["paper1_complete"])
paper1_incomplete = QueryMaker.in1d("PGC", _list_by_pgc["paper1_incomplete"])
paper1 = QueryMaker.in1d("PGC", _list_by_pgc["paper1_complete"] + _list_by_pgc["paper1_incomplete"])
paper1_observed = paper1

paper2 = paper2_complete = QueryMaker.in1d("PGC", _list_by_pgc["paper2_complete"])
paper3 = QueryMaker.in1d("PGC", _list_by_pgc["paper3"])

observed_definition = Query("specs_ours_rvir >= 100", "specs_ours_rvir < 999999")

complete_definition2 = Query(
    "paper2_need_spec / paper2_total < 0.2",
    "paper2_total > 0",
    "paper2_need_spec < 999999",
)
complete_definition3 = Query(
    "paper3_need_spec / paper3_total < 0.4",
    "paper3_total > 0",
    "paper3_need_spec < 999999",
)
complete_definition = complete_definition3

observed = good_hosts & observed_definition
unobserved = good & (~observed_definition)
complete = observed & complete_definition
incomplete = observed & (~complete_definition)

environment_allowed = Query(
    "abs(GLAT) >= 25",
    "BRIGHTEST_K_R1 >= K_TC + 1",
    "BRIGHTEST_STAR_R1 >= 5",
    "M_HALO < 13",
    "REMOVED_BY_HAND == 0",
)
environment_preferred = Query(environment_allowed, "BRIGHTEST_K_R1 >= K_TC + 1.6")
mass_allowed = Query("K_ABS >= -24.7", "K_ABS <= -22.9")
mass_preferred = Query("K_ABS >= -24.6", "K_ABS <= -23.0")
distance_allowed = Query("DIST >= 20", "DIST <= 42", "V_HELIO >= 1400")
distance_preferred = Query("DIST >= 25", "DIST <= 40.75", "V_HELIO >= 1400")

has_mw_mass_neighbor = QueryMaker.not_equal("NEARBY_MW_COUNT_1.5", 0)
is_local_group_like = QueryMaker.equal("NEARBY_MW_COUNT_1.5", 1)
<commit_msg>update host cuts<commit_after>from easyquery import Query, QueryMaker

# fmt: off
_hostlist_v1_pgc = (
    1952, 2052, 2081, 2437, 2778, 3089, 3743, 4948, 5139, 5619, 6110, 6656,
    6993, 8160, 8631, 9057, 9747, 9788, 9843, 9988, 10330, 10965, 771919,
    11819, 12007, 12431, 12626, 13130, 13166, 13602, 13881, 13999, 14084,
    16716, 16906, 16949, 17436, 18394, 18880, 20458, 22270, 23028, 23701,
    24464, 25069, 25248, 25886, 25955, 25999, 26246, 26483, 26649, 26939,
    27004, 27635, 27723, 27982, 28805, 28876, 29009, 30176, 31166, 31428,
    31636, 32533, 32854, 33371, 33623, 33860, 34561, 34731, 34908, 35088,
    35097, 35285, 35294, 35539, 35869, 36138, 36266, 36304, 36737, 36827,
    36914, 37178, 37366, 37483, 37845, 38031, 38049, 38555, 38588, 38802,
    38851, 39414, 40055, 40265, 40284, 41083, 41150, 42139, 44025, 44145,
    44600, 45269, 45795, 46304, 46938, 47394, 47721, 48258, 48307, 48767,
    48815, 48816, 49342, 49563, 49604, 49618, 50031, 50670, 50732, 51055,
    51340, 51471, 51564, 51620, 51787, 51932, 52116, 52266, 52365, 52411,
    52735, 52825, 53037, 53043, 53499, 53630, 53724, 53755, 54119, 54234,
    54348, 54364, 54429, 54761, 54849, 55081, 55281, 55588, 56337, 57345,
    57924, 58470, 59426, 61742, 61833, 61972, 63797, 64427, 64446, 64772,
    65436, 66318, 66566, 66648, 66812, 66934, 67018, 67146, 67546, 67663,
    67817, 67839, 67892, 67932, 68128, 68389, 68469, 68511, 68712, 68743,
    68826, 69094, 69253, 69349, 69521, 69661, 70025, 70094, 70128, 70419,
    70505, 70676, 70795, 71548, 71729, 71883, 72001, 73163,
)

_paper1_complete_hostid = (
    'nsa132339', 'nsa33446', 'nsa165536', 'nsa166313',
    'nsa147100', 'nsa149781', 'nsa150887', 'nsa61945',
)

_paper1_incomplete_hostid = (
    'nsa126115', 'nsa129237', 'nsa85746', 'nsa137625',
    'nsa140594', 'nsa13927', 'nsa161174', 'nsa145729',
)

_paper2_hostid = (
    'nsa129237', 'nsa132339', 'nsa133355', 'nsa133549', 'pgc18880', 'nsa33446',
    'nsa16235', 'nsa157374', 'nsa32', 'nsa139467', 'nsa140458', 'nsa161174',
    'nsa3469', 'nsa141465', 'nsa143856', 'nsa94340', 'nsa69028', 'nsa144953',
    'nsa35340', 'nsa51348', 'nsa145372', 'nsa165536', 'nsa145729', 'nsa145879',
    'nsa166313', 'nsa147100', 'nsa147606', 'pgc64725', 'pgc66318', 'pgc66934',
    'pgc67782', 'pgc67817', 'nsa149781', 'nsa149977', 'nsa150887', 'nsa61945',
)

_paper3_hostid = (
    'nsa126115', 'nsa127226', 'nsa129237', 'nsa129387', 'nsa130133', 'nsa132339',
    'nsa132368', 'nsa133115', 'nsa133141', 'nsa133355', 'nsa133549', 'nsa135296',
    'nsa135591', 'nsa135739', 'nsa13927', 'nsa139467', 'nsa140458', 'nsa141315',
    'nsa141465', 'nsa143856', 'nsa14409', 'nsa144151', 'nsa144953', 'nsa145297',
    'nsa145372', 'nsa145729', 'nsa145879', 'nsa146486', 'nsa147100', 'nsa147606',
    'nsa149781', 'nsa149977', 'nsa150307', 'nsa150578', 'nsa150887', 'nsa153017',
    'nsa155005', 'nsa157300', 'nsa157374', 'nsa158901', 'nsa159488', 'nsa159520',
    'nsa159593', 'nsa160302', 'nsa161174', 'nsa16235', 'nsa163956', 'nsa164865',
    'nsa165162', 'nsa165316', 'nsa165536', 'nsa16559', 'nsa165707', 'nsa165956',
    'nsa165980', 'nsa166313', 'nsa169439', 'nsa32', 'nsa33446', 'nsa3469',
    'nsa35340', 'nsa47117', 'nsa51348', 'nsa52773', 'nsa56434', 'nsa61945',
    'nsa67593', 'nsa69028', 'nsa85746', 'nsa94217', 'nsa94340', 'pgc10330',
    'pgc12342', 'pgc13108', 'pgc13166', 'pgc16044', 'pgc18880', 'pgc1952',
    'pgc26034', 'pgc31636', 'pgc36304', 'pgc45643', 'pgc50670', 'pgc51055',
    'pgc53630', 'pgc57345', 'pgc64725', 'pgc66318', 'pgc66566', 'pgc66669',
    'pgc66934', 'pgc67146', 'pgc67532', 'pgc67663', 'pgc67782', 'pgc67817',
    'pgc68312', 'pgc69677', 'pgc72009', 'pgc72060', 'pgc8279', 'pgc9747',
)

_cos_saga_hostid = (
    "nsa85746", "nsa85746", "nsa85746", "nsa140594", "nsa13927",
    "nsa144953", "nsa165082", "nsa165153", "nsa165707", "nsa147606",
)
# fmt: on

has_decals_image = Query("COVERAGE_DECALS_DR9 >= 0.95")
has_des_image = Query("COVERAGE_DES_DR1 >= 0.95")
has_sdss_image = Query("COVERAGE_SDSS >= 0.95")
has_decals_dr8_image = Query("COVERAGE_DECALS_DR8 >= 0.95")
has_decals_dr9_image = Query("COVERAGE_DECALS_DR9 >= 0.95")
has_decals_dr9_image_99 = Query("COVERAGE_DECALS_DR9 >= 0.99")

has_image = Query("HAS_IMAGE > 0")
has_better_image = Query("HAS_IMAGE > 1")

potential_hosts = Query("HOST_SCORE > 0")
good_hosts = Query("HOST_SCORE >= 4")
build_default = good = good_hosts & has_image

hostlist_v1 = QueryMaker.isin("PGC", _hostlist_v1_pgc)
paper1_complete = QueryMaker.isin("HOSTID", _paper1_complete_hostid)
paper1_incomplete = QueryMaker.isin("HOSTID", _paper1_incomplete_hostid)
paper1 = paper1_observed = QueryMaker.isin("HOSTID", _paper1_complete_hostid + _paper1_incomplete_hostid)
paper2 = paper2_complete = QueryMaker.isin("HOSTID", _paper2_hostid)
paper3 = paper3_complete = QueryMaker.isin("HOSTID", _paper3_hostid)
cos_saga = QueryMaker.isin("HOSTID", _cos_saga_hostid)

observed_definition = Query("specs_ours_rvir >= 100", "specs_ours_rvir < 999999")

complete_definition2 = Query(
    "paper2_need_spec / paper2_total < 0.2",
    "paper2_total > 0",
    "paper2_need_spec < 999999",
)
complete_definition3 = Query(
    "paper3_need_spec / paper3_total < 0.4",
    "paper3_total > 0",
    "paper3_need_spec < 999999",
)
complete_definition = complete_definition3

observed = good & observed_definition
unobserved = good & (~observed_definition)
complete = observed & complete_definition
incomplete = observed & (~complete_definition)

environment_allowed = Query(
    "abs(GLAT) >= 25",
    "BRIGHTEST_K_R1 >= K_TC + 1",
    "BRIGHTEST_STAR_R1 >= 5",
    "M_HALO < 13",
    "REMOVED_BY_HAND == 0",
)
environment_preferred = Query(environment_allowed, "BRIGHTEST_K_R1 >= K_TC + 1.6")
mass_allowed = Query("K_ABS >= -24.7", "K_ABS <= -22.9")
mass_preferred = Query("K_ABS >= -24.6", "K_ABS <= -23.0")
distance_allowed = Query("DIST >= 20", "DIST <= 42", "V_HELIO >= 1400")
distance_preferred = Query("DIST >= 25", "DIST <= 40.75", "V_HELIO >= 1400")

has_mw_mass_neighbor = QueryMaker.not_equal("NEARBY_MW_COUNT_1.5", 0)
is_local_group_like = QueryMaker.equal("NEARBY_MW_COUNT_1.5", 1)
<|endoftext|>


<commit_before>import os
import subprocess
import re
import codecs
import tempfile
import time
from functools import partial

import sublime

try:
    from . import git_helper
    from .git_gutter_settings import settings
    from .promise import Promise
except (ImportError, ValueError):
    import git_helper
    from git_gutter_settings import settings
    from promise import Promise

class GitGutterHandler(object):
    # the git repo won't change that often so we can easily wait few seconds
    # between updates for performance
    git_file_update_interval_secs = 5

    def __init__(self, view):
        self.view = view

        self.git_temp_file = None
        self.buf_temp_file = None

        self.git_tree = None
        self.git_dir = None
        self.git_path = None

        self._last_refresh_time_git_file = 0

    def __del__(self):
        if self.git_temp_file:
            os.unlink(self.git_temp_file)
        if self.buf_temp_file:
            os.unlink(self.buf_temp_file)

    @staticmethod
    def tmp_file():
        """Create a temp file and return the filepath to it.

        CAUTION: Caller is responsible for clean up
        """
        fd, filepath = tempfile.mkstemp(prefix='git_gutter_')
        os.close(fd)
        return filepath

    def clear_git_time(self):
        self._last_refresh_time_git_file = 0

    def update_git_time(self):
        self._last_refresh_time_git_file = time.time()

    def git_time(self):
        return time.time() - self._last_refresh_time_git_file

    def _get_view_encoding(self):
        # get encoding and clean it for python ex: "Western (ISO 8859-1)"
        # NOTE(maelnor): are we need regex here?
        pattern = re.compile(r'.+\((.*)\)')
        encoding = self.view.encoding()
        if encoding == "Undefined":
            encoding = self.view.settings().get('default_encoding')
        if pattern.match(encoding):
            encoding = pattern.sub(r'\1', encoding)

        encoding = encoding.replace('with BOM', '')
        encoding = encoding.replace('Windows', 'cp')
        encoding = encoding.replace('-', '_')
        encoding = encoding.replace(' ', '')

        # work around with ConvertToUTF8 plugin
        origin_encoding = self.view.settings().get('origin_encoding')
        return origin_encoding or encoding

    def on_disk(self):
        # if the view is saved to disk
        on_disk = os.path.exists(self.view.file_name())
        if on_disk:
            self.git_tree = self.git_tree or git_helper.git_tree(self.view)
            self.git_dir = self.git_dir or git_helper.git_dir(self.git_tree)
            self.git_path = self.git_path or git_helper.git_file_path(
                self.view, self.git_tree)
        return on_disk

    def update_buf_file(self):
        chars = self.view.size()
        region = sublime.Region(0, chars)

        # Try conversion
        try:
            contents = self.view.substr(
                region).encode(self._get_view_encoding())
        except UnicodeError:
            # Fallback to utf8-encoding
            contents = self.view.substr(region).encode('utf-8')
        except LookupError:
            # May encounter an encoding we don't have a codec for
            contents = self.view.substr(region).encode('utf-8')

        contents = contents.replace(b'\r\n', b'\n')
        contents = contents.replace(b'\r', b'\n')

        if not self.buf_temp_file:
            self.buf_temp_file = GitGutterHandler.tmp_file()

        with open(self.buf_temp_file, 'wb') as f:
            if self.view.encoding() == "UTF-8 with BOM":
                f.write(codecs.BOM_UTF8)

            f.write(contents)

    def update_git_file(self):

        def write_file(contents):
            contents = contents.replace(b'\r\n', b'\n')
            contents = contents.replace(b'\r', b'\n')
            with open(self.git_temp_file, 'wb') as f:
                f.write(contents)

        if self.git_time() > self.git_file_update_interval_secs:
            if not self.git_temp_file:
                self.git_temp_file = GitGutterHandler.tmp_file()

            args = [
                settings.git_binary_path,
                '--git-dir=' + self.git_dir,
                '--work-tree=' + self.git_tree,
                'show',
                '%s:%s' % (
                    settings.get_compare_against(self.view), self.git_path),
            ]

            try:
                self.update_git_time()
                return GitGutterHandler.run_command(args).then(write_file)
            except Exception:
                pass
        return Promise.resolve()

    # Parse unified diff with 0 lines of context.
    # Hunk range info format:
    #   @@ -3,2 +4,0 @@
    #     Hunk originally starting at line 3, and occupying 2 lines, now
    #     starts at line 4, and occupies 0 lines, i.e. it was deleted.
    #   @@ -9 +10,2 @@
    #     Hunk size can be omitted, and defaults to one line.
    # Dealing with ambiguous hunks:
    #   "A\nB\n" -> "C\n"
    #   Was 'A' modified, and 'B' deleted? Or 'B' modified, 'A' deleted?
    #   Or both deleted? To minimize confusion, let's simply mark the
    #   hunk as modified.
    def process_diff(self, diff_str):
        inserted = []
        modified = []
        deleted = []
        hunk_re = '^@@ \-(\d+),?(\d*) \+(\d+),?(\d*) @@'
        hunks = re.finditer(hunk_re, diff_str, re.MULTILINE)
        for hunk in hunks:
            start = int(hunk.group(3))
            old_size = int(hunk.group(2) or 1)
            new_size = int(hunk.group(4) or 1)
            if not old_size:
                inserted += range(start, start + new_size)
            elif not new_size:
                deleted += [start + 1]
            else:
                modified += range(start, start + new_size)
        return (inserted, modified, deleted)

    def diff_str(self):
        def decode_diff(results):
            encoding = self._get_view_encoding()
            try:
                decoded_results = results.decode(encoding.replace(' ', ''))
            except UnicodeError:
                try:
                    decoded_results = results.decode("utf-8")
                except UnicodeDecodeError:
                    decoded_results = ""
            except LookupError:
                try:
                    decoded_results = codecs.decode(results)
                except UnicodeDecodeError:
                    decoded_results = ""
            return decoded_results

        def run_diff(_unused):
            self.update_buf_file()
            args = [
                settings.git_binary_path,
                'diff', '-U0', '--no-color', '--no-index',
                settings.ignore_whitespace,
                settings.patience_switch,
                self.git_temp_file,
                self.buf_temp_file,
            ]
            args = list(filter(None, args))  # Remove empty args
            return GitGutterHandler.run_command(args).then(decode_diff)

        if self.on_disk() and self.git_path:
            return self.update_git_file().then(run_diff)
        else:
            return Promise.resolve("")

    def process_diff_line_change(self, line_nr, diff_str):
        hunk_re = '^@@ \-(\d+),?(\d*) \+(\d+),?(\d*) @@'
        hunks = re.finditer(hunk_re, diff_str, re.MULTILINE)

        # we also want to extract the position of the surrounding changes
        first_change = prev_change = next_change = None

        for hunk in hunks:
            start = int(hunk.group(3))
            size = int(hunk.group(4) or 1)
            if first_change is None:
                first_change = start
            # special handling to also match the line below deleted
            # content
            if size == 0 and line_nr == start + 1:
                pass
            # continue if the hunk is before the line
            elif start + size < line_nr:
                prev_change = start
                continue
            # break if the hunk is after the line
            elif line_nr < start:
                break
            # in the following the line is inside the hunk
            try:
                next_hunk = next(hunks)
                hunk_end = next_hunk.start()
                next_change = int(next_hunk.group(3))
            except:
                hunk_end = len(diff_str)

            # if wrap is disable avoid wrapping
            wrap = settings.get('next_prev_change_wrap', True)
            if not wrap:
                if prev_change is None:
                    prev_change = start
                if next_change is None:
                    next_change = start

            # if prev change is None set it to the wrap around the
            # document: prev -> last hunk, next -> first hunk
            if prev_change is None:
                try:
                    remaining_hunks = list(hunks)
                    if remaining_hunks:
                        last_hunk = remaining_hunks[-1]
                        prev_change = int(last_hunk.group(3))
                    elif next_change is not None:
                        prev_change = next_change
                    else:
                        prev_change = start
                except:
                    prev_change = start
            if next_change is None:
                next_change = first_change

            # extract the content of the hunk
            hunk_content = diff_str[hunk.start():hunk_end]
            # store all deleted lines (starting with -)
            hunk_lines = hunk_content.splitlines()[1:]
            deleted_lines = [
                line[1:] for line in hunk_lines if line.startswith("-")
            ]
            added_lines = [line[1:] for line in hunk_lines
                           if line.startswith("+")]
            meta = {
                "added_lines": added_lines,
                "first_change": first_change,
                "next_change": next_change,
                "prev_change": prev_change
            }
            return (deleted_lines, start, size, meta)
        return ([], -1, -1, {})

    def diff_line_change(self, line):
        return self.diff_str().then(
            partial(self.process_diff_line_change, line))

    def diff(self):
        return self.diff_str().then(self.process_diff)

    def untracked(self):
        return self.handle_files([])

    def ignored(self):
        return self.handle_files(['-i'])

    def handle_files(self, additional_args):
        if self.on_disk() and self.git_path:
            def is_nonempty(results):
                encoding = self._get_view_encoding()
                try:
                    decoded_results = results.decode(encoding.replace(' ', ''))
                except UnicodeError:
                    decoded_results = results.decode("utf-8")
                return (decoded_results != "")

            args = [
                settings.git_binary_path,
                '--git-dir=' + self.git_dir,
                '--work-tree=' + self.git_tree,
                'ls-files', '--other', '--exclude-standard',
            ] + additional_args + [
                os.path.join(self.git_tree, self.git_path),
            ]
            args = list(filter(None, args))  # Remove empty args
            return GitGutterHandler.run_command(args).then(is_nonempty)
        return Promise.resolve(False)

    def git_commits(self):
        args = [
            settings.git_binary_path,
            '--git-dir=' + self.git_dir,
            '--work-tree=' + self.git_tree,
            'log', '--all',
            '--pretty=%s\a%h %an <%aE>\a%ad (%ar)',
            '--date=local', '--max-count=9000'
        ]
        return GitGutterHandler.run_command(args)

    def git_branches(self):
        args = [
            settings.git_binary_path,
            '--git-dir=' + self.git_dir,
            '--work-tree=' + self.git_tree,
            'for-each-ref',
            '--sort=-committerdate',
            '--format=%(subject)\a%(refname)\a%(objectname)',
            'refs/heads/'
        ]
        return GitGutterHandler.run_command(args)

    def git_tags(self):
        args = [
            settings.git_binary_path,
            '--git-dir=' + self.git_dir,
            '--work-tree=' + self.git_tree,
            'show-ref',
            '--tags',
            '--abbrev=7'
        ]
        return GitGutterHandler.run_command(args)

    def git_current_branch(self):
        args = [
            settings.git_binary_path,
            '--git-dir=' + self.git_dir,
            '--work-tree=' + self.git_tree,
            'rev-parse',
            '--abbrev-ref',
            'HEAD'
        ]
        return GitGutterHandler.run_command(args)

    @staticmethod
    def run_command(args):
        def read_output(resolve):
            startupinfo = None
            if os.name == 'nt':
                startupinfo = subprocess.STARTUPINFO()
                startupinfo.dwFlags |= subprocess.STARTF_USESHOWWINDOW
            proc = subprocess.Popen(
                args, stdout=subprocess.PIPE, startupinfo=startupinfo,
                stderr=subprocess.PIPE)
            stdout_output = proc.stdout.read()
            resolve(stdout_output)

        def run_async(resolve):
            if hasattr(sublime, 'set_timeout_async'):
                sublime.set_timeout_async(lambda: read_output(resolve), 0)
            else:
                read_output(resolve)

        return Promise(run_async)
<commit_msg>Avoid exception by os.path.exist in on_disk() with new files with no filename.<commit_after>import os
import subprocess
import re
import codecs
import tempfile
import time
from functools import partial

import sublime

try:
    from . import git_helper
    from .git_gutter_settings import settings
    from .promise import Promise
except (ImportError, ValueError):
    import git_helper
    from git_gutter_settings import settings
    from promise import Promise

class GitGutterHandler(object):
    # the git repo won't change that often so we can easily wait few seconds
    # between updates for performance
    git_file_update_interval_secs = 5

    def __init__(self, view):
        self.view = view

        self.git_temp_file = None
        self.buf_temp_file = None

        self.git_tree = None
        self.git_dir = None
        self.git_path = None

        self._last_refresh_time_git_file = 0

    def __del__(self):
        if self.git_temp_file:
            os.unlink(self.git_temp_file)
        if self.buf_temp_file:
            os.unlink(self.buf_temp_file)

    @staticmethod
    def tmp_file():
        """Create a temp file and return the filepath to it.

        CAUTION: Caller is responsible for clean up
        """
        fd, filepath = tempfile.mkstemp(prefix='git_gutter_')
        os.close(fd)
        return filepath

    def clear_git_time(self):
        self._last_refresh_time_git_file = 0

    def update_git_time(self):
        self._last_refresh_time_git_file = time.time()

    def git_time(self):
        return time.time() - self._last_refresh_time_git_file

    def _get_view_encoding(self):
        # get encoding and clean it for python ex: "Western (ISO 8859-1)"
        # NOTE(maelnor): are we need regex here?
        pattern = re.compile(r'.+\((.*)\)')
        encoding = self.view.encoding()
        if encoding == "Undefined":
            encoding = self.view.settings().get('default_encoding')
        if pattern.match(encoding):
            encoding = pattern.sub(r'\1', encoding)

        encoding = encoding.replace('with BOM', '')
        encoding = encoding.replace('Windows', 'cp')
        encoding = encoding.replace('-', '_')
        encoding = encoding.replace(' ', '')

        # work around with ConvertToUTF8 plugin
        origin_encoding = self.view.settings().get('origin_encoding')
        return origin_encoding or encoding

    def on_disk(self):
        # if the view is saved to disk
        fn = self.view.file_name()
        on_disk = (fn is not None) and os.path.exists(fn)
        if on_disk:
            self.git_tree = self.git_tree or git_helper.git_tree(self.view)
            self.git_dir = self.git_dir or git_helper.git_dir(self.git_tree)
            self.git_path = self.git_path or git_helper.git_file_path(
                self.view, self.git_tree)
        return on_disk

    def update_buf_file(self):
        chars = self.view.size()
        region = sublime.Region(0, chars)

        # Try conversion
        try:
            contents = self.view.substr(
                region).encode(self._get_view_encoding())
        except UnicodeError:
            # Fallback to utf8-encoding
            contents = self.view.substr(region).encode('utf-8')
        except LookupError:
            # May encounter an encoding we don't have a codec for
            contents = self.view.substr(region).encode('utf-8')

        contents = contents.replace(b'\r\n', b'\n')
        contents = contents.replace(b'\r', b'\n')

        if not self.buf_temp_file:
            self.buf_temp_file = GitGutterHandler.tmp_file()

        with open(self.buf_temp_file, 'wb') as f:
            if self.view.encoding() == "UTF-8 with BOM":
                f.write(codecs.BOM_UTF8)

            f.write(contents)

    def update_git_file(self):

        def write_file(contents):
            contents = contents.replace(b'\r\n', b'\n')
            contents = contents.replace(b'\r', b'\n')
            with open(self.git_temp_file, 'wb') as f:
                f.write(contents)

        if self.git_time() > self.git_file_update_interval_secs:
            if not self.git_temp_file:
                self.git_temp_file = GitGutterHandler.tmp_file()

            args = [
                settings.git_binary_path,
                '--git-dir=' + self.git_dir,
                '--work-tree=' + self.git_tree,
                'show',
                '%s:%s' % (
                    settings.get_compare_against(self.view), self.git_path),
            ]

            try:
                self.update_git_time()
                return GitGutterHandler.run_command(args).then(write_file)
            except Exception:
                pass
        return Promise.resolve()

    # Parse unified diff with 0 lines of context.
    # Hunk range info format:
    #   @@ -3,2 +4,0 @@
    #     Hunk originally starting at line 3, and occupying 2 lines, now
    #     starts at line 4, and occupies 0 lines, i.e. it was deleted.
    #   @@ -9 +10,2 @@
    #     Hunk size can be omitted, and defaults to one line.
    # Dealing with ambiguous hunks:
    #   "A\nB\n" -> "C\n"
    #   Was 'A' modified, and 'B' deleted? Or 'B' modified, 'A' deleted?
    #   Or both deleted? To minimize confusion, let's simply mark the
    #   hunk as modified.
    def process_diff(self, diff_str):
        inserted = []
        modified = []
        deleted = []
        hunk_re = '^@@ \-(\d+),?(\d*) \+(\d+),?(\d*) @@'
        hunks = re.finditer(hunk_re, diff_str, re.MULTILINE)
        for hunk in hunks:
            start = int(hunk.group(3))
            old_size = int(hunk.group(2) or 1)
            new_size = int(hunk.group(4) or 1)
            if not old_size:
                inserted += range(start, start + new_size)
            elif not new_size:
                deleted += [start + 1]
            else:
                modified += range(start, start + new_size)
        return (inserted, modified, deleted)

    def diff_str(self):
        def decode_diff(results):
            encoding = self._get_view_encoding()
            try:
                decoded_results = results.decode(encoding.replace(' ', ''))
            except UnicodeError:
                try:
                    decoded_results = results.decode("utf-8")
                except UnicodeDecodeError:
                    decoded_results = ""
            except LookupError:
                try:
                    decoded_results = codecs.decode(results)
                except UnicodeDecodeError:
                    decoded_results = ""
            return decoded_results

        def run_diff(_unused):
            self.update_buf_file()
            args = [
                settings.git_binary_path,
                'diff', '-U0', '--no-color', '--no-index',
                settings.ignore_whitespace,
                settings.patience_switch,
                self.git_temp_file,
                self.buf_temp_file,
            ]
            args = list(filter(None, args))  # Remove empty args
            return GitGutterHandler.run_command(args).then(decode_diff)

        if self.on_disk() and self.git_path:
            return self.update_git_file().then(run_diff)
        else:
            return Promise.resolve("")

    def process_diff_line_change(self, line_nr, diff_str):
        hunk_re = '^@@ \-(\d+),?(\d*) \+(\d+),?(\d*) @@'
        hunks = re.finditer(hunk_re, diff_str, re.MULTILINE)

        # we also want to extract the position of the surrounding changes
        first_change = prev_change = next_change = None

        for hunk in hunks:
            start = int(hunk.group(3))
            size = int(hunk.group(4) or 1)
            if first_change is None:
                first_change = start
            # special handling to also match the line below deleted
            # content
            if size == 0 and line_nr == start + 1:
                pass
            # continue if the hunk is before the line
            elif start + size < line_nr:
                prev_change = start
                continue
            # break if the hunk is after the line
            elif line_nr < start:
                break
            # in the following the line is inside the hunk
            try:
                next_hunk = next(hunks)
                hunk_end = next_hunk.start()
                next_change = int(next_hunk.group(3))
            except:
                hunk_end = len(diff_str)

            # if wrap is disable avoid wrapping
            wrap = settings.get('next_prev_change_wrap', True)
            if not wrap:
                if prev_change is None:
                    prev_change = start
                if next_change is None:
                    next_change = start

            # if prev change is None set it to the wrap around the
            # document: prev -> last hunk, next -> first hunk
            if prev_change is None:
                try:
                    remaining_hunks = list(hunks)
                    if remaining_hunks:
                        last_hunk = remaining_hunks[-1]
                        prev_change = int(last_hunk.group(3))
                    elif next_change is not None:
                        prev_change = next_change
                    else:
                        prev_change = start
                except:
                    prev_change = start
            if next_change is None:
                next_change = first_change

            # extract the content of the hunk
            hunk_content = diff_str[hunk.start():hunk_end]
            # store all deleted lines (starting with -)
            hunk_lines = hunk_content.splitlines()[1:]
            deleted_lines = [
                line[1:] for line in hunk_lines if line.startswith("-")
            ]
            added_lines = [line[1:] for line in hunk_lines
                           if line.startswith("+")]
            meta = {
                "added_lines": added_lines,
                "first_change": first_change,
                "next_change": next_change,
                "prev_change": prev_change
            }
            return (deleted_lines, start, size, meta)
        return ([], -1, -1, {})

    def diff_line_change(self, line):
        return self.diff_str().then(
            partial(self.process_diff_line_change, line))

    def diff(self):
        return self.diff_str().then(self.process_diff)

    def untracked(self):
        return self.handle_files([])

    def ignored(self):
        return self.handle_files(['-i'])

    def handle_files(self, additional_args):
        if self.on_disk() and self.git_path:
            def is_nonempty(results):
                encoding = self._get_view_encoding()
                try:
                    decoded_results = results.decode(encoding.replace(' ', ''))
                except UnicodeError:
                    decoded_results = results.decode("utf-8")
                return (decoded_results != "")

            args = [
                settings.git_binary_path,
                '--git-dir=' + self.git_dir,
                '--work-tree=' + self.git_tree,
                'ls-files', '--other', '--exclude-standard',
            ] + additional_args + [
                os.path.join(self.git_tree, self.git_path),
            ]
            args = list(filter(None, args))  # Remove empty args
            return GitGutterHandler.run_command(args).then(is_nonempty)
        return Promise.resolve(False)

    def git_commits(self):
        args = [
            settings.git_binary_path,
            '--git-dir=' + self.git_dir,
            '--work-tree=' + self.git_tree,
            'log', '--all',
            '--pretty=%s\a%h %an <%aE>\a%ad (%ar)',
            '--date=local', '--max-count=9000'
        ]
        return GitGutterHandler.run_command(args)

    def git_branches(self):
        args = [
            settings.git_binary_path,
            '--git-dir=' + self.git_dir,
            '--work-tree=' + self.git_tree,
            'for-each-ref',
            '--sort=-committerdate',
            '--format=%(subject)\a%(refname)\a%(objectname)',
            'refs/heads/'
        ]
        return GitGutterHandler.run_command(args)

    def git_tags(self):
        args = [
            settings.git_binary_path,
            '--git-dir=' + self.git_dir,
            '--work-tree=' + self.git_tree,
            'show-ref',
            '--tags',
            '--abbrev=7'
        ]
        return GitGutterHandler.run_command(args)

    def git_current_branch(self):
        args = [
            settings.git_binary_path,
            '--git-dir=' + self.git_dir,
            '--work-tree=' + self.git_tree,
            'rev-parse',
            '--abbrev-ref',
            'HEAD'
        ]
        return GitGutterHandler.run_command(args)

    @staticmethod
    def run_command(args):
        def read_output(resolve):
            startupinfo = None
            if os.name == 'nt':
                startupinfo = subprocess.STARTUPINFO()
                startupinfo.dwFlags |= subprocess.STARTF_USESHOWWINDOW
            proc = subprocess.Popen(
                args, stdout=subprocess.PIPE, startupinfo=startupinfo,
                stderr=subprocess.PIPE)
            stdout_output = proc.stdout.read()
            resolve(stdout_output)

        def run_async(resolve):
            if hasattr(sublime, 'set_timeout_async'):
                sublime.set_timeout_async(lambda: read_output(resolve), 0)
            else:
                read_output(resolve)

        return Promise(run_async)
<|endoftext|>


<commit_before>import os
import sys
import zmq
from zmq import ssh
import pickle
import socket
import time

from utils import Message
from utils import RetMessage, bcolors
#===================================================================================================
#++++++Class: SchedulerClient+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
#===================================================================================================

class SchedulerClient(object):

    def __init__(self, url,tcp_port, local=True):
        self.context = zmq.Context()
        self.socket = self.context.socket(zmq.REQ)
        if(local):
            self.socket.connect("tcp://127.0.0.1:%s"%tcp_port)
        else:
            #try:
                #server_name = url.split("@")[0]+"@"+canIHasIP(url.split("@")[1],3)
            #except e:
                #raise(e)
                #return
            #server_name = url.split("@")[0]+"@"+socket.gethostbyname(url.split("@")[1])
            server_name = DNSResolve(url)
            print(server_name)
            try:
                #self.socket.connect("tcp://%s:%s"%(server_name,tcp_port))
                self.tunnel = ssh.tunnel_connection(self.socket,"tcp://127.0.0.1:%s"%tcp_port, server_name, timeout=3)
            except e:
                raise(e)
                return
        self.socket.setsockopt(zmq.LINGER, 0)

        self.poller = zmq.Poller()
        self.poller.register(self.socket, zmq.POLLIN)
#___________________________________________________________________________________________________
    def send_msg(self, msg):
        try:
            self.socket.send(msg, zmq.NOBLOCK)
        except:
            print(bcolors.BOLD+bcolors.FAIL+"Failed to send message"+bcolors.ENDC)
        return_msg = ""

        try:
            while(True):
                socks = dict(self.poller.poll(1000))
                if socks:

                    if socks.get(self.socket) == zmq.POLLIN:
                        constructed_message = False
                        return_msg = ''
                        while(not constructed_message):
                            return_msg += self.socket.recv(zmq.NOBLOCK)
                            try:
                                pickle.loads(retmsg)
                                constructed_message = True
                            except:
                                constructed_message = False
                        break
                else:
                    print(bcolors.BOLD+bcolors.FAIL+"Timed out"+bcolors.ENDC)
                    break
        except:
            print(bcolors.BOLD+bcolors.FAIL+"Error while recieveing message"+bcolors.ENDC)

        return return_msg
#___________________________________________________________________________________________________
    def submit_simple_jobs(self, cmd_list, outlog_path  = None, errlog_path = None, user = "Unknown", env = None, wdir = None, shell = True):
        if(outlog_path == None or errlog_path == None):
            msg = Message('SUBMIT_JOBS', 'SIMPLE', user)
        else:
            msg = Message( 'SUBMIT_JOBS', 'SIMPLE_LOG', user)
            msg.msg["outlog_path"] = outlog_path
            msg.msg["errlog_path"] = errlog_path

        msg.msg["cmd_list"] = cmd_list
        msg.msg["env"] = env
        msg.msg["wdir"] = wdir
        msg.msg["shell"] = shell

        return self.send_msg(msg.compose())

#___________________________________________________________________________________________________
    def submit_job_description(self, exe, args , outlog  = None, errlog = None, user = "Unknown", env = None, shell = False):
        msg = Message( 'SUBMIT_JOBS', 'JOB_DESCRIPTION', user)
        msg.msg["executable"] = exe
        msg.msg["args"] = args
        msg.msg["env"] = env
        msg.msg["shell"] = shell
        msg.msg["outlog"] = outlog
        msg.msg["errlog"] = errlog

        return self.send_msg(msg.compose())
#___________________________________________________________________________________________________
    def classical_submit(self, executable, var, out, err, user, queue, init_dir):
        print("Not implemented yet")
#___________________________________________________________________________________________________
    def stop_scheduler(self, opt = "NOW", user = ""):
        msg = Message('STOP', opt, user)
        return self.send_msg(msg.compose())
#___________________________________________________________________________________________________
    def change_nproc_limit(self, nproc, user = ""):
        msg = Message('CONFIGURE', 'NPROC', user)
        msg.msg["n-proc"] = nproc
        return self.send_msg(msg.compose())

#___________________________________________________________________________________________________
    def get_avg_load(self, user = ""):
        msg = Message('AVG_LOAD', '', user)
        return self.send_msg(msg.compose())
#___________________________________________________________________________________________________
    def ping(self, user = ""):
        msg = Message('PING', '', user)
        start_time = time.time()
        rec = self.send_msg(msg.compose()).split()
        finish_time = time.time()
        return (rec[1],rec[2],finish_time-start_time)
#___________________________________________________________________________________________________
    def remove_jobs(self, ids = None, user = "Unknown"):

        if(ids == None):
            opt = 'ALL'
        else:
            opt = 'LIST'
        msg = Message('REMOVE_JOBS', opt, user)
        msg.msg["job_ids"] = ids
        return self.send_msg(msg.compose())
#___________________________________________________________________________________________________
    def request_job(self, id = None, user = "Unknown"):
        msg = Message('REQUEST_JOBS', id, user)

        retmsg = self.send_msg(msg.compose())
        #print(retmsg)
        retmsg = pickle.loads(retmsg)#RetMessage()
        #retmsg.decompose(self.send_msg(msg.compose()))

        #print(retmsg.status)
        if('error' in retmsg.msg.keys()):
            print(bcolors.BOLD+bcolors.FAIL+retmsg.msg['error']+bcolors.ENDC)
            sys.exit(0)
        return retmsg.msg['jobs'], retmsg
#___________________________________________________________________________________________________
import signal, socket
try:
    import DNS
except:
    DNS = False

#___________________________________________________________________________________________________
def DNSResolve( s ):
    if DNS:
        DNS.ParseResolvConf() # Windows?
        r = DNS.DnsRequest(name=s,qtype='A')
        a = r.req()
        return a.answers[0]['data']
    else:
        return socket.gethostbyname( s )

#___________________________________________________________________________________________________
def dns_timeout(a,b):
    raise Exception("Oh Noes! a DNS lookup timeout!")

#___________________________________________________________________________________________________
def canIHasIP( domain_name, timeout=3 ):
    signal.signal(signal.SIGALRM, dns_timeout)
    signal.alarm( timeout )
    try:
        ip = DNSResolve( domain_name )
    except Exception, exc:
        print exc
        return False
    signal.alarm(0)
    return ip<commit_msg>Fixed a typo making the last commit useless.<commit_after>import os
import sys
import zmq
from zmq import ssh
import pickle
import socket
import time

from utils import Message
from utils import RetMessage, bcolors
#===================================================================================================
#++++++Class: SchedulerClient+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
#===================================================================================================

class SchedulerClient(object):

    def __init__(self, url,tcp_port, local=True):
        self.context = zmq.Context()
        self.socket = self.context.socket(zmq.REQ)
        if(local):
            self.socket.connect("tcp://127.0.0.1:%s"%tcp_port)
        else:
            #try:
                #server_name = url.split("@")[0]+"@"+canIHasIP(url.split("@")[1],3)
            #except e:
                #raise(e)
                #return
            #server_name = url.split("@")[0]+"@"+socket.gethostbyname(url.split("@")[1])
            server_name = DNSResolve(url)
            print(server_name)
            try:
                #self.socket.connect("tcp://%s:%s"%(server_name,tcp_port))
                self.tunnel = ssh.tunnel_connection(self.socket,"tcp://127.0.0.1:%s"%tcp_port, server_name, timeout=3)
            except e:
                raise(e)
                return
        self.socket.setsockopt(zmq.LINGER, 0)

        self.poller = zmq.Poller()
        self.poller.register(self.socket, zmq.POLLIN)
#___________________________________________________________________________________________________
    def send_msg(self, msg):
        try:
            self.socket.send(msg, zmq.NOBLOCK)
        except:
            print(bcolors.BOLD+bcolors.FAIL+"Failed to send message"+bcolors.ENDC)
        return_msg = ""

        try:
            while(True):
                socks = dict(self.poller.poll(1000))
                if socks:

                    if socks.get(self.socket) == zmq.POLLIN:
                        constructed_message = False
                        return_msg = ''
                        while(not constructed_message):
                            return_msg += self.socket.recv(zmq.NOBLOCK)
                            try:
                                pickle.loads(return_msg)
                                constructed_message = True
                            except Exception as e:
                                constructed_message = False
                        break
                else:
                    print(bcolors.BOLD+bcolors.FAIL+"Timed out"+bcolors.ENDC)
                    break
        except:
            print(bcolors.BOLD+bcolors.FAIL+"Error while recieveing message"+bcolors.ENDC)

        return return_msg
#___________________________________________________________________________________________________
    def submit_simple_jobs(self, cmd_list, outlog_path  = None, errlog_path = None, user = "Unknown", env = None, wdir = None, shell = True):
        if(outlog_path == None or errlog_path == None):
            msg = Message('SUBMIT_JOBS', 'SIMPLE', user)
        else:
            msg = Message( 'SUBMIT_JOBS', 'SIMPLE_LOG', user)
            msg.msg["outlog_path"] = outlog_path
            msg.msg["errlog_path"] = errlog_path

        msg.msg["cmd_list"] = cmd_list
        msg.msg["env"] = env
        msg.msg["wdir"] = wdir
        msg.msg["shell"] = shell

        return self.send_msg(msg.compose())

#___________________________________________________________________________________________________
    def submit_job_description(self, exe, args , outlog  = None, errlog = None, user = "Unknown", env = None, shell = False):
        msg = Message( 'SUBMIT_JOBS', 'JOB_DESCRIPTION', user)
        msg.msg["executable"] = exe
        msg.msg["args"] = args
        msg.msg["env"] = env
        msg.msg["shell"] = shell
        msg.msg["outlog"] = outlog
        msg.msg["errlog"] = errlog

        return self.send_msg(msg.compose())
#___________________________________________________________________________________________________
    def classical_submit(self, executable, var, out, err, user, queue, init_dir):
        print("Not implemented yet")
#___________________________________________________________________________________________________
    def stop_scheduler(self, opt = "NOW", user = ""):
        msg = Message('STOP', opt, user)
        return self.send_msg(msg.compose())
#___________________________________________________________________________________________________
    def change_nproc_limit(self, nproc, user = ""):
        msg = Message('CONFIGURE', 'NPROC', user)
        msg.msg["n-proc"] = nproc
        return self.send_msg(msg.compose())

#___________________________________________________________________________________________________
    def get_avg_load(self, user = ""):
        msg = Message('AVG_LOAD', '', user)
        return self.send_msg(msg.compose())
#___________________________________________________________________________________________________
    def ping(self, user = ""):
        msg = Message('PING', '', user)
        start_time = time.time()
        rec = self.send_msg(msg.compose()).split()
        finish_time = time.time()
        return (rec[1],rec[2],finish_time-start_time)
#___________________________________________________________________________________________________
    def remove_jobs(self, ids = None, user = "Unknown"):

        if(ids == None):
            opt = 'ALL'
        else:
            opt = 'LIST'
        msg = Message('REMOVE_JOBS', opt, user)
        msg.msg["job_ids"] = ids
        return self.send_msg(msg.compose())
#___________________________________________________________________________________________________
    def request_job(self, id = None, user = "Unknown"):
        msg = Message('REQUEST_JOBS', id, user)

        retmsg = self.send_msg(msg.compose())
        #print(retmsg)
        retmsg = pickle.loads(retmsg)#RetMessage()
        #retmsg.decompose(self.send_msg(msg.compose()))

        #print(retmsg.status)
        if('error' in retmsg.msg.keys()):
            print(bcolors.BOLD+bcolors.FAIL+retmsg.msg['error']+bcolors.ENDC)
            sys.exit(0)
        return retmsg.msg['jobs'], retmsg
#___________________________________________________________________________________________________
import signal, socket
try:
    import DNS
except:
    DNS = False

#___________________________________________________________________________________________________
def DNSResolve( s ):
    if DNS:
        DNS.ParseResolvConf() # Windows?
        r = DNS.DnsRequest(name=s,qtype='A')
        a = r.req()
        return a.answers[0]['data']
    else:
        return socket.gethostbyname( s )

#___________________________________________________________________________________________________
def dns_timeout(a,b):
    raise Exception("Oh Noes! a DNS lookup timeout!")

#___________________________________________________________________________________________________
def canIHasIP( domain_name, timeout=3 ):
    signal.signal(signal.SIGALRM, dns_timeout)
    signal.alarm( timeout )
    try:
        ip = DNSResolve( domain_name )
    except Exception, exc:
        print exc
        return False
    signal.alarm(0)
    return ip<|endoftext|>


<commit_before>#! /usr/bin/env python

# borrowed from elsewhere.

import sys
from osgeo import osr

def esriprj2standards(shapeprj_path):
   prj_file = open(shapeprj_path, 'r')
   prj_txt = prj_file.read()
   srs = osr.SpatialReference()
   srs.ImportFromESRI([prj_txt])
   print 'Shape prj is: %s' % prj_txt
   print 'WKT is: %s' % srs.ExportToWkt()
   print 'Proj4 is: %s' % srs.ExportToProj4()
   srs.AutoIdentifyEPSG()
   print 'EPSG is: %s' % srs.GetAuthorityCode(None)

esriprj2standards(sys.argv[1])

<commit_msg>Added attribution and reference<commit_after>#! /usr/bin/env python

# I didn't write this. It was borrowed from elsewhere.
# http://gis.stackexchange.com/questions/55196/how-can-i-get-the-proj4-string-or-epsg-code-from-a-shapefile-prj-file
# Accessed on Feb 21 2015

import sys
from osgeo import osr

def esriprj2standards(shapeprj_path):
   prj_file = open(shapeprj_path, 'r')
   prj_txt = prj_file.read()
   srs = osr.SpatialReference()
   srs.ImportFromESRI([prj_txt])
   print 'Shape prj is: %s' % prj_txt
   print 'WKT is: %s' % srs.ExportToWkt()
   print 'Proj4 is: %s' % srs.ExportToProj4()
   srs.AutoIdentifyEPSG()
   print 'EPSG is: %s' % srs.GetAuthorityCode(None)

esriprj2standards(sys.argv[1])

<|endoftext|>


<commit_before>"""Import data."""
import h5py
import os
import mne
import numpy as np
import pandas as pd

from nipype.utils.filemanip import split_filename as split_f
from scipy.io import loadmat
from mne.io import read_raw_ctf

# -------------------- nodes (Function)
def _convert_ds_to_raw_fif(ds_file):
    """CTF .ds to .fif and save result in pipeline folder structure."""

    _, basename, ext = split_f(ds_file)
    raw = read_raw_ctf(ds_file)

    raw_fif_file = os.path.abspath(basename + "_raw.fif")

    if not os.path.isfile(raw_fif_file):
        raw = read_raw_ctf(ds_file)
        raw.save(raw_fif_file)
    else:
        print(('*** RAW FIF file %s exists!!!' % raw_fif_file))

    return raw_fif_file

def write_hdf5(filename, data, dataset_name='dataset', dtype='f'):
    """
    Create hdf5 file

    Inputs
        filename : str
            hdf5 filename
        data : array, shape (n_vertices, n_times)
            raw data for whose the dataset is created
        dataset_name : str
            name of dataset to create
        dtype : str
            data type for new dataset

    """

    hf = h5py.File(filename, 'w')
    hf.create_dataset(dataset_name, data=data, dtype=dtype)
    hf.close()

def read_hdf5(filename, dataset_name='dataset'):
    """
    Read hdf5 file

    Inputs
        filename : str
            hdf5 filename
        dataset_name : str
            name of dataset to create

    Outputs
        data : array, shape (n_vertices, n_times)
            raw data for whose the dataset is created
    """

    hf = h5py.File(filename, 'r')
    data = hf[dataset_name][()]

    return data

def npy2hdf5(filename, dataset_name='dataset', dtype='f'):

    data = np.load(filename)
    print('converting to hdf5')

    write_hdf5(filename, data, dataset_name=dataset_name, dtype=dtype)

def import_hdf5_file(hdf5_file, data_field_name='F'):

    subj_path, basename, ext = split_f(hdf5_file)

    hf = h5py.File(hdf5_file, 'r')
    raw_data = hf[data_field_name][()]

    print(raw_data)
    print((raw_data.shape))
    print((raw_data.__class__))

    ts_file = os.path.abspath(basename + '.npy')
    np.save(ts_file, raw_data)

    return ts_file

def import_mat_to_conmat(mat_file, data_field_name='F',
                         orig_channel_names_file=None,
                         orig_channel_coords_file=None):
    """Import mat to conmat."""

    subj_path, basename, ext = split_f(mat_file)

    mat = loadmat(mat_file)
    print((mat[data_field_name].shape))

    raw_data = np.array(mat[data_field_name], dtype='f')
    print(raw_data)
    print((raw_data.shape))

    ts_file = os.path.abspath(basename + '.npy')
    np.save(ts_file, raw_data)

    if orig_channel_names_file is not None:
        correct_channel_coords = np.loadtxt(orig_channel_coords_file)
        print(correct_channel_coords)

        # save channel coords
        channel_coords_file = os.path.abspath('correct_channel_coords.txt')
        np.savetxt(channel_coords_file, correct_channel_coords, fmt='%s')

    if orig_channel_coords_file is not None:
        correct_channel_names = np.loadtxt(orig_channel_coords_file)
        print(correct_channel_names)

        # save channel names
        channel_names_file = os.path.abspath('correct_channel_names.txt')
        np.savetxt(channel_names_file, correct_channel_names, fmt='%s')

    if all([k is not None for k in (orig_channel_names_file,
                                    orig_channel_coords_file)]):
        return ts_file, channel_coords_file, channel_names_file
    else:
        return ts_file

def import_tsmat_to_ts(tsmat_file, data_field_name='F',
                       good_channels_field_name='ChannelFlag'):
    """Import tsmat to ts."""

    print(tsmat_file)

    subj_path, basename, ext = split_f(tsmat_file)

    mat = loadmat(tsmat_file)

    raw_data = np.array(mat[data_field_name], dtype="f")
    print((raw_data.shape))

    print([key for key in list(mat.keys())])

    if good_channels_field_name is not None:
        print("Using good channels to sort channels")
        good_channels = np.array(mat[good_channels_field_name])
        print((good_channels.shape))

        good_channels = good_channels.reshape(good_channels.shape[0])
        print((good_channels.shape))

        good_data = raw_data[good_channels == 1, :]

        print((good_data.shape))

    else:
        print("No channel sorting")
        good_data = raw_data

    # save data
    print((good_data.shape))

    ts_file = os.path.abspath("tsmat.npy")
    np.save(ts_file, good_data)
    return ts_file

def import_amplmat_to_ts(tsmat_file):
    """Import amplmat to ts."""

    print(tsmat_file)

    subj_path, basename, ext = split_f(tsmat_file)

    mat = loadmat(tsmat_file)

    # field_name = basename.split('_')[0]
    # field_name = basename.split('_')[1]
    # print field_name

    raw_data = np.array(mat['F'], dtype="f")
    print((raw_data.shape))

    good_channels = np.array(mat['ChannelFlag'])

    good_channels = good_channels.reshape(good_channels.shape[0])

    print("Good channels:")
    print((good_channels.shape))

    good_data = raw_data[good_channels == 1, :]

    print((good_data.shape))

    # save data
    ts_file = os.path.abspath("amplmat.npy")

    np.save(ts_file, good_data)
    # np.save(ts_file,raw_data)

    return ts_file

def import_mat_to_ts(mat_file, orig_channel_names_file,
                     orig_channel_coords_file):
    """Import mat to ts."""

    subj_path, basename, ext = split_f(mat_file)

    mat = loadmat(mat_file)

    field_name = basename.split('_')[0]
    print('************************************ \n')
    print(field_name)
    print((mat[field_name].shape))

    raw_data = np.array(mat[field_name], dtype="f")
    print((raw_data.shape))

    # load electrode names
    elec_names = [line.strip() for line in open(orig_channel_names_file)]

    print(elec_names)

    cond = [elec.startswith('EMG') or elec.startswith(
        'EOG') for elec in elec_names]
    select_sensors, = np.where(np.array(cond, dtype='bool') is False)
    print(select_sensors)

    # save electrode locations
    elec_loc = np.loadtxt(orig_channel_coords_file)
    print(elec_loc)

    correct_elec_loc = np.roll(elec_loc[select_sensors, :], shift=2, axis=1)

    print((correct_elec_loc[0, :]))
    print((correct_elec_loc[7, :]))

    channel_coords_file = os.path.abspath("correct_channel_coords.txt")
    np.savetxt(channel_coords_file, correct_elec_loc, fmt='%s')

    # save electrode names
    correct_elec_names = np.array([elec_names[pos]
                                   for pos in select_sensors], dtype="str")

    channel_names_file = os.path.abspath("correct_channel_names.txt")
    np.savetxt(channel_names_file, correct_elec_names, fmt='%s')

    # save data (reorganise dimensions)
    new_data = raw_data[select_sensors, :].swapaxes(0, 2).swapaxes(1, 2)
    print((new_data.shape))

    ts_file = os.path.abspath(basename + ".npy")

    np.save(ts_file, new_data)

    return ts_file, channel_coords_file, channel_names_file

def concat_ts(all_ts_files):
    """Concat ts."""

    print(all_ts_files)

    for i, ts_file in enumerate(all_ts_files):

        # loading ROI coordinates
        ts = np.load(ts_file)

        # print "all_ts: "
        print((ts.shape))

        if i == 0:
            concat_ts = ts.copy()
        else:
            concat_ts = np.concatenate((concat_ts, ts), axis=0)

    print((concat_ts.shape))

    # saving time series
    concat_ts_file = os.path.abspath("concat_ts.npy")
    np.save(concat_ts_file, concat_ts)

    return concat_ts_file

def split_txt(sample_size, txt_file, sep_label_name, repair=True, sep=";",
              keep_electrodes=""):
    """Split txt."""

    if repair is True:
        df_data = []
        elec_names = []

        with open(txt_file) as f:
            lines = f.readlines()
            for line in lines:
                # print line
                if line.startswith('"') and line.strip().endswith('"'):
                    line = line.strip()[1:-1]
                print(line)

                splitted_line = line.strip().split(sep, 1)
                name = splitted_line[0]
                print(name)

                elec_names.append(name)
                data = splitted_line[1]

                new_data = data.replace(" ", sep)
                df_data.append([float(d.replace(",", "."))
                                for d in new_data.split(sep)])

        print(df_data)
        print((np.array(df_data).shape))

        df = pd.DataFrame(np.array(df_data), index=elec_names)
    else:
        df = pd.read_table(txt_file, sep=sep, decimal=",",
                           header=None, index_col=0)

    # electrode names:
    np_indexes = df.index.values
    list_indexes = np_indexes.tolist()
    print(list_indexes)

    if keep_electrodes != "":
        list_keep_electrodes = keep_electrodes.split("-")
    else:
        list_keep_electrodes = []

    print(list_keep_electrodes)

    if sep_label_name != "":
        if len(list_keep_electrodes) == 0:

            keep = [len(index.split(
                        sep_label_name)) == 2 for index in list_indexes]
        else:

            keep = [len(index.split(sep_label_name)) == 2 and index in
                    list_keep_electrodes for index in list_indexes]
    else:
        if len(list_keep_electrodes) == 0:
            keep = np.ones(shape=np_indexes.shape)
        else:
            keep = [index in list_keep_electrodes for index in list_indexes]

    keep = keep.astype(int)
    print(keep)
    print((np_indexes[keep == 1]))

    elec_names_file = os.path.abspath("correct_channel_names.txt")
    np.savetxt(elec_names_file, np_indexes[keep == 1], fmt="%s")

    # splitting data_path
    print((df.shape))
    if df.shape[1] % sample_size != 0:
        print("Error, sample_size is not a multiple of ts shape")
        print(sample_size)
        print(df)
        return

    nb_epochs = df.shape[1] / sample_size
    print(nb_epochs)
    splitted_ts = np.split(df.values[keep == 1, :], nb_epochs, axis=1)
    print((len(splitted_ts)))
    print((splitted_ts[0]))

    np_splitted_ts = np.array(splitted_ts, dtype='float')
    print((np_splitted_ts.shape))

    splitted_ts_file = os.path.abspath("splitted_ts.npy")
    np.save(splitted_ts_file, np_splitted_ts)

    return splitted_ts_file, elec_names_file

def _read_brainvision_vhdr(vhdr_file, sample_size):
    """Read brainvision vhdr files."""
    data_raw = mne.io.read_raw_brainvision(vhdr_file, verbose=True)
    print((data_raw.ch_names))

    data, times = data_raw[:]
    print((data.shape))

    nb_epochs = data.shape[1] / sample_size
    print(nb_epochs)

    splitted_ts = np.split(data, nb_epochs, axis=1)
    print((len(splitted_ts)))
    print((splitted_ts[0]))

    np_splitted_ts = np.array(splitted_ts, dtype='float')
    print((np_splitted_ts.shape))

    return np_splitted_ts, data_raw.ch_names
<commit_msg>add private function<commit_after>"""Import data."""
import h5py
import os
import mne
import numpy as np
import pandas as pd

from nipype.utils.filemanip import split_filename as split_f
from scipy.io import loadmat
from mne.io import read_raw_ctf

# -------------------- nodes (Function)
def _convert_ds_to_raw_fif(ds_file):
    """CTF .ds to .fif and save result in pipeline folder structure."""

    _, basename, ext = split_f(ds_file)
    raw = read_raw_ctf(ds_file)

    raw_fif_file = os.path.abspath(basename + "_raw.fif")

    if not os.path.isfile(raw_fif_file):
        raw = read_raw_ctf(ds_file)
        raw.save(raw_fif_file)
    else:
        print(('*** RAW FIF file %s exists!!!' % raw_fif_file))

    return raw_fif_file

def _write_hdf5(filename, data, dataset_name='dataset', dtype='f'):
    """
    Create hdf5 file

    Inputs
        filename : str
            hdf5 filename
        data : array, shape (n_vertices, n_times)
            raw data for whose the dataset is created
        dataset_name : str
            name of dataset to create
        dtype : str
            data type for new dataset

    """

    hf = h5py.File(filename, 'w')
    hf.create_dataset(dataset_name, data=data, dtype=dtype)
    hf.close()

def _read_hdf5(filename, dataset_name='dataset'):
    """
    Read hdf5 file

    Inputs
        filename : str
            hdf5 filename
        dataset_name : str
            name of dataset to create

    Outputs
        data : array, shape (n_vertices, n_times)
            raw data for whose the dataset is created
    """

    hf = h5py.File(filename, 'r')
    data = hf[dataset_name][()]

    return data

def npy2hdf5(filename, dataset_name='dataset', dtype='f'):

    data = np.load(filename)
    print('converting to hdf5')

    _write_hdf5(filename, data, dataset_name=dataset_name, dtype=dtype)

def import_hdf5_file(hdf5_file, data_field_name='F'):

    subj_path, basename, ext = split_f(hdf5_file)

    hf = h5py.File(hdf5_file, 'r')
    raw_data = hf[data_field_name][()]

    print(raw_data)
    print((raw_data.shape))
    print((raw_data.__class__))

    ts_file = os.path.abspath(basename + '.npy')
    np.save(ts_file, raw_data)

    return ts_file

def import_mat_to_conmat(mat_file, data_field_name='F',
                         orig_channel_names_file=None,
                         orig_channel_coords_file=None):
    """Import mat to conmat."""

    subj_path, basename, ext = split_f(mat_file)

    mat = loadmat(mat_file)
    print((mat[data_field_name].shape))

    raw_data = np.array(mat[data_field_name], dtype='f')
    print(raw_data)
    print((raw_data.shape))

    ts_file = os.path.abspath(basename + '.npy')
    np.save(ts_file, raw_data)

    if orig_channel_names_file is not None:
        correct_channel_coords = np.loadtxt(orig_channel_coords_file)
        print(correct_channel_coords)

        # save channel coords
        channel_coords_file = os.path.abspath('correct_channel_coords.txt')
        np.savetxt(channel_coords_file, correct_channel_coords, fmt='%s')

    if orig_channel_coords_file is not None:
        correct_channel_names = np.loadtxt(orig_channel_coords_file)
        print(correct_channel_names)

        # save channel names
        channel_names_file = os.path.abspath('correct_channel_names.txt')
        np.savetxt(channel_names_file, correct_channel_names, fmt='%s')

    if all([k is not None for k in (orig_channel_names_file,
                                    orig_channel_coords_file)]):
        return ts_file, channel_coords_file, channel_names_file
    else:
        return ts_file

def import_tsmat_to_ts(tsmat_file, data_field_name='F',
                       good_channels_field_name='ChannelFlag'):
    """Import tsmat to ts."""

    print(tsmat_file)

    subj_path, basename, ext = split_f(tsmat_file)

    mat = loadmat(tsmat_file)

    raw_data = np.array(mat[data_field_name], dtype="f")
    print((raw_data.shape))

    print([key for key in list(mat.keys())])

    if good_channels_field_name is not None:
        print("Using good channels to sort channels")
        good_channels = np.array(mat[good_channels_field_name])
        print((good_channels.shape))

        good_channels = good_channels.reshape(good_channels.shape[0])
        print((good_channels.shape))

        good_data = raw_data[good_channels == 1, :]

        print((good_data.shape))

    else:
        print("No channel sorting")
        good_data = raw_data

    # save data
    print((good_data.shape))

    ts_file = os.path.abspath("tsmat.npy")
    np.save(ts_file, good_data)
    return ts_file

def import_amplmat_to_ts(tsmat_file):
    """Import amplmat to ts."""

    print(tsmat_file)

    subj_path, basename, ext = split_f(tsmat_file)

    mat = loadmat(tsmat_file)

    # field_name = basename.split('_')[0]
    # field_name = basename.split('_')[1]
    # print field_name

    raw_data = np.array(mat['F'], dtype="f")
    print((raw_data.shape))

    good_channels = np.array(mat['ChannelFlag'])

    good_channels = good_channels.reshape(good_channels.shape[0])

    print("Good channels:")
    print((good_channels.shape))

    good_data = raw_data[good_channels == 1, :]

    print((good_data.shape))

    # save data
    ts_file = os.path.abspath("amplmat.npy")

    np.save(ts_file, good_data)
    # np.save(ts_file,raw_data)

    return ts_file

def import_mat_to_ts(mat_file, orig_channel_names_file,
                     orig_channel_coords_file):
    """Import mat to ts."""

    subj_path, basename, ext = split_f(mat_file)

    mat = loadmat(mat_file)

    field_name = basename.split('_')[0]
    print('************************************ \n')
    print(field_name)
    print((mat[field_name].shape))

    raw_data = np.array(mat[field_name], dtype="f")
    print((raw_data.shape))

    # load electrode names
    elec_names = [line.strip() for line in open(orig_channel_names_file)]

    print(elec_names)

    cond = [elec.startswith('EMG') or elec.startswith(
        'EOG') for elec in elec_names]
    select_sensors, = np.where(np.array(cond, dtype='bool') is False)
    print(select_sensors)

    # save electrode locations
    elec_loc = np.loadtxt(orig_channel_coords_file)
    print(elec_loc)

    correct_elec_loc = np.roll(elec_loc[select_sensors, :], shift=2, axis=1)

    print((correct_elec_loc[0, :]))
    print((correct_elec_loc[7, :]))

    channel_coords_file = os.path.abspath("correct_channel_coords.txt")
    np.savetxt(channel_coords_file, correct_elec_loc, fmt='%s')

    # save electrode names
    correct_elec_names = np.array([elec_names[pos]
                                   for pos in select_sensors], dtype="str")

    channel_names_file = os.path.abspath("correct_channel_names.txt")
    np.savetxt(channel_names_file, correct_elec_names, fmt='%s')

    # save data (reorganise dimensions)
    new_data = raw_data[select_sensors, :].swapaxes(0, 2).swapaxes(1, 2)
    print((new_data.shape))

    ts_file = os.path.abspath(basename + ".npy")

    np.save(ts_file, new_data)

    return ts_file, channel_coords_file, channel_names_file

def concat_ts(all_ts_files):
    """Concat ts."""

    print(all_ts_files)

    for i, ts_file in enumerate(all_ts_files):

        # loading ROI coordinates
        ts = np.load(ts_file)

        # print "all_ts: "
        print((ts.shape))

        if i == 0:
            concat_ts = ts.copy()
        else:
            concat_ts = np.concatenate((concat_ts, ts), axis=0)

    print((concat_ts.shape))

    # saving time series
    concat_ts_file = os.path.abspath("concat_ts.npy")
    np.save(concat_ts_file, concat_ts)

    return concat_ts_file

def split_txt(sample_size, txt_file, sep_label_name, repair=True, sep=";",
              keep_electrodes=""):
    """Split txt."""

    if repair is True:
        df_data = []
        elec_names = []

        with open(txt_file) as f:
            lines = f.readlines()
            for line in lines:
                # print line
                if line.startswith('"') and line.strip().endswith('"'):
                    line = line.strip()[1:-1]
                print(line)

                splitted_line = line.strip().split(sep, 1)
                name = splitted_line[0]
                print(name)

                elec_names.append(name)
                data = splitted_line[1]

                new_data = data.replace(" ", sep)
                df_data.append([float(d.replace(",", "."))
                                for d in new_data.split(sep)])

        print(df_data)
        print((np.array(df_data).shape))

        df = pd.DataFrame(np.array(df_data), index=elec_names)
    else:
        df = pd.read_table(txt_file, sep=sep, decimal=",",
                           header=None, index_col=0)

    # electrode names:
    np_indexes = df.index.values
    list_indexes = np_indexes.tolist()
    print(list_indexes)

    if keep_electrodes != "":
        list_keep_electrodes = keep_electrodes.split("-")
    else:
        list_keep_electrodes = []

    print(list_keep_electrodes)

    if sep_label_name != "":
        if len(list_keep_electrodes) == 0:

            keep = [len(index.split(
                        sep_label_name)) == 2 for index in list_indexes]
        else:

            keep = [len(index.split(sep_label_name)) == 2 and index in
                    list_keep_electrodes for index in list_indexes]
    else:
        if len(list_keep_electrodes) == 0:
            keep = np.ones(shape=np_indexes.shape)
        else:
            keep = [index in list_keep_electrodes for index in list_indexes]

    keep = keep.astype(int)
    print(keep)
    print((np_indexes[keep == 1]))

    elec_names_file = os.path.abspath("correct_channel_names.txt")
    np.savetxt(elec_names_file, np_indexes[keep == 1], fmt="%s")

    # splitting data_path
    print((df.shape))
    if df.shape[1] % sample_size != 0:
        print("Error, sample_size is not a multiple of ts shape")
        print(sample_size)
        print(df)
        return

    nb_epochs = df.shape[1] / sample_size
    print(nb_epochs)
    splitted_ts = np.split(df.values[keep == 1, :], nb_epochs, axis=1)
    print((len(splitted_ts)))
    print((splitted_ts[0]))

    np_splitted_ts = np.array(splitted_ts, dtype='float')
    print((np_splitted_ts.shape))

    splitted_ts_file = os.path.abspath("splitted_ts.npy")
    np.save(splitted_ts_file, np_splitted_ts)

    return splitted_ts_file, elec_names_file

def _read_brainvision_vhdr(vhdr_file, sample_size):
    """Read brainvision vhdr files."""
    data_raw = mne.io.read_raw_brainvision(vhdr_file, verbose=True)
    print((data_raw.ch_names))

    data, times = data_raw[:]
    print((data.shape))

    nb_epochs = data.shape[1] / sample_size
    print(nb_epochs)

    splitted_ts = np.split(data, nb_epochs, axis=1)
    print((len(splitted_ts)))
    print((splitted_ts[0]))

    np_splitted_ts = np.array(splitted_ts, dtype='float')
    print((np_splitted_ts.shape))

    return np_splitted_ts, data_raw.ch_names
<|endoftext|>


<commit_before># Copyright 2020 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""This script is used to synthesize generated parts of this library."""

import synthtool as s
import synthtool.gcp as gcp
import subprocess
import logging

logging.basicConfig(level=logging.DEBUG)

# run the gapic generator
gapic = gcp.GAPICMicrogenerator()
versions = ['v1beta1']
name = 'servicedirectory'
for version in versions:
  library = gapic.typescript_library(
    name,
    generator_args={
      "grpc-service-config": f"google/cloud/{name}/{version}/{name}_grpc_service_config.json",
      'package-name': '@google-cloud/service-directory',
      "main-service": "ServiceDirectory"
    },
    extra_proto_files=['google/cloud/common_resources.proto'],
    proto_path=f'/google/cloud/{name}/{version}',
    version=version)
s.copy(library, excludes=['README.md', 'package.json'])

# Copy common templates
common_templates = gcp.CommonTemplates()
templates = common_templates.node_library(source_location='build/src')
s.copy(templates, excludes=[])

# Node.js specific cleanup
subprocess.run(['npm', 'install'])
subprocess.run(['npm', 'run', 'fix'])
subprocess.run(['npx', 'compileProtos', 'src'])

<commit_msg>build: set AUTOSYNTH_MULTIPLE_COMMITS=true for context aware commits (#20)<commit_after># Copyright 2020 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""This script is used to synthesize generated parts of this library."""

import synthtool as s
import synthtool.gcp as gcp
import subprocess
import logging

logging.basicConfig(level=logging.DEBUG)

AUTOSYNTH_MULTIPLE_COMMITS = True

# run the gapic generator
gapic = gcp.GAPICMicrogenerator()
versions = ['v1beta1']
name = 'servicedirectory'
for version in versions:
  library = gapic.typescript_library(
    name,
    generator_args={
      "grpc-service-config": f"google/cloud/{name}/{version}/{name}_grpc_service_config.json",
      'package-name': '@google-cloud/service-directory',
      "main-service": "ServiceDirectory"
    },
    extra_proto_files=['google/cloud/common_resources.proto'],
    proto_path=f'/google/cloud/{name}/{version}',
    version=version)
s.copy(library, excludes=['README.md', 'package.json'])

# Copy common templates
common_templates = gcp.CommonTemplates()
templates = common_templates.node_library(source_location='build/src')
s.copy(templates, excludes=[])

# Node.js specific cleanup
subprocess.run(['npm', 'install'])
subprocess.run(['npm', 'run', 'fix'])
subprocess.run(['npx', 'compileProtos', 'src'])

<|endoftext|>