<commit_before>########
# Copyright (c) 2015 GigaSpaces Technologies Ltd. All rights reserved
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#        http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
#    * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#    * See the License for the specific language governing permissions and
#    * limitations under the License.

# Built-in Imports
import requests
from requests import Request,Session,Response
import json
import constants
import sys
import auth
import utils
import os
from cloudify.exceptions import NonRecoverableError
from cloudify import ctx
from cloudify.decorators import operation

#virtualmachine:

@operation
def creation_validation(**_):
    for property_key in constants.VM_REQUIRED_PROPERTIES:
        _validate_node_properties(property_key, ctx.node.properties)

    vm_name_exists = _get_vm_name()
    if ctx.node.properties['use_external_resource'] and not vm_name_exists:
    raise NonRecoverableError(
    'External resource, but the supplied '
    'vm does not exist in the account.')
    if not ctx.node.properties['use_external_resource'] and vm_name_exists:
    raise NonRecoverableError(
    'Not external resource, but the supplied '
    'vm exists in the account.')

@operation
def create_vm(**_):

    RANDOM_SUFFIX_VALUE = utils.random_suffix_generator()
    vm_name = ctx.node.properties['vm_name']+RANDOM_SUFFIX_VALUE
    resource_group_name = ctx.instance.runtime_properties['resource_group']
    storage_account_name = storageaccount.storage_account_name
    location = ctx.node.properties['location']
    vnet_name = vnet.vnet_name
    nic_name = nic.nic_name
    public_ip_name= publicip.public_ip_name
    RANDOM_SUFFIX_VALUE = utils.random_suffix_generator()
    vm_name = contants.VM_PREFIX +RANDOM_SUFFIX_VALUE
    credentials='Bearer '+auth.get_token_from_client_credentials()
    subscription_id = ctx.node.properties['subscription_id']
    headers = {"Content-Type": "application/json", "Authorization": credentials}

    ctx.logger.info("Checking availability of virtual network: " + vm_name)
    if 1:
        try:
            ctx.logger.info("Creating new virtual machine: " + vm_name)
            virtual_machine_params=json.dumps(
            {
                "id":"/subscriptions/"+subscription_id+"/resourceGroups/"+resource_group_name+"/providers/Microsoft.Compute/virtualMachines/"+vm_name,
                "name":vm_name,
                "type":"Microsoft.Compute/virtualMachines",
                "location":location,
                "properties": {
                    "hardwareProfile": {
                        "vmSize": ctx.node.properties['vm_size']
                    },
                    "osProfile": {
                        "computername": vm_name,
                        "adminUsername": ctx.node.properties['ssh_username'],
                        "linuxConfiguration": {
                            "disablePasswordAuthentication": "true",
                            "ssh": {
                                "publicKeys": [
                                    {
                                        "path": "/home/"+ctx.node.properties['ssh_username']+"/.ssh/authorized_keys",
                                        "keyData": ctx.node.properties['key_data']}
                                ]
                            }
                        }
                    },
                    "storageProfile": {
                        "imageReference": {
                            "publisher": ctx.node.properties['image_reference_publisher'],
                            "offer": ctx.node.properties['image_reference_offer'],
                            "sku" : ctx.node.properties['image_reference_sku'],
                            "version":constants.vm_version
                        },
                        "osDisk" : {
                            "name": constants.os_disk_name,
                            "vhd": {
                                "uri": "http://"+storage_account_name+".blob.core.windows.net/vhds/osdisk.vhd"
                            },
                            "caching": "ReadWrite",
                            "createOption": "FromImage"
                        }
                    },
                    "networkProfile": {
                        "networkInterfaces": [
                            {
                                "id": "/subscriptions/"+subscription_id+"/resourceGroups/"+resource_group_name+"/providers/Microsoft.Network/networkInterfaces/"+nic_name
                            }
                        ]                        }
                    }
                })
            virtual_machine_url=constants.azure_url+'/subscriptions/'+subscription_id+'/resourceGroups/'+resource_group_name+'/providers/Microsoft.Compute/virtualMachines/'+vm_name+'?validating=true&api-version='+constants.api_version
            response_vm = requests.put(url=virtual_machine_url, data=virtual_machine_params, headers=headers)
            print(response_vm.text)
        except:
          ctx.logger.info("Virtual Machine " + vm_name + "could not be created.")
          sys.exit(1)
    else:
     ctx.logger.info("Virtual Machine" + vm_name + "has already been provisioned by another user.")

#start_vm
@operation
def start_vm(**_):
    credentials='Bearer '+auth.get_token_from_client_credentials()

    headers = {"Content-Type": "application/json", "Authorization": credentials}

    subscription_id = ctx.node.properties['subscription_id']
    resource_group_name = resourcegroup.resource_group_name
    public_ip_name= publicip.public_ip_name
    get_pip_info_url= constants.azure_url+'/subscriptions/'+subscription_id+'/resourceGroups/'+resource_group_name+'/providers/microsoft.network/publicIPAddresses/'+public_ip_name+'?api-version='+constants.api_version
    raw_response = requests.get(url=get_pip_info_url, headers=headers)
    ctx.logger.info("raw_response : " + str(raw_response))
    response_get_info = raw_response.json()
    ctx.logger.info("response_get_info : " + str(response_get_info))
    currProperties = response_get_info[u'properties']
    ctx.logger.info("currProperties : " + str(currProperties))
    currIpAddress = currProperties[u'ipAddress']
    ctx.logger.info("Current public IP address is " + str(currIpAddress))
    ctx.instance.runtime_properties['vm_public_ip']= currIpAddress
    resource_group_name = resourcegroup.resource_group_name
    start_vm_url=constants.azure_url+'/subscriptions/'+subscription_id+'/resourceGroups/'+resource_group_name+'/providers/Microsoft.Compute/virtualMachines/'+vm_name+'/start?api-version='+constants.api_version
    response_start_vm=requests.post(start_vm_url,headers=headers)
    print (response_start_vm.text)

#stop_vm
@operation
def stop_vm(**_):
    subscription_id = ctx.node.properties['subscription_id']

    credentials='Bearer '+auth.get_token_from_client_credentials()

    headers = {"Content-Type": "application/json", "Authorization": credentials}

    resource_group_name = resourcegroup.resource_group_name
    stop_vm_url=constants.azure_url+'/subscriptions/'+subscription_id+'/resourceGroups/'+resource_group_name+'/providers/Microsoft.Compute/virtualMachines/'+vm_name+'/start?api-version='+constants.api_version
    response_stop_vm=requests.post(stop_vm_url,headers=headers)
    print (response_stop_vm.text)

@operation
def delete_virtual_machine(**_):
    resource_group_name = ctx.node.runtime_properties['resource_group']
    subscription_id = ctx.node.properties['subscription_id']

    credentials='Bearer '+auth.get_token_from_client_credentials()
    headers = {"Content-Type": "application/json", "Authorization": credentials}

    ctx.logger.info("Checking availability of virtual network: " + vm_name)
    if 1:
        try:
            ctx.logger.info("Deleting the virtual machine: " + vm_name)
            vm_url=constants.azure_url+'/subscriptions/'+subscription_id+'/resourceGroups/'+resource_group_name+'/providers/Microsoft.Compute/virtualMachines/'+vm_name+'?validating=true&api-version='+constants.api_version
            response_vm = requests.delete(url=vm_url,headers=headers)
            print(response_vm.text)

        except:
            ctx.logger.info("Virtual Machine " + vm_name + " could not be deleted.")
        sys.exit(1)
    else:
        ctx.logger.info("Virtual Machine " + vm_name + " does not exist.")

@operation
def set_dependent_resources_names(azure_config,**kwargs):
    ctx.logger.info("Setting set_private_ip")
    vm_private_ip = ctx.target.instance.runtime_properties['private_ip']
    ctx.logger.info("vm_private_ip is " + vm_private_ip)
    ctx.source.instance.runtime_properties['ip'] = vm_private_ip
    ctx.source.instance.runtime_properties['resource_group'] = ctx.target.instance.runtime_properties['resource_group']
    ctx.source.instance.runtime_properties['storage_account'] = ctx.target.instance.runtime_properties['storage_account']
    ctx.source.instance.runtime_properties['vnet'] = ctx.target.instance.runtime_properties['vnet']
    ctx.source.instance.runtime_properties['publicip'] = ctx.target.instance.runtime_properties['publicip']
    ctx.source.instance.runtime_properties['nic'] = ctx.target.instance.runtime_properties['nic']

def _validate_node_properties(key, ctx_node_properties):
    if key not in ctx_node_properties:
        raise NonRecoverableError('{0} is a required input. Unable to create.'.format(key))
<commit_msg>Update server.py<commit_after>########
# Copyright (c) 2015 GigaSpaces Technologies Ltd. All rights reserved
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#        http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
#    * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#    * See the License for the specific language governing permissions and
#    * limitations under the License.

# Built-in Imports
import requests
from requests import Request,Session,Response
import json
import constants
import sys
import auth
import utils
import os
from cloudify.exceptions import NonRecoverableError
from cloudify import ctx
from cloudify.decorators import operation

#virtualmachine:

@operation
def creation_validation(**_):
    for property_key in constants.VM_REQUIRED_PROPERTIES:
        _validate_node_properties(property_key, ctx.node.properties)

    vm_name_exists = _get_vm_name()
    if ctx.node.properties['use_external_resource'] and not vm_name_exists:
    raise NonRecoverableError(
    'External resource, but the supplied '
    'vm does not exist in the account.')
    if not ctx.node.properties['use_external_resource'] and vm_name_exists:
    raise NonRecoverableError(
    'Not external resource, but the supplied '
    'vm exists in the account.')

@operation
def create_vm(**_):

    RANDOM_SUFFIX_VALUE = utils.random_suffix_generator()
    vm_name = ctx.node.properties['vm_name']+RANDOM_SUFFIX_VALUE
    resource_group_name = ctx.instance.runtime_properties['resource_group']
    storage_account_name = ctx.instance.runtime_properties['storage_account']
    location = ctx.node.properties['location']
    vnet_name = ctx.instance.runtime_properties['vnet']
    nic_name = ctx.instance.runtime_properties['nic']
    public_ip_name= ctx.instance.runtime_properties['publicip']
    RANDOM_SUFFIX_VALUE = utils.random_suffix_generator()
    vm_name = contants.VM_PREFIX +RANDOM_SUFFIX_VALUE
    credentials='Bearer '+auth.get_token_from_client_credentials()
    subscription_id = ctx.node.properties['subscription_id']
    headers = {"Content-Type": "application/json", "Authorization": credentials}

    ctx.logger.info("Checking availability of virtual network: " + vm_name)
    if 1:
        try:
            ctx.logger.info("Creating new virtual machine: " + vm_name)
            virtual_machine_params=json.dumps(
            {
                "id":"/subscriptions/"+subscription_id+"/resourceGroups/"+resource_group_name+"/providers/Microsoft.Compute/virtualMachines/"+vm_name,
                "name":vm_name,
                "type":"Microsoft.Compute/virtualMachines",
                "location":location,
                "properties": {
                    "hardwareProfile": {
                        "vmSize": ctx.node.properties['vm_size']
                    },
                    "osProfile": {
                        "computername": vm_name,
                        "adminUsername": ctx.node.properties['ssh_username'],
                        "linuxConfiguration": {
                            "disablePasswordAuthentication": "true",
                            "ssh": {
                                "publicKeys": [
                                    {
                                        "path": "/home/"+ctx.node.properties['ssh_username']+"/.ssh/authorized_keys",
                                        "keyData": ctx.node.properties['key_data']}
                                ]
                            }
                        }
                    },
                    "storageProfile": {
                        "imageReference": {
                            "publisher": ctx.node.properties['image_reference_publisher'],
                            "offer": ctx.node.properties['image_reference_offer'],
                            "sku" : ctx.node.properties['image_reference_sku'],
                            "version":constants.vm_version
                        },
                        "osDisk" : {
                            "name": constants.os_disk_name,
                            "vhd": {
                                "uri": "http://"+storage_account_name+".blob.core.windows.net/vhds/osdisk.vhd"
                            },
                            "caching": "ReadWrite",
                            "createOption": "FromImage"
                        }
                    },
                    "networkProfile": {
                        "networkInterfaces": [
                            {
                                "id": "/subscriptions/"+subscription_id+"/resourceGroups/"+resource_group_name+"/providers/Microsoft.Network/networkInterfaces/"+nic_name
                            }
                        ]                        }
                    }
                })
            virtual_machine_url=constants.azure_url+'/subscriptions/'+subscription_id+'/resourceGroups/'+resource_group_name+'/providers/Microsoft.Compute/virtualMachines/'+vm_name+'?validating=true&api-version='+constants.api_version
            response_vm = requests.put(url=virtual_machine_url, data=virtual_machine_params, headers=headers)
            print(response_vm.text)
        except:
          ctx.logger.info("Virtual Machine " + vm_name + "could not be created.")
          sys.exit(1)
    else:
     ctx.logger.info("Virtual Machine" + vm_name + "has already been provisioned by another user.")

#start_vm
@operation
def start_vm(**_):
    credentials='Bearer '+auth.get_token_from_client_credentials()

    headers = {"Content-Type": "application/json", "Authorization": credentials}

    subscription_id = ctx.node.properties['subscription_id']
    resource_group_name = resourcegroup.resource_group_name
    public_ip_name= publicip.public_ip_name
    get_pip_info_url= constants.azure_url+'/subscriptions/'+subscription_id+'/resourceGroups/'+resource_group_name+'/providers/microsoft.network/publicIPAddresses/'+public_ip_name+'?api-version='+constants.api_version
    raw_response = requests.get(url=get_pip_info_url, headers=headers)
    ctx.logger.info("raw_response : " + str(raw_response))
    response_get_info = raw_response.json()
    ctx.logger.info("response_get_info : " + str(response_get_info))
    currProperties = response_get_info[u'properties']
    ctx.logger.info("currProperties : " + str(currProperties))
    currIpAddress = currProperties[u'ipAddress']
    ctx.logger.info("Current public IP address is " + str(currIpAddress))
    ctx.instance.runtime_properties['vm_public_ip']= currIpAddress
    resource_group_name = resourcegroup.resource_group_name
    start_vm_url=constants.azure_url+'/subscriptions/'+subscription_id+'/resourceGroups/'+resource_group_name+'/providers/Microsoft.Compute/virtualMachines/'+vm_name+'/start?api-version='+constants.api_version
    response_start_vm=requests.post(start_vm_url,headers=headers)
    print (response_start_vm.text)

#stop_vm
@operation
def stop_vm(**_):
    subscription_id = ctx.node.properties['subscription_id']

    credentials='Bearer '+auth.get_token_from_client_credentials()

    headers = {"Content-Type": "application/json", "Authorization": credentials}

    resource_group_name = resourcegroup.resource_group_name
    stop_vm_url=constants.azure_url+'/subscriptions/'+subscription_id+'/resourceGroups/'+resource_group_name+'/providers/Microsoft.Compute/virtualMachines/'+vm_name+'/start?api-version='+constants.api_version
    response_stop_vm=requests.post(stop_vm_url,headers=headers)
    print (response_stop_vm.text)

@operation
def delete_virtual_machine(**_):
    resource_group_name = ctx.node.runtime_properties['resource_group']
    subscription_id = ctx.node.properties['subscription_id']

    credentials='Bearer '+auth.get_token_from_client_credentials()
    headers = {"Content-Type": "application/json", "Authorization": credentials}

    ctx.logger.info("Checking availability of virtual network: " + vm_name)
    if 1:
        try:
            ctx.logger.info("Deleting the virtual machine: " + vm_name)
            vm_url=constants.azure_url+'/subscriptions/'+subscription_id+'/resourceGroups/'+resource_group_name+'/providers/Microsoft.Compute/virtualMachines/'+vm_name+'?validating=true&api-version='+constants.api_version
            response_vm = requests.delete(url=vm_url,headers=headers)
            print(response_vm.text)

        except:
            ctx.logger.info("Virtual Machine " + vm_name + " could not be deleted.")
        sys.exit(1)
    else:
        ctx.logger.info("Virtual Machine " + vm_name + " does not exist.")

@operation
def set_dependent_resources_names(azure_config,**kwargs):
    ctx.logger.info("Setting set_private_ip")
    vm_private_ip = ctx.target.instance.runtime_properties['private_ip']
    ctx.logger.info("vm_private_ip is " + vm_private_ip)
    ctx.source.instance.runtime_properties['ip'] = vm_private_ip
    ctx.source.instance.runtime_properties['resource_group'] = ctx.target.instance.runtime_properties['resource_group']
    ctx.source.instance.runtime_properties['storage_account'] = ctx.target.instance.runtime_properties['storage_account']
    ctx.source.instance.runtime_properties['vnet'] = ctx.target.instance.runtime_properties['vnet']
    ctx.source.instance.runtime_properties['publicip'] = ctx.target.instance.runtime_properties['publicip']
    ctx.source.instance.runtime_properties['nic'] = ctx.target.instance.runtime_properties['nic']

def _validate_node_properties(key, ctx_node_properties):
    if key not in ctx_node_properties:
        raise NonRecoverableError('{0} is a required input. Unable to create.'.format(key))
<|endoftext|>


<commit_before>#!/usr/bin/env python
import os
import sys
import re
import logging
import urllib2
import json

TOKEN_DIR = '/tmp/tokens'
SHARE_DIR = '/var/www/virtualenv/bioshare/include/bioshare/media/files/'
ORIGINAL_COMMAND = None
USER = None
TEST = False
PERMISSIONS = {}
WRITE_PERMISSIONS = ['write_to_share','delete_share_files']
READ_PERMISSIONS = ['download_share_files']

import ConfigParser
config = ConfigParser.ConfigParser()
config.read('sshwrapper.config')

# r = re.compile('^hg -R (S%2B) serve --stdio$')
# match = re.search(r, os.environ['SSH_ORIGINAL_COMMAND'])
# if match:
#     repo_path = match.groups()[0]
#     if os.path.exists(repo_path):
#         os.execvp('hg', ['hg', '-R', repo_path, 'serve', '--stdio'])

def get_token_data(token_file):
    import ConfigParser
    config = ConfigParser.ConfigParser()
    config.read(token_file)
    directory = config.get('details','directory')
    user = config.get('details','user')
    return {'directory':directory,'user':user}

def get_permissions(username,share):
    if not PERMISSIONS.has_key(share):
        url  = config.get('config','perm_api_url') % (share,username)
        response = urllib2.urlopen(url)
        response = json.load(response)
        PERMISSIONS[share]= response['permissions']
    return PERMISSIONS[share]

def can_write(username,share):
    perms = get_permissions(username,share)
    for perm in WRITE_PERMISSIONS:
        if perm not in perms:
            return False
    return True

def can_read(username,share):
    perms = get_permissions(username,share)
    for perm in READ_PERMISSIONS:
        if perm not in perms:
            return False
    return True

def transform_path(path):
    match = re.match('/(?P<token>[a-zA-Z0-9]{10})(?:/(?P<subpath>.*))', path)
    try:
        matches = match.groupdict()
        if not matches.has_key('token'):
            return None
        if matches.has_key('subpath'):
            if '..' in matches['subpath']:
                print 'Illegal subpath: %s' % matches['subpath']
                return None
        token_file = join(TOKEN_DIR,match.group('token'))
        if isfile(token_file):
            data = get_token_data(token_file)
            if match.groupdict().has_key('subpath'):
                return join(data['directory'],match.group('subpath'))
            return data['directory']
        else:
            return None
    except:
        print 'Bad path: %s' % path
        return None

def analyze_path(path):
#     print path
    match = re.match('/(?P<share>[a-zA-Z0-9]{15})(?:/(?P<subpath>.*))', path)
    try:
        matches = match.groupdict()
        if not matches.has_key('share'):
            raise Exception, 'analyze_path: Bad path: %s' % path
        if matches.has_key('subpath'):
#             print matches['subpath']
            if '..' in matches['subpath']:
                raise Exception, 'Illegal subpath: %s' % matches['subpath']
        if matches.has_key('subpath'):
            path = join(config.get('config','share_dir'), matches['share'], match.group('subpath'))
        else:
            path = join(config.get('config','share_dir'), matches['share'])
        return {'share':matches['share'],'path':path}
    except Exception, e:
        raise Exception, 'analyze_path: Bad path: %s' % path
        logger.info('analyze_path exception: %s' % e.message)

def handle_rsync(parts):
    try:
        paths_data = [analyze_path(path) for path in parts[parts.index('.')+1:]]
        paths = [path_data['path'] for path_data in paths_data]
        shares = [path_data['share'] for path_data in paths_data]
        if '--sender' in parts:#server->client
            for share in shares:
                if not can_read(USER, share):
                    raise Exception, 'User %s cannot read from share %s' % (USER,share)
            command = ['rsync', '--server', '--sender', '-vogDtprze.iLsf', '.'] + paths
        else:#client->server
            # --no-p --no-g --chmod=ugo=rwX  //destination default permissions
            for share in shares:
                if not can_write(USER, share):
                    raise Exception, 'User %s cannot write to share %s' % (USER,share)
            command = ['rsync', '--server', '-voDtrze.iLsf', '.'] + paths
        if TEST:
            print command
        else:
            os.execvp('rsync', command)
    except Exception, e:
        logger.info('handle_rsync exception: %s' % e.message)

def handle_ls(parts):
    path_data = analyze_path(parts[1])
    print path_data
    if path_data is not None:
        if can_read(USER, path_data['share']):
            print 'can read'
            command = ['ls', path_data['path']]
#             if TEST:
#                 print command
#             else:
            os.execvp('ls', command)
        else:
            print 'Permission denied'
    else:
        print 'Bad command'

def main():
    try:
        parts = re.split('\s+',ORIGINAL_COMMAND)
        logger.info('SSH_ORIGINAL_COMMAND: '+ORIGINAL_COMMAND)
        if parts[0] == 'rsync':
            handle_rsync(parts)
        elif parts[0] == 'ls':
            handle_ls(parts)
        else:
            print 'Unsupported command: %s' % parts[0]
    #     os.execvp('ls', ['ls', '/var/www'])
    except Exception, e:
        logger.error('Bad or missing parameter "SSH_ORIGINAL_COMMAND"')

if __name__ == '__main__':
    #Should probably use argument parsing library, but trying to keep dependencies to a minumum

    if len(sys.argv)==2:
        USER = sys.argv[1]
    elif len(sys.argv)==3:
        USER = sys.argv[1]
        ORIGINAL_COMMAND = sys.argv[2] #for testing: ssh-wrapper.py username 'rsync /local/file remote:/TOKEN/subdir
        TEST = True
    if ORIGINAL_COMMAND is None:
        ORIGINAL_COMMAND = os.environ['SSH_ORIGINAL_COMMAND']
    logger = logging.getLogger('bioshare')
    hdlr = logging.FileHandler('/var/log/sshwrapper.log')
    formatter = logging.Formatter('%(asctime)s %(levelname)s %(message)s')
    hdlr.setFormatter(formatter)
    logger.addHandler(hdlr)
    logger.setLevel(logging.INFO)
    from os.path import join, isfile
    logger.info('user: %s' % USER)
    logger.info('command: %s' % ORIGINAL_COMMAND)
    main()
<commit_msg>wrapper<commit_after>#!/usr/bin/env python
import os
import sys
import re
import logging
import urllib2
import json

TOKEN_DIR = '/tmp/tokens'
SHARE_DIR = '/var/www/virtualenv/bioshare/include/bioshare/media/files/'
ORIGINAL_COMMAND = None
USER = None
TEST = False
PERMISSIONS = {}
WRITE_PERMISSIONS = ['write_to_share','delete_share_files']
READ_PERMISSIONS = ['download_share_files']

import ConfigParser
config = ConfigParser.ConfigParser()
config.read('sshwrapper.config')

# r = re.compile('^hg -R (S%2B) serve --stdio$')
# match = re.search(r, os.environ['SSH_ORIGINAL_COMMAND'])
# if match:
#     repo_path = match.groups()[0]
#     if os.path.exists(repo_path):
#         os.execvp('hg', ['hg', '-R', repo_path, 'serve', '--stdio'])

def get_token_data(token_file):
    import ConfigParser
    config = ConfigParser.ConfigParser()
    config.read(token_file)
    directory = config.get('details','directory')
    user = config.get('details','user')
    return {'directory':directory,'user':user}

def get_permissions(username,share):
    if not PERMISSIONS.has_key(share):
        url  = config.get('config','perm_api_url') % (share,username)
        response = urllib2.urlopen(url)
        response = json.load(response)
        PERMISSIONS[share]= response['permissions']
    return PERMISSIONS[share]

def can_write(username,share):
    perms = get_permissions(username,share)
    for perm in WRITE_PERMISSIONS:
        if perm not in perms:
            return False
    return True

def can_read(username,share):
    perms = get_permissions(username,share)
    for perm in READ_PERMISSIONS:
        if perm not in perms:
            return False
    return True

def transform_path(path):
    match = re.match('/(?P<token>[a-zA-Z0-9]{10})(?:/(?P<subpath>.*))', path)
    try:
        matches = match.groupdict()
        if not matches.has_key('token'):
            return None
        if matches.has_key('subpath'):
            if '..' in matches['subpath']:
                print 'Illegal subpath: %s' % matches['subpath']
                return None
        token_file = join(TOKEN_DIR,match.group('token'))
        if isfile(token_file):
            data = get_token_data(token_file)
            if match.groupdict().has_key('subpath'):
                return join(data['directory'],match.group('subpath'))
            return data['directory']
        else:
            return None
    except:
        print 'Bad path: %s' % path
        return None

def analyze_path(path):
#     print path
    match = re.match('/(?P<share>[a-zA-Z0-9]{15})(?:/(?P<subpath>.*))', path)
    try:
        matches = match.groupdict()
        if not matches.has_key('share'):
            raise Exception, 'analyze_path: Bad path: %s' % path
        if matches.has_key('subpath'):
#             print matches['subpath']
            if '..' in matches['subpath']:
                raise Exception, 'Illegal subpath: %s' % matches['subpath']
        if matches.has_key('subpath'):
            path = join(config.get('config','share_dir'), matches['share'], match.group('subpath'))
        else:
            path = join(config.get('config','share_dir'), matches['share'])
        return {'share':matches['share'],'path':path}
    except Exception, e:
        raise Exception, 'analyze_path: Bad path: %s' % path
        logger.info('analyze_path exception: %s' % e.message)

def handle_rsync(parts):
    try:
        paths_data = [analyze_path(path) for path in parts[parts.index('.')+1:]]
        paths = [path_data['path'] for path_data in paths_data]
        shares = [path_data['share'] for path_data in paths_data]
        if '--sender' in parts:#server->client
            for share in shares:
                if not can_read(USER, share):
                    raise Exception, 'User %s cannot read from share %s' % (USER,share)
            command = ['rsync', '--server', '--sender', '-vogDtprze.iLsf', '.'] + paths
        else:#client->server
            # --no-p --no-g --chmod=ugo=rwX  //destination default permissions
            for share in shares:
                if not can_write(USER, share):
                    raise Exception, 'User %s cannot write to share %s' % (USER,share)
            command = ['rsync', '--server', parts[2] , '.'] + paths#'-voDtrze.iLsf'
        if TEST:
            print command
        else:
            logger.info('running rsync command: ' + command)
            os.execvp('rsync', command)
    except Exception, e:
        logger.info('handle_rsync exception: %s' % e.message)

def handle_ls(parts):
    path_data = analyze_path(parts[1])
    print path_data
    if path_data is not None:
        if can_read(USER, path_data['share']):
            print 'can read'
            command = ['ls', path_data['path']]
#             if TEST:
#                 print command
#             else:
            os.execvp('ls', command)
        else:
            print 'Permission denied'
    else:
        print 'Bad command'

def main():
    try:
        parts = re.split('\s+',ORIGINAL_COMMAND)
        logger.info('SSH_ORIGINAL_COMMAND: '+ORIGINAL_COMMAND)
        if parts[0] == 'rsync':
            handle_rsync(parts)
        elif parts[0] == 'ls':
            handle_ls(parts)
        else:
            print 'Unsupported command: %s' % parts[0]
    #     os.execvp('ls', ['ls', '/var/www'])
    except Exception, e:
        logger.error('Bad or missing parameter "SSH_ORIGINAL_COMMAND"')

if __name__ == '__main__':
    #Should probably use argument parsing library, but trying to keep dependencies to a minumum

    if len(sys.argv)==2:
        USER = sys.argv[1]
    elif len(sys.argv)==3:
        USER = sys.argv[1]
        ORIGINAL_COMMAND = sys.argv[2] #for testing: ssh-wrapper.py username 'rsync /local/file remote:/TOKEN/subdir
        TEST = True
    if ORIGINAL_COMMAND is None:
        ORIGINAL_COMMAND = os.environ['SSH_ORIGINAL_COMMAND']
    logger = logging.getLogger('bioshare')
    hdlr = logging.FileHandler('/var/log/sshwrapper.log')
    formatter = logging.Formatter('%(asctime)s %(levelname)s %(message)s')
    hdlr.setFormatter(formatter)
    logger.addHandler(hdlr)
    logger.setLevel(logging.INFO)
    from os.path import join, isfile
    logger.info('user: %s' % USER)
    logger.info('command: %s' % ORIGINAL_COMMAND)
    main()
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-

# OpenFisca -- A versatile microsimulation software
# By: OpenFisca Team <contact@openfisca.fr>
#
# Copyright (C) 2011, 2012, 2013, 2014, 2015 OpenFisca Team
# https://github.com/openfisca
#
# This file is part of OpenFisca.
#
# OpenFisca is free software; you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as
# published by the Free Software Foundation, either version 3 of the
# License, or (at your option) any later version.
#
# OpenFisca is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program.  If not, see <http://www.gnu.org/licenses/>.

import json

from webob import Request
from nose.tools import assert_equal, assert_in, assert_not_in, assert_is_instance, assert_not_equal

from . import common

TARGET_URL = '/api/2/formula/'
INPUT_VARIABLE = 'salaire_de_base'
VALID_PERIOD = '2015-01'
INVALID_PERIOD = 'herp'
VALID_FORMULA = 'salaire_net_a_payer'
DATED_FORMULA = 'allegement_fillon'
VALID_OTHER_FORMULA = 'sal'
INVALID_FORMULA = 'inexistent'
PARAM_VALUE = 1300
VALID_QUERY_STRING = '{0}={1}'.format(INPUT_VARIABLE, PARAM_VALUE)
INVALID_QUERY_STRING = '{0}={1}'.format(INVALID_FORMULA, PARAM_VALUE)

def send(formula = VALID_FORMULA, method = 'GET', period = '', query_string = ''):
    if period:
        period += '/'

    target = TARGET_URL + period + formula + '?' + query_string

    req = Request.blank(target, method = method)
    res = req.get_response(common.app)

    return {
        'status_code': res.status_code,
        'payload': json.loads(res.body)
        }

def setup_module(module):
    common.get_or_load_app()

def test_formula_get_status_code():
    assert_equal(send()['status_code'], 200)

def test_formula_post_status_code():
    assert_equal(send(method = 'POST')['status_code'], 405)

def test_formula_put_status_code():
    assert_equal(send(method = 'PUT')['status_code'], 405)

def test_formula_delete_status_code():
    assert_equal(send(method = 'DELETE')['status_code'], 405)

def test_formula_api_version():
    assert_equal(send()['payload']['apiVersion'], '2.0.0-alpha.1')

def test_dated_formula_status_code():
    assert_equal(send(formula = DATED_FORMULA)['status_code'], 200)

def test_not_a_formula_status_code():
    assert_equal(send(formula = INPUT_VARIABLE)['status_code'], 422)

def test_not_a_formula_error_message():
    message = send(formula = INPUT_VARIABLE)['payload']['error']['message']

    assert_in(INPUT_VARIABLE, message)
    assert_in('input variable', message)
    assert_in('cannot be computed', message)
    assert_not_in('{', message)  # serialisation failed

def test_invalid_formula_status_code():
    assert_equal(send(formula = INVALID_FORMULA)['status_code'], 404)

def test_invalid_formula_error_message():
    message = send(formula = INVALID_FORMULA)['payload']['error']['message']

    assert_in(INVALID_FORMULA, message)
    assert_in('does not exist', message)
    assert_not_in('{', message)  # serialisation failed

def test_invalid_formula_params():
    params = send(formula = INVALID_FORMULA, query_string = VALID_QUERY_STRING)['payload']['params']

    assert_equal({INPUT_VARIABLE: PARAM_VALUE}, params)

def test_invalid_formula_with_valid_formula_status_code():
    assert_equal(send(formula = VALID_FORMULA + '+' + INVALID_FORMULA)['status_code'], 404)

def test_invalid_formula_with_valid_formula_error_message():
    message = send(formula = VALID_FORMULA + '+' + INVALID_FORMULA)['payload']['error']['message']

    assert_in(INVALID_FORMULA, message)
    assert_in('does not exist', message)
    assert_not_in('{', message)  # serialisation failed

def test_formula_value_without_params():
    value = send()['payload']['values'][VALID_FORMULA]
    assert_is_instance(value, float)
    assert_equal(value, 0)

def test_formula_value_with_params():
    value = send(query_string = VALID_QUERY_STRING)['payload']['values'][VALID_FORMULA]
    assert_is_instance(value, float)
    assert_not_equal(value, 0)

def test_formula_echo_params_without_params():
    params = send()['payload']['params']
    assert_equal({}, params)

def test_formula_echo_params_with_params():
    params = send(query_string = VALID_QUERY_STRING)['payload']['params']
    assert_equal({INPUT_VARIABLE: PARAM_VALUE}, params)

def test_bad_params_status_code():
    assert_equal(send(query_string = INVALID_QUERY_STRING)['status_code'], 400)

def test_bad_params_error_message():
    message = send(query_string = INVALID_QUERY_STRING)['payload']['error']['message']

    assert_in(INVALID_FORMULA, message)
    assert_in('does not exist', message)
    assert_not_in('{', message)  # serialisation failed

def test_unnormalizable_params_status_code():
    assert_equal(send(query_string = 'birth=herp')['status_code'], 400)

def test_unnormalizable_params_error_message():
    message = send(query_string = 'birth=herp')['payload']['error']['message']

    assert_in('birth', message)
    assert_in('normalized', message)
    assert_not_in('{', message)  # serialisation failed

def test_multiple_formulas_value_without_params():
    values = send(formula = VALID_FORMULA + '+' + VALID_OTHER_FORMULA)['payload']['values']

    for formula_name in (VALID_FORMULA, VALID_OTHER_FORMULA):
        assert_in(formula_name, values)
        assert_is_instance(values[formula_name], float)
        assert_equal(values[formula_name], 0)

def test_period_year():
    assert_equal(send(period = '2012')['payload']['period'], ['year', [2012, 1, 1], 1])

def test_period_month():
    assert_equal(send(period = '2013-02')['payload']['period'], ['month', [2013, 2, 1], 1])

def test_invalid_period_status_code():
    assert_equal(send(period = INVALID_PERIOD)['status_code'], 400)

def test_invalid_period_error_message():
    message = send(period = INVALID_PERIOD)['payload']['error']['message']

    assert_in(INVALID_PERIOD, message)
    assert_in('could not be parsed', message)
    assert_not_in('{', message)  # serialisation failed
<commit_msg>Add failing test for echoing date param in /formula<commit_after># -*- coding: utf-8 -*-

# OpenFisca -- A versatile microsimulation software
# By: OpenFisca Team <contact@openfisca.fr>
#
# Copyright (C) 2011, 2012, 2013, 2014, 2015 OpenFisca Team
# https://github.com/openfisca
#
# This file is part of OpenFisca.
#
# OpenFisca is free software; you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as
# published by the Free Software Foundation, either version 3 of the
# License, or (at your option) any later version.
#
# OpenFisca is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program.  If not, see <http://www.gnu.org/licenses/>.

import json

from webob import Request
from nose.tools import assert_equal, assert_in, assert_not_in, assert_is_instance, assert_not_equal

from . import common

TARGET_URL = '/api/2/formula/'
INPUT_VARIABLE = 'salaire_de_base'
VALID_PERIOD = '2015-01'
INVALID_PERIOD = 'herp'
VALID_FORMULA = 'salaire_net_a_payer'
DATED_FORMULA = 'allegement_fillon'
DATE_PARAM = 'apprentissage_contrat_debut'
DATE_PARAM_VALUE = VALID_PERIOD + '-03'
VALID_OTHER_FORMULA = 'sal'
INVALID_FORMULA = 'inexistent'
PARAM_VALUE = 1300
VALID_QUERY_STRING = '{0}={1}'.format(INPUT_VARIABLE, PARAM_VALUE)
INVALID_QUERY_STRING = '{0}={1}'.format(INVALID_FORMULA, PARAM_VALUE)

def send(formula = VALID_FORMULA, method = 'GET', period = '', query_string = ''):
    if period:
        period += '/'

    target = TARGET_URL + period + formula + '?' + query_string

    req = Request.blank(target, method = method)
    res = req.get_response(common.app)

    return {
        'status_code': res.status_code,
        'payload': json.loads(res.body)
        }

def setup_module(module):
    common.get_or_load_app()

def test_formula_get_status_code():
    assert_equal(send()['status_code'], 200)

def test_formula_post_status_code():
    assert_equal(send(method = 'POST')['status_code'], 405)

def test_formula_put_status_code():
    assert_equal(send(method = 'PUT')['status_code'], 405)

def test_formula_delete_status_code():
    assert_equal(send(method = 'DELETE')['status_code'], 405)

def test_formula_api_version():
    assert_equal(send()['payload']['apiVersion'], '2.0.0-alpha.1')

def test_dated_formula_status_code():
    assert_equal(send(formula = DATED_FORMULA)['status_code'], 200)

def test_not_a_formula_status_code():
    assert_equal(send(formula = INPUT_VARIABLE)['status_code'], 422)

def test_not_a_formula_error_message():
    message = send(formula = INPUT_VARIABLE)['payload']['error']['message']

    assert_in(INPUT_VARIABLE, message)
    assert_in('input variable', message)
    assert_in('cannot be computed', message)
    assert_not_in('{', message)  # serialisation failed

def test_invalid_formula_status_code():
    assert_equal(send(formula = INVALID_FORMULA)['status_code'], 404)

def test_invalid_formula_error_message():
    message = send(formula = INVALID_FORMULA)['payload']['error']['message']

    assert_in(INVALID_FORMULA, message)
    assert_in('does not exist', message)
    assert_not_in('{', message)  # serialisation failed

def test_invalid_formula_params():
    params = send(formula = INVALID_FORMULA, query_string = VALID_QUERY_STRING)['payload']['params']

    assert_equal({INPUT_VARIABLE: PARAM_VALUE}, params)

def test_invalid_formula_with_valid_formula_status_code():
    assert_equal(send(formula = VALID_FORMULA + '+' + INVALID_FORMULA)['status_code'], 404)

def test_invalid_formula_with_valid_formula_error_message():
    message = send(formula = VALID_FORMULA + '+' + INVALID_FORMULA)['payload']['error']['message']

    assert_in(INVALID_FORMULA, message)
    assert_in('does not exist', message)
    assert_not_in('{', message)  # serialisation failed

def test_formula_value_without_params():
    value = send()['payload']['values'][VALID_FORMULA]
    assert_is_instance(value, float)
    assert_equal(value, 0)

def test_formula_value_with_params():
    value = send(query_string = VALID_QUERY_STRING)['payload']['values'][VALID_FORMULA]
    assert_is_instance(value, float)
    assert_not_equal(value, 0)

def test_formula_echo_params_without_params():
    params = send()['payload']['params']
    assert_equal({}, params)

def test_formula_echo_params_with_params():
    params = send(query_string = VALID_QUERY_STRING)['payload']['params']
    assert_equal({INPUT_VARIABLE: PARAM_VALUE}, params)

def test_echo_params_date():
    params = send(query_string = DATE_PARAM + '=' + DATE_PARAM_VALUE)['payload']['params']
    assert_equal({DATE_PARAM: DATE_PARAM_VALUE}, params)

def test_bad_params_status_code():
    assert_equal(send(query_string = INVALID_QUERY_STRING)['status_code'], 400)

def test_bad_params_error_message():
    message = send(query_string = INVALID_QUERY_STRING)['payload']['error']['message']

    assert_in(INVALID_FORMULA, message)
    assert_in('does not exist', message)
    assert_not_in('{', message)  # serialisation failed

def test_unnormalizable_params_status_code():
    assert_equal(send(query_string = 'birth=herp')['status_code'], 400)

def test_unnormalizable_params_error_message():
    message = send(query_string = 'birth=herp')['payload']['error']['message']

    assert_in('birth', message)
    assert_in('normalized', message)
    assert_not_in('{', message)  # serialisation failed

def test_multiple_formulas_value_without_params():
    values = send(formula = VALID_FORMULA + '+' + VALID_OTHER_FORMULA)['payload']['values']

    for formula_name in (VALID_FORMULA, VALID_OTHER_FORMULA):
        assert_in(formula_name, values)
        assert_is_instance(values[formula_name], float)
        assert_equal(values[formula_name], 0)

def test_period_year():
    assert_equal(send(period = '2012')['payload']['period'], ['year', [2012, 1, 1], 1])

def test_period_month():
    assert_equal(send(period = '2013-02')['payload']['period'], ['month', [2013, 2, 1], 1])

def test_invalid_period_status_code():
    assert_equal(send(period = INVALID_PERIOD)['status_code'], 400)

def test_invalid_period_error_message():
    message = send(period = INVALID_PERIOD)['payload']['error']['message']

    assert_in(INVALID_PERIOD, message)
    assert_in('could not be parsed', message)
    assert_not_in('{', message)  # serialisation failed
<|endoftext|>


<commit_before>from __future__ import division

import os, os.path
import json
import itertools
from collections import namedtuple
import matplotlib.pyplot as plt
import numpy as np

from parsers.article import ArticleData

BACKGROUND_COLOR = '#e3e1dd'
LIGHT_COLOR = '#f0efed'
DARK_COLOR =  '#4c4c4c'

def get_subdirs(parent_path):
    return [d for d in os.listdir(parent_path) if os.path.isdir(os.path.join(parent_path, d))]

def get_articles(json_filepath):
    with open(json_filepath, 'r') as f:
        json_content = json.load(f)
        articles_string = json_content['articles']
        return [ArticleData.from_json(article_s) for article_s in articles_string]

def get_flat_article_list(provider_path):
    all_days = get_subdirs(provider_path)
    all_articles = []

    for day in all_days:
        day_path = os.path.join(provider_path, day)
        all_batches = get_subdirs(day_path)

        for batch in all_batches:
            batch_path = os.path.join(day_path, batch)
            all_articles.extend(get_articles(os.path.join(batch_path, 'articles.json')))

    return all_articles

def categorize_articles(articles):
    def keyfunc(article):
        return article.category

    groups = []
    uniquekeys = []
    data = sorted(articles, key=keyfunc)
    for k, g in itertools.groupby(data, keyfunc):
        groups.append(list(g))      # Store group iterator as a list
        uniquekeys.append(k)

    return zip(uniquekeys, groups)

def count_links(articles):
    ext_links = sum([len(art.external_links) for art in articles])
    int_links = sum([len(art.internal_links) for art in articles])

    return ext_links, int_links

CategoryCounters = namedtuple('CategoryCounters', 'name total_links total_articles  link_article_ratio')

def make_barchart_in_subplot(ax, xs, title, labels):
    ind = np.arange(len(xs))
    ax.barh(ind, xs, color=LIGHT_COLOR)
    ax.set_yticklabels(ind+0.35, labels, fontsize='small', fontname='sans')
    ax.set_title(title)

def make_barchart(xs, title, labels):
    ind = np.arange(len(xs))
    plt.barh(ind, xs, color=LIGHT_COLOR)
    plt.yticks(ind+0.35, labels, fontsize='small')
    plt.title(title)

def sort_categories_by_links_article_ratio(categorized_articles):
    link_counters = list()

    max_total_articles = len(max(categorized_articles, key=lambda a: len(a[1]))[1])

    for (group, articles) in categorized_articles:
        total_articles = len(articles)

        total_links = sum(count_links(articles))
        if total_links and max_total_articles:
            ratio =  (total_articles / total_links) / max_total_articles
            link_counters.append(CategoryCounters(name=group,
                                                  total_links=total_links,
                                                  total_articles=total_articles,
                                                  link_article_ratio=ratio))

    def keyfunc(counter):
        return counter.link_article_ratio
    link_counters.sort(key=keyfunc)

    return link_counters

def plot_categories_by_links_article_ratio_in_subplot(ax, categorized_articles, source_name):
    link_counters = sort_categories_by_links_article_ratio(categorized_articles)

    x = np.array([c.link_article_ratio for c in link_counters])

    def make_label(counter):
        return u'{0} (n_a={1}  n_l={2})'.format(u'/'.join(counter.name),
                                               counter.total_articles,
                                               counter.total_links)

    labels = [make_label(c) for c in link_counters]
    make_barchart_in_subplot(ax, x, source_name, labels)

def plot_categories_by_links_article_ratio(name, categorized_articles, outdir):
    link_counters = sort_categories_by_links_article_ratio(categorized_articles)

    for counter in link_counters:
        print counter

    x = np.array([c.link_article_ratio for c in link_counters])

    def make_label(counter):
        return u'{0} (n_a={1}  n_l={2})'.format(u'/'.join(counter.name),
                                               counter.total_articles,
                                               counter.total_links)

    plt.clf()
    labels = [make_label(c) for c in link_counters]
    make_barchart(x, 'Categories by article/links ratio ({0})'.format(name), labels)
    plt.savefig(os.path.join(outdir, name+'_article_link_ratio.png'))

def plot_categories_by_number_of_articles(name, categorized_articles, outdir):
    article_counters = list()
    for (group, articles) in categorized_articles:
        article_counters.append((group, len(articles)))

    def keyfunc(counter):
        return counter[1]
    article_counters.sort(key=keyfunc)

    x = np.array([counter[1] for counter in article_counters])

    def make_label(article_counter):
        return u'{0}'.format(u'/'.join(article_counter[0]))

    plt.clf()
    labels = [make_label(c) for c in article_counters]
    make_barchart(x, '# Articles per category ({0})'.format(name), labels)
    plt.savefig(os.path.join(outdir, name+'_articles_by_category.png'))

def plot_categories_by_number_of_links(name, categorized_articles, outdir):

    LinkCounter = namedtuple('LinkCounter', 'name total_ext_links total_int_links total_links')

    link_counters = list()
    for (group, articles) in categorized_articles:
        n_ext_links, n_int_links = count_links(articles)
        link_counters.append(LinkCounter(name=group,
                                         total_ext_links=n_ext_links,
                                         total_int_links=n_int_links,
                                         total_links=n_ext_links+n_int_links))

    def keyfunc(counter):
        return counter.total_links
    link_counters.sort(key=keyfunc)

    print link_counters

    x1 = np.array([counter.total_ext_links for counter in link_counters])
    x2 = np.array([counter.total_int_links for counter in link_counters])

    def make_label(link_counter):
        return u'{0}'.format(u'/'.join(link_counter.name))
    labels = [make_label(c) for c in link_counters]

    plt.clf()
    plt.autumn()
    ind = np.arange(len(x1))
    p1 = plt.barh(ind, x1, color=DARK_COLOR)
    p2 = plt.barh(ind, x2, left=x1, color=LIGHT_COLOR)
    plt.yticks(ind+0.35, labels, fontsize='small', fontname='sans')
    plt.title('Number of links per category ({0})'.format(name))
    plt.legend( (p1[0], p2[0]), ('External links', 'Internal links'), 'lower right' )
    plt.savefig(os.path.join(outdir, name+'_number_of_links.png'))

def make_all_figures(db_root, outdir):
    if not os.path.exists(outdir):
        os.mkdir(outdir)

    for source_dir in get_subdirs(db_root):
        articles = get_flat_article_list(os.path.join(db_root, source_dir))

        categorized_articles = categorize_articles(articles)

        for (cat_name, articles) in categorized_articles:
            print u'{0} : \t {1} articles'.format(u'/'.join(cat_name), len(articles))

        plot_categories_by_links_article_ratio(source_dir, categorized_articles, outdir)
        plot_categories_by_number_of_articles(source_dir, categorized_articles, outdir)
        plot_categories_by_number_of_links(source_dir, categorized_articles, outdir)

if __name__=='__main__':
    import argparse

    parser = argparse.ArgumentParser(description='Make (hopefully) interesting figures from the json db')
    parser.add_argument('--dir', type=str, dest='input_dir', required=True, help='json db directory')
    parser.add_argument('--outdir', type=str, dest='output_dir', required=True, help='directory to dump the figures in')
    args = parser.parse_args()
    make_all_figures(args.input_dir, args.output_dir)

<commit_msg>removed *all* font selections<commit_after>from __future__ import division

import os, os.path
import json
import itertools
from collections import namedtuple
import matplotlib.pyplot as plt
import numpy as np

from parsers.article import ArticleData

BACKGROUND_COLOR = '#e3e1dd'
LIGHT_COLOR = '#f0efed'
DARK_COLOR =  '#4c4c4c'

def get_subdirs(parent_path):
    return [d for d in os.listdir(parent_path) if os.path.isdir(os.path.join(parent_path, d))]

def get_articles(json_filepath):
    with open(json_filepath, 'r') as f:
        json_content = json.load(f)
        articles_string = json_content['articles']
        return [ArticleData.from_json(article_s) for article_s in articles_string]

def get_flat_article_list(provider_path):
    all_days = get_subdirs(provider_path)
    all_articles = []

    for day in all_days:
        day_path = os.path.join(provider_path, day)
        all_batches = get_subdirs(day_path)

        for batch in all_batches:
            batch_path = os.path.join(day_path, batch)
            all_articles.extend(get_articles(os.path.join(batch_path, 'articles.json')))

    return all_articles

def categorize_articles(articles):
    def keyfunc(article):
        return article.category

    groups = []
    uniquekeys = []
    data = sorted(articles, key=keyfunc)
    for k, g in itertools.groupby(data, keyfunc):
        groups.append(list(g))      # Store group iterator as a list
        uniquekeys.append(k)

    return zip(uniquekeys, groups)

def count_links(articles):
    ext_links = sum([len(art.external_links) for art in articles])
    int_links = sum([len(art.internal_links) for art in articles])

    return ext_links, int_links

CategoryCounters = namedtuple('CategoryCounters', 'name total_links total_articles  link_article_ratio')

def make_barchart_in_subplot(ax, xs, title, labels):
    ind = np.arange(len(xs))
    ind = np.arange(len(xs))
    ax.barh(ind, xs, color=LIGHT_COLOR)
    ax.set_yticklabels(ind+0.35, labels, fontsize='small')
    ax.set_title(title)

def make_barchart(xs, title, labels):
    ind = np.arange(len(xs))
    plt.barh(ind, xs, color=LIGHT_COLOR)
    plt.yticks(ind+0.35, labels, fontsize='small')
    plt.title(title)

def sort_categories_by_links_article_ratio(categorized_articles):
    link_counters = list()

    max_total_articles = len(max(categorized_articles, key=lambda a: len(a[1]))[1])

    for (group, articles) in categorized_articles:
        total_articles = len(articles)

        total_links = sum(count_links(articles))
        if total_links and max_total_articles:
            ratio =  (total_articles / total_links) / max_total_articles
            link_counters.append(CategoryCounters(name=group,
                                                  total_links=total_links,
                                                  total_articles=total_articles,
                                                  link_article_ratio=ratio))

    def keyfunc(counter):
        return counter.link_article_ratio
    link_counters.sort(key=keyfunc)

    return link_counters

def plot_categories_by_links_article_ratio_in_subplot(ax, categorized_articles, source_name):
    link_counters = sort_categories_by_links_article_ratio(categorized_articles)

    x = np.array([c.link_article_ratio for c in link_counters])

    def make_label(counter):
        return u'{0} (n_a={1}  n_l={2})'.format(u'/'.join(counter.name),
                                               counter.total_articles,
                                               counter.total_links)

    labels = [make_label(c) for c in link_counters]
    make_barchart_in_subplot(ax, x, source_name, labels)

def plot_categories_by_links_article_ratio(name, categorized_articles, outdir):
    link_counters = sort_categories_by_links_article_ratio(categorized_articles)

    for counter in link_counters:
        print counter

    x = np.array([c.link_article_ratio for c in link_counters])

    def make_label(counter):
        return u'{0} (n_a={1}  n_l={2})'.format(u'/'.join(counter.name),
                                               counter.total_articles,
                                               counter.total_links)

    plt.clf()
    labels = [make_label(c) for c in link_counters]
    make_barchart(x, 'Categories by article/links ratio ({0})'.format(name), labels)
    plt.savefig(os.path.join(outdir, name+'_article_link_ratio.png'))

def plot_categories_by_number_of_articles(name, categorized_articles, outdir):
    article_counters = list()
    for (group, articles) in categorized_articles:
        article_counters.append((group, len(articles)))

    def keyfunc(counter):
        return counter[1]
    article_counters.sort(key=keyfunc)

    x = np.array([counter[1] for counter in article_counters])

    def make_label(article_counter):
        return u'{0}'.format(u'/'.join(article_counter[0]))

    plt.clf()
    labels = [make_label(c) for c in article_counters]
    make_barchart(x, '# Articles per category ({0})'.format(name), labels)
    plt.savefig(os.path.join(outdir, name+'_articles_by_category.png'))

def plot_categories_by_number_of_links(name, categorized_articles, outdir):

    LinkCounter = namedtuple('LinkCounter', 'name total_ext_links total_int_links total_links')

    link_counters = list()
    for (group, articles) in categorized_articles:
        n_ext_links, n_int_links = count_links(articles)
        link_counters.append(LinkCounter(name=group,
                                         total_ext_links=n_ext_links,
                                         total_int_links=n_int_links,
                                         total_links=n_ext_links+n_int_links))

    def keyfunc(counter):
        return counter.total_links
    link_counters.sort(key=keyfunc)

    print link_counters

    x1 = np.array([counter.total_ext_links for counter in link_counters])
    x2 = np.array([counter.total_int_links for counter in link_counters])

    def make_label(link_counter):
        return u'{0}'.format(u'/'.join(link_counter.name))
    labels = [make_label(c) for c in link_counters]

    plt.clf()
    plt.autumn()
    ind = np.arange(len(x1))
    p1 = plt.barh(ind, x1, color=DARK_COLOR)
    p2 = plt.barh(ind, x2, left=x1, color=LIGHT_COLOR)
    plt.yticks(ind+0.35, labels, fontsize='small')
    plt.title('Number of links per category ({0})'.format(name))
    plt.legend( (p1[0], p2[0]), ('External links', 'Internal links'), 'lower right' )
    plt.savefig(os.path.join(outdir, name+'_number_of_links.png'))

def make_all_figures(db_root, outdir):
    if not os.path.exists(outdir):
        os.mkdir(outdir)

    for source_dir in get_subdirs(db_root):
        articles = get_flat_article_list(os.path.join(db_root, source_dir))

        categorized_articles = categorize_articles(articles)

        for (cat_name, articles) in categorized_articles:
            print u'{0} : \t {1} articles'.format(u'/'.join(cat_name), len(articles))

        plot_categories_by_links_article_ratio(source_dir, categorized_articles, outdir)
        plot_categories_by_number_of_articles(source_dir, categorized_articles, outdir)
        plot_categories_by_number_of_links(source_dir, categorized_articles, outdir)

if __name__=='__main__':
    import argparse

    parser = argparse.ArgumentParser(description='Make (hopefully) interesting figures from the json db')
    parser.add_argument('--dir', type=str, dest='input_dir', required=True, help='json db directory')
    parser.add_argument('--outdir', type=str, dest='output_dir', required=True, help='directory to dump the figures in')
    args = parser.parse_args()
    make_all_figures(args.input_dir, args.output_dir)

<|endoftext|>


<commit_before>import numpy as np
import struct
import socket
from contextlib import closing
import time
import cPickle

def get_udp_packets(ri,npkts,streamid,stream_reg='streamid',addr=('192.168.1.1',12345)):
    ri.r.write_int(stream_reg,0)

    with closing(socket.socket(socket.AF_INET,socket.SOCK_DGRAM)) as s:
        s.bind(addr)
        s.settimeout(0)
        nstale = 0
        try:
            while s.recv(2000):
                nstale +=1
            if nstale:
                print "flushed",nstale,"packets"
        except:
            pass
        s.settimeout(1)

        ri.r.write_int(stream_reg,streamid)
        pkts = []
        while len(pkts) < npkts:
            pkt = s.recv(2000)
            if pkt:
                pkts.append(pkt)
            else:
                print "breaking"
                break

        ri.r.write_int(stream_reg,0)

    return pkts

def get_udp_data(ri,npkts,streamid,chans,nfft,stream_reg='streamid',addr=('192.168.1.1',12345)):
    pkts = get_udp_packets(ri, npkts, streamid, stream_reg=stream_reg, addr=addr)
    darray,seqnos = decode_packets(pkts,streamid,chans,nfft)
    return darray,seqnos

ptype = np.dtype([('idle','>i2'),
                  ('idx', '>i2'),
                ('stream', '>i2'),
                ('chan', '>i2'),
                ('mcntr', '>i4')])

hdr_fmt = ">4HI"
hdr_size = struct.calcsize(hdr_fmt)
pkt_size = hdr_size + 1024
null_pkt = "\x00"*1024
def decode_packets(plist,streamid,chans,nfft,pkts_per_chunk = 16):
    nchan = chans.shape[0]
    mcnt_inc = nfft*2**12/nchan
    next_seqno = None
    mcnt_top = 0
    dset = []
    mcntoff = None
    last_mcnt_ovf = None
    seqnos = []
    nextseqnos = []
    chan0 = None
    for pnum,pkt in enumerate(plist):
        if len(pkt) != pkt_size:
            print "got packet size",len(pkt), "expected",pkt_size
            continue
        pidle,pidx,pstream,pchan,pmcnt = struct.unpack(hdr_fmt,pkt[:hdr_size])
        #print pmcnt
        if pstream != streamid:
            print "got stream id",pstream,"expected",streamid
            continue
        if next_seqno is None:
            mcnt_top = 0
        else:
            if pmcnt < mcnt_inc:
                if last_mcnt_ovf != pmcnt:
                    print "detected mcnt overflow",pmcnt,pidx,next_seqno,(mcnt_top/2**32),pnum,mcntoff
                    mcnt_top += 2**32
                    last_mcnt_ovf = pmcnt
                else:
#                    print "continuation of previous mcnt overflow",pidx
                    pass
            else:
                last_mcnt_ovf = None
        chunkno,pmcntoff = divmod(pmcnt+mcnt_top,mcnt_inc)
        seqno = (chunkno)*pkts_per_chunk + pidx
        seqnos.append(seqno)
        nextseqnos.append(next_seqno)
        if next_seqno is None:
            chan0 = pchan
#            print "found first packet",seqno,pidx
            next_seqno = seqno
            mcntoff = pmcntoff
#            print pchan
        if mcntoff != pmcntoff:
            print "mcnt offset jumped. Was",mcntoff,"now",pmcntoff,"dropping.."
            continue
        if pchan != chan0:
            print "warning! channel id changed from",chan0,"to",pchan
        if seqno - next_seqno < 0:
            print "seqno diff",(seqno-next_seqno),seqno,next_seqno
            continue # trying to go backwards
        if seqno == next_seqno:
            dset.append(pkt[hdr_size:])
            next_seqno += 1
        else:
            print "sequence number skip, expected:",next_seqno,"got",seqno,"inserting",(seqno-next_seqno),"null packets",pnum,pidx
            if True: #seqno-next_seqno == 32768:
                print "caught special case, writing to disk"
                fname = time.strftime("udp_skip_%Y-%m-%d_%H%M%S.pkl")
                fh = open(fname,'w')
                cPickle.dump(dict(plist=plist,dset=dset,pnum=pnum,pkt=pkt,streamid=streamid,chans=chans,nfft=nfft),
                             fh,cPickle.HIGHEST_PROTOCOL)
                fh.close()
                print "wrote data to:",fname
            for k in range(seqno - next_seqno+1):
                dset.append(null_pkt)
                next_seqno += 1
    dset = ''.join(dset)
    ns = (len(dset)//(4*nchan))
    dset = dset[:ns*(4*nchan)]
    darray = np.fromstring(dset,dtype='>i2').astype('float32').view('complex64')
    darray.shape = (ns,nchan)
    shift = np.flatnonzero(chans==(chan0))[0] - (nchan-1)
    darray = np.roll(darray,shift,axis=1)
    return darray,np.array(seqnos)<commit_msg>Tentative fix for extraneous 2**15 extra packets problem<commit_after>import numpy as np
import struct
import socket
from contextlib import closing
import time
import cPickle

def get_udp_packets(ri,npkts,streamid,stream_reg='streamid',addr=('192.168.1.1',12345)):
    ri.r.write_int(stream_reg,0)

    with closing(socket.socket(socket.AF_INET,socket.SOCK_DGRAM)) as s:
        s.bind(addr)
        s.settimeout(0)
        nstale = 0
        try:
            while s.recv(2000):
                nstale +=1
            if nstale:
                print "flushed",nstale,"packets"
        except:
            pass
        s.settimeout(1)

        ri.r.write_int(stream_reg,streamid)
        pkts = []
        while len(pkts) < npkts:
            pkt = s.recv(2000)
            if pkt:
                pkts.append(pkt)
            else:
                print "breaking"
                break

        ri.r.write_int(stream_reg,0)

    return pkts

def get_udp_data(ri,npkts,streamid,chans,nfft,stream_reg='streamid',addr=('192.168.1.1',12345)):
    pkts = get_udp_packets(ri, npkts, streamid, stream_reg=stream_reg, addr=addr)
    darray,seqnos = decode_packets(pkts,streamid,chans,nfft)
    return darray,seqnos

ptype = np.dtype([('idle','>i2'),
                  ('idx', '>i2'),
                ('stream', '>i2'),
                ('chan', '>i2'),
                ('mcntr', '>i4')])

hdr_fmt = ">4HI"
hdr_size = struct.calcsize(hdr_fmt)
pkt_size = hdr_size + 1024
null_pkt = "\x00"*1024
def decode_packets(plist,streamid,chans,nfft,pkts_per_chunk = 16,capture_failures=True):
    nchan = chans.shape[0]
    mcnt_inc = nfft*2**12/nchan
    next_seqno = None
    mcnt_top = 0
    dset = []
    mcntoff = None
    last_mcnt_ovf = None
    seqnos = []
    nextseqnos = []
    chan0 = None
    for pnum,pkt in enumerate(plist):
        if len(pkt) != pkt_size:
            print "got packet size",len(pkt), "expected",pkt_size
            continue
        pidle,pidx,pstream,pchan,pmcnt = struct.unpack(hdr_fmt,pkt[:hdr_size])
        #print pmcnt
        if pstream != streamid:
            print "got stream id",pstream,"expected",streamid
            continue
        if next_seqno is None:
            mcnt_top = 0
            last_mcnt_ovf = pmcnt
        else:
            if pmcnt < mcnt_inc:
                if last_mcnt_ovf != pmcnt:
                    print "detected mcnt overflow",last_mcnt_ovf,pmcnt,pidx,next_seqno,(mcnt_top/2**32),pnum,mcntoff
                    mcnt_top += 2**32
                    last_mcnt_ovf = pmcnt
                else:
#                    print "continuation of previous mcnt overflow",pidx
                    pass
            else:
                last_mcnt_ovf = None
        chunkno,pmcntoff = divmod(pmcnt+mcnt_top,mcnt_inc)
        #print chunkno,pmcnt,pmcntoff,pidx
        seqno = (chunkno)*pkts_per_chunk + pidx
        #print seqno
        seqnos.append(seqno)
        nextseqnos.append(next_seqno)
        if next_seqno is None:
            chan0 = pchan
#            print "found first packet",seqno,pidx
            next_seqno = seqno
            mcntoff = pmcntoff
#            print pchan
        if mcntoff != pmcntoff:
            print "mcnt offset jumped. Was",mcntoff,"now",pmcntoff,"dropping.."
            continue
        if pchan != chan0:
            print "warning! channel id changed from",chan0,"to",pchan
        if seqno - next_seqno < 0:
            print "seqno diff",(seqno-next_seqno),seqno,next_seqno
            continue # trying to go backwards
        if seqno == next_seqno:
            dset.append(pkt[hdr_size:])
            next_seqno += 1
        else:
            print "sequence number skip, expected:",next_seqno,"got",seqno,"inserting",(seqno-next_seqno),"null packets",pnum,pidx
            if capture_failures: #seqno-next_seqno == 32768:
                print "caught special case, writing to disk"
                fname = time.strftime("udp_skip_%Y-%m-%d_%H%M%S.pkl")
                fh = open(fname,'w')
                cPickle.dump(dict(plist=plist,dset=dset,pnum=pnum,pkt=pkt,streamid=streamid,chans=chans,nfft=nfft),
                             fh,cPickle.HIGHEST_PROTOCOL)
                fh.close()
                print "wrote data to:",fname
            for k in range(seqno - next_seqno+1):
                dset.append(null_pkt)
                next_seqno += 1
    dset = ''.join(dset)
    ns = (len(dset)//(4*nchan))
    dset = dset[:ns*(4*nchan)]
    darray = np.fromstring(dset,dtype='>i2').astype('float32').view('complex64')
    darray.shape = (ns,nchan)
    shift = np.flatnonzero(chans==(chan0))[0] - (nchan-1)
    darray = np.roll(darray,shift,axis=1)
    return darray,np.array(seqnos)<|endoftext|>


<commit_before># sublimelint.py
# SublimeLint is a code linting support framework for Sublime Text 2
#
# Project: https://github.com/lunixbochs/sublimelint
# License: MIT

import sublime
import sublime_plugin

import os
import time
import thread

from lint.modules import Modules
from lint.linter import Linter
from lint.highlight import HighlightSet
import lint.persist as persist

default_user_settings = '''{
	"debug": false
}
'''
cwd = os.getcwd()

class SublimeLint(sublime_plugin.EventListener):
	def __init__(self, *args, **kwargs):
		sublime_plugin.EventListener.__init__(self, *args, **kwargs)

		self.settings = sublime.load_settings('SublimeLint.sublime-settings')
		self.settings.add_on_change('lint-settings', self.update_settings)
		self.update_settings()

		self.loaded = set()
		self.linted = set()
		self.modules = Modules(cwd, 'languages').load_all()
		self.last_syntax = {}
		persist.queue.start(self.lint)

		# this gives us a chance to lint the active view on fresh install
		window = sublime.active_window()
		if window:
			sublime.set_timeout(
				lambda: self.on_activated(window.active_view()), 100
			)

		self.start = time.time()

	def update_settings(self):
		pass

	def lint(self, view_id):
		view = Linter.get_view(view_id)

		sections = {}
		for sel, _ in Linter.get_selectors(view_id):
			sections[sel] = []
			for result in view.find_by_selector(sel):
				sections[sel].append(
					(view.rowcol(result.a)[0], result.a, result.b)
				)

		if view is not None:
			filename = view.file_name()
			persist.debug('SublimeLint: running on `%s`' % os.path.split(filename or 'untitled')[1])
			code = Linter.text(view)
			thread.start_new_thread(Linter.lint_view, (view_id, filename, code, sections, self.finish))

	def finish(self, view, linters):
		errors = {}
		highlights = HighlightSet()

		for linter in linters:
			if linter.highlight:
				highlights.add(linter.highlight)

			if linter.errors:
				errors.update(linter.errors)

		highlights.clear(view)
		highlights.draw(view)
		persist.errors[view.id()] = errors
		self.on_selection_modified(view)

	# helpers

	def hit(self, view):
		self.linted.add(view.id())
		if view.size() == 0:
			for l in Linter.get_linters(view.id()):
				l.clear()

			return

		persist.queue.hit(view)

	def check_syntax(self, view, lint=False):
		vid = view.id()
		syntax = view.settings().get('syntax')

		# syntax either has never been set or just changed
		if not vid in self.last_syntax or self.last_syntax[vid] != syntax:
			self.last_syntax[vid] = syntax

			# assign a linter, then maybe trigger a lint if we get one
			if Linter.assign(view) and lint:
				self.hit(view)

	# callins
	def on_modified(self, view):
		self.check_syntax(view)
		self.hit(view)

	def on_load(self, view):
		self.on_new(view)

	def on_activated(self, view):
		sublime.set_timeout(lambda: self.check_syntax(view, True), 50)

		view_id = view.id()
		if not view_id in self.linted:
			if not view_id in self.loaded:
				# it seems on_activated can be called before loaded on first start
				if time.time() - self.start < 5: return
				self.on_new(view)

			self.hit(view)

	def on_new(self, view):
		# handle new user preferences file
		if view.file_name() and os.path.split(view.file_name())[1] == 'SublimeLint.sublime-settings':
			if view.size() == 0:
				edit = view.begin_edit()
				view.insert(edit, 0, default_user_settings)
				view.end_edit(edit)

		vid = view.id()
		self.loaded.add(vid)
		self.last_syntax[vid] = view.settings().get('syntax')
		Linter.assign(view)

	def on_post_save(self, view):
		# this will reload submodules if they are saved with sublime text
		for name, module in self.modules.modules.items():
			if os.name == 'posix' and (
				os.stat(module.__file__).st_ino == os.stat(view.file_name()).st_ino
			) or module.__file__ == view.file_name():
				self.modules.reload(module)
				Linter.reload(name)
				break

		# linting here doesn't matter, because we lint on load and on modify
		# self.hit(view)

	def on_selection_modified(self, view):
		vid = view.id()
		lineno = view.rowcol(view.sel()[0].end())[0]

		view.erase_status('sublimelint')
		if vid in persist.errors:
			errors = persist.errors[vid]
			if errors:
				plural = 's' if len(errors) > 1 else ''
				if lineno in errors:
					status = ''
					if plural:
						num = sorted(list(errors)).index(lineno) + 1
						status += '%i/%i errors: ' % (num, len(errors))

					# sublime statusbar can't hold unicode
					status += '; '.join(errors[lineno]).encode('ascii', 'replace')
				else:
					status = '%i error%s' % (len(errors), plural)

				view.set_status('sublimelint', status)

		persist.queue.delay()
<commit_msg>delay on_selection_modified to prevent sublimetext error<commit_after># sublimelint.py
# SublimeLint is a code linting support framework for Sublime Text 2
#
# Project: https://github.com/lunixbochs/sublimelint
# License: MIT

import sublime
import sublime_plugin

import os
import time
import thread

from lint.modules import Modules
from lint.linter import Linter
from lint.highlight import HighlightSet
import lint.persist as persist

default_user_settings = '''{
	"debug": false
}
'''
cwd = os.getcwd()

class SublimeLint(sublime_plugin.EventListener):
	def __init__(self, *args, **kwargs):
		sublime_plugin.EventListener.__init__(self, *args, **kwargs)

		self.settings = sublime.load_settings('SublimeLint.sublime-settings')
		self.settings.add_on_change('lint-settings', self.update_settings)
		self.update_settings()

		self.loaded = set()
		self.linted = set()
		self.modules = Modules(cwd, 'languages').load_all()
		self.last_syntax = {}
		persist.queue.start(self.lint)

		# this gives us a chance to lint the active view on fresh install
		window = sublime.active_window()
		if window:
			sublime.set_timeout(
				lambda: self.on_activated(window.active_view()), 100
			)

		self.start = time.time()

	def update_settings(self):
		pass

	def lint(self, view_id):
		view = Linter.get_view(view_id)

		sections = {}
		for sel, _ in Linter.get_selectors(view_id):
			sections[sel] = []
			for result in view.find_by_selector(sel):
				sections[sel].append(
					(view.rowcol(result.a)[0], result.a, result.b)
				)

		if view is not None:
			filename = view.file_name()
			persist.debug('SublimeLint: running on `%s`' % os.path.split(filename or 'untitled')[1])
			code = Linter.text(view)
			thread.start_new_thread(Linter.lint_view, (view_id, filename, code, sections, self.finish))

	def finish(self, view, linters):
		errors = {}
		highlights = HighlightSet()

		for linter in linters:
			if linter.highlight:
				highlights.add(linter.highlight)

			if linter.errors:
				errors.update(linter.errors)

		highlights.clear(view)
		highlights.draw(view)
		persist.errors[view.id()] = errors
		self.on_selection_modified(view)

	# helpers

	def hit(self, view):
		self.linted.add(view.id())
		if view.size() == 0:
			for l in Linter.get_linters(view.id()):
				l.clear()

			return

		persist.queue.hit(view)

	def check_syntax(self, view, lint=False):
		vid = view.id()
		syntax = view.settings().get('syntax')

		# syntax either has never been set or just changed
		if not vid in self.last_syntax or self.last_syntax[vid] != syntax:
			self.last_syntax[vid] = syntax

			# assign a linter, then maybe trigger a lint if we get one
			if Linter.assign(view) and lint:
				self.hit(view)

	# callins
	def on_modified(self, view):
		self.check_syntax(view)
		self.hit(view)

	def on_load(self, view):
		self.on_new(view)

	def on_activated(self, view):
		sublime.set_timeout(lambda: self.check_syntax(view, True), 50)

		view_id = view.id()
		if not view_id in self.linted:
			if not view_id in self.loaded:
				# it seems on_activated can be called before loaded on first start
				if time.time() - self.start < 5: return
				self.on_new(view)

			self.hit(view)

	def on_new(self, view):
		# handle new user preferences file
		if view.file_name() and os.path.split(view.file_name())[1] == 'SublimeLint.sublime-settings':
			if view.size() == 0:
				edit = view.begin_edit()
				view.insert(edit, 0, default_user_settings)
				view.end_edit(edit)

		vid = view.id()
		self.loaded.add(vid)
		self.last_syntax[vid] = view.settings().get('syntax')
		Linter.assign(view)

	def on_post_save(self, view):
		# this will reload submodules if they are saved with sublime text
		for name, module in self.modules.modules.items():
			if os.name == 'posix' and (
				os.stat(module.__file__).st_ino == os.stat(view.file_name()).st_ino
			) or module.__file__ == view.file_name():
				self.modules.reload(module)
				Linter.reload(name)
				break

		# linting here doesn't matter, because we lint on load and on modify
		# self.hit(view)

	def on_selection_modified(self, view):
		def defer():
			vid = view.id()
			lineno = view.rowcol(view.sel()[0].end())[0]

			view.erase_status('sublimelint')
			if vid in persist.errors:
				errors = persist.errors[vid]
				if errors:
					plural = 's' if len(errors) > 1 else ''
					if lineno in errors:
						status = ''
						if plural:
							num = sorted(list(errors)).index(lineno) + 1
							status += '%i/%i errors: ' % (num, len(errors))

						# sublime statusbar can't hold unicode
						status += '; '.join(errors[lineno]).encode('ascii', 'replace')
					else:
						status = '%i error%s' % (len(errors), plural)

					view.set_status('sublimelint', status)

			persist.queue.delay()

		sublime.set_timeout(defer, 1);
<|endoftext|>