<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-

import csv
import sys
from apiclient.http import MediaFileUpload
import apiclient.errors
import urllib
import requests
import json
import pprint
import re

import logging
logger = logging.getLogger("gdput")
# logger.setLevel(logging.ERROR)

import random
import os

import json

from gdcmdtools.base import GDBase
from gdcmdtools.perm import GDPerm
from gdcmdtools.auth import GDAuth

DICT_OF_CONVERTIBLE_FILE_TYPE = {
    'raw': [
        "Raw file",
        []],
    'ss': [
        "Spreadsheet",
        ['xls', 'xlsx', 'ods', 'csv', 'tsv', 'tab']],
    'ft': [
        "Fusion Table",
        ['csv']],
    'pt': [
        "Presentation",
        ['ppt', 'pps', 'pptx']],
    'dr': [
        "Drawing",
        ['wmf']],
    'ocr': [
        "OCR",
        ['jpg', 'git', 'png', 'pdf']],
    'doc': [
        "Document",
        ['doc', 'docx', 'html', 'htm', 'txt', 'rtf']],
    'gas': [
        "GAS project",
        ['json']],
}

# FIXME: naming
class GDPut:

    def __init__(self, **kwargs):

        logger.debug(kwargs)

        for key, value in kwargs.items():
            setattr(self, key, value)

        self.ft_headers = None

        self.csv_latlng_suffix = "_latlng_%04x.csv" % random.getrandbits(16)

        # base
        auth = GDAuth()
        creds = auth.get_credentials()
        self.auth_user = creds.id_token.get("email", None)
        if creds is None:
            raise Exception("Failed to retrieve credentials")
        self.http = auth.get_authorized_http()

        base = GDBase()

        self.file_id = None
        if self.replace_id:
            self.file_id = base.get_id_from_url(self.replace_id)

        self.service = base.get_drive_service(self.http)
        self.root = base.get_root()

        # ft service
        if self.target_type == "ft":
            self.ft_service = base.get_ft_service(self.http)

    def if_folder_writable(self):
        try:
            permissions = self.service.permissions().list(fileId=self.folder_id).execute()
            valid_roles = ["writer", "owner"]
            logger.debug(pprint.pformat(permissions))
            for p in permissions["items"]:
                email = p.get("emailAddress", None).lower()
                role = p.get("role", None).lower()

                logger.debug("email: %s, role: %s" % (email, role))

                if(email == self.auth_user) and (role in valid_roles):
                    return True
        except:
            return False

        return False

    def run(self):
        # check folder_id
        if self.folder_id:
            if self.if_folder_writable() == False:
                raise Exception(
                    "folder_id doesn't exist or insufficient permission: %s" %
                    self.folder_id)

        try:
            result = getattr(self, self.target_type + "_put")()
        except AttributeError as e:
            logger.error(e)
            raise
        except Exception as e:
            logger.error(e)
            raise

        return result

    def get_current_user(self):
        pass

    def raw_put(self):
        return self.generic_put(False)

    def check_gas(self):
        # have "id",
        # have at least one file
        # the file should have type, id, name items.
        with open(self.source_file, "rb") as f:
            jsons = json.loads(f.read())

            if_ok = False

            if not isinstance(jsons, dict):
                return False

            if not self.gas_new:
                assert(jsons.get("id"))
                self.file_id = jsons.get("id")

            if(len(jsons["files"]) > 0):
                for j in jsons["files"]:
                    if j["type"] and j["name"]:
                        if_ok = True
                    else:
                        return False

        return if_ok

    def gas_pack(self):
        map_type_ext = {"server_js": "js", "html": "html"}
        json_packed = {}
        try:
            with open(self.source_file, "rb") as fr1:
                jsons = json.loads(fr1.read())

                path = os.path.split(self.source_file)[0]
                for j in jsons["files"]:
                    file_name = os.path.join(path, "%s.%s" % (
                        j["name"], map_type_ext[j["type"]]))
                    with open(file_name, "rb") as fr2:
                        file_content = fr2.read()

                    j["source"] = file_content

                new_json = "%s.packed" % self.source_file
                with open(new_json, "wb+") as fw:
                    fw.write(json.dumps(jsons, indent=4))
        except:
            return False
        else:
            return True

    def gas_put(self):
        if not self.check_gas():
            raise Exception(
                "The target file is not a GAS project json, if you like to raw-upload a json, try '-t raw'")

        if not self.gas_pack():
            raise Exception("Failed to pack the GAS project files")

        return self.generic_put(True, file_name="%s.packed" % self.source_file)

    def check_csv(self):
        self.csv_delimiter = ','
        with open(self.source_file, 'rb') as csv_file:
            try:
                dialect = csv.Sniffer().sniff(csv_file.readline())
                if dialect.delimiter == self.csv_delimiter:
                    return True
            except:
                logger.error("Failed at calling csv.Sniffer().sniff)")

        return False

    def csv_save_latlng(self):
        rows = []
        # read csv header
        with open(self.source_file, 'rb') as csv_file:
            csv_reader = csv.reader(csv_file)
            self.ft_headers = csv_reader.next()

            if self.ft_location_column and self.ft_latlng_column:
                self.ft_headers.append(self.ft_latlng_column)
                rows.append(self.ft_headers)

            # TODO: check if location in the list
            index_latlng = self.ft_headers.index(self.ft_latlng_column)
            index_location = self.ft_headers.index(self.ft_location_column)

            for row in csv_reader:
                latlng = self.ft_geocoding(row[index_location])
                row.insert(index_latlng, latlng)
                rows.append(row)

        # logger.debug(rows)

        # save new file
        csv_file_dir = os.path.dirname(self.source_file)
        csv_file_basename = os.path.basename(self.source_file)
        csv_file_noextension = os.path.splitext(csv_file_basename)[0]
        latlng_file = os.path.join(
            csv_file_dir,
            csv_file_noextension +
            self.csv_latlng_suffix)
        # write csv header with latlng
        with open(latlng_file, 'wb+') as csv_file:
            csv_writer = csv.writer(csv_file, lineterminator='\n')
            csv_writer.writerows(rows)

        return latlng_file

    def ss_put(self):
        if not self.check_csv():
            raise Exception(
                "The delimiter of the source csv file is not '%s'" %
                self.csv_delimiter)

        return self.generic_put(True)

    def user_define_column(self, cols, csv_column_define):
        return_cols = []
        for (col, col_type) in zip(cols, self.csv_column_define):
            d = {"type": col_type, "name": col}
            return_cols.append(d)

        return return_cols

    # read csv and convert to the fusion table
    def create_ft(self, target_file):
        table = {
            "name": self.title,
            "description": self.description,
            "isExportable": True,    # FIXME
            "columns": []
        }

        with open(target_file, 'rb') as csv_file:
            csv_reader = csv.reader(csv_file)
            cols = csv_reader.next()

            self.ft_headers = cols

            # FIXME:
            if self.ft_location_column and self.ft_latlng_column:
                if self.ft_location_column not in cols:
                    raise Exception(
                        "Column %s not found in the csv file" %
                        self.ft_location_column)

                if self.csv_column_define is None:
                    for c in cols:
                        if c == self.ft_latlng_column:
                            d = {"type": "LOCATION"}
                        else:
                            d = {"type": "STRING"}
                        d["name"] = c

                        table["columns"].append(d)
                else:
                    table["columns"] = self.user_define_column(
                        cols, self.csv_column_define)

            elif self.ft_location_column and not self.ft_latlng_column:
                if self.ft_location_column not in cols:
                    raise Exception(
                        "Column %s not found in the csv file" %
                        self.ft_location_column)
                if self.csv_column_define is None:
                    for c in cols:
                        if c == self.ft_location_column:
                            d = {"type": "LOCATION"}
                        else:
                            d = {"type": "STRING"}
                        d["name"] = c

                        table["columns"].append(d)
                else:
                    table["columns"] = self.user_define_column(
                        cols, self.csv_column_define)

            else:

                if self.csv_column_define is None:
                    for c in cols:
                        d = {"type": "STRING", "name": c}
                        table["columns"].append(d)
                else:
                    table["columns"] = self.user_define_column(
                        cols, self.csv_column_define)

        return table

    def ft_put(self):
        if not self.check_csv():
            raise Exception(
                "The delimiter of the source csv file is not '%s'" %
                self.csv_delimiter)

        # save new csv file with latlng data
        if self.ft_location_column and self.ft_latlng_column:
            target_file = self.csv_save_latlng()
            table = self.create_ft(target_file)
        else:
            table = self.create_ft(self.source_file)
        #logger.debug('body=%s' % body)

        # table columns are created, get tableId
        service_response = self.ft_service.table().insert(body=table).execute()
        #logger.debug("service_response=%s" % service_response)
        table_id = service_response["tableId"]

        # move to target folder
        if self.folder_id is not None:
            new_parent = {'id': self.folder_id}

            try:
                self.service.parents().insert(fileId=table_id, body=new_parent).execute()
            except apiclient.errors.HttpError as error:
                raise Exception('An error occurred: %s' % error)

            # remove from root folder
            try:
                self.service.parents().delete(fileId=table_id, parentId=self.root).execute()
            except apiclient.errors.HttpError as error:
                raise Exception('Atable_idn error occurred: %s' % error)

        if self.ft_location_column and self.ft_latlng_column:
            url = self.ft_put_body(table_id, target_file)
        else:
            url = self.ft_put_body(table_id, self.source_file)

        if self.permission is not None:
            GDPerm.insert(
                self.service,
                service_response['tableId'],
                self.permission)

        ft_url = "https://www.google.com/fusiontables/data?docid=%s" % table_id

        return service_response

    def ft_put_body(self, table_id, target_file):
        params = urllib.urlencode({'isStrict': "false"})
        URI = "https://www.googleapis.com/upload/fusiontables/v1/tables/%s/import?%s" % (
            table_id, params)
        METHOD = "POST"

        with open(target_file) as ft_file:
            # get the rows
            # ft_file.next()
            rows = ft_file.read()
            i_newline = rows.index('\n') + 1
            rows = rows[i_newline:]
            # weird issue here: the URI should be encoded with UTF-8 if body is
            # UTF-8 too.
            utf8_body = rows.decode('utf-8').encode('utf-8')
            # logger.debug(utf8_body)
            try:
                response, content = self.http.request(
                    URI.encode('utf-8'), METHOD, body=utf8_body)
            except:
                raise Exception('Failed at calling http.request(%s, %s, %s)'
                                % (URI.encode('utf-8'), METHOD, utf8_body))

            content = json.loads(content)
            # logger.debug(content)

    @staticmethod
    def ft_geocoding(address):
        GEOCODING_URL = "http://maps.googleapis.com/maps/api/geocode/json"

        params = {'address': address, 'sensor': 'false'}
        response = requests.get(GEOCODING_URL, params=params)
        response_json = (response.json())

        # FIXME
        lat = response_json["results"][0]["geometry"]["location"]["lat"]
        lng = response_json["results"][0]["geometry"]["location"]["lng"]

        latlng = str(lat) + "," + str(lng)

        return latlng

    def generic_put(self, if_convert, file_name=None):
        if(file_name):
            self.source_file = file_name

        media_body = MediaFileUpload(
            self.source_file,
            mimetype=self.mime_type,
            resumable=True)

        if self.folder_id is None:
            parents = []
        else:
            parents = [{
                "kind": "drive#fileLink",
                "id": self.folder_id}]

        body = {
            'title': self.title,
            'description': self.description,
            'mimeType': self.mime_type,
            'parents': parents}

        if(self.file_id):
            if(self.gas_new):
                request = self.service.files().insert(
                    body=body, media_body=media_body, convert=if_convert)
            else:
                request = self.service.files().update(body=body, media_body=media_body,
                                                      convert=if_convert, fileId=self.file_id)
        else:
            request = self.service.files().insert(
                body=body, media_body=media_body, convert=if_convert)

        service_response = None

        print "Uploading file: %s" % self.source_file
        while service_response is None:
            status, service_response = request.next_chunk(num_retries=10)
            if status:
                sys.stdout.write(
                    "\rCompleted: %.2f%%" %
                    (status.progress() * 100))
                sys.stdout.flush()
            else:
                sys.stdout.write("\rCompleted!%s\n" % (" " * 10))
                sys.stdout.flush()

                if self.permission is not None:
                    GDPerm.insert(
                        self.service,
                        service_response['id'],
                        self.permission)

                return service_response

    def pt_put(self):
        return self.generic_put(True)

    def dr_put(self):
        return self.generic_put(True)

    def ocr_put(self):
        return self.generic_put(True)

    def doc_put(self):
        return self.generic_put(True)
        #raise Exception("this function is not supported yet")
<commit_msg>fixed put.py set permission<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-

import csv
import sys
from apiclient.http import MediaFileUpload
import apiclient.errors
import urllib
import requests
import json
import pprint
import re

import logging
logger = logging.getLogger("gdput")
# logger.setLevel(logging.ERROR)

import random
import os

import json

from gdcmdtools.base import GDBase
from gdcmdtools.perm import GDPerm
from gdcmdtools.auth import GDAuth

DICT_OF_CONVERTIBLE_FILE_TYPE = {
    'raw': [
        "Raw file",
        []],
    'ss': [
        "Spreadsheet",
        ['xls', 'xlsx', 'ods', 'csv', 'tsv', 'tab']],
    'ft': [
        "Fusion Table",
        ['csv']],
    'pt': [
        "Presentation",
        ['ppt', 'pps', 'pptx']],
    'dr': [
        "Drawing",
        ['wmf']],
    'ocr': [
        "OCR",
        ['jpg', 'git', 'png', 'pdf']],
    'doc': [
        "Document",
        ['doc', 'docx', 'html', 'htm', 'txt', 'rtf']],
    'gas': [
        "GAS project",
        ['json']],
}

# FIXME: naming
class GDPut:

    def __init__(self, **kwargs):

        logger.debug(kwargs)

        for key, value in kwargs.items():
            setattr(self, key, value)

        self.ft_headers = None

        self.csv_latlng_suffix = "_latlng_%04x.csv" % random.getrandbits(16)

        # base
        auth = GDAuth()
        creds = auth.get_credentials()
        self.auth_user = creds.id_token.get("email", None)
        if creds is None:
            raise Exception("Failed to retrieve credentials")
        self.http = auth.get_authorized_http()

        base = GDBase()

        self.file_id = None
        if self.replace_id:
            self.file_id = base.get_id_from_url(self.replace_id)

        self.service = base.get_drive_service(self.http)
        self.root = base.get_root()

        # ft service
        if self.target_type == "ft":
            self.ft_service = base.get_ft_service(self.http)

    def if_folder_writable(self):
        try:
            permissions = self.service.permissions().list(fileId=self.folder_id).execute()
            valid_roles = ["writer", "owner"]
            logger.debug(pprint.pformat(permissions))
            for p in permissions["items"]:
                email = p.get("emailAddress", None).lower()
                role = p.get("role", None).lower()

                logger.debug("email: %s, role: %s" % (email, role))

                if(email == self.auth_user) and (role in valid_roles):
                    return True
        except:
            return False

        return False

    def run(self):
        # check folder_id
        if self.folder_id:
            if self.if_folder_writable() == False:
                raise Exception(
                    "folder_id doesn't exist or insufficient permission: %s" %
                    self.folder_id)

        try:
            result = getattr(self, self.target_type + "_put")()
        except AttributeError as e:
            logger.error(e)
            raise
        except Exception as e:
            logger.error(e)
            raise

        return result

    def get_current_user(self):
        pass

    def raw_put(self):
        return self.generic_put(False)

    def check_gas(self):
        # have "id",
        # have at least one file
        # the file should have type, id, name items.
        with open(self.source_file, "rb") as f:
            jsons = json.loads(f.read())

            if_ok = False

            if not isinstance(jsons, dict):
                return False

            if not self.gas_new:
                assert(jsons.get("id"))
                self.file_id = jsons.get("id")

            if(len(jsons["files"]) > 0):
                for j in jsons["files"]:
                    if j["type"] and j["name"]:
                        if_ok = True
                    else:
                        return False

        return if_ok

    def gas_pack(self):
        map_type_ext = {"server_js": "js", "html": "html"}
        json_packed = {}
        try:
            with open(self.source_file, "rb") as fr1:
                jsons = json.loads(fr1.read())

                path = os.path.split(self.source_file)[0]
                for j in jsons["files"]:
                    file_name = os.path.join(path, "%s.%s" % (
                        j["name"], map_type_ext[j["type"]]))
                    with open(file_name, "rb") as fr2:
                        file_content = fr2.read()

                    j["source"] = file_content

                new_json = "%s.packed" % self.source_file
                with open(new_json, "wb+") as fw:
                    fw.write(json.dumps(jsons, indent=4))
        except:
            return False
        else:
            return True

    def gas_put(self):
        if not self.check_gas():
            raise Exception(
                "The target file is not a GAS project json, if you like to raw-upload a json, try '-t raw'")

        if not self.gas_pack():
            raise Exception("Failed to pack the GAS project files")

        return self.generic_put(True, file_name="%s.packed" % self.source_file)

    def check_csv(self):
        self.csv_delimiter = ','
        with open(self.source_file, 'rb') as csv_file:
            try:
                dialect = csv.Sniffer().sniff(csv_file.readline())
                if dialect.delimiter == self.csv_delimiter:
                    return True
            except:
                logger.error("Failed at calling csv.Sniffer().sniff)")

        return False

    def csv_save_latlng(self):
        rows = []
        # read csv header
        with open(self.source_file, 'rb') as csv_file:
            csv_reader = csv.reader(csv_file)
            self.ft_headers = csv_reader.next()

            if self.ft_location_column and self.ft_latlng_column:
                self.ft_headers.append(self.ft_latlng_column)
                rows.append(self.ft_headers)

            # TODO: check if location in the list
            index_latlng = self.ft_headers.index(self.ft_latlng_column)
            index_location = self.ft_headers.index(self.ft_location_column)

            for row in csv_reader:
                latlng = self.ft_geocoding(row[index_location])
                row.insert(index_latlng, latlng)
                rows.append(row)

        # logger.debug(rows)

        # save new file
        csv_file_dir = os.path.dirname(self.source_file)
        csv_file_basename = os.path.basename(self.source_file)
        csv_file_noextension = os.path.splitext(csv_file_basename)[0]
        latlng_file = os.path.join(
            csv_file_dir,
            csv_file_noextension +
            self.csv_latlng_suffix)
        # write csv header with latlng
        with open(latlng_file, 'wb+') as csv_file:
            csv_writer = csv.writer(csv_file, lineterminator='\n')
            csv_writer.writerows(rows)

        return latlng_file

    def ss_put(self):
        if not self.check_csv():
            raise Exception(
                "The delimiter of the source csv file is not '%s'" %
                self.csv_delimiter)

        return self.generic_put(True)

    def user_define_column(self, cols, csv_column_define):
        return_cols = []
        for (col, col_type) in zip(cols, self.csv_column_define):
            d = {"type": col_type, "name": col}
            return_cols.append(d)

        return return_cols

    # read csv and convert to the fusion table
    def create_ft(self, target_file):
        table = {
            "name": self.title,
            "description": self.description,
            "isExportable": True,    # FIXME
            "columns": []
        }

        with open(target_file, 'rb') as csv_file:
            csv_reader = csv.reader(csv_file)
            cols = csv_reader.next()

            self.ft_headers = cols

            # FIXME:
            if self.ft_location_column and self.ft_latlng_column:
                if self.ft_location_column not in cols:
                    raise Exception(
                        "Column %s not found in the csv file" %
                        self.ft_location_column)

                if self.csv_column_define is None:
                    for c in cols:
                        if c == self.ft_latlng_column:
                            d = {"type": "LOCATION"}
                        else:
                            d = {"type": "STRING"}
                        d["name"] = c

                        table["columns"].append(d)
                else:
                    table["columns"] = self.user_define_column(
                        cols, self.csv_column_define)

            elif self.ft_location_column and not self.ft_latlng_column:
                if self.ft_location_column not in cols:
                    raise Exception(
                        "Column %s not found in the csv file" %
                        self.ft_location_column)
                if self.csv_column_define is None:
                    for c in cols:
                        if c == self.ft_location_column:
                            d = {"type": "LOCATION"}
                        else:
                            d = {"type": "STRING"}
                        d["name"] = c

                        table["columns"].append(d)
                else:
                    table["columns"] = self.user_define_column(
                        cols, self.csv_column_define)

            else:

                if self.csv_column_define is None:
                    for c in cols:
                        d = {"type": "STRING", "name": c}
                        table["columns"].append(d)
                else:
                    table["columns"] = self.user_define_column(
                        cols, self.csv_column_define)

        return table

    def ft_put(self):
        if not self.check_csv():
            raise Exception(
                "The delimiter of the source csv file is not '%s'" %
                self.csv_delimiter)

        # save new csv file with latlng data
        if self.ft_location_column and self.ft_latlng_column:
            target_file = self.csv_save_latlng()
            table = self.create_ft(target_file)
        else:
            table = self.create_ft(self.source_file)
        #logger.debug('body=%s' % body)

        # table columns are created, get tableId
        service_response = self.ft_service.table().insert(body=table).execute()
        #logger.debug("service_response=%s" % service_response)
        table_id = service_response["tableId"]

        # move to target folder
        if self.folder_id is not None:
            new_parent = {'id': self.folder_id}

            try:
                self.service.parents().insert(fileId=table_id, body=new_parent).execute()
            except apiclient.errors.HttpError as error:
                raise Exception('An error occurred: %s' % error)

            # remove from root folder
            try:
                self.service.parents().delete(fileId=table_id, parentId=self.root).execute()
            except apiclient.errors.HttpError as error:
                raise Exception('Atable_idn error occurred: %s' % error)

        if self.ft_location_column and self.ft_latlng_column:
            url = self.ft_put_body(table_id, target_file)
        else:
            url = self.ft_put_body(table_id, self.source_file)

        if self.permission is not None:
            GDPerm.insert(
                self.service,
                service_response['tableId'],
                self.permission)

        ft_url = "https://www.google.com/fusiontables/data?docid=%s" % table_id

        return service_response

    def ft_put_body(self, table_id, target_file):
        params = urllib.urlencode({'isStrict': "false"})
        URI = "https://www.googleapis.com/upload/fusiontables/v1/tables/%s/import?%s" % (
            table_id, params)
        METHOD = "POST"

        with open(target_file) as ft_file:
            # get the rows
            # ft_file.next()
            rows = ft_file.read()
            i_newline = rows.index('\n') + 1
            rows = rows[i_newline:]
            # weird issue here: the URI should be encoded with UTF-8 if body is
            # UTF-8 too.
            utf8_body = rows.decode('utf-8').encode('utf-8')
            # logger.debug(utf8_body)
            try:
                response, content = self.http.request(
                    URI.encode('utf-8'), METHOD, body=utf8_body)
            except:
                raise Exception('Failed at calling http.request(%s, %s, %s)'
                                % (URI.encode('utf-8'), METHOD, utf8_body))

            content = json.loads(content)
            # logger.debug(content)

    @staticmethod
    def ft_geocoding(address):
        GEOCODING_URL = "http://maps.googleapis.com/maps/api/geocode/json"

        params = {'address': address, 'sensor': 'false'}
        response = requests.get(GEOCODING_URL, params=params)
        response_json = (response.json())

        # FIXME
        lat = response_json["results"][0]["geometry"]["location"]["lat"]
        lng = response_json["results"][0]["geometry"]["location"]["lng"]

        latlng = str(lat) + "," + str(lng)

        return latlng

    def generic_put(self, if_convert, file_name=None):
        if(file_name):
            self.source_file = file_name

        media_body = MediaFileUpload(
            self.source_file,
            mimetype=self.mime_type,
            resumable=True)

        if self.folder_id is None:
            parents = []
        else:
            parents = [{
                "kind": "drive#fileLink",
                "id": self.folder_id}]

        body = {
            'title': self.title,
            'description': self.description,
            'mimeType': self.mime_type,
            'parents': parents}

        if(self.file_id):
            if(self.gas_new):
                request = self.service.files().insert(
                    body=body, media_body=media_body, convert=if_convert)
            else:
                request = self.service.files().update(body=body, media_body=media_body,
                                                      convert=if_convert, fileId=self.file_id)
        else:
            request = self.service.files().insert(
                body=body, media_body=media_body, convert=if_convert)

        service_response = None

        print "Uploading file: %s" % self.source_file
        while service_response is None:
            status, service_response = request.next_chunk(num_retries=10)
            if status:
                sys.stdout.write(
                    "\rCompleted: %.2f%%" %
                    (status.progress() * 100))
                sys.stdout.flush()
            else:
                sys.stdout.write("\rCompleted!%s\n" % (" " * 10))
                sys.stdout.flush()

                if self.permission is not None:
                    pass_action = {
                        "name":"insert",
                        "param":self.permission}

                    perm = GDPerm(service_response['id'], pass_action)
                    result = perm.run()
                    logger.debug(result)

                return service_response

    def pt_put(self):
        return self.generic_put(True)

    def dr_put(self):
        return self.generic_put(True)

    def ocr_put(self):
        return self.generic_put(True)

    def doc_put(self):
        return self.generic_put(True)
        #raise Exception("this function is not supported yet")
<|endoftext|>


<commit_before>from collections import OrderedDict, namedtuple
import functools

import numpy

from PyQt4 import QtGui
from PyQt4.QtGui import QTreeView, QStandardItemModel, QStandardItem
from PyQt4.QtCore import Qt

import Orange.data
import Orange.classification

from Orange.evaluation import testing, scoring

from Orange.widgets import widget, gui, settings

Input = namedtuple("Input", ["learner", "results", "stats"])

def classification_stats(results):
    stats = (CA(results),
             F1(results),
             Precision(results),
             Recall(results))
    if len(results.data.domain.class_var.values) == 2:
        return (AUC(results),) + stats
    return stats

classification_stats.headers = ["CA", "F1", "Precision", "Recall"]
classification_stats.headers_binary = ["AUC"]

def regression_stats(results):
    return (MSE(results),
            RMSE(results),
            MAE(results),
            R2(results))

regression_stats.headers = ["MSE", "RMSE", "MAE", "R2"]

def is_discrete(var):
    return isinstance(var, Orange.data.DiscreteVariable)

class OWTestLearners(widget.OWWidget):
    name = "Test Learners"
    description = ""
    icon = "icons/TestLearners1.svg"
    priority = 100

    inputs = [("Learner", Orange.classification.Fitter,
               "set_learner", widget.Multiple),
              ("Data", Orange.data.Table, "set_train_data", widget.Default),
              ("Test Data", Orange.data.Table, "set_test_data")]

    outputs = [("Evaluation Results", testing.Results)]

    #: Resampling/testing types
    KFold, LeaveOneOut, Bootstrap, TestOnTrain, TestOnTest = 0, 1, 2, 3, 4

    #: Selected resampling type
    resampling = settings.Setting(0)
    #: Number of folds for K-fold cross validation
    k_folds = settings.Setting(10)
    #: Number of repeats for bootstrap sampling
    n_repeat = settings.Setting(10)
    #: Bootstrap sampling p
    sample_p = settings.Setting(75)

    def __init__(self, parent=None):
        super().__init__(parent)

        self.train_data = None
        self.test_data = None

        #: An Ordered dictionary with current inputs and their testing
        #: results.
        self.learners = OrderedDict()

        sbox = gui.widgetBox(self.controlArea, "Sampling")
        rbox = gui.radioButtons(
            sbox, self, "resampling", callback=self._param_changed
        )
        gui.appendRadioButton(rbox, "Cross validation")
        ibox = gui.indentedBox(rbox)
        gui.spin(ibox, self, "k_folds", 2, 50, label="Number of folds:",
                 callback=self._param_changed)
        gui.appendRadioButton(rbox, "Leave one out")
        gui.appendRadioButton(rbox, "Random sampling")
        ibox = gui.indentedBox(rbox)
        gui.spin(ibox, self, "n_repeat", 2, 50, label="Repeat train/test",
                 callback=self._param_changed)
        gui.widgetLabel(ibox, "Relative training set size:")
        gui.hSlider(ibox, self, "sample_p", minValue=1, maxValue=100,
                    ticks=20, vertical=False,
                    callback=self._param_changed)

        gui.appendRadioButton(rbox, "Test on train data")
        gui.appendRadioButton(rbox, "Test on test data")

        rbox.layout().addSpacing(5)
        gui.button(rbox, self, "Apply", callback=self.apply)

        gui.rubber(self.controlArea)

        self.view = QTreeView(
            rootIsDecorated=False,
            uniformRowHeights=True,
            wordWrap=True,
            editTriggers=QTreeView.NoEditTriggers
        )

        self.result_model = QStandardItemModel()
        self.view.setModel(self.result_model)
        self._update_header()
        box = gui.widgetBox(self.mainArea, "Evaluation Results")
        box.layout().addWidget(self.view)

    def set_learner(self, learner, key):
        if key in self.learners and learner is None:
            del self.learners[key]
        else:
            self.learners[key] = Input(learner, None, ())
        self._update_stats_model()

    def set_train_data(self, data):
        self.train_data = data
        self._update_header()
        self._invalidate()

    def set_test_data(self, data):
        self.test_data = data
        if self.resampling == OWTestLearners.TestOnTest:
            self._invalidate()

    def handleNewSignals(self):
        self.update_results()
        self.commit()

    def _param_changed(self):
        self._invalidate()

    def update_results(self):
        self.warning(1, "")
        if self.train_data is None:
            return

        # items in need of an update
        items = [(key, input) for key, input in self.learners.items()
                 if input.results is None]
        learners = [input.learner for _, input in items]

        self.setStatusMessage("Running")
        if self.test_data is not None and \
                self.resampling != OWTestLearners.TestOnTest:
            self.warning(1, "Test data is present but unused. "
                            "Select 'Test on test data' to use it.")

        # TODO: Test each learner individually

        if self.resampling == OWTestLearners.KFold:
            results = testing.CrossValidation(
                self.train_data, learners, k=self.k_folds, store_data=True
            )
        elif self.resampling == OWTestLearners.LeaveOneOut:
            results = testing.LeaveOneOut(
                self.train_data, learners, store_data=True
            )
        elif self.resampling == OWTestLearners.Bootstrap:
            p = self.sample_p / 100.0
            results = testing.Bootstrap(
                self.train_data, learners, n_resamples=self.n_repeat, p=p,
                store_data=True
            )
        elif self.resampling == OWTestLearners.TestOnTrain:
            results = testing.TestOnTrainingData(
                self.train_data, learners, store_data=True
            )
        elif self.resampling == OWTestLearners.TestOnTest:
            if self.test_data is None:
                return
            results = testing.TestOnTestData(
                self.train_data, self.test_data, learners, store_data=True
            )
        else:
            assert False

        results = list(split_by_model(results))
        class_var = self.train_data.domain.class_var

        if is_discrete(class_var):
            test_stats = classification_stats
        else:
            test_stats = regression_stats

        self._update_header()

        stats = [test_stats(res) for res in results]
        for (key, input), res, stat in zip(items, results, stats):
            self.learners[key] = input._replace(results=res, stats=stat)

        self.setStatusMessage("")

        self._update_stats_model()

    def _update_header(self):
        headers = ["Method"]
        if self.train_data is not None:
            if is_discrete(self.train_data.domain.class_var):
                if len(self.train_data.domain.class_var.values) == 2:
                    headers.extend(classification_stats.headers_binary)
                headers.extend(classification_stats.headers)
            else:
                headers.extend(regression_stats.headers)
        for i in reversed(range(len(headers),
                                self.result_model.columnCount())):
            self.result_model.takeColumn(i)

        self.result_model.setHorizontalHeaderLabels(headers)

    def _update_stats_model(self):
        model = self.view.model()

        for r in reversed(range(model.rowCount())):
            model.takeRow(r)

        for input in self.learners.values():
            name = learner_name(input.learner)
            row = []
            head = QStandardItem()
            head.setData(name, Qt.DisplayRole)
            row.append(head)
            for stat in input.stats:
                item = QStandardItem()
                item.setData(float(stat[0]), Qt.DisplayRole)
                row.append(item)
            model.appendRow(row)

        self.view.resizeColumnToContents(0)

    def _invalidate(self, which=None):
        if which is None:
            which = self.learners.keys()

        all_keys = list(self.learners.keys())
        model = self.view.model()

        for key in which:
            self.learners[key] = \
                self.learners[key]._replace(results=None, stats=None)

            if key in self.learners:
                row = all_keys.index(key)
                for c in range(1, model.columnCount()):
                    item = model.item(row, c)
                    if item is not None:
                        item.setData(None, Qt.DisplayRole)

    def apply(self):
        self.update_results()
        self.commit()

    def commit(self):
        results = [val.results for val in self.learners.values()
                   if val.results is not None]
        if results:
            combined = results_merge(results)
            combined.fitter_names = [learner_name(val.learner)
                                     for val in self.learners.values()]
        else:
            combined = None
        self.send("Evaluation Results", combined)

def learner_name(learner):
    return getattr(learner, "name", type(learner).__name__)

def split_by_model(results):
    """
    Split evaluation results by models
    """
    data = results.data
    nmethods = len(results.predicted)
    for i in range(nmethods):
        res = testing.Results()
        res.data = data
        res.row_indices = results.row_indices
        res.actual = results.actual
        res.predicted = results.predicted[(i,), :]
        if results.probabilities is not None:
            res.probabilities = results.probabilities[(i,), :, :]

        if results.models:
            res.models = [mf[i] for mf in results.models]

        if results.folds:
            res.folds = results.folds

        yield res

def results_add_by_model(x, y):
    def is_empty(res):
        return (getattr(res, "models", None) is None
                and getattr(res, "row_indices", None) is None)

    if is_empty(x):
        return y
    elif is_empty(y):
        return x

    assert (x.row_indices == y.row_indices).all()
    assert (x.actual == y.actual).all()

    res = testing.Results()
    res.data = x.data
    res.row_indices = x.row_indices
    res.folds = x.folds
    res.actual = x.actual
    res.predicted = numpy.vstack((x.predicted, y.predicted))
    if x.probabilities is not None and y.probabilities is not None:
        res.probabilities = numpy.vstack((x.probabilities, y.probabilities))

    if x.models is not None:
        res.models = [xm + ym for xm, ym in zip(x.models, y.models)]
    return res

def results_merge(results):
    return functools.reduce(results_add_by_model, results, testing.Results())

import sklearn.metrics
import numpy as np

def _skl_metric(results, metric):
    return np.fromiter(
        (metric(results.actual, predicted)
         for predicted in results.predicted),
        dtype=np.float64, count=len(results.predicted))

def CA(results):
    return _skl_metric(results, sklearn.metrics.accuracy_score)

def Precision(results):
    return _skl_metric(results, sklearn.metrics.precision_score)

def Recall(results):
    return _skl_metric(results, sklearn.metrics.recall_score)

def AUC(results):
    return _skl_metric(results, sklearn.metrics.roc_auc_score)

def F1(results):
    return _skl_metric(results, sklearn.metrics.f1_score)

def MSE(results):
    return _skl_metric(results, sklearn.metrics.mean_squared_error)

def RMSE(results):
    return np.sqrt(MSE(results))

def MAE(results):
    return _skl_metric(results, sklearn.metrics.mean_absolute_error)

def R2(results):
    return _skl_metric(results, sklearn.metrics.r2_score)

def main():
    from Orange.classification import \
        logistic_regression as lr, naive_bayes as nb

    app = QtGui.QApplication([])
    data = Orange.data.Table("iris")
    w = OWTestLearners()
    w.show()
    w.set_train_data(data)
    w.set_test_data(data)
    w.set_learner(lr.LogisticRegressionLearner(), 1)
    w.set_learner(nb.BayesLearner(), 2)
    w.handleNewSignals()
    return app.exec_()

if __name__ == "__main__":
    import sys
    sys.exit(main())
<commit_msg>Fix number of decimals and layout of Test Learners table<commit_after>from collections import OrderedDict, namedtuple
import functools

import numpy

from PyQt4 import QtGui
from PyQt4.QtGui import QTreeView, QStandardItemModel, QStandardItem, \
    QHeaderView, QItemDelegate
from PyQt4.QtCore import Qt, QSize

import Orange.data
import Orange.classification

from Orange.evaluation import testing, scoring

from Orange.widgets import widget, gui, settings

Input = namedtuple("Input", ["learner", "results", "stats"])

def classification_stats(results):
    stats = (CA(results),
             F1(results),
             Precision(results),
             Recall(results))
    if len(results.data.domain.class_var.values) == 2:
        return (AUC(results),) + stats
    return stats

classification_stats.headers = ["CA", "F1", "Precision", "Recall"]
classification_stats.headers_binary = ["AUC"]

def regression_stats(results):
    return (MSE(results),
            RMSE(results),
            MAE(results),
            R2(results))

regression_stats.headers = ["MSE", "RMSE", "MAE", "R2"]

def is_discrete(var):
    return isinstance(var, Orange.data.DiscreteVariable)

class ItemDelegate(QItemDelegate):
    def sizeHint(self, *args):
        size = super().sizeHint(*args)
        return QSize(size.width(), size.height() + 6)

class OWTestLearners(widget.OWWidget):
    name = "Test Learners"
    description = ""
    icon = "icons/TestLearners1.svg"
    priority = 100

    inputs = [("Learner", Orange.classification.Fitter,
               "set_learner", widget.Multiple),
              ("Data", Orange.data.Table, "set_train_data", widget.Default),
              ("Test Data", Orange.data.Table, "set_test_data")]

    outputs = [("Evaluation Results", testing.Results)]

    #: Resampling/testing types
    KFold, LeaveOneOut, Bootstrap, TestOnTrain, TestOnTest = 0, 1, 2, 3, 4

    #: Selected resampling type
    resampling = settings.Setting(0)
    #: Number of folds for K-fold cross validation
    k_folds = settings.Setting(10)
    #: Number of repeats for bootstrap sampling
    n_repeat = settings.Setting(10)
    #: Bootstrap sampling p
    sample_p = settings.Setting(75)

    def __init__(self, parent=None):
        super().__init__(parent)

        self.train_data = None
        self.test_data = None

        #: An Ordered dictionary with current inputs and their testing
        #: results.
        self.learners = OrderedDict()

        sbox = gui.widgetBox(self.controlArea, "Sampling")
        rbox = gui.radioButtons(
            sbox, self, "resampling", callback=self._param_changed
        )
        gui.appendRadioButton(rbox, "Cross validation")
        ibox = gui.indentedBox(rbox)
        gui.spin(ibox, self, "k_folds", 2, 50, label="Number of folds:",
                 callback=self._param_changed)
        gui.appendRadioButton(rbox, "Leave one out")
        gui.appendRadioButton(rbox, "Random sampling")
        ibox = gui.indentedBox(rbox)
        gui.spin(ibox, self, "n_repeat", 2, 50, label="Repeat train/test",
                 callback=self._param_changed)
        gui.widgetLabel(ibox, "Relative training set size:")
        gui.hSlider(ibox, self, "sample_p", minValue=1, maxValue=100,
                    ticks=20, vertical=False,
                    callback=self._param_changed)

        gui.appendRadioButton(rbox, "Test on train data")
        gui.appendRadioButton(rbox, "Test on test data")

        rbox.layout().addSpacing(5)
        gui.button(rbox, self, "Apply", callback=self.apply)

        gui.rubber(self.controlArea)

        self.view = QTreeView(
            rootIsDecorated=False,
            uniformRowHeights=True,
            wordWrap=True,
            editTriggers=QTreeView.NoEditTriggers
        )
        header = self.view.header()
        header.setResizeMode(QHeaderView.ResizeToContents)
        header.setDefaultAlignment(Qt.AlignCenter)
        header.setStretchLastSection(False)

        self.result_model = QStandardItemModel()
        self.view.setModel(self.result_model)
        self.view.setItemDelegate(ItemDelegate())
        self._update_header()
        box = gui.widgetBox(self.mainArea, "Evaluation Results")
        box.layout().addWidget(self.view)

    def set_learner(self, learner, key):
        if key in self.learners and learner is None:
            del self.learners[key]
        else:
            self.learners[key] = Input(learner, None, ())
        self._update_stats_model()

    def set_train_data(self, data):
        self.train_data = data
        self._update_header()
        self._invalidate()

    def set_test_data(self, data):
        self.test_data = data
        if self.resampling == OWTestLearners.TestOnTest:
            self._invalidate()

    def handleNewSignals(self):
        self.update_results()
        self.commit()

    def _param_changed(self):
        self._invalidate()

    def update_results(self):
        self.warning(1, "")
        if self.train_data is None:
            return

        # items in need of an update
        items = [(key, input) for key, input in self.learners.items()
                 if input.results is None]
        learners = [input.learner for _, input in items]

        self.setStatusMessage("Running")
        if self.test_data is not None and \
                self.resampling != OWTestLearners.TestOnTest:
            self.warning(1, "Test data is present but unused. "
                            "Select 'Test on test data' to use it.")

        # TODO: Test each learner individually

        if self.resampling == OWTestLearners.KFold:
            results = testing.CrossValidation(
                self.train_data, learners, k=self.k_folds, store_data=True
            )
        elif self.resampling == OWTestLearners.LeaveOneOut:
            results = testing.LeaveOneOut(
                self.train_data, learners, store_data=True
            )
        elif self.resampling == OWTestLearners.Bootstrap:
            p = self.sample_p / 100.0
            results = testing.Bootstrap(
                self.train_data, learners, n_resamples=self.n_repeat, p=p,
                store_data=True
            )
        elif self.resampling == OWTestLearners.TestOnTrain:
            results = testing.TestOnTrainingData(
                self.train_data, learners, store_data=True
            )
        elif self.resampling == OWTestLearners.TestOnTest:
            if self.test_data is None:
                return
            results = testing.TestOnTestData(
                self.train_data, self.test_data, learners, store_data=True
            )
        else:
            assert False

        results = list(split_by_model(results))
        class_var = self.train_data.domain.class_var

        if is_discrete(class_var):
            test_stats = classification_stats
        else:
            test_stats = regression_stats

        self._update_header()

        stats = [test_stats(res) for res in results]
        for (key, input), res, stat in zip(items, results, stats):
            self.learners[key] = input._replace(results=res, stats=stat)

        self.setStatusMessage("")

        self._update_stats_model()

    def _update_header(self):
        headers = ["Method"]
        if self.train_data is not None:
            if is_discrete(self.train_data.domain.class_var):
                if len(self.train_data.domain.class_var.values) == 2:
                    headers.extend(classification_stats.headers_binary)
                headers.extend(classification_stats.headers)
            else:
                headers.extend(regression_stats.headers)
        for i in reversed(range(len(headers),
                                self.result_model.columnCount())):
            self.result_model.takeColumn(i)

        self.result_model.setHorizontalHeaderLabels(headers)

    def _update_stats_model(self):
        model = self.view.model()

        for r in reversed(range(model.rowCount())):
            model.takeRow(r)

        for input in self.learners.values():
            name = learner_name(input.learner)
            row = []
            head = QStandardItem()
            head.setData(name, Qt.DisplayRole)
            row.append(head)
            for stat in input.stats:
                item = QStandardItem()
                item.setData(" {:.3f} ".format(stat[0]), Qt.DisplayRole)
                row.append(item)
            model.appendRow(row)

    def _invalidate(self, which=None):
        if which is None:
            which = self.learners.keys()

        all_keys = list(self.learners.keys())
        model = self.view.model()

        for key in which:
            self.learners[key] = \
                self.learners[key]._replace(results=None, stats=None)

            if key in self.learners:
                row = all_keys.index(key)
                for c in range(1, model.columnCount()):
                    item = model.item(row, c)
                    if item is not None:
                        item.setData(None, Qt.DisplayRole)

    def apply(self):
        self.update_results()
        self.commit()

    def commit(self):
        results = [val.results for val in self.learners.values()
                   if val.results is not None]
        if results:
            combined = results_merge(results)
            combined.fitter_names = [learner_name(val.learner)
                                     for val in self.learners.values()]
        else:
            combined = None
        self.send("Evaluation Results", combined)

def learner_name(learner):
    return getattr(learner, "name", type(learner).__name__)

def split_by_model(results):
    """
    Split evaluation results by models
    """
    data = results.data
    nmethods = len(results.predicted)
    for i in range(nmethods):
        res = testing.Results()
        res.data = data
        res.row_indices = results.row_indices
        res.actual = results.actual
        res.predicted = results.predicted[(i,), :]
        if results.probabilities is not None:
            res.probabilities = results.probabilities[(i,), :, :]

        if results.models:
            res.models = [mf[i] for mf in results.models]

        if results.folds:
            res.folds = results.folds

        yield res

def results_add_by_model(x, y):
    def is_empty(res):
        return (getattr(res, "models", None) is None
                and getattr(res, "row_indices", None) is None)

    if is_empty(x):
        return y
    elif is_empty(y):
        return x

    assert (x.row_indices == y.row_indices).all()
    assert (x.actual == y.actual).all()

    res = testing.Results()
    res.data = x.data
    res.row_indices = x.row_indices
    res.folds = x.folds
    res.actual = x.actual
    res.predicted = numpy.vstack((x.predicted, y.predicted))
    if x.probabilities is not None and y.probabilities is not None:
        res.probabilities = numpy.vstack((x.probabilities, y.probabilities))

    if x.models is not None:
        res.models = [xm + ym for xm, ym in zip(x.models, y.models)]
    return res

def results_merge(results):
    return functools.reduce(results_add_by_model, results, testing.Results())

import sklearn.metrics
import numpy as np

def _skl_metric(results, metric):
    return np.fromiter(
        (metric(results.actual, predicted)
         for predicted in results.predicted),
        dtype=np.float64, count=len(results.predicted))

def CA(results):
    return _skl_metric(results, sklearn.metrics.accuracy_score)

def Precision(results):
    return _skl_metric(results, sklearn.metrics.precision_score)

def Recall(results):
    return _skl_metric(results, sklearn.metrics.recall_score)

def AUC(results):
    return _skl_metric(results, sklearn.metrics.roc_auc_score)

def F1(results):
    return _skl_metric(results, sklearn.metrics.f1_score)

def MSE(results):
    return _skl_metric(results, sklearn.metrics.mean_squared_error)

def RMSE(results):
    return np.sqrt(MSE(results))

def MAE(results):
    return _skl_metric(results, sklearn.metrics.mean_absolute_error)

def R2(results):
    return _skl_metric(results, sklearn.metrics.r2_score)

def main():
    from Orange.classification import \
        logistic_regression as lr, naive_bayes as nb

    app = QtGui.QApplication([])
    data = Orange.data.Table("iris")
    w = OWTestLearners()
    w.show()
    w.set_train_data(data)
    w.set_test_data(data)
    w.set_learner(lr.LogisticRegressionLearner(), 1)
    w.set_learner(nb.BayesLearner(), 2)
    w.handleNewSignals()
    return app.exec_()

if __name__ == "__main__":
    import sys
    sys.exit(main())
<|endoftext|>


<commit_before>import sys
import PyFBA
from PyFBA import lp, log_and_message

def create_stoichiometric_matrix(reactions_to_run, reactions, compounds, media, biomass_equation,
                                 uptake_secretion=None, verbose=False):
    """Given the reactions data and a list of RIDs to include, build a
    stoichiometric matrix and load that into the linear solver.

    The uptake and secretion reactions (sometimes called boundary reactions) are reactions that allow
    compounds to flow into the media and secreted compounds away from the cell. You can either provide these (e.g. if
    you are parsing an XML (SBML) file, or we will calculate them for you based on your media and reactions.

    We also take this opportunity to set the objective function (as it is a member of the SM).

    :param uptake_secretion: An optional hash of uptake and secretion reactions that should be added to the model
    :type uptake_secretion: a dict of Reactions
    :param compounds: a set of the compounds present
    :type compounds: set[PyFBA.metabolism.CompoundWithLocation]
    :param reactions: a dict of all the reactions
    :type reactions: dict[PyFBA.metabolism.Reaction]
    :param reactions_to_run: just the reaction ids that we want to include in our model
    :type reactions_to_run: set
    :param media: a set of compounds that would make up the media
    :type media: set
    :param biomass_equation: the biomass_equation equation as a Reaction object
    :type biomass_equation: metabolism.Reaction
    :param verbose: print more information
    :type verbose: bool
    :returns: Sorted lists of all the compounds and reactions in the model, and a revised reactions dict that includes the uptake and secretion reactions
    :rtype: list, list, dict

    """

    sm = {}  # the matrix
    reaction_cpds = set()  # all the cpds in the matrix

    # unify compounds from dicts and sets
    if type(compounds) == dict:
        log_and_message("create_stoichiometric_matrix is converting compounds from a dict to a set")
        compounds = set(compounds.values())

    # initialize the compounds set and the stoichiometric matrix with
    # everything in the media. The order of compounds is irrelevant
    for c in media:
        if c not in compounds:
            log_and_message(f"create_stoichiometric_matrix is adding compound {c} from media to compounds", stderr=True)
            compounds.add(c)
        if verbose and type(c) != PyFBA.metabolism.compound.CompoundWithLocation:
            log_and_message(f"create_stoichiometric_matrix is parsing the media, {c} is a {type(c)} " +
                            f"(not a cpod with location)", stderr=True)
        reaction_cpds.add(c)
        sm[c] = {}

    # iterate through the reactions
    for r in reactions_to_run:
        for c in reactions[r].left_compounds:
            reaction_cpds.add(c)
            if c not in sm:
                sm[c] = {}
            if verbose and type(c) != PyFBA.metabolism.compound.CompoundWithLocation:
                log_and_message(f"In parsing left compounds, {c} is a {type(c)}", stderr=True)
            sm[c][r] = 0 - reactions[r].get_left_compound_abundance(c)

        for c in reactions[r].right_compounds:
            reaction_cpds.add(c)
            if c not in sm:
                sm[c] = {}
            if verbose and type(c) != PyFBA.metabolism.compound.CompoundWithLocation:
                log_and_message(f"In parsing right compounds, {c} is a {type(c)}", stderr=True)
            sm[c][r] = reactions[r].get_right_compound_abundance(c)

    for c in biomass_equation.left_compounds:
        if c not in compounds:
            compounds.add(c)
            log_and_message(f"create_stoichiometric_matrix added compound {c} from biomass to compounds", stderr=True)
        if verbose and type(c) != PyFBA.metabolism.compound.CompoundWithLocation:
            log_and_message(f"In parsing biomass left, {c} is a {type(c)}", stderr=True)
        reaction_cpds.add(c)
        if c not in sm:
            sm[c] = {}
        sm[c]["BIOMASS_EQN"] = 0 - biomass_equation.get_left_compound_abundance(c)
    for c in biomass_equation.right_compounds:
        if c not in compounds:
            compounds.add(c)
            log_and_message(f"create_stoichiometric_matrix added compound {c} from biomass to compounds", stderr=True)
        if verbose and type(c) != PyFBA.metabolism.compound.CompoundWithLocation:
            log_and_message(f"In parsing biomass right, {c} is a {type(c)}", stderr=True)
        reaction_cpds.add(c)
        if c not in sm:
            sm[c] = {}
        sm[c]["BIOMASS_EQN"] = biomass_equation.get_right_compound_abundance(c)

    # Add the uptake/secretion reactions. These are reactions that allow things to flow from the media
    # into the reaction, or from the cell outwards.
    #
    # The reactions are determined by the external compounds (and biomass_equation), but we only add the left side
    # of the equation to the stoichiometric matrix. This means that they can appear and disappear at will!
    #
    # When we set the reaction bounds we determine which things are in the media unless they are provided for you

    if not uptake_secretion:
        uptake_secretion = PyFBA.fba.uptake_and_secretion_reactions(reaction_cpds)
        log_and_message(f"create_stoichiometric_matrix found {len(uptake_secretion)} reactions", stderr=True)
    for r in uptake_secretion:
        reactions[uptake_secretion[r].id] = uptake_secretion[r]
        for c in uptake_secretion[r].left_compounds:
            reaction_cpds.add(c)
            if c not in sm:
                sm[c] = {}
            sm[c][uptake_secretion[r].id] = 0 - uptake_secretion[r].get_left_compound_abundance(c)

    # now we need to make this into a matrix sorted by
    # reaction id and by cpds
    cp = list(reaction_cpds)
    cp.sort()
    rc = list(reactions_to_run)
    rc.sort()
    rc += [uptake_secretion[x].id for x in uptake_secretion]

    # it is important that we add these at the end
    rc.append("BIOMASS_EQN")

    if verbose:
        sys.stderr.write(f"In the model there are : {len(cp)} compounds and {len(rc)} reactions\n")

    # here we create the matrix from our sm hash
    data = []
    for i, j in enumerate(cp):
        if j not in sm:
            sys.exit("Error while parsing: no " + j + " in sm")
        data.append([])
        for c in rc:
            if c in sm[j]:
                data[i].append(sm[j][c])
            else:
                data[i].append(0.0)

    # load the data into the model
    PyFBA.lp.load(data, [str(c) for c in cp], [str(r) for r in rc], verbose=verbose)

    # now set the objective function.It is the biomass_equation
    # equation which is the last reaction in the network
    ob = [0.0 for r in rc]
    ob[-1] = 1

    PyFBA.lp.objective_coefficients(ob)

    return cp, rc, reactions
<commit_msg>Clarifying output message<commit_after>import sys
import PyFBA
from PyFBA import lp, log_and_message

def create_stoichiometric_matrix(reactions_to_run, reactions, compounds, media, biomass_equation,
                                 uptake_secretion=None, verbose=False):
    """Given the reactions data and a list of RIDs to include, build a
    stoichiometric matrix and load that into the linear solver.

    The uptake and secretion reactions (sometimes called boundary reactions) are reactions that allow
    compounds to flow into the media and secreted compounds away from the cell. You can either provide these (e.g. if
    you are parsing an XML (SBML) file, or we will calculate them for you based on your media and reactions.

    We also take this opportunity to set the objective function (as it is a member of the SM).

    :param uptake_secretion: An optional hash of uptake and secretion reactions that should be added to the model
    :type uptake_secretion: a dict of Reactions
    :param compounds: a set of the compounds present
    :type compounds: set[PyFBA.metabolism.CompoundWithLocation]
    :param reactions: a dict of all the reactions
    :type reactions: dict[PyFBA.metabolism.Reaction]
    :param reactions_to_run: just the reaction ids that we want to include in our model
    :type reactions_to_run: set
    :param media: a set of compounds that would make up the media
    :type media: set
    :param biomass_equation: the biomass_equation equation as a Reaction object
    :type biomass_equation: metabolism.Reaction
    :param verbose: print more information
    :type verbose: bool
    :returns: Sorted lists of all the compounds and reactions in the model, and a revised reactions dict that includes the uptake and secretion reactions
    :rtype: list, list, dict

    """

    sm = {}  # the matrix
    reaction_cpds = set()  # all the cpds in the matrix

    # unify compounds from dicts and sets
    if type(compounds) == dict:
        log_and_message("create_stoichiometric_matrix is converting compounds from a dict to a set")
        compounds = set(compounds.values())

    # initialize the compounds set and the stoichiometric matrix with
    # everything in the media. The order of compounds is irrelevant
    for c in media:
        if c not in compounds:
            log_and_message(f"create_stoichiometric_matrix is adding compound {c} from media to compounds", stderr=True)
            compounds.add(c)
        if verbose and type(c) != PyFBA.metabolism.compound.CompoundWithLocation:
            log_and_message(f"create_stoichiometric_matrix is parsing the media, {c} is a {type(c)} " +
                            f"(not a cpod with location)", stderr=True)
        reaction_cpds.add(c)
        sm[c] = {}

    # iterate through the reactions
    for r in reactions_to_run:
        for c in reactions[r].left_compounds:
            reaction_cpds.add(c)
            if c not in sm:
                sm[c] = {}
            if verbose and type(c) != PyFBA.metabolism.compound.CompoundWithLocation:
                log_and_message(f"In parsing left compounds, {c} is a {type(c)}", stderr=True)
            sm[c][r] = 0 - reactions[r].get_left_compound_abundance(c)

        for c in reactions[r].right_compounds:
            reaction_cpds.add(c)
            if c not in sm:
                sm[c] = {}
            if verbose and type(c) != PyFBA.metabolism.compound.CompoundWithLocation:
                log_and_message(f"In parsing right compounds, {c} is a {type(c)}", stderr=True)
            sm[c][r] = reactions[r].get_right_compound_abundance(c)

    for c in biomass_equation.left_compounds:
        if c not in compounds:
            compounds.add(c)
            log_and_message(f"create_stoichiometric_matrix added compound {c} from biomass to compounds", stderr=True)
        if verbose and type(c) != PyFBA.metabolism.compound.CompoundWithLocation:
            log_and_message(f"In parsing biomass left, {c} is a {type(c)}", stderr=True)
        reaction_cpds.add(c)
        if c not in sm:
            sm[c] = {}
        sm[c]["BIOMASS_EQN"] = 0 - biomass_equation.get_left_compound_abundance(c)
    for c in biomass_equation.right_compounds:
        if c not in compounds:
            compounds.add(c)
            log_and_message(f"create_stoichiometric_matrix added compound {c} from biomass to compounds", stderr=True)
        if verbose and type(c) != PyFBA.metabolism.compound.CompoundWithLocation:
            log_and_message(f"In parsing biomass right, {c} is a {type(c)}", stderr=True)
        reaction_cpds.add(c)
        if c not in sm:
            sm[c] = {}
        sm[c]["BIOMASS_EQN"] = biomass_equation.get_right_compound_abundance(c)

    # Add the uptake/secretion reactions. These are reactions that allow things to flow from the media
    # into the reaction, or from the cell outwards.
    #
    # The reactions are determined by the external compounds (and biomass_equation), but we only add the left side
    # of the equation to the stoichiometric matrix. This means that they can appear and disappear at will!
    #
    # When we set the reaction bounds we determine which things are in the media unless they are provided for you

    if not uptake_secretion:
        uptake_secretion = PyFBA.fba.uptake_and_secretion_reactions(reaction_cpds)
        log_and_message(f"create_stoichiometric_matrix found {len(uptake_secretion)} uptake and secretion reactions",
                        stderr=True)
    for r in uptake_secretion:
        reactions[uptake_secretion[r].id] = uptake_secretion[r]
        for c in uptake_secretion[r].left_compounds:
            reaction_cpds.add(c)
            if c not in sm:
                sm[c] = {}
            sm[c][uptake_secretion[r].id] = 0 - uptake_secretion[r].get_left_compound_abundance(c)

    # now we need to make this into a matrix sorted by
    # reaction id and by cpds
    cp = list(reaction_cpds)
    cp.sort()
    rc = list(reactions_to_run)
    rc.sort()
    rc += [uptake_secretion[x].id for x in uptake_secretion]

    # it is important that we add these at the end
    rc.append("BIOMASS_EQN")

    if verbose:
        sys.stderr.write(f"In the model there are : {len(cp)} compounds and {len(rc)} reactions\n")

    # here we create the matrix from our sm hash
    data = []
    for i, j in enumerate(cp):
        if j not in sm:
            sys.exit("Error while parsing: no " + j + " in sm")
        data.append([])
        for c in rc:
            if c in sm[j]:
                data[i].append(sm[j][c])
            else:
                data[i].append(0.0)

    # load the data into the model
    PyFBA.lp.load(data, [str(c) for c in cp], [str(r) for r in rc], verbose=verbose)

    # now set the objective function.It is the biomass_equation
    # equation which is the last reaction in the network
    ob = [0.0 for r in rc]
    ob[-1] = 1

    PyFBA.lp.objective_coefficients(ob)

    return cp, rc, reactions
<|endoftext|>


<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-
#
# Output Format:
# [grid-at]
# JSON

import os
import logging
import getopt
import sys
import time
import json
from multiprocessing import Process
from util.dbopts import connectMongo
from util.GridPropSup import GridPropSup
from util.preprocess import chunks, getCityLocs
from util.preprocess import parseFormatGID

def mergeGrids(basepath, jobsNum):
	# 加载所有网格
	locs = getCityLocs('beijing')
	SPLIT = 0.0005
	LATNUM = int((locs['north'] - locs['south']) / SPLIT + 1)
	LNGNUM = int((locs['east'] - locs['west']) / SPLIT + 1)

	totalLen = LATNUM * LNGNUM
	result = [parseFormatGID(locs, i) for i in xrange(0, totalLen)]

	# 根据每个文件对网格进行更新
	for i in xrange(0, jobsNum):
		with open(os.path.join(basepath, "BJ-MID-SQL", "grid-j%d" % j)) as f:
			for each in f:
				each = each.strip().split(',')
				nid = int(each[0])
				pid = int(each[1])

				if result[nid]['pid'] != -1:
					result[nid]['pid'] = pid
		f.close()

	# 结果写入文件
	result = [i for i in result if i['pid'] != -1]
	with open(os.path.join(basepath, 'BJ-MID-SQL', 'grid-at'), 'ab') as res:
		res.write(json.dumps(result).encode('utf-8'))
	res.close()

def processTask(INDEX, city, basepath, pidList):
	task = GridPropSup({
		'INDEX': INDEX,
		'city': city,
		'basepath': basepath,
		'pidList': pidList
	})
	task.run()

def usage():
	pass

def main(argv):
	try:
		opts, args = getopt.getopt(argv, "hc:d:", ["help", "city=", 'directory='])
	except getopt.GetoptError as err:
		print str(err)
		usage()
		sys.exit(2)

	city, directory = 'beijing', '/home/tao.jiang/datasets/JingJinJi/records'
	for opt, arg in opts:
		if opt == '-h':
			usage()
			sys.exit()
		elif opt in ("-c", "--city"):
			city = arg
		elif opt in ("-d", "--directory"):
			directory = arg

	STARTTIME = time.time()
	print "Start approach at %s" % STARTTIME

	conn, db = connectMongo('stvis')
	plist = list(db['pois'].find({}, {
		'properties.pid': 1,
		'properties.coordinates': 1
	}))
	pois = list(chunks(plist, 20))
	conn.close()

	# @多进程运行程序 START
	jobs = []

	for x in xrange(0, 20):
		tProcess = Process(target=processTask, args=(x, city, directory, pois[x]))
		jobs.append(tProcess)
		jobs[x].start()

	for job in jobs:
		job.join()

	# 处理剩余数据进文件
	# 合并操作
	mergeGrids(directory, 20)

	# @多进程运行程序 END

	print "END TIME: %s" % time.time()

if __name__ == '__main__':
	logging.basicConfig(filename='logger-gengridsubprop.log', level=logging.DEBUG)
	main(sys.argv[1:])
<commit_msg>update: mongo query properties<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-
#
# Output Format:
# [grid-at]
# JSON

import os
import logging
import getopt
import sys
import time
import json
from multiprocessing import Process
from util.dbopts import connectMongo
from util.GridPropSup import GridPropSup
from util.preprocess import chunks, getCityLocs
from util.preprocess import parseFormatGID

def mergeGrids(basepath, jobsNum):
	# 加载所有网格
	locs = getCityLocs('beijing')
	SPLIT = 0.0005
	LATNUM = int((locs['north'] - locs['south']) / SPLIT + 1)
	LNGNUM = int((locs['east'] - locs['west']) / SPLIT + 1)

	totalLen = LATNUM * LNGNUM
	result = [parseFormatGID(locs, i) for i in xrange(0, totalLen)]

	# 根据每个文件对网格进行更新
	for i in xrange(0, jobsNum):
		with open(os.path.join(basepath, "BJ-MID-SQL", "grid-j%d" % j)) as f:
			for each in f:
				each = each.strip().split(',')
				nid = int(each[0])
				pid = int(each[1])

				if result[nid]['pid'] != -1:
					result[nid]['pid'] = pid
		f.close()

	# 结果写入文件
	result = [i for i in result if i['pid'] != -1]
	with open(os.path.join(basepath, 'BJ-MID-SQL', 'grid-at'), 'ab') as res:
		res.write(json.dumps(result).encode('utf-8'))
	res.close()

def processTask(INDEX, city, basepath, pidList):
	task = GridPropSup({
		'INDEX': INDEX,
		'city': city,
		'basepath': basepath,
		'pidList': pidList
	})
	task.run()

def usage():
	pass

def main(argv):
	try:
		opts, args = getopt.getopt(argv, "hc:d:", ["help", "city=", 'directory='])
	except getopt.GetoptError as err:
		print str(err)
		usage()
		sys.exit(2)

	city, directory = 'beijing', '/home/tao.jiang/datasets/JingJinJi/records'
	for opt, arg in opts:
		if opt == '-h':
			usage()
			sys.exit()
		elif opt in ("-c", "--city"):
			city = arg
		elif opt in ("-d", "--directory"):
			directory = arg

	STARTTIME = time.time()
	print "Start approach at %s" % STARTTIME

	conn, db = connectMongo('stvis')
	plist = list(db['pois'].find({}, {
		'pid': 1,
		'coordinates': 1
	}))
	pois = list(chunks(plist, 20))
	conn.close()

	# @多进程运行程序 START
	jobs = []

	for x in xrange(0, 20):
		tProcess = Process(target=processTask, args=(x, city, directory, pois[x]))
		jobs.append(tProcess)
		jobs[x].start()

	for job in jobs:
		job.join()

	# 处理剩余数据进文件
	# 合并操作
	mergeGrids(directory, 20)

	# @多进程运行程序 END

	print "END TIME: %s" % time.time()

if __name__ == '__main__':
	logging.basicConfig(filename='logger-gengridsubprop.log', level=logging.DEBUG)
	main(sys.argv[1:])
<|endoftext|>


<commit_before># Copyright 2017 Rice University
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import argparse
import os
import shutil
import zipfile
import urllib.request

if __name__ == '__main__':

    # Parse command line args.
    parser = argparse.ArgumentParser()
    parser.add_argument('--name', type=str, required=True)
    parser.add_argument('--url',  type=str, required=True)
    parser.add_argument('--model_dir',  type=str, required=True)
    args = parser.parse_args()

    model_name=args.name
    model_dir=args.model_dir
    fetch_url_base=args.url

    # Ensure the model directory exists prior to proceeding.
    if not os.path.exists(model_dir) or not os.path.isdir(model_dir):
        print("No such dir: " + model_dir)
        exit(1)

    model_file = os.path.join(model_dir, model_name + ".zip")

    # Check if we already have the model.
    if os.path.exists(model_file):
        exit(0)

    # We don't have the model. Delete any existing files (probably old model files).
    print("Delete existing model? (if any) [y/n]")
    confirm = input()
    if confirm != "y" and confirm != "Y":
        exit(0)
    shutil.rmtree(model_dir)
    os.makedirs(model_dir)

    # Download new model files from remote server.
    fetch_url = fetch_url_base + model_name + ".zip"
    model_zip_file = os.path.join(model_dir,  model_name + ".zip")
    print("Downloading " + fetch_url)
    urllib.request.urlretrieve(fetch_url, model_zip_file)

    # Unzip model files and delete ziped model.
    with zipfile.ZipFile(model_zip_file,"r") as zip_ref:
        zip_ref.extractall(model_dir)

<commit_msg>Do not ask user input (y/n) when downloading model<commit_after># Copyright 2017 Rice University
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import argparse
import os
import shutil
import zipfile
import urllib.request

if __name__ == '__main__':

    # Parse command line args.
    parser = argparse.ArgumentParser()
    parser.add_argument('--name', type=str, required=True)
    parser.add_argument('--url',  type=str, required=True)
    parser.add_argument('--model_dir',  type=str, required=True)
    args = parser.parse_args()

    model_name=args.name
    model_dir=args.model_dir
    fetch_url_base=args.url

    # Ensure the model directory exists prior to proceeding.
    if not os.path.exists(model_dir) or not os.path.isdir(model_dir):
        print("No such dir: " + model_dir)
        exit(1)

    model_file = os.path.join(model_dir, model_name + ".zip")

    # Check if we already have the model.
    if os.path.exists(model_file):
        exit(0)

    # We don't have the model. Delete any existing files (probably old model files).
    shutil.rmtree(model_dir)
    os.makedirs(model_dir)

    # Download new model files from remote server.
    fetch_url = fetch_url_base + model_name + ".zip"
    model_zip_file = os.path.join(model_dir,  model_name + ".zip")
    print("Downloading " + fetch_url)
    urllib.request.urlretrieve(fetch_url, model_zip_file)

    # Unzip model files and delete ziped model.
    with zipfile.ZipFile(model_zip_file,"r") as zip_ref:
        zip_ref.extractall(model_dir)

<|endoftext|>


<commit_before>import re

from typing import List, Union

__all__ = [
    'Indent', 'OpenParen', 'CloseParen', 'Name', 'Class', 'Operator',
    'Lexer',
    'lex',
]

# Regular expression patterns.
INDENT_SIZE = 4
RE_LEADING_INDENT = re.compile(fr'^((?: {{{INDENT_SIZE}}})*)(.*)$')

RE_CLASS = re.compile(r'[A-Z][-_a-zA-Z0-9]*')
RE_NAME = re.compile(r'_+|(?:_*[a-z][_a-zA-Z0-9]*(?:-[_a-zA-Z0-9]*)*[!@$%^&*?a-zA-Z0-9]?)')
RE_OPERATOR = re.compile(r'[!@$%^&*()-=+|/?<>\[\]{}~]+')

RE_PREFIX_OP = re.compile(r'(?P<op>' + RE_OPERATOR.pattern + r')(?P<val>(?:' + RE_NAME.pattern + '|' + RE_CLASS.pattern + '))')
RE_POSTFIX_OP = re.compile(r'(?P<val>(?:' + RE_NAME.pattern + '|' + RE_CLASS.pattern + '))(?P<op>' + RE_OPERATOR.pattern + r')')
RE_OPEN_PAREN = re.compile(r'(?P<left>' + RE_NAME.pattern + ')?\((?P<right>' + RE_NAME.pattern + ')?')
RE_CLOSE_PAREN = re.compile(r'(?P<left>' + RE_NAME.pattern + ')?\)(?P<right>' + RE_NAME.pattern + ')?')
RE_BOTH_PAREN = re.compile(r'(?P<left>' + RE_NAME.pattern + ')?\((?P<inside>' + RE_NAME.pattern + ')?\)(?P<right>' + RE_NAME.pattern + ')?')

# Regular expression magic class for making if/else matching simpler. Idea from:
#   http://code.activestate.com/recipes/456151-using-rematch-research-and-regroup-in-if-elif-elif/
class RegexMatcher:
    def __init__(self, token):
        self._token = token

    def match(self, pattern: re._pattern_type):
        self._match = pattern.match(self._token)
        return self._match

    def fullmatch(self, pattern: re._pattern_type):
        self._match = pattern.fullmatch(self._token)
        return self._match

    def search(self, pattern: re._pattern_type):
        self._match = pattern.search(self._token)
        return self._match

    def group(self, grp: Union[int, str]):
        return self._match.group(grp)

    def groups(self):
        return self._match.groups()

# Tokens
class Lexeme:
    def __init__(self, text, repl_with_text=True):
        self.text = text
        self._repl_with_text = repl_with_text

    def __repr__(self):
        if self._repl_with_text:
            return f'{type(self).__name__}({self.text})'
        else:
            return type(self).__name__

    def __str__(self):
        return self.text

    def __eq__(self, other):
        if not isinstance(other, type(self)):
            return False
        return self.text == other.text

    def __ne__(self, other):
        return not self.__eq__(other)

class Indent(Lexeme):
    def __init__(self):
        super().__init__(' ' * INDENT_SIZE, False)

class NewLine(Lexeme):
    def __init__(self):
        super().__init__('\n', False)

class OpenParen(Lexeme):
    def __init__(self):
        super().__init__('(', False)

class CloseParen(Lexeme):
    def __init__(self):
        super().__init__(')', False)

class Name(Lexeme):
    pass

class Class(Lexeme):
    pass

class Operator(Lexeme):
    pass

INDENT = Indent()
NEWLINE = NewLine()
OPEN_PAREN = OpenParen()
CLOSE_PAREN = CloseParen()

# Lexer errors.
class LexerError(Exception):
    pass

# Lexer implementation.
class Lexer:
    @classmethod
    def lex(cls, text: str) -> List[Lexeme]:
        lexemes = []
        for line in text.splitlines():
            lexemes.extend(cls.lex_line(line))
            lexemes.append(NEWLINE)
        return lexemes

    @classmethod
    def lex_line(cls, line: str) -> List[Lexeme]:
        lexemes = []
        if not line:
            return lexemes
        # Remove properly-indented leading whitespace.
        match = RE_LEADING_INDENT.match(line)
        if match is None:
            raise LexerError(f"invalid line given: '{line}'")
        indentation, rest = match.groups()
        for indentation_level in range(len(indentation) // INDENT_SIZE):
            lexemes.append(Indent())
        for token in rest.split():
            for lexeme in cls.lex_token(token):
                lexemes.append(lexeme)
        return lexemes

    @classmethod
    def lex_token(cls, token: str) -> List[Lexeme]:
        matcher = RegexMatcher(token)
        lexemes = []
        if matcher.fullmatch(RE_NAME):
            lexemes.append(Name(matcher.group(0)))
        elif matcher.fullmatch(RE_CLASS):
            lexemes.append(Class(matcher.group(0)))
        elif matcher.fullmatch(RE_OPERATOR):
            lexemes.append(Operator(matcher.group(0)))
        elif matcher.fullmatch(RE_PREFIX_OP):
            lexemes.append(Operator(matcher.group('op')))
            lexemes.extend(cls.lex_token(matcher.group('val')))
        elif matcher.fullmatch(RE_POSTFIX_OP):
            lexemes.extend(cls.lex_token(matcher.group('val')))
            lexemes.append(Operator(matcher.group('op')))
        elif matcher.fullmatch(RE_OPEN_PAREN):
            lexemes.extend(cls.lex_token(matcher.group('left')))
            lexemes.append(OPEN_PAREN)
            lexemes.extend(cls.lex_token(matcher.group('right')))
        elif matcher.fullmatch(RE_CLOSE_PAREN):
            lexemes.extend(cls.lex_token(matcher.group('left')))
            lexemes.append(CLOSE_PAREN)
            lexemes.extend(cls.lex_token(matcher.group('right')))
        elif matcher.fullmatch(RE_BOTH_PAREN):
            lexemes.extend(cls.lex_token(matcher.group('left')))
            lexemes.append(OPEN_PAREN)
            lexemes.extend(cls.lex_token(matcher.group('inside')))
            lexemes.append(CLOSE_PAREN)
            lexemes.extend(cls.lex_token(matcher.group('right')))
        else:
            raise LexerError(f"invalid token: '{token}'")
        return lexemes

lex = Lexer.lex
<commit_msg>Fix warnings and add file-lexing method<commit_after>import re

from re import _pattern_type as PatternType
from typing import List, Union

__all__ = [
    'Indent', 'OpenParen', 'CloseParen', 'Name', 'Class', 'Operator',
    'lex_file', 'lex_line',
]

# Regular expression patterns.
INDENT_SIZE = 4
RE_LEADING_INDENT = re.compile(fr'^((?: {{{INDENT_SIZE}}})*)(.*)$')

RE_CLASS = re.compile(r'[A-Z][-_a-zA-Z0-9]*')
RE_NAME = re.compile(r'_+|(?:_*[a-z][_a-zA-Z0-9]*(?:-[_a-zA-Z0-9]*)*[!@$%^&*?a-zA-Z0-9]?)')
RE_OPERATOR = re.compile(r'[!@$%^&*()-=+|/?<>\[\]{}~]+')

RE_PREFIX_OP = re.compile(r'(?P<op>' + RE_OPERATOR.pattern + r')(?P<val>(?:' + RE_NAME.pattern + '|' +
                          RE_CLASS.pattern + '))')
RE_POSTFIX_OP = re.compile(r'(?P<val>(?:' + RE_NAME.pattern + '|' + RE_CLASS.pattern + '))(?P<op>' +
                           RE_OPERATOR.pattern + r')')
RE_OPEN_PAREN = re.compile(r'(?P<left>' + RE_NAME.pattern + ')?\((?P<right>' + RE_NAME.pattern + ')?')
RE_CLOSE_PAREN = re.compile(r'(?P<left>' + RE_NAME.pattern + ')?\)(?P<right>' + RE_NAME.pattern + ')?')
RE_BOTH_PAREN = re.compile(r'(?P<left>' + RE_NAME.pattern + ')?\((?P<inside>' + RE_NAME.pattern + ')?\)(?P<right>' +
                           RE_NAME.pattern + ')?')

# Regular expression magic class for making if/else matching simpler. Idea from:
#   http://code.activestate.com/recipes/456151-using-rematch-research-and-regroup-in-if-elif-elif/
class RegexMatcher:
    def __init__(self, token):
        self._token = token
        self._match = None

    def match(self, pattern: PatternType):
        self._match = pattern.match(self._token)
        return self._match

    def fullmatch(self, pattern: PatternType):
        self._match = pattern.fullmatch(self._token)
        return self._match

    def search(self, pattern: PatternType):
        self._match = pattern.search(self._token)
        return self._match

    def group(self, grp: Union[int, str]):
        return self._match.group(grp)

    def groups(self):
        return self._match.groups()

# Tokens
class Lexeme:
    def __init__(self, text, repl_with_text=True):
        self.text = text
        self._repl_with_text = repl_with_text

    def __repr__(self):
        if self._repl_with_text:
            return f'{type(self).__name__}({self.text})'
        else:
            return type(self).__name__

    def __str__(self):
        return self.text

    def __eq__(self, other):
        if not isinstance(other, type(self)):
            return False
        return self.text == other.text

    def __ne__(self, other):
        return not self.__eq__(other)

class Indent(Lexeme):
    def __init__(self):
        super().__init__(' ' * INDENT_SIZE, False)

class NewLine(Lexeme):
    def __init__(self):
        super().__init__('\n', False)

class OpenParen(Lexeme):
    def __init__(self):
        super().__init__('(', False)

class CloseParen(Lexeme):
    def __init__(self):
        super().__init__(')', False)

class Name(Lexeme):
    pass

class Class(Lexeme):
    pass

class Operator(Lexeme):
    pass

INDENT = Indent()
NEWLINE = NewLine()
OPEN_PAREN = OpenParen()
CLOSE_PAREN = CloseParen()

# Lexer errors.
class LexerError(Exception):
    pass

# Lexer implementation.
class Lexer:
    @classmethod
    def lex(cls, text: str) -> List[Lexeme]:
        lexemes = []
        for line in text.splitlines():
            lexemes.extend(cls.lex_line(line))
            lexemes.append(NEWLINE)
        return lexemes

    @classmethod
    def lex_file(cls, file: str) -> List[Lexeme]:
        lexemes = []
        with open(file) as f:
            for line in f:
                lexemes.extend(cls.lex_line(line))
                lexemes.append(NEWLINE)
        return lexemes

    @classmethod
    def lex_line(cls, line: str) -> List[Lexeme]:
        lexemes = []
        if not line:
            return lexemes
        # Remove properly-indented leading whitespace.
        match = RE_LEADING_INDENT.match(line)
        if match is None:
            raise LexerError(f"invalid line given: '{line}'")
        indentation, rest = match.groups()
        for indentation_level in range(len(indentation) // INDENT_SIZE):
            lexemes.append(Indent())
        for token in rest.split():
            for lexeme in cls.lex_token(token):
                lexemes.append(lexeme)
        return lexemes

    @classmethod
    def lex_token(cls, token: str) -> List[Lexeme]:
        matcher = RegexMatcher(token)
        lexemes = []
        if matcher.fullmatch(RE_NAME):
            lexemes.append(Name(matcher.group(0)))
        elif matcher.fullmatch(RE_CLASS):
            lexemes.append(Class(matcher.group(0)))
        elif matcher.fullmatch(RE_OPERATOR):
            lexemes.append(Operator(matcher.group(0)))
        elif matcher.fullmatch(RE_PREFIX_OP):
            lexemes.append(Operator(matcher.group('op')))
            lexemes.extend(cls.lex_token(matcher.group('val')))
        elif matcher.fullmatch(RE_POSTFIX_OP):
            lexemes.extend(cls.lex_token(matcher.group('val')))
            lexemes.append(Operator(matcher.group('op')))
        elif matcher.fullmatch(RE_OPEN_PAREN):
            lexemes.extend(cls.lex_token(matcher.group('left')))
            lexemes.append(OPEN_PAREN)
            lexemes.extend(cls.lex_token(matcher.group('right')))
        elif matcher.fullmatch(RE_CLOSE_PAREN):
            lexemes.extend(cls.lex_token(matcher.group('left')))
            lexemes.append(CLOSE_PAREN)
            lexemes.extend(cls.lex_token(matcher.group('right')))
        elif matcher.fullmatch(RE_BOTH_PAREN):
            lexemes.extend(cls.lex_token(matcher.group('left')))
            lexemes.append(OPEN_PAREN)
            lexemes.extend(cls.lex_token(matcher.group('inside')))
            lexemes.append(CLOSE_PAREN)
            lexemes.extend(cls.lex_token(matcher.group('right')))
        else:
            raise LexerError(f"invalid token: '{token}'")
        return lexemes

lex_file = Lexer.lex_file
lex_line = Lexer.lex_line
<|endoftext|>


<commit_before>import os
import sys
import logging
import subprocess
import virtool.routes
import virtool.sessions

from aiohttp import web
from motor import motor_asyncio

logger = logging.getLogger(__name__)

def create_app(loop, db_name="virtool"):
    app = web.Application(loop=loop, middlewares=[virtool.sessions.middleware_factory])

    app["db"] = motor_asyncio.AsyncIOMotorClient(io_loop=loop)[db_name]

    virtool.routes.setup_routes(app)

    return app

def reload():
    exe = sys.executable

    if exe.endswith("python"):
        os.execl(exe, exe, *sys.argv)

    if exe.endswith("run"):
        os.execv(exe, sys.argv)

    raise SystemError("Could not determine executable type")

def shutdown(app, exit_code=0):
    """
    Shutdown the server by cancelling all jobs and calling :func:`sys.exit`.

    :param exit_code: the exit code to return.
    :type exit_code: int

    """
    logging.info("Shutting down")

    job_manager = app.get("job_manager")

    if job_manager:
        for job_id in job_manager.jobs_dict:
            job_manager.cancel(job_id)

    logging.info("Exiting")

    sys.exit(exit_code)

def find_server_version(install_path="."):
    try:
        return subprocess.check_output(["git", "describe", "--tags"]).decode().rstrip()
    except (subprocess.CalledProcessError, FileNotFoundError):
        try:
            version_file_path = os.path.join(install_path, "VERSION")

            with open(version_file_path, "r") as version_file:
                return version_file.read().rstrip()

        except FileNotFoundError:
            logger.critical("Could not determine software version.")
            return "Unknown"
<commit_msg>Add web.configure_ssl function<commit_after>import os
import sys
import ssl
import logging
import subprocess
import virtool.routes
import virtool.sessions

from aiohttp import web
from motor import motor_asyncio

logger = logging.getLogger(__name__)

def create_app(loop, db_name="virtool"):
    app = web.Application(loop=loop, middlewares=[virtool.sessions.middleware_factory])

    app["db"] = motor_asyncio.AsyncIOMotorClient(io_loop=loop)[db_name]

    virtool.routes.setup_routes(app)

    return app

def configure_ssl(cert_path, key_path):
    ssl_ctx = ssl.create_default_context(ssl.Purpose.CLIENT_AUTH)
    ssl_ctx.load_cert_chain(cert_path, key_path)

    return ssl_ctx

def reload():
    exe = sys.executable

    if exe.endswith("python"):
        os.execl(exe, exe, *sys.argv)

    if exe.endswith("run"):
        os.execv(exe, sys.argv)

    raise SystemError("Could not determine executable type")

def shutdown(app, exit_code=0):
    """
    Shutdown the server by cancelling all jobs and calling :func:`sys.exit`.

    :param exit_code: the exit code to return.
    :type exit_code: int

    """
    logging.info("Shutting down")

    job_manager = app.get("job_manager")

    if job_manager:
        for job_id in job_manager.jobs_dict:
            job_manager.cancel(job_id)

    logging.info("Exiting")

    sys.exit(exit_code)

def find_server_version(install_path="."):
    try:
        return subprocess.check_output(["git", "describe", "--tags"]).decode().rstrip()
    except (subprocess.CalledProcessError, FileNotFoundError):
        try:
            version_file_path = os.path.join(install_path, "VERSION")

            with open(version_file_path, "r") as version_file:
                return version_file.read().rstrip()

        except FileNotFoundError:
            logger.critical("Could not determine software version.")
            return "Unknown"
<|endoftext|>