<commit_before>import xml.etree.cElementTree as ET
import logging
from UI.resources.constants import DEFAULT_SHARD_SIZE,\
    DEFAULT_MAX_BRIDGE_REQUEST_TIMEOUT

CONFIG_FILE = 'storj_client_config.xml'

# Configuration backend section
class Configuration:

    __logger = logging.getLogger('%s.Configuration' % __name__)

    def __init__(self, sameFileNamePrompt=None, sameFileHashPrompt=None,
                 load_config=False):
        if load_config:
            et = None
            try:
                et = ET.parse(CONFIG_FILE)
            except:
                self.__logger.error('Unspecified XML parse error')

            for tags in et.iter('same_file_name_prompt'):
                if tags.text == '1':
                    self.sameFileNamePrompt = True
                elif tags.text == '0':
                    self.sameFileNamePrompt = False
                else:
                    self.sameFileNamePrompt = True
            for tags in et.iter('same_file_hash_prompt'):
                if tags.text == '1':
                    self.sameFileHashPrompt = True
                elif tags.text == '0':
                    self.sameFileHashPrompt = False
                else:
                    self.sameFileHashPrompt = True
            for tags in et.iter('max_chunk_size_for_download'):
                if tags.text is not None:
                    self.maxDownloadChunkSize = int(tags.text)
                else:
                    self.maxDownloadChunkSize = 1024

    def get_config_parametr_value(self, parametr):
        output = ''
        try:
            et = ET.parse(CONFIG_FILE)
            for tags in et.iter(str(parametr)):
                output = tags.text
        except:
            self.__logger.error('Unspecified error')

        return output

    def load_config_from_xml(self):
        try:
            et = ET.parse(CONFIG_FILE)
            for tags in et.iter('password'):
                output = tags.text
        except:
            self.__logger.error('Unspecified error')

    def paint_config_to_ui(self, settings_ui):
        et = None

        try:
            et = ET.parse(CONFIG_FILE)

            for tags in et.iter('max_shard_size'):
                settings_ui.max_shard_size.setValue(int(tags.text))
            for tags in et.iter('max_connections_onetime'):
                settings_ui.connections_onetime.setValue(int(tags.text))
            for tags in et.iter('bridge_request_timeout'):
                settings_ui.bridge_request_timeout.setValue(int(tags.text))
            for tags in et.iter('crypto_keys_location'):
                settings_ui.crypto_keys_location.setText(str(tags.text))
            for tags in et.iter('max_download_bandwidth'):
                settings_ui.max_download_bandwidth.setText(str(tags.text))
            for tags in et.iter('max_upload_bandwidth'):
                settings_ui.max_upload_bandwidth.setText(str(tags.text))
            for tags in et.iter('default_file_encryption_algorithm'):
                settings_ui.default_crypto_algorithm.setCurrentIndex(int(tags.text))
            for tags in et.iter('shard_size_unit'):
                settings_ui.shard_size_unit.setCurrentIndex(int(tags.text))
            for tags in et.iter('custom_max_shard_size_enabled'):
                if int(tags.text) == 1:
                    settings_ui.max_shard_size_enabled_checkBox.setChecked(True)
                else:
                    settings_ui.max_shard_size_enabled_checkBox.setChecked(False)

        except Exception as e:
            self.__logger.error('Unspecified XML parse error')
            self.__logger.error(e)

    def save_client_configuration(self, settings_ui):
        root = ET.Element('configuration')
        doc = ET.SubElement(root, 'client')

        # settings_ui = Ui_
        if settings_ui.max_shard_size_enabled_checkBox.isChecked():
            custom_max_shard_size_enabled_checkbox = '1'
        else:
            custom_max_shard_size_enabled_checkbox = '0'

        ET.SubElement(doc, 'custom_max_shard_size_enabled').text = \
            str(custom_max_shard_size_enabled_checkbox)
        ET.SubElement(doc, 'max_shard_size').text = \
            str(settings_ui.max_shard_size.text())
        ET.SubElement(doc, 'max_connections_onetime').text = \
            str(settings_ui.connections_onetime.text())
        ET.SubElement(doc, 'shard_size_unit').text = \
            str(settings_ui.shard_size_unit.currentIndex())
        ET.SubElement(doc, 'max_download_bandwidth').text = \
            str(settings_ui.max_download_bandwidth.text())
        ET.SubElement(doc, 'max_upload_bandwidth').text = \
            str(settings_ui.max_upload_bandwidth.text())
        ET.SubElement(doc, 'default_file_encryption_algorithm').text = \
            str(settings_ui.default_crypto_algorithm.currentIndex())
        ET.SubElement(doc, 'bridge_request_timeout').text = \
            str(settings_ui.bridge_request_timeout.text())
        ET.SubElement(doc, 'crypto_keys_location').text = \
            str(settings_ui.crypto_keys_location.text())
        tree = ET.ElementTree(root)
        tree.write(CONFIG_FILE)

    def max_shard_size(self):
        et = None

        max_shard_size = DEFAULT_SHARD_SIZE

        try:

            et = ET.parse(CONFIG_FILE)
            shard_size_unit = 2
            max_shard_size_sterile = None
            for tags in et.iter('custom_max_shard_size_enabled'):
                if int(tags.text) == 1:
                    for tags2 in et.iter('max_shard_size'):
                        max_shard_size_sterile = int(tags2.text)
                    for tags3 in et.iter('shard_size_unit'):
                        shard_size_unit = int(tags3.text)

                    if shard_size_unit == 0:  # KB:
                        max_shard_size = max_shard_size_sterile * 2048
                    elif shard_size_unit == 1:  # MB:
                        max_shard_size = max_shard_size_sterile * 1024 * 2048
                    elif shard_size_unit == 2:  # GB:
                        max_shard_size = max_shard_size_sterile * (1024 ** 2) * 2048
                    elif shard_size_unit == 3:  # TB:
                        max_shard_size = max_shard_size_sterile * (1024 ** 3) * 2048
                else:
                    max_shard_size = DEFAULT_SHARD_SIZE

        except Exception as e:
            self.__logger.error('Unspecified XML parse error')
            self.__logger.error(e)

        return max_shard_size

    def get_max_bridge_request_timeout(self):
        max_bridge_request_timeout = DEFAULT_MAX_BRIDGE_REQUEST_TIMEOUT
        et = None
        try:
            et = ET.parse(CONFIG_FILE)
            for tags in et.iter(str('bridge_request_timeout')):
                max_bridge_request_timeout = int(tags.text)

        except Exception as e:
            self.__logger.error('Unspecified XML parse error')
            self.__logger.error(e)

        return max_bridge_request_timeout
<commit_msg>Update backend_config.py<commit_after>import xml.etree.cElementTree as ET
from log_manager import logger
from BeautifulSoup import BeautifulStoneSoup as Soup
from UI.resources.constants import DEFAULT_MAX_SHARD_SIZE, DEFAULT_SHARD_SIZE, DEFAULT_MAX_BRIDGE_REQUEST_TIMEOUT

CONFIG_FILE = "storj_client_config.xml"

# Configuration backend section
class Configuration:

    def __init__(self, sameFileNamePrompt=None, sameFileHashPrompt=None,
                 load_config=False):
        if load_config:

            et = None

            try:
                et = ET.parse(CONFIG_FILE)
            except:
                logger.error("Unspecified XML parse error")

            for tags in et.iter(str("same_file_name_prompt")):
                if tags.text == "1":
                    self.sameFileNamePrompt = True
                elif tags.text == "0":
                    self.sameFileNamePrompt = False
                else:
                    self.sameFileNamePrompt = True
            for tags in et.iter(str("same_file_hash_prompt")):
                if tags.text == "1":
                    self.sameFileHashPrompt = True
                elif tags.text == "0":
                    self.sameFileHashPrompt = False
                else:
                    self.sameFileHashPrompt = True
            for tags in et.iter(str("max_chunk_size_for_download")):
                if tags.text is not None:
                    self.maxDownloadChunkSize = int(tags.text)
                else:
                    self.maxDownloadChunkSize = 1024

    def get_config_parametr_value(self, parametr):
        output = ""
        try:
            et = ET.parse(CONFIG_FILE)
            for tags in et.iter(str(parametr)):
                output = tags.text
        except:
            logger.error("Unspecified error")

        return output

    def load_config_from_xml(self):
        try:
            et = ET.parse(CONFIG_FILE)
            for tags in et.iter('password'):
                output = tags.text
        except:
            logger.error("Unspecified error")

    def paint_config_to_ui(self, settings_ui):
        et = None

        try:
            et = ET.parse(CONFIG_FILE)

            for tags in et.iter(str("max_shard_size")):
                settings_ui.max_shard_size.setValue(int(tags.text))
            for tags in et.iter(str("max_connections_onetime")):
                settings_ui.connections_onetime.setValue(int(tags.text))
            for tags in et.iter(str("bridge_request_timeout")):
                settings_ui.bridge_request_timeout.setValue(int(tags.text))
            for tags in et.iter(str("crypto_keys_location")):
                settings_ui.crypto_keys_location.setText(str(tags.text))
            for tags in et.iter(str("max_download_bandwidth")):
                settings_ui.max_download_bandwidth.setText(str(tags.text))
            for tags in et.iter(str("max_upload_bandwidth")):
                settings_ui.max_upload_bandwidth.setText(str(tags.text))
            for tags in et.iter(str("default_file_encryption_algorithm")):
                settings_ui.default_crypto_algorithm.setCurrentIndex(int(tags.text))
            for tags in et.iter(str("shard_size_unit")):
                settings_ui.shard_size_unit.setCurrentIndex(int(tags.text))
            for tags in et.iter(str("custom_max_shard_size_enabled")):
                if int(tags.text) == 1:
                    settings_ui.max_shard_size_enabled_checkBox.setChecked(True)
                else:
                    settings_ui.max_shard_size_enabled_checkBox.setChecked(False)

        except Exception as e:
            logger.error("Unspecified XML parse error" + str(e))

    def save_client_configuration(self, settings_ui):
        root = ET.Element("configuration")
        doc = ET.SubElement(root, "client")

        # settings_ui = Ui_
        if settings_ui.max_shard_size_enabled_checkBox.isChecked():
            custom_max_shard_size_enabled_checkbox = '1'
        else:
            custom_max_shard_size_enabled_checkbox = '0'

        ET.SubElement(doc, "custom_max_shard_size_enabled").text = str(custom_max_shard_size_enabled_checkbox)
        ET.SubElement(doc, "max_shard_size").text = str(settings_ui.max_shard_size.text())
        ET.SubElement(doc, "max_connections_onetime").text = str(settings_ui.connections_onetime.text())
        ET.SubElement(doc, "shard_size_unit").text = str(settings_ui.shard_size_unit.currentIndex())
        ET.SubElement(doc, "max_download_bandwidth").text = str(settings_ui.max_download_bandwidth.text())
        ET.SubElement(doc, "max_upload_bandwidth").text = str(settings_ui.max_upload_bandwidth.text())
        ET.SubElement(doc, "default_file_encryption_algorithm").text = str(
                                    settings_ui.default_crypto_algorithm.currentIndex())
        ET.SubElement(doc, "bridge_request_timeout").text = str(settings_ui.bridge_request_timeout.text())
        ET.SubElement(doc, "crypto_keys_location").text = str(settings_ui.crypto_keys_location.text())
        #ET.SubElement(doc, "custom_tmp_path").text = "1"
        ET.SubElement(doc, "custom_tmp_path").set('path', 'ABC')
        tree = ET.ElementTree(root)
        tree.write(CONFIG_FILE)

        custom_tmp_path = self.get_custom_temp_path()
        print custom_tmp_path

    def max_shard_size(self):
        et = None

        max_shard_size = DEFAULT_SHARD_SIZE

        try:

            et = ET.parse(CONFIG_FILE)
            shard_size_unit = 2
            max_shard_size_sterile = None
            for tags in et.iter(str("custom_max_shard_size_enabled")):
                if int(tags.text) == 1:
                    for tags2 in et.iter(str("max_shard_size")):
                        max_shard_size_sterile = int(tags2.text)
                    for tags3 in et.iter(str("shard_size_unit")):
                        shard_size_unit = int(tags3.text)

                    if shard_size_unit == 0:  # KB:
                        max_shard_size = (max_shard_size_sterile * 2048)
                    elif shard_size_unit == 1:  # MB:
                        max_shard_size = (max_shard_size_sterile * 1024 * 2048)
                    elif shard_size_unit == 2:  # GB:
                        max_shard_size = (max_shard_size_sterile * 1024 * 1024 * 2048)
                    elif shard_size_unit == 3:  # TB:
                        max_shard_size = (max_shard_size_sterile * 1024 * 1024 * 1024 * 2048)
                else:
                    max_shard_size = DEFAULT_SHARD_SIZE

        except Exception as e:
            logger.error("Unspecified XML parse error" + str(e))

        return (max_shard_size/2)

    def get_max_bridge_request_timeout(self):
        max_bridge_request_timeout = DEFAULT_MAX_BRIDGE_REQUEST_TIMEOUT

        et = None

        try:
            et = ET.parse(CONFIG_FILE)
            for tags in et.iter(str("bridge_request_timeout")):
                max_bridge_request_timeout = int(tags.text)

        except Exception as e:
            logger.error("Unspecified XML parse error" + str(e))

        return max_bridge_request_timeout

    def get_custom_temp_path(self):
        et = ET.parse(CONFIG_FILE)
        custom_temp_path = ""
        for tags in et.iter(str("custom_tmp_path")):
            custom_temp_path = str(tags.get('path'))

        return custom_temp_path

    def save_custom_temp_path(self, custom_path):

        with open(CONFIG_FILE, 'r') as conf_file:
            XML_conf_data = conf_file.read().replace('\n', '')
        soup = Soup(XML_conf_data)
        custom_tmp_path_tag = soup.configuration.client.custom_temp_path
        custom_tmp_path_tag['path'] = 'Updated'

        return True
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
###
# (C) Copyright (2012-2017) Hewlett Packard Enterprise Development LP
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in
# all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
# THE SOFTWARE.
###

from config_loader import try_load_from_file
from hpOneView.oneview_client import OneViewClient
from pprint import pprint

config = {
    "ip": "<oneview_ip>",
    "credentials": {
        "userName": "<username>",
        "password": "<password>"
    }
}

# Try load config from a file (if there is a config file)
config = try_load_from_file(config)
oneview_client = OneViewClient(config)

# Example license Key to be used
# ACDE B9MA H9PY CHW3 U7B5 HWW5 Y9JL KMPL B89H MZVU DXAU 2CSM GHTG L762 KQL7 EG5M KJVT D5KM AFVW TT5J F77K NXWW BPSM YF26 28JS EWTZ X36Q
# M5G7 WZJL HH5Q L975 SNJT 288F ADT2 LK44 56UG V8MC 2K9X 7KG2 F6AD EMVA 9GEB 95Y6 XBM3 HVDY LBSS PU24 KEWY JSJC FPZC 2JJE
# ZLAB\"24R2-02192-002 T1111A HP_OneView_w/o_iLO_Explicit_Feature J4E8IAMANEON\"

options = {
    "key": "Use the example key mentioned above",
    "type": "LicenseV500"
}

# Add a License
license = oneview_client.licenses.create(options)
print("\n\nLicense added '%s' successfully.\n" % license['key'])

# Get all licenses
print("\n\n\n\n **********   Displaying all the Licenses loaded on DCS:  ********** \n\n\n\n")
licenses = oneview_client.licenses.get_all()
pprint(licenses)

# Get License by ID
print("\n\n ************   Getting the ID to fetch the license by ID:  ************ \n")
uri = license['uri']
print uri
print("\n License fetched by ID is: \n")
license = oneview_client.licenses.get_by_id(uri)
pprint(license)

# Delete License by ID
print ("\n\n   ********** Delete the license by ID:  **********")
print uri
oneview_client.licenses.delete(uri)

print("\n Check if the license is Deleted: \n")
lic = oneview_client.licenses.get_by_id(uri)
pprint(lic)
<commit_msg>incorporated Madhav's comments<commit_after># -*- coding: utf-8 -*-
###
# (C) Copyright (2012-2017) Hewlett Packard Enterprise Development LP
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in
# all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
# THE SOFTWARE.
###

from config_loader import try_load_from_file
from hpOneView.oneview_client import OneViewClient
from pprint import pprint

config = {
    "ip": "<oneview_ip>",
    "credentials": {
        "userName": "<username>",
        "password": "<password>"
    }
}

# Try load config from a file (if there is a config file)
config = try_load_from_file(config)
oneview_client = OneViewClient(config)

# Example license Key to be used
# ACDE B9MA H9PY CHW3 U7B5 HWW5 Y9JL KMPL B89H MZVU DXAU 2CSM GHTG L762 KQL7 EG5M KJVT D5KM AFVW TT5J F77K NXWW BPSM YF26 28JS EWTZ X36Q
# M5G7 WZJL HH5Q L975 SNJT 288F ADT2 LK44 56UG V8MC 2K9X 7KG2 F6AD EMVA 9GEB 95Y6 XBM3 HVDY LBSS PU24 KEWY JSJC FPZC 2JJE
# ZLAB\"24R2-02192-002 T1111A HP_OneView_w/o_iLO_Explicit_Feature J4E8IAMANEON\"

options = {
    "key": "Your license key",
    "type": "LicenseV500"
}

# Add a License
license = oneview_client.licenses.create(options)
print("\n\nLicense added '%s' successfully.\n" % license['key'])

# Get all licenses
print("\n\n\n\n **********   Displaying all the Licenses loaded on the appliance:  ********** \n\n\n\n")
licenses = oneview_client.licenses.get_all()
pprint(licenses)

# Get License by ID
uri = license['uri']
print uri
print("\n License fetched by ID is: \n")
license = oneview_client.licenses.get_by_id(uri)
pprint(license)

# Delete License by ID
print ("\n\n   ********** Delete the license by ID:  **********")
print uri
oneview_client.licenses.delete(uri)

print("\n Check if the license is Deleted: \n")
lic = oneview_client.licenses.get_by_id(uri)
pprint(lic)
<|endoftext|>


<commit_before>from django.db import transaction
from django.db.backends.mysql.base import quote_name
from MySQLdb import ProgrammingError, OperationalError
from MySQLdb.constants import FIELD_TYPE
import re

foreign_key_re = re.compile(r"\sCONSTRAINT `[^`]*` FOREIGN KEY \(`([^`]*)`\) REFERENCES `([^`]*)` \(`([^`]*)`\)")

def get_table_list(cursor):
    "Returns a list of table names in the current database."
    cursor.execute("SHOW TABLES")
    return [row[0] for row in cursor.fetchall()]

def get_table_description(cursor, table_name):
    "Returns a description of the table, with the DB-API cursor.description interface."
    cursor.execute("SELECT * FROM %s LIMIT 1" % quote_name(table_name))
    return cursor.description

def _name_to_index(cursor, table_name):
    """
    Returns a dictionary of {field_name: field_index} for the given table.
    Indexes are 0-based.
    """
    return dict([(d[0], i) for i, d in enumerate(get_table_description(cursor, table_name))])

def get_relations(cursor, table_name):
    """
    Returns a dictionary of {field_index: (field_index_other_table, other_table)}
    representing all relationships to the given table. Indexes are 0-based.
    """
    my_field_dict = _name_to_index(cursor, table_name)
    constraints = []
    relations = {}
    try:
        # This should work for MySQL 5.0.
        cursor.execute("""
            SELECT column_name, referenced_table_name, referenced_column_name
            FROM information_schema.key_column_usage
            WHERE table_name = %s
                AND referenced_table_name IS NOT NULL
                AND referenced_column_name IS NOT NULL""", [table_name])
        constraints.extend(cursor.fetchall())
    except (ProgrammingError, OperationalError):
        # Fall back to "SHOW CREATE TABLE", for previous MySQL versions.
        # Go through all constraints and save the equal matches.
        cursor.execute("SHOW CREATE TABLE %s" % table_name)
        for row in cursor.fetchall():
            pos = 0
            while True:
                match = foreign_key_re.search(row[1], pos)
                if match == None:
                    break
                pos = match.end()
                constraints.append(match.groups())

    for my_fieldname, other_table, other_field in constraints:
        other_field_index = _name_to_index(cursor, other_table)[other_field]
        my_field_index = my_field_dict[my_fieldname]
        relations[my_field_index] = (other_field_index, other_table)

    return relations

def get_indexes(cursor, table_name):
    """
    Returns a dictionary of fieldname -> infodict for the given table,
    where each infodict is in the format:
        {'primary_key': boolean representing whether it's the primary key,
         'unique': boolean representing whether it's a unique index}
    """
    cursor.execute("SHOW INDEX FROM %s" % quote_name(table_name))
    indexes = {}
    for row in cursor.fetchall():
        indexes[row[4]] = {'primary_key': (row[2] == 'PRIMARY'), 'unique': not bool(row[1])}
    return indexes

DATA_TYPES_REVERSE = {
    FIELD_TYPE.BLOB: 'TextField',
    FIELD_TYPE.CHAR: 'CharField',
    FIELD_TYPE.DECIMAL: 'FloatField',
    FIELD_TYPE.DATE: 'DateField',
    FIELD_TYPE.DATETIME: 'DateTimeField',
    FIELD_TYPE.DOUBLE: 'FloatField',
    FIELD_TYPE.FLOAT: 'FloatField',
    FIELD_TYPE.INT24: 'IntegerField',
    FIELD_TYPE.LONG: 'IntegerField',
    FIELD_TYPE.LONGLONG: 'IntegerField',
    FIELD_TYPE.SHORT: 'IntegerField',
    FIELD_TYPE.STRING: 'TextField',
    FIELD_TYPE.TIMESTAMP: 'DateTimeField',
    FIELD_TYPE.TINY_BLOB: 'TextField',
    FIELD_TYPE.MEDIUM_BLOB: 'TextField',
    FIELD_TYPE.LONG_BLOB: 'TextField',
    FIELD_TYPE.VAR_STRING: 'CharField',
}
<commit_msg>Fixed #1830 -- MySQL inspectdb now detects tinyint field properly<commit_after>from django.db import transaction
from django.db.backends.mysql.base import quote_name
from MySQLdb import ProgrammingError, OperationalError
from MySQLdb.constants import FIELD_TYPE
import re

foreign_key_re = re.compile(r"\sCONSTRAINT `[^`]*` FOREIGN KEY \(`([^`]*)`\) REFERENCES `([^`]*)` \(`([^`]*)`\)")

def get_table_list(cursor):
    "Returns a list of table names in the current database."
    cursor.execute("SHOW TABLES")
    return [row[0] for row in cursor.fetchall()]

def get_table_description(cursor, table_name):
    "Returns a description of the table, with the DB-API cursor.description interface."
    cursor.execute("SELECT * FROM %s LIMIT 1" % quote_name(table_name))
    return cursor.description

def _name_to_index(cursor, table_name):
    """
    Returns a dictionary of {field_name: field_index} for the given table.
    Indexes are 0-based.
    """
    return dict([(d[0], i) for i, d in enumerate(get_table_description(cursor, table_name))])

def get_relations(cursor, table_name):
    """
    Returns a dictionary of {field_index: (field_index_other_table, other_table)}
    representing all relationships to the given table. Indexes are 0-based.
    """
    my_field_dict = _name_to_index(cursor, table_name)
    constraints = []
    relations = {}
    try:
        # This should work for MySQL 5.0.
        cursor.execute("""
            SELECT column_name, referenced_table_name, referenced_column_name
            FROM information_schema.key_column_usage
            WHERE table_name = %s
                AND referenced_table_name IS NOT NULL
                AND referenced_column_name IS NOT NULL""", [table_name])
        constraints.extend(cursor.fetchall())
    except (ProgrammingError, OperationalError):
        # Fall back to "SHOW CREATE TABLE", for previous MySQL versions.
        # Go through all constraints and save the equal matches.
        cursor.execute("SHOW CREATE TABLE %s" % table_name)
        for row in cursor.fetchall():
            pos = 0
            while True:
                match = foreign_key_re.search(row[1], pos)
                if match == None:
                    break
                pos = match.end()
                constraints.append(match.groups())

    for my_fieldname, other_table, other_field in constraints:
        other_field_index = _name_to_index(cursor, other_table)[other_field]
        my_field_index = my_field_dict[my_fieldname]
        relations[my_field_index] = (other_field_index, other_table)

    return relations

def get_indexes(cursor, table_name):
    """
    Returns a dictionary of fieldname -> infodict for the given table,
    where each infodict is in the format:
        {'primary_key': boolean representing whether it's the primary key,
         'unique': boolean representing whether it's a unique index}
    """
    cursor.execute("SHOW INDEX FROM %s" % quote_name(table_name))
    indexes = {}
    for row in cursor.fetchall():
        indexes[row[4]] = {'primary_key': (row[2] == 'PRIMARY'), 'unique': not bool(row[1])}
    return indexes

DATA_TYPES_REVERSE = {
    FIELD_TYPE.BLOB: 'TextField',
    FIELD_TYPE.CHAR: 'CharField',
    FIELD_TYPE.DECIMAL: 'FloatField',
    FIELD_TYPE.DATE: 'DateField',
    FIELD_TYPE.DATETIME: 'DateTimeField',
    FIELD_TYPE.DOUBLE: 'FloatField',
    FIELD_TYPE.FLOAT: 'FloatField',
    FIELD_TYPE.INT24: 'IntegerField',
    FIELD_TYPE.LONG: 'IntegerField',
    FIELD_TYPE.LONGLONG: 'IntegerField',
    FIELD_TYPE.SHORT: 'IntegerField',
    FIELD_TYPE.STRING: 'TextField',
    FIELD_TYPE.TIMESTAMP: 'DateTimeField',
    FIELD_TYPE.TINY: 'IntegerField',
    FIELD_TYPE.TINY_BLOB: 'TextField',
    FIELD_TYPE.MEDIUM_BLOB: 'TextField',
    FIELD_TYPE.LONG_BLOB: 'TextField',
    FIELD_TYPE.VAR_STRING: 'CharField',
}
<|endoftext|>


<commit_before>"""CLI client for creep"""

import cmd # Command interpreter logic. Gives us the base class for the client
import distutils.dir_util # Directory utilities
import inspect # Functions to inspect live objects
import json # JSON encoder and decoder
import os # Miscellaneous operating system interfaces
import shlex # Lexical analysis of user input.
import shutil # High-level file operations
import subprocess # Spawn subprocesses, connect in/out pipes, obtain return codes
import sys # System specific parameters and functions
import tempfile # Temporary file utilities
import zipfile # Zip file utilities

from qi.console.client import Client
from operator import attrgetter
from repository import Repository

class CreepClient(Client, cmd.Cmd):
    """Creep Mod Package Manager Client"""

    # Version of this client
    VERSION = '0.1.2'

    # Absolute path where this client is installed
    installdir = ''

    # Absolute path to the dotfile for this client
    appdir = ''

    # Absolute path to the minecraft dir
    minecraftdir = ''

    # Version of minecraft to target for mods
    minecraft_target = '1.7.10'

    def __init__(self, **kwargs):
        """Constructor"""
        cmd.Cmd.__init__(self)
        Client.__init__(self, **kwargs)

        self.updateVersionWithGitDescribe()
        self.updatePaths()
        self.load_target()

        self.createRepository()

    def do_version(self, args):
        """Display creep version"""
        print self.colortext("Creep v{}".format(self.VERSION), self.terminal.C_GREEN)
        self.display_target()

    def do_target(self, args):
        """Set the targeted minecraft version
Usage: creep target <minecraft_version>

Examples:
  creep target 1.7.10
     Set the version of minecraft for mods
"""
        if len(args) > 0:
            # TODO: Validate against ~/.minecraft/launcher_profiles.json for valid versions to use
            self.minecraft_target = args

        self.display_target()
        self.repository.set_minecraft_target(self.minecraft_target)
        self.save_options()

    def display_target(self):
        print self.colortext("Targetting minecraft version {}".format(self.minecraft_target), self.terminal.C_GREEN)

    def load_target(self):
        # TODO: More robust user options file handling. It should be its own object to load options
        options_path = self.appdir + os.sep + 'options.json'
        if os.path.isfile(options_path):
            options = json.load(open(options_path))
            self.minecraft_target = options['minecraft_target']

    def save_options(self):
        options_path = self.appdir + os.sep + 'options.json'
        with open(options_path, 'w') as outfile:
            json.dump({'minecraft_target': self.minecraft_target}, outfile)

    def do_list(self, args):
        """List packages (mods)
Usage: creep list [installed]

Examples:
  creep list
     List available packages in repository

  creep list installed
     List installed packages
"""
        if args == 'installed':
            installdir = self.minecraftdir + os.sep + 'mods' + os.sep + self.minecraft_target
            files = os.listdir(installdir)
            if not files:
                print "No mods installed"
                return False

            packages = []
            unknownfiles = []
            for name in files:
                package = self.repository.fetch_package_byfilename(name)
                if not package:
                    unknownfiles.append(name)
                else:
                    packages.append(package)

            packages.sort(key=attrgetter('name'))

            print self.colortext("Installed mods (in {}):".format(installdir), self.terminal.C_YELLOW)
            for package in packages:
                self.print_package(package)
            for name in unknownfiles:
                print self.colortext(name, self.terminal.C_RED)
        else:
            self.display_packages()

    def display_packages(self):
        """Display list of packages available"""
        for package in self.repository.unique_packages:
            self.print_package(package)

    def print_package(self, package):
        message = '{name} {version} - {description} [{mcversion}]'.format(
            name=package.name,
            version=self.colortext('(' + package.version + ')', self.terminal.C_YELLOW),
            description=self.colortext(package.description, self.terminal.C_MAGENTA),
            mcversion=self.colortext(package.get_minecraft_version(), self.terminal.C_YELLOW)
        )

        if package.type == 'collection':
            message = message + self.colortext(' [collection]', self.terminal.C_CYAN)

        print message

    def print_package_details(self, package):
        print package.name
        print '--------'
        print "Version: " + package.version
        print "Description: " + package.description
        print "Package Type: " + package.type
        print "Keywords: " + package.keywords
        print "Homepage: " + package.homepage
        print "Dependencies: "
        for key,value in package.require.iteritems():
            print " - " + key + ": " + value

    def do_search(self, args):
        """Search for a package (mod)
Usage: creep search <packagename>

Example: creep search nei
"""
        if args == '':
            return False

        packages = self.repository.search(args)
        for package in packages:
            self.print_package(package)

    def do_info(self, args):
        """Display details for a specific package (mod)
Usage: creep info <packagename>

Example: creep info slimeknights/tconstruct
"""
        if len(args) == 0:
            print self.colortext("Missing argument", self.terminal.C_RED)
            return 1

        package = self.repository.fetch_package(args)
        if not package:
            print self.colortext("Unknown package '{}'".format(args), self.terminal.C_RED)
            return 1

        self.print_package_details(package)

    def do_install(self, args):
        """Install a package (mod)
Usage: creep install <packagename>
       creep install -l <listfilename>

Example: creep install thecricket/chisel2
         creep install -l mymodlist.txt

<listfilename> is a list of packages to install consisting of one package name per line
"""
        args = shlex.split(args)

        if len(args) == 0:
            print self.colortext("Missing argument", self.terminal.C_RED)
            return 1

        if args[0] == '-l':
            # Handle install from listfile
            try:
                listfile = args[1]
            except IndexError:
                print self.colortext("Missing argument for listfile", self.terminal.C_RED)
                return 1
            self.install_from_listfile(listfile)
        else:
            # Handle install individual package
            self.install_package(args[0])

    def install_package(self, args):
        package = self.repository.fetch_package(args)
        if not package:
            print self.colortext("Unknown package '{}'".format(args), self.terminal.C_RED)
            return 1

        print self.colortext("Installing package {}".format(package), self.terminal.C_BLUE)

        # Install any required packages
        # Warning: does not handle versions or circular dependencies
        for dependency in package.require:
            if dependency == 'minecraft' or dependency == 'forge':
                continue
            print self.colortext("Installing dependency '{}'".format(dependency), self.terminal.C_CYAN)
            self.do_install(dependency)

        if package.type == 'collection':
            # Collection only has dependencies
            print self.colortext("Installed collection '{0}'".format(package.name), self.terminal.C_GREEN)
        else:
            cachedir = self.appdir + os.sep + 'cache' + os.sep + package.installdir

            if not os.path.isdir(cachedir):
                os.mkdir(cachedir)

            if not os.path.isfile(cachedir + os.sep + package.get_local_filename()):
                print self.colortext("Downloading mod '{0}' from {1}".format(package.name, package.get_download_location()), self.terminal.C_YELLOW)
                downloadResult = package.download(cachedir)

                if not downloadResult:
                    print "Download failed."
                    return False

            # Most of the time this is the '~/.minecraft/mods' dir, but some mods have an alternate location for artifacts
            savedir = self.minecraftdir + os.sep + package.installdir

            # Tack on the minecraft version to the savedir
            if package.installdir == 'mods':
                savedir = savedir + os.sep + package.require['minecraft']

            if not os.path.isdir(savedir):
                print self.colortext("Creating directory '{0}'".format(savedir))
                os.mkdir(savedir)

            if package.installstrategy:
                self.install_with_strategy(package.installstrategy, package, cachedir, savedir)

            shutil.copyfile(cachedir + os.sep + package.get_local_filename(), savedir + os.sep + package.get_local_filename())

            print self.colortext("Installed mod '{0}' in '{1}'".format(package.name, savedir + os.sep + package.get_local_filename()), self.terminal.C_GREEN)

    def install_from_listfile(self, listfile):
        print "Reading packages from file '{}'...".format(listfile)

        if not os.path.isfile(listfile):
            print self.colortext("File '{}' not found".format(listfile), self.terminal.C_RED)
            return 1

        # Read file and attempt to parse each line as a package name
        with open(listfile) as fp:
            for line in fp:
                args = line.split()
                self.install_package(args[0])

    def install_with_strategy(self, installstrategy, package, cachedir, savedir):

        print "Installing with strategy: " + installstrategy

        if ';' in installstrategy:
            strategies = installstrategy.split(';')
        else:
            strategies = [installstrategy]

        # set up a temppath where we will work
        tmppath = tempfile.gettempdir() + os.sep + package.name.replace('/', '_')
        if os.path.exists(tmppath):
            shutil.rmtree(tmppath)
        os.mkdir(tmppath)

        for strategy in strategies:
            args = shlex.split(strategy)
            if args[0] == 'unzip':
                print 'Unzipping archive: ' + cachedir + os.sep + package.get_local_filename()
                self.unzip(cachedir + os.sep + package.get_local_filename(), tmppath)
            elif args[0] == 'move':
                print 'Moving files: ' + args[1]
                path = args[1]
                if path[-2:] == '/*':
                    path = path.replace('/*', '')
                    distutils.dir_util.copy_tree(tmppath + os.sep + path, savedir)
                else:
                    shutil.copytree(tmppath + os.sep + path, savedir)

    def do_uninstall(self, args):
        """Uninstall a package (mod)
Usage: creep uninstall <packagename>

Example: creep uninstall thecricket/chisel2
"""
        package = self.repository.fetch_package(args)
        if not package:
            print 'Unknown package {}'.format(args)
            return 1

        if package.installdir == 'mods':
            savedir = self.minecraftdir + os.sep + package.installdir + os.sep + self.minecraft_target
        else:
            savedir = self.minecraftdir + os.sep + package.installdir

        os.remove(savedir + os.sep + package.get_local_filename())
        print "Removed mod '{0}' from '{1}'".format(package.name, savedir)

    def do_purge(self, args):
        """Purge all installed packages (mods)
Usage: creep purge
"""
        installdir = self.minecraftdir + os.sep + 'mods' + os.sep + self.minecraft_target
        print "Purging all installed mods in {}...".format(installdir)
        self.delete_path(installdir)
        print "Done."

    def do_refresh(self, args):
        """Force an refresh of the package repository"""

        self.repository.clear_cache()
        self.repository.populate()
        print self.colortext("Repository updated to version {} ({}).".format(self.repository.version_hash, self.repository.version_date), self.terminal.C_GREEN)
        print "Count: {} packages.".format(self.repository.count_packages())

    def delete_path(self, rootdir):
        files = os.listdir(rootdir)
        for f in files:
            if os.path.isdir(rootdir + os.sep + f):
                self.delete_path(rootdir + os.sep + f)
                os.rmdir(rootdir + os.sep + f)
            else:
                print self.colortext('Removing file {}'.format(f), self.terminal.C_RED)
                try:
                    os.remove(rootdir + os.sep + f)
                except OSError:
                    print 'opa'
                    continue

    def createRepository(self):
        self.repository = Repository(self.appdir)
        self.repository.set_minecraft_target(self.minecraft_target)

        # Check if local packages repository exists and load it too
        if os.path.isfile(self.appdir + os.sep + 'local-packages.json'):
            self.repository.populate('', False)
            self.repository.populate(self.appdir + os.sep + 'local-packages.json')
        else:
            self.repository.populate('', True)

    def updatePaths(self):
        self.installdir = os.path.dirname(os.path.dirname(os.path.abspath(inspect.getfile(inspect.currentframe()))))

        self.appdir = self.getHomePath('.creep')
        if not os.path.isdir(self.appdir):
            os.mkdir(self.appdir)

        if not os.path.isdir(self.appdir + os.sep + 'cache'):
            os.mkdir(self.appdir + os.sep + 'cache')

        if sys.platform[:3] == 'win':
            self.minecraftdir = self.getHomePath('AppData\\Roaming\\.minecraft')
        elif sys.platform == 'darwin':
            self.minecraftdir = self.getHomePath('Library/Application Support/minecraft')
        elif sys.platform == 'cygwin':
            self.minecraftdir = os.getenv('APPDATA') + os.sep + '.minecraft'
        else:
            self.minecraftdir = self.getHomePath('.minecraft')

        if not os.path.isdir(self.minecraftdir):
            print "Minecraft dir not found ({})".format(self.minecraftdir)
            print "Is Minecraft installed?"
            sys.exit(2)

        if not os.path.isdir(self.minecraftdir + os.sep + 'mods'):
            os.mkdir(self.minecraftdir + os.sep + 'mods')

    def getHomePath(self, path=''):
        """Get the home path for this user from the OS"""
        home = os.getenv('HOME')
        if home == None:
            home = os.getenv('USERPROFILE')

        return home + os.sep + path

    def updateVersionWithGitDescribe(self):
        """Update the version of this client to reflect any local changes in git"""

        appdir = os.path.dirname(os.path.abspath(inspect.getfile(inspect.currentframe())))

        try:
            self.VERSION = subprocess.check_output(['git', '-C', appdir, 'describe'], stderr=subprocess.STDOUT).strip()
        except OSError:
            pass
        except subprocess.CalledProcessError:
            # Oh well, we tried, just use the VERSION as it was
            pass

    def colortext(self, text, forecolor=None, backcolor=None, isbold=None):

        if forecolor is None and backcolor is None and isbold is None:
            return text

        if isbold:
            boldstart = self.terminal.bold()
            boldend = self.terminal.sgr0()
        else:
            boldstart = ''
            boldend = ''

        return '{bold}{start}{text}{end}{unbold}'.format(
            bold=boldstart,
            start=self.colorstart(forecolor, backcolor),
            text=text,
            end=self.colorend(),
            unbold=boldend
        )

    def colorstart(self, forecolor=None, backcolor=None):
        coloring = ''
        if forecolor is not None:
            coloring += self.terminal.setaf(forecolor)
        if backcolor is not None:
            coloring += self.terminal.setab(backcolor)

        return coloring

    def colorend(self):
        return self.terminal.op()

    def unzip(self, source_filename, dest_dir):
        with zipfile.ZipFile(source_filename) as zf:
            zf.extractall(dest_dir)
<commit_msg>Handle when target mods directory is empty<commit_after>"""CLI client for creep"""

import cmd # Command interpreter logic. Gives us the base class for the client
import distutils.dir_util # Directory utilities
import inspect # Functions to inspect live objects
import json # JSON encoder and decoder
import os # Miscellaneous operating system interfaces
import shlex # Lexical analysis of user input.
import shutil # High-level file operations
import subprocess # Spawn subprocesses, connect in/out pipes, obtain return codes
import sys # System specific parameters and functions
import tempfile # Temporary file utilities
import zipfile # Zip file utilities

from qi.console.client import Client
from operator import attrgetter
from repository import Repository

class CreepClient(Client, cmd.Cmd):
    """Creep Mod Package Manager Client"""

    # Version of this client
    VERSION = '0.1.2'

    # Absolute path where this client is installed
    installdir = ''

    # Absolute path to the dotfile for this client
    appdir = ''

    # Absolute path to the minecraft dir
    minecraftdir = ''

    # Version of minecraft to target for mods
    minecraft_target = '1.7.10'

    def __init__(self, **kwargs):
        """Constructor"""
        cmd.Cmd.__init__(self)
        Client.__init__(self, **kwargs)

        self.updateVersionWithGitDescribe()
        self.updatePaths()
        self.load_target()

        self.createRepository()

    def do_version(self, args):
        """Display creep version"""
        print self.colortext("Creep v{}".format(self.VERSION), self.terminal.C_GREEN)
        self.display_target()

    def do_target(self, args):
        """Set the targeted minecraft version
Usage: creep target <minecraft_version>

Examples:
  creep target 1.7.10
     Set the version of minecraft for mods
"""
        if len(args) > 0:
            # TODO: Validate against ~/.minecraft/launcher_profiles.json for valid versions to use
            self.minecraft_target = args

        self.display_target()
        self.repository.set_minecraft_target(self.minecraft_target)
        self.save_options()

    def display_target(self):
        print self.colortext("Targetting minecraft version {}".format(self.minecraft_target), self.terminal.C_GREEN)

    def load_target(self):
        # TODO: More robust user options file handling. It should be its own object to load options
        options_path = self.appdir + os.sep + 'options.json'
        if os.path.isfile(options_path):
            options = json.load(open(options_path))
            self.minecraft_target = options['minecraft_target']

    def save_options(self):
        options_path = self.appdir + os.sep + 'options.json'
        with open(options_path, 'w') as outfile:
            json.dump({'minecraft_target': self.minecraft_target}, outfile)

    def do_list(self, args):
        """List packages (mods)
Usage: creep list [installed]

Examples:
  creep list
     List available packages in repository

  creep list installed
     List installed packages
"""
        if args == 'installed':
            installdir = self.minecraftdir + os.sep + 'mods' + os.sep + self.minecraft_target
            files = []
            try:
                files = os.listdir(installdir)
            except OSError:
                pass

            if not files:
                print self.colortext("Looking in {}".format(installdir), self.terminal.C_YELLOW)
                print "No mods installed"
                return False

            packages = []
            unknownfiles = []
            for name in files:
                package = self.repository.fetch_package_byfilename(name)
                if not package:
                    unknownfiles.append(name)
                else:
                    packages.append(package)

            packages.sort(key=attrgetter('name'))

            print self.colortext("Installed mods (in {}):".format(installdir), self.terminal.C_YELLOW)
            for package in packages:
                self.print_package(package)
            for name in unknownfiles:
                print self.colortext(name, self.terminal.C_RED)
        else:
            self.display_packages()

    def display_packages(self):
        """Display list of packages available"""
        for package in self.repository.unique_packages:
            self.print_package(package)

    def print_package(self, package):
        message = '{name} {version} - {description} [{mcversion}]'.format(
            name=package.name,
            version=self.colortext('(' + package.version + ')', self.terminal.C_YELLOW),
            description=self.colortext(package.description, self.terminal.C_MAGENTA),
            mcversion=self.colortext(package.get_minecraft_version(), self.terminal.C_YELLOW)
        )

        if package.type == 'collection':
            message = message + self.colortext(' [collection]', self.terminal.C_CYAN)

        print message

    def print_package_details(self, package):
        print package.name
        print '--------'
        print "Version: " + package.version
        print "Description: " + package.description
        print "Package Type: " + package.type
        print "Keywords: " + package.keywords
        print "Homepage: " + package.homepage
        print "Dependencies: "
        for key,value in package.require.iteritems():
            print " - " + key + ": " + value

    def do_search(self, args):
        """Search for a package (mod)
Usage: creep search <packagename>

Example: creep search nei
"""
        if args == '':
            return False

        packages = self.repository.search(args)
        for package in packages:
            self.print_package(package)

    def do_info(self, args):
        """Display details for a specific package (mod)
Usage: creep info <packagename>

Example: creep info slimeknights/tconstruct
"""
        if len(args) == 0:
            print self.colortext("Missing argument", self.terminal.C_RED)
            return 1

        package = self.repository.fetch_package(args)
        if not package:
            print self.colortext("Unknown package '{}'".format(args), self.terminal.C_RED)
            return 1

        self.print_package_details(package)

    def do_install(self, args):
        """Install a package (mod)
Usage: creep install <packagename>
       creep install -l <listfilename>

Example: creep install thecricket/chisel2
         creep install -l mymodlist.txt

<listfilename> is a list of packages to install consisting of one package name per line
"""
        args = shlex.split(args)

        if len(args) == 0:
            print self.colortext("Missing argument", self.terminal.C_RED)
            return 1

        if args[0] == '-l':
            # Handle install from listfile
            try:
                listfile = args[1]
            except IndexError:
                print self.colortext("Missing argument for listfile", self.terminal.C_RED)
                return 1
            self.install_from_listfile(listfile)
        else:
            # Handle install individual package
            self.install_package(args[0])

    def install_package(self, args):
        package = self.repository.fetch_package(args)
        if not package:
            print self.colortext("Unknown package '{}'".format(args), self.terminal.C_RED)
            return 1

        print self.colortext("Installing package {}".format(package), self.terminal.C_BLUE)

        # Install any required packages
        # Warning: does not handle versions or circular dependencies
        for dependency in package.require:
            if dependency == 'minecraft' or dependency == 'forge':
                continue
            print self.colortext("Installing dependency '{}'".format(dependency), self.terminal.C_CYAN)
            self.do_install(dependency)

        if package.type == 'collection':
            # Collection only has dependencies
            print self.colortext("Installed collection '{0}'".format(package.name), self.terminal.C_GREEN)
        else:
            cachedir = self.appdir + os.sep + 'cache' + os.sep + package.installdir

            if not os.path.isdir(cachedir):
                os.mkdir(cachedir)

            if not os.path.isfile(cachedir + os.sep + package.get_local_filename()):
                print self.colortext("Downloading mod '{0}' from {1}".format(package.name, package.get_download_location()), self.terminal.C_YELLOW)
                downloadResult = package.download(cachedir)

                if not downloadResult:
                    print "Download failed."
                    return False

            # Most of the time this is the '~/.minecraft/mods' dir, but some mods have an alternate location for artifacts
            savedir = self.minecraftdir + os.sep + package.installdir

            # Tack on the minecraft version to the savedir
            if package.installdir == 'mods':
                savedir = savedir + os.sep + package.require['minecraft']

            if not os.path.isdir(savedir):
                print self.colortext("Creating directory '{0}'".format(savedir))
                os.mkdir(savedir)

            if package.installstrategy:
                self.install_with_strategy(package.installstrategy, package, cachedir, savedir)

            shutil.copyfile(cachedir + os.sep + package.get_local_filename(), savedir + os.sep + package.get_local_filename())

            print self.colortext("Installed mod '{0}' in '{1}'".format(package.name, savedir + os.sep + package.get_local_filename()), self.terminal.C_GREEN)

    def install_from_listfile(self, listfile):
        print "Reading packages from file '{}'...".format(listfile)

        if not os.path.isfile(listfile):
            print self.colortext("File '{}' not found".format(listfile), self.terminal.C_RED)
            return 1

        # Read file and attempt to parse each line as a package name
        with open(listfile) as fp:
            for line in fp:
                args = line.split()
                self.install_package(args[0])

    def install_with_strategy(self, installstrategy, package, cachedir, savedir):

        print "Installing with strategy: " + installstrategy

        if ';' in installstrategy:
            strategies = installstrategy.split(';')
        else:
            strategies = [installstrategy]

        # set up a temppath where we will work
        tmppath = tempfile.gettempdir() + os.sep + package.name.replace('/', '_')
        if os.path.exists(tmppath):
            shutil.rmtree(tmppath)
        os.mkdir(tmppath)

        for strategy in strategies:
            args = shlex.split(strategy)
            if args[0] == 'unzip':
                print 'Unzipping archive: ' + cachedir + os.sep + package.get_local_filename()
                self.unzip(cachedir + os.sep + package.get_local_filename(), tmppath)
            elif args[0] == 'move':
                print 'Moving files: ' + args[1]
                path = args[1]
                if path[-2:] == '/*':
                    path = path.replace('/*', '')
                    distutils.dir_util.copy_tree(tmppath + os.sep + path, savedir)
                else:
                    shutil.copytree(tmppath + os.sep + path, savedir)

    def do_uninstall(self, args):
        """Uninstall a package (mod)
Usage: creep uninstall <packagename>

Example: creep uninstall thecricket/chisel2
"""
        package = self.repository.fetch_package(args)
        if not package:
            print 'Unknown package {}'.format(args)
            return 1

        if package.installdir == 'mods':
            savedir = self.minecraftdir + os.sep + package.installdir + os.sep + self.minecraft_target
        else:
            savedir = self.minecraftdir + os.sep + package.installdir

        os.remove(savedir + os.sep + package.get_local_filename())
        print "Removed mod '{0}' from '{1}'".format(package.name, savedir)

    def do_purge(self, args):
        """Purge all installed packages (mods)
Usage: creep purge
"""
        installdir = self.minecraftdir + os.sep + 'mods' + os.sep + self.minecraft_target
        print "Purging all installed mods in {}...".format(installdir)
        self.delete_path(installdir)
        print "Done."

    def do_refresh(self, args):
        """Force an refresh of the package repository"""

        self.repository.clear_cache()
        self.repository.populate()
        print self.colortext("Repository updated to version {} ({}).".format(self.repository.version_hash, self.repository.version_date), self.terminal.C_GREEN)
        print "Count: {} packages.".format(self.repository.count_packages())

    def delete_path(self, rootdir):
        files = os.listdir(rootdir)
        for f in files:
            if os.path.isdir(rootdir + os.sep + f):
                self.delete_path(rootdir + os.sep + f)
                os.rmdir(rootdir + os.sep + f)
            else:
                print self.colortext('Removing file {}'.format(f), self.terminal.C_RED)
                try:
                    os.remove(rootdir + os.sep + f)
                except OSError:
                    print 'opa'
                    continue

    def createRepository(self):
        self.repository = Repository(self.appdir)
        self.repository.set_minecraft_target(self.minecraft_target)

        # Check if local packages repository exists and load it too
        if os.path.isfile(self.appdir + os.sep + 'local-packages.json'):
            self.repository.populate('', False)
            self.repository.populate(self.appdir + os.sep + 'local-packages.json')
        else:
            self.repository.populate('', True)

    def updatePaths(self):
        self.installdir = os.path.dirname(os.path.dirname(os.path.abspath(inspect.getfile(inspect.currentframe()))))

        self.appdir = self.getHomePath('.creep')
        if not os.path.isdir(self.appdir):
            os.mkdir(self.appdir)

        if not os.path.isdir(self.appdir + os.sep + 'cache'):
            os.mkdir(self.appdir + os.sep + 'cache')

        if sys.platform[:3] == 'win':
            self.minecraftdir = self.getHomePath('AppData\\Roaming\\.minecraft')
        elif sys.platform == 'darwin':
            self.minecraftdir = self.getHomePath('Library/Application Support/minecraft')
        elif sys.platform == 'cygwin':
            self.minecraftdir = os.getenv('APPDATA') + os.sep + '.minecraft'
        else:
            self.minecraftdir = self.getHomePath('.minecraft')

        if not os.path.isdir(self.minecraftdir):
            print "Minecraft dir not found ({})".format(self.minecraftdir)
            print "Is Minecraft installed?"
            sys.exit(2)

        if not os.path.isdir(self.minecraftdir + os.sep + 'mods'):
            os.mkdir(self.minecraftdir + os.sep + 'mods')

    def getHomePath(self, path=''):
        """Get the home path for this user from the OS"""
        home = os.getenv('HOME')
        if home == None:
            home = os.getenv('USERPROFILE')

        return home + os.sep + path

    def updateVersionWithGitDescribe(self):
        """Update the version of this client to reflect any local changes in git"""

        appdir = os.path.dirname(os.path.abspath(inspect.getfile(inspect.currentframe())))

        try:
            self.VERSION = subprocess.check_output(['git', '-C', appdir, 'describe'], stderr=subprocess.STDOUT).strip()
        except OSError:
            pass
        except subprocess.CalledProcessError:
            # Oh well, we tried, just use the VERSION as it was
            pass

    def colortext(self, text, forecolor=None, backcolor=None, isbold=None):

        if forecolor is None and backcolor is None and isbold is None:
            return text

        if isbold:
            boldstart = self.terminal.bold()
            boldend = self.terminal.sgr0()
        else:
            boldstart = ''
            boldend = ''

        return '{bold}{start}{text}{end}{unbold}'.format(
            bold=boldstart,
            start=self.colorstart(forecolor, backcolor),
            text=text,
            end=self.colorend(),
            unbold=boldend
        )

    def colorstart(self, forecolor=None, backcolor=None):
        coloring = ''
        if forecolor is not None:
            coloring += self.terminal.setaf(forecolor)
        if backcolor is not None:
            coloring += self.terminal.setab(backcolor)

        return coloring

    def colorend(self):
        return self.terminal.op()

    def unzip(self, source_filename, dest_dir):
        with zipfile.ZipFile(source_filename) as zf:
            zf.extractall(dest_dir)
<|endoftext|>


<commit_before>"""Utils for simplifying building neural networks with TT-layers"""

from itertools import count
import numpy as np
from tensorflow.keras.layers import Layer
from tensorflow.keras.layers import Activation
import t3f
import tensorflow.compat.v1 as tf

class KerasDense(Layer):
  _counter = count(0)

  def __init__(self, input_dims, output_dims, tt_rank=2,
               activation=None, use_bias=True, kernel_initializer='glorot',
               bias_initializer=0.1, **kwargs):
    """Creates a TT-Matrix based Dense Keras layer.

    Args:
      input_dims: an array, tensor shape of the matrix row index
      ouput_dims: an array, tensor shape of the matrix column index
      tt_rank: a number or an array, desired tt-rank of the TT-Matrix
      activation: [None] string or None, specifies the activation function.
      use_bias: bool, whether to use bias
      kernel_initializer: string specifying initializer for the TT-Matrix.
          Possible values are 'glorot', 'he', and 'lecun'.
      bias_initializer: a number, initialization value of the bias

    Returns:
      Layer object corresponding to multiplication by a TT-Matrix
          followed by addition of a bias and applying
          an elementwise activation

    Raises:
        ValueError if the provided activation or kernel_initializer is
        unknown.
    """
    self.counter = next(self._counter)
    self.tt_shape = [input_dims, output_dims]
    self.output_dim = np.prod(output_dims)
    self.tt_rank = tt_rank
    self.activation = activation
    self.use_bias = use_bias
    self.kernel_initializer = kernel_initializer
    self.bias_initializer = bias_initializer
    super(KerasDense, self).__init__(**kwargs)

  def build(self, input_shape):
    if self.kernel_initializer == 'glorot':
      initializer = t3f.glorot_initializer(self.tt_shape,
                                           tt_rank=self.tt_rank)
    elif self.kernel_initializer == 'he':
      initializer = t3f.he_initializer(self.tt_shape,
                                       tt_rank=self.tt_rank)
    elif self.kernel_initializer == 'lecun':
      initializer = t3f.lecun_initializer(self.tt_shape,
                                          tt_rank=self.tt_rank)
    else:
      raise ValueError('Unknown kernel_initializer "%s", only "glorot",'
                       '"he", and "lecun"  are supported'
                       % self.kernel_initializer)
    name = 'tt_dense_{}'.format(self.counter)
    with tf.variable_scope(name):
      self.matrix = t3f.get_variable('matrix', initializer=initializer)
      self.b = None
      if self.use_bias:
        b_init = tf.constant_initializer(self.bias_initializer)
        self.b = tf.get_variable('bias', shape=self.output_dim,
                                 initializer=b_init)
    self._trainable_weights = list(self.matrix.tt_cores)
    if self.b is not None:
      self._trainable_weights.append(self.b)

  def call(self, x):
    res = t3f.matmul(x, self.matrix)
    if self.use_bias:
      res += self.b
    if self.activation is not None:
      res = Activation(self.activation)(res)
    return res

  def compute_output_shape(self, input_shape):
    return (input_shape[0], self.output_dim)
<commit_msg>Use new Keras API for adding weights<commit_after>"""Utils for simplifying building neural networks with TT-layers"""

from itertools import count
import numpy as np
from tensorflow.keras.layers import Layer
from tensorflow.keras.layers import Activation
import t3f
import tensorflow.compat.v1 as tf

class KerasDense(Layer):
  _counter = count(0)

  def __init__(self, input_dims, output_dims, tt_rank=2,
               activation=None, use_bias=True, kernel_initializer='glorot',
               bias_initializer=0.1, **kwargs):
    """Creates a TT-Matrix based Dense Keras layer.

    Args:
      input_dims: an array, tensor shape of the matrix row index
      ouput_dims: an array, tensor shape of the matrix column index
      tt_rank: a number or an array, desired tt-rank of the TT-Matrix
      activation: [None] string or None, specifies the activation function.
      use_bias: bool, whether to use bias
      kernel_initializer: string specifying initializer for the TT-Matrix.
          Possible values are 'glorot', 'he', and 'lecun'.
      bias_initializer: a number, initialization value of the bias

    Returns:
      Layer object corresponding to multiplication by a TT-Matrix
          followed by addition of a bias and applying
          an elementwise activation

    Raises:
        ValueError if the provided activation or kernel_initializer is
        unknown.
    """
    self.counter = next(self._counter)
    self.tt_shape = [input_dims, output_dims]
    self.output_dim = np.prod(output_dims)
    self.tt_rank = tt_rank
    self.activation = activation
    self.use_bias = use_bias
    self.kernel_initializer = kernel_initializer
    self.bias_initializer = bias_initializer
    super(KerasDense, self).__init__(**kwargs)

  def build(self, input_shape):
    if self.kernel_initializer == 'glorot':
      initializer = t3f.glorot_initializer(self.tt_shape,
                                           tt_rank=self.tt_rank)
    elif self.kernel_initializer == 'he':
      initializer = t3f.he_initializer(self.tt_shape,
                                       tt_rank=self.tt_rank)
    elif self.kernel_initializer == 'lecun':
      initializer = t3f.lecun_initializer(self.tt_shape,
                                          tt_rank=self.tt_rank)
    else:
      raise ValueError('Unknown kernel_initializer "%s", only "glorot",'
                       '"he", and "lecun"  are supported'
                       % self.kernel_initializer)
    name = 'tt_dense_{}'.format(self.counter)
    with tf.variable_scope(name):
      self.matrix = t3f.get_variable('matrix', initializer=initializer)
      self.b = None
      if self.use_bias:
        b_init = tf.constant_initializer(self.bias_initializer)
        self.b = self.add_weight('bias', shape=self.output_dim,
                                 initializer=b_init)
    for i,  v in enumerate(self.matrix.tt_cores):
      self.add_weight('%d' % i, initializer=v.initialized_value())

  def call(self, x):
    res = t3f.matmul(x, self.matrix)
    if self.use_bias:
      res += self.b
    if self.activation is not None:
      res = Activation(self.activation)(res)
    return res

  def compute_output_shape(self, input_shape):
    return (input_shape[0], self.output_dim)
<|endoftext|>


<commit_before>import os

from rest_framework import status
from rest_framework.response import Response

from django.conf import settings

from .models import State
from .utils import (
    get_message,
    get_question,
    save_answer,
    check_school,
    verify_answer,
    get_date,
    check_data_validity
)

from schools.models import School
from common.views import KLPAPIView

# Exotel numbers. Find them at http://my.exotel.in/viamentis/apps#installed-apps
PRI = "08039236431"
PRE = "08039510414"
GKA_DEV = "08039510185"
GKA_SMS = "08039514048"
GKA_SERVER = "08039591332"

class SMSView(KLPAPIView):
    def get(self, request):
        # In the SMS functionality of exotel, it expects a 200 response for all
        # its requests if it is to send the 'body' of your response as an sms
        # reply to the sender. Hence you will find places below where
        # status.HTTP_200_OK has been hardcoded in the response.
        content_type = "text/plain"

        date = request.QUERY_PARAMS.get('Date', None)
        data = request.QUERY_PARAMS.get('Body', None)
        ivrs_type = request.QUERY_PARAMS.get('To', None)
        telephone = request.QUERY_PARAMS.get('From', None)
        session_id = request.QUERY_PARAMS.get('SmsSid', None)

        state, created = State.objects.get_or_create(
            session_id=session_id
        )
        state.telephone = telephone
        state.date_of_visit = get_date(date.strip("'"))
        state.answers = []
        # Ignoring index 0 since question_numbers start from 1
        state.answers.append('IGNORED_INDEX')
        state.save()

        original_data = data # Used in the reply sms.
        data = data.split(',')
        data, valid, message = check_data_validity(original_data, data)
        if not valid:
            return Response(
                message,
                status=status.HTTP_200_OK,
                content_type=content_type
            )

        school_id = data.pop(0)

        state, status_code, message = check_school(state, school_id, ivrs_type)
        if status_code != status.HTTP_200_OK:
            return Response(
                message,
                status=status.HTTP_200_OK,
                content_type=content_type
            )

        # Loop over the entire data array and try to validate and save
        # each answer.
        for question_number, response in enumerate(data):
            # Blank data corresponds to NA and indicates that we should
            # skip the corresponding question.
            if response == '':
                continue
            else:
                # question_number starts from 0, and hence we need to add 1
                # to it in order to get the correct sequence of questions.
                # question_number corresponds to questiongroupquestions__sequence
                # while querying for the corresponding Question.
                state, status_code, message = verify_answer(
                    session_id,
                    question_number+1,
                    response,
                    ivrs_type,
                    original_data=original_data
                )
                if status_code != status.HTTP_200_OK:
                    # If we find any of the answers are corrupt, we return
                    # an error response.
                    return Response(
                        message,
                        status=status.HTTP_200_OK,
                        content_type=content_type
                    )
        else:
            message = get_message(
                valid=True,
                date=date,
                data=original_data
            )

        return Response(
            message,
            status=status.HTTP_200_OK,
            content_type=content_type
        )

# This view is on hold for now.
class DynamicResponse(KLPAPIView):
    def get(self, request):
        sound_file_paths = ""

        if settings.IVRS_VOICE_FILES_DIR:
            status_code = status.HTTP_200_OK

            digits = request.QUERY_PARAMS.get('digits', None)

            digits = digits.strip('"')
            path = settings.IVRS_VOICE_FILES_DIR
            path = os.path.join(path, '') # Adds a trailing slash if not present.

            for digit in digits:
                file_name = digit + ".wav\n"
                sound_file_paths += path + file_name
        else:
            status_code = status.HTTP_404_NOT_FOUND

        return Response(
            sound_file_paths,
            status=status_code,
            content_type="text/plain"
        )

class CheckSchool(KLPAPIView):
    def get(self, request):
        status_code = status.HTTP_200_OK

        ivrs_type = request.QUERY_PARAMS.get('To', None)
        telephone = request.QUERY_PARAMS.get('From', None)
        date = request.QUERY_PARAMS.get('StartTime', None)
        school_id = request.QUERY_PARAMS.get('digits', None)
        session_id = request.QUERY_PARAMS.get('CallSid', None)

        state, created = State.objects.get_or_create(
            session_id=session_id
        )
        state.telephone = telephone
        state.date_of_visit = get_date(date)
        state.answers = []
        # Ignoring index 0 since question_numbers start from 1
        state.answers.append('IGNORED_INDEX')

        if school_id:
            school_id = school_id.strip('"')

        state, status_code, message = check_school(state, school_id, ivrs_type)

        return Response("", status=status_code)

class ReadSchool(KLPAPIView):
    def get(self, request):
        session_id = request.QUERY_PARAMS.get('CallSid', None)
        if State.objects.filter(session_id=session_id).exists():
            state = State.objects.get(session_id=session_id)

            status_code = status.HTTP_200_OK
            school = School.objects.get(id=state.school_id)
            # Removes special characters from the school name. Apparently,
            # that was causing Exotel to break.
            school_name = ''.join(
                char for char in school.name if char.isalnum() or char.isspace()
            )
            data = "The ID you have entered is " + \
                   " ".join(str(school.id)) + \
                   " and school name is " + \
                   school_name
        else:
            status_code = status.HTTP_404_NOT_FOUND
            data = ''

        return Response(data, status=status_code, content_type="text/plain")

class VerifyAnswer(KLPAPIView):
    def get(self, request, question_number):
        ivrs_type = request.QUERY_PARAMS.get('To', None)
        session_id = request.QUERY_PARAMS.get('CallSid', None)
        response = request.QUERY_PARAMS.get('digits', None)

        state, status_code, message = verify_answer(
            session_id, question_number, response, ivrs_type,
        )

        return Response("", status=status_code)
<commit_msg>Fixes merge conflicts<commit_after>import os

from rest_framework import status
from rest_framework.response import Response

from django.conf import settings

from .models import State
from .utils import (
    get_message,
    get_question,
    save_answer,
    check_school,
    verify_answer,
    get_date,
    check_data_validity
)

from schools.models import School
from common.views import KLPAPIView

# Exotel numbers. Find them at http://my.exotel.in/viamentis/apps#installed-apps
PRI = "08039236431"
PRE = "08039510414"
GKA_DEV = "08039510185"
GKA_SMS = "08039514048"
GKA_SERVER = "08039591332"

class SMSView(KLPAPIView):
    def get(self, request):
        # In the SMS functionality of exotel, it expects a 200 response for all
        # its requests if it is to send the 'body' of your response as an sms
        # reply to the sender. Hence you will find places below where
        # status.HTTP_200_OK has been hardcoded in the response.
        content_type = "text/plain"

        date = request.QUERY_PARAMS.get('Date', None)
        data = request.QUERY_PARAMS.get('Body', None)
        ivrs_type = request.QUERY_PARAMS.get('To', None)
        telephone = request.QUERY_PARAMS.get('From', None)
        session_id = request.QUERY_PARAMS.get('SmsSid', None)

        state, created = State.objects.get_or_create(
            session_id=session_id
        )
        state.telephone = telephone
        state.date_of_visit = get_date(date.strip("'"))
        state.answers = []
        # Ignoring index 0 since question_numbers start from 1
        state.answers.append('IGNORED_INDEX')
        state.save()

        original_data = data # Used in the reply sms.
        data = data.split(',')
        data, valid, message = check_data_validity(original_data, data)
        if not valid:
            return Response(
                message,
                status=status.HTTP_200_OK,
                content_type=content_type
            )

        school_id = data.pop(0)

        state, status_code, message = check_school(state, school_id, ivrs_type)
        if status_code != status.HTTP_200_OK:
            return Response(
                message,
                status=status.HTTP_200_OK,
                content_type=content_type
            )

        # Loop over the entire data array and try to validate and save
        # each answer.
        for question_number, response in enumerate(data):
            # Blank data corresponds to NA and indicates that we should
            # skip the corresponding question.
            if response == '':
                continue
            else:
                # question_number starts from 0, and hence we need to add 1
                # to it in order to get the correct sequence of questions.
                # question_number corresponds to questiongroupquestions__sequence
                # while querying for the corresponding Question.
                state, status_code, message = verify_answer(
                    session_id,
                    question_number+1,
                    response,
                    ivrs_type,
                    original_data=original_data
                )
                if status_code != status.HTTP_200_OK:
                    # If we find any of the answers are corrupt, we return
                    # an error response.
                    return Response(
                        message,
                        status=status.HTTP_200_OK,
                        content_type=content_type
                    )
        else:
            message = get_message(
                valid=True,
                date=date,
                data=original_data
            )

        return Response(
            message,
            status=status.HTTP_200_OK,
            content_type=content_type
        )

# This view is on hold for now.
class DynamicResponse(KLPAPIView):
    def get(self, request):
        sound_file_paths = ""

        if settings.IVRS_VOICE_FILES_DIR:
            status_code = status.HTTP_200_OK

            digits = request.QUERY_PARAMS.get('digits', None)

            digits = digits.strip('"')
            path = settings.IVRS_VOICE_FILES_DIR
            path = os.path.join(path, '') # Adds a trailing slash if not present.

            for digit in digits:
                file_name = digit + ".wav\n"
                sound_file_paths += path + file_name
        else:
            status_code = status.HTTP_404_NOT_FOUND

        return Response(
            sound_file_paths,
            status=status_code,
            content_type="text/plain"
        )

class CheckSchool(KLPAPIView):
    def get(self, request):
        status_code = status.HTTP_200_OK

        ivrs_type = request.QUERY_PARAMS.get('To', None)
        telephone = request.QUERY_PARAMS.get('From', None)
        date = request.QUERY_PARAMS.get('StartTime', None)
        school_id = request.QUERY_PARAMS.get('digits', None)
        session_id = request.QUERY_PARAMS.get('CallSid', None)

        state, created = State.objects.get_or_create(
            session_id=session_id
        )
        state.telephone = telephone
        state.date_of_visit = get_date(date)
        state.answers = []
        # Ignoring index 0 since question_numbers start from 1
        state.answers.append('IGNORED_INDEX')

        if school_id:
            school_id = school_id.strip('"')

        state, status_code, message = check_school(state, school_id, ivrs_type)

        return Response("", status=status_code)

class ReadSchool(KLPAPIView):
    def get(self, request):
        session_id = request.QUERY_PARAMS.get('CallSid', None)
        if State.objects.filter(session_id=session_id).exists():
            state = State.objects.get(session_id=session_id)

            status_code = status.HTTP_200_OK
            school = School.objects.get(id=state.school_id)
            # Removes special characters from the school name. Apparently,
            # that was causing Exotel to break.
            school_name = ''.join(
                char for char in school.name if char.isalnum() or char.isspace()
            )
            data = "The ID you have entered is " + \
                   " ".join(str(school.id)) + \
                   " and school name is " + \
                   school_name
        else:
            status_code = status.HTTP_404_NOT_FOUND
            data = ''

        return Response(data, status=status_code, content_type="text/plain")

class VerifyAnswer(KLPAPIView):
    def get(self, request, question_number):
        ivrs_type = request.QUERY_PARAMS.get('To', None)
        session_id = request.QUERY_PARAMS.get('CallSid', None)
        response = request.QUERY_PARAMS.get('digits', None)

        state, status_code, message = verify_answer(
            session_id, question_number, response, ivrs_type,
        )

        return Response("", status=status_code)

def get_date(date):
    date = datetime.datetime.strptime(
        date, '%Y-%m-%d %H-%M-%S'
    )
    date = timezone.make_aware(
        date, timezone.get_current_timezone()
    )
    return date

<|endoftext|>