<commit_before>"""module contains integration code for rust-gn3"""
import json
from functools import reduce
from utility.db_tools import mescape
from utility.db_tools import create_in_clause
from wqflask.correlation.correlation_functions import get_trait_symbol_and_tissue_values
from wqflask.correlation.correlation_gn3_api import create_target_this_trait
from wqflask.correlation.correlation_gn3_api import lit_for_trait_list
from wqflask.correlation.correlation_gn3_api import do_lit_correlation
from gn3.computations.correlations import compute_all_lit_correlation
from gn3.computations.rust_correlation import run_correlation
from gn3.computations.rust_correlation import get_sample_corr_data
from gn3.computations.rust_correlation import parse_tissue_corr_data
from gn3.db_utils import database_connector

def chunk_dataset(dataset,steps,name):

    results = []

    query = """
            SELECT ProbeSetXRef.DataId,ProbeSet.Name
            FROM ProbeSet, ProbeSetXRef, ProbeSetFreeze
            WHERE ProbeSetFreeze.Name = '{}' AND
                  ProbeSetXRef.ProbeSetFreezeId = ProbeSetFreeze.Id AND
                  ProbeSetXRef.ProbeSetId = ProbeSet.Id
    """.format(name)

    with database_connector() as conn:
        curr = conn.cursor()

        curr.execute(query)

        traits_name_dict = dict(curr.fetchall())

    for i in range(0, len(dataset), steps):
        matrix = list(dataset[i:i + steps])
        trait_name = traits_name_dict[matrix[0][0]]

        strains = [trait_name] + [str(value) for (trait_name, strain, value) in matrix]
        results.append(",".join(strains))

    return results

def compute_top_n_sample(start_vars, dataset, trait_list):
    """check if dataset is of type probeset"""

    if dataset.type.lower()!= "probeset":
        return  {}

    def __fetch_sample_ids__(samples_vals, samples_group):

        all_samples = json.loads(samples_vals)
        sample_data = get_sample_corr_data(
            sample_type=samples_group, all_samples=all_samples,
            dataset_samples=dataset.group.all_samples_ordered())

        with database_connector() as conn:

            curr = conn.cursor()

            curr.execute(
                """
                    SELECT Strain.Name, Strain.Id FROM Strain, Species
                    WHERE Strain.Name IN {}
                    and Strain.SpeciesId=Species.Id
                    and Species.name = '{}'
                    """.format(create_in_clause(list(sample_data.keys())),
                               *mescape(dataset.group.species))

            )

            return (sample_data,dict(curr.fetchall()))

    (sample_data,sample_ids) = __fetch_sample_ids__(start_vars["sample_vals"], start_vars["corr_samples_group"])

    with database_connector() as conn:

        curr = conn.cursor()

        #fetching strain data in bulk

        curr.execute(

        """
            SELECT * from ProbeSetData
            where StrainID in {}
            and id in (SELECT ProbeSetXRef.DataId
            FROM (ProbeSet, ProbeSetXRef, ProbeSetFreeze)
            WHERE ProbeSetXRef.ProbeSetFreezeId = ProbeSetFreeze.Id
            and ProbeSetFreeze.Name = '{}'
            and ProbeSet.Name in {}
            and ProbeSet.Id = ProbeSetXRef.ProbeSetId)
           """.format(create_in_clause(list(sample_ids.values())),dataset.name,create_in_clause(trait_list))

        )

        corr_data = chunk_dataset(list(curr.fetchall()),len(sample_ids.values()),dataset.name)

        return run_correlation(corr_data,list(sample_data.values()),"pearson",",")

def compute_top_n_lit(corr_results, this_dataset, this_trait) -> dict:
    (this_trait_geneid, geneid_dict, species) = do_lit_correlation(
        this_trait, this_dataset)

    geneid_dict = {trait_name: geneid for (trait_name, geneid) in geneid_dict.items() if
                   corr_results.get(trait_name)}
    with database_connector() as conn:
        return reduce(
            lambda acc, corr: {**acc, **corr},
            compute_all_lit_correlation(
                conn=conn, trait_lists=list(geneid_dict.items()),
                species=species, gene_id=this_trait_geneid),
            {})

    return {}

def compute_top_n_tissue(this_dataset, this_trait, traits, method):

    # refactor lots of rpt

    trait_symbol_dict = dict({
        trait_name: symbol
        for (trait_name, symbol)
        in this_dataset.retrieve_genes("Symbol").items()
        if traits.get(trait_name)})

    corr_result_tissue_vals_dict = get_trait_symbol_and_tissue_values(
        symbol_list=list(trait_symbol_dict.values()))

    data = parse_tissue_corr_data(symbol_name=this_trait.symbol,
                                  symbol_dict=get_trait_symbol_and_tissue_values(
                                      symbol_list=[this_trait.symbol]),
                                  dataset_symbols=trait_symbol_dict,
                                  dataset_vals=corr_result_tissue_vals_dict)

    if data:
        return run_correlation(
            data[1], data[0], method, ",", "tissue")

    return {}

def merge_results(dict_a: dict, dict_b: dict, dict_c: dict) -> list[dict]:
    """code to merge diff corr  into individual dicts
    a"""

    def __merge__(trait_name, trait_corrs):
        return {
            trait_name: {
                **trait_corrs,
                **dict_b.get(trait_name, {}),
                **dict_c.get(trait_name, {})
            }
        }
    return [__merge__(tname, tcorrs) for tname, tcorrs in dict_a.items()]

def __compute_sample_corr__(
        start_vars: dict, corr_type: str, method: str, n_top: int,
        target_trait_info: tuple):
    """Compute the sample correlations"""
    (this_dataset, this_trait, target_dataset, sample_data) = target_trait_info
    all_samples = json.loads(start_vars["sample_vals"])
    sample_data = get_sample_corr_data(
        sample_type=start_vars["corr_samples_group"], all_samples=all_samples,
        dataset_samples=this_dataset.group.all_samples_ordered())
    target_dataset.get_trait_data(list(sample_data.keys()))

    target_data = []
    for (key, val) in target_dataset.trait_data.items():
        lts = [key] + [str(x) for x in val]
        r = ",".join(lts)
        target_data.append(r)

    return run_correlation(
        target_data, list(sample_data.values()), method, ",", corr_type,
        n_top)

def __compute_tissue_corr__(
        start_vars: dict, corr_type: str, method: str, n_top: int,
        target_trait_info: tuple):
    """Compute the tissue correlations"""
    (this_dataset, this_trait, target_dataset, sample_data) = target_trait_info
    trait_symbol_dict = this_dataset.retrieve_genes("Symbol")
    corr_result_tissue_vals_dict = get_trait_symbol_and_tissue_values(
        symbol_list=list(trait_symbol_dict.values()))

    data = parse_tissue_corr_data(
        symbol_name=this_trait.symbol,
        symbol_dict=get_trait_symbol_and_tissue_values(
            symbol_list=[this_trait.symbol]),
        dataset_symbols=trait_symbol_dict,
        dataset_vals=corr_result_tissue_vals_dict)

    if data:
        return run_correlation(data[1], data[0], method, ",", "tissue")
    return {}

def __compute_lit_corr__(
        start_vars: dict, corr_type: str, method: str, n_top: int,
        target_trait_info: tuple):
    """Compute the literature correlations"""
    (this_dataset, this_trait, target_dataset, sample_data) = target_trait_info
    target_dataset_type = target_dataset.type
    this_dataset_type = this_dataset.type
    (this_trait_geneid, geneid_dict, species) = do_lit_correlation(
        this_trait, this_dataset)

    with database_connector() as conn:
        return compute_all_lit_correlation(
            conn=conn, trait_lists=list(geneid_dict.items()),
            species=species, gene_id=this_trait_geneid)
    return {}

def compute_correlation_rust(
        start_vars: dict, corr_type: str, method: str = "pearson",
        n_top: int = 500, compute_all: bool = False):
    """function to compute correlation"""
    target_trait_info = create_target_this_trait(start_vars)
    (this_dataset, this_trait, target_dataset, sample_data) = (
        target_trait_info)

    # Replace this with `match ...` once we hit Python 3.10
    corr_type_fns = {
        "sample": __compute_sample_corr__,
        "tissue": __compute_tissue_corr__,
        "lit": __compute_lit_corr__
    }
    results = corr_type_fns[corr_type](
        start_vars, corr_type, method, n_top, target_trait_info)

    # END: Replace this with `match ...` once we hit Python 3.10

    top_a = top_b = {}

    if compute_all:

        if corr_type == "sample":

            top_a = compute_top_n_tissue(
            this_dataset, this_trait, results, method)

            top_b = compute_top_n_lit(results, this_dataset, this_trait)

        elif corr_type == "lit":

            #currently fails for lit

            top_a = compute_top_n_sample(start_vars,target_dataset,list(results.keys()))
            top_b =  compute_top_n_tissue(
            this_dataset, this_trait, results, method)

        else:

            top_a = compute_top_n_sample(start_vars,target_dataset,list(results.keys()))

            top_b = compute_top_n_lit(results, this_dataset, this_trait)

    return  {
        "correlation_results": merge_results(
            results, top_a, top_b),
        "this_trait": this_trait.name,
        "target_dataset": start_vars['corr_dataset'],
        "return_results": n_top
    }
<commit_msg>linter fixes<commit_after>"""module contains integration code for rust-gn3"""
import json
from functools import reduce
from utility.db_tools import mescape
from utility.db_tools import create_in_clause
from wqflask.correlation.correlation_functions\
    import get_trait_symbol_and_tissue_values
from wqflask.correlation.correlation_gn3_api import create_target_this_trait
from wqflask.correlation.correlation_gn3_api import lit_for_trait_list
from wqflask.correlation.correlation_gn3_api import do_lit_correlation
from gn3.computations.correlations import compute_all_lit_correlation
from gn3.computations.rust_correlation import run_correlation
from gn3.computations.rust_correlation import get_sample_corr_data
from gn3.computations.rust_correlation import parse_tissue_corr_data
from gn3.db_utils import database_connector

def chunk_dataset(dataset, steps, name):

    results = []

    query = """
            SELECT ProbeSetXRef.DataId,ProbeSet.Name
            FROM ProbeSet, ProbeSetXRef, ProbeSetFreeze
            WHERE ProbeSetFreeze.Name = '{}' AND
                  ProbeSetXRef.ProbeSetFreezeId = ProbeSetFreeze.Id AND
                  ProbeSetXRef.ProbeSetId = ProbeSet.Id
    """.format(name)

    with database_connector() as conn:
        curr = conn.cursor()

        curr.execute(query)

        traits_name_dict = dict(curr.fetchall())

    for i in range(0, len(dataset), steps):
        matrix = list(dataset[i:i + steps])
        trait_name = traits_name_dict[matrix[0][0]]

        strains = [trait_name] + [str(value)
                                  for (trait_name, strain, value) in matrix]
        results.append(",".join(strains))

    return results

def compute_top_n_sample(start_vars, dataset, trait_list):
    """check if dataset is of type probeset"""

    if dataset.type.lower() != "probeset":
        return {}

    def __fetch_sample_ids__(samples_vals, samples_group):

        all_samples = json.loads(samples_vals)
        sample_data = get_sample_corr_data(
            sample_type=samples_group, all_samples=all_samples,
            dataset_samples=dataset.group.all_samples_ordered())

        with database_connector() as conn:

            curr = conn.cursor()

            curr.execute(
                """
                    SELECT Strain.Name, Strain.Id FROM Strain, Species
                    WHERE Strain.Name IN {}
                    and Strain.SpeciesId=Species.Id
                    and Species.name = '{}'
                    """.format(create_in_clause(list(sample_data.keys())),
                               *mescape(dataset.group.species))

            )

            return (sample_data, dict(curr.fetchall()))

    (sample_data, sample_ids) = __fetch_sample_ids__(
        start_vars["sample_vals"], start_vars["corr_samples_group"])

    with database_connector() as conn:

        curr = conn.cursor()

        # fetching strain data in bulk

        curr.execute(

            """
            SELECT * from ProbeSetData
            where StrainID in {}
            and id in (SELECT ProbeSetXRef.DataId
            FROM (ProbeSet, ProbeSetXRef, ProbeSetFreeze)
            WHERE ProbeSetXRef.ProbeSetFreezeId = ProbeSetFreeze.Id
            and ProbeSetFreeze.Name = '{}'
            and ProbeSet.Name in {}
            and ProbeSet.Id = ProbeSetXRef.ProbeSetId)
           """.format(create_in_clause(list(sample_ids.values())), dataset.name, create_in_clause(trait_list))

        )

        corr_data = chunk_dataset(list(curr.fetchall()), len(
            sample_ids.values()), dataset.name)

        return run_correlation(corr_data,
                               list(sample_data.values()),
                               "pearson", ",")

def compute_top_n_lit(corr_results, this_dataset, this_trait) -> dict:
    (this_trait_geneid, geneid_dict, species) = do_lit_correlation(
        this_trait, this_dataset)

    geneid_dict = {trait_name: geneid for (trait_name, geneid)
                   in geneid_dict.items() if
                   corr_results.get(trait_name)}
    with database_connector() as conn:
        return reduce(
            lambda acc, corr: {**acc, **corr},
            compute_all_lit_correlation(
                conn=conn, trait_lists=list(geneid_dict.items()),
                species=species, gene_id=this_trait_geneid),
            {})

    return {}

def compute_top_n_tissue(this_dataset, this_trait, traits, method):

    # refactor lots of rpt

    trait_symbol_dict = dict({
        trait_name: symbol
        for (trait_name, symbol)
        in this_dataset.retrieve_genes("Symbol").items()
        if traits.get(trait_name)})

    corr_result_tissue_vals_dict = get_trait_symbol_and_tissue_values(
        symbol_list=list(trait_symbol_dict.values()))

    data = parse_tissue_corr_data(symbol_name=this_trait.symbol,
                                  symbol_dict=get_trait_symbol_and_tissue_values(
                                      symbol_list=[this_trait.symbol]),
                                  dataset_symbols=trait_symbol_dict,
                                  dataset_vals=corr_result_tissue_vals_dict)

    if data:
        return run_correlation(
            data[1], data[0], method, ",", "tissue")

    return {}

def merge_results(dict_a: dict, dict_b: dict, dict_c: dict) -> list[dict]:
    """code to merge diff corr  into individual dicts
    a"""

    def __merge__(trait_name, trait_corrs):
        return {
            trait_name: {
                **trait_corrs,
                **dict_b.get(trait_name, {}),
                **dict_c.get(trait_name, {})
            }
        }
    return [__merge__(tname, tcorrs) for tname, tcorrs in dict_a.items()]

def __compute_sample_corr__(
        start_vars: dict, corr_type: str, method: str, n_top: int,
        target_trait_info: tuple):
    """Compute the sample correlations"""
    (this_dataset, this_trait, target_dataset, sample_data) = target_trait_info
    all_samples = json.loads(start_vars["sample_vals"])
    sample_data = get_sample_corr_data(
        sample_type=start_vars["corr_samples_group"], all_samples=all_samples,
        dataset_samples=this_dataset.group.all_samples_ordered())
    target_dataset.get_trait_data(list(sample_data.keys()))

    target_data = []
    for (key, val) in target_dataset.trait_data.items():
        lts = [key] + [str(x) for x in val]
        r = ",".join(lts)
        target_data.append(r)

    return run_correlation(
        target_data, list(sample_data.values()), method, ",", corr_type,
        n_top)

def __compute_tissue_corr__(
        start_vars: dict, corr_type: str, method: str, n_top: int,
        target_trait_info: tuple):
    """Compute the tissue correlations"""
    (this_dataset, this_trait, target_dataset, sample_data) = target_trait_info
    trait_symbol_dict = this_dataset.retrieve_genes("Symbol")
    corr_result_tissue_vals_dict = get_trait_symbol_and_tissue_values(
        symbol_list=list(trait_symbol_dict.values()))

    data = parse_tissue_corr_data(
        symbol_name=this_trait.symbol,
        symbol_dict=get_trait_symbol_and_tissue_values(
            symbol_list=[this_trait.symbol]),
        dataset_symbols=trait_symbol_dict,
        dataset_vals=corr_result_tissue_vals_dict)

    if data:
        return run_correlation(data[1], data[0], method, ",", "tissue")
    return {}

def __compute_lit_corr__(
        start_vars: dict, corr_type: str, method: str, n_top: int,
        target_trait_info: tuple):
    """Compute the literature correlations"""
    (this_dataset, this_trait, target_dataset, sample_data) = target_trait_info
    target_dataset_type = target_dataset.type
    this_dataset_type = this_dataset.type
    (this_trait_geneid, geneid_dict, species) = do_lit_correlation(
        this_trait, this_dataset)

    with database_connector() as conn:
        return compute_all_lit_correlation(
            conn=conn, trait_lists=list(geneid_dict.items()),
            species=species, gene_id=this_trait_geneid)
    return {}

def compute_correlation_rust(
        start_vars: dict, corr_type: str, method: str = "pearson",
        n_top: int = 500, compute_all: bool = False):
    """function to compute correlation"""
    target_trait_info = create_target_this_trait(start_vars)
    (this_dataset, this_trait, target_dataset, sample_data) = (
        target_trait_info)

    # Replace this with `match ...` once we hit Python 3.10
    corr_type_fns = {
        "sample": __compute_sample_corr__,
        "tissue": __compute_tissue_corr__,
        "lit": __compute_lit_corr__
    }
    results = corr_type_fns[corr_type](
        start_vars, corr_type, method, n_top, target_trait_info)

    # END: Replace this with `match ...` once we hit Python 3.10

    top_a = top_b = {}

    if compute_all:

        if corr_type == "sample":

            top_a = compute_top_n_tissue(
                this_dataset, this_trait, results, method)

            top_b = compute_top_n_lit(results, this_dataset, this_trait)

        elif corr_type == "lit":

            # currently fails for lit

            top_a = compute_top_n_sample(
                start_vars, target_dataset, list(results.keys()))
            top_b = compute_top_n_tissue(
                this_dataset, this_trait, results, method)

        else:

            top_a = compute_top_n_sample(
                start_vars, target_dataset, list(results.keys()))

            top_b = compute_top_n_lit(results, this_dataset, this_trait)

    return {
        "correlation_results": merge_results(
            results, top_a, top_b),
        "this_trait": this_trait.name,
        "target_dataset": start_vars['corr_dataset'],
        "return_results": n_top
    }
<|endoftext|>


<commit_before>__version__ = '0.6'
<commit_msg>version++<commit_after>__version__ = '0.7'
<|endoftext|>


<commit_before># coding: utf-8

# Copyright:
#  Copyright (C) 2014 by Ronnie Sahlberg<ronniesahlberg@gmail.com>
#  Copyright (C) 2015 by Markus Rosjat<markus.rosjat@gmail.com>
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU Lesser General Public License as published by
# the Free Software Foundation; either version 2.1 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public License
# along with this program; if not, see <http://www.gnu.org/licenses/>.

from pyscsi.pyscsi.scsi_cdb_exchangemedium import ExchangeMedium
from pyscsi.pyscsi.scsi_cdb_getlbastatus import GetLBAStatus
from pyscsi.pyscsi.scsi_cdb_initelementstatus import InitializeElementStatus
from pyscsi.pyscsi.scsi_cdb_initelementstatuswithrange import InitializeElementStatusWithRange
from pyscsi.pyscsi.scsi_cdb_inquiry import Inquiry
from pyscsi.pyscsi.scsi_cdb_modesense6 import ModeSense6, ModeSelect6
from pyscsi.pyscsi.scsi_cdb_modesense10 import ModeSense10, ModeSelect10
from pyscsi.pyscsi.scsi_cdb_movemedium import MoveMedium
from pyscsi.pyscsi.scsi_cdb_openclose_exportimport_element import OpenCloseImportExportElement
from pyscsi.pyscsi.scsi_cdb_positiontoelement import PositionToElement
from pyscsi.pyscsi.scsi_cdb_preventallow_mediumremoval import PreventAllowMediumRemoval
from pyscsi.pyscsi.scsi_cdb_read10 import Read10
from pyscsi.pyscsi.scsi_cdb_read12 import Read12
from pyscsi.pyscsi.scsi_cdb_read16 import Read16
from pyscsi.pyscsi.scsi_cdb_readcapacity10 import ReadCapacity10
from pyscsi.pyscsi.scsi_cdb_readcapacity16 import ReadCapacity16
from pyscsi.pyscsi.scsi_cdb_readelementstatus import ReadElementStatus
from pyscsi.pyscsi.scsi_cdb_report_luns import ReportLuns
from pyscsi.pyscsi.scsi_cdb_report_priority import ReportPriority
from pyscsi.pyscsi.scsi_cdb_testunitready import TestUnitReady
from pyscsi.pyscsi.scsi_cdb_write10 import Write10
from pyscsi.pyscsi.scsi_cdb_write12 import Write12
from pyscsi.pyscsi.scsi_cdb_write16 import Write16
from pyscsi.pyscsi.scsi_cdb_writesame10 import WriteSame10
from pyscsi.pyscsi.scsi_cdb_writesame16 import WriteSame16
from pyscsi.pyscsi.scsi_enum_command import spc, sbc, smc, ssc, mmc

class SCSI(object):
    """
    The interface to  the specialized scsi classes
    """
    def __init__(self, dev):
        self.device = dev
        self._blocksize = 0
        self.__init_opcode()

    def __call__(self, dev):
        self.device = dev
        self.__init_opcode()

    def __init_opcode(self):
        """
        Small helper method to terminate the type of
        the scsi device and assigning a proper opcode
        mapper.
        """
        if self.device is not None:
            self.device.devicetype = self.inquiry().result['peripheral_device_type']
            if self.device.devicetype in (0x00, 0x04, 0x07, ):  # sbc
                self.device.opcodes = sbc
            elif self.device.devicetype in (0x01, 0x02, 0x09):  # ssc
                self.device.opcodes = ssc
            elif self.device.devicetype in (0x03,):  # spc
                self.device.opcodes = spc
            elif self.device.devicetype in (0x08,):  # smc
                self.device.opcodes = smc
            elif self.device.devicetype in (0x05,):  # mmc
                self.device.opcodes = mmc

    @property
    def blocksize(self):
        """
        getter method of the blocksize property

        :return: blocksize in bytes
        """
        return self._blocksize

    @blocksize.setter
    def blocksize(self, value):
        """
        setter method of the blocksize property

        :param: blocksize in bytes
        """
        self._blocksize = value

    def exchangemedium(self, xfer, source, dest1, dest2, **kwargs):
        """
        Returns a ExchangeMedium Instance

        :param xfer: medium transport element to use
        :param source: source element
        :param dest1: destination 1 element
        :param dest2: destination 2 element
        :param inv1=0: invert/rotate the medium before loading
        :param inv2=0: invert/rotate the medium before loading
        :return: an ExchangeMedium instance
        """
        return ExchangeMedium(self, xfer, source, dest1, dest2, **kwargs)

    def getlbastatus(self, lba, **kwargs):
        """
        Returns a GetLBAStatus Instance

        :param lba: starting lba
        :param alloc_len = 16384: size of requested datain
        :return: a getlbastatus instance
        """
        return GetLBAStatus(self, lba, **kwargs)

    def inquiry(self, **kwargs):
        """
        Returns a Inquiry Instance

        :param evpd = 0: a byte indicating if vital product data is supported
        :param page_code = 0: a byte representing a page code for vpd
        :param alloc_len = 96: a integer , the size of the data_in buffer
        :return: a Inquiry instance
        """
        return Inquiry(self, **kwargs)

    def initializeelementstatus(self):
        """
        Returns a InitializeElementStatus Instance

        :return: a InitializeElementStatus instance
        """
        return InitializeElementStatus(self)

    def initializeelementstatuswithrange(self, xfer, elements, **kwargs):
        """
        Returns a InitializeElementStatusWithRange Instance

        :param xfer: two byte indicating the address of the starting element
        :param elements: two byte representing a range of elements that should be initialized
        :param range = 0: a integer indicating if elements should be ignored
        :param fast = 0: a integer indicating if  elements should be scanned for media presence
        :return: a InitializeElementStatusWithRange instance
        """
        return InitializeElementStatusWithRange(self, xfer, elements, **kwargs)

    def modeselect6(self, data, **kwargs):
        """
        Returns a ModeSelect6 Instance

        :param data: a dict containing the mode page to set
        :param pf = 0: Page Format flag
        :param sp = 0: Save Pages flag
        :return: a ModeSelect6 instance
        """
        return ModeSelect6(self, data, **kwargs)

    def modesense6(self, page_code, **kwargs):
        """
        Returns a ModeSense6 Instance

        :param page_code:  The page requested
        :param sub_page_code = 0: Requested subpage
        :param dbd = 0: Disable Block Descriptors flag
        :param pc = 0: Page Control flag
        :param alloclen = 96
        :return: a ModeSense6 instance
        """
        return ModeSense6(self, page_code, **kwargs)

    def modesense10(self, page_code, **kwargs):
        """
        Returns a ModeSense10 Instance

        :param page_code:  The page requested
        :param sub_page_code = 0: Requested subpage
        :param llbaa = 0:
        :param dbd = 0: Disable Block Descriptors flag
        :param pc = 0: Page Control flag
        :param alloclen = 96
        :return: a ModeSense10 instance
        """
        return ModeSense10(self, page_code, **kwargs)

    def modeselect10(self, data, **kwargs):
        """
        Returns a ModeSelect10 Instance

        :param data: a dict containing the mode page to set
        :param pf = 0: Page Format flag
        :param sp = 0: Save Pages flag
        :return: a ModeSelect10 instance
        """
        return ModeSelect10(self, data, **kwargs)

    def opencloseimportexportelement(self, xfer, acode):
        """
        Returns a OpenCloseImportExportElement Instance

        :param data: a dict containing the mode page to set
        :param pf = 0: Page Format flag
        :param sp = 0: Save Pages flag
        :return: a OpenCloseImportExportElement instance
        """
        return OpenCloseImportExportElement(self, xfer, acode)

    def positiontoelement(self, xfer, dest, **kwargs):
        """
        Returns a PositionToElement Instance

        :param xfer: medium transport element to use
        :param dest: destination element
        :param invert=0: invert/rotate the medium before loading
        :return: an PositionToElement instance
        """
        return PositionToElement(self, xfer, dest, **kwargs)

    def preventallowmediumremoval(self, **kwargs):
        """
        Returns a PreventAllowMediumRemoval Instance

        :param prevent=0: prevent/allow the medium to be removed
        :return: an PreventAllowMediumRemoval instance
        """
        return PreventAllowMediumRemoval(self, **kwargs)

    def read10(self, lba, tl, **kwargs):
        """
        Returns a Read10 Instance

        :param lba: Logical Block Address
        :param tl: Transfer Length
        :param rdprotect = 0: ReadProtect flags
        :param dpo = 0: DisablePageOut flag
        :param fua = 0: Force Unit Access flag
        :param rarc = 0: Rebuild Assist Recovery control flag
        :param group = 0: Group Number
        :returns the data returned in self.datain
        """
        return Read10(self, lba, tl, **kwargs)

    def read12(self, lba, tl, **kwargs):
        """
        Returns a Read12 Instance

        :param lba: Logical Block Address
        :param tl: Transfer Length
        :param rdprotect = 0: ReadProtect flags
        :param dpo = 0: DisablePageOut flag
        :param fua = 0: Force Unit Access flag
        :param rarc = 0: Rebuild Assist Recovery control flag
        :param group = 0: Group Number
        :returns the data returned in self.datain
        """
        return Read12(self, lba, tl, **kwargs)

    def read16(self, lba, tl, **kwargs):
        """
        Returns a Read16 Instance

        :param lba: Logical Block Address
        :param tl: Transfer Length
        :param rdprotect = 0: ReadProtect flags
        :param dpo = 0: DisablePageOut flag
        :param fua = 0: Force Unit Access flag
        :param rarc = 0: Rebuild Assist Recovery control flag
        :param group = 0: Group Number
        :returns the data returned in self.datain
        """
        return Read16(self, lba, tl, **kwargs)

    def readcapacity10(self, **kwargs):
        """
        Returns a ReadCapacity10 Instance

        :param alloc_len = 8: size of requested datain
        :return: a ReadCapacity10 instance
        """
        return ReadCapacity10(self, **kwargs)

    def readcapacity16(self, **kwargs):
        """
        Returns a ReadCapacity16 Instance

        :param alloc_len = 32: size of requested datain
        :return: a ReadCapacity16 instance
        """
        return ReadCapacity16(self, **kwargs)

    def readelementstatus(self, start, num, **kwargs):
        """
        Returns a ReadElementStatus Instance

        :param start: starting address for first element to return
        :param num: numbver of elements to return
        :param element_type=ELEMENT_TYPE.ALL: type of elements to return
        :param voltag = 0: whether volume tag data should be returned
        :param curdata = 1: check current data
        :param dvcid = 0: whether to return device identifiers
        :param alloclen=16384: max amount od data to return
        :return: an ReadElementStatus instance
        """
        return ReadElementStatus(self, start, num, **kwargs)

    def movemedium(self, xfer, source, dest, **kwargs):
        """
        Returns a MoveMedium Instance

        :param xfer: medium transport element to use
        :param source: source element
        :param dest: destination element
        :param invert=0: invert/rotate the medium before loading
        :return: an MoveMedium instance
        """
        return MoveMedium(self, xfer, source, dest, **kwargs)

    def testunitready(self):
        """
        Returns a TestUnitReady Instance

        No params
        No return value
        """
        return TestUnitReady(self)

    def write10(self, lba, tl, data, **kwargs):
        """
        Returns a Write10 Instance

        :param lba: Logical Block Address to write to
        :param tl: Transfer Length in blocks
        :param data: bytearray containing the data to write
        :param wrprotect = 0: WriteProtect flags
        :param dpo = 0: disable Page Out flag
        :param fua = 0: Force Unit Access flag
        :param group = 0: Group Number
        :return: a Write10 instance
        """
        return Write10(self, lba, tl, data, **kwargs)

    def write12(self, lba, tl, data, **kwargs):
        """
        Returns a Write12 Instance

        :param lba: Logical Block Address to write to
        :param tl: Transfer Length in blocks
        :param data: bytearray containing the data to write
        :param wrprotect = 0: WriteProtect flags
        :param dpo = 0: disable Page Out flag
        :param fua = 0: Force Unit Access flag
        :param group = 0: Group Number
        :return: a Write12 instance
        """
        return Write12(self, lba, tl, data, **kwargs)

    def write16(self, lba, tl, data, **kwargs):
        """
        Returns a Write16 Instance

        :param lba: Logical Block Address to write to
        :param tl: Transfer Length in blocks
        :param data: bytearray containing the data to write
        :param wrprotect = 0: WriteProtect flags
        :param dpo = 0: disable Page Out flag
        :param fua = 0: Force Unit Access flag
        :param group = 0: Group Number
        :return: a Write16 instance
        """
        return Write16(self, lba, tl, data, **kwargs)

    def writesame16(self, lba, nb, data, **kwargs):
        """
        Returns a WriteSame16 Instance

        :param lba: Logical Block Address to write to
        :param nb: Number of blocks
        :param data: bytearray containing the block to write
        :param wrprotect = 0: WriteProtect flags
        :param anchor = False: Anchor flag
        :param unmap = False: Unmap flag
        :param ndob = False: NoDataOutBuffer flag. When set data is None.
        :param group = 0: Group Number
        :return: a WriteSame16 instance
        """
        return WriteSame16(self, lba, nb, data, **kwargs)

    def writesame10(self, lba, nb, data, **kwargs):
        """
        Returns a WriteSame10 Instance

        :param lba: Logical Block Address to write to
        :param nb: Number of blocks
        :param data: bytearray containing the block to write
        :param wrprotect = 0: WriteProtect flags
        :param anchor = False: Anchor flag
        :param unmap = False: Unmap flag
        :param group = 0: Group Number
        :return: a WriteSame10 instance
        """
        return WriteSame10(self, lba, nb, data, **kwargs)

    def reportluns(self, **kwargs):
        """
        Return a ReportLuns Instance

        :param report=0x00: type of logical unit addresses that shall be reported
        :param alloclen=96: size of requested datain
        :return: a ReportLuns instance
        """
        return ReportLuns(self, **kwargs)

    def reportpriority(self, **kwargs):
        """
        Return a ReportPriority Instance

        :param priority=0: specifies information to be returned in data_in buffer
        :param alloclen=16384: size of requested datain
        :return: a ReportLuns instance
        """
        return ReportPriority(self, **kwargs)<commit_msg>adding blocksize as arg to the SCSI __init__  and __call__ method<commit_after># coding: utf-8

# Copyright:
#  Copyright (C) 2014 by Ronnie Sahlberg<ronniesahlberg@gmail.com>
#  Copyright (C) 2015 by Markus Rosjat<markus.rosjat@gmail.com>
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU Lesser General Public License as published by
# the Free Software Foundation; either version 2.1 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public License
# along with this program; if not, see <http://www.gnu.org/licenses/>.

from pyscsi.pyscsi.scsi_cdb_exchangemedium import ExchangeMedium
from pyscsi.pyscsi.scsi_cdb_getlbastatus import GetLBAStatus
from pyscsi.pyscsi.scsi_cdb_initelementstatus import InitializeElementStatus
from pyscsi.pyscsi.scsi_cdb_initelementstatuswithrange import InitializeElementStatusWithRange
from pyscsi.pyscsi.scsi_cdb_inquiry import Inquiry
from pyscsi.pyscsi.scsi_cdb_modesense6 import ModeSense6, ModeSelect6
from pyscsi.pyscsi.scsi_cdb_modesense10 import ModeSense10, ModeSelect10
from pyscsi.pyscsi.scsi_cdb_movemedium import MoveMedium
from pyscsi.pyscsi.scsi_cdb_openclose_exportimport_element import OpenCloseImportExportElement
from pyscsi.pyscsi.scsi_cdb_positiontoelement import PositionToElement
from pyscsi.pyscsi.scsi_cdb_preventallow_mediumremoval import PreventAllowMediumRemoval
from pyscsi.pyscsi.scsi_cdb_read10 import Read10
from pyscsi.pyscsi.scsi_cdb_read12 import Read12
from pyscsi.pyscsi.scsi_cdb_read16 import Read16
from pyscsi.pyscsi.scsi_cdb_readcapacity10 import ReadCapacity10
from pyscsi.pyscsi.scsi_cdb_readcapacity16 import ReadCapacity16
from pyscsi.pyscsi.scsi_cdb_readbuffer10 import ReadBuffer10
from pyscsi.pyscsi.scsi_cdb_readelementstatus import ReadElementStatus
from pyscsi.pyscsi.scsi_cdb_report_luns import ReportLuns
from pyscsi.pyscsi.scsi_cdb_report_priority import ReportPriority
from pyscsi.pyscsi.scsi_cdb_testunitready import TestUnitReady
from pyscsi.pyscsi.scsi_cdb_write10 import Write10
from pyscsi.pyscsi.scsi_cdb_write12 import Write12
from pyscsi.pyscsi.scsi_cdb_write16 import Write16
from pyscsi.pyscsi.scsi_cdb_writesame10 import WriteSame10
from pyscsi.pyscsi.scsi_cdb_writesame16 import WriteSame16
from pyscsi.pyscsi.scsi_enum_command import spc, sbc, smc, ssc, mmc

class SCSI(object):
    """
    The interface to  the specialized scsi classes
    """
    def __init__(self, dev, blocksize=0):
        self.device = dev
        self._blocksize = blocksize
        self.__init_opcode()

    def __call__(self, dev, blocksize=0):
        self.device = dev
        elf._blocksize = blocksize
        self.__init_opcode()

    def __init_opcode(self):
        """
        Small helper method to terminate the type of
        the scsi device and assigning a proper opcode
        mapper.
        """
        if self.device is not None:
            self.device.devicetype = self.inquiry().result['peripheral_device_type']
            if self.device.devicetype in (0x00, 0x04, 0x07, ):  # sbc
                self.device.opcodes = sbc
            elif self.device.devicetype in (0x01, 0x02, 0x09):  # ssc
                self.device.opcodes = ssc
            elif self.device.devicetype in (0x03,):  # spc
                self.device.opcodes = spc
            elif self.device.devicetype in (0x08,):  # smc
                self.device.opcodes = smc
            elif self.device.devicetype in (0x05,):  # mmc
                self.device.opcodes = mmc

    @property
    def blocksize(self):
        """
        getter method of the blocksize property

        :return: blocksize in bytes
        """
        return self._blocksize

    @blocksize.setter
    def blocksize(self, value):
        """
        setter method of the blocksize property

        :param: blocksize in bytes
        """
        self._blocksize = value

    def exchangemedium(self, xfer, source, dest1, dest2, **kwargs):
        """
        Returns a ExchangeMedium Instance

        :param xfer: medium transport element to use
        :param source: source element
        :param dest1: destination 1 element
        :param dest2: destination 2 element
        :param inv1=0: invert/rotate the medium before loading
        :param inv2=0: invert/rotate the medium before loading
        :return: an ExchangeMedium instance
        """
        return ExchangeMedium(self, xfer, source, dest1, dest2, **kwargs)

    def getlbastatus(self, lba, **kwargs):
        """
        Returns a GetLBAStatus Instance

        :param lba: starting lba
        :param alloc_len = 16384: size of requested datain
        :return: a getlbastatus instance
        """
        return GetLBAStatus(self, lba, **kwargs)

    def inquiry(self, **kwargs):
        """
        Returns a Inquiry Instance

        :param evpd = 0: a byte indicating if vital product data is supported
        :param page_code = 0: a byte representing a page code for vpd
        :param alloc_len = 96: a integer , the size of the data_in buffer
        :return: a Inquiry instance
        """
        return Inquiry(self, **kwargs)

    def initializeelementstatus(self):
        """
        Returns a InitializeElementStatus Instance

        :return: a InitializeElementStatus instance
        """
        return InitializeElementStatus(self)

    def initializeelementstatuswithrange(self, xfer, elements, **kwargs):
        """
        Returns a InitializeElementStatusWithRange Instance

        :param xfer: two byte indicating the address of the starting element
        :param elements: two byte representing a range of elements that should be initialized
        :param range = 0: a integer indicating if elements should be ignored
        :param fast = 0: a integer indicating if  elements should be scanned for media presence
        :return: a InitializeElementStatusWithRange instance
        """
        return InitializeElementStatusWithRange(self, xfer, elements, **kwargs)

    def modeselect6(self, data, **kwargs):
        """
        Returns a ModeSelect6 Instance

        :param data: a dict containing the mode page to set
        :param pf = 0: Page Format flag
        :param sp = 0: Save Pages flag
        :return: a ModeSelect6 instance
        """
        return ModeSelect6(self, data, **kwargs)

    def modesense6(self, page_code, **kwargs):
        """
        Returns a ModeSense6 Instance

        :param page_code:  The page requested
        :param sub_page_code = 0: Requested subpage
        :param dbd = 0: Disable Block Descriptors flag
        :param pc = 0: Page Control flag
        :param alloclen = 96
        :return: a ModeSense6 instance
        """
        return ModeSense6(self, page_code, **kwargs)

    def modesense10(self, page_code, **kwargs):
        """
        Returns a ModeSense10 Instance

        :param page_code:  The page requested
        :param sub_page_code = 0: Requested subpage
        :param llbaa = 0:
        :param dbd = 0: Disable Block Descriptors flag
        :param pc = 0: Page Control flag
        :param alloclen = 96
        :return: a ModeSense10 instance
        """
        return ModeSense10(self, page_code, **kwargs)

    def modeselect10(self, data, **kwargs):
        """
        Returns a ModeSelect10 Instance

        :param data: a dict containing the mode page to set
        :param pf = 0: Page Format flag
        :param sp = 0: Save Pages flag
        :return: a ModeSelect10 instance
        """
        return ModeSelect10(self, data, **kwargs)

    def opencloseimportexportelement(self, xfer, acode):
        """
        Returns a OpenCloseImportExportElement Instance

        :param data: a dict containing the mode page to set
        :param pf = 0: Page Format flag
        :param sp = 0: Save Pages flag
        :return: a OpenCloseImportExportElement instance
        """
        return OpenCloseImportExportElement(self, xfer, acode)

    def positiontoelement(self, xfer, dest, **kwargs):
        """
        Returns a PositionToElement Instance

        :param xfer: medium transport element to use
        :param dest: destination element
        :param invert=0: invert/rotate the medium before loading
        :return: an PositionToElement instance
        """
        return PositionToElement(self, xfer, dest, **kwargs)

    def preventallowmediumremoval(self, **kwargs):
        """
        Returns a PreventAllowMediumRemoval Instance

        :param prevent=0: prevent/allow the medium to be removed
        :return: an PreventAllowMediumRemoval instance
        """
        return PreventAllowMediumRemoval(self, **kwargs)

    def read10(self, lba, tl, **kwargs):
        """
        Returns a Read10 Instance

        :param lba: Logical Block Address
        :param tl: Transfer Length
        :param rdprotect = 0: ReadProtect flags
        :param dpo = 0: DisablePageOut flag
        :param fua = 0: Force Unit Access flag
        :param rarc = 0: Rebuild Assist Recovery control flag
        :param group = 0: Group Number
        :returns the data returned in self.datain
        """
        return Read10(self, lba, tl, **kwargs)

    def read12(self, lba, tl, **kwargs):
        """
        Returns a Read12 Instance

        :param lba: Logical Block Address
        :param tl: Transfer Length
        :param rdprotect = 0: ReadProtect flags
        :param dpo = 0: DisablePageOut flag
        :param fua = 0: Force Unit Access flag
        :param rarc = 0: Rebuild Assist Recovery control flag
        :param group = 0: Group Number
        :returns the data returned in self.datain
        """
        return Read12(self, lba, tl, **kwargs)

    def read16(self, lba, tl, **kwargs):
        """
        Returns a Read16 Instance

        :param lba: Logical Block Address
        :param tl: Transfer Length
        :param rdprotect = 0: ReadProtect flags
        :param dpo = 0: DisablePageOut flag
        :param fua = 0: Force Unit Access flag
        :param rarc = 0: Rebuild Assist Recovery control flag
        :param group = 0: Group Number
        :returns the data returned in self.datain
        """
        return Read16(self, lba, tl, **kwargs)

    def readcapacity10(self, **kwargs):
        """
        Returns a ReadCapacity10 Instance

        :param alloc_len = 8: size of requested datain
        :return: a ReadCapacity10 instance
        """
        return ReadCapacity10(self, **kwargs)

    def readcapacity16(self, **kwargs):
        """
        Returns a ReadCapacity16 Instance

        :param alloc_len = 32: size of requested datain
        :return: a ReadCapacity16 instance
        """
        return ReadCapacity16(self, **kwargs)

    def readelementstatus(self, start, num, **kwargs):
        """
        Returns a ReadElementStatus Instance

        :param start: starting address for first element to return
        :param num: numbver of elements to return
        :param element_type=ELEMENT_TYPE.ALL: type of elements to return
        :param voltag = 0: whether volume tag data should be returned
        :param curdata = 1: check current data
        :param dvcid = 0: whether to return device identifiers
        :param alloclen=16384: max amount od data to return
        :return: an ReadElementStatus instance
        """
        return ReadElementStatus(self, start, num, **kwargs)

    def movemedium(self, xfer, source, dest, **kwargs):
        """
        Returns a MoveMedium Instance

        :param xfer: medium transport element to use
        :param source: source element
        :param dest: destination element
        :param invert=0: invert/rotate the medium before loading
        :return: an MoveMedium instance
        """
        return MoveMedium(self, xfer, source, dest, **kwargs)

    def testunitready(self):
        """
        Returns a TestUnitReady Instance

        No params
        No return value
        """
        return TestUnitReady(self)

    def write10(self, lba, tl, data, **kwargs):
        """
        Returns a Write10 Instance

        :param lba: Logical Block Address to write to
        :param tl: Transfer Length in blocks
        :param data: bytearray containing the data to write
        :param wrprotect = 0: WriteProtect flags
        :param dpo = 0: disable Page Out flag
        :param fua = 0: Force Unit Access flag
        :param group = 0: Group Number
        :return: a Write10 instance
        """
        return Write10(self, lba, tl, data, **kwargs)

    def write12(self, lba, tl, data, **kwargs):
        """
        Returns a Write12 Instance

        :param lba: Logical Block Address to write to
        :param tl: Transfer Length in blocks
        :param data: bytearray containing the data to write
        :param wrprotect = 0: WriteProtect flags
        :param dpo = 0: disable Page Out flag
        :param fua = 0: Force Unit Access flag
        :param group = 0: Group Number
        :return: a Write12 instance
        """
        return Write12(self, lba, tl, data, **kwargs)

    def write16(self, lba, tl, data, **kwargs):
        """
        Returns a Write16 Instance

        :param lba: Logical Block Address to write to
        :param tl: Transfer Length in blocks
        :param data: bytearray containing the data to write
        :param wrprotect = 0: WriteProtect flags
        :param dpo = 0: disable Page Out flag
        :param fua = 0: Force Unit Access flag
        :param group = 0: Group Number
        :return: a Write16 instance
        """
        return Write16(self, lba, tl, data, **kwargs)

    def writesame16(self, lba, nb, data, **kwargs):
        """
        Returns a WriteSame16 Instance

        :param lba: Logical Block Address to write to
        :param nb: Number of blocks
        :param data: bytearray containing the block to write
        :param wrprotect = 0: WriteProtect flags
        :param anchor = False: Anchor flag
        :param unmap = False: Unmap flag
        :param ndob = False: NoDataOutBuffer flag. When set data is None.
        :param group = 0: Group Number
        :return: a WriteSame16 instance
        """
        return WriteSame16(self, lba, nb, data, **kwargs)

    def writesame10(self, lba, nb, data, **kwargs):
        """
        Returns a WriteSame10 Instance

        :param lba: Logical Block Address to write to
        :param nb: Number of blocks
        :param data: bytearray containing the block to write
        :param wrprotect = 0: WriteProtect flags
        :param anchor = False: Anchor flag
        :param unmap = False: Unmap flag
        :param group = 0: Group Number
        :return: a WriteSame10 instance
        """
        return WriteSame10(self, lba, nb, data, **kwargs)

    def reportluns(self, **kwargs):
        """
        Return a ReportLuns Instance

        :param report=0x00: type of logical unit addresses that shall be reported
        :param alloclen=96: size of requested datain
        :return: a ReportLuns instance
        """
        return ReportLuns(self, **kwargs)

    def reportpriority(self, **kwargs):
        """
        Return a ReportPriority Instance

        :param priority=0: specifies information to be returned in data_in buffer
        :param alloclen=16384: size of requested datain
        :return: a ReportLuns instance
        """
        return ReportPriority(self, **kwargs)

    def readbuffer10(self, mode, mode_spec, buf_id, buf_offset, alloclen, **kwargs):
        """
        Return a ReportPriority Instance

        :param priority=0: specifies information to be returned in data_in buffer
        :param alloclen=16384: size of requested datain
        :return: a ReportLuns instance
        """
        return ReadBuffer10(self, mode, mode_spec, buf_id, buf_offset, alloclen, **kwargs)
<|endoftext|>


<commit_before>"""cli.py is the command line interface for CCDC.

"""

from ccdc import core
import click

def context_settings():
    """Normalized tokens for Click cli

    Returns:
        dict
    """

    return dict(token_normalize_func=lambda x: x.lower())

@click.group(context_settings=context_settings())
def entrypoint():
    """Placeholder function to group Click commands"""
    pass

@entrypoint.command()
@click.option('--x',          '-x', required=True)
@click.option('--y',          '-y', required=True)
@click.option('--acquired',   '-a', required=False, default=core.acquired())
@click.option('--number',     '-n', required=False, default=2500)
@click.option('--chunk_size', '-c', required=False, default=1)
def changedetection(x, y, acquired, number, chunk_size):
    """Run change detection for a tile and save results to Cassandra.

    Args:
        x        (int): tile x coordinate
        y        (int): tile y coordinate
        acquired (str): ISO8601 date range
        number   (int): Number of chips to run change detection on.  Testing only.

    Returns:
        count of saved segments
    """

    return core.changedetection(x=x,
                                y=y,
                                acquired=acquired,
                                number=number,
                                chunk_size=chunk_size)

@entrypoint.command()
@click.option('--x', '-x', required=True)
@click.option('--y', '-y', required=True)
@click.option('--msday', '-s', required=True)
@click.option('--meday', '-e', required=True)
@click.option('--acquired', '-a', required=False, default=acquired())
def classification(x, y, msday, meday, acquired):
    """
    Classify a tile.

    Args:
        acquired (str): ISO8601 date range
        x        (int): x coordinate in tile
        y        (int): y coordinate in tile
        msday    (int): ordinal day, beginning of training period
        meday    (int): ordinal day, end of training period
        acquired (str): date range of change segments to classify
    """

    return core.classification(x=x,
                               y=y,
                               msday=msday,
                               meday=meday,
                               acquired=acquired)

if __name__ == '__main__':
    entrypoint()
<commit_msg>correct cli for acquired<commit_after>"""cli.py is the command line interface for CCDC.

"""

from ccdc import core
import click

def context_settings():
    """Normalized tokens for Click cli

    Returns:
        dict
    """

    return dict(token_normalize_func=lambda x: x.lower())

@click.group(context_settings=context_settings())
def entrypoint():
    """Placeholder function to group Click commands"""
    pass

@entrypoint.command()
@click.option('--x',          '-x', required=True)
@click.option('--y',          '-y', required=True)
@click.option('--acquired',   '-a', required=False, default=core.acquired())
@click.option('--number',     '-n', required=False, default=2500)
@click.option('--chunk_size', '-c', required=False, default=1)
def changedetection(x, y, acquired, number, chunk_size):
    """Run change detection for a tile and save results to Cassandra.

    Args:
        x        (int): tile x coordinate
        y        (int): tile y coordinate
        acquired (str): ISO8601 date range
        number   (int): Number of chips to run change detection on.  Testing only.

    Returns:
        count of saved segments
    """

    return core.changedetection(x=x,
                                y=y,
                                acquired=acquired,
                                number=number,
                                chunk_size=chunk_size)

@entrypoint.command()
@click.option('--x', '-x', required=True)
@click.option('--y', '-y', required=True)
@click.option('--msday', '-s', required=True)
@click.option('--meday', '-e', required=True)
@click.option('--acquired', '-a', required=False, default=core.acquired())
def classification(x, y, msday, meday, acquired):
    """
    Classify a tile.

    Args:
        acquired (str): ISO8601 date range
        x        (int): x coordinate in tile
        y        (int): y coordinate in tile
        msday    (int): ordinal day, beginning of training period
        meday    (int): ordinal day, end of training period
        acquired (str): date range of change segments to classify
    """

    return core.classification(x=x,
                               y=y,
                               msday=msday,
                               meday=meday,
                               acquired=acquired)

if __name__ == '__main__':
    entrypoint()
<|endoftext|>


<commit_before>import os
import argparse

def default_db():
    return os.path.join(os.path.expanduser("~"), "~/.pysswords")

def parse_args(args):
    parser = argparse.ArgumentParser(prog="Pysswords")

    group_init = parser.add_argument_group("Init options")
    group_init.add_argument("-I", "--init", action="store_true")
    group_init.add_argument("-D", "--database", default=default_db())

    group_cred = parser.add_argument_group("Credential options")
    group_cred.add_argument("-a", "--add", action="store_true")
    group_cred.add_argument("-g", "--get")
    group_cred.add_argument("-u", "--update")
    group_cred.add_argument("-r", "--remove")
    group_cred.add_argument("-s", "--search")

    return parser.parse_args(args)
<commit_msg>Refactor parse args db options<commit_after>import os
import argparse

def default_db():
    return os.path.join(os.path.expanduser("~"), "~/.pysswords")

def parse_args(args):
    parser = argparse.ArgumentParser(prog="Pysswords")

    group_db = parser.add_argument_group("Databse options")
    group_db.add_argument("-I", "--init", action="store_true")
    group_db.add_argument("-D", "--database", default=default_db())

    group_cred = parser.add_argument_group("Credential options")
    group_cred.add_argument("-a", "--add", action="store_true")
    group_cred.add_argument("-g", "--get")
    group_cred.add_argument("-u", "--update")
    group_cred.add_argument("-r", "--remove")
    group_cred.add_argument("-s", "--search")

    return parser.parse_args(args)
<|endoftext|>


<commit_before>"""Methods for reading files.
"""

import numpy as np

def read_timeseries(inputfns, importer, **kwargs):
    """Read a list of input files using io tools and stack them into a 3d array.

    Parameters
    ----------
    inputfns : list
        List of input files returned by any function implemented in archive.
    importer : function
        Any function implemented in importers.
    kwargs : dict
        Optional keyword arguments for the importer.

    Returns
    -------
    out : tuple
        A three-element tuple containing the precipitation fields read, the quality field,
        and associated metadata.
    """
    R = []
    Q = []
    for ifn in inputfns[0]:
        R_, Q_, metadata = importer(ifn, **kwargs)
        R.append(R_)
        Q.append(Q_)

    R = np.concatenate([R_[None, :, :] for R_ in R])
    Q = np.concatenate([Q_[None, :, :] for Q_ in Q])

    return R, Q, metadata
<commit_msg>Q passed as list<commit_after>"""Methods for reading files.
"""

import numpy as np

def read_timeseries(inputfns, importer, **kwargs):
    """Read a list of input files using io tools and stack them into a 3d array.

    Parameters
    ----------
    inputfns : list
        List of input files returned by any function implemented in archive.
    importer : function
        Any function implemented in importers.
    kwargs : dict
        Optional keyword arguments for the importer.

    Returns
    -------
    out : tuple
        A three-element tuple containing the precipitation fields read, the quality field,
        and associated metadata.
    """
    R = []
    Q = []
    for ifn in inputfns[0]:
        R_, Q_, metadata = importer(ifn, **kwargs)
        R.append(R_)
        Q.append(Q_)

    R = np.concatenate([R_[None, :, :] for R_ in R])
    #TODO: Q should be organized as R, but this is not trivial as Q_ can be also None or a scalar

    return R, Q, metadata
<|endoftext|>


<commit_before>"""Step parsers."""

from __future__ import absolute_import
import re as base_re
import warnings

import parse as base_parse
from parse_type import cfparse as base_cfparse

import six

from .exceptions import InvalidStepParserError

RE_TYPE = type(base_re.compile(''))

class StepParser(object):

    """Parser of the individual step."""

    def __init__(self, name):
        self.name = name

    def parse_arguments(self, name):
        """Get step arguments from the given step name.

        :return: `dict` of step arguments
        """
        raise NotImplementedError()

    def is_matching(self, name):
        """Match given name with the step name."""
        raise NotImplementedError()

class re(StepParser):

    """Regex step parser."""

    def __init__(self, name, *args, **kwargs):
        """Compile regex."""
        super(re, self).__init__(name)
        self.regex = base_re.compile(self.name, *args, **kwargs)

    def parse_arguments(self, name):
        """Get step arguments.

        :return: `dict` of step arguments
        """
        return self.regex.match(name).groupdict()

    def is_matching(self, name):
        """Match given name with the step name."""
        return bool(self.regex.match(name))

class parse(StepParser):

    """parse step parser."""

    def __init__(self, name, *args, **kwargs):
        """Compile parse expression."""
        super(parse, self).__init__(name)
        self.parser = base_parse.compile(self.name, *args, **kwargs)

    def parse_arguments(self, name):
        """Get step arguments.

        :return: `dict` of step arguments
        """
        return self.parser.parse(name).named

    def is_matching(self, name):
        """Match given name with the step name."""
        try:
            return bool(self.parser.parse(name))
        except ValueError:
            return False

class cfparse(parse):

    """cfparse step parser."""

    def __init__(self, name, *args, **kwargs):
        """Compile parse expression."""
        super(parse, self).__init__(name)
        self.parser = base_cfparse.Parser(self.name, *args, **kwargs)

class string(StepParser):

    """Exact string step parser."""

    def parse_arguments(self, name):
        """No parameters are available for simple string step.

        :return: `dict` of step arguments
        """
        return {}

    def is_matching(self, name):
        """Match given name with the step name."""
        return self.name == name

def get_parser(step_name):
    """Get parser by given name.

    :param step_name: name of the step to parse

    :return: step parser object
    :rtype: StepArgumentParser
    """
    if isinstance(step_name, six.string_types):
        if isinstance(step_name, six.binary_type):  # Python 2 compatibility
            step_name = step_name.decode('utf-8')
        return string(step_name)
    elif not hasattr(step_name, 'is_matching') or not hasattr(step_name, 'parse_arguments'):
        raise InvalidStepParserError(step_name)
    else:
        return step_name
<commit_msg>Reorganize imports<commit_after>"""Step parsers."""

from __future__ import absolute_import

import re as base_re

import parse as base_parse
import six
from parse_type import cfparse as base_cfparse

from .exceptions import InvalidStepParserError

class StepParser(object):
    """Parser of the individual step."""

    def __init__(self, name):
        self.name = name

    def parse_arguments(self, name):
        """Get step arguments from the given step name.

        :return: `dict` of step arguments
        """
        raise NotImplementedError()

    def is_matching(self, name):
        """Match given name with the step name."""
        raise NotImplementedError()

class re(StepParser):
    """Regex step parser."""

    def __init__(self, name, *args, **kwargs):
        """Compile regex."""
        super(re, self).__init__(name)
        self.regex = base_re.compile(self.name, *args, **kwargs)

    def parse_arguments(self, name):
        """Get step arguments.

        :return: `dict` of step arguments
        """
        return self.regex.match(name).groupdict()

    def is_matching(self, name):
        """Match given name with the step name."""
        return bool(self.regex.match(name))

class parse(StepParser):
    """parse step parser."""

    def __init__(self, name, *args, **kwargs):
        """Compile parse expression."""
        super(parse, self).__init__(name)
        self.parser = base_parse.compile(self.name, *args, **kwargs)

    def parse_arguments(self, name):
        """Get step arguments.

        :return: `dict` of step arguments
        """
        return self.parser.parse(name).named

    def is_matching(self, name):
        """Match given name with the step name."""
        try:
            return bool(self.parser.parse(name))
        except ValueError:
            return False

class cfparse(parse):
    """cfparse step parser."""

    def __init__(self, name, *args, **kwargs):
        """Compile parse expression."""
        super(parse, self).__init__(name)
        self.parser = base_cfparse.Parser(self.name, *args, **kwargs)

class string(StepParser):
    """Exact string step parser."""

    def parse_arguments(self, name):
        """No parameters are available for simple string step.

        :return: `dict` of step arguments
        """
        return {}

    def is_matching(self, name):
        """Match given name with the step name."""
        return self.name == name

def get_parser(step_name):
    """Get parser by given name.

    :param step_name: name of the step to parse

    :return: step parser object
    :rtype: StepArgumentParser
    """
    if isinstance(step_name, six.string_types):
        if isinstance(step_name, six.binary_type):  # Python 2 compatibility
            step_name = step_name.decode('utf-8')
        return string(step_name)
    elif not hasattr(step_name, 'is_matching') or not hasattr(step_name, 'parse_arguments'):
        raise InvalidStepParserError(step_name)
    else:
        return step_name
<|endoftext|>


<commit_before>import time
import threading
from functools import partial
from datetime import datetime
from restful_api_socket import RESTfulApiSocket
from exchange import ExchangeGateway
from market_data import L2Depth, Trade
from util import Logger
from instrument import Instrument
from sql_client_template import SqlClientTemplate

class ExchGwApiQuoine(RESTfulApiSocket):
    """
    Exchange gateway RESTfulApi
    """
    def __init__(self):
        RESTfulApiSocket.__init__(self)

    @classmethod
    def get_timestamp_offset(cls):
        return 1

    @classmethod
    def get_trades_timestamp_field_name(cls):
        return 'created_at'

    @classmethod
    def get_bids_field_name(cls):
        return 'buy_price_levels'

    @classmethod
    def get_asks_field_name(cls):
        return 'sell_price_levels'

    @classmethod
    def get_trade_side_field_name(cls):
        return 'taker_side'

    @classmethod
    def get_trade_id_field_name(cls):
        return 'id'

    @classmethod
    def get_trade_price_field_name(cls):
        return 'price'

    @classmethod
    def get_trade_volume_field_name(cls):
        return 'quantity'

    @classmethod
    def get_order_book_link(cls, instmt):
        return "https://api.quoine.com/products/%s/price_levels" % instmt.get_instmt_code()

    @classmethod
    def get_trades_link(cls, instmt, page=1):
            return "https://api.quoine.com/executions?product_id=%s&page=%d" % \
                   (instmt.get_instmt_code(), page)

    @classmethod
    def parse_l2_depth(cls, instmt, raw):
        """
        Parse raw data to L2 depth
        :param instmt: Instrument
        :param raw: Raw data in JSON
        """
        l2_depth = L2Depth(20)
        keys = list(raw.keys())
        if cls.get_bids_field_name() in keys and \
           cls.get_asks_field_name() in keys:

            # Date time
            l2_depth.date_time = datetime.utcnow().strftime("%Y%m%d %H:%M:%S.%f")

            # Bids
            bids = raw[cls.get_bids_field_name()]
            bids = sorted(bids, key=lambda x: x[0], reverse=True)
            for i in range(0, 5):
                l2_depth.bids[i].price = float(bids[i][0]) if type(bids[i][0]) != float else bids[i][0]
                l2_depth.bids[i].volume = float(bids[i][1]) if type(bids[i][1]) != float else bids[i][1]

            # Asks
            asks = raw[cls.get_asks_field_name()]
            asks = sorted(asks, key=lambda x: x[0])
            for i in range(0, 5):
                l2_depth.asks[i].price = float(asks[i][0]) if type(asks[i][0]) != float else asks[i][0]
                l2_depth.asks[i].volume = float(asks[i][1]) if type(asks[i][1]) != float else asks[i][1]
        else:
            raise Exception('Does not contain order book keys in instmt %s-%s.\nOriginal:\n%s' % \
                (instmt.get_exchange_name(), instmt.get_instmt_name(), \
                 raw))

        return l2_depth

    @classmethod
    def parse_trade(cls, instmt, raw):
        """
        :param instmt: Instrument
        :param raw: Raw data in JSON
        :return:
        """
        trade = Trade()
        keys = list(raw.keys())

        if cls.get_trades_timestamp_field_name() in keys and \
           cls.get_trade_id_field_name() in keys and \
           cls.get_trade_price_field_name() in keys and \
           cls.get_trade_volume_field_name() in keys and \
           cls.get_trade_side_field_name() in keys:

            # Date time
            date_time = float(raw[cls.get_trades_timestamp_field_name()])
            date_time = date_time / cls.get_timestamp_offset()
            trade.date_time = datetime.utcfromtimestamp(date_time).strftime("%Y%m%d %H:%M:%S.%f")

            # Trade side
            trade.trade_side = 1

            # Trade id
            trade.trade_id = str(raw[cls.get_trade_id_field_name()])

            # Trade price
            trade.trade_price = float(str(raw[cls.get_trade_price_field_name()]))

            # Trade volume
            trade.trade_volume = float(str(raw[cls.get_trade_volume_field_name()]))
        else:
            raise Exception('Does not contain trade keys in instmt %s-%s.\nOriginal:\n%s' % \
                (instmt.get_exchange_name(), instmt.get_instmt_name(), \
                 raw))

        return trade

    @classmethod
    def get_order_book(cls, instmt):
        """
        Get order book
        :param instmt: Instrument
        :return: Object L2Depth
        """
        link = cls.get_order_book_link(instmt)
        res = cls.request(link)
        if len(res) > 0:
            return cls.parse_l2_depth(instmt=instmt,
                                       raw=res)
        else:
            return None

    @classmethod
    def get_trades(cls, instmt):
        """
        Get trades
        :param instmt: Instrument
        :param trade_id: Trade id
        :return: List of trades
        """
        exch_trade_id = int(instmt.get_exch_trade_id())
        page_limit = 10 if exch_trade_id > 0 else 1
        trades = []
        is_continue = True

        for page in range(1, page_limit+1):
            link = cls.get_trades_link(instmt, page)
            res = cls.request(link)
            Logger.info('test', res)
            if len(res) > 0:
                if 'models' not in res.keys():
                    break
                res = res['models']
                for t in res:
                    trade = cls.parse_trade(instmt=instmt,
                                             raw=t)
                    if int(trade.trade_id) > exch_trade_id:
                        trades.append(trade)
                    else:
                        is_continue = False
                        break

                if not is_continue:
                    break

        return trades

class ExchGwQuoine(ExchangeGateway):
    """
    Exchange gateway
    """
    def __init__(self, db_client):
        """
        Constructor
        :param db_client: Database client
        """
        ExchangeGateway.__init__(self, ExchGwApiQuoine(), db_client)

    @classmethod
    def get_exchange_name(cls):
        """
        Get exchange name
        :return: Exchange name string
        """
        return 'Quoine'

    def get_order_book_worker(self, instmt):
        """
        Get order book worker
        :param instmt: Instrument
        """
        instmt.set_order_book_id(self.get_order_book_init(instmt))

        while True:
            try:
                l2_depth = self.api_socket.get_order_book(instmt)
                if l2_depth is not None and l2_depth.is_diff(instmt.get_l2_depth()):
                    instmt.set_prev_l2_depth(instmt.get_l2_depth())
                    instmt.set_l2_depth(l2_depth)
                    instmt.incr_order_book_id()
                    self.insert_order_book(instmt)
            except Exception as e:
                Logger.error(self.__class__.__name__, "Error in order book: %s" % e)
            time.sleep(1)

    def get_trades_worker(self, instmt):
        """
        Get order book worker thread
        :param instmt: Instrument name
        """
        trade_id, exch_trade_id = self.get_trades_init(instmt)
        instmt.set_trade_id(trade_id)
        instmt.set_exch_trade_id(exch_trade_id)

        while True:
            try:
                ret = self.api_socket.get_trades(instmt)
                if ret is None or len(ret) == 0:
                    time.sleep(1)
                    continue
            except Exception as e:
                Logger.error(self.__class__.__name__, "Error in trades: %s" % e)

            for trade in ret:
                assert isinstance(trade.trade_id, str), "trade.trade_id(%s) = %s" % (type(trade.trade_id), trade.trade_id)
                assert isinstance(instmt.get_exch_trade_id(), str), \
                       "instmt.get_exch_trade_id()(%s) = %s" % (type(instmt.get_exch_trade_id()), instmt.get_exch_trade_id())
                if int(trade.trade_id) > int(instmt.get_exch_trade_id()):
                    instmt.set_exch_trade_id(trade.trade_id)
                    instmt.incr_trade_id()
                    self.insert_trade(instmt, trade)

            # After the first time of getting the trade, indicate the instrument
            # is recovered
            if not instmt.get_recovered():
                instmt.set_recovered(True)

            time.sleep(1)

    def start(self, instmt):
        """
        Start the exchange gateway
        :param instmt: Instrument
        :return List of threads
        """
        instmt.set_l2_depth(L2Depth(5))
        instmt.set_prev_l2_depth(L2Depth(5))
        instmt.set_order_book_table_name(self.get_order_book_table_name(instmt.get_exchange_name(),
                                                                        instmt.get_instmt_name()))
        instmt.set_trades_table_name(self.get_trades_table_name(instmt.get_exchange_name(),
                                                                instmt.get_instmt_name()))
        instmt.set_recovered(False)
        t1 = threading.Thread(target=partial(self.get_order_book_worker, instmt))
        t1.start()
        t2 = threading.Thread(target=partial(self.get_trades_worker, instmt))
        t2.start()
        return [t1, t2]

if __name__ == '__main__':
    Logger.init_log()
    exchange_name = 'Quoine'
    instmt_name = 'BTCUSD'
    instmt_code = '1'
    instmt = Instrument(exchange_name, instmt_name, instmt_code)
    db_client = SqlClientTemplate()
    exch = ExchGwQuoine(db_client)
    instmt.set_l2_depth(L2Depth(5))
    instmt.set_prev_l2_depth(L2Depth(5))
    instmt.set_order_book_table_name(exch.get_order_book_table_name(instmt.get_exchange_name(),
                                                                    instmt.get_instmt_name()))
    instmt.set_trades_table_name(exch.get_trades_table_name(instmt.get_exchange_name(),
                                                            instmt.get_instmt_name()))
    instmt.set_recovered(False)
    # exch.get_order_book_worker(instmt)
    exch.get_trades_worker(instmt)<commit_msg>Resolved an issue from #20<commit_after>import time
import threading
from functools import partial
from datetime import datetime
from restful_api_socket import RESTfulApiSocket
from exchange import ExchangeGateway
from market_data import L2Depth, Trade
from util import Logger
from instrument import Instrument
from sql_client_template import SqlClientTemplate

class ExchGwApiQuoine(RESTfulApiSocket):
    """
    Exchange gateway RESTfulApi
    """
    def __init__(self):
        RESTfulApiSocket.__init__(self)

    @classmethod
    def get_timestamp_offset(cls):
        return 1

    @classmethod
    def get_trades_timestamp_field_name(cls):
        return 'created_at'

    @classmethod
    def get_bids_field_name(cls):
        return 'buy_price_levels'

    @classmethod
    def get_asks_field_name(cls):
        return 'sell_price_levels'

    @classmethod
    def get_trade_side_field_name(cls):
        return 'taker_side'

    @classmethod
    def get_trade_id_field_name(cls):
        return 'id'

    @classmethod
    def get_trade_price_field_name(cls):
        return 'price'

    @classmethod
    def get_trade_volume_field_name(cls):
        return 'quantity'

    @classmethod
    def get_order_book_link(cls, instmt):
        return "https://api.quoine.com/products/%s/price_levels" % instmt.get_instmt_code()

    @classmethod
    def get_trades_link(cls, instmt, page=1):
            return "https://api.quoine.com/executions?product_id=%s&page=%d" % \
                   (instmt.get_instmt_code(), page)

    @classmethod
    def parse_l2_depth(cls, instmt, raw):
        """
        Parse raw data to L2 depth
        :param instmt: Instrument
        :param raw: Raw data in JSON
        """
        l2_depth = L2Depth()
        keys = list(raw.keys())
        if cls.get_bids_field_name() in keys and \
           cls.get_asks_field_name() in keys:

            # Date time
            l2_depth.date_time = datetime.utcnow().strftime("%Y%m%d %H:%M:%S.%f")

            # Bids
            bids = raw[cls.get_bids_field_name()]
            bids = sorted(bids, key=lambda x: x[0], reverse=True)
            for i in range(0, 5):
                l2_depth.bids[i].price = float(bids[i][0]) if type(bids[i][0]) != float else bids[i][0]
                l2_depth.bids[i].volume = float(bids[i][1]) if type(bids[i][1]) != float else bids[i][1]

            # Asks
            asks = raw[cls.get_asks_field_name()]
            asks = sorted(asks, key=lambda x: x[0])
            for i in range(0, 5):
                l2_depth.asks[i].price = float(asks[i][0]) if type(asks[i][0]) != float else asks[i][0]
                l2_depth.asks[i].volume = float(asks[i][1]) if type(asks[i][1]) != float else asks[i][1]
        else:
            raise Exception('Does not contain order book keys in instmt %s-%s.\nOriginal:\n%s' % \
                (instmt.get_exchange_name(), instmt.get_instmt_name(), \
                 raw))

        return l2_depth

    @classmethod
    def parse_trade(cls, instmt, raw):
        """
        :param instmt: Instrument
        :param raw: Raw data in JSON
        :return:
        """
        trade = Trade()
        keys = list(raw.keys())

        if cls.get_trades_timestamp_field_name() in keys and \
           cls.get_trade_id_field_name() in keys and \
           cls.get_trade_price_field_name() in keys and \
           cls.get_trade_volume_field_name() in keys and \
           cls.get_trade_side_field_name() in keys:

            # Date time
            date_time = float(raw[cls.get_trades_timestamp_field_name()])
            date_time = date_time / cls.get_timestamp_offset()
            trade.date_time = datetime.utcfromtimestamp(date_time).strftime("%Y%m%d %H:%M:%S.%f")

            # Trade side
            trade.trade_side = 1

            # Trade id
            trade.trade_id = str(raw[cls.get_trade_id_field_name()])

            # Trade price
            trade.trade_price = float(str(raw[cls.get_trade_price_field_name()]))

            # Trade volume
            trade.trade_volume = float(str(raw[cls.get_trade_volume_field_name()]))
        else:
            raise Exception('Does not contain trade keys in instmt %s-%s.\nOriginal:\n%s' % \
                (instmt.get_exchange_name(), instmt.get_instmt_name(), \
                 raw))

        return trade

    @classmethod
    def get_order_book(cls, instmt):
        """
        Get order book
        :param instmt: Instrument
        :return: Object L2Depth
        """
        link = cls.get_order_book_link(instmt)
        res = cls.request(link)
        if len(res) > 0:
            return cls.parse_l2_depth(instmt=instmt,
                                       raw=res)
        else:
            return None

    @classmethod
    def get_trades(cls, instmt):
        """
        Get trades
        :param instmt: Instrument
        :param trade_id: Trade id
        :return: List of trades
        """
        exch_trade_id = int(instmt.get_exch_trade_id())
        page_limit = 10 if exch_trade_id > 0 else 1
        trades = []
        is_continue = True

        for page in range(1, page_limit+1):
            link = cls.get_trades_link(instmt, page)
            res = cls.request(link)
            Logger.info('test', res)
            if len(res) > 0:
                if 'models' not in res.keys():
                    break
                res = res['models']
                for t in res:
                    trade = cls.parse_trade(instmt=instmt,
                                             raw=t)
                    if int(trade.trade_id) > exch_trade_id:
                        trades.append(trade)
                    else:
                        is_continue = False
                        break

                if not is_continue:
                    break

        return trades

class ExchGwQuoine(ExchangeGateway):
    """
    Exchange gateway
    """
    def __init__(self, db_client):
        """
        Constructor
        :param db_client: Database client
        """
        ExchangeGateway.__init__(self, ExchGwApiQuoine(), db_client)

    @classmethod
    def get_exchange_name(cls):
        """
        Get exchange name
        :return: Exchange name string
        """
        return 'Quoine'

    def get_order_book_worker(self, instmt):
        """
        Get order book worker
        :param instmt: Instrument
        """
        instmt.set_order_book_id(self.get_order_book_init(instmt))

        while True:
            try:
                l2_depth = self.api_socket.get_order_book(instmt)
                if l2_depth is not None and l2_depth.is_diff(instmt.get_l2_depth()):
                    instmt.set_prev_l2_depth(instmt.get_l2_depth())
                    instmt.set_l2_depth(l2_depth)
                    instmt.incr_order_book_id()
                    self.insert_order_book(instmt)
            except Exception as e:
                Logger.error(self.__class__.__name__, "Error in order book: %s" % e)
            time.sleep(1)

    def get_trades_worker(self, instmt):
        """
        Get order book worker thread
        :param instmt: Instrument name
        """
        trade_id, exch_trade_id = self.get_trades_init(instmt)
        instmt.set_trade_id(trade_id)
        instmt.set_exch_trade_id(exch_trade_id)

        while True:
            try:
                ret = self.api_socket.get_trades(instmt)
                if ret is None or len(ret) == 0:
                    time.sleep(1)
                    continue
            except Exception as e:
                Logger.error(self.__class__.__name__, "Error in trades: %s" % e)

            for trade in ret:
                assert isinstance(trade.trade_id, str), "trade.trade_id(%s) = %s" % (type(trade.trade_id), trade.trade_id)
                assert isinstance(instmt.get_exch_trade_id(), str), \
                       "instmt.get_exch_trade_id()(%s) = %s" % (type(instmt.get_exch_trade_id()), instmt.get_exch_trade_id())
                if int(trade.trade_id) > int(instmt.get_exch_trade_id()):
                    instmt.set_exch_trade_id(trade.trade_id)
                    instmt.incr_trade_id()
                    self.insert_trade(instmt, trade)

            # After the first time of getting the trade, indicate the instrument
            # is recovered
            if not instmt.get_recovered():
                instmt.set_recovered(True)

            time.sleep(1)

    def start(self, instmt):
        """
        Start the exchange gateway
        :param instmt: Instrument
        :return List of threads
        """
        instmt.set_l2_depth(L2Depth(5))
        instmt.set_prev_l2_depth(L2Depth(5))
        instmt.set_order_book_table_name(self.get_order_book_table_name(instmt.get_exchange_name(),
                                                                        instmt.get_instmt_name()))
        instmt.set_trades_table_name(self.get_trades_table_name(instmt.get_exchange_name(),
                                                                instmt.get_instmt_name()))
        instmt.set_recovered(False)
        t1 = threading.Thread(target=partial(self.get_order_book_worker, instmt))
        t1.start()
        t2 = threading.Thread(target=partial(self.get_trades_worker, instmt))
        t2.start()
        return [t1, t2]

if __name__ == '__main__':
    Logger.init_log()
    exchange_name = 'Quoine'
    instmt_name = 'BTCUSD'
    instmt_code = '1'
    instmt = Instrument(exchange_name, instmt_name, instmt_code)
    db_client = SqlClientTemplate()
    exch = ExchGwQuoine(db_client)
    instmt.set_l2_depth(L2Depth(5))
    instmt.set_prev_l2_depth(L2Depth(5))
    instmt.set_order_book_table_name(exch.get_order_book_table_name(instmt.get_exchange_name(),
                                                                    instmt.get_instmt_name()))
    instmt.set_trades_table_name(exch.get_trades_table_name(instmt.get_exchange_name(),
                                                            instmt.get_instmt_name()))
    instmt.set_recovered(False)
    # exch.get_order_book_worker(instmt)
    exch.get_trades_worker(instmt)<|endoftext|>