<commit_before># -*- coding: utf-8 -*-
# encoding: utf-8
"""
The class :py:class:`Reader` has been designed to selectively extract data
from a mzML file and to expose the data as a python object.
Necessary information are read in and stored in a fast
accessible format.
The reader itself is an iterator, thus looping over all spectra
follows the classical pythonian syntax.
Additionally one can random access spectra by their nativeID
if the file if not truncated by a conversion Program.

Note:
    The class :py:class:`Writer` is still in development.

"""

# Python mzML module - pymzml
# Copyright (C) 2010-2019 M. Kösters, C. Fufezan
#     The MIT License (MIT)

#     Permission is hereby granted, free of charge, to any person obtaining a copy
#     of this software and associated documentation files (the "Software"), to deal
#     in the Software without restriction, including without limitation the rights
#     to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
#     copies of the Software, and to permit persons to whom the Software is
#     furnished to do so, subject to the following conditions:

#     The above copyright notice and this permission notice shall be included in all
#     copies or substantial portions of the Software.

#     THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
#     IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
#     FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
#     AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
#     LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
#     OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
#     SOFTWARE.

import re
import os
import xml.etree.ElementTree as ElementTree
from collections import defaultdict as ddict
from io import BytesIO

from . import spec
from . import obo
from . import regex_patterns
from .file_interface import FileInterface
from .file_classes.standardMzml import StandardMzml

class Reader(object):
    """
    Initialize Reader object for a given mzML file.

    Arguments:
        path (str): path to the mzml file to parse.

    Keyword Arguments:
        MS_precisions (dict): measured precisions for the different MS levels.
            e.g.::

                {
                    1 : 5e-6,
                    2 : 20e-6
                }

        obo_version (str, optional): obo version number as string. If not
            specified the version will be extracted from the mzML file

    Note:
        Setting the precision for MS1 and MSn spectra has changed in version 1.2.
        However, the old syntax as kwargs is still compatible ( e.g. 'MS1_Precision=5e-6').
    """

    def __init__(
        self,
        path_or_file,
        MS_precisions=None,
        obo_version=None,
        build_index_from_scratch=False,
        skip_chromatogram=True,
        **kwargs
    ):
        """Initialize and set required attributes."""
        self.build_index_from_scratch = build_index_from_scratch
        self.skip_chromatogram = skip_chromatogram
        if MS_precisions is None:
            MS_precisions = {}
            if "MS1_Precision" in kwargs.keys():
                MS_precisions[1] = kwargs["MS1_Precision"]
            if "MSn_Precision" in kwargs.keys():
                MS_precisions[2] = kwargs["MSn_Precision"]
                MS_precisions[3] = kwargs["MSn_Precision"]

        # Parameters
        self.ms_precisions = {
            None: 0.0001,  # if spectra does not contain ms_level information
                         # e.g. UV-chromatograms (thanks pyeguy) then ms_level is
                         # returned as None
            1: 5e-6,
            2: 20e-6,
        }
        self.ms_precisions.update(MS_precisions)

        # File info
        self.info = ddict()
        if isinstance(path_or_file, str):
            self.info["file_name"] = path_or_file
            self.info["encoding"] = self._determine_file_encoding(path_or_file)
        else:
            self.info["encoding"] = self._guess_encoding(path_or_file)

        self.info["file_object"] = self._open_file(path_or_file)
        self.info["offset_dict"] = self.info["file_object"].offset_dict
        if obo_version:
            self.info["obo_version"] = self._obo_version_validator(obo_version)
        else:
            # obo version not specified -> try to identify from mzML by self._init_iter
            self.info["obo_version"] = None

        self.iter = self._init_iter()
        self.OT = self._init_obo_translator()

    def __next__(self):
        """
        Iterator for the class :py:class:`Run`.

        Iterates all of the spectra in the file.

        Returns:
            Spectrum (:py:class:`Spectrum`): a spectrum object with interface
                to the original spectrum element.

        Example:

        >>> for spectrum in Reader:
        ...     print(spectrum.mz, end='\\r')

        """
        has_ref_group = self.info.get("referenceable_param_group_list", False)
        while True:
            event, element = next(self.iter, ("END", "END"))
            if event == "end":
                if element.tag.endswith("}spectrum"):
                    spectrum = spec.Spectrum(element)
                    if has_ref_group:
                        spectrum._set_params_from_reference_group(
                            self.info["referenceable_param_group_list_element"]
                        )
                    ms_level = spectrum.ms_level
                    spectrum.measured_precision = self.ms_precisions[ms_level]
                    spectrum.calling_instance = self
                    return spectrum
                if element.tag.endswith("}chromatogram"):
                    if self.skip_chromatogram:
                        continue
                    spectrum = spec.Chromatogram(element)
                    # if has_ref_group:
                    #     spectrum._set_params_from_reference_group(
                    #         self.info['referenceable_param_group_list_element']
                    #     )
                    spectrum.calling_instance = self
                    return spectrum
            elif event == "END":
                raise StopIteration

    def __getitem__(self, identifier):
        """
        Access spectrum with native id 'identifier'.

        Arguments:
            identifier (str or int): last number in the id tag of the spectrum
                element

        Returns:
            spectrum (Spectrum or Chromatogram): spectrum/chromatogram object
            with native id 'identifier'
        """
        try:
            if int(identifier) > self.get_spectrum_count():
                raise Exception("Requested identifier is out of range")
        except:
            pass
        spectrum = self.info["file_object"][identifier]
        spectrum.calling_instance = self
        if isinstance(spectrum, spec.Spectrum):
            spectrum.measured_precision = self.ms_precisions[spectrum.ms_level]
        return spectrum

    def __enter__(self):
        return self

    def __exit__(self, type, value, traceback):
        self.close()

    @property
    def file_class(self):
        """Return file object in use."""
        return type(self.info["file_object"].file_handler)

    def _open_file(self, path_or_file):
        """
        Open the path using the FileInterface class as a wrapper.

        Arguments:
            path (str): path to the file to parse

        Returns:
            (FileInterface): Wrapper class for compressed and uncompressed
                mzml files
        """
        return FileInterface(
            path_or_file,
            self.info["encoding"],
            build_index_from_scratch=self.build_index_from_scratch,
        )

    def _guess_encoding(self, mzml_file):
        """
        Determine the encoding used for the file.

        Arguments:
            mzml_file (IOBase): an mzml file

        Returns:
            mzml_encoding (str): encoding type of the file
        """
        match = regex_patterns.FILE_ENCODING_PATTERN.search(mzml_file.readline())
        if match:
            return bytes.decode(match.group("encoding"))
        else:
            return "utf-8"

    def _determine_file_encoding(self, path):
        """
        Determine the encoding used for the file in path.

        Arguments:
            path (str): path to the mzml files

        Returns:
            mzml_encoding (str): encoding type of the file
        """
        if os.path.exists(path):
            if path.endswith(".gz") or path.endswith(".igz"):
                import gzip

                _open = gzip.open
            else:
                _open = open
            with _open(path, "rb") as sniffer:
                return self._guess_encoding(sniffer)

    @staticmethod
    def _obo_version_validator(version):

        """
        The obo version should fit file names in the obo folder.
        However, some software generate mzML with built in obo version string like:
        '23:06:2017' or even newer version that not in obo folder yet.
        This is to check obo version and try to fit the best obo version
        to the obo version in the mzML file.

        Arguments:
            version (str): The original version to check.

        Returns:
            version_fixed (str): The checked obo version.
        """
        obo_rgx = re.compile(r"(\d\.\d{1,2}\.\d{1,2})(_[rR][cC]\d{0,2})?")
        obo_years_rgx = re.compile(r"20\d\d")
        obo_year_version_dct = {
            2012: "3.40.0",
            2013: "3.50.0",
            2014: "3.60.0",
            2015: "3.75.0",
            2016: "4.0.1",
            2017: "4.1.0",
            2018: "4.1.10",
            2019: "4.1.22",
        }
        version_fixed = None
        if obo_rgx.match(version):
            version_fixed = version
        else:
            if obo_years_rgx.search(version):
                years_found = obo_years_rgx.search(version)
                if years_found:
                    try:
                        year = int(years_found.group(0))
                    except ValueError:
                        year = 2000

                    if year in obo_year_version_dct:
                        version_fixed = obo_year_version_dct[year]
                    else:
                        if year > 2019:
                            version_fixed = "4.1.0"

        if version_fixed:
            # Check if the corresponding obo file existed in obo folder
            obo_root = os.path.dirname(__file__)
            obo_file = os.path.join(
                obo_root,
                "obo",
                "psi-ms{0}.obo".format("-" + version_fixed if version_fixed else ""),
            )
            if os.path.exists(obo_file) or os.path.exists(obo_file + ".gz"):
                pass
            else:
                version_fixed = "1.1.0"
        else:
            version_fixed = "1.1.0"

        return version_fixed

    def _init_obo_translator(self):
        """
        Initialize the obo translator with the minimum requirement
        and extra Accessions.

        Returns:
            obo_translator (OboTranslator): translator class to translate
                accessions to names
        """
        # parse obo, check MS tags and if they are ok in minimum.py (minimum
        # required) ...
        if self.info.get("obo_version", None) is None:
            self.info["obo_version"] = "1.1.0"
        obo_translator = obo.OboTranslator(version=self.info["obo_version"])

        return obo_translator

    def _init_iter(self):
        """
        Initalize the iterator for the spectra and sets it to the start
        of the spectrumList element.

        Returns:
            mzml_iter (xml.etree.ElementTree._IterParseIterator): Iterator over
                all element in the file starting with the first spectrum
        """
        mzml_iter = iter(
            ElementTree.iterparse(self.info["file_object"], events=("end", "start"))
        )  # NOTE: end might be sufficient
        _, self.root = next(mzml_iter)
        self.info["chromatogram_count"] = None
        self.info["spectrum_count"] = None
        while True:
            event, element = next(mzml_iter, ("END", "END"))
            if element.tag.endswith("}mzML"):
                if "version" in element.attrib and len(element.attrib["version"]) > 0:
                    self.info["mzml_version"] = element.attrib["version"]
                else:
                    s = element.attrib[
                        "{http://www.w3.org/2001/XMLSchema-instance}" "schemaLocation"
                    ]
                    self.info["mzml_version"] = re.search(
                        r"[0-9]*\.[0-9]*\.[0-9]*", s
                    ).group()
            elif element.tag.endswith("}cv"):
                if (
                    not self.info["obo_version"]
                    and element.attrib.get("id", None) == "MS"
                ):
                    obo_in_mzml = element.attrib.get("version", "1.1.0")
                    self.info["obo_version"] = self._obo_version_validator(obo_in_mzml)

            elif element.tag.endswith("}referenceableParamGroupList"):
                self.info["referenceable_param_group_list"] = True
                self.info["referenceable_param_group_list_element"] = element
            elif element.tag.endswith("}spectrumList"):
                spec_cnt = element.attrib.get("count")
                self.info["spectrum_count"] = int(spec_cnt) if spec_cnt else None
                break
            elif element.tag.endswith("}chromatogramList"):
                chrom_cnt = element.attrib.get("count", None)
                if chrom_cnt:
                    self.info["chromatogram_count"] = int(chrom_cnt)
                break
            elif element.tag.endswith("}run"):
                run_id = element.attrib.get("id")
                start_time = element.attrib.get("startTimeStamp")
                self.info["run_id"] = run_id
                self.info["start_time"] = start_time
            else:
                pass
        self.root.clear()
        return mzml_iter

    def __iter__(self):
        """Return self."""
        return self

    def next(self):
        """Function to return the next Spectrum element."""
        return self.__next__()

    def get_spectrum_count(self):
        """
        Number of spectra in file.

        Returns:
            spectrum count (int): Number of spectra in file.
        """
        return self.info["spectrum_count"]

    def get_chromatogram_count(self):
        """
        Number of chromatograms in file.

        Returns:
            chromatogram count (int): Number of chromatograms in file.
        """
        return self.info["chromatogram_count"]

    def close(self):
        self.info["file_object"].close()

if __name__ == "__main__":
    print(__doc__)
<commit_msg>Reset iter after StopIteration<commit_after># -*- coding: utf-8 -*-
# encoding: utf-8
"""
The class :py:class:`Reader` has been designed to selectively extract data
from a mzML file and to expose the data as a python object.
Necessary information are read in and stored in a fast
accessible format.
The reader itself is an iterator, thus looping over all spectra
follows the classical pythonian syntax.
Additionally one can random access spectra by their nativeID
if the file if not truncated by a conversion Program.

Note:
    The class :py:class:`Writer` is still in development.

"""

# Python mzML module - pymzml
# Copyright (C) 2010-2019 M. Kösters, C. Fufezan
#     The MIT License (MIT)

#     Permission is hereby granted, free of charge, to any person obtaining a copy
#     of this software and associated documentation files (the "Software"), to deal
#     in the Software without restriction, including without limitation the rights
#     to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
#     copies of the Software, and to permit persons to whom the Software is
#     furnished to do so, subject to the following conditions:

#     The above copyright notice and this permission notice shall be included in all
#     copies or substantial portions of the Software.

#     THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
#     IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
#     FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
#     AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
#     LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
#     OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
#     SOFTWARE.

import re
import os
import xml.etree.ElementTree as ElementTree
from collections import defaultdict as ddict
from io import BytesIO

from . import spec
from . import obo
from . import regex_patterns
from .file_interface import FileInterface
from .file_classes.standardMzml import StandardMzml

class Reader(object):
    """
    Initialize Reader object for a given mzML file.

    Arguments:
        path (str): path to the mzml file to parse.

    Keyword Arguments:
        MS_precisions (dict): measured precisions for the different MS levels.
            e.g.::

                {
                    1 : 5e-6,
                    2 : 20e-6
                }

        obo_version (str, optional): obo version number as string. If not
            specified the version will be extracted from the mzML file

    Note:
        Setting the precision for MS1 and MSn spectra has changed in version 1.2.
        However, the old syntax as kwargs is still compatible ( e.g. 'MS1_Precision=5e-6').
    """

    def __init__(
        self,
        path_or_file,
        MS_precisions=None,
        obo_version=None,
        build_index_from_scratch=False,
        skip_chromatogram=True,
        **kwargs
    ):
        """Initialize and set required attributes."""
        self.build_index_from_scratch = build_index_from_scratch
        self.skip_chromatogram = skip_chromatogram
        if MS_precisions is None:
            MS_precisions = {}
            if "MS1_Precision" in kwargs.keys():
                MS_precisions[1] = kwargs["MS1_Precision"]
            if "MSn_Precision" in kwargs.keys():
                MS_precisions[2] = kwargs["MSn_Precision"]
                MS_precisions[3] = kwargs["MSn_Precision"]

        # Parameters
        self.ms_precisions = {
            None: 0.0001,  # if spectra does not contain ms_level information
                         # e.g. UV-chromatograms (thanks pyeguy) then ms_level is
                         # returned as None
            1: 5e-6,
            2: 20e-6,
        }
        self.ms_precisions.update(MS_precisions)

        # File info
        self.info = ddict()
        self.path_or_file = path_or_file
        if isinstance(path_or_file, str):
            self.info["file_name"] = path_or_file
            self.info["encoding"] = self._determine_file_encoding(path_or_file)
        else:
            self.info["encoding"] = self._guess_encoding(path_or_file)

        self.info["file_object"] = self._open_file(path_or_file)
        self.info["offset_dict"] = self.info["file_object"].offset_dict
        if obo_version:
            self.info["obo_version"] = self._obo_version_validator(obo_version)
        else:
            # obo version not specified -> try to identify from mzML by self._init_iter
            self.info["obo_version"] = None

        self.iter = self._init_iter()
        self.OT = self._init_obo_translator()

    def __next__(self):
        """
        Iterator for the class :py:class:`Run`.

        Iterates all of the spectra in the file.

        Returns:
            Spectrum (:py:class:`Spectrum`): a spectrum object with interface
                to the original spectrum element.

        Example:

        >>> for spectrum in Reader:
        ...     print(spectrum.mz, end='\\r')

        """
        has_ref_group = self.info.get("referenceable_param_group_list", False)
        while True:
            event, element = next(self.iter, ("END", "END"))
            if event == "end":
                if element.tag.endswith("}spectrum"):
                    spectrum = spec.Spectrum(element)
                    if has_ref_group:
                        spectrum._set_params_from_reference_group(
                            self.info["referenceable_param_group_list_element"]
                        )
                    ms_level = spectrum.ms_level
                    spectrum.measured_precision = self.ms_precisions[ms_level]
                    spectrum.calling_instance = self
                    return spectrum
                if element.tag.endswith("}chromatogram"):
                    if self.skip_chromatogram:
                        continue
                    spectrum = spec.Chromatogram(element)
                    # if has_ref_group:
                    #     spectrum._set_params_from_reference_group(
                    #         self.info['referenceable_param_group_list_element']
                    #     )
                    spectrum.calling_instance = self
                    return spectrum
            elif event == "END":
                # reinit iter
                self.info["file_object"] = self._open_file(self.path_or_file)
                self.iter = self._init_iter()
                raise StopIteration

    def __getitem__(self, identifier):
        """
        Access spectrum with native id 'identifier'.

        Arguments:
            identifier (str or int): last number in the id tag of the spectrum
                element

        Returns:
            spectrum (Spectrum or Chromatogram): spectrum/chromatogram object
            with native id 'identifier'
        """
        try:
            if int(identifier) > self.get_spectrum_count():
                raise Exception("Requested identifier is out of range")
        except:
            pass
        spectrum = self.info["file_object"][identifier]
        spectrum.calling_instance = self
        if isinstance(spectrum, spec.Spectrum):
            spectrum.measured_precision = self.ms_precisions[spectrum.ms_level]
        return spectrum

    def __enter__(self):
        return self

    def __exit__(self, type, value, traceback):
        self.close()

    @property
    def file_class(self):
        """Return file object in use."""
        return type(self.info["file_object"].file_handler)

    def _open_file(self, path_or_file):
        """
        Open the path using the FileInterface class as a wrapper.

        Arguments:
            path (str): path to the file to parse

        Returns:
            (FileInterface): Wrapper class for compressed and uncompressed
                mzml files
        """
        return FileInterface(
            path_or_file,
            self.info["encoding"],
            build_index_from_scratch=self.build_index_from_scratch,
        )

    def _guess_encoding(self, mzml_file):
        """
        Determine the encoding used for the file.

        Arguments:
            mzml_file (IOBase): an mzml file

        Returns:
            mzml_encoding (str): encoding type of the file
        """
        match = regex_patterns.FILE_ENCODING_PATTERN.search(mzml_file.readline())
        if match:
            return bytes.decode(match.group("encoding"))
        else:
            return "utf-8"

    def _determine_file_encoding(self, path):
        """
        Determine the encoding used for the file in path.

        Arguments:
            path (str): path to the mzml files

        Returns:
            mzml_encoding (str): encoding type of the file
        """
        if os.path.exists(path):
            if path.endswith(".gz") or path.endswith(".igz"):
                import gzip

                _open = gzip.open
            else:
                _open = open
            with _open(path, "rb") as sniffer:
                return self._guess_encoding(sniffer)

    @staticmethod
    def _obo_version_validator(version):

        """
        The obo version should fit file names in the obo folder.
        However, some software generate mzML with built in obo version string like:
        '23:06:2017' or even newer version that not in obo folder yet.
        This is to check obo version and try to fit the best obo version
        to the obo version in the mzML file.

        Arguments:
            version (str): The original version to check.

        Returns:
            version_fixed (str): The checked obo version.
        """
        obo_rgx = re.compile(r"(\d\.\d{1,2}\.\d{1,2})(_[rR][cC]\d{0,2})?")
        obo_years_rgx = re.compile(r"20\d\d")
        obo_year_version_dct = {
            2012: "3.40.0",
            2013: "3.50.0",
            2014: "3.60.0",
            2015: "3.75.0",
            2016: "4.0.1",
            2017: "4.1.0",
            2018: "4.1.10",
            2019: "4.1.22",
        }
        version_fixed = None
        if obo_rgx.match(version):
            version_fixed = version
        else:
            if obo_years_rgx.search(version):
                years_found = obo_years_rgx.search(version)
                if years_found:
                    try:
                        year = int(years_found.group(0))
                    except ValueError:
                        year = 2000

                    if year in obo_year_version_dct:
                        version_fixed = obo_year_version_dct[year]
                    else:
                        if year > 2019:
                            version_fixed = "4.1.0"

        if version_fixed:
            # Check if the corresponding obo file existed in obo folder
            obo_root = os.path.dirname(__file__)
            obo_file = os.path.join(
                obo_root,
                "obo",
                "psi-ms{0}.obo".format("-" + version_fixed if version_fixed else ""),
            )
            if os.path.exists(obo_file) or os.path.exists(obo_file + ".gz"):
                pass
            else:
                version_fixed = "1.1.0"
        else:
            version_fixed = "1.1.0"

        return version_fixed

    def _init_obo_translator(self):
        """
        Initialize the obo translator with the minimum requirement
        and extra Accessions.

        Returns:
            obo_translator (OboTranslator): translator class to translate
                accessions to names
        """
        # parse obo, check MS tags and if they are ok in minimum.py (minimum
        # required) ...
        if self.info.get("obo_version", None) is None:
            self.info["obo_version"] = "1.1.0"
        obo_translator = obo.OboTranslator(version=self.info["obo_version"])

        return obo_translator

    def _init_iter(self):
        """
        Initalize the iterator for the spectra and sets it to the start
        of the spectrumList element.

        Returns:
            mzml_iter (xml.etree.ElementTree._IterParseIterator): Iterator over
                all element in the file starting with the first spectrum
        """
        mzml_iter = iter(
            ElementTree.iterparse(self.info["file_object"], events=("end", "start"))
        )  # NOTE: end might be sufficient
        _, self.root = next(mzml_iter)
        self.info["chromatogram_count"] = None
        self.info["spectrum_count"] = None
        while True:
            event, element = next(mzml_iter, ("END", "END"))
            if element.tag.endswith("}mzML"):
                if "version" in element.attrib and len(element.attrib["version"]) > 0:
                    self.info["mzml_version"] = element.attrib["version"]
                else:
                    s = element.attrib[
                        "{http://www.w3.org/2001/XMLSchema-instance}" "schemaLocation"
                    ]
                    self.info["mzml_version"] = re.search(
                        r"[0-9]*\.[0-9]*\.[0-9]*", s
                    ).group()
            elif element.tag.endswith("}cv"):
                if (
                    not self.info["obo_version"]
                    and element.attrib.get("id", None) == "MS"
                ):
                    obo_in_mzml = element.attrib.get("version", "1.1.0")
                    self.info["obo_version"] = self._obo_version_validator(obo_in_mzml)

            elif element.tag.endswith("}referenceableParamGroupList"):
                self.info["referenceable_param_group_list"] = True
                self.info["referenceable_param_group_list_element"] = element
            elif element.tag.endswith("}spectrumList"):
                spec_cnt = element.attrib.get("count")
                self.info["spectrum_count"] = int(spec_cnt) if spec_cnt else None
                break
            elif element.tag.endswith("}chromatogramList"):
                chrom_cnt = element.attrib.get("count", None)
                if chrom_cnt:
                    self.info["chromatogram_count"] = int(chrom_cnt)
                break
            elif element.tag.endswith("}run"):
                run_id = element.attrib.get("id")
                start_time = element.attrib.get("startTimeStamp")
                self.info["run_id"] = run_id
                self.info["start_time"] = start_time
            else:
                pass
        self.root.clear()
        return mzml_iter

    def __iter__(self):
        """Return self."""
        return self

    def next(self):
        """Function to return the next Spectrum element."""
        return self.__next__()

    def get_spectrum_count(self):
        """
        Number of spectra in file.

        Returns:
            spectrum count (int): Number of spectra in file.
        """
        return self.info["spectrum_count"]

    def get_chromatogram_count(self):
        """
        Number of chromatograms in file.

        Returns:
            chromatogram count (int): Number of chromatograms in file.
        """
        return self.info["chromatogram_count"]

    def close(self):
        self.info["file_object"].close()

if __name__ == "__main__":
    print(__doc__)
<|endoftext|>


<commit_before>"""
Base information for pysal meta package
"""

federation_hierarchy = {
    'explore': ['esda', 'giddy', 'segregation',
                'pointpats', 'inequality',
                'segregation', 'spaghetti'],
    'model': ['spreg', 'spglm', 'tobler', 'spint',
              'mgwr', 'spvcm'],
    'viz': ['splot', 'mapclassify'],
    'lib': ['libpysal']
}

memberships = {'libpysal': 'lib',
               'esda': 'explore',
               'giddy': 'explore',
               'segregation': 'explore',
               'pointpats': 'explore',
               'inequality': 'explore',
               'spaghetti': 'explore',
               'spreg': 'model',
               'spglm': 'model',
               'spint': 'model',
               'tobler': 'model',
               'spvcm': 'model',
               'splot': 'viz',
               'mapclassify': 'viz'
}

class cached_property(object):
    """ A property that is only computed once per instance and then replaces
        itself with an ordinary attribute. Deleting the attribute resets the
        property.

        Source: https://github.com/bottlepy/bottle/commit/fa7733e075da0d790d809aa3d2f53071897e6f76
        """

    def __init__(self, func):
        self.__doc__ = getattr(func, '__doc__')
        self.func = func

    def __get__(self, obj, cls):
        if obj is None:
            return self
        value = obj.__dict__[self.func.__name__] = self.func(obj)
        return value

def _installed_version(package):
    try:
        exec(f'import {package}')
    except ModuleNotFoundError:
        v = 'NA'
    try:
        v = eval(f'{package}.__version__')
    except AttributeError:
        v = 'NA'
    return v

def _installed_versions():
    ver = {}
    for package in memberships.keys():
        ver[package] = _installed_version(package)
    return ver

def _released_versions():
    from .frozen import frozen_packages
    return frozen_packages

class Versions:
    @cached_property
    def installed(self):
        """
        Inventory versions of pysal packages that are installed

        Attributes
        ----------
        installed: dict
                   key is package name, value is version string
        """
        return _installed_versions()

    @cached_property
    def released(self):
        """
        Inventory versions of pysal packages that are released in the meta
        package.

        Attributes
        ----------
        released: dict
                   key is package name, value is version string
        """

        return _released_versions()

    def check(self):
        """
        Print a tabular string that reports installed and released versions of
        PySAL packages.
        """
        table = []
        package = "Package"
        installed = "Installed"
        released = "Released"
        match = "Match"
        s = f'{package:>12} | {installed:>15} | {released:>15} | {match:>5}'
        table.append(s)
        table.append("-"*len(s))
        for package in self.installed:
            installed = self.installed[package]
            released = self.released[package]
            match = installed == released
            s = f'{package:>12} | {installed:>15} | {released:>15} | {match:>5}'
            table.append(s)
        print("\n".join(table))

versions = Versions()

<commit_msg>[ENH] Adding access<commit_after>"""
Base information for pysal meta package
"""

federation_hierarchy = {
    'explore': ['esda', 'giddy', 'segregation',
                'pointpats', 'inequality',
                'segregation', 'spaghetti'],
    'model': ['spreg', 'spglm', 'tobler', 'spint',
              'mgwr', 'spvcm', 'access'],
    'viz': ['splot', 'mapclassify'],
    'lib': ['libpysal']
}

memberships = {'libpysal': 'lib',
               'esda': 'explore',
               'giddy': 'explore',
               'segregation': 'explore',
               'pointpats': 'explore',
               'inequality': 'explore',
               'spaghetti': 'explore',
               'access': 'access',
               'spreg': 'model',
               'spglm': 'model',
               'spint': 'model',
               'tobler': 'model',
               'spvcm': 'model',
               'splot': 'viz',
               'mapclassify': 'viz'
}

class cached_property(object):
    """ A property that is only computed once per instance and then replaces
        itself with an ordinary attribute. Deleting the attribute resets the
        property.

        Source: https://github.com/bottlepy/bottle/commit/fa7733e075da0d790d809aa3d2f53071897e6f76
        """

    def __init__(self, func):
        self.__doc__ = getattr(func, '__doc__')
        self.func = func

    def __get__(self, obj, cls):
        if obj is None:
            return self
        value = obj.__dict__[self.func.__name__] = self.func(obj)
        return value

def _installed_version(package):
    try:
        exec(f'import {package}')
    except ModuleNotFoundError:
        v = 'NA'
    try:
        v = eval(f'{package}.__version__')
    except AttributeError:
        v = 'NA'
    return v

def _installed_versions():
    ver = {}
    for package in memberships.keys():
        ver[package] = _installed_version(package)
    return ver

def _released_versions():
    from .frozen import frozen_packages
    return frozen_packages

class Versions:
    @cached_property
    def installed(self):
        """
        Inventory versions of pysal packages that are installed

        Attributes
        ----------
        installed: dict
                   key is package name, value is version string
        """
        return _installed_versions()

    @cached_property
    def released(self):
        """
        Inventory versions of pysal packages that are released in the meta
        package.

        Attributes
        ----------
        released: dict
                   key is package name, value is version string
        """

        return _released_versions()

    def check(self):
        """
        Print a tabular string that reports installed and released versions of
        PySAL packages.
        """
        table = []
        package = "Package"
        installed = "Installed"
        released = "Released"
        match = "Match"
        s = f'{package:>12} | {installed:>15} | {released:>15} | {match:>5}'
        table.append(s)
        table.append("-"*len(s))
        for package in self.installed:
            installed = self.installed[package]
            released = self.released[package]
            match = installed == released
            s = f'{package:>12} | {installed:>15} | {released:>15} | {match:>5}'
            table.append(s)
        print("\n".join(table))

versions = Versions()

<|endoftext|>


<commit_before># coding: utf-8
"""
Public endpoint to import API classes

Instead of importing each module individually (eg.
``from pysuru.apps import AppsAPI``), import from this module.
"""
from .apps import AppsAPI
from .services import ServiceInstanceAPI

__all__ = (
    'AppsAPI',
    'ServiceInstanceAPI',
)
<commit_msg>Remove ``pysuru.api`` module<commit_after><|endoftext|>


<commit_before>import sqlite3

import requests

from .HashStore import LocationService, MimetypeService, IPLDHashService
from .Multihash import Multihash
from .NameResolver import NameResolverDir, NameResolverFile
from .miscutils import httpget

class DOI(NameResolverDir):
    """
    DOI is a class for Document Object Identifiers, an academic namespace
    typically of form   10.1234/abc123
    URL: `/xxx/doi/10.pub_id/pub_specific_id` (forwarded here by ServerGateway methods)

    Implements name resolution of the DOI namespace, via a sqlite database (provided by Brian Reynolds)

    Case insensitive, have some significant punctuation, but sometimes presented with insignificant punctuation

    Future Work
    * Build way to preload the hashstore with the hashes and URLs from the sqlite

    TODO - ssome of this will end up in NameResolverDir as we build other classe and see commonalities
    """

    """
    $ cd data; sqlite3 idents_files_urls.sqlite .schema
    CREATE TABLE files_id_doi (doi text not null, sha1 char(40) not null, type text);
    CREATE UNIQUE INDEX file_id_doi_sha1 on files_id_doi (sha1);
    CREATE INDEX files_doi on files_id_doi (doi);
    CREATE TABLE files_metadata (sha1 char(40) not null, mimetype text, size_bytes integer, md5 char(32));
    CREATE INDEX files_metadata_sha1 on files_metadata (sha1);
    CREATE TABLE urls (sha1 char(40) not null, url text not null, datetime integer);
    CREATE INDEX url_sha1 on urls (sha1);
    """

    # SQLITE="../data/idents_files_urls_sqlite"   # Old version in Python2 when working dir was "python"
    SQLITE="data/idents_files_urls.sqlite"
    _sqliteconnection=None

    def __init__(self, namespace, publisher, *identifier, **kwargs):
        """
        Creates the object

        :param namespace:
        :param publisher:   Publisher ID allocated by DOI.org, always of form 10.nnnn
        :param identifier:  Id for an academic article or resource, allocated by the publisher. Case insensitive, Alphanumeric plus a few (undefined) punctuation.
                            Its an array, because may be multiple fields seperated by /, can safely re-concatenate
        :param kwargs:      Any other args to the URL, ignored for now.
        """
        #Note if you dont have your own way of using sqlite I suggest SqliteWrap from https://github.com/mitra42/sqlite_models
        """
        Pseudo-code
        * Canonicalize pub_specific_id (lowercase, strip chars)
        * Look up in sqlite to find the [hash, location]* of the file (no need to retrieve it)
        * Convert hashes to multihashes
        * For each multihash: location_store(multihash, location)
        *   Create a DOIfile(..)
        *   self.push(DOIfile)
        """
        verbose=kwargs.get("verbose",False)
        if verbose: print("DOI.__init__",namespace,publisher,identifier)
        super(DOI,self).__init__(namespace, publisher, *identifier)
        db = self.sqliteconnection(verbose)                     # Lazy connection to database
        self.doi = self.canonical(publisher, *identifier)    # "10.nnnn/xxxx/yyyy"
        self.metadata = {}
        if verbose: print("DOI.__init__ getting metadata for",self.doi)
        self.doi_org_metadata = {}  # Will hold metadata retrieved from doi.org
        self.get_doi_metadata(verbose)
        if verbose: print("DOI.__init__ looking up",self.doi)
        sha1_list = list(db.execute('SELECT * FROM files_id_doi WHERE doi = ?;', [self.doi]))

        if verbose: print("DOI.__init__ iterating over",len(sha1_list),"rows")
        for row in sha1_list:
            _, sha1_hex, _ = row
            doifile = DOIfile(doi=self.doi, multihash=Multihash(sha1_hex=sha1_hex))
            self.push(doifile)
        if verbose: print("DOI.__init__ completing")

    @classmethod
    def sqliteconnection(cls, verbose=False):
        if not cls._sqliteconnection:
            if verbose: print("DOI.sqliteconnection connecting to DB")
            cls._sqliteconnection = sqlite3.connect(cls.SQLITE)
            if verbose: print("DOI.sqliteconnection connected to DB")
        return cls._sqliteconnection

    @staticmethod
    def archive_url(row):
        """
        Take a tuple of sha-1 URL datetime and return a direct URL to file content
        Currently the sqlite database has URLs with an optional datetime column.
        If the url starts "https://archive.org/download/" then it's an item/file pointer, and there is no datetime.
        If the url starts with something else, it's a URL from wayback, and the datetime can be used to construct a
        "https://web.archive.org/web/<datetime>/<url>" URL. TODO: double-check that wayback gateway supports range-requests
        :return: url to file content
        """
        _, url, datetime = row  # sha1, url, datetime(optional)
        return 'https://web.archive.org/web/{}/{}'.format(datetime, url) if datetime else url

    def push(self,doifile):
        """
        Add a DOIfile to a DOI -
        :return:    undefined
        """
        # Currently Nothing done here other than superclass adding to list.
        super(DOI, self).push(doifile)

    def content(self, verbose=False):
        return {'Content-type': 'application/json',
            'data': {
                'metadata': self.metadata,  # Archive generated metadata - there isnt any, its all at files level for DOI
                'doi_org_metadata': self.doi_org_metadata,  # Metadata as supplied by DOI.org
                "files": [
                    doifile.metadata for doifile in self.files()
                ]
            }
        }

    @classmethod
    def canonical(cls, publisher, *identifier):
        """
        As a pre-mature heads up, the DOI format is sort of underspecified.
        It is case-insensitive (can be used in URLs, sometimes with encoding),
        but can have weird shit like whitespace.
        The vast majority are more "well behaved", but things like parentheses are common.
        They do always start with "10\.\d+\/" (aka, 10, period, numerical digits, slash)
        Brian says its a long-tail, the vast majority of correct DOI appear to be case insensitive alphanumeric with some allowed punctuation
        This will require lookin in the Sqlite (or asking Brian) to determine suitable characters for a OK DOI
        """
        #TODO convert this identifier into a canonicised form,
        #TDO check publisher is canonicised 10.nnnn
        return publisher.lower() + "/" + "/".join([i.lower() for i in identifier])

    def check_if_link_works(self, url, verbose=False):
        """
        See if a link is valid (i.e., returns a '200' to the HTML request).
        """
        if verbose:
            print("check_if_link_works",url)
        print("XXX@check_if_link_works - dummied out")
        #return True
        try:
            headers = {"accept": "*/*"}
            # This next link can fail, it follows a redirection and then can fail on th actual PDF, which isnt what we want cos we'll use a Archive URL
            request = requests.get(url, headers=headers)
        except Exception as e:
            raise e
        if verbose: print("result=", request.status_code)
        return request.status_code == 200

    def get_doi_metadata(self, verbose):
        """
        For a DOI, get metadata from doi.org about that file
        TODO: pick which fields want to analyze, e.g.
        :return: metadata on the doi in json format
        """
        url = "http://dx.doi.org/" + self.doi
        headers = {"accept": "application/vnd.citationstyles.csl+json"}
        r = requests.get(url, headers=headers) # Note that with headers it wont redirect, without it will go to doc which may fail
        if verbose: print("get_doi_metadata returned:",r)
        if r.status_code == 200:
            self.doi_org_metadata = r.json()
        # If dont get metadata, the rest of our info may still be valid

class DOIfile(NameResolverFile):
    """
    Class for one file
    """

    # TODO get ContentHash to build a DOIfile

    def __init__(self, doi=None, multihash=None, metadata=None, verbose=False):
        super(NameResolverFile, self).__init__(metadata)
        self.doi = doi
        self.metadata = metadata or {}    # For now all in one dict
        self.multihash = multihash
        if multihash and not self.doi:
            # Lookup DOI from sha1_hex if not supplied.
            if verbose: print("DOIfile.__init__ looking up", multihash.sha1_hex)
            self.doi, _, _ = list(DOI.sqliteconnection(verbose).execute('SELECT * FROM files_id_doi WHERE sha1 = ?;', [multihash.sha1_hex]))[0]
        if multihash:
            self.sqlite_metadata(verbose)

    def sqlite_metadata(self, verbose):
            files_metadata_list = list(DOI.sqliteconnection(verbose).execute('SELECT * FROM files_metadata WHERE sha1 = ?;', [self.multihash.sha1_hex]))
            _, mimetype, size_bytes, md5 = files_metadata_list[0]
            files_list = list(DOI.sqliteconnection(verbose).execute('SELECT * FROM urls WHERE sha1 = ?;', [self.multihash.sha1_hex]))
            self.metadata = { 'mimetype': mimetype, 'size_bytes': size_bytes, 'md5': md5, 'multihash58': self.multihash.multihash58,
                    'files': [DOI.archive_url(file) for file in files_list]}
            if verbose: print("multihash base58=",self.multihash.multihash58)
            #multihash58_sha256 = Multihash(data=doifile.retrieve(), code=SHA256)
            #print("Saving location", multihash58_sha256, doifile.metadata["urls"][0]  )
            LocationService.set(self.multihash.multihash58, self.metadata["files"][0],verbose=verbose)
            MimetypeService.set(self.multihash.multihash58, self.metadata["mimetype"],verbose=verbose)
            ipldhash = IPLDHashService.get(self.multihash.multihash58)    # May be None, we don't know it
            if not ipldhash:
                data = httpget(self.metadata["files"][0])
                ipldhash = requests.post('http://localhost:5001/api/v0/add', files={'file': ('', data, self.metadata["mimetype"])}).json()['Hash']
                IPLDHashService.set(self.multihash.multihash58, ipldhash)
            self.metadata["ipldhash"] = ipldhash

    def retrieve(self):
        return httpget(self.metadata["urls"][0])

    def content(self):
        #TODO iterate over urls and find first matching hash
        return { "Content-type": self.metadata["mimetype"], "data": self.retrieve() }

if __name__ == '__main__':
    import sys
    if len(sys.argv) != 2:
        print('hey I expected a single doi!!')
        sys.exit(-1)
    doi = DOI("doi", *sys.argv[1].split('/'))

    for i in doi.files():
        print(i)
        print(i.content())
<commit_msg>Changes made on server<commit_after>import sqlite3

import requests

from .HashStore import LocationService, MimetypeService, IPLDHashService
from .Multihash import Multihash
from .NameResolver import NameResolverDir, NameResolverFile
from .miscutils import httpget

class DOI(NameResolverDir):
    """
    DOI is a class for Document Object Identifiers, an academic namespace
    typically of form   10.1234/abc123
    URL: `/xxx/doi/10.pub_id/pub_specific_id` (forwarded here by ServerGateway methods)

    Implements name resolution of the DOI namespace, via a sqlite database (provided by Brian Reynolds)

    Case insensitive, have some significant punctuation, but sometimes presented with insignificant punctuation

    Future Work
    * Build way to preload the hashstore with the hashes and URLs from the sqlite

    TODO - ssome of this will end up in NameResolverDir as we build other classe and see commonalities
    """

    """
    $ cd data; sqlite3 idents_files_urls.sqlite .schema
    CREATE TABLE files_id_doi (doi text not null, sha1 char(40) not null, type text);
    CREATE UNIQUE INDEX file_id_doi_sha1 on files_id_doi (sha1);
    CREATE INDEX files_doi on files_id_doi (doi);
    CREATE TABLE files_metadata (sha1 char(40) not null, mimetype text, size_bytes integer, md5 char(32));
    CREATE INDEX files_metadata_sha1 on files_metadata (sha1);
    CREATE TABLE urls (sha1 char(40) not null, url text not null, datetime integer);
    CREATE INDEX url_sha1 on urls (sha1);
    """

    # SQLITE="../data/idents_files_urls_sqlite"   # Old version in Python2 when working dir was "python"
    SQLITE="data/idents_files_urls.sqlite"
    _sqliteconnection=None

    def __init__(self, namespace, publisher, *identifier, **kwargs):
        """
        Creates the object

        :param namespace:
        :param publisher:   Publisher ID allocated by DOI.org, always of form 10.nnnn
        :param identifier:  Id for an academic article or resource, allocated by the publisher. Case insensitive, Alphanumeric plus a few (undefined) punctuation.
                            Its an array, because may be multiple fields seperated by /, can safely re-concatenate
        :param kwargs:      Any other args to the URL, ignored for now.
        """
        #Note if you dont have your own way of using sqlite I suggest SqliteWrap from https://github.com/mitra42/sqlite_models
        """
        Pseudo-code
        * Canonicalize pub_specific_id (lowercase, strip chars)
        * Look up in sqlite to find the [hash, location]* of the file (no need to retrieve it)
        * Convert hashes to multihashes
        * For each multihash: location_store(multihash, location)
        *   Create a DOIfile(..)
        *   self.push(DOIfile)
        """
        verbose=kwargs.get("verbose",False)
        if verbose: print("DOI.__init__",namespace,publisher,identifier)
        super(DOI,self).__init__(namespace, publisher, *identifier)
        db = self.sqliteconnection(verbose)                     # Lazy connection to database
        self.doi = self.canonical(publisher, *identifier)    # "10.nnnn/xxxx/yyyy"
        self.metadata = {}
        if verbose: print("DOI.__init__ getting metadata for",self.doi)
        self.doi_org_metadata = {}  # Will hold metadata retrieved from doi.org
        self.get_doi_metadata(verbose)
        if verbose: print("DOI.__init__ looking up",self.doi)
        sha1_list = list(db.execute('SELECT * FROM files_id_doi WHERE doi = ?;', [self.doi]))

        if verbose: print("DOI.__init__ iterating over",len(sha1_list),"rows")
        for row in sha1_list:
            _, sha1_hex, _ = row
            doifile = DOIfile(doi=self.doi, multihash=Multihash(sha1_hex=sha1_hex))
            self.push(doifile)
        if verbose: print("DOI.__init__ completing")

    @classmethod
    def sqliteconnection(cls, verbose=False):
        if not cls._sqliteconnection:
            if verbose: print("DOI.sqliteconnection connecting to DB")
            cls._sqliteconnection = sqlite3.connect(cls.SQLITE)
            if verbose: print("DOI.sqliteconnection connected to DB")
        return cls._sqliteconnection

    @staticmethod
    def archive_url(row):
        """
        Take a tuple of sha-1 URL datetime and return a direct URL to file content
        Currently the sqlite database has URLs with an optional datetime column.
        If the url starts "https://archive.org/download/" then it's an item/file pointer, and there is no datetime.
        If the url starts with something else, it's a URL from wayback, and the datetime can be used to construct a
        "https://web.archive.org/web/<datetime>/<url>" URL. TODO: double-check that wayback gateway supports range-requests
        :return: url to file content
        """
        _, url, datetime = row  # sha1, url, datetime(optional)
        return 'https://web.archive.org/web/{}/{}'.format(datetime, url) if datetime else url

    def push(self,doifile):
        """
        Add a DOIfile to a DOI -
        :return:    undefined
        """
        # Currently Nothing done here other than superclass adding to list.
        super(DOI, self).push(doifile)

    def content(self, verbose=False):
        return {'Content-type': 'application/json',
            'data': {
                'metadata': self.metadata,  # Archive generated metadata - there isnt any, its all at files level for DOI
                'doi_org_metadata': self.doi_org_metadata,  # Metadata as supplied by DOI.org
                "files": [
                    doifile.metadata for doifile in self.files()
                ]
            }
        }

    @classmethod
    def canonical(cls, publisher, *identifier):
        """
        As a pre-mature heads up, the DOI format is sort of underspecified.
        It is case-insensitive (can be used in URLs, sometimes with encoding),
        but can have weird shit like whitespace.
        The vast majority are more "well behaved", but things like parentheses are common.
        They do always start with "10\.\d+\/" (aka, 10, period, numerical digits, slash)
        Brian says its a long-tail, the vast majority of correct DOI appear to be case insensitive alphanumeric with some allowed punctuation
        This will require lookin in the Sqlite (or asking Brian) to determine suitable characters for a OK DOI
        """
        #TODO convert this identifier into a canonicised form,
        #TDO check publisher is canonicised 10.nnnn
        return publisher.lower() + "/" + "/".join([i.lower() for i in identifier])

    def check_if_link_works(self, url, verbose=False):
        """
        See if a link is valid (i.e., returns a '200' to the HTML request).
        """
        if verbose:
            print("check_if_link_works",url)
        print("XXX@check_if_link_works - dummied out")
        #return True
        try:
            headers = {"accept": "*/*"}
            # This next link can fail, it follows a redirection and then can fail on th actual PDF, which isnt what we want cos we'll use a Archive URL
            request = requests.get(url, headers=headers)
        except Exception as e:
            raise e
        if verbose: print("result=", request.status_code)
        return request.status_code == 200

    def get_doi_metadata(self, verbose):
        """
        For a DOI, get metadata from doi.org about that file
        TODO: pick which fields want to analyze, e.g.
        :return: metadata on the doi in json format
        """
        url = "http://dx.doi.org/" + self.doi
        headers = {"accept": "application/vnd.citationstyles.csl+json"}
        r = requests.get(url, headers=headers) # Note that with headers it wont redirect, without it will go to doc which may fail
        if verbose: print("get_doi_metadata returned:",r)
        if r.status_code == 200:
            self.doi_org_metadata = r.json()
        else:
            print("Failed to read metadata at",url)
        # If dont get metadata, the rest of our info may still be valid

class DOIfile(NameResolverFile):
    """
    Class for one file
    """

    # TODO get ContentHash to build a DOIfile

    def __init__(self, doi=None, multihash=None, metadata=None, verbose=False):
        super(NameResolverFile, self).__init__(metadata)
        self.doi = doi
        self.metadata = metadata or {}    # For now all in one dict
        self.multihash = multihash
        if multihash and not self.doi:
            # Lookup DOI from sha1_hex if not supplied.
            if verbose: print("DOIfile.__init__ looking up", multihash.sha1_hex)
            self.doi, _, _ = list(DOI.sqliteconnection(verbose).execute('SELECT * FROM files_id_doi WHERE sha1 = ?;', [multihash.sha1_hex]))[0]
        if multihash:
            self.sqlite_metadata(verbose)

    def sqlite_metadata(self, verbose):
            files_metadata_list = list(DOI.sqliteconnection(verbose).execute('SELECT * FROM files_metadata WHERE sha1 = ?;', [self.multihash.sha1_hex]))
            _, mimetype, size_bytes, md5 = files_metadata_list[0]
            files_list = list(DOI.sqliteconnection(verbose).execute('SELECT * FROM urls WHERE sha1 = ?;', [self.multihash.sha1_hex]))
            self.metadata = { 'mimetype': mimetype, 'size_bytes': size_bytes, 'md5': md5, 'multihash58': self.multihash.multihash58,
                    'files': [DOI.archive_url(file) for file in files_list]}
            if verbose: print("multihash base58=",self.multihash.multihash58)
            #multihash58_sha256 = Multihash(data=doifile.retrieve(), code=SHA256)
            #print("Saving location", multihash58_sha256, doifile.metadata["urls"][0]  )
            LocationService.set(self.multihash.multihash58, self.metadata["files"][0],verbose=verbose)
            MimetypeService.set(self.multihash.multihash58, self.metadata["mimetype"],verbose=verbose)
            ipldhash = IPLDHashService.get(self.multihash.multihash58)    # May be None, we don't know it
            if not ipldhash:
                data = httpget(self.metadata["files"][0])
                ipldhash = requests.post('http://localhost:5001/api/v0/add', files={'file': ('', data, self.metadata["mimetype"])}).json()['Hash']
                IPLDHashService.set(self.multihash.multihash58, ipldhash)
            self.metadata["ipldhash"] = ipldhash

    def retrieve(self):
        return httpget(self.metadata["urls"][0])

    def content(self):
        #TODO iterate over urls and find first matching hash
        return { "Content-type": self.metadata["mimetype"], "data": self.retrieve() }

if __name__ == '__main__':
    import sys
    if len(sys.argv) != 2:
        print('hey I expected a single doi!!')
        sys.exit(-1)
    doi = DOI("doi", *sys.argv[1].split('/'))

    for i in doi.files():
        print(i)
        print(i.content())
<|endoftext|>


<commit_before># Copyright 2018 Google LLC
#
# Use of this source code is governed by an MIT-style
# license that can be found in the LICENSE file or at
# https://opensource.org/licenses/MIT.

import Crypto.Cipher.AES

import cipher

class AES(cipher.Blockcipher):
    def set_keylen(self, k):
        self.choose_variant(lambda v: v["lengths"]["key"] == k)

    def variant_name(self):
        l = self.lengths()
        return "{}{}".format(self.name(), l['key'] * 8)

    def variants(self):
        for kl in [16, 24, 32]:
            yield {
                'cipher': 'AES',
                'lengths': {
                    'block': 16,
                    'key': kl
                }
            }

    def encrypt(self, pt, key):
        assert len(key) == self.lengths()['key']
        a = Crypto.Cipher.AES.new(key)
        return a.encrypt(pt)

    def decrypt(self, ct, key):
        assert len(key) == self.lengths()['key']
        a = Crypto.Cipher.AES.new(key)
        return a.decrypt(ct)
<commit_msg>aes.py: support pycryptodome<commit_after># Copyright 2018 Google LLC
#
# Use of this source code is governed by an MIT-style
# license that can be found in the LICENSE file or at
# https://opensource.org/licenses/MIT.

import Crypto.Cipher.AES

import cipher

class AES(cipher.Blockcipher):
    def set_keylen(self, k):
        self.choose_variant(lambda v: v["lengths"]["key"] == k)

    def variant_name(self):
        l = self.lengths()
        return "{}{}".format(self.name(), l['key'] * 8)

    def variants(self):
        for kl in [16, 24, 32]:
            yield {
                'cipher': 'AES',
                'lengths': {
                    'block': 16,
                    'key': kl
                }
            }

    def encrypt(self, pt, key):
        assert len(key) == self.lengths()['key']
        a = Crypto.Cipher.AES.new(key, Crypto.Cipher.AES.MODE_ECB)
        return a.encrypt(pt)

    def decrypt(self, ct, key):
        assert len(key) == self.lengths()['key']
        a = Crypto.Cipher.AES.new(key, Crypto.Cipher.AES.MODE_ECB)
        return a.decrypt(ct)
<|endoftext|>


<commit_before>#!/usr/bin/python
import glob
import os
import sys
import logging
import json
from optparse import OptionParser
from lxml import etree
from xml.etree.ElementTree import QName

log = logging.getLogger("count_descriptive_text")
log.setLevel(logging.DEBUG)
namespace = "http://schemas.android.com/apk/res/android"

def count_elements(filename, element_name):
    tree = etree.parse(filename)
    attribute_full_name = str(QName(namespace, 'android:contentDescription'))
    total = tree.xpath('count(//' + element_name + ')')
    with_count = tree.xpath('count(//' + element_name +
        '[@android:contentDescription])',
        namespaces={'android': 'http://schemas.android.com/apk/res/android'})
    without_count = tree.xpath('count(//' + element_name +
            '[not(@android:contentDescription)])',
            namespaces={'android': 'http://schemas.android.com/apk/res/android'})
    return (total, with_count, without_count)

def count_descriptive_text(element_name, source_dir, target_dir):
    count = 0
    result = []
    for f in glob.iglob(os.path.join(source_dir, '**/', '*.xml')):
        log.info("%i - Checking the number of " + element_name + " in %s",
                count, os.path.abspath(f))
        count +=1
        base_name = os.path.basename(f)
        base_name = os.path.splitext(base_name)[0]
        package_name = base_name.rsplit('-', 1)[0]
        version_code = base_name.rsplit('-', 1)[1]
        d = {"id": package_name + "-" + version_code,
                "packageName" : package_name, "versionCode": version_code,
                "total_" + element_name: 0, "count_with_contentDescription": 0,
                "count_without_contentDescription": 0}
        total, with_count, without_count = count_elements(f, element_name)
        d["total_"+element_name] = total
        d["count_with_contentDescription"] = with_count
        d["count_without_contentDescription"] = without_count
        result.append(d)
    out_file = os.path.join(target_dir, element_name + '_descriptive_text_count.json')
    with open(out_file, 'w+') as fw:
        json.dump(result, fw)
    log.info("Saved the result at %s", out_file)

def main(args):
    formatter = logging.Formatter('%(asctime)s - %(levelname)s - %(message)s')
    logging_console = logging.StreamHandler(sys.stdout)
    logging_console.setFormatter(formatter)
    logging_console.setLevel(logging.DEBUG)
    log.addHandler(logging_console)
    usage_info = ''' python %prog <View> <source_directory> <target_directory> [options]

    Parses ui-xml files and counts how many <View> elements are defined with
    the android:contentDescription attribute in each app.

    View elements can be any Android View such as, ImageButton, ImageView, CheckBox, etc.
    The tool will attempt to recursively find xml files in source_directory.
    Each xml file is expected to be named in the following naming schemes packageName-versionCode.xml.
    The result is a JSON file with the name "View_descriptive_text_count.json"
    '''
    parser = OptionParser(usage=usage_info, version="%prog 1.0")
    parser.add_option("-l", "--log", dest="log_file",
            help="write logs to FILE.", metavar="FILE")
    (options, args) = parser.parse_args()
    if options.log_file:
        logging_file = logging.FileHandler(options.log_file, mode='a',
                encoding='utf-8', delay=False)
        logging_file.setLevel(logging_level)
        logging_file.setFormatter(formatter)
        log.addHandler(logging_file)
    if len(args) != 3:
        parser.error("incorrect number of arguments.")
    cmd_list = ['ImageButton', 'ImageView', 'CheckBox']
    if args[0] not in cmd_list:
        sys.exit(args[0] + ". Error: unknown command. Valid commands are: " + ", ".join(cmd_list))
    if not os.path.isdir(args[1]):
        sys.exit("Error: source directory " + args[1] + " does not exist.")
    if not os.path.isdir(args[2]):
        sys.exit("Error: target directory " + args[2] + " does not exist.")
    count_descriptive_text(args[0], args[1], args[2])

if __name__ == '__main__':
    if sys.version_info >= (3,5):
        main(sys.argv[1:])
    else:
        sys.exit("ERROR: This tool requires Python version 3.5 or higher")
<commit_msg>parse through broken XML<commit_after>#!/usr/bin/python
import glob
import os
import sys
import logging
import json
from optparse import OptionParser
from lxml import etree
from xml.etree.ElementTree import QName
from lxml.etree import ParserError
from lxml.etree import XMLSyntaxError

log = logging.getLogger("count_descriptive_text")
log.setLevel(logging.DEBUG)
namespace = "http://schemas.android.com/apk/res/android"

def count_elements(filename, element_name):
    parser = etree.XMLParser(recover=True)
    tree = etree.parse(filename, parser)
    attribute_full_name = str(QName(namespace, 'android:contentDescription'))
    total = tree.xpath('count(//' + element_name + ')')
    with_count = tree.xpath('count(//' + element_name +
        '[@android:contentDescription])',
        namespaces={'android': 'http://schemas.android.com/apk/res/android'})
    without_count = tree.xpath('count(//' + element_name +
            '[not(@android:contentDescription)])',
            namespaces={'android': 'http://schemas.android.com/apk/res/android'})
    return (total, with_count, without_count)

def count_descriptive_text(element_name, source_dir, target_dir):
    count = 0
    result = []
    for f in glob.iglob(os.path.join(source_dir, '**/', '*.xml'), recursive=True):
        log.info("%i - Checking the number of " + element_name + " in %s",
                count, os.path.abspath(f))
        count +=1
        base_name = os.path.basename(f)
        base_name = os.path.splitext(base_name)[0]
        package_name = base_name.rsplit('-', 1)[0]
        version_code = base_name.rsplit('-', 1)[1]
        d = {"id": package_name + "-" + version_code,
                "packageName" : package_name, "versionCode": version_code,
                "total_" + element_name: 0, "count_with_contentDescription": 0,
                "count_without_contentDescription": 0}
        try:
            total, with_count, without_count = count_elements(f, element_name)
            d["total_"+element_name] = total
            d["count_with_contentDescription"] = with_count
            d["count_without_contentDescription"] = without_count
            result.append(d)
        except (ParserError, XMLSyntaxError) as e:
            log.error("Error in parsing file: %s", f)
            continue
    out_file = os.path.join(target_dir, element_name + '_descriptive_text_count.json')
    with open(out_file, 'w+') as fw:
        json.dump(result, fw, indent=0)
    log.info("Saved the result at %s", out_file)

def main(args):
    formatter = logging.Formatter('%(asctime)s - %(levelname)s - %(message)s')
    logging_console = logging.StreamHandler(sys.stdout)
    logging_console.setFormatter(formatter)
    logging_console.setLevel(logging.DEBUG)
    log.addHandler(logging_console)
    usage_info = ''' python %prog <View> <source_directory> <target_directory> [options]

    Parses ui-xml files and counts how many <View> elements are defined with
    the android:contentDescription attribute in each app.

    View elements can be any Android View such as, ImageButton, ImageView, CheckBox, etc.
    The tool will attempt to recursively find xml files in source_directory.
    Each xml file is expected to be named in the following naming schemes packageName-versionCode.xml.
    The result is a JSON file with the name "View_descriptive_text_count.json"
    '''
    parser = OptionParser(usage=usage_info, version="%prog 1.0")
    parser.add_option("-l", "--log", dest="log_file",
            help="write logs to FILE.", metavar="FILE")
    (options, args) = parser.parse_args()
    if options.log_file:
        logging_file = logging.FileHandler(options.log_file, mode='a',
                encoding='utf-8', delay=False)
        logging_file.setLevel(logging_level)
        logging_file.setFormatter(formatter)
        log.addHandler(logging_file)
    if len(args) != 3:
        parser.error("incorrect number of arguments.")
    cmd_list = ['ImageButton', 'ImageView', 'CheckBox']
    if args[0] not in cmd_list:
        sys.exit(args[0] + ". Error: unknown command. Valid commands are: " + ", ".join(cmd_list))
    if not os.path.isdir(args[1]):
        sys.exit("Error: source directory " + args[1] + " does not exist.")
    if not os.path.isdir(args[2]):
        sys.exit("Error: target directory " + args[2] + " does not exist.")
    count_descriptive_text(args[0], args[1], args[2])

if __name__ == '__main__':
    if sys.version_info >= (3,5):
        main(sys.argv[1:])
    else:
        sys.exit("ERROR: This tool requires Python version 3.5 or higher")
<|endoftext|>


<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-

# === haiku.pickle.simple__test -------------------------------------------===
# Copyright © 2011-2012, RokuSigma Inc. and contributors. See AUTHORS for more
# details.
#
# Some rights reserved.
#
# Redistribution and use in source and binary forms of the software as well as
# documentation, with or without modification, are permitted provided that the
# following conditions are met:
#
#  * Redistributions of source code must retain the above copyright notice,
#    this list of conditions and the following disclaimer.
#  * Redistributions in binary form must reproduce the above copyright notice,
#    this list of conditions and the following disclaimer in the documentation
#    and/or other materials provided with the distribution.
#  * The names of the copyright holders or contributors may not be used to
#    endorse or promote products derived from this software without specific
#    prior written permission.
#
# THIS SOFTWARE AND DOCUMENTATION IS PROVIDED BY THE COPYRIGHT HOLDERS AND
# CONTRIBUTORS “AS IS” AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT
# NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
# PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
# OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
# WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR
# OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE AND
# DOCUMENTATION, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
# ===----------------------------------------------------------------------===

# Python standard library, unit-testing
import unittest2

# Python patterns, scenario unit-testing
from python_patterns.unittest.scenario import ScenarioMeta, ScenarioTest

# Python standard library, string input/output
from StringIO import StringIO

# Haiku language, s-expression pickler
from haiku.pickle import SimpleExpressionPickler

SCENARIOS = [
  # Empty string (edge case):
  dict(lisp=u'',                      python={}),

  # Symbols:
  dict(lisp=u'abc',                   python={0:'abc'}),
  dict(lisp=u'a b c',                 python={0:'a',1:'b',2:'c'}),
  dict(lisp=u'#nil',                  python={0:''}, skip=['load']),
  dict(lisp=u'[byte-array IA==]',     python={0:' '}, skip=['load']),
  dict(lisp=u'[byte-array AA==]',     python={0:'\0'}, skip=['load']),
  dict(lisp=u'[byte-array AQIDBA==]', python={0:'\x01\x02\x03\x04'}, skip=['load']),

  # Constants:
  dict(lisp=u'#nil',                  python={0:None}),
  dict(lisp=u'#f',                    python={0:False}),
  dict(lisp=u'#t',                    python={0:True}),

  # Integers:
  dict(lisp=u'0',                     python={0:0L}),
  dict(lisp=u'1',                     python={0:1L}),
  dict(lisp=u'-1',                    python={0:-1L}),
  dict(lisp=u'36893488147419103232',  python={0:long(2**65)}),
  dict(lisp=u'-36893488147419103232', python={0:-long(2**65)}),

  # Unicode strings:
  dict(lisp=u'""',                    python={0:u''}),
  dict(lisp=u'"-"',                   python={0:u'-'}),
  dict(lisp=u'"\\""',                 python={0:u'"'}),
  dict(lisp=u'"tsch\xfcss!"',         python={0:u'tschüss!'}),
  dict(lisp=u'"\u3053\u3093\u306b\u3061\u306f\u4e16\u754c\uff01"', python={0:u'こんにちは世界！'}),

  # Lists:
  dict(lisp=u'[]',                    python={0:{}}),
  dict(lisp=u'[a]',                   python={0:{0:'a'}}),
  dict(lisp=u'[a b c]',               python={0:{0:'a',1:'b',2:'c'}}),
  dict(lisp=u'[1 -2 3]',              python={0:{0:1L,1:-2L,2:3L}}),
  dict(lisp=u'[a [b 1] [c 2]]',       python={0:{0:'a',1:{0:'b',1:1L},2:{0:'c',1:2L}}}),

  # Dictionaries:
  #dict(lisp=u'[:]',                               python={0:{}}),
  #dict(lisp=u'[: one 1]',                         python={0:{'one':1L}}),
  #dict(lisp=u'[: one 1 : two 2]',                 python={0:{'one':1L,'two':2L}}),
  #dict(lisp=u'[: one 1 : three (+ 1 2) : two 2]', python={0:{'one':1L,'two':2L,'three':['+',1L,2L]}}),

  # Integer arithmetic operators:
  dict(lisp=u'[+ 0 1]',               python={0:{0:'+',1:0L,2:1L}}),
  dict(lisp=u'[+ 2 3 4]',             python={0:{0:'+',1:2L,2:3L,3:4L}}),
  dict(lisp=u'[+ 5 6 7 8]',           python={0:{0:'+',1:5L,2:6L,3:7L,4:8L}}),
  dict(lisp=u'[- 0 1]',               python={0:{0:'-',1:0L,2:1L}}),
  dict(lisp=u'[- 2 3 4]',             python={0:{0:'-',1:2L,2:3L,3:4L}}),
  dict(lisp=u'[- 5 6 7 8]',           python={0:{0:'-',1:5L,2:6L,3:7L,4:8L}}),
  dict(lisp=u'[/ 0 1]',               python={0:{0:'/',1:0L,2:1L}}),
  dict(lisp=u'[/ 2 3 4]',             python={0:{0:'/',1:2L,2:3L,3:4L}}),
  dict(lisp=u'[/ 5 6 7 8]',           python={0:{0:'/',1:5L,2:6L,3:7L,4:8L}}),
  dict(lisp=u'[* 0 1]',               python={0:{0:'*',1:0L,2:1L}}),
  dict(lisp=u'[* 2 3 4]',             python={0:{0:'*',1:2L,2:3L,3:4L}}),
  dict(lisp=u'[* 5 6 7 8]',           python={0:{0:'*',1:5L,2:6L,3:7L,4:8L}}),
  dict(lisp=u'[** 0 8]',              python={0:{0:'**',1:0L,2:8L}}),
  dict(lisp=u'[** 1 4]',              python={0:{0:'**',1:1L,2:4L}}),
  dict(lisp=u'[** 2 3]',              python={0:{0:'**',1:2L,2:3L}}),
  dict(lisp=u'[** 3 3]',              python={0:{0:'**',1:3L,2:3L}}),
  dict(lisp=u'[** 7 2 5]',            python={0:{0:'**',1:7L,2:2L,3:5L}}),

  # Logical operators:
  dict(lisp=u'[& #f #f]',             python={0:{0:'&',1:False,2:False}}),
  dict(lisp=u'[& #f #t]',             python={0:{0:'&',1:False,2: True}}),
  dict(lisp=u'[& #t #f]',             python={0:{0:'&',1: True,2:False}}),
  dict(lisp=u'[& #t #t]',             python={0:{0:'&',1: True,2: True}}),
  dict(lisp=u'[| #f #f]',             python={0:{0:'|',1:False,2:False}}),
  dict(lisp=u'[| #f #t]',             python={0:{0:'|',1:False,2: True}}),
  dict(lisp=u'[| #t #f]',             python={0:{0:'|',1: True,2:False}}),
  dict(lisp=u'[| #t #t]',             python={0:{0:'|',1: True,2: True}}),
  dict(lisp=u'[^ #f #f]',             python={0:{0:'^',1:False,2:False}}),
  dict(lisp=u'[^ #f #t]',             python={0:{0:'^',1:False,2: True}}),
  dict(lisp=u'[^ #t #f]',             python={0:{0:'^',1: True,2:False}}),
  dict(lisp=u'[^ #t #t]',             python={0:{0:'^',1: True,2: True}}),
  dict(lisp=u'[~ #f]',                python={0:{0:'~',1:False}}),
  dict(lisp=u'[~ #t]',                python={0:{0:'~',1: True}}),

  # Relational operators:
  dict(lisp=u'[< 0 0]',               python={0:{0: '<',1:0L,2:0L}}),
  dict(lisp=u'[< 0 1]',               python={0:{0: '<',1:0L,2:1L}}),
  dict(lisp=u'[< 1 0]',               python={0:{0: '<',1:1L,2:0L}}),
  dict(lisp=u'[< 1 1]',               python={0:{0: '<',1:1L,2:1L}}),
  dict(lisp=u'[<= 0 0]',              python={0:{0:'<=',1:0L,2:0L}}),
  dict(lisp=u'[<= 0 1]',              python={0:{0:'<=',1:0L,2:1L}}),
  dict(lisp=u'[<= 1 0]',              python={0:{0:'<=',1:1L,2:0L}}),
  dict(lisp=u'[<= 1 1]',              python={0:{0:'<=',1:1L,2:1L}}),
  dict(lisp=u'[= 0 0]',               python={0:{0: '=',1:0L,2:0L}}),
  dict(lisp=u'[= 0 1]',               python={0:{0: '=',1:0L,2:1L}}),
  dict(lisp=u'[= 1 0]',               python={0:{0: '=',1:1L,2:0L}}),
  dict(lisp=u'[= 1 1]',               python={0:{0: '=',1:1L,2:1L}}),
  dict(lisp=u'[>= 0 0]',              python={0:{0:'>=',1:0L,2:0L}}),
  dict(lisp=u'[>= 0 1]',              python={0:{0:'>=',1:0L,2:1L}}),
  dict(lisp=u'[>= 1 0]',              python={0:{0:'>=',1:1L,2:0L}}),
  dict(lisp=u'[>= 1 1]',              python={0:{0:'>=',1:1L,2:1L}}),
  dict(lisp=u'[> 0 0]',               python={0:{0: '>',1:0L,2:0L}}),
  dict(lisp=u'[> 0 1]',               python={0:{0: '>',1:0L,2:1L}}),
  dict(lisp=u'[> 1 0]',               python={0:{0: '>',1:1L,2:0L}}),
  dict(lisp=u'[> 1 1]',               python={0:{0: '>',1:1L,2:1L}}),
  dict(lisp=u'[~= 0 0]',              python={0:{0:'~=',1:0L,2:0L}}),
  dict(lisp=u'[~= 0 1]',              python={0:{0:'~=',1:0L,2:1L}}),
  dict(lisp=u'[~= 1 0]',              python={0:{0:'~=',1:1L,2:0L}}),
  dict(lisp=u'[~= 1 1]',              python={0:{0:'~=',1:1L,2:1L}}),

  # String operators:
  dict(lisp=u'[+ "Hello, " "world!"]', python={0:{0:'+',1:u"Hello, ",2:u"world!"}}),
]

class TestSimpleExpressionPickler(unittest2.TestCase):
  """Test serialization and deserialization of Lisp code to/from Python
  objects using the `SimpleExpressionPickler` class."""
  __metaclass__ = ScenarioMeta
  _pickler = SimpleExpressionPickler
  def setUp(self, *args, **kwargs):
    self.pickler = self._pickler()
  class test_dump(ScenarioTest):
    scenarios = SCENARIOS
    def __test__(self, lisp, python, skip=[]):
      # Check if it is okay to run this scenario:
      if 'dump' in skip:
        self.skipTest(u"Scenario not compatible with pickle.dump(); skipping...")
      # Compare dump(StringIO) vs the prepared Lisp:
      ostream = StringIO()
      self.pickler.dump(ostream, python)
      self.assertEqual(
        u"".join([u"[", lisp, u"]"]),
        ostream.getvalue().decode('utf-8'))
  class test_dump_utf_16(ScenarioTest):
    scenarios = SCENARIOS
    def __test__(self, lisp, python, skip=[]):
      # Check if it is okay to run this scenario:
      if 'dump' in skip:
        self.skipTest(u"Scenario not compatible with pickle.dump(); skipping...")
      # Compare dump(StringIO) vs the prepared Lisp:
      ostream = StringIO()
      self.pickler.dump(ostream, python, encoding='utf-16')
      self.assertEqual(
        u"".join([u"[", lisp, u"]"]),
        ostream.getvalue().decode('utf-16'))
  class test_dumps(ScenarioTest):
    scenarios = SCENARIOS
    def __test__(self, lisp, python, skip=[]):
      # Check if it is okay to run this scenario:
      if 'dump' in skip:
        self.skipTest(u"Scenario not compatible with pickle.dump(); skipping...")
      # Compare dumps() vs the prepared Lisp:
      self.assertEqual(
        u"".join([u"[", lisp, u"]"]),
        self.pickler.dumps(python))
  class test_load(ScenarioTest):
    scenarios = SCENARIOS
    def __test__(self, lisp, python, skip=[]):
      # Check if it is okay to run this scenario:
      if 'load' in skip:
        self.skipTest(u"Scenario not compatible with pickle.load(); skipping...")
      # Compare loads(StringIO) vs the prepared Python:
      istream = StringIO(lisp.encode('utf-8')) # utf-8 is load()'s default
      actual = self.pickler.load(istream)
      self.assertEqual(actual, python)
  class test_load_utf_16(ScenarioTest):
    scenarios = SCENARIOS
    def __test__(self, lisp, python, skip=[]):
      # Check if it is okay to run this scenario:
      if 'load' in skip:
        self.skipTest(u"Scenario not compatible with pickle.load(); skipping...")
      # Compare loads(StringIO) vs the prepared Python:
      istream = StringIO(lisp.encode('utf-16'))
      actual = self.pickler.load(istream, encoding='utf-16')
      self.assertEqual(actual, python)
  class test_loads(ScenarioTest):
    scenarios = SCENARIOS
    def __test__(self, lisp, python, skip=[]):
      # Check if it is okay to run this scenario:
      if 'load' in skip:
        self.skipTest(u"Scenario not compatible with pickle.load(); skipping...")
      # Compare loads() vs the prepared Python:
      self.assertEqual(self.pickler.loads(lisp), python)

# ===----------------------------------------------------------------------===
# End of File
# ===----------------------------------------------------------------------===
<commit_msg>Removed mutable value from default parameter list (no side effect, but more pythonic and avoids future mistakes).<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-

# === haiku.pickle.simple__test -------------------------------------------===
# Copyright © 2011-2012, RokuSigma Inc. and contributors. See AUTHORS for more
# details.
#
# Some rights reserved.
#
# Redistribution and use in source and binary forms of the software as well as
# documentation, with or without modification, are permitted provided that the
# following conditions are met:
#
#  * Redistributions of source code must retain the above copyright notice,
#    this list of conditions and the following disclaimer.
#  * Redistributions in binary form must reproduce the above copyright notice,
#    this list of conditions and the following disclaimer in the documentation
#    and/or other materials provided with the distribution.
#  * The names of the copyright holders or contributors may not be used to
#    endorse or promote products derived from this software without specific
#    prior written permission.
#
# THIS SOFTWARE AND DOCUMENTATION IS PROVIDED BY THE COPYRIGHT HOLDERS AND
# CONTRIBUTORS “AS IS” AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT
# NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
# PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
# OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
# WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR
# OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE AND
# DOCUMENTATION, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
# ===----------------------------------------------------------------------===

# Python standard library, unit-testing
import unittest2

# Python patterns, scenario unit-testing
from python_patterns.unittest.scenario import ScenarioMeta, ScenarioTest

# Python standard library, string input/output
from StringIO import StringIO

# Haiku language, s-expression pickler
from haiku.pickle import SimpleExpressionPickler

SCENARIOS = [
  # Empty string (edge case):
  dict(lisp=u'',                      python={}),

  # Symbols:
  dict(lisp=u'abc',                   python={0:'abc'}),
  dict(lisp=u'a b c',                 python={0:'a',1:'b',2:'c'}),
  dict(lisp=u'#nil',                  python={0:''}, skip=['load']),
  dict(lisp=u'[byte-array IA==]',     python={0:' '}, skip=['load']),
  dict(lisp=u'[byte-array AA==]',     python={0:'\0'}, skip=['load']),
  dict(lisp=u'[byte-array AQIDBA==]', python={0:'\x01\x02\x03\x04'}, skip=['load']),

  # Constants:
  dict(lisp=u'#nil',                  python={0:None}),
  dict(lisp=u'#f',                    python={0:False}),
  dict(lisp=u'#t',                    python={0:True}),

  # Integers:
  dict(lisp=u'0',                     python={0:0L}),
  dict(lisp=u'1',                     python={0:1L}),
  dict(lisp=u'-1',                    python={0:-1L}),
  dict(lisp=u'36893488147419103232',  python={0:long(2**65)}),
  dict(lisp=u'-36893488147419103232', python={0:-long(2**65)}),

  # Unicode strings:
  dict(lisp=u'""',                    python={0:u''}),
  dict(lisp=u'"-"',                   python={0:u'-'}),
  dict(lisp=u'"\\""',                 python={0:u'"'}),
  dict(lisp=u'"tsch\xfcss!"',         python={0:u'tschüss!'}),
  dict(lisp=u'"\u3053\u3093\u306b\u3061\u306f\u4e16\u754c\uff01"', python={0:u'こんにちは世界！'}),

  # Lists:
  dict(lisp=u'[]',                    python={0:{}}),
  dict(lisp=u'[a]',                   python={0:{0:'a'}}),
  dict(lisp=u'[a b c]',               python={0:{0:'a',1:'b',2:'c'}}),
  dict(lisp=u'[1 -2 3]',              python={0:{0:1L,1:-2L,2:3L}}),
  dict(lisp=u'[a [b 1] [c 2]]',       python={0:{0:'a',1:{0:'b',1:1L},2:{0:'c',1:2L}}}),

  # Dictionaries:
  #dict(lisp=u'[:]',                               python={0:{}}),
  #dict(lisp=u'[: one 1]',                         python={0:{'one':1L}}),
  #dict(lisp=u'[: one 1 : two 2]',                 python={0:{'one':1L,'two':2L}}),
  #dict(lisp=u'[: one 1 : three (+ 1 2) : two 2]', python={0:{'one':1L,'two':2L,'three':['+',1L,2L]}}),

  # Integer arithmetic operators:
  dict(lisp=u'[+ 0 1]',               python={0:{0:'+',1:0L,2:1L}}),
  dict(lisp=u'[+ 2 3 4]',             python={0:{0:'+',1:2L,2:3L,3:4L}}),
  dict(lisp=u'[+ 5 6 7 8]',           python={0:{0:'+',1:5L,2:6L,3:7L,4:8L}}),
  dict(lisp=u'[- 0 1]',               python={0:{0:'-',1:0L,2:1L}}),
  dict(lisp=u'[- 2 3 4]',             python={0:{0:'-',1:2L,2:3L,3:4L}}),
  dict(lisp=u'[- 5 6 7 8]',           python={0:{0:'-',1:5L,2:6L,3:7L,4:8L}}),
  dict(lisp=u'[/ 0 1]',               python={0:{0:'/',1:0L,2:1L}}),
  dict(lisp=u'[/ 2 3 4]',             python={0:{0:'/',1:2L,2:3L,3:4L}}),
  dict(lisp=u'[/ 5 6 7 8]',           python={0:{0:'/',1:5L,2:6L,3:7L,4:8L}}),
  dict(lisp=u'[* 0 1]',               python={0:{0:'*',1:0L,2:1L}}),
  dict(lisp=u'[* 2 3 4]',             python={0:{0:'*',1:2L,2:3L,3:4L}}),
  dict(lisp=u'[* 5 6 7 8]',           python={0:{0:'*',1:5L,2:6L,3:7L,4:8L}}),
  dict(lisp=u'[** 0 8]',              python={0:{0:'**',1:0L,2:8L}}),
  dict(lisp=u'[** 1 4]',              python={0:{0:'**',1:1L,2:4L}}),
  dict(lisp=u'[** 2 3]',              python={0:{0:'**',1:2L,2:3L}}),
  dict(lisp=u'[** 3 3]',              python={0:{0:'**',1:3L,2:3L}}),
  dict(lisp=u'[** 7 2 5]',            python={0:{0:'**',1:7L,2:2L,3:5L}}),

  # Logical operators:
  dict(lisp=u'[& #f #f]',             python={0:{0:'&',1:False,2:False}}),
  dict(lisp=u'[& #f #t]',             python={0:{0:'&',1:False,2: True}}),
  dict(lisp=u'[& #t #f]',             python={0:{0:'&',1: True,2:False}}),
  dict(lisp=u'[& #t #t]',             python={0:{0:'&',1: True,2: True}}),
  dict(lisp=u'[| #f #f]',             python={0:{0:'|',1:False,2:False}}),
  dict(lisp=u'[| #f #t]',             python={0:{0:'|',1:False,2: True}}),
  dict(lisp=u'[| #t #f]',             python={0:{0:'|',1: True,2:False}}),
  dict(lisp=u'[| #t #t]',             python={0:{0:'|',1: True,2: True}}),
  dict(lisp=u'[^ #f #f]',             python={0:{0:'^',1:False,2:False}}),
  dict(lisp=u'[^ #f #t]',             python={0:{0:'^',1:False,2: True}}),
  dict(lisp=u'[^ #t #f]',             python={0:{0:'^',1: True,2:False}}),
  dict(lisp=u'[^ #t #t]',             python={0:{0:'^',1: True,2: True}}),
  dict(lisp=u'[~ #f]',                python={0:{0:'~',1:False}}),
  dict(lisp=u'[~ #t]',                python={0:{0:'~',1: True}}),

  # Relational operators:
  dict(lisp=u'[< 0 0]',               python={0:{0: '<',1:0L,2:0L}}),
  dict(lisp=u'[< 0 1]',               python={0:{0: '<',1:0L,2:1L}}),
  dict(lisp=u'[< 1 0]',               python={0:{0: '<',1:1L,2:0L}}),
  dict(lisp=u'[< 1 1]',               python={0:{0: '<',1:1L,2:1L}}),
  dict(lisp=u'[<= 0 0]',              python={0:{0:'<=',1:0L,2:0L}}),
  dict(lisp=u'[<= 0 1]',              python={0:{0:'<=',1:0L,2:1L}}),
  dict(lisp=u'[<= 1 0]',              python={0:{0:'<=',1:1L,2:0L}}),
  dict(lisp=u'[<= 1 1]',              python={0:{0:'<=',1:1L,2:1L}}),
  dict(lisp=u'[= 0 0]',               python={0:{0: '=',1:0L,2:0L}}),
  dict(lisp=u'[= 0 1]',               python={0:{0: '=',1:0L,2:1L}}),
  dict(lisp=u'[= 1 0]',               python={0:{0: '=',1:1L,2:0L}}),
  dict(lisp=u'[= 1 1]',               python={0:{0: '=',1:1L,2:1L}}),
  dict(lisp=u'[>= 0 0]',              python={0:{0:'>=',1:0L,2:0L}}),
  dict(lisp=u'[>= 0 1]',              python={0:{0:'>=',1:0L,2:1L}}),
  dict(lisp=u'[>= 1 0]',              python={0:{0:'>=',1:1L,2:0L}}),
  dict(lisp=u'[>= 1 1]',              python={0:{0:'>=',1:1L,2:1L}}),
  dict(lisp=u'[> 0 0]',               python={0:{0: '>',1:0L,2:0L}}),
  dict(lisp=u'[> 0 1]',               python={0:{0: '>',1:0L,2:1L}}),
  dict(lisp=u'[> 1 0]',               python={0:{0: '>',1:1L,2:0L}}),
  dict(lisp=u'[> 1 1]',               python={0:{0: '>',1:1L,2:1L}}),
  dict(lisp=u'[~= 0 0]',              python={0:{0:'~=',1:0L,2:0L}}),
  dict(lisp=u'[~= 0 1]',              python={0:{0:'~=',1:0L,2:1L}}),
  dict(lisp=u'[~= 1 0]',              python={0:{0:'~=',1:1L,2:0L}}),
  dict(lisp=u'[~= 1 1]',              python={0:{0:'~=',1:1L,2:1L}}),

  # String operators:
  dict(lisp=u'[+ "Hello, " "world!"]', python={0:{0:'+',1:u"Hello, ",2:u"world!"}}),
]

class TestSimpleExpressionPickler(unittest2.TestCase):
  """Test serialization and deserialization of Lisp code to/from Python
  objects using the `SimpleExpressionPickler` class."""
  __metaclass__ = ScenarioMeta
  _pickler = SimpleExpressionPickler
  def setUp(self, *args, **kwargs):
    self.pickler = self._pickler()
  class test_dump(ScenarioTest):
    scenarios = SCENARIOS
    def __test__(self, lisp, python, skip=None):
      skip = skip or []
      # Check if it is okay to run this scenario:
      if 'dump' in skip:
        self.skipTest(u"Scenario not compatible with pickle.dump(); skipping...")
      # Compare dump(StringIO) vs the prepared Lisp:
      ostream = StringIO()
      self.pickler.dump(ostream, python)
      self.assertEqual(
        u"".join([u"[", lisp, u"]"]),
        ostream.getvalue().decode('utf-8'))
  class test_dump_utf_16(ScenarioTest):
    scenarios = SCENARIOS
    def __test__(self, lisp, python, skip=None):
      skip = skip or []
      # Check if it is okay to run this scenario:
      if 'dump' in skip:
        self.skipTest(u"Scenario not compatible with pickle.dump(); skipping...")
      # Compare dump(StringIO) vs the prepared Lisp:
      ostream = StringIO()
      self.pickler.dump(ostream, python, encoding='utf-16')
      self.assertEqual(
        u"".join([u"[", lisp, u"]"]),
        ostream.getvalue().decode('utf-16'))
  class test_dumps(ScenarioTest):
    scenarios = SCENARIOS
    def __test__(self, lisp, python, skip=None):
      skip = skip or []
      # Check if it is okay to run this scenario:
      if 'dump' in skip:
        self.skipTest(u"Scenario not compatible with pickle.dump(); skipping...")
      # Compare dumps() vs the prepared Lisp:
      self.assertEqual(
        u"".join([u"[", lisp, u"]"]),
        self.pickler.dumps(python))
  class test_load(ScenarioTest):
    scenarios = SCENARIOS
    def __test__(self, lisp, python, skip=None):
      skip = skip or []
      # Check if it is okay to run this scenario:
      if 'load' in skip:
        self.skipTest(u"Scenario not compatible with pickle.load(); skipping...")
      # Compare loads(StringIO) vs the prepared Python:
      istream = StringIO(lisp.encode('utf-8')) # utf-8 is load()'s default
      actual = self.pickler.load(istream)
      self.assertEqual(actual, python)
  class test_load_utf_16(ScenarioTest):
    scenarios = SCENARIOS
    def __test__(self, lisp, python, skip=None):
      skip = skip or []
      # Check if it is okay to run this scenario:
      if 'load' in skip:
        self.skipTest(u"Scenario not compatible with pickle.load(); skipping...")
      # Compare loads(StringIO) vs the prepared Python:
      istream = StringIO(lisp.encode('utf-16'))
      actual = self.pickler.load(istream, encoding='utf-16')
      self.assertEqual(actual, python)
  class test_loads(ScenarioTest):
    scenarios = SCENARIOS
    def __test__(self, lisp, python, skip=None):
      skip = skip or []
      # Check if it is okay to run this scenario:
      if 'load' in skip:
        self.skipTest(u"Scenario not compatible with pickle.load(); skipping...")
      # Compare loads() vs the prepared Python:
      self.assertEqual(self.pickler.loads(lisp), python)

# ===----------------------------------------------------------------------===
# End of File
# ===----------------------------------------------------------------------===
<|endoftext|>