<commit_before>import os, sys, hashlib, json, fnmatch
from igf_data.igfdb.baseadaptor import BaseAdaptor
from igf_data.igfdb.igfTables import Seqrun
from igf_data.igfdb.seqrunadaptor import SeqrunAdaptor
from igf_data.igfdb.collectionadaptor import CollectionAdaptor
from igf_data.igfdb.fileadaptor import FileAdaptor
from igf_data.igfdb.pipelineadaptor import PipelineAdaptor
from igf_data.igfdb.platformadaptor import PlatformAdaptor
from igf_data.illumina.runinfo_xml import RunInfo_xml
from igf_data.illumina.runparameters_xml import RunParameter_xml
from igf_data.utils.fileutils import calculate_file_checksum

def find_new_seqrun_dir(path, dbconfig):
  '''
  A method for check and finding new sequencing run directory
  '''
  all_seqrun_dir=[f for f in os.listdir(path) if os.path.isdir(os.path.join(path,f))]    # list of all directories present under path
  new_seqrun_dir=check_seqrun_dir_in_db(all_seqrun_dir,dbconfig)
  valid_seqrun_dir=check_finished_seqrun_dir(seqrun_dir=new_seqrun_dir, seqrun_path=path)
  return valid_seqrun_dir

def check_finished_seqrun_dir(seqrun_dir, seqrun_path, required_files=['RTAComplete.txt','SampleSheet.csv','RunInfo.xml']):
  '''
  A method for checking complete sequencing run directory
  '''
  valid_seqruns=dict()
  for seqrun in seqrun_dir:
    skip=0
    for serun_file in required_files:
      required_path=os.path.join(seqrun_path,seqrun,serun_file)
      if not os.path.exists(required_path):
        skip=1
      elif int(os.path.getsize(required_path))==0:
        skip=1
    if skip==0:
      valid_seqruns[seqrun]=os.path.join(seqrun_path,seqrun)
  return valid_seqruns

def check_seqrun_dir_in_db(all_seqrun_dir,dbconfig):
  '''
  A method for checking existing seqrun dirs in database
  required params:
  all_seqrun_dir: list of seqrun dirs to check
  dbconfig: dbconfig
  '''
  dbparam=None
  with open(dbconfig, 'r') as json_data:
    dbparam=json.load(json_data)

  sra=SeqrunAdaptor(**dbparam)
  sra.start_session()
  sra_data=sra.fetch_records(sra.session.query(Seqrun.seqrun_igf_id),output_mode='object')
  existing_runs=set(s[0] for s in sra_data)
  sra.close_session()
  all_runs=set(all_seqrun_dir)
  new_runs=list(all_runs.difference(existing_runs))
  return new_runs

def calculate_file_md5(seqrun_info, md5_out, seqrun_path, file_suffix='md5.json', exclude_dir=[]):
  '''
  A method for file md5 calculation for all the sequencing run files
  Output is a dictionary of json files
  {seqrun_name: seqrun_md5_list_path}
  Format of the json file
  [{"seqrun_file_name":"file_path","file_md5":"md5_value"}]
  '''
  seqrun_and_md5=dict()
  for seqrun_name, seqrun_path in seqrun_info.items():
    file_list_with_md5=list()
    output_json_file=os.path.join(md5_out,'{0}.{1}'.format(seqrun_name, file_suffix))
    for root_path,dirs,files in os.walk(seqrun_path, topdown=True):
      dirs[:]=[ d for d in dirs if d not in exclude_dir ]                                     # exclude listed dires from search
      if len(files)>0:
        for file_name in files:
          if not fnmatch.fnmatch(file, '*.fastq.gz'):
            file_path=os.path.join(root_path,file_name)
            if os.path.exists(file_path):
              file_md5=calculate_file_checksum(filepath=file_path)                              # calculate file checksum
              file_rel_path=os.path.relpath(file_path, start=seqrun_path)                       # get relative filepath
              file_list_with_md5.append({"seqrun_file_name":file_rel_path,"file_md5":file_md5})

    with open(output_json_file, 'w') as output_json:
      json.dump(file_list_with_md5, output_json, indent=4)                                    # write json md5 list

    seqrun_and_md5[seqrun_name]=output_json_file
  return seqrun_and_md5

def prepare_seqrun_for_db(seqrun_name, seqrun_path, session_class):
  '''
  A method for preparing seqrun data for database
  '''
  try:
    runinfo_file=os.path.join(seqrun_path,'RunInfo.xml')
    runinfo_data=RunInfo_xml(xml_file=runinfo_file)
    platform_name=runinfo_data.get_platform_number()
    reads_stats=runinfo_data.get_reads_stats()
    flowcell_id=runinfo_data.get_flowcell_name()

    seqrun_data=dict()
    seqrun_data['seqrun_igf_id']=seqrun_name
    seqrun_data['platform_igf_id']=platform_name
    seqrun_data['flowcell_id']=flowcell_id

    for read_id in reads_stats.keys():
      if reads_stats[read_id]['isindexedread'] == 'Y':
        # its index
        seqrun_data['index{0}'.format(read_id)]=reads_stats[read_id]['numcycles']
      elif  reads_stats[read_id]['isindexedread'] == 'N':
        # its read
        seqrun_data['read{0}'.format(read_id)]=reads_stats[read_id]['numcycles']
      else:
        raise ValueError('unknown value for isindexedread: {0}'.format(reads_stats[read_id]['isindexedread']))

    pl=PlatformAdaptor(**{'session_class':session_class})
    pl.start_session()
    pl_data=pl.fetch_platform_records_igf_id(platform_igf_id=platform_name)
    pl.close_session()

    if (pl_data.model_name=='HISEQ4000'):
      runparameters_file=os.path.join(seqrun_path,'runParameters.xml')
      runparameters_data=RunParameter_xml(xml_file=runparameters_file)
      flowcell_type=runparameters_data.get_hiseq_flowcell()
      # add flowcell information for hiseq runs
      seqrun_data['flowcell']=flowcell_type

      if flowcell_type is None:
        raise ValueError('unknown flowcell type for sequencing run model {0}'.format(pl_data.model_name))
    else:
      seqrun_data['flowcell']=pl_data.model_name  # adding flowcell info for remaining platforms
    return seqrun_data
  except:
    raise

def seed_pipeline_table_for_new_seqrun(pipeline_name, dbconfig):
  '''
  A method for seeding pipelines for the new seqruns
  required params:
  pipeline_name: A pipeline name
  dbconfig: A dbconfig file
  '''
  dbparam=None
  with open(dbconfig, 'r') as json_data:
    dbparam=json.load(json_data)

  try:
    pa=PipelineAdaptor(**dbparam)
    pa.start_session()
    pa.seed_new_seqruns(pipeline_name=pipeline_name)
  except:
    raise
  finally:
    pa.close_session()

def load_seqrun_files_to_db(seqrun_info, seqrun_md5_info, dbconfig, file_type='ILLUMINA_BCL_MD5'):
  '''
  A method for loading md5 lists to collection and files table
  '''
  dbparam=None
  with open(dbconfig, 'r') as json_data:
    dbparam=json.load(json_data)

  seqrun_data=list()
  seqrun_md5_collection_data=list()
  seqrun_md5_file_data=list()
  seqrun_file_collection=list()

  base=BaseAdaptor(**dbparam)
  session_class=base.get_session_class()

  for seqrun_name, seqrun_path in seqrun_info.items():
    seqrun_data.append(prepare_seqrun_for_db(seqrun_name, seqrun_path, session_class))
    seqrun_md5_collection_data.append({'name':seqrun_name, 'type':file_type,'table':'seqrun' })
    seqrun_md5_file=seqrun_md5_info[seqrun_name]
    file_md5=calculate_file_checksum(seqrun_md5_file)
    file_size=os.path.getsize(seqrun_md5_file)
    seqrun_md5_file_data.append({'file_path':seqrun_md5_file,'location':'ORWELL','md5':file_md5, 'size':file_size})
    seqrun_file_collection.append({'name':seqrun_name, 'type':file_type, 'file_path':seqrun_md5_file})

  try:
    base.start_session()
    # store seqrun info
    sra=SeqrunAdaptor(**{'session':base.session})
    sra.store_seqrun_and_attribute_data(data=seqrun_data, autosave=False)

    # store collection
    ca=CollectionAdaptor(**{'session':base.session})
    ca.store_collection_and_attribute_data(data=seqrun_md5_collection_data, autosave=False)
    ca.session.flush()

    # store files
    fa=FileAdaptor(**{'session':base.session})
    fa.store_file_and_attribute_data(data=seqrun_md5_file_data, autosave=False)
    fa.session.flush()

    # store file collection
    ca.create_collection_group(data=seqrun_file_collection, autosave=False)

    base.commit_session()
  except:
    base.rollback_session()
    raise
  finally:
    base.close_session()

<commit_msg>fixed seqrun processing class<commit_after>import os, sys, hashlib, json, fnmatch
from igf_data.igfdb.baseadaptor import BaseAdaptor
from igf_data.igfdb.igfTables import Seqrun
from igf_data.igfdb.seqrunadaptor import SeqrunAdaptor
from igf_data.igfdb.collectionadaptor import CollectionAdaptor
from igf_data.igfdb.fileadaptor import FileAdaptor
from igf_data.igfdb.pipelineadaptor import PipelineAdaptor
from igf_data.igfdb.platformadaptor import PlatformAdaptor
from igf_data.illumina.runinfo_xml import RunInfo_xml
from igf_data.illumina.runparameters_xml import RunParameter_xml
from igf_data.utils.fileutils import calculate_file_checksum

def find_new_seqrun_dir(path, dbconfig):
  '''
  A method for check and finding new sequencing run directory
  '''
  all_seqrun_dir=[f for f in os.listdir(path) if os.path.isdir(os.path.join(path,f))]    # list of all directories present under path
  new_seqrun_dir=check_seqrun_dir_in_db(all_seqrun_dir,dbconfig)
  valid_seqrun_dir=check_finished_seqrun_dir(seqrun_dir=new_seqrun_dir, seqrun_path=path)
  return valid_seqrun_dir

def check_finished_seqrun_dir(seqrun_dir, seqrun_path, required_files=['RTAComplete.txt','SampleSheet.csv','RunInfo.xml']):
  '''
  A method for checking complete sequencing run directory
  '''
  valid_seqruns=dict()
  for seqrun in seqrun_dir:
    skip=0
    for serun_file in required_files:
      required_path=os.path.join(seqrun_path,seqrun,serun_file)
      if not os.path.exists(required_path):
        skip=1
      elif int(os.path.getsize(required_path))==0:
        skip=1
    if skip==0:
      valid_seqruns[seqrun]=os.path.join(seqrun_path,seqrun)
  return valid_seqruns

def check_seqrun_dir_in_db(all_seqrun_dir,dbconfig):
  '''
  A method for checking existing seqrun dirs in database
  required params:
  all_seqrun_dir: list of seqrun dirs to check
  dbconfig: dbconfig
  '''
  dbparam=None
  with open(dbconfig, 'r') as json_data:
    dbparam=json.load(json_data)

  sra=SeqrunAdaptor(**dbparam)
  sra.start_session()
  sra_data=sra.fetch_records(sra.session.query(Seqrun.seqrun_igf_id),output_mode='object')
  existing_runs=set(s[0] for s in sra_data)
  sra.close_session()
  all_runs=set(all_seqrun_dir)
  new_runs=list(all_runs.difference(existing_runs))
  return new_runs

def calculate_file_md5(seqrun_info, md5_out, seqrun_path, file_suffix='md5.json', exclude_dir=[]):
  '''
  A method for file md5 calculation for all the sequencing run files
  Output is a dictionary of json files
  {seqrun_name: seqrun_md5_list_path}
  Format of the json file
  [{"seqrun_file_name":"file_path","file_md5":"md5_value"}]
  '''
  seqrun_and_md5=dict()
  for seqrun_name, seqrun_path in seqrun_info.items():
    file_list_with_md5=list()
    output_json_file=os.path.join(md5_out,'{0}.{1}'.format(seqrun_name, file_suffix))
    for root_path,dirs,files in os.walk(seqrun_path, topdown=True):
      dirs[:]=[ d for d in dirs if d not in exclude_dir ]                                     # exclude listed dires from search
      if len(files)>0:
        for file_name in files:
          if not fnmatch.fnmatch(file_name, '*.fastq.gz'):
            file_path=os.path.join(root_path,file_name)
            if os.path.exists(file_path):
              file_md5=calculate_file_checksum(filepath=file_path)                              # calculate file checksum
              file_rel_path=os.path.relpath(file_path, start=seqrun_path)                       # get relative filepath
              file_list_with_md5.append({"seqrun_file_name":file_rel_path,"file_md5":file_md5})

    with open(output_json_file, 'w') as output_json:
      json.dump(file_list_with_md5, output_json, indent=4)                                    # write json md5 list

    seqrun_and_md5[seqrun_name]=output_json_file
  return seqrun_and_md5

def prepare_seqrun_for_db(seqrun_name, seqrun_path, session_class):
  '''
  A method for preparing seqrun data for database
  '''
  try:
    runinfo_file=os.path.join(seqrun_path,'RunInfo.xml')
    runinfo_data=RunInfo_xml(xml_file=runinfo_file)
    platform_name=runinfo_data.get_platform_number()
    reads_stats=runinfo_data.get_reads_stats()
    flowcell_id=runinfo_data.get_flowcell_name()

    seqrun_data=dict()
    seqrun_data['seqrun_igf_id']=seqrun_name
    seqrun_data['platform_igf_id']=platform_name
    seqrun_data['flowcell_id']=flowcell_id

    for read_id in reads_stats.keys():
      if reads_stats[read_id]['isindexedread'] == 'Y':
        # its index
        seqrun_data['index{0}'.format(read_id)]=reads_stats[read_id]['numcycles']
      elif  reads_stats[read_id]['isindexedread'] == 'N':
        # its read
        seqrun_data['read{0}'.format(read_id)]=reads_stats[read_id]['numcycles']
      else:
        raise ValueError('unknown value for isindexedread: {0}'.format(reads_stats[read_id]['isindexedread']))

    pl=PlatformAdaptor(**{'session_class':session_class})
    pl.start_session()
    pl_data=pl.fetch_platform_records_igf_id(platform_igf_id=platform_name)
    pl.close_session()

    if (pl_data.model_name=='HISEQ4000'):
      runparameters_file=os.path.join(seqrun_path,'runParameters.xml')
      runparameters_data=RunParameter_xml(xml_file=runparameters_file)
      flowcell_type=runparameters_data.get_hiseq_flowcell()
      # add flowcell information for hiseq runs
      seqrun_data['flowcell']=flowcell_type

      if flowcell_type is None:
        raise ValueError('unknown flowcell type for sequencing run model {0}'.format(pl_data.model_name))
    else:
      seqrun_data['flowcell']=pl_data.model_name  # adding flowcell info for remaining platforms
    return seqrun_data
  except:
    raise

def seed_pipeline_table_for_new_seqrun(pipeline_name, dbconfig):
  '''
  A method for seeding pipelines for the new seqruns
  required params:
  pipeline_name: A pipeline name
  dbconfig: A dbconfig file
  '''
  dbparam=None
  with open(dbconfig, 'r') as json_data:
    dbparam=json.load(json_data)

  try:
    pa=PipelineAdaptor(**dbparam)
    pa.start_session()
    pa.seed_new_seqruns(pipeline_name=pipeline_name)
  except:
    raise
  finally:
    pa.close_session()

def load_seqrun_files_to_db(seqrun_info, seqrun_md5_info, dbconfig, file_type='ILLUMINA_BCL_MD5'):
  '''
  A method for loading md5 lists to collection and files table
  '''
  dbparam=None
  with open(dbconfig, 'r') as json_data:
    dbparam=json.load(json_data)

  seqrun_data=list()
  seqrun_md5_collection_data=list()
  seqrun_md5_file_data=list()
  seqrun_file_collection=list()

  base=BaseAdaptor(**dbparam)
  session_class=base.get_session_class()

  for seqrun_name, seqrun_path in seqrun_info.items():
    seqrun_data.append(prepare_seqrun_for_db(seqrun_name, seqrun_path, session_class))
    seqrun_md5_collection_data.append({'name':seqrun_name, 'type':file_type,'table':'seqrun' })
    seqrun_md5_file=seqrun_md5_info[seqrun_name]
    file_md5=calculate_file_checksum(seqrun_md5_file)
    file_size=os.path.getsize(seqrun_md5_file)
    seqrun_md5_file_data.append({'file_path':seqrun_md5_file,'location':'ORWELL','md5':file_md5, 'size':file_size})
    seqrun_file_collection.append({'name':seqrun_name, 'type':file_type, 'file_path':seqrun_md5_file})

  try:
    base.start_session()
    # store seqrun info
    sra=SeqrunAdaptor(**{'session':base.session})
    sra.store_seqrun_and_attribute_data(data=seqrun_data, autosave=False)

    # store collection
    ca=CollectionAdaptor(**{'session':base.session})
    ca.store_collection_and_attribute_data(data=seqrun_md5_collection_data, autosave=False)
    ca.session.flush()

    # store files
    fa=FileAdaptor(**{'session':base.session})
    fa.store_file_and_attribute_data(data=seqrun_md5_file_data, autosave=False)
    fa.session.flush()

    # store file collection
    ca.create_collection_group(data=seqrun_file_collection, autosave=False)

    base.commit_session()
  except:
    base.rollback_session()
    raise
  finally:
    base.close_session()

<|endoftext|>


<commit_before>#!/usr/bin/env python3.8
# ^ Use python 3.8 since Pip isn't configured for newer versions (3.9+)

# Copyright 2020 Google LLC.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import json
import os
import sys

from python import invoker

if len(sys.argv) != 2:
    raise ValueError('Please specify exactly one [root] directory.')

root_dir = sys.argv[1]
output_path = os.path.join(root_dir, 'polyglot_snippet_data.json')

json_array = invoker.get_json_for_dir(root_dir)
with open(output_path, 'w') as file:
    json.dump(json_array, file)

print(f'JSON written to: {output_path}')
print('Do not move this file!')
<commit_msg>Revert "Backdate python version to fix tests"<commit_after>#!/usr/bin/env python3

# Copyright 2020 Google LLC.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import json
import os
import sys

from python import invoker

if len(sys.argv) != 2:
    raise ValueError('Please specify exactly one [root] directory.')

root_dir = sys.argv[1]
output_path = os.path.join(root_dir, 'polyglot_snippet_data.json')

json_array = invoker.get_json_for_dir(root_dir)
with open(output_path, 'w') as file:
    json.dump(json_array, file)

print(f'JSON written to: {output_path}')
print('Do not move this file!')
<|endoftext|>


<commit_before>import json
import os
import re
from typing import Any, Dict, List, Mapping

import markdown
from django.utils.html import escape as escape_html
from markdown.extensions import Extension
from markdown.preprocessors import Preprocessor

from zerver.openapi.openapi import get_openapi_parameters

REGEXP = re.compile(r'\{generate_api_arguments_table\|\s*(.+?)\s*\|\s*(.+)\s*\}')

class MarkdownArgumentsTableGenerator(Extension):
    def __init__(self, configs: Mapping[str, Any] = {}) -> None:
        self.config = {
            'base_path': ['.', 'Default location from which to evaluate relative paths for the JSON files.'],
        }
        for key, value in configs.items():
            self.setConfig(key, value)

    def extendMarkdown(self, md: markdown.Markdown, md_globals: Dict[str, Any]) -> None:
        md.preprocessors.add(
            'generate_api_arguments', APIArgumentsTablePreprocessor(md, self.getConfigs()), '_begin',
        )

class APIArgumentsTablePreprocessor(Preprocessor):
    def __init__(self, md: markdown.Markdown, config: Dict[str, Any]) -> None:
        super().__init__(md)
        self.base_path = config['base_path']

    def run(self, lines: List[str]) -> List[str]:
        done = False
        while not done:
            for line in lines:
                loc = lines.index(line)
                match = REGEXP.search(line)

                if not match:
                    continue

                filename = match.group(1)
                doc_name = match.group(2)
                filename = os.path.expanduser(filename)

                is_openapi_format = filename.endswith('.yaml')

                if not os.path.isabs(filename):
                    parent_dir = self.base_path
                    filename = os.path.normpath(os.path.join(parent_dir, filename))

                if is_openapi_format:
                    endpoint, method = doc_name.rsplit(':', 1)
                    arguments: List[Dict[str, Any]] = []

                    try:
                        arguments = get_openapi_parameters(endpoint, method)
                    except KeyError as e:
                        # Don't raise an exception if the "parameters"
                        # field is missing; we assume that's because the
                        # endpoint doesn't accept any parameters
                        if e.args != ('parameters',):
                            raise e
                else:
                    with open(filename) as fp:
                        json_obj = json.load(fp)
                        arguments = json_obj[doc_name]

                if arguments:
                    text = self.render_table(arguments)
                else:
                    text = ['This endpoint does not consume any arguments.']
                # The line that contains the directive to include the macro
                # may be preceded or followed by text or tags, in that case
                # we need to make sure that any preceding or following text
                # stays the same.
                line_split = REGEXP.split(line, maxsplit=0)
                preceding = line_split[0]
                following = line_split[-1]
                text = [preceding] + text + [following]
                lines = lines[:loc] + text + lines[loc+1:]
                break
            else:
                done = True
        return lines

    def render_table(self, arguments: List[Dict[str, Any]]) -> List[str]:
        # TODO: Fix naming now that this no longer renders a table.
        table = []
        argument_template = """

{argument} {required}

        Example: {example}

{description}

"""

        md_engine = markdown.Markdown(extensions=[])

        for argument in arguments:
            description = argument['description']

            oneof = ['`' + str(item) + '`'
                     for item in argument.get('schema', {}).get('enum', [])]
            if oneof:
                description += '\nMust be one of: {}.'.format(', '.join(oneof))

            default = argument.get('schema', {}).get('default')
            if default is not None:
                description += f'\nDefaults to `{json.dumps(default)}`.'

            # TODO: OpenAPI allows indicating where the argument goes
            # (path, querystring, form data...).  We should document this detail.
            example = ""
            if 'example' in argument:
                example = argument['example']
            else:
                example = json.dumps(argument['content']['application/json']['example'])

            table.append(argument_template.format(
                argument=argument.get('argument') or argument.get('name'),
                example=escape_html(example),
                required='required' if argument.get('required')
                else 'optional',
                description=md_engine.convert(description),
            ))

        return table

def makeExtension(*args: Any, **kwargs: str) -> MarkdownArgumentsTableGenerator:
    return MarkdownArgumentsTableGenerator(kwargs)
<commit_msg>api docs: Use better language for no parameters allowed.<commit_after>import json
import os
import re
from typing import Any, Dict, List, Mapping

import markdown
from django.utils.html import escape as escape_html
from markdown.extensions import Extension
from markdown.preprocessors import Preprocessor

from zerver.openapi.openapi import get_openapi_parameters

REGEXP = re.compile(r'\{generate_api_arguments_table\|\s*(.+?)\s*\|\s*(.+)\s*\}')

class MarkdownArgumentsTableGenerator(Extension):
    def __init__(self, configs: Mapping[str, Any] = {}) -> None:
        self.config = {
            'base_path': ['.', 'Default location from which to evaluate relative paths for the JSON files.'],
        }
        for key, value in configs.items():
            self.setConfig(key, value)

    def extendMarkdown(self, md: markdown.Markdown, md_globals: Dict[str, Any]) -> None:
        md.preprocessors.add(
            'generate_api_arguments', APIArgumentsTablePreprocessor(md, self.getConfigs()), '_begin',
        )

class APIArgumentsTablePreprocessor(Preprocessor):
    def __init__(self, md: markdown.Markdown, config: Dict[str, Any]) -> None:
        super().__init__(md)
        self.base_path = config['base_path']

    def run(self, lines: List[str]) -> List[str]:
        done = False
        while not done:
            for line in lines:
                loc = lines.index(line)
                match = REGEXP.search(line)

                if not match:
                    continue

                filename = match.group(1)
                doc_name = match.group(2)
                filename = os.path.expanduser(filename)

                is_openapi_format = filename.endswith('.yaml')

                if not os.path.isabs(filename):
                    parent_dir = self.base_path
                    filename = os.path.normpath(os.path.join(parent_dir, filename))

                if is_openapi_format:
                    endpoint, method = doc_name.rsplit(':', 1)
                    arguments: List[Dict[str, Any]] = []

                    try:
                        arguments = get_openapi_parameters(endpoint, method)
                    except KeyError as e:
                        # Don't raise an exception if the "parameters"
                        # field is missing; we assume that's because the
                        # endpoint doesn't accept any parameters
                        if e.args != ('parameters',):
                            raise e
                else:
                    with open(filename) as fp:
                        json_obj = json.load(fp)
                        arguments = json_obj[doc_name]

                if arguments:
                    text = self.render_table(arguments)
                else:
                    text = ['This endpoint does not accept any parameters.']
                # The line that contains the directive to include the macro
                # may be preceded or followed by text or tags, in that case
                # we need to make sure that any preceding or following text
                # stays the same.
                line_split = REGEXP.split(line, maxsplit=0)
                preceding = line_split[0]
                following = line_split[-1]
                text = [preceding] + text + [following]
                lines = lines[:loc] + text + lines[loc+1:]
                break
            else:
                done = True
        return lines

    def render_table(self, arguments: List[Dict[str, Any]]) -> List[str]:
        # TODO: Fix naming now that this no longer renders a table.
        table = []
        argument_template = """

{argument} {required}

        Example: {example}

{description}

"""

        md_engine = markdown.Markdown(extensions=[])

        for argument in arguments:
            description = argument['description']

            oneof = ['`' + str(item) + '`'
                     for item in argument.get('schema', {}).get('enum', [])]
            if oneof:
                description += '\nMust be one of: {}.'.format(', '.join(oneof))

            default = argument.get('schema', {}).get('default')
            if default is not None:
                description += f'\nDefaults to `{json.dumps(default)}`.'

            # TODO: OpenAPI allows indicating where the argument goes
            # (path, querystring, form data...).  We should document this detail.
            example = ""
            if 'example' in argument:
                example = argument['example']
            else:
                example = json.dumps(argument['content']['application/json']['example'])

            table.append(argument_template.format(
                argument=argument.get('argument') or argument.get('name'),
                example=escape_html(example),
                required='required' if argument.get('required')
                else 'optional',
                description=md_engine.convert(description),
            ))

        return table

def makeExtension(*args: Any, **kwargs: str) -> MarkdownArgumentsTableGenerator:
    return MarkdownArgumentsTableGenerator(kwargs)
<|endoftext|>


<commit_before>#!/usr/bin/python
import sys
from PySide.QtGui import *
from PySide.QtCore import *
import os
import voting
from time import *
try:
    from SimpleCV import Camera
except:
    print "no simple cv"
from chat_client.pearclient import PearClient as PC

_main_path = "test_images/"
LOCALHOST, PORT, BUFFER_S = '127.0.0.1', 8008, 1024
IO_TIMEOUT_S = 0.1
MESSAGE_LIMIT = 40

class CryptoPear(QWidget):

    def __init__(self):
        super(CryptoPear, self).__init__()

        self.initUI()

    def list_files(self):
        path = os.getcwd() + '/test_images/'
        files = os.listdir(path)
        return map(lambda f: path + f, files)

    def initUI(self):
        self.paths = self.list_files()

        pixmap = QPixmap(self.paths.pop(0)).scaledToHeight(200)
        self.lbl = QLabel(self)
        self.lbl.setPixmap(pixmap)

        self.btn_connect = QPushButton("Connect")
        self.btn_connect.clicked.connect(self.connect_to_server)
        self.btn_accept = QPushButton("Accept")
        self.btn_reject = QPushButton("Reject")
        self.btn_accept.clicked.connect(self.accept_participent)
        self.btn_reject.clicked.connect(self.reject_participent)
        self.btn_accept.setDisabled(True)
        self.btn_reject.setDisabled(True)

        hbox = QHBoxLayout()
        hbox.addStretch(1)
        hbox.addWidget(self.btn_connect)
        hbox.addWidget(self.btn_accept)
        hbox.addWidget(self.btn_reject)

        self.chatbox = QTextEdit(self)
        self.chatbox.setMinimumHeight(400)
        self.chatbox.setReadOnly(True)

        self.text_entry = QTextEdit(self)
        self.text_entry.installEventFilter(self)
        btn_submit = QPushButton("Submit")
        btn_submit.clicked.connect(self.submit_message)
        text_hbox = QHBoxLayout()
        text_hbox.addWidget(self.text_entry)
        text_hbox.addWidget(btn_submit)

        vbox = QVBoxLayout()
        vbox.addStretch(1)
        vbox.addWidget(self.lbl)
        vbox.addLayout(hbox)
        vbox.addWidget(self.chatbox)
        vbox.addLayout(text_hbox)
        self.setLayout(vbox)

        #starting x, starting y, width, height
        self.setGeometry(0, 0, 500, 700)
        self.setWindowTitle('CryptoPear')
        self.show()

    def eventFilter(self, obj, event):
        if obj == self.text_entry:
            if isinstance(event, QKeyEvent):
                if event.key() == Qt.Key_Return:
                    if event.type() == QEvent.KeyPress:
                        self.submit_message()
                        return True

        return False

    def get_camera(self):
        try:
            cam = Camera()
            img = cam.getImage()
            img.save("mZOMGGUYS.png")
        except:
            pass
        return voting.encode_image("test_images/test_james.jpeg")

    def get_user(self):
        text, ok = QInputDialog.getText(self, 'Input Dialog',
            'Enter your name:')
        if ok:
            return text
        else:
            return None

    def get_json_dict(self):
        data = { 'name':self.get_user(), 'photo':self.get_camera()}
        return data

    def connect_to_server(self):
        self.server_handler = PC("198.211.120.146").ident(self.get_json_dict())
        self.btn_accept.setDisabled(False)
        self.btn_reject.setDisabled(False)
        self.btn_connect.setDisabled(True)

        self.thread = MessageThread(self.chatbox, self.server_handler)
        self.thread.message.connect(self.append_messages, Qt.QueuedConnection)
        self.handletoggle()

    def append_messages(self, messages):
        if messages:
            self.chatbox.setTextColor('red')
            self.chatbox.setReadOnly(False)
            self.chatbox.clear()
            self.chatbox.append(messages)
            self.chatbox.setReadOnly(True)

    def handletoggle(self):
        if self.thread.isRunning():
            self.thread.exiting=True
        else:
            self.thread.exiting=False
            self.thread.start()

    def submit_message(self):
        message = self.text_entry.toPlainText().encode('ascii')
        if message:
            self.server_handler.encrypted_send(message+'\n')
            self.text_entry.clear()

    def set_picture(self, file_path):
        image = QImage(file_path)
        if image.isNull():
            QMessageBox.information(self, "Image Viewer",
                    "Cannot load file")
            return

        self.lbl.setPixmap(QPixmap.fromImage(image).scaledToHeight(200))
        self.lbl.adjustSize()

    def accept_participent(self):
        self.set_picture(self.paths.pop(0))

    def reject_participent(self):
        print "participant rejected"

    def pretty_json(self, msg_json):
        return msg_json

class MessageThread(QThread):

    message = Signal(str)

    def __init__(self, chatbox, server_handler):
        super(MessageThread, self).__init__()
        self.chatbox = chatbox
        self.server_handler = server_handler

    def run(self):
        while True:
            new_messages = None
            try:
                new_messages = "".join(self.server_handler.get_recent_messages())
            except:
                pass
            self.message.emit(new_messages)
            sleep(1)

def main():
    app = QApplication(sys.argv)
    cp = CryptoPear()
    sys.exit(app.exec_())

if __name__ == '__main__':
    main()
<commit_msg>got rid of sleep in receive thread<commit_after>#!/usr/bin/python
import sys
from PySide.QtGui import *
from PySide.QtCore import *
import os
import voting
from time import *
try:
    from SimpleCV import Camera
except:
    print "no simple cv"
from chat_client.pearclient import PearClient as PC

_main_path = "test_images/"
LOCALHOST, PORT, BUFFER_S = '127.0.0.1', 8008, 1024
IO_TIMEOUT_S = 0.1
MESSAGE_LIMIT = 40

class CryptoPear(QWidget):

    def __init__(self):
        super(CryptoPear, self).__init__()

        self.initUI()

    def list_files(self):
        path = os.getcwd() + '/test_images/'
        files = os.listdir(path)
        return map(lambda f: path + f, files)

    def initUI(self):
        self.paths = self.list_files()

        pixmap = QPixmap(self.paths.pop(0)).scaledToHeight(200)
        self.lbl = QLabel(self)
        self.lbl.setPixmap(pixmap)

        self.btn_connect = QPushButton("Connect")
        self.btn_connect.clicked.connect(self.connect_to_server)
        self.btn_accept = QPushButton("Accept")
        self.btn_reject = QPushButton("Reject")
        self.btn_accept.clicked.connect(self.accept_participent)
        self.btn_reject.clicked.connect(self.reject_participent)
        self.btn_accept.setDisabled(True)
        self.btn_reject.setDisabled(True)

        hbox = QHBoxLayout()
        hbox.addStretch(1)
        hbox.addWidget(self.btn_connect)
        hbox.addWidget(self.btn_accept)
        hbox.addWidget(self.btn_reject)

        self.chatbox = QTextEdit(self)
        self.chatbox.setMinimumHeight(400)
        self.chatbox.setReadOnly(True)

        self.text_entry = QTextEdit(self)
        self.text_entry.installEventFilter(self)
        btn_submit = QPushButton("Submit")
        btn_submit.clicked.connect(self.submit_message)
        text_hbox = QHBoxLayout()
        text_hbox.addWidget(self.text_entry)
        text_hbox.addWidget(btn_submit)

        vbox = QVBoxLayout()
        vbox.addStretch(1)
        vbox.addWidget(self.lbl)
        vbox.addLayout(hbox)
        vbox.addWidget(self.chatbox)
        vbox.addLayout(text_hbox)
        self.setLayout(vbox)

        #starting x, starting y, width, height
        self.setGeometry(0, 0, 500, 700)
        self.setWindowTitle('CryptoPear')
        self.show()

    def eventFilter(self, obj, event):
        if obj == self.text_entry:
            if isinstance(event, QKeyEvent):
                if event.key() == Qt.Key_Return:
                    if event.type() == QEvent.KeyPress:
                        self.submit_message()
                        return True

        return False

    def get_camera(self):
        try:
            cam = Camera()
            img = cam.getImage()
            img.save("mZOMGGUYS.png")
        except:
            pass
        return voting.encode_image("test_images/test_james.jpeg")

    def get_user(self):
        text, ok = QInputDialog.getText(self, 'Input Dialog',
            'Enter your name:')
        if ok:
            return text
        else:
            return None

    def get_json_dict(self):
        data = { 'name':self.get_user(), 'photo':self.get_camera()}
        return data

    def connect_to_server(self):
        self.server_handler = PC("198.211.120.146").ident(self.get_json_dict())
        self.btn_accept.setDisabled(False)
        self.btn_reject.setDisabled(False)
        self.btn_connect.setDisabled(True)

        self.thread = MessageThread(self.chatbox, self.server_handler)
        self.thread.message.connect(self.append_messages, Qt.QueuedConnection)
        self.handletoggle()

    def append_messages(self, messages):
        if messages:
            self.chatbox.setTextColor('red')
            self.chatbox.setReadOnly(False)
            self.chatbox.clear()
            self.chatbox.append(messages)
            self.chatbox.setReadOnly(True)

    def handletoggle(self):
        if self.thread.isRunning():
            self.thread.exiting=True
        else:
            self.thread.exiting=False
            self.thread.start()

    def submit_message(self):
        message = self.text_entry.toPlainText().encode('ascii')
        if message:
            self.server_handler.encrypted_send(message+'\n')
            self.text_entry.clear()

    def set_picture(self, file_path):
        image = QImage(file_path)
        if image.isNull():
            QMessageBox.information(self, "Image Viewer",
                    "Cannot load file")
            return

        self.lbl.setPixmap(QPixmap.fromImage(image).scaledToHeight(200))
        self.lbl.adjustSize()

    def accept_participent(self):
        self.set_picture(self.paths.pop(0))

    def reject_participent(self):
        print "participant rejected"

    def pretty_json(self, msg_json):
        return msg_json

class MessageThread(QThread):

    message = Signal(str)

    def __init__(self, chatbox, server_handler):
        super(MessageThread, self).__init__()
        self.chatbox = chatbox
        self.server_handler = server_handler

    def run(self):
        while True:
            new_messages = None
            try:
                new_messages = "".join(self.server_handler.get_recent_messages())
            except:
                pass
            self.message.emit(new_messages)
            #sleep(1)

def main():
    app = QApplication(sys.argv)
    cp = CryptoPear()
    sys.exit(app.exec_())

if __name__ == '__main__':
    main()
<|endoftext|>


<commit_before>#!/usr/bin/env python2
# -*- coding: utf-8 -*-
#
#  This program distributed under Apache 2.0 license.
#  See LICENSE.txt for more details.
#  © UselessFire

# v. 0.3.1 Beta Public

mvDir = "tagged"

import sys, os, gtk, shutil, gc

gc.enable()

from mutagen.easyid3 import EasyID3
from mutagen.mp3 import MP3

default_label_text = \
u'''Название файла: "%s" (%i/%i)
Длина: %i секунд (%i минут %i секунд)
Битрейт: %i КБ\с
Файл будет переименован в '''

Help = u'''UTag - изменение тегов mp3 файлов

Использование:
--help | -h | help - эта справка

UTag <имена файлов> - изменить теги этих файлов
'''

class utag_window:
	def __init__(self, files):
		self.files = files
		self.All = len(self.files)
		self.number = int()

		self.window = gtk.Window()

		self.window.set_title('UTag')
#		self.window.set_border_width(0)

		self.window.connect("delete_event", self.dialog)
		self.window.connect("destroy", gtk.main_quit)

		self.Vbox = gtk.VBox(False, 0)

		self.labelframe = gtk.Frame(u'Информация')
		self.labelframe.show()
		self.label = gtk.Label()
		self.label.show()
		self.labelframe.add(self.label)
		self.Vbox.add(self.labelframe)

		# entrances

		# artist
		response, self.artist = self.tag_entry(u'Исполнитель')
		self.Vbox.add(response)
		self.artist.connect("activate", self.update_newName)

		# album
		response, self.album = self.tag_entry(u'Альбом')
		self.Vbox.add(response)

		# title
		response, self.title = self.tag_entry(u'Название')
		self.Vbox.add(response)
		self.title.connect("activate", self.update_newName)

		# date
		response, self.date = self.tag_entry(u'Год')
		self.Vbox.add(response)

		# genre
		response, self.genre = self.tag_entry(u'Жанр')
		self.Vbox.add(response)

		# tracknumber
		response, self.tracknumber = self.tag_entry(u'Номер трека')
		self.Vbox.add(response)

		self.entrances = {
			'artist': self.artist,
			'album': self.album,
			'title': self.title,
			'date': self.date,
			'genre': self.genre,
			'tracknumber': self.tracknumber
		}

		self.Hbox = gtk.HBox(False, 0)

		self.delete_and_next_button = gtk.Button(u'Удалить и следующий')
		self.delete_and_next_button.connect('clicked', self.delete_and_next_file)
		self.Hbox.add(self.delete_and_next_button)
		self.delete_and_next_button.show()

		self.next_button = gtk.Button(u'Пропустить')
		self.next_button.connect('clicked', self.next_file)
		self.Hbox.add(self.next_button)
		self.next_button.show()

		self.save_and_next_button = gtk.Button(u'Сохранить и следующий')
		self.save_and_next_button.connect('clicked', self.save_and_next_file)
		self.save_and_next_button.show()
		self.Hbox.add(self.save_and_next_button)

		self.Vbox.show()
		self.window.add(self.Vbox)

		self.Vbox.add(self.Hbox)
		self.Hbox.show()

		self.window.show()

		self.next_file()

	def save_and_next_file(self, widget=None):
		for name, entry in self.entrances.items():
			text = entry.get_text()
			if text:
				self.mp3[name] = text
			elif name in self.mp3:
				del self.mp3[name]

		self.mp3.save()
		print u'Файл "%s" был сохранён' % self.mp3.filename

		newName = (u'%s - %s.mp3' % \
			(
				(self.artist.get_text()	if	self.artist.get_text()	else    u'Неизвестно'),
				(self.title.get_text()	if	self.title.get_text()	else    u'Неизвестно')
			)
		)

		if os.path.exists("%s/%s" % (mvDir, newName)):
			if not self.dialog(title=u'Заменить?', message=u'Файл "%s" уже существует в папке "%s"' % (newName, mvDir)):
				os.remove("%s/%s" % (mvDir, newName))
				shutil.move(self.mp3.filename, "%s/%s" % (mvDir, newName))
				print u'Файл "%s" был перемещён в папку "%s" с названием "%s" с перезаписью' % (self.mp3.filename, mvDir, newName)
				self.next_file()
		else:
			shutil.move(self.mp3.filename, "%s/%s" % (mvDir, newName))
			print u'Файл "%s" был перемещён в папку "%s" с названием "%s"' % (self.mp3.filename, mvDir, newName)
			self.next_file()

	def delete_and_next_file(self, widget=None):
		os.remove(self.mp3.filename)
		print u'Файл "%s" был удалён' % self.mp3.filename
		self.next_file()

	def tag_entry(self, title):
		frame = gtk.Frame(title)
		entry = gtk.Entry(max=64)
		entry.show()
		frame.add(entry)
		frame.show()
		return (frame, entry)

	def next_file(self, widget=None):
		if self.files:
			self.mp3 = MP3(self.files.pop(0), ID3=EasyID3)
			self.number += 1
			for name, entry in self.entrances.items():
				if name in self.mp3.keys():
					entry.set_text(self.mp3[name][0])
				else:
					entry.set_text(str())
			self.update_label()
			print u'Переход к файлу "%s"' % self.mp3.filename
		else:
			gtk.main_quit()

	def update_label(self):
		self.label_text = default_label_text % \
			(
			self.mp3.filename,
			self.number,
			self.All,
			self.mp3.info.length,
			self.mp3.info.length / 60,
			self.mp3.info.length % 60,
			self.mp3.info.bitrate / 1024
			) + '"%s"'
		self.update_newName()

	update_newName = lambda self, widget=None: self.label.set_text(
		self.label_text % \
				(u'%s - %s.mp3' % \
					(
						(self.artist.get_text()    if   self.artist.get_text()   else    u'Неизвестно'),
						(self.title.get_text()      if   self.title.get_text()     else    u'Неизвестно')
					)
				)
		)

	def dialog(self, widget=None, event=None, title=u'Уверены?', message=u'Несохранённые изменения будут утеряны!'):
		self.dialog = gtk.MessageDialog(
			parent=self.window,
			type=gtk.MESSAGE_QUESTION,
			buttons=gtk.BUTTONS_YES_NO,
			message_format=message
		)

		self.dialog.set_title(title)

		response = self.dialog.run()
		self.dialog.destroy()

		return not response == gtk.RESPONSE_YES

#		if response == gtk.RESPONSE_YES:
#			return False
#		else:
#			return True

def main():
	sys.argv = sys.argv[1:]
	if sys.argv and (sys.argv[0] in ('--help', 'help', '-h')):
		print(Help)
	else:
		if not os.path.exists(mvDir):
			os.makedirs(mvDir)
		files = (sys.argv if sys.argv else os.listdir(chr(46)))
		mp3files = list()
		for x in files:
			if (not os.path.isdir(x)) and x.endswith('mp3'):
				mp3files.append(x)
			else:
				print(u'Невозможно открыть файл "%s": это не MP3 файл' % x)
		if mp3files:
			os.chdir(os.path.split(os.path.abspath(mp3files[0]))[0])
			utag_window(mp3files)
			gtk.main()
		else:
			print(u'\n\nНе указано файлов')

if __name__ == '__main__':
	try:
		main()
	except KeyboardInterrupt:
		sys.exit()

<commit_msg>added error dialog (gui.py)<commit_after>#!/usr/bin/env python2
# -*- coding: utf-8 -*-
#
#  This program distributed under Apache 2.0 license.
#  See LICENSE.txt for more details.
#  © UselessFire

# v. 0.3.2 Beta Public

mvDir = "tagged"

import sys, os, gtk, shutil, gc

gc.enable()

from traceback import format_exc

from mutagen.easyid3 import EasyID3
from mutagen.mp3 import MP3

default_label_text = \
u'''Название файла: "%s" (%i/%i)
Длина: %i секунд (%i минут %i секунд)
Битрейт: %i КБ\с
Файл будет переименован в '''

Help = u'''UTag - изменение тегов mp3 файлов

Использование:
--help | -h | help - эта справка

UTag <имена файлов> - изменить теги этих файлов
'''

class utag_window:
	def __init__(self, files):
		self.files = files
		self.All = len(self.files)
		self.number = int()

		self.window = gtk.Window()

		self.window.set_title('UTag')
#		self.window.set_border_width(0)

		self.window.connect("delete_event", self.dialog)
		self.window.connect("destroy", gtk.main_quit)

		self.Vbox = gtk.VBox(False, 0)

		self.labelframe = gtk.Frame(u'Информация')
		self.labelframe.show()
		self.label = gtk.Label()
		self.label.show()
		self.labelframe.add(self.label)
		self.Vbox.add(self.labelframe)

		# entrances

		# artist
		response, self.artist = self.tag_entry(u'Исполнитель')
		self.Vbox.add(response)
		self.artist.connect("activate", self.update_newName)

		# album
		response, self.album = self.tag_entry(u'Альбом')
		self.Vbox.add(response)

		# title
		response, self.title = self.tag_entry(u'Название')
		self.Vbox.add(response)
		self.title.connect("activate", self.update_newName)

		# date
		response, self.date = self.tag_entry(u'Год')
		self.Vbox.add(response)

		# genre
		response, self.genre = self.tag_entry(u'Жанр')
		self.Vbox.add(response)

		# tracknumber
		response, self.tracknumber = self.tag_entry(u'Номер трека')
		self.Vbox.add(response)

		self.entrances = {
			'artist': self.artist,
			'album': self.album,
			'title': self.title,
			'date': self.date,
			'genre': self.genre,
			'tracknumber': self.tracknumber
		}

		self.Hbox = gtk.HBox(False, 0)

		self.delete_and_next_button = gtk.Button(u'Удалить и следующий')
		self.delete_and_next_button.connect('clicked', self.delete_and_next_file)
		self.Hbox.add(self.delete_and_next_button)
		self.delete_and_next_button.show()

		self.next_button = gtk.Button(u'Пропустить')
		self.next_button.connect('clicked', self.next_file)
		self.Hbox.add(self.next_button)
		self.next_button.show()

		self.save_and_next_button = gtk.Button(u'Сохранить и следующий')
		self.save_and_next_button.connect('clicked', self.save_and_next_file)
		self.save_and_next_button.show()
		self.Hbox.add(self.save_and_next_button)

		self.Vbox.show()
		self.window.add(self.Vbox)

		self.Vbox.add(self.Hbox)
		self.Hbox.show()

		self.window.show()

		self.next_file()

	def save_and_next_file(self, widget=None):
		for name, entry in self.entrances.items():
			text = entry.get_text()
			if text:
				self.mp3[name] = text
			elif name in self.mp3:
				try:
					del self.mp3[name]
				except:
					if self.dialog(title=u'Произошла ошибка', message=u'Произошла ошибка при удалении тега "%s" в файле "%s":\n%s.\nПовторить?' % (name, self.mp3.filename, format_exc())):
						self.next_file()
					else:
						self.save_and_next_file()
					return

		try:
			self.mp3.save()
		except:
			if self.dialog(title=u'Произошла ошибка', message=u'Произошла ошибка при сохранении файла "%s":\n%s.\nПовторить?' % (self.mp3.filename, format_exc())):
				self.next_file()
			else:
				self.save_and_next_file()
		else:
			print u'Файл "%s" был сохранён' % self.mp3.filename

			newName = (u'%s - %s.mp3' % \
				(
					(self.artist.get_text()	if	self.artist.get_text()	else    u'Неизвестно'),
					(self.title.get_text()		if	self.title.get_text()		else    u'Неизвестно')
				)
			)

			if os.path.exists("%s/%s" % (mvDir, newName)):
				if not self.dialog(title=u'Заменить?', message=u'Файл "%s" уже существует в папке "%s"' % (newName, mvDir)):
					os.remove("%s/%s" % (mvDir, newName))
					shutil.move(self.mp3.filename, "%s/%s" % (mvDir, newName))
					print u'Файл "%s" был перемещён в папку "%s" с названием "%s" с перезаписью' % (self.mp3.filename, mvDir, newName)
					self.next_file()
			else:
				shutil.move(self.mp3.filename, "%s/%s" % (mvDir, newName))
				print u'Файл "%s" был перемещён в папку "%s" с названием "%s"' % (self.mp3.filename, mvDir, newName)
				self.next_file()

	def delete_and_next_file(self, widget=None):
		try:
			os.remove(self.mp3.filename)
		except:
			if self.dialog(title=u'Произошла ошибка', message=u'Произошла ошибка при удалении файла "%s":\n%s.\nПовторить?' % (self.mp3.filename, format_exc())):
				self.next_file()
			else:
				self.delete_and_next_file()
		else:
			print u'Файл "%s" был удалён' % self.mp3.filename
			self.next_file()

	def tag_entry(self, title):
		frame = gtk.Frame(title)
		entry = gtk.Entry(max=64)
		entry.show()
		frame.add(entry)
		frame.show()
		return (frame, entry)

	def next_file(self, widget=None):
		if self.files:
			self.mp3 = MP3(self.files.pop(0), ID3=EasyID3)
			self.number += 1
			for name, entry in self.entrances.items():
				if name in self.mp3.keys():
					entry.set_text(self.mp3[name][0])
				else:
					entry.set_text(str())
			self.update_label()
			print u'Переход к файлу "%s"' % self.mp3.filename
		else:
			gtk.main_quit()

	def update_label(self):
		self.label_text = default_label_text % \
			(
			self.mp3.filename,
			self.number,
			self.All,
			self.mp3.info.length,
			self.mp3.info.length / 60,
			self.mp3.info.length % 60,
			self.mp3.info.bitrate / 1024
			) + '"%s"'
		self.update_newName()

	update_newName = lambda self, widget=None: self.label.set_text(
		self.label_text % \
				(u'%s - %s.mp3' % \
					(
						(self.artist.get_text()    if   self.artist.get_text()   else    u'Неизвестно'),
						(self.title.get_text()      if   self.title.get_text()     else    u'Неизвестно')
					)
				)
		)

	def dialog(self, widget=None, event=None, title=u'Уверены?', message=u'Несохранённые изменения будут утеряны!'):
		self.dialog = gtk.MessageDialog(
			parent=self.window,
			type=gtk.MESSAGE_QUESTION,
			buttons=gtk.BUTTONS_YES_NO,
			message_format=message
		)

		self.dialog.set_title(title)

		response = self.dialog.run()
		self.dialog.destroy()

		return not response == gtk.RESPONSE_YES

#		if response == gtk.RESPONSE_YES:
#			return False
#		else:
#			return True

def main():
	sys.argv = sys.argv[1:]
	if sys.argv and (sys.argv[0] in ('--help', 'help', '-h')):
		print(Help)
	else:
		if not os.path.exists(mvDir):
			os.makedirs(mvDir)
		files = (sys.argv if sys.argv else os.listdir(chr(46)))
		mp3files = list()
		for x in files:
			if (not os.path.isdir(x)) and x.endswith('mp3'):
				mp3files.append(x)
			else:
				print(u'Невозможно открыть файл "%s": это не MP3 файл' % x)
		if mp3files:
			os.chdir(os.path.split(os.path.abspath(mp3files[0]))[0])
			utag_window(mp3files)
			gtk.main()
		else:
			print(u'\n\nНе указано файлов')

if __name__ == '__main__':
	try:
		main()
	except KeyboardInterrupt:
		sys.exit()

<|endoftext|>


<commit_before>import argparse

def get_parser():
	parser = argparse.ArgumentParser()

	parser.add_argument('--batch-size', action='store', dest='batch_size',help='batch-size , default 64',type=int,default = 64)

	parser.add_argument('--epochs', action='store', dest='n_epochs',help='# of epochs, default = 500',type=int,default = 500)

	parser.add_argument('--patience', action='store', dest='patience',help='# of epochs for patience, default = 10',type=int,default = 10)

	parser.add_argument('--model', action='store', dest='model',help='model type {enc2dec, attention, pointer},default = pointer', default = 'pointer')

	parser.add_argument('--unit', action='store', dest='unit',help='train with {lstm gru rnn} units,default = lstm', default = 'lstm')

	parser.add_argument('--hidden', action='store', dest='n_hidden',help='hidden size of softmax layer, default = 256',type=int,default = 256)

	parser.add_argument('--layers', action='store', dest='n_layers',help='# of hidden layers, default = 1',type=int,default = 1)

	parser.add_argument('--train', action='store', dest='tr',help='training conll file ',default = '../data/PTB_SD_3_3_0/train.conll')

	parser.add_argument('--val', action='store', dest='val',help='validation conll file',default = '../data/PTB_SD_3_3_0/dev.conll')

	parser.add_argument('--test', action='store', dest='test',help='test conll file',default = '../data/PTB_SD_3_3_0/test.conll')

	parser.add_argument('--prefix', action='store', dest='prefix',help='exp log prefix to append exp/{} default = 0',default = '0')

	parser.add_argument('--vector', action='store', dest='vector',help='vector pkl file', default = '../embeddings/sskip100.pkl')

	return parser

def get_tester():
	parser = argparse.ArgumentParser()

	parser.add_argument('--prefix', action='store', dest='prefix',help='prefix to model files')

	parser.add_argument('--test', action='store', dest='test',help='test conll file',default = '../data/PTB_SD_3_3_0/test.conll')

	parser.add_argument('--max-length', action='store', dest='max_length',help='max length of sequence',type=int)

	parser.add_argument('--vector', action='store', dest='vector',help='vector pkl file', default = '../embeddings/sskip100.pkl')

	parser.add_argument('--predict', action='store', dest='predict',help='prediction(output) file')

	return parser

def get_lengths(f_name):
	s = []
	lengths = []
	for line in open(f_name):
		l = line.strip().split('\t')
		if len(l) <= 1:
			lengths.append(len(s))
			s = []
			continue
		s.append(l)
	if s != []:
		lengths.append(len(s))
	return lengths

def convert2conll(prediction):

	out = []
	for i,pred in enumerate(prediction):
		if int(pred) == i:
			out.append('0')
		else:
			out.append(str(int(pred) +1))
	return out
<commit_msg>a small fix for utils.py<commit_after>import argparse

def get_parser():
	parser = argparse.ArgumentParser()

	parser.add_argument('--batch-size', action='store', dest='batch_size',help='batch-size , default 64',type=int,default = 64)

	parser.add_argument('--epochs', action='store', dest='n_epochs',help='# of epochs, default = 500',type=int,default = 500)

	parser.add_argument('--patience', action='store', dest='patience',help='# of epochs for patience, default = 10',type=int,default = 10)

	parser.add_argument('--model', action='store', dest='model',help='model type {enc2dec, attention, pointer},default = pointer', default = 'pointer')

	parser.add_argument('--unit', action='store', dest='unit',help='train with {lstm gru rnn} units,default = lstm', default = 'lstm')

	parser.add_argument('--hidden', action='store', dest='n_hidden',help='hidden size of neural networks, default = 256',type=int,default = 256)

	parser.add_argument('--layers', action='store', dest='n_layers',help='# of hidden layers, default = 1',type=int,default = 1)

	parser.add_argument('--train', action='store', dest='tr',help='training conll file ',default = '../data/PTB_SD_3_3_0/train.conll')

	parser.add_argument('--val', action='store', dest='val',help='validation conll file',default = '../data/PTB_SD_3_3_0/dev.conll')

	parser.add_argument('--test', action='store', dest='test',help='test conll file',default = '../data/PTB_SD_3_3_0/test.conll')

	parser.add_argument('--prefix', action='store', dest='prefix',help='exp log prefix to append exp/{} default = 0',default = '0')

	parser.add_argument('--vector', action='store', dest='vector',help='vector pkl file', default = '../embeddings/sskip100.pkl')

	return parser

def get_tester():
	parser = argparse.ArgumentParser()

	parser.add_argument('--prefix', action='store', dest='prefix',help='prefix to model files')

	parser.add_argument('--test', action='store', dest='test',help='test conll file',default = '../data/PTB_SD_3_3_0/test.conll')

	parser.add_argument('--max-length', action='store', dest='max_length',help='max length of sequence',type=int)

	parser.add_argument('--vector', action='store', dest='vector',help='vector pkl file', default = '../embeddings/sskip100.pkl')

	parser.add_argument('--predict', action='store', dest='predict',help='prediction(output) file')

	return parser

def get_lengths(f_name):
	s = []
	lengths = []
	for line in open(f_name):
		l = line.strip().split('\t')
		if len(l) <= 1:
			lengths.append(len(s))
			s = []
			continue
		s.append(l)
	if s != []:
		lengths.append(len(s))
	return lengths

def convert2conll(prediction):

	out = []
	for i,pred in enumerate(prediction):
		if int(pred) == i:
			out.append('0')
		else:
			out.append(str(int(pred) +1))
	return out
<|endoftext|>


<commit_before># Copyright (c) 2016 Cisco Systems, Inc.
# All Rights Reserved
#
#    Licensed under the Apache License, Version 2.0 (the "License"); you may
#    not use this file except in compliance with the License. You may obtain
#    a copy of the License at
#
#         http://www.apache.org/licenses/LICENSE-2.0
#
#    Unless required by applicable law or agreed to in writing, software
#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
#    License for the specific language governing permissions and limitations
#    under the License.

import grp
import os
import pwd
import vpp_papi
from oslo_log import log as logging

LOG = logging.getLogger(__name__)

def mac_to_bytes(mac):
    return str(''.join(chr(int(x, base=16)) for x in mac.split(':')))

def fix_string(s):
    return s.rstrip("\0").decode(encoding='ascii')

def _vpp_cb(*args, **kwargs):
    # sw_interface_set_flags comes back when you delete interfaces
    # print 'callback:', args, kwargs
    pass

def _check_retval(t):
    if t.retval != 0:
        print ('FAIL? retval here is %s' % t.retval)
#        raise Exception('failed in backend')

# Sometimes a callback fires unexpectedly.  We need to catch them
# because vpp_papi will traceback otherwise
vpp_papi.register_event_callback(_vpp_cb)

class VPPInterface(object):

    def get_interfaces(self):
        t = vpp_papi.sw_interface_dump(0, b'ignored')

        for interface in t:
            if interface.vl_msg_id == vpp_papi.VL_API_SW_INTERFACE_DETAILS:
                yield (fix_string(interface.interface_name), interface)

    def get_interface(self, name):
        for (ifname, f) in self.get_interfaces():
            if ifname == name:
                return f

    def get_version(self):
        t = vpp_papi.show_version()

        _check_retval(t)

        return fix_string(t.version)

    ########################################

    def create_tap(self, ifname, mac):
        t = vpp_papi.tap_connect(False,  # random MAC
                                 str(ifname), # TODO is this not a string?
                                 mac_to_bytes(mac),
                                 False,  # renumber - who knows, no doc
                                 0)  # customdevinstance - who knows, no doc

        _check_retval(t)

        return t.sw_if_index  # will be -1 on failure (e.g. 'already exists')

    def delete_tap(self, idx):
        vpp_papi.tap_delete(idx)

        # Err, I just got a sw_interface_set_flags here, not a delete tap?
        # _check_retval(t)

    #############################

    def create_vhostuser(self, ifpath, mac, qemu_user, qemu_group):
        self.LOG.info('Creating %s as a port' % ifpath)
        t = vpp_papi.create_vhost_user_if(True,  # is a server?
                                          str(ifpath),  # unicode not allowed.
                                          False,  # Who knows what renumber is?
                                          0,  # custom_dev_instance
                                          True,  # use custom MAC
                                          mac_to_bytes(mac)
                                          )

        print("Created vhost user interface object: %s" % str(t))
        _check_retval(t)

        # The permission that qemu runs as.
        self.LOG.info('Changing vhostuser interface file permission to %s:%s'
                      % (qemu_user, qemu_group))
        uid = pwd.getpwnam(qemu_user).pw_uid
        gid = grp.getgrnam(qemu_group).gr_gid

        os.chown(ifpath, uid, gid)
        os.chmod(ifpath, 0o770)

        return t.sw_if_index

    def delete_vhostuser(self, idx):
        #LOG.debug("Deleting VPP interface - index: %s" % idx)
        print("Deleting VPP interface - index: %s" % idx)
        t = vpp_papi.delete_vhost_user_if(idx)

        _check_retval(t)

    ########################################

    def __init__(self, log):
        self.LOG = log
        self.r = vpp_papi.connect("test_papi")

    def disconnect(self):
        vpp_papi.disconnect()

    def create_bridge_domain(self, id):
        t = vpp_papi.bridge_domain_add_del(
            id,  # the numeric ID of this domain
            True,  # enable bcast and mcast flooding
            True,  # enable unknown ucast flooding
            True,  # enable forwarding on all interfaces
            True,  # enable learning on all interfaces
            False,  # enable ARP termination in the BD
            True  # is an add
        )

        _check_retval(t)

    def create_vlan_subif(self, if_id, vlan_tag):
        t = vpp_papi.create_vlan_subif(
            if_id,
            vlan_tag)

        _check_retval(t)

        return t.sw_if_index

    def create_srcrep_vxlan_subif(self, vrf_id, src_addr, bcast_addr, vnid):
        t = vpp_papi.vxlan_add_del_tunnel(
            true,  # is_add
            src_addr,
            bcast_addr,
            vrf_id,
            decap_next_index,   # what is this?
            vni)

        _check_retval(t)

        return t.sw_if_index

    ########################################

    def add_to_bridge(self, bridx, *ifidxes):
        for ifidx in ifidxes:
            t = vpp_papi.sw_interface_set_l2_bridge(
                ifidx, bridx,
                False,                  # BVI (no thanks)
                0,                      # shared horizon group
                True)                   # enable bridge mode
            _check_retval(t)

    def ifup(self, *ifidxes):
        for ifidx in ifidxes:
            vpp_papi.sw_interface_set_flags(
                ifidx,
                1, 1,               # admin and link up
                0)                   # err, I can set the delected flag?
<commit_msg>fix bug attribute error vpp.py<commit_after># Copyright (c) 2016 Cisco Systems, Inc.
# All Rights Reserved
#
#    Licensed under the Apache License, Version 2.0 (the "License"); you may
#    not use this file except in compliance with the License. You may obtain
#    a copy of the License at
#
#         http://www.apache.org/licenses/LICENSE-2.0
#
#    Unless required by applicable law or agreed to in writing, software
#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
#    License for the specific language governing permissions and limitations
#    under the License.

import grp
import os
import pwd
import vpp_papi
from oslo_log import log as logging

LOG = logging.getLogger(__name__)

def mac_to_bytes(mac):
    return str(''.join(chr(int(x, base=16)) for x in mac.split(':')))

def fix_string(s):
    return s.rstrip("\0").decode(encoding='ascii')

def _vpp_cb(*args, **kwargs):
    # sw_interface_set_flags comes back when you delete interfaces
    # print 'callback:', args, kwargs
    pass

def _check_retval(t):
    try:
        if t.retval != 0:
            print ('FAIL? retval here is %s' % t.retval)
    #        raise Exception('failed in backend')
    except AttributeError as e:
        print("Error: %s" % e)

# Sometimes a callback fires unexpectedly.  We need to catch them
# because vpp_papi will traceback otherwise
vpp_papi.register_event_callback(_vpp_cb)

class VPPInterface(object):

    def get_interfaces(self):
        t = vpp_papi.sw_interface_dump(0, b'ignored')

        for interface in t:
            if interface.vl_msg_id == vpp_papi.VL_API_SW_INTERFACE_DETAILS:
                yield (fix_string(interface.interface_name), interface)

    def get_interface(self, name):
        for (ifname, f) in self.get_interfaces():
            if ifname == name:
                return f

    def get_version(self):
        t = vpp_papi.show_version()

        _check_retval(t)

        return fix_string(t.version)

    ########################################

    def create_tap(self, ifname, mac):
        t = vpp_papi.tap_connect(False,  # random MAC
                                 str(ifname), # TODO is this not a string?
                                 mac_to_bytes(mac),
                                 False,  # renumber - who knows, no doc
                                 0)  # customdevinstance - who knows, no doc

        _check_retval(t)

        return t.sw_if_index  # will be -1 on failure (e.g. 'already exists')

    def delete_tap(self, idx):
        vpp_papi.tap_delete(idx)

        # Err, I just got a sw_interface_set_flags here, not a delete tap?
        # _check_retval(t)

    #############################

    def create_vhostuser(self, ifpath, mac, qemu_user, qemu_group):
        self.LOG.info('Creating %s as a port' % ifpath)
        t = vpp_papi.create_vhost_user_if(True,  # is a server?
                                          str(ifpath),  # unicode not allowed.
                                          False,  # Who knows what renumber is?
                                          0,  # custom_dev_instance
                                          True,  # use custom MAC
                                          mac_to_bytes(mac)
                                          )

        print("Created vhost user interface object: %s" % str(t))
        _check_retval(t)

        # The permission that qemu runs as.
        self.LOG.info('Changing vhostuser interface file permission to %s:%s'
                      % (qemu_user, qemu_group))
        uid = pwd.getpwnam(qemu_user).pw_uid
        gid = grp.getgrnam(qemu_group).gr_gid

        os.chown(ifpath, uid, gid)
        os.chmod(ifpath, 0o770)

        return t.sw_if_index

    def delete_vhostuser(self, idx):
        #LOG.debug("Deleting VPP interface - index: %s" % idx)
        print("Deleting VPP interface - index: %s" % idx)
        t = vpp_papi.delete_vhost_user_if(idx)

        _check_retval(t)

    ########################################

    def __init__(self, log):
        self.LOG = log
        self.r = vpp_papi.connect("test_papi")

    def disconnect(self):
        vpp_papi.disconnect()

    def create_bridge_domain(self, id):
        t = vpp_papi.bridge_domain_add_del(
            id,  # the numeric ID of this domain
            True,  # enable bcast and mcast flooding
            True,  # enable unknown ucast flooding
            True,  # enable forwarding on all interfaces
            True,  # enable learning on all interfaces
            False,  # enable ARP termination in the BD
            True  # is an add
        )

        _check_retval(t)

    def create_vlan_subif(self, if_id, vlan_tag):
        t = vpp_papi.create_vlan_subif(
            if_id,
            vlan_tag)

        _check_retval(t)

        return t.sw_if_index

    def create_srcrep_vxlan_subif(self, vrf_id, src_addr, bcast_addr, vnid):
        t = vpp_papi.vxlan_add_del_tunnel(
            true,  # is_add
            src_addr,
            bcast_addr,
            vrf_id,
            decap_next_index,   # what is this?
            vni)

        _check_retval(t)

        return t.sw_if_index

    ########################################

    def add_to_bridge(self, bridx, *ifidxes):
        for ifidx in ifidxes:
            t = vpp_papi.sw_interface_set_l2_bridge(
                ifidx, bridx,
                False,                  # BVI (no thanks)
                0,                      # shared horizon group
                True)                   # enable bridge mode
            _check_retval(t)

    def ifup(self, *ifidxes):
        for ifidx in ifidxes:
            vpp_papi.sw_interface_set_flags(
                ifidx,
                1, 1,               # admin and link up
                0)                   # err, I can set the delected flag?
<|endoftext|>


<commit_before>load("//internal:node.bzl", "nodejs_test")
load("//internal:devmode_js_sources.bzl", "devmode_js_sources")

def jasmine_node_test(name, srcs, data = [], deps = [], **kwargs):
  devmode_js_sources(
      name = "%s_devmode_srcs" % name,
      deps = srcs + deps,
      testonly = 1,
  )

  data += srcs + deps
  data += [Label("//internal:jasmine_runner.js")]
  data += [":%s_devmode_srcs.MF" % name]
  entry_point = "build_bazel_rules_nodejs/internal/jasmine_runner.js"

  nodejs_test(
      name = name,
      data = data,
      entry_point = entry_point,
      templated_args = ["$(location :%s_devmode_srcs.MF)" % name],
      testonly = 1,
      **kwargs
  )
<commit_msg>Fix += for lists in jasmine_node_test<commit_after>load("//internal:node.bzl", "nodejs_test")
load("//internal:devmode_js_sources.bzl", "devmode_js_sources")

def jasmine_node_test(name, srcs, data = [], deps = [], **kwargs):
  devmode_js_sources(
      name = "%s_devmode_srcs" % name,
      deps = srcs + deps,
      testonly = 1,
  )

  all_data = data + srcs + deps
  all_data += [Label("//internal:jasmine_runner.js")]
  all_data += [":%s_devmode_srcs.MF" % name]
  entry_point = "build_bazel_rules_nodejs/internal/jasmine_runner.js"

  nodejs_test(
      name = name,
      data = all_data,
      entry_point = entry_point,
      templated_args = ["$(location :%s_devmode_srcs.MF)" % name],
      testonly = 1,
      **kwargs
  )
<|endoftext|>


<commit_before>#!/usr/bin/env python

#Copyright (C) 2013 by Glenn Hickey
#
# Class derived from _BaseHMM and MultinomialHMM from sklearn/tests/hmm.py
# (2010 - 2013, scikit-learn developers (BSD License))
#
#Released under the MIT license, see LICENSE.txt
#!/usr/bin/env python

import os
import sys
import numpy as np
import pickle
import string
import copy
import logging
from collections import Iterable
from numpy.testing import assert_array_equal, assert_array_almost_equal

from .emission import IndependentMultinomialEmissionModel
from .track import TrackList, TrackTable, Track
from .common import EPSILON, myLog
from sklearn.hmm import _BaseHMM
from sklearn.hmm import MultinomialHMM
from sklearn.hmm import _hmmc
from sklearn.hmm import NEGINF
from sklearn.utils import check_random_state, deprecated

"""
This class is based on the MultinomialHMM from sckikit-learn, but we make
the emission model a parameter. The custom emission model we support at this
point is a multi-*dimensional* multinomial.
"""
class MultitrackHmm(_BaseHMM):
    def __init__(self, emissionModel=None,
                 startprob=None,
                 transmat=None, startprob_prior=None, transmat_prior=None,
                 algorithm="viterbi", random_state=None,
                 n_iter=10, thresh=1e-2, params=string.ascii_letters,
                 init_params=string.ascii_letters,
                 state_name_map=None,
                 fudge=0.0,
                 fixTrans=False,
                 fixEmission=False):
        if emissionModel is not None:
            n_components = emissionModel.getNumStates()
        else:
            n_components = 1

        """Create a hidden Markov model that supports multiple tracks.
        emissionModel must have already been created"""
        _BaseHMM.__init__(self, n_components=n_components,
                          startprob=startprob,
                          transmat=transmat,
                          startprob_prior=startprob_prior,
                          transmat_prior=transmat_prior,
                          algorithm=algorithm,
                          random_state=random_state,
                          n_iter=n_iter,
                          thresh=thresh,
                          params=params,
                          init_params=init_params)
        # remember init_params
        self.init_params = init_params
        #: emission model is specified as parameter here
        self.emissionModel = emissionModel
        #: a TrackList object specifying information about the tracks
        self.trackList = None
        #: a map between state values and names (track.CategoryMap)
        self.stateNameMap = state_name_map
        # little constant that gets added to frequencies during training
        # to prevent zero probabilities.  The bigger it is, the flatter
        # the distribution... (note that emission class has its own)
        self.fudge = fudge
        # freeze input transmat
        self.fixTrans = fixTrans
        # freeze input EmissionModel
        self.fixEmission = fixEmission
        # keep track of number of EM iterations performed
        self.current_iteration = None

    def train(self, trackData):
        """ Use EM to estimate best parameters from scratch (unsupervised)"""
        self.trackList = trackData.getTrackList()
        self.fit(trackData.getTrackTableList())
        self.validate()

    def supervisedTrain(self, trackData, bedIntervals):
        """ Train directly from set of known states (4th column in the
        bedIntervals provided.  We assume that the most likely parameters
        are also just the expected values, which works for our basic
        multinomial distribution. Note that the states should already
        have been mapped to integers"""
        # NOTE bedIntervals must be sorted!
        self.trackList = trackData.getTrackList()
        N = self.emissionModel.getNumStates()
        transitionCount = self.fudge + np.zeros((N,N), np.float)
        freqCount = self.fudge + np.zeros((N,), np.float)
        prevInterval = None
        logging.debug("beginning supervised transition stats")
        for interval in bedIntervals:
            state = int(interval[3])
            assert state < N
            transitionCount[state,state] += interval[2] - interval[1] - 1
            freqCount[state] += interval[2] - interval[1]
            if prevInterval is not None and prevInterval[0] == interval[0]:
                if interval[1] < prevInterval[2]:
                    raise RuntimeError(
                        "Overlapping or out of order training intervals"
                        " detected: %s and %s."
                        (prevInterval, interval))
                elif interval[1] == prevInterval[2]:
                    transitionCount[prevInterval[3], state] += 1
            prevInterval = interval
        for row in xrange(len(transitionCount)):
            transitionCount[row] /= np.sum(transitionCount[row])
        self.transmat_ = np.copy(transitionCount)
        # scikit learn is too chicken to have 0-probs.  so we go back and
        # hack them in if necessary
        self._log_transmat = myLog(transitionCount)
        freqCount /= np.sum(freqCount)
        self.startprob_ = freqCount
        self.emissionModel.supervisedTrain(trackData, bedIntervals)
        self.validate()

    def logProb(self, trackData):
        """ Return the log probability of the data (one score for each
        interval"""
        logProbList = []
        for trackTable in trackData.getTrackTableList():
            totalLogProb.append(self.score(trackTable))
        return logProbList

    def viterbi(self, trackData, numThreads = 1):
        """ Return the output of the Viterbi algorithm on the loaded
        data: a tuple of (log likelihood of best path, and the path itself)
        (one data point of each interval of track data)
        """
        # Thread interface provided but not implemented:
        assert numThreads == 1
        output = []
        for trackTable in trackData.getTrackTableList():
            prob, states = self.decode(trackTable)
            if self.stateNameMap is not None:
                states = map(self.stateNameMap.getMapBack, states)
            output.append((prob,states))
        return output

    def __str__(self):
        states = [x for x in xrange(self.n_components)]
        if self.stateNameMap is not None:
            states = map(self.stateNameMap.getMapBack, states)
        s = "\nNumStates = %d:\n%s\n" % (self.n_components, str(states))
        sp = [(states[i], self.startprob_[i])
              for i in xrange(self.n_components)]
        s += "\nStart probs =\n%s\n" % str(sp)
        s += "\nTransitions =\n%s\n" % str(self.transmat_)
        s += "\nlogTransitions = \n%s\n" % str(myLog(self.transmat_))
        em = self.emissionModel
        s += "\nNumber of symbols per track=\n%s\n" % str(
            em.getNumSymbolsPerTrack())
        s += "\nEmissions =\n"
        emProbs = em.getLogProbs()
        for state, stateName in enumerate(states):
            s += "State %s:\n" % stateName
            for trackNo in xrange(em.getNumTracks()):
                track = self.trackList.getTrackByNumber(trackNo)
                s += "  Track %d %s (%s):\n" % (track.getNumber(),
                                                track.getName(),
                                                track.getDist())
                numSymbolsPerTrack =  em.getNumSymbolsPerTrack()
                for idx, symbol in enumerate(em.getTrackSymbols(trackNo)):
                    symbolName = track.getValueMap().getMapBack(symbol)
                    prob = np.exp(emProbs[trackNo][state][symbol])
                    if idx <= 2 or prob > 0.005:
                        logval = str(myLog(prob))
                        if prob == 0.0:
                            logval = "-inf"
                        s += "    %s) %s: %f (log=%s)\n" % (symbol, symbolName,
                                                            prob, logval)
        return s

    def getTrackList(self):
        return self.trackList

    def getStartProbs(self):
        return self.startprob_

    def getTransitionProbs(self):
        return self.transmat_

    def getStateNameMap(self):
        return self.stateNameMap

    def getEmissionModel(self):
        return self.emissionModel

    def validate(self):
        assert len(self.startprob_) == self.emissionModel.getNumStates()
        assert not isinstance(self.startprob_[0], Iterable)
        assert len(self.transmat_) == self.emissionModel.getNumStates()
        assert len(self.transmat_[0]) == self.emissionModel.getNumStates()
        assert_array_almost_equal(np.sum(self.startprob_), 1.)
        for i in xrange(self.emissionModel.getNumStates()):
            assert_array_almost_equal(np.sum(self.transmat_[i]), 1.)
        self.emissionModel.validate()

    ###########################################################################
    #       SCIKIT LEARN BASEHMM OVERRIDES BELOW
    ###########################################################################

    def _compute_log_likelihood(self, obs):
        return self.emissionModel.allLogProbs(obs)

    def _generate_sample_from_state(self, state, random_state=None):
        return self.emissionModel.sample(state)

    def _init(self, obs, params='ste'):
        self.params = params
        if self.fixTrans is True:
            self.params = self.params.replace("t", "")
        if self.fixEmission is True:
            self.params = self.params.replace("e", "")
        super(MultitrackHmm, self)._init(obs, params=params)
        self.random_state = check_random_state(self.random_state)
        randomize = 'e' in params
        self.emissionModel.initParams(randomize=randomize)

    def _initialize_sufficient_statistics(self):
        stats = super(MultitrackHmm, self)._initialize_sufficient_statistics()
        stats['obs'] = self.emissionModel.initStats()
        return stats

    def _accumulate_sufficient_statistics(self, stats, obs, framelogprob,
                                          posteriors, fwdlattice, bwdlattice,
                                          params):
        logging.debug("%d: beginning MultitrackHMM E-step" %
                      self.current_iteration)
        super(MultitrackHmm, self)._accumulate_sufficient_statistics(
            stats, obs, framelogprob, posteriors, fwdlattice, bwdlattice,
            params)
        if 'e' in params:
            logging.debug("beginning Emissions E-substep")
            self.emissionModel.accumulateStats(obs, stats['obs'], posteriors)

        logging.debug("ending MultitrackHMM E-step")

    def _do_mstep(self, stats, params):
        logging.debug("%d: beginning MultitrackHMM M-step" %
                      self.current_iteration)
        super(MultitrackHmm, self)._do_mstep(stats, params)
        if 'e' in params:
            self.emissionModel.maximize(stats['obs'])
        logging.debug("%d: ending MultitrackHMM M-step" %
                      self.current_iteration)
        self.current_iteration += 1

    def fit(self, obs, **kwargs):
        self.current_iteration = 1
        return _BaseHMM.fit(self, obs, **kwargs)

    # Getting annoyed with epsilons being added by scikit learn
    # so redo tranmat property to allow zeros (should probably do
    # for start probs as well at some point)
    def _get_transmat(self):
        """Matrix of transition probabilities."""
        return np.exp(self._log_transmat)

    def _set_transmat(self, transmat):
        if transmat is None:
            transmat = np.tile(1.0 / self.n_components,
                               (self.n_components, self.n_components))

        if (np.asarray(transmat).shape
                != (self.n_components, self.n_components)):
            raise ValueError('transmat must have shape '
                             '(n_components, n_components)')
        if not np.all(np.allclose(np.sum(transmat, axis=1), 1.0)):
            raise ValueError('Rows of transmat must sum to 1.0')

        self._log_transmat = myLog(np.asarray(transmat).copy())

    transmat_ = property(_get_transmat, _set_transmat)
<commit_msg>fix bug added to _init where self.params was getting overrided with init_params.  also remove line in same function where emission model is initialized.  this must now be done outside of hmm<commit_after>#!/usr/bin/env python

#Copyright (C) 2013 by Glenn Hickey
#
# Class derived from _BaseHMM and MultinomialHMM from sklearn/tests/hmm.py
# (2010 - 2013, scikit-learn developers (BSD License))
#
#Released under the MIT license, see LICENSE.txt
#!/usr/bin/env python

import os
import sys
import numpy as np
import pickle
import string
import copy
import logging
from collections import Iterable
from numpy.testing import assert_array_equal, assert_array_almost_equal

from .emission import IndependentMultinomialEmissionModel
from .track import TrackList, TrackTable, Track
from .common import EPSILON, myLog
from sklearn.hmm import _BaseHMM
from sklearn.hmm import MultinomialHMM
from sklearn.hmm import _hmmc
from sklearn.hmm import NEGINF
from sklearn.utils import check_random_state, deprecated

"""
This class is based on the MultinomialHMM from sckikit-learn, but we make
the emission model a parameter. The custom emission model we support at this
point is a multi-*dimensional* multinomial.
"""
class MultitrackHmm(_BaseHMM):
    def __init__(self, emissionModel=None,
                 startprob=None,
                 transmat=None, startprob_prior=None, transmat_prior=None,
                 algorithm="viterbi", random_state=None,
                 n_iter=10, thresh=1e-2, params=string.ascii_letters,
                 init_params=string.ascii_letters,
                 state_name_map=None,
                 fudge=0.0,
                 fixTrans=False,
                 fixEmission=False):
        if emissionModel is not None:
            n_components = emissionModel.getNumStates()
        else:
            n_components = 1

        """Create a hidden Markov model that supports multiple tracks.
        emissionModel must have already been created"""
        _BaseHMM.__init__(self, n_components=n_components,
                          startprob=startprob,
                          transmat=transmat,
                          startprob_prior=startprob_prior,
                          transmat_prior=transmat_prior,
                          algorithm=algorithm,
                          random_state=random_state,
                          n_iter=n_iter,
                          thresh=thresh,
                          params=params,
                          init_params=init_params)
        # remember init_params
        self.init_params = init_params
        #: emission model is specified as parameter here
        self.emissionModel = emissionModel
        #: a TrackList object specifying information about the tracks
        self.trackList = None
        #: a map between state values and names (track.CategoryMap)
        self.stateNameMap = state_name_map
        # little constant that gets added to frequencies during training
        # to prevent zero probabilities.  The bigger it is, the flatter
        # the distribution... (note that emission class has its own)
        self.fudge = fudge
        # freeze input transmat
        self.fixTrans = fixTrans
        # freeze input EmissionModel
        self.fixEmission = fixEmission
        # keep track of number of EM iterations performed
        self.current_iteration = None

    def train(self, trackData):
        """ Use EM to estimate best parameters from scratch (unsupervised)"""
        self.trackList = trackData.getTrackList()
        self.fit(trackData.getTrackTableList())
        self.validate()

    def supervisedTrain(self, trackData, bedIntervals):
        """ Train directly from set of known states (4th column in the
        bedIntervals provided.  We assume that the most likely parameters
        are also just the expected values, which works for our basic
        multinomial distribution. Note that the states should already
        have been mapped to integers"""
        # NOTE bedIntervals must be sorted!
        self.trackList = trackData.getTrackList()
        N = self.emissionModel.getNumStates()
        transitionCount = self.fudge + np.zeros((N,N), np.float)
        freqCount = self.fudge + np.zeros((N,), np.float)
        prevInterval = None
        logging.debug("beginning supervised transition stats")
        for interval in bedIntervals:
            state = int(interval[3])
            assert state < N
            transitionCount[state,state] += interval[2] - interval[1] - 1
            freqCount[state] += interval[2] - interval[1]
            if prevInterval is not None and prevInterval[0] == interval[0]:
                if interval[1] < prevInterval[2]:
                    raise RuntimeError(
                        "Overlapping or out of order training intervals"
                        " detected: %s and %s."
                        (prevInterval, interval))
                elif interval[1] == prevInterval[2]:
                    transitionCount[prevInterval[3], state] += 1
            prevInterval = interval
        for row in xrange(len(transitionCount)):
            transitionCount[row] /= np.sum(transitionCount[row])
        self.transmat_ = np.copy(transitionCount)
        # scikit learn is too chicken to have 0-probs.  so we go back and
        # hack them in if necessary
        self._log_transmat = myLog(transitionCount)
        freqCount /= np.sum(freqCount)
        self.startprob_ = freqCount
        self.emissionModel.supervisedTrain(trackData, bedIntervals)
        self.validate()

    def logProb(self, trackData):
        """ Return the log probability of the data (one score for each
        interval"""
        logProbList = []
        for trackTable in trackData.getTrackTableList():
            totalLogProb.append(self.score(trackTable))
        return logProbList

    def viterbi(self, trackData, numThreads = 1):
        """ Return the output of the Viterbi algorithm on the loaded
        data: a tuple of (log likelihood of best path, and the path itself)
        (one data point of each interval of track data)
        """
        # Thread interface provided but not implemented:
        assert numThreads == 1
        output = []
        for trackTable in trackData.getTrackTableList():
            prob, states = self.decode(trackTable)
            if self.stateNameMap is not None:
                states = map(self.stateNameMap.getMapBack, states)
            output.append((prob,states))
        return output

    def __str__(self):
        states = [x for x in xrange(self.n_components)]
        if self.stateNameMap is not None:
            states = map(self.stateNameMap.getMapBack, states)
        s = "\nNumStates = %d:\n%s\n" % (self.n_components, str(states))
        sp = [(states[i], self.startprob_[i])
              for i in xrange(self.n_components)]
        s += "\nStart probs =\n%s\n" % str(sp)
        s += "\nTransitions =\n%s\n" % str(self.transmat_)
        s += "\nlogTransitions = \n%s\n" % str(myLog(self.transmat_))
        em = self.emissionModel
        s += "\nNumber of symbols per track=\n%s\n" % str(
            em.getNumSymbolsPerTrack())
        s += "\nEmissions =\n"
        emProbs = em.getLogProbs()
        for state, stateName in enumerate(states):
            s += "State %s:\n" % stateName
            for trackNo in xrange(em.getNumTracks()):
                track = self.trackList.getTrackByNumber(trackNo)
                s += "  Track %d %s (%s):\n" % (track.getNumber(),
                                                track.getName(),
                                                track.getDist())
                numSymbolsPerTrack =  em.getNumSymbolsPerTrack()
                for idx, symbol in enumerate(em.getTrackSymbols(trackNo)):
                    symbolName = track.getValueMap().getMapBack(symbol)
                    prob = np.exp(emProbs[trackNo][state][symbol])
                    if idx <= 2 or prob > 0.005:
                        logval = str(myLog(prob))
                        if prob == 0.0:
                            logval = "-inf"
                        s += "    %s) %s: %f (log=%s)\n" % (symbol, symbolName,
                                                            prob, logval)
        return s

    def getTrackList(self):
        return self.trackList

    def getStartProbs(self):
        return self.startprob_

    def getTransitionProbs(self):
        return self.transmat_

    def getStateNameMap(self):
        return self.stateNameMap

    def getEmissionModel(self):
        return self.emissionModel

    def validate(self):
        assert len(self.startprob_) == self.emissionModel.getNumStates()
        assert not isinstance(self.startprob_[0], Iterable)
        assert len(self.transmat_) == self.emissionModel.getNumStates()
        assert len(self.transmat_[0]) == self.emissionModel.getNumStates()
        assert_array_almost_equal(np.sum(self.startprob_), 1.)
        for i in xrange(self.emissionModel.getNumStates()):
            assert_array_almost_equal(np.sum(self.transmat_[i]), 1.)
        self.emissionModel.validate()

    ###########################################################################
    #       SCIKIT LEARN BASEHMM OVERRIDES BELOW
    ###########################################################################

    def _compute_log_likelihood(self, obs):
        return self.emissionModel.allLogProbs(obs)

    def _generate_sample_from_state(self, state, random_state=None):
        return self.emissionModel.sample(state)

    def _init(self, obs, params='ste'):
        if self.fixTrans is True:
            self.params = self.params.replace("t", "")
        if self.fixEmission is True:
            self.params = self.params.replace("e", "")
        super(MultitrackHmm, self)._init(obs, params=params)
        self.random_state = check_random_state(self.random_state)

    def _initialize_sufficient_statistics(self):
        stats = super(MultitrackHmm, self)._initialize_sufficient_statistics()
        stats['obs'] = self.emissionModel.initStats()
        return stats

    def _accumulate_sufficient_statistics(self, stats, obs, framelogprob,
                                          posteriors, fwdlattice, bwdlattice,
                                          params):
        logging.debug("%d: beginning MultitrackHMM E-step" %
                      self.current_iteration)
        super(MultitrackHmm, self)._accumulate_sufficient_statistics(
            stats, obs, framelogprob, posteriors, fwdlattice, bwdlattice,
            params)
        if 'e' in params:
            logging.debug("beginning Emissions E-substep")
            self.emissionModel.accumulateStats(obs, stats['obs'], posteriors)

        logging.debug("ending MultitrackHMM E-step")

    def _do_mstep(self, stats, params):
        logging.debug("%d: beginning MultitrackHMM M-step" %
                      self.current_iteration)
        super(MultitrackHmm, self)._do_mstep(stats, params)
        if 'e' in params:
            self.emissionModel.maximize(stats['obs'])
        logging.debug("%d: ending MultitrackHMM M-step" %
                      self.current_iteration)
        self.current_iteration += 1

    def fit(self, obs, **kwargs):
        self.current_iteration = 1
        return _BaseHMM.fit(self, obs, **kwargs)

    # Getting annoyed with epsilons being added by scikit learn
    # so redo tranmat property to allow zeros (should probably do
    # for start probs as well at some point)
    def _get_transmat(self):
        """Matrix of transition probabilities."""
        return np.exp(self._log_transmat)

    def _set_transmat(self, transmat):
        if transmat is None:
            transmat = np.tile(1.0 / self.n_components,
                               (self.n_components, self.n_components))

        if (np.asarray(transmat).shape
                != (self.n_components, self.n_components)):
            raise ValueError('transmat must have shape '
                             '(n_components, n_components)')
        if not np.all(np.allclose(np.sum(transmat, axis=1), 1.0)):
            raise ValueError('Rows of transmat must sum to 1.0')

        self._log_transmat = myLog(np.asarray(transmat).copy())

    transmat_ = property(_get_transmat, _set_transmat)
<|endoftext|>