<commit_before># audiovisualizer
# Copyright (C) 2015 Timothy Woodford
# Licensed under the Apache License, version 2.0
# You can obtain a copy of the license at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied.  See the License for the
# specific language governing permissions and limitations
# under the License.
import numpy
import numpy.fft
import scipy

def _bin_approx_search(lst, tg):
    """
    Find the index of the element in lst which is closest to the number tg
    """
    top = len(lst) - 1
    bottom = 0
    while top > bottom:
        curri = (top - bottom)//2 + bottom
        if lst[curri] < tg:
            bottom = curri
        else:
            top = curri
        if top - bottom == 1:
            if abs(lst[top] - tg) < abs(lst[bottom] - tg):
                return top
            else:
                return bottom
    return top

def fft_in_range(audiomatrix, startindex, endindex, channel):
    """
    Do an FFT in the specified range of indices

    The audiomatrix should have the first index as its time domain and
    second index as the channel number.  The startindex and endinex
    select the time range to use, and the channel parameter selects
    which channel to do the FFT on.

    Returns a vector of data in the frequency domain
    """
    n = endindex - startindex
    indat = audiomatrix[startindex:endindex, channel]
    outdat = (numpy.fft.fft(indat)[range(n//2)])/n
    return outdat

def get_x_axis(samplerate, samplelength):
    """
    Find the actual frequencies to include along the x axis

    The samplerate is the sample rate of the audio matrix in Hertz
    and the samplelength is the number of samples fed to the FFT.

    Returns a matrix with the x axis numbers.
    """
    time = samplelength / samplerate # The sample time of a single fft
    return scipy.arange(samplelength // 2) / time

def moving_fft(audiomatrix, sampletime, fps, samplerate, channel=0):
    """
    Get a number of FFT samples over the time of an audio sample

    This is basically like a moving average for DFTs

    Args:
        audiomatrix: A matrix of audio data with time for the first
            dimension and channel for the second dimension
        sampletime: The length of a sample in seconds
        fps: The number of output samples per second
        samplerate: The sample frequency of the audio in hertz
        channel: The audio channel to use
    Returns:
        A matrix where the first dimension is the frame number
        and the second dimension is the frequency
    """
    samplelength = int(sampletime * samplerate)
    frame_increment = samplerate // fps
    frame = 0
    frames = (audiomatrix.shape[0] - samplelength) // frame_increment
    #ret = numpy.zeros((frames, frame_increment//2), numpy.complex128)
    ret = []
    for startindex in range(0, audiomatrix.shape[0] - samplelength, frame_increment):
        x = fft_in_range(audiomatrix, startindex, startindex + frame_increment, channel)
        ret.append(x)
    return numpy.array(ret)

def freq_range_graph(fftmatrix, freqrange, samplerate, sampletime):
    """
    Create a row vector of the average amplitude of a frequency range over time

    Args:
        fftmatrix: The moving_fft() output
        freqrange: A tuple of form (minimum frequency, maximum frequency)
        samplerate: The sample frequency of the audio in hertz
        sampletime: The length of a sample in seconds
    """
    frq = get_x_axis(samplerate, sampletime * samplerate)
    bottomindex = _bin_approx_search(frq, freqrange[0])
    topindex = _bin_approx_search(frq, freqrange[1])
    sliced = fftmatrix[:,bottomindex:topindex]
    return numpy.average(sliced, 1)

def isolate_freq_range(fftmatrix, freqrange, samplerate, sampletime):
    """
    Create a moving DFT matrix with only the given frequency range

    Args:
        fftmatrix: The moving_fft() output
        freqrange: A tuple of form (minimum frequency, maximum frequency)
        samplerate: The sample frequency of the audio in hertz
        sampletime: The length of a sample in seconds
    """
    frq = get_x_axis(samplerate, sampletime * samplerate)
    bottomindex = _bin_approx_search(frq, freqrange[0])
    topindex = _bin_approx_search(frq, freqrange[1])
    return fftmatrix[:,bottomindex:topindex]

<commit_msg>Add extract_freq() method for extracting drum hits, etc.<commit_after># audiovisualizer
# Copyright (C) 2015 Timothy Woodford
# Licensed under the Apache License, version 2.0
# You can obtain a copy of the license at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied.  See the License for the
# specific language governing permissions and limitations
# under the License.
import numpy
import numpy.fft
import scipy

def _bin_approx_search(lst, tg):
    """
    Find the index of the element in lst which is closest to the number tg
    """
    top = len(lst) - 1
    bottom = 0
    while top > bottom:
        curri = (top - bottom)//2 + bottom
        if lst[curri] < tg:
            bottom = curri
        else:
            top = curri
        if top - bottom == 1:
            if abs(lst[top] - tg) < abs(lst[bottom] - tg):
                return top
            else:
                return bottom
    return top

def fft_in_range(audiomatrix, startindex, endindex, channel):
    """
    Do an FFT in the specified range of indices

    The audiomatrix should have the first index as its time domain and
    second index as the channel number.  The startindex and endinex
    select the time range to use, and the channel parameter selects
    which channel to do the FFT on.

    Returns a vector of data in the frequency domain
    """
    n = endindex - startindex
    indat = audiomatrix[startindex:endindex, channel]
    outdat = (numpy.fft.fft(indat)[range(n//2)])/n
    return outdat

def get_x_axis(samplerate, samplelength):
    """
    Find the actual frequencies to include along the x axis

    The samplerate is the sample rate of the audio matrix in Hertz
    and the samplelength is the number of samples fed to the FFT.

    Returns a matrix with the x axis numbers.
    """
    time = samplelength / samplerate # The sample time of a single fft
    return scipy.arange(samplelength // 2) / time

def moving_fft(audiomatrix, sampletime, fps, samplerate, channel=0):
    """
    Get a number of FFT samples over the time of an audio sample

    This is basically like a moving average for DFTs

    Args:
        audiomatrix: A matrix of audio data with time for the first
            dimension and channel for the second dimension
        sampletime: The length of a sample in seconds
        fps: The number of output samples per second
        samplerate: The sample frequency of the audio in hertz
        channel: The audio channel to use
    Returns:
        A matrix where the first dimension is the frame number
        and the second dimension is the frequency
    """
    samplelength = int(sampletime * samplerate)
    frame_increment = samplerate // fps
    frame = 0
    frames = (audiomatrix.shape[0] - samplelength) // frame_increment
    #ret = numpy.zeros((frames, frame_increment//2), numpy.complex128)
    ret = []
    for startindex in range(0, audiomatrix.shape[0] - samplelength, frame_increment):
        x = fft_in_range(audiomatrix, startindex, startindex + frame_increment, channel)
        ret.append(x)
    return numpy.array(ret)

def freq_range_graph(fftmatrix, freqrange, samplerate, sampletime):
    """
    Create a row vector of the average amplitude of a frequency range over time

    Args:
        fftmatrix: The moving_fft() output
        freqrange: A tuple of form (minimum frequency, maximum frequency)
        samplerate: The sample frequency of the audio in hertz
        sampletime: The length of a sample in seconds
    """
    frq = get_x_axis(samplerate, sampletime * samplerate)
    bottomindex = _bin_approx_search(frq, freqrange[0])
    topindex = _bin_approx_search(frq, freqrange[1])
    sliced = fftmatrix[:,bottomindex:topindex]
    return numpy.average(sliced, 1)

def isolate_freq_range(fftmatrix, freqrange, samplerate, sampletime):
    """
    Create a moving DFT matrix with only the given frequency range

    Args:
        fftmatrix: The moving_fft() output
        freqrange: A tuple of form (minimum frequency, maximum frequency)
        samplerate: The sample frequency of the audio in hertz
        sampletime: The length of a sample in seconds
    """
    frq = get_x_axis(samplerate, sampletime * samplerate)
    bottomindex = _bin_approx_search(frq, freqrange[0])
    topindex = _bin_approx_search(frq, freqrange[1])
    return fftmatrix[:,bottomindex:topindex]

def extract_freq(fftmatrix, backgroundfreq, targetfreq, samplerate, sampletime):
    """
    Extract occurrences of a specific note of drum by comparing its frequencies
    to background frequencies

    Args:
        fftmatrix: The moving_fft() output
        backgroundfreq: Frequencies to look for background noise
        targetfreq: Frequency range of the instrument
        samplerate: The sample frequency of the audio in hertz
        sampletime: The length of a sample in seconds
    """
    target = freq_range_graph(fftmatrix, targetfreq, samplerate, sampletime)
    background = freq_range_graph(fftmatrix, backgroundfreq, samplerate, sampletime)
    return abs(target)/abs(background)

<|endoftext|>


<commit_before>"""
Import Hackney
"""
from time import sleep

from django.contrib.gis.geos import Point

from data_collection.management.commands import BaseCsvStationsCsvAddressesImporter
from data_finder.helpers import geocode, geocode_point_only, PostcodeError
from addressbase.models import Address

class Command(BaseCsvStationsCsvAddressesImporter):
    """
    Imports the Polling Station data from Hackney Council
    """
    private = True
    council_id      = 'E09000012'
    addresses_name  = 'addresses.csv'
    stations_name   = 'stations.csv'
    csv_delimiter   = ','
    elections       = [
        'ref.2016-06-23'
    ]

    def station_record_to_dict(self, record):
        address = record.situation_of_polling_station
        while "\n\n" in address:
            address = address.replace("\n\n", "\n").strip()

        location = None
        location_data = None
        if record.postcode_if_available:
            try:
                postcode = record.postcode_if_available.strip()
                postcode = postcode.replace('\n', '')
                if len(postcode) > 5:
                    location_data = geocode_point_only(postcode)
            except PostcodeError:
                pass

            if location_data:
                location = Point(
                    location_data['wgs84_lon'],
                    location_data['wgs84_lat'],
                    srid=4326)

        desc = record.description_of_persons_entitled_to_vote
        district = desc.split('-')[0].strip()

        return {
            'internal_council_id': district,
            'polling_district_id': district,
            'postcode'           : None,
            'address'            : address,
            'location'           : location
        }

    def address_record_to_dict(self, record):
        address = record.address

        return {
            'address'           : address,
            'postcode'          : record.postcode.strip(),
            'polling_station_id': record.district
        }
<commit_msg>update Hackney import script for parl.2017-06-08 (closes #847)<commit_after>from data_collection.management.commands import BaseXpressDemocracyClubCsvImporter

class Command(BaseXpressDemocracyClubCsvImporter):
    council_id = 'E09000012'
    addresses_name = 'parl.2017-06-08/Version 1/Democracy_Club__08June2017.CSV'
    stations_name = 'parl.2017-06-08/Version 1/Democracy_Club__08June2017.CSV'
    elections = ['parl.2017-06-08']
<|endoftext|>


<commit_before>from data_collection.management.commands import BaseXpressDCCsvInconsistentPostcodesImporter

class Command(BaseXpressDCCsvInconsistentPostcodesImporter):
    council_id = 'E06000038'
    addresses_name = 'parl.2017-06-08/Version 1/Democracy_Club__08June2017 (17).tsv'
    stations_name = 'parl.2017-06-08/Version 1/Democracy_Club__08June2017 (17).tsv'
    elections = ['parl.2017-06-08']
    csv_delimiter = '\t'
<commit_msg>import script for Reading (closes #1226)<commit_after>from data_collection.management.commands import BaseXpressDCCsvInconsistentPostcodesImporter

class Command(BaseXpressDCCsvInconsistentPostcodesImporter):
    council_id = 'E06000038'
    addresses_name = 'local.2018-05-03/Version 1/Democracy_Club__03May2018 Reading.CSV'
    stations_name = 'local.2018-05-03/Version 1/Democracy_Club__03May2018 Reading.CSV'
    elections = ['local.2018-05-03']
    csv_delimiter = ','

    def address_record_to_dict(self, record):
        if record.property_urn.strip() in ['310082434', '310082435']:
            rec = super().address_record_to_dict(record)
            rec['postcode'] = 'RG2 6AB'
            return rec

        if record.property_urn.strip() == '310074687':
            rec = super().address_record_to_dict(record)
            rec['postcode'] = 'RG1 5NN'
            return rec

        if record.property_urn.strip() == '310080353':
            rec = super().address_record_to_dict(record)
            rec['postcode'] = 'RG6 1DJ'
            return rec

        return super().address_record_to_dict(record)
<|endoftext|>


<commit_before>from __future__ import absolute_import

import autograd.numpy as np
import scipy.stats
from autograd.extend import primitive, defvjp
from autograd.numpy.numpy_vjps import unbroadcast_f
from autograd.scipy.special import gamma

cdf = primitive(scipy.stats.chi2.cdf)
logpdf = primitive(scipy.stats.chi2.logpdf)
pdf = primitive(scipy.stats.chi2.pdf)

def grad_chi2_logpdf(x, df):
    return np.where(df % 1 == 0, (df - x - 2) / (2 * x), 0)

defvjp(cdf, lambda ans, x, df: unbroadcast_f(x, lambda g: g * np.power(2., -df/2) * np.exp(-x/2) * np.power(x, df/2 - 1) / gamma(df/2)), argnums=[0])
defvjp(logpdf, lambda ans, x, df: unbroadcast_f(x, lambda g: g * grad_chi2_logpdf(x, df)), argnums=[0])
defvjp(pdf, lambda ans, x, df: unbroadcast_f(x, lambda g: g * ans * grad_chi2_logpdf(x, df)), argnums=[0])
<commit_msg>Imported __future__.division to avoid integer division in Python 2<commit_after>from __future__ import absolute_import, division

import autograd.numpy as np
import scipy.stats
from autograd.extend import primitive, defvjp
from autograd.numpy.numpy_vjps import unbroadcast_f
from autograd.scipy.special import gamma

cdf = primitive(scipy.stats.chi2.cdf)
logpdf = primitive(scipy.stats.chi2.logpdf)
pdf = primitive(scipy.stats.chi2.pdf)

def grad_chi2_logpdf(x, df):
    return np.where(df % 1 == 0, (df - x - 2) / (2 * x), 0)

defvjp(cdf, lambda ans, x, df: unbroadcast_f(x, lambda g: g * np.power(2., -df/2) * np.exp(-x/2) * np.power(x, df/2 - 1) / gamma(df/2)), argnums=[0])
defvjp(logpdf, lambda ans, x, df: unbroadcast_f(x, lambda g: g * grad_chi2_logpdf(x, df)), argnums=[0])
defvjp(pdf, lambda ans, x, df: unbroadcast_f(x, lambda g: g * ans * grad_chi2_logpdf(x, df)), argnums=[0])
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
from rest_framework import viewsets

from apps.broadcasts.models import Broadcast, Series
from apps.games.models import Game

from .serializers import (BroadcastSerializer, GameSerializer, RaidSerializer,
    SeriesSerializer)

class BroadcastViewSet(viewsets.ModelViewSet):
    queryset = Broadcast.objects.all()
    serializer_class = BroadcastSerializer

class RaidViewSet(viewsets.ModelViewSet):
    queryset = Raid.objects.all()
    serializer_class = RaidSerializer
<commit_msg>Hwhoops.<commit_after># -*- coding: utf-8 -*-
from rest_framework import viewsets

from apps.broadcasts.models import Broadcast, Raid, Series
from apps.games.models import Game

from .serializers import (BroadcastSerializer, GameSerializer, RaidSerializer,
    SeriesSerializer)

class BroadcastViewSet(viewsets.ModelViewSet):
    queryset = Broadcast.objects.all()
    serializer_class = BroadcastSerializer

class RaidViewSet(viewsets.ModelViewSet):
    queryset = Raid.objects.all()
    serializer_class = RaidSerializer
<|endoftext|>


<commit_before>#!/usr/bin/env python

import jsonobject
import model

class Fleet(jsonobject.JSONSerializableObject):

    id = jsonobject.ReadonlyJSONProperty('id', value_type=int)
    """ID."""
    name = jsonobject.ReadonlyJSONProperty('name', value_type=unicode)
    """Name."""
    ship_ids = jsonobject.JSONProperty('ship_ids', value_type=list,
                                       element_type=int)
    """IDs of ships belonging to this fleet."""
    mission_id = jsonobject.JSONProperty('mission_id', value_type=int)
    """ID of mission which this fleet is undertaking."""
    mission_complete = jsonobject.JSONProperty('mission_complete',
                                               value_type=bool)
    """True if the mission is complete. Updated when the screen transitioned to
    PORT_MAIN."""

class FleetList(model.KCAAObject):
    """List of fleets (decks)."""

    fleets = jsonobject.JSONProperty('fleets', [], value_type=list,
                                     element_type=Fleet)
    """Fleets.

    Note that this list has 0-origin, while other objects use 1-origin index to
    reference a fleet."""

    def update(self, api_name, request, response, objects, debug):
        super(FleetList, self).update(api_name, request, response, objects,
                                      debug)
        self.fleets = []
        for data in response.api_data.api_deck_port:
            mission_id = None
            mission_complete = None
            if data.api_mission[0] != 0:
                mission_id = data.api_mission[1]
                # TODO: Fix this. This is not accurate.
                # Probably better to use Mission's eta and the current
                # time?
                mission_complete = (
                    True if data.api_mission[3] == 1 else False)
            self.fleets.append(Fleet(
                id=data.api_id,
                name=data.api_name,
                ship_ids=filter(lambda x: x != -1, data.api_ship),
                mission_id=mission_id,
                mission_complete=mission_complete))
<commit_msg>Get bach the API name check in fleet.py.<commit_after>#!/usr/bin/env python

import jsonobject
import model

class Fleet(jsonobject.JSONSerializableObject):

    id = jsonobject.ReadonlyJSONProperty('id', value_type=int)
    """ID."""
    name = jsonobject.ReadonlyJSONProperty('name', value_type=unicode)
    """Name."""
    ship_ids = jsonobject.JSONProperty('ship_ids', value_type=list,
                                       element_type=int)
    """IDs of ships belonging to this fleet."""
    mission_id = jsonobject.JSONProperty('mission_id', value_type=int)
    """ID of mission which this fleet is undertaking."""
    mission_complete = jsonobject.JSONProperty('mission_complete',
                                               value_type=bool)
    """True if the mission is complete. Updated when the screen transitioned to
    PORT_MAIN."""

class FleetList(model.KCAAObject):
    """List of fleets (decks)."""

    fleets = jsonobject.JSONProperty('fleets', [], value_type=list,
                                     element_type=Fleet)
    """Fleets.

    Note that this list has 0-origin, while other objects use 1-origin index to
    reference a fleet."""

    def update(self, api_name, request, response, objects, debug):
        super(FleetList, self).update(api_name, request, response, objects,
                                      debug)
        if api_name == '/api_port/port':
            self.fleets = []
            for data in response.api_data.api_deck_port:
                mission_id = None
                mission_complete = None
                if data.api_mission[0] != 0:
                    mission_id = data.api_mission[1]
                    # TODO: Fix this. This is not accurate.
                    # Probably better to use Mission's eta and the current
                    # time?
                    mission_complete = (
                        True if data.api_mission[3] == 1 else False)
                self.fleets.append(Fleet(
                    id=data.api_id,
                    name=data.api_name,
                    ship_ids=filter(lambda x: x != -1, data.api_ship),
                    mission_id=mission_id,
                    mission_complete=mission_complete))
<|endoftext|>


<commit_before># coding: utf-8
__author__ = "Marcel Martin"

from collections import namedtuple
import sys
if sys.version_info[0] < 3:
	from itertools import izip as zip
else:
	basestring = str
from cutadapt.xopen import xopen
from os.path import splitext
import sys

Sequence = namedtuple("Sequence", "name sequence qualities")

class FormatError(Exception):
	"""
	Raised when an input file (FASTA or FASTQ) is malformatted.
	"""
	pass

class FileWithPrependedLine(object):
	"""
	A file-like object that allows to "prepend" a single
	line to an already opened file. That is, further
	reads on the file will return the provided line and
	only then the actual content. This is needed to solve
	the problem of autodetecting input from a stream:
	As soon as the first line has been read, we know
	the file type, but also that line is "gone" and
	unavailable for further processing.
	"""
	def __init__(self, file, line):
		"""
		file is an already opened file-like object.
		line is a single string (newline will be appended if not included)
		"""
		if not line.endswith('\n'):
			line += '\n'
		self.first_line = line
		self.file = file

	def __iter__(self):
		yield self.first_line
		for line in self.file:
			yield line

class UnknownFileType(Exception):
	"""
	Raised when SequenceReader could not autodetect the file type.
	"""
	pass

def SequenceReader(file, colorspace=False, fileformat=None):
	"""
	Reader for FASTA and FASTQ files that autodetects the file format.
	Returns either an instance of FastaReader or of FastqReader,
	depending on file type.

	The autodetection can be skipped by setting fileformat to the string
	'fasta' or 'fastq'

	file is a filename or a file-like object.
	If file is a filename, then .gz files are supported.
	If the file name is available, the file type is detected
	by looking at the file name.
	If the file name is not available (for example, reading
	from standard input), then the file is read and the file
	type determined from the content.
	"""
	if fileformat is not None:
		fileformat = fileformat.lower()
		if fileformat == 'fasta':
			return FastaReader(file)
		elif fileformat == 'fastq':
			return FastqReader(file)
		else:
			raise UnknownFileType("File format {} is unknown (expected 'fasta' or 'fastq').".format(fileformat))

	name = None
	if file == "-":
		file = sys.stdin
	elif isinstance(file, basestring):
		name = file
	elif hasattr(file, "name"):
		name = file.name
	if name is not None:
		if name.endswith('.gz'):
			name = name[:-3]
		name, ext = splitext(name)
		ext = ext.lower()
		if ext in ['.fasta', '.fa', '.fna', '.csfasta', '.csfa']:
			return FastaReader(file)
		elif ext in ['.fastq']:
			return FastqReader(file, colorspace)
		else:
			raise UnknownFileType("Could not determine whether this is FASTA or FASTQ: file name extension {} not recognized".format(ext))

	# No name available.
	# Assume that 'file' is an open file
	# and autodetect its type by reading from it.
	for line in file:
		if line.startswith('#'):
			# Skip comment lines (needed for csfasta)
			continue
		if line.startswith('>'):
			return FastaReader(FileWithPrependedLine(file, line))
		if line.startswith('@'):
			return FastqReader(FileWithPrependedLine(file, line), colorspace)
	raise UnknownFileType("File is neither FASTQ nor FASTA.")

class FastaReader(object):
	"""
	Reader for FASTA files.
	"""
	def __init__(self, file, wholefile=False, keep_linebreaks=False):
		"""
		file is a filename or a file-like object.
		If file is a filename, then .gz files are supported.
		If wholefile is True, then it is ok to read the entire file
		into memory. This is faster when there are many newlines in
		the file, but may obviously need a lot of memory.
		keep_linebreaks -- whether to keep the newline characters in the sequence
		"""
		if isinstance(file, basestring):
			file = xopen(file, "r")
		self.fp = file
		self.wholefile = wholefile
		self.keep_linebreaks = keep_linebreaks
		assert not (wholefile and keep_linebreaks), "not supported"

	def __iter__(self):
		"""
		Return instances of the Sequence class.
		The qualities attribute is always None.
		"""
		return self._wholefile_iter() if self.wholefile else self._streaming_iter()

	def _streaming_iter(self):
		"""
		Read next entry from the file (single entry at a time).

		# TODO this can be quadratic since += is used for the sequence string
		"""
		name = None
		seq = ""
		appendchar = '\n' if self.keep_linebreaks else ''
		for line in self.fp:
			# strip() should also take care of DOS line breaks
			line = line.strip()
			if line and line[0] == ">":
				if name is not None:
					assert self.keep_linebreaks or seq.find('\n') == -1
					yield Sequence(name, seq, None)
				name = line[1:]
				seq = ""
			else:
				seq += line + appendchar
		if name is not None:
			assert self.keep_linebreaks or seq.find('\n') == -1
			yield Sequence(name, seq, None)

	def _wholefile_iter(self):
		"""
		This reads in the entire file at once, but is faster than the above code when there are lots of newlines.
		The idea comes from the TAMO package (http://fraenkel.mit.edu/TAMO/), module TAMO.seq.Fasta (author is
		David Benjamin Gordon).
		"""
		wholefile = self.fp.read()
		assert '\r' not in wholefile, "Sorry, currently don't know how to deal with files that contain \\r linebreaks"
		assert len(wholefile) == 0 or wholefile[0] == '>', "FASTA file must start with '>'"
		parts = wholefile.split('\n>')
		# first part has '>' in front
		parts[0] = parts[0][1:]
		for part in parts:
			lines = part.split('\n', 1)
			yield Sequence(name=lines[0], sequence=lines[1].replace('\n', ''), qualities=None)

	def __enter__(self):
		if self.fp is None:
			raise ValueError("I/O operation on closed FastaReader")
		return self

	def __exit__(self, *args):
		self.fp.close()

class FastqReader(object):
	"""
	Reader for FASTQ files. Does not support multi-line FASTQ files.
	"""
	def __init__(self, file, colorspace=False):
		"""
		file is a filename or a file-like object.
		If file is a filename, then .gz files are supported.

		colorspace -- Usually (when this is False), there must be n characters in the sequence and
		n quality values. When this is True, there must be n+1 characters in the sequence and n quality values.
		"""
		if isinstance(file, basestring):
			file = xopen(file, "r")
		self.fp = file
		self.colorspace = colorspace
		self.twoheaders = False

	def __iter__(self):
		"""
		Return tuples: (name, sequence, qualities).
		qualities is a string and it contains the unmodified, encoded qualities.
		"""
		lengthdiff = 1 if self.colorspace else 0
		for i, line in enumerate(self.fp):
			if i % 4 == 0:
				if not line.startswith('@'):
					raise FormatError("at line {}, expected a line starting with '+'".format(i+1))
				name = line.strip()[1:]
			elif i % 4 == 1:
				sequence = line.strip()
			elif i % 4 == 2:
				line = line.strip()
				if not line.startswith('+'):
					raise FormatError("at line {}, expected a line starting with '+'".format(i+1))
				if len(line) > 1:
					self.twoheaders = True
					if not line[1:] == name:
						raise FormatError(
							"At line {}: Two sequence descriptions are given in "
							"the FASTQ file, but they don't match "
							"('{}' != '{}')".format(i+1, name, line.rstrip()[1:]))
			elif i % 4 == 3:
				qualities = line.rstrip("\n\r")
				if len(qualities) + lengthdiff != len(sequence):
					raise ValueError("Length of quality sequence and length of read do not match (%d+%d!=%d)" % (len(qualities), lengthdiff, len(sequence)))
				yield Sequence(name, sequence, qualities)

	def __enter__(self):
		if self.fp is None:
			raise ValueError("I/O operation on closed FastqReader")
		return self

	def __exit__(self, *args):
		self.fp.close()

def _quality_to_ascii(qualities, base=33):
	"""
	Convert a list containing qualities given as integer to a string of
	ASCII-encoded qualities.

	base -- ASCII code of quality zero (sensible values are 33 and 64).

	>>> _quality_to_ascii([17, 4, 29, 18])
	'2%>3'
	"""
	qualities = ''.join(chr(q+base) for q in qualities)
	return qualities

class FastaQualReader(object):
	"""
	Reader for reads that are stored in .(CS)FASTA and .QUAL files.
	"""
	def __init__(self, fastafile, qualfile, colorspace=False):
		"""
		fastafile and qualfile are filenames file-like objects.
		If file is a filename, then .gz files are supported.

		colorspace -- Usually (when this is False), there must be n characters in the sequence and
		n quality values. When this is True, there must be n+1 characters in the sequence and n quality values.
		"""
		self.fastareader = FastaReader(fastafile)
		self.qualreader = FastaReader(qualfile, keep_linebreaks=True)
		self.colorspace = colorspace

	def __iter__(self):
		"""
		Return tuples: (name, sequence, qualities).
		qualities is a string and it contains the qualities encoded as ascii(qual+33).
		"""
		lengthdiff = 1 if self.colorspace else 0
		for fastaread, qualread in zip(self.fastareader, self.qualreader):
			qualities = _quality_to_ascii(map(int, qualread.sequence.split()))
			assert fastaread.name == qualread.name
			if len(qualities) + lengthdiff != len(fastaread.sequence):
				raise ValueError("Length of quality sequence and length of read do not match (%d+%d!=%d)" % (
					len(qualities), lengthdiff, len(fastaread.sequence)))
			yield Sequence(fastaread.name, fastaread.sequence, qualities)

	def __enter__(self):
		if self.fastafile is None:
			raise ValueError("I/O operation on closed FastaQualReader")
		return self

	def __exit__(self, *args):
		self.fastareader.close()
		self.qualreader.close()

################################################

# use this with str.translate to convert encoded quality values from
# ascii(phred_quality + 64) to ascii(phred_quality + 33)
translate_64_to_33 = ''.join(chr(max(c-64+33,0)) for c in range(256))

# benchmarks for the following two functions (reading in chr1 and printing its size):
# the strange timings of the first function are always reproducible on my machine
#readfasta 247249719 2.03
#readfasta 247249719 30.98
#readfasta 247249719 2.01
#readfasta 247249719 30.92
#readfasta 247249719 2.05
#
#readfasta2 247249719 2.13
#readfasta2 247249719 2.11
#readfasta2 247249719 2.13
#readfasta2 247249719 2.11
#readfasta2 247249719 2.13

def writefasta(f, seqlist, linelength=None):
	"""
	Print out a FASTA-formatted file from data given in seqlist.

	seqlist -- iterable over (name, sequence) tuples
	f -- output file
	linelength -- If this isn't None, wrap lines after linelength characters.
	"""
	if linelength is not None:
		for name, seq in seqlist:
			f.write('>')
			f.write(name)
			f.write('\n')
			for i in xrange(0, len(seq), linelength):
				f.write(seq[i:i+linelength])
				f.write('\n')
	else:
		for name, seq in seqlist:
			f.write('>%s\n%s\n' % (name, seq))

def writefastq(f, seqlist, twoheaders=False):
	"""
	seqlist must contain (description, sequence, qualities) tuples

	If twoheaders is True, the sequence name (description) is also written
	after the "+" character.
	"""
	for description, sequence, qualities in seqlist:
		tmp = description if twoheaders else ''
		f.write('@%s\n%s\n+%s\n%s\n' % (description, sequence, tmp, qualities))
<commit_msg>recognize files ending with .fq as FASTQ files<commit_after># coding: utf-8
__author__ = "Marcel Martin"

from collections import namedtuple
import sys
if sys.version_info[0] < 3:
	from itertools import izip as zip
else:
	basestring = str
from cutadapt.xopen import xopen
from os.path import splitext
import sys

Sequence = namedtuple("Sequence", "name sequence qualities")

class FormatError(Exception):
	"""
	Raised when an input file (FASTA or FASTQ) is malformatted.
	"""
	pass

class FileWithPrependedLine(object):
	"""
	A file-like object that allows to "prepend" a single
	line to an already opened file. That is, further
	reads on the file will return the provided line and
	only then the actual content. This is needed to solve
	the problem of autodetecting input from a stream:
	As soon as the first line has been read, we know
	the file type, but also that line is "gone" and
	unavailable for further processing.
	"""
	def __init__(self, file, line):
		"""
		file is an already opened file-like object.
		line is a single string (newline will be appended if not included)
		"""
		if not line.endswith('\n'):
			line += '\n'
		self.first_line = line
		self.file = file

	def __iter__(self):
		yield self.first_line
		for line in self.file:
			yield line

class UnknownFileType(Exception):
	"""
	Raised when SequenceReader could not autodetect the file type.
	"""
	pass

def SequenceReader(file, colorspace=False, fileformat=None):
	"""
	Reader for FASTA and FASTQ files that autodetects the file format.
	Returns either an instance of FastaReader or of FastqReader,
	depending on file type.

	The autodetection can be skipped by setting fileformat to the string
	'fasta' or 'fastq'

	file is a filename or a file-like object.
	If file is a filename, then .gz files are supported.
	If the file name is available, the file type is detected
	by looking at the file name.
	If the file name is not available (for example, reading
	from standard input), then the file is read and the file
	type determined from the content.
	"""
	if fileformat is not None:
		fileformat = fileformat.lower()
		if fileformat == 'fasta':
			return FastaReader(file)
		elif fileformat == 'fastq':
			return FastqReader(file)
		else:
			raise UnknownFileType("File format {} is unknown (expected 'fasta' or 'fastq').".format(fileformat))

	name = None
	if file == "-":
		file = sys.stdin
	elif isinstance(file, basestring):
		name = file
	elif hasattr(file, "name"):
		name = file.name
	if name is not None:
		if name.endswith('.gz'):
			name = name[:-3]
		name, ext = splitext(name)
		ext = ext.lower()
		if ext in ['.fasta', '.fa', '.fna', '.csfasta', '.csfa']:
			return FastaReader(file)
		elif ext in ['.fastq', '.fq']:
			return FastqReader(file, colorspace)
		else:
			raise UnknownFileType("Could not determine whether this is FASTA or FASTQ: file name extension {} not recognized".format(ext))

	# No name available.
	# Assume that 'file' is an open file
	# and autodetect its type by reading from it.
	for line in file:
		if line.startswith('#'):
			# Skip comment lines (needed for csfasta)
			continue
		if line.startswith('>'):
			return FastaReader(FileWithPrependedLine(file, line))
		if line.startswith('@'):
			return FastqReader(FileWithPrependedLine(file, line), colorspace)
	raise UnknownFileType("File is neither FASTQ nor FASTA.")

class FastaReader(object):
	"""
	Reader for FASTA files.
	"""
	def __init__(self, file, wholefile=False, keep_linebreaks=False):
		"""
		file is a filename or a file-like object.
		If file is a filename, then .gz files are supported.
		If wholefile is True, then it is ok to read the entire file
		into memory. This is faster when there are many newlines in
		the file, but may obviously need a lot of memory.
		keep_linebreaks -- whether to keep the newline characters in the sequence
		"""
		if isinstance(file, basestring):
			file = xopen(file, "r")
		self.fp = file
		self.wholefile = wholefile
		self.keep_linebreaks = keep_linebreaks
		assert not (wholefile and keep_linebreaks), "not supported"

	def __iter__(self):
		"""
		Return instances of the Sequence class.
		The qualities attribute is always None.
		"""
		return self._wholefile_iter() if self.wholefile else self._streaming_iter()

	def _streaming_iter(self):
		"""
		Read next entry from the file (single entry at a time).

		# TODO this can be quadratic since += is used for the sequence string
		"""
		name = None
		seq = ""
		appendchar = '\n' if self.keep_linebreaks else ''
		for line in self.fp:
			# strip() should also take care of DOS line breaks
			line = line.strip()
			if line and line[0] == ">":
				if name is not None:
					assert self.keep_linebreaks or seq.find('\n') == -1
					yield Sequence(name, seq, None)
				name = line[1:]
				seq = ""
			else:
				seq += line + appendchar
		if name is not None:
			assert self.keep_linebreaks or seq.find('\n') == -1
			yield Sequence(name, seq, None)

	def _wholefile_iter(self):
		"""
		This reads in the entire file at once, but is faster than the above code when there are lots of newlines.
		The idea comes from the TAMO package (http://fraenkel.mit.edu/TAMO/), module TAMO.seq.Fasta (author is
		David Benjamin Gordon).
		"""
		wholefile = self.fp.read()
		assert '\r' not in wholefile, "Sorry, currently don't know how to deal with files that contain \\r linebreaks"
		assert len(wholefile) == 0 or wholefile[0] == '>', "FASTA file must start with '>'"
		parts = wholefile.split('\n>')
		# first part has '>' in front
		parts[0] = parts[0][1:]
		for part in parts:
			lines = part.split('\n', 1)
			yield Sequence(name=lines[0], sequence=lines[1].replace('\n', ''), qualities=None)

	def __enter__(self):
		if self.fp is None:
			raise ValueError("I/O operation on closed FastaReader")
		return self

	def __exit__(self, *args):
		self.fp.close()

class FastqReader(object):
	"""
	Reader for FASTQ files. Does not support multi-line FASTQ files.
	"""
	def __init__(self, file, colorspace=False):
		"""
		file is a filename or a file-like object.
		If file is a filename, then .gz files are supported.

		colorspace -- Usually (when this is False), there must be n characters in the sequence and
		n quality values. When this is True, there must be n+1 characters in the sequence and n quality values.
		"""
		if isinstance(file, basestring):
			file = xopen(file, "r")
		self.fp = file
		self.colorspace = colorspace
		self.twoheaders = False

	def __iter__(self):
		"""
		Return tuples: (name, sequence, qualities).
		qualities is a string and it contains the unmodified, encoded qualities.
		"""
		lengthdiff = 1 if self.colorspace else 0
		for i, line in enumerate(self.fp):
			if i % 4 == 0:
				if not line.startswith('@'):
					raise FormatError("at line {}, expected a line starting with '+'".format(i+1))
				name = line.strip()[1:]
			elif i % 4 == 1:
				sequence = line.strip()
			elif i % 4 == 2:
				line = line.strip()
				if not line.startswith('+'):
					raise FormatError("at line {}, expected a line starting with '+'".format(i+1))
				if len(line) > 1:
					self.twoheaders = True
					if not line[1:] == name:
						raise FormatError(
							"At line {}: Two sequence descriptions are given in "
							"the FASTQ file, but they don't match "
							"('{}' != '{}')".format(i+1, name, line.rstrip()[1:]))
			elif i % 4 == 3:
				qualities = line.rstrip("\n\r")
				if len(qualities) + lengthdiff != len(sequence):
					raise ValueError("Length of quality sequence and length of read do not match (%d+%d!=%d)" % (len(qualities), lengthdiff, len(sequence)))
				yield Sequence(name, sequence, qualities)

	def __enter__(self):
		if self.fp is None:
			raise ValueError("I/O operation on closed FastqReader")
		return self

	def __exit__(self, *args):
		self.fp.close()

def _quality_to_ascii(qualities, base=33):
	"""
	Convert a list containing qualities given as integer to a string of
	ASCII-encoded qualities.

	base -- ASCII code of quality zero (sensible values are 33 and 64).

	>>> _quality_to_ascii([17, 4, 29, 18])
	'2%>3'
	"""
	qualities = ''.join(chr(q+base) for q in qualities)
	return qualities

class FastaQualReader(object):
	"""
	Reader for reads that are stored in .(CS)FASTA and .QUAL files.
	"""
	def __init__(self, fastafile, qualfile, colorspace=False):
		"""
		fastafile and qualfile are filenames file-like objects.
		If file is a filename, then .gz files are supported.

		colorspace -- Usually (when this is False), there must be n characters in the sequence and
		n quality values. When this is True, there must be n+1 characters in the sequence and n quality values.
		"""
		self.fastareader = FastaReader(fastafile)
		self.qualreader = FastaReader(qualfile, keep_linebreaks=True)
		self.colorspace = colorspace

	def __iter__(self):
		"""
		Return tuples: (name, sequence, qualities).
		qualities is a string and it contains the qualities encoded as ascii(qual+33).
		"""
		lengthdiff = 1 if self.colorspace else 0
		for fastaread, qualread in zip(self.fastareader, self.qualreader):
			qualities = _quality_to_ascii(map(int, qualread.sequence.split()))
			assert fastaread.name == qualread.name
			if len(qualities) + lengthdiff != len(fastaread.sequence):
				raise ValueError("Length of quality sequence and length of read do not match (%d+%d!=%d)" % (
					len(qualities), lengthdiff, len(fastaread.sequence)))
			yield Sequence(fastaread.name, fastaread.sequence, qualities)

	def __enter__(self):
		if self.fastafile is None:
			raise ValueError("I/O operation on closed FastaQualReader")
		return self

	def __exit__(self, *args):
		self.fastareader.close()
		self.qualreader.close()

################################################

# use this with str.translate to convert encoded quality values from
# ascii(phred_quality + 64) to ascii(phred_quality + 33)
translate_64_to_33 = ''.join(chr(max(c-64+33,0)) for c in range(256))

# benchmarks for the following two functions (reading in chr1 and printing its size):
# the strange timings of the first function are always reproducible on my machine
#readfasta 247249719 2.03
#readfasta 247249719 30.98
#readfasta 247249719 2.01
#readfasta 247249719 30.92
#readfasta 247249719 2.05
#
#readfasta2 247249719 2.13
#readfasta2 247249719 2.11
#readfasta2 247249719 2.13
#readfasta2 247249719 2.11
#readfasta2 247249719 2.13

def writefasta(f, seqlist, linelength=None):
	"""
	Print out a FASTA-formatted file from data given in seqlist.

	seqlist -- iterable over (name, sequence) tuples
	f -- output file
	linelength -- If this isn't None, wrap lines after linelength characters.
	"""
	if linelength is not None:
		for name, seq in seqlist:
			f.write('>')
			f.write(name)
			f.write('\n')
			for i in xrange(0, len(seq), linelength):
				f.write(seq[i:i+linelength])
				f.write('\n')
	else:
		for name, seq in seqlist:
			f.write('>%s\n%s\n' % (name, seq))

def writefastq(f, seqlist, twoheaders=False):
	"""
	seqlist must contain (description, sequence, qualities) tuples

	If twoheaders is True, the sequence name (description) is also written
	after the "+" character.
	"""
	for description, sequence, qualities in seqlist:
		tmp = description if twoheaders else ''
		f.write('@%s\n%s\n+%s\n%s\n' % (description, sequence, tmp, qualities))
<|endoftext|>


<commit_before>"""
VHDL Mode for Sublime Text 3

This package attempts to recreate to some level of fidelity the features
in the vhdl-mode in Emacs.
"""
import os
import time
import re
import textwrap
import sublime
import sublime_plugin

from . import vhdl_lang as vhdl
from . import vhdl_util as util

#----------------------------------------------------------------
class vhdlModeVersionCommand(sublime_plugin.TextCommand):
    """
    Prints the version to the console.
    """
    def run(self, edit):
        print("vhdl-mode: VHDL Mode Version 1.7.2")

#----------------------------------------------------------------
class vhdlModeInsertHeaderCommand(sublime_plugin.TextCommand):
    """
    This command is used to insert a predefined header into the
    current text file.
    """
    def run(self, edit):
        # Assigning this to a string to keep command shorter later.
        template = "Packages/VHDL Mode/Snippets/vhdl-header.sublime-snippet"

        # Looking for a name, first the buffer name, then the file name,
        # then finally a default value.
        buffname = self.view.name()
        longname = self.view.file_name()
        if buffname:
            filename = buffname
        elif longname:
            # Convert Windows slashes to Unix slashes (if any)
            longname = re.sub(r'\\', '/', longname)
            namechunks = longname.split('/')
            filename = namechunks[len(namechunks)-1]
        else:
            filename = '<filename>'

        # Get the other fields out of settings.
        project = util.get_vhdl_setting(self, 'vhdl-project-name')
        author = util.get_vhdl_setting(self, 'vhdl-user')
        company = util.get_vhdl_setting(self, 'vhdl-company')
        platform = util.get_vhdl_setting(self, 'vhdl-platform')
        standard = util.get_vhdl_setting(self, 'vhdl-standard')
        mtime_prefix = util.get_vhdl_setting(self, 'vhdl-modified-time-string')
        use_copyright = util.get_vhdl_setting(self, 'vhdl-use-copyright-block')
        use_revision = util.get_vhdl_setting(self, 'vhdl-use-revision-block')
        copyright_list = util.get_vhdl_setting(self, 'vhdl-copyright-block')
        revision_list = util.get_vhdl_setting(self, 'vhdl-revision-block')

        # Get the current time and create the modified time string.
        date = time.ctime(time.time())
        year = time.strftime("%Y",time.localtime())
        mod_time = mtime_prefix + date

        # Create the copyright block and revision block.  Both need
        # prefixed newlines because they are optional and the
        # snippet field is at the end of the preceding line.
        if use_copyright:
            copyright = '\n'.join(copyright_list)
            copyright = re.sub(r'\${YEAR}', year, copyright)
            copyright = re.sub(r'\${COMPANY}', company, copyright)
            copyright = '\n' + copyright
        else:
            copyright = ''
        if use_revision:
            revision = '\n'.join(revision_list)
            revision = '\n' + revision
        else:
            revision = ''

        # Moving insertion point to the beginning of the file.
        bof = self.view.text_point(0,0)
        self.view.sel().clear()
        self.view.sel().add(sublime.Region(bof))
        self.view.show(bof)

        # Inserting template/snippet
        self.view.run_command("insert_snippet",
            {
                "name"     : template,
                "PROJECT"  : project,
                "FILENAME" : filename,
                "AUTHOR"   : author,
                "COMPANY"  : company,
                "CDATE"    : date,
                "MODIFIED_TIME_STRING" : mod_time,
                "MDATE"    : date,
                "YEAR"     : year,
                "PLATFORM" : platform,
                "STANDARD" : standard,
                "COPYRIGHT_BLOCK" : copyright,
                "REVISION_BLOCK" : revision
            })
        print('vhdl-mode: Inserted header template.')

#----------------------------------------------------------------
class vhdlModeToggleCommentRegionCommand(sublime_plugin.TextCommand):
    """
    The command analyzes the block delivered to the command
    and attempts to find the leftmost point and uses that for
    the location of the commenting characters so that it provides
    an even margin and eases removal later.

    If the starting line of the region begins with a comment,
    the command attempts to remove the comment from that and
    each subsequent line.
    """
    def run(self, edit):
        # This could theoretically run on multiple regions but
        # it's not a likely application and I'm just going to
        # worry about the first one for now.
        region = self.view.sel()[0]
        # The line method when applied to a region says it
        # returns a new region that is blocked to the
        # beginning of the line and the end of the line.
        # Exactly what I want, so let's try it.
        region = self.view.line(region)
        block = self.view.substr(region)
        lines = block.split('\n')

        # Setting the value to an absurd value for
        # comparison.  Search for the first non-
        # whitespace character to determine the
        # left margin.
        margin = 1000
        for line in lines:
            s = re.search(r'\S', line)
            if s:
                if s.start() < margin:
                    margin = s.start()

        # Check for comment on first line.  This
        # determines if we're commenting or
        # uncommenting.
        comment = True
        s = re.search(r'^\s*--', lines[0])
        if s:
            comment = False

        # Process lines.
        for index, line in enumerate(lines):
            if comment:
                lines[index] = lines[index][0:margin] + \
                               '--' + \
                               lines[index][margin:]
            else:
                # Assuming this is a commented block, we replace
                # only the first comment designator.  Weird things
                # will happen if there are uncommented lines in the
                # block and there's also inline comments.
                lines[index] = re.sub('--', '', lines[index], 1)

        # Put together into big string
        block = '\n'.join(lines)
        #print(block)
        # Replace the current region with the new region
        self.view.replace(edit, region, block)

#----------------------------------------------------------------
class vhdlModeBeautifyBufferCommand(sublime_plugin.TextCommand):
    """
    This is a Sublime Text variation of the standalone beautify
    code program.  Sets the region to the entire buffer, obtains
    the lines, then processes them and writes them back.
    """
    def run(self, edit):
        # Save original point, and convert to row col.  Beautify
        # will change the number of characters in the file, so
        # need coordinates to know where to go back to.
        original_region = self.view.sel()[0]
        original_point = original_region.begin()
        orig_x, orig_y = self.view.rowcol(original_point)

        # Create points for a region that define beginning and end.
        begin = 0
        end = self.view.size()-1

        # Slurp up entire buffer
        whole_region = sublime.Region(begin, end)
        buffer_str = self.view.substr(whole_region)
        lines = buffer_str.split('\n')

        # Get the scope for column 0 of each line.
        point = 0
        scope_list = []
        while not util.is_end_line(self, point):
            scope_list.append(self.view.scope_name(point))
            point = util.move_down(self, point)
        scope_list.append(self.view.scope_name(point))

        # Process each line
        # Left justify
        vhdl.left_justify(lines)

        # Because there are some really terrible typists out there
        # I end up having to MAKE SURE that symbols like : := <= and =>
        # have spaces to either side of them.  I'm just going to wholesale
        # replace them all with padded versions and then I remove extra space
        # later, which seems wasteful, but easier than trying to come up with
        # a ton of complicated patterns.
        vhdl.pad_vhdl_symbols(lines)

        # Remove extra blank space and convert tabs to spaces
        vhdl.remove_extra_space(lines)

        # Align
        print('vhdl-mode: Aligning symbols.')
        vhdl.align_block_on_re(lines=lines, regexp=r':(?!=)', scope_data=scope_list)
        vhdl.align_block_on_re(
            lines=lines,
            regexp=r':(?!=)\s?(?:in\b|out\b|inout\b|buffer\b)?\s*',
            padside='post',
            scope_data=scope_list)
        vhdl.align_block_on_re(lines=lines, regexp=r'<|:(?==)', scope_data=scope_list)
        vhdl.align_block_on_re(lines=lines, regexp=r'=>', scope_data=scope_list)

        # Indent!  Get some settings first.
        use_spaces = util.get_vhdl_setting(self, 'translate_tabs_to_spaces')
        tab_size = util.get_vhdl_setting(self, 'tab_size')
        print('vhdl-mode: Indenting.')
        vhdl.indent_vhdl(lines=lines, initial=0, tab_size=tab_size,
                         use_spaces=use_spaces)

        # Post indent alignment
        vhdl.align_block_on_re(lines=lines, regexp=r'\bwhen\b', scope_data=scope_list)

        # Recombine into one big blobbed string.
        buffer_str = '\n'.join(lines)

        # Annnd if all went well, write it back into the buffer
        self.view.replace(edit, whole_region, buffer_str)

        # Put cursor back to original point (roughly)
        original_point = self.view.text_point(orig_x, orig_y)
        util.set_cursor(self, original_point)

#----------------------------------------------------------------
class vhdlModeUpdateLastUpdatedCommand(sublime_plugin.TextCommand):
    """
    Finds the last updated field in the header and updates the time
    in the field.
    """
    def run(self, edit):
        """Sublime Text plugin run method."""
        # Note, if one changes the header, this might need to change too.
        pattern = util.get_vhdl_setting(self, 'vhdl-modified-time-string')
        region = self.view.find(pattern, 0)
        #print('Region Diagnostics')
        #print('------------------')
        #print('Begin: {}'.format(region.begin()))
        #print('End:   {}'.format(region.end()))
        #print('Empty? {}'.format(region.empty()))
        if not region.empty():
            region = self.view.line(region)
            date = time.ctime(time.time())
            new_mtime = pattern + '{}'.format(date)
            self.view.replace(edit, region, new_mtime)
            print('vhdl-mode: Updated last modified time.')
        else:
            print('vhdl-mode: No last modified time field found.')

#----------------------------------------------------------------
class vhdlModeUpdateModifiedTimeOnSave(sublime_plugin.EventListener):
    """
    Watches for a save event and updates the Last update
    field in the header.
    """
    def on_pre_save(self, view):
        """
        Gets passed the view that is being saved and scans for the
        Last updated field.
        """
        # MUST CHECK FOR VHDL FILE TYPE (otherwise it
        # starts executing on this very source file which
        # is problematic!)
        if util.is_vhdl_file(view.scope_name(0)):
            view.run_command("vhdl_mode_update_last_updated")

#----------------------------------------------------------------
class vhdlModeScopeSnifferCommand(sublime_plugin.TextCommand):
    """
    My own scope sniffing command that prints to
    console instead of a popup window.
    """
    def run(self, edit):
        """ST3 Run Method"""
        region = self.view.sel()[0]
        sniff_point = region.begin()
        print('vhdl-mode: {}'.format(self.view.scope_name(sniff_point)))

#----------------------------------------------------------------
class vhdlModeInsertCommentLine(sublime_plugin.TextCommand):
    """
    This should insert a line out to the margine (80 characters)
    starting where the point is.  This is intended to run after
    the user types '---' (see keybindings)
    """
    def run(self, edit):
        """Standard TextCommand Run method"""
        # Get the current point.
        region = self.view.sel()[0]
        original_point = region.begin()
        point_r, point_c = self.view.rowcol(original_point)
        # Figure out if any tab characters were used.
        line = self.view.substr(self.view.line(original_point))
        numtabs = line.count('\t')
        # Get the current tab size
        tabsize = util.get_vhdl_setting(self, 'tab_size')
        # Create string of correct amount of dashes.  A tab consumed
        # one character but generates tabsize-1 space.
        line = '-'*(80-point_c-(tabsize-1)*numtabs)
        num_chars = self.view.insert(edit, original_point, line)
        print('vhdl-mode: Inserted comment line.')

#----------------------------------------------------------------
class vhdlModeInsertCommentBox(sublime_plugin.TextCommand):
    """
    This should insert a box out to the margin (80 characters)
    starting where the point is, and taking into account tabs.
    This is intended to run after the user types '----' (see
    keybindings)
    """
    def run(self, edit):
        """Standard TextCommand Run method"""
        # Get the current point.
        region = self.view.sel()[0]
        original_point = region.begin()
        point_r, point_c = self.view.rowcol(original_point)
        # Figure out if any tab characters were used.
        line = self.view.substr(self.view.line(original_point))
        numtabs = line.count('\t')
        # Get the current tab size
        tabsize = util.get_vhdl_setting(self, 'tab_size')
        # Create string of correct amount of dashes.  A tab consumed
        # one character but generates tabsize-1 space.
        line = '-'*(80-point_c-(tabsize-1)*numtabs)
        # Create snippet object.
        snippet = line + '\n' + '-- $0' + '\n' + line + '\n'
        # Inserting template/snippet
        self.view.run_command("insert_snippet",
            {
                "contents" : snippet
            })

#----------------------------------------------------------------
class vhdlModeSettingSniffer(sublime_plugin.TextCommand):
    '''
    Creating a command to check settings in various
    contexts
    '''
    def run(self, edit):
        '''
        Standard TextCommand Run Method
        '''
        print('Preference Settings')
        print('vhdl-mode: {}: {}'.format('tab_size', util.get_vhdl_setting(self, 'tab_size')))
        print('vhdl-mode: {}: {}'.format('translate_tabs_to_spaces', util.get_vhdl_setting(self, 'translate_tabs_to_spaces')))
        vhdl_settings = sublime.load_settings('vhdl_mode.sublime-settings')
        keys = ['vhdl-user',
                'vhdl-company',
                'vhdl-project-name',
                'vhdl-platform',
                'vhdl-standard',
                'vhdl-modified-time-string',
                'vhdl-use-copyright-block',
                'vhdl-use-revision-block',
                'vhdl-copyright-block',
                'vhdl-revision-block']
        print('Package Settings')
        for key in keys:
            print('vhdl-mode: {}: "{}"'.format(key, vhdl_settings.get(key)))

        print('View Settings')
        for key in keys:
            print('vhdl-mode: {}: {}'.format(key, util.get_vhdl_setting(self, key)))

<commit_msg>Documentation update<commit_after>"""
VHDL Mode for Sublime Text 3

This package attempts to recreate to some level of fidelity the features
in the vhdl-mode in Emacs.
"""
import os
import time
import re
import textwrap
import sublime
import sublime_plugin

from . import vhdl_lang as vhdl
from . import vhdl_util as util

#----------------------------------------------------------------
class vhdlModeVersionCommand(sublime_plugin.TextCommand):
    """
    Prints the version to the console.
    """
    def run(self, edit):
        print("vhdl-mode: VHDL Mode Version 1.7.3")

#----------------------------------------------------------------
class vhdlModeInsertHeaderCommand(sublime_plugin.TextCommand):
    """
    This command is used to insert a predefined header into the
    current text file.
    """
    def run(self, edit):
        # Assigning this to a string to keep command shorter later.
        template = "Packages/VHDL Mode/Snippets/vhdl-header.sublime-snippet"

        # Looking for a name, first the buffer name, then the file name,
        # then finally a default value.
        buffname = self.view.name()
        longname = self.view.file_name()
        if buffname:
            filename = buffname
        elif longname:
            # Convert Windows slashes to Unix slashes (if any)
            longname = re.sub(r'\\', '/', longname)
            namechunks = longname.split('/')
            filename = namechunks[len(namechunks)-1]
        else:
            filename = '<filename>'

        # Get the other fields out of settings.
        project = util.get_vhdl_setting(self, 'vhdl-project-name')
        author = util.get_vhdl_setting(self, 'vhdl-user')
        company = util.get_vhdl_setting(self, 'vhdl-company')
        platform = util.get_vhdl_setting(self, 'vhdl-platform')
        standard = util.get_vhdl_setting(self, 'vhdl-standard')
        mtime_prefix = util.get_vhdl_setting(self, 'vhdl-modified-time-string')
        use_copyright = util.get_vhdl_setting(self, 'vhdl-use-copyright-block')
        use_revision = util.get_vhdl_setting(self, 'vhdl-use-revision-block')
        copyright_list = util.get_vhdl_setting(self, 'vhdl-copyright-block')
        revision_list = util.get_vhdl_setting(self, 'vhdl-revision-block')

        # Get the current time and create the modified time string.
        date = time.ctime(time.time())
        year = time.strftime("%Y",time.localtime())
        mod_time = mtime_prefix + date

        # Create the copyright block and revision block.  Both need
        # prefixed newlines because they are optional and the
        # snippet field is at the end of the preceding line.
        if use_copyright:
            copyright = '\n'.join(copyright_list)
            copyright = re.sub(r'\${YEAR}', year, copyright)
            copyright = re.sub(r'\${COMPANY}', company, copyright)
            copyright = '\n' + copyright
        else:
            copyright = ''
        if use_revision:
            revision = '\n'.join(revision_list)
            revision = '\n' + revision
        else:
            revision = ''

        # Moving insertion point to the beginning of the file.
        bof = self.view.text_point(0,0)
        self.view.sel().clear()
        self.view.sel().add(sublime.Region(bof))
        self.view.show(bof)

        # Inserting template/snippet
        self.view.run_command("insert_snippet",
            {
                "name"     : template,
                "PROJECT"  : project,
                "FILENAME" : filename,
                "AUTHOR"   : author,
                "COMPANY"  : company,
                "CDATE"    : date,
                "MODIFIED_TIME_STRING" : mod_time,
                "MDATE"    : date,
                "YEAR"     : year,
                "PLATFORM" : platform,
                "STANDARD" : standard,
                "COPYRIGHT_BLOCK" : copyright,
                "REVISION_BLOCK" : revision
            })
        print('vhdl-mode: Inserted header template.')

#----------------------------------------------------------------
class vhdlModeToggleCommentRegionCommand(sublime_plugin.TextCommand):
    """
    The command analyzes the block delivered to the command
    and attempts to find the leftmost point and uses that for
    the location of the commenting characters so that it provides
    an even margin and eases removal later.

    If the starting line of the region begins with a comment,
    the command attempts to remove the comment from that and
    each subsequent line.
    """
    def run(self, edit):
        # This could theoretically run on multiple regions but
        # it's not a likely application and I'm just going to
        # worry about the first one for now.
        region = self.view.sel()[0]
        # The line method when applied to a region says it
        # returns a new region that is blocked to the
        # beginning of the line and the end of the line.
        # Exactly what I want, so let's try it.
        region = self.view.line(region)
        block = self.view.substr(region)
        lines = block.split('\n')

        # Setting the value to an absurd value for
        # comparison.  Search for the first non-
        # whitespace character to determine the
        # left margin.
        margin = 1000
        for line in lines:
            s = re.search(r'\S', line)
            if s:
                if s.start() < margin:
                    margin = s.start()

        # Check for comment on first line.  This
        # determines if we're commenting or
        # uncommenting.
        comment = True
        s = re.search(r'^\s*--', lines[0])
        if s:
            comment = False

        # Process lines.
        for index, line in enumerate(lines):
            if comment:
                lines[index] = lines[index][0:margin] + \
                               '--' + \
                               lines[index][margin:]
            else:
                # Assuming this is a commented block, we replace
                # only the first comment designator.  Weird things
                # will happen if there are uncommented lines in the
                # block and there's also inline comments.
                lines[index] = re.sub('--', '', lines[index], 1)

        # Put together into big string
        block = '\n'.join(lines)
        #print(block)
        # Replace the current region with the new region
        self.view.replace(edit, region, block)

#----------------------------------------------------------------
class vhdlModeBeautifyBufferCommand(sublime_plugin.TextCommand):
    """
    This is a Sublime Text variation of the standalone beautify
    code program.  Sets the region to the entire buffer, obtains
    the lines, then processes them and writes them back.
    """
    def run(self, edit):
        # Save original point, and convert to row col.  Beautify
        # will change the number of characters in the file, so
        # need coordinates to know where to go back to.
        original_region = self.view.sel()[0]
        original_point = original_region.begin()
        orig_x, orig_y = self.view.rowcol(original_point)

        # Create points for a region that define beginning and end.
        begin = 0
        end = self.view.size()-1

        # Slurp up entire buffer
        whole_region = sublime.Region(begin, end)
        buffer_str = self.view.substr(whole_region)
        lines = buffer_str.split('\n')

        # Get the scope for column 0 of each line.
        point = 0
        scope_list = []
        while not util.is_end_line(self, point):
            scope_list.append(self.view.scope_name(point))
            point = util.move_down(self, point)
        scope_list.append(self.view.scope_name(point))

        # Process each line
        # Left justify
        vhdl.left_justify(lines)

        # Because there are some really terrible typists out there
        # I end up having to MAKE SURE that symbols like : := <= and =>
        # have spaces to either side of them.  I'm just going to wholesale
        # replace them all with padded versions and then I remove extra space
        # later, which seems wasteful, but easier than trying to come up with
        # a ton of complicated patterns.
        vhdl.pad_vhdl_symbols(lines)

        # Remove extra blank space and convert tabs to spaces
        vhdl.remove_extra_space(lines)

        # Align
        print('vhdl-mode: Aligning symbols.')
        vhdl.align_block_on_re(lines=lines, regexp=r':(?!=)', scope_data=scope_list)
        vhdl.align_block_on_re(
            lines=lines,
            regexp=r':(?!=)\s?(?:in\b|out\b|inout\b|buffer\b)?\s*',
            padside='post',
            scope_data=scope_list)
        vhdl.align_block_on_re(lines=lines, regexp=r'<|:(?==)', scope_data=scope_list)
        vhdl.align_block_on_re(lines=lines, regexp=r'=>', scope_data=scope_list)

        # Indent!  Get some settings first.
        use_spaces = util.get_vhdl_setting(self, 'translate_tabs_to_spaces')
        tab_size = util.get_vhdl_setting(self, 'tab_size')
        print('vhdl-mode: Indenting.')
        vhdl.indent_vhdl(lines=lines, initial=0, tab_size=tab_size,
                         use_spaces=use_spaces)

        # Post indent alignment
        vhdl.align_block_on_re(lines=lines, regexp=r'\bwhen\b', scope_data=scope_list)

        # Recombine into one big blobbed string.
        buffer_str = '\n'.join(lines)

        # Annnd if all went well, write it back into the buffer
        self.view.replace(edit, whole_region, buffer_str)

        # Put cursor back to original point (roughly)
        original_point = self.view.text_point(orig_x, orig_y)
        util.set_cursor(self, original_point)

#----------------------------------------------------------------
class vhdlModeUpdateLastUpdatedCommand(sublime_plugin.TextCommand):
    """
    Finds the last updated field in the header and updates the time
    in the field.
    """
    def run(self, edit):
        """Sublime Text plugin run method."""
        # Note, if one changes the header, this might need to change too.
        pattern = util.get_vhdl_setting(self, 'vhdl-modified-time-string')
        region = self.view.find(pattern, 0)
        #print('Region Diagnostics')
        #print('------------------')
        #print('Begin: {}'.format(region.begin()))
        #print('End:   {}'.format(region.end()))
        #print('Empty? {}'.format(region.empty()))
        if not region.empty():
            region = self.view.line(region)
            date = time.ctime(time.time())
            new_mtime = pattern + '{}'.format(date)
            self.view.replace(edit, region, new_mtime)
            print('vhdl-mode: Updated last modified time.')
        else:
            print('vhdl-mode: No last modified time field found.')

#----------------------------------------------------------------
class vhdlModeUpdateModifiedTimeOnSave(sublime_plugin.EventListener):
    """
    Watches for a save event and updates the Last update
    field in the header.
    """
    def on_pre_save(self, view):
        """
        Gets passed the view that is being saved and scans for the
        Last updated field.
        """
        # MUST CHECK FOR VHDL FILE TYPE (otherwise it
        # starts executing on this very source file which
        # is problematic!)
        if util.is_vhdl_file(view.scope_name(0)):
            view.run_command("vhdl_mode_update_last_updated")

#----------------------------------------------------------------
class vhdlModeScopeSnifferCommand(sublime_plugin.TextCommand):
    """
    My own scope sniffing command that prints to
    console instead of a popup window.
    """
    def run(self, edit):
        """ST3 Run Method"""
        region = self.view.sel()[0]
        sniff_point = region.begin()
        print('vhdl-mode: {}'.format(self.view.scope_name(sniff_point)))

#----------------------------------------------------------------
class vhdlModeInsertCommentLine(sublime_plugin.TextCommand):
    """
    This should insert a line out to the margine (80 characters)
    starting where the point is.  This is intended to run after
    the user types '---' (see keybindings)
    """
    def run(self, edit):
        """Standard TextCommand Run method"""
        # Get the current point.
        region = self.view.sel()[0]
        original_point = region.begin()
        point_r, point_c = self.view.rowcol(original_point)
        # Figure out if any tab characters were used.
        line = self.view.substr(self.view.line(original_point))
        numtabs = line.count('\t')
        # Get the current tab size
        tabsize = util.get_vhdl_setting(self, 'tab_size')
        # Create string of correct amount of dashes.  A tab consumed
        # one character but generates tabsize-1 space.
        line = '-'*(80-point_c-(tabsize-1)*numtabs)
        num_chars = self.view.insert(edit, original_point, line)
        print('vhdl-mode: Inserted comment line.')

#----------------------------------------------------------------
class vhdlModeInsertCommentBox(sublime_plugin.TextCommand):
    """
    This should insert a box out to the margin (80 characters)
    starting where the point is, and taking into account tabs.
    This is intended to run after the user types '----' (see
    keybindings)
    """
    def run(self, edit):
        """Standard TextCommand Run method"""
        # Get the current point.
        region = self.view.sel()[0]
        original_point = region.begin()
        point_r, point_c = self.view.rowcol(original_point)
        # Figure out if any tab characters were used.
        line = self.view.substr(self.view.line(original_point))
        numtabs = line.count('\t')
        # Get the current tab size
        tabsize = util.get_vhdl_setting(self, 'tab_size')
        # Create string of correct amount of dashes.  A tab consumed
        # one character but generates tabsize-1 space.
        line = '-'*(80-point_c-(tabsize-1)*numtabs)
        # Create snippet object.
        snippet = line + '\n' + '-- $0' + '\n' + line + '\n'
        # Inserting template/snippet
        self.view.run_command("insert_snippet",
            {
                "contents" : snippet
            })

#----------------------------------------------------------------
class vhdlModeSettingSniffer(sublime_plugin.TextCommand):
    '''
    Creating a command to check settings in various
    contexts
    '''
    def run(self, edit):
        '''
        Standard TextCommand Run Method
        '''
        print('Preference Settings')
        print('vhdl-mode: {}: {}'.format('tab_size', util.get_vhdl_setting(self, 'tab_size')))
        print('vhdl-mode: {}: {}'.format('translate_tabs_to_spaces', util.get_vhdl_setting(self, 'translate_tabs_to_spaces')))
        vhdl_settings = sublime.load_settings('vhdl_mode.sublime-settings')
        keys = ['vhdl-user',
                'vhdl-company',
                'vhdl-project-name',
                'vhdl-platform',
                'vhdl-standard',
                'vhdl-modified-time-string',
                'vhdl-use-copyright-block',
                'vhdl-use-revision-block',
                'vhdl-copyright-block',
                'vhdl-revision-block']
        print('Package Settings')
        for key in keys:
            print('vhdl-mode: {}: "{}"'.format(key, vhdl_settings.get(key)))

        print('View Settings')
        for key in keys:
            print('vhdl-mode: {}: {}'.format(key, util.get_vhdl_setting(self, key)))

<|endoftext|>