<commit_before>import re, hashlib, pprint
from collections import Counter
from html.parser import HTMLParser
from classes.fingerprints import Fingerprints
from classes.requester2 import Requester
from classes.matcher import Match
from classes.request import PageFetcher, Request

class DiscoverRedirect(object):

	def __init__(self, options):
		self.org = options['host']
		self.url = options['host']

		fetcher = PageFetcher(self.url)
		try:
			r = fetcher.get()
			request_url = r.get_url()

			if not request_url == self.url:
				# ensure that folders and files are removed
				parts = request_url.split('//')
				http, netloc = parts[0:2]

				# remove subfolders and/or files
				# http://example.com/test -> http://example.com/
				if '/' in netloc:
					self.url = http + '//' + netloc.split('/')[0] + '/'
				else:
					self.url = http + '//' + netloc + '/'
		except:
			self.url = None

	# check if the host redirects to another location
	def is_redirected(self):
		return not self.org == self.url

	# return a cleaned URL
	def get_valid_url(self):
		return self.url

class DiscoverErrorPage(object):
	# find error pages on the site
	# the requester has a built-in list of items and patterns
	# to remove before calculating a checksum of pages that
	# should not exists

	def __init__(self, options, data):
		self.host = options['host']
		self.urls = data['fingerprints'].get_error_urls()
		self.error_pages = set()
		self.requester = data['requester']

	def run(self):
		urls = [ [{'host': self.host, 'url': u}] for u in self.urls ]

		self.requester.set_fingerprints(urls)
		self.requester.set_find_404(True)

		results = self.requester.run()
		while results.qsize() > 0:
			response = results.get()
			self.error_pages.add(response)

		self.requester.set_find_404(False)

	def get_error_pages(self):
		return self.error_pages

class DiscoverCMS(object):

	def __init__(self, options, data):
		self.chunk_size = options['chunk_size']
		self.matcher = data['matcher']
		self.requester = data['requester']
		self.fps = data['fingerprints'].get_ordered_list()
		self.fps_iter = iter(self.fps)
		self.index = 0

	def is_done(self):
		return self.index >= len(self.fps)

	def run(self):
		i = self.index
		cs = self.chunk_size

		# extract a chunck
		chunk = self.fps[i:i+cs]
		self.index += cs

		self.requester.set_fingerprints(chunk)
		results = self.requester.run()

		# process the results and find matches
		while results.qsize() > 0:
			fps,response = results.get()
			matches = self.matcher.get_result(fps, response)
			if matches:
				return [cms['cms'] for cms in matches]

		return []

class DiscoverVersion(object):
	def __init__(self, options, data):
		self.chunk_size = options['chunk_size']
		self.result = data['results']
		self.matcher = data['matcher']
		self.requester = data['requester']
		self.fingerprints = data['fingerprints']

	def run(self, cms):
		cs = self.chunk_size
		fps = self.fingerprints.get_fingerprints_for_cms(cms)

		for i in range(0, len(fps), cs):
			chunk = fps[i:i+cs]

			self.requester.set_fingerprints(chunk)
			results = self.requester.run()

			while results.qsize() > 0:
				res_fps,response = results.get()
				for fp in self.matcher.get_result(res_fps, response):
					self.result.add_cms(fp)

class DiscoverOS(object):
	def __init__(self, data):
		self.cache = data['cache']
		self.results = data['results']
		self.fingerprints = data['fingerprints'].get_os_fingerprints()

		self.category = "Operating System"
		self.os = Counter()
		self.packages = Counter()
		self.oss = []
		self.matched_packages = set()

	def find_match(self, response):
		headers = response.headers
		if 'server' in headers:
			line = headers['server']
			if "(" in line:
				os = line[line.find('(')+1:line.find(')')]
				line = line[:line.find('(')-1] + line[line.find(')')+1: ]
			else:
				os = False

			if os: self.oss.append(os.lower())

			for part in line.split(" "):
				try:
					pkg,version = list(map(str.lower, part.split('/')))
					self.packages[pkg] += 1

					os_list = self.fingerprints[pkg][version]

					for i in os_list:
						if len(i) == 2:
							os, os_version = i
							weight = 1
						elif len(i) == 3:
							os, os_version, weight = i

						self.matched_packages.add( (os, os_version, pkg, version) )
						self.os[(os, os_version)] += weight

				except Exception as e:
					continue

	def finalize(self):
		# this might not be stable for items other than php at the moment
		# recheck this process
		# changed in order to eliminate the 'X-Powered-by' check here
		platforms = self.results.get_platform_results()
		for pkg in platforms:
			for version in platforms[pkg]:
				score = platforms[pkg][version]
				try:
					for i in self.fingerprints[pkg.lower()][version]:
						if len(i) == 2:
							os, os_version = i
							weight = 1
						elif len(i) == 3:
							os, os_version, weight = i

						self.matched_packages.add( (os, os_version, pkg, version) )
						self.os[(os, os_version)] += score*weight

				except Exception as e:
					pass

		# if an os string 'self.oss' has been found in the header
		# prioritize the identified os's in self.os

		# iterate over the list of os strings found
		for os in self.oss:
			# iterate over the fingerprinted os's
			for key in self.os:
				if os in key[0].lower():
					self.os[key] += 100

		# add OS to results: self.os: {(os, version): weight, ...}
		results = []
		for p in self.os:
			results.append({'version': p[1], 'os': p[0], 'count': self.os[p]})

		if len(results) == 0: return

		prio = sorted(results, key=lambda x:x['count'], reverse=True)
		max_count = prio[0]['count']
		relevant = []
		for i in prio:
			if i['count'] == max_count:
				if len(relevant) > 0  and i[0] == "": continue
				self.results.add(self.category, i['os'], i['version'], weight=i['count'])
			else:
				break

	def run(self):
		headers = set()
		responses = self.cache.get_responses()
		for response in responses:
			self.find_match(response)

		self.finalize()

# Used by the DiscoverMore crawler
# The
class LinkExtractor(HTMLParser):
	def __init__(self, strict):
		super().__init__(strict=strict)
		self.results = set()

	def get_results(self):
		return self.results

	def handle_starttag(self, tag, attrs):
		try:
			url = ''
			if tag == 'script' or tag == 'img':
				for attr in attrs:
					if attr[0] == 'src':  self.results.add(attr[1])
			if tag == 'link':
				for attr in attrs:
					if attr[0] == 'href': self.results.add(attr[1])
		except:
			pass

class DiscoverMore(object):

	def __init__(self, options, data):
		self.host = options['host']
		self.threads = options['threads']
		self.cache = data['cache']
		self.result = data['results']
		self.matcher = data['matcher']
		self.requester = data['requester']
		self.fingerprints = data['fingerprints']

	def _get_urls(self, response):
		# only get urls from elements that use 'src' to avoid
		# fetching resources provided by <a>-tags, as this could
		# lead to the crawling of the whole application
		regexes = [ 'src="(.+?)"', "src='(.+?)'"]

		urls = set()
		for regex in regexes:
			for match in re.findall(regex, response.body):
				urls.add( match )

		return urls

	def run(self):
		resources = set()
		parser = LinkExtractor(strict=False)

		for req in self.cache.get_responses():
			# skip pages that do not set 'content-type'
			# these might be binaries
			if not 'content-type' in req.headers:
				continue

			# only scrape pages that can contain links/references
			if 'text/html' in req.headers['content-type']:
				tmp = self._get_urls(req)

				parser.feed(req.body)
				tmp = tmp.union( parser.get_results())

				for i in tmp:

					# ensure that only resources located on the domain /sub-domain is requested
					if i.startswith('http') or i.startswith('//'):
						parts = i.split('/')
						host = parts[2]

						# if the resource is out side of the domain, skip it
						if not host in self.host.split('/')[2]:
							continue

						# else update the url so that it only contains the relative location
						else:
							i = '/'.join(parts[3:])

					resources.add( i )

		# the items in the resource set should mimic a list of fingerprints:
		# a fingerprint is a dict with at least an URL key
		urls = [ [{'url':i}] for i in resources ]

		# fetch the discovered resources.
		# As this class' purpose only is to fetch the resource (add to cache)
		# there is no return value, or further actions needed
		for i in range(0, len(urls), self.threads):
			self.requester.set_fingerprints( urls[i:i+self.threads] )
			results = self.requester.run()

class DiscoverAllCMS(object):
	# match all fingerprints against all responses
	# this might generate false positives

	def __init__(self, data):
		self.cache = data['cache']
		self.fps = data['fingerprints'].get_all()
		self.results = data['results']
		self.matcher = data['matcher']

	def run(self):
		# find matches for all the responses in the cache
		for response in self.cache.get_responses():
			matches = self.matcher.get_result(self.fps, response)
			for fp in matches:
				self.results.add_cms(fp)

class DiscoverJavaScript(object):
	def __init__(self, data):
		self.cache = data['cache']
		self.fingerprints = data['fingerprints'].get_js_fingerprints()
		self.matcher = data['matcher']
		self.result = data['results']

	def run(self):
		for response in self.cache.get_responses():

			# match only if the response is JavaScript
			#  check content type
			content_type = response.headers['content-type'] if 'content-type' in response.headers else ''
			# and extension
			is_js = 'javascript' in content_type or '.js' in response.url.split('.')[-1]

			# if the response is JavaScript try to match it to the known fingerprints
			if is_js:
				matches = self.matcher.get_result(self.fingerprints, response)
				for fp in matches:
					self.result.add( fp['category'], fp['name'], fp['output'], fingerprint=fp, weight=1)

class DiscoverInteresting(object):
	def __init__(self, options, data):
		self.requester = data['requester']
		self.interesting = data['fingerprints'].get_interesting_fingerprints()
		self.matcher = data['matcher']
		self.result = data['results']
		self.threads = options['threads']
		self.category = "Interesting"

	def run(self):
		cs = self.threads

		for i in range(0, len(self.interesting), cs):
			chunk = self.interesting[i:i+cs]

			self.requester.set_fingerprints(chunk)
			results = self.requester.run()
			while results.qsize() > 0:
				fps,response = results.get()

				matches = self.matcher.get_result(fps, response)

				# their should not have been a redirection
				# when requesting these files
				if len(response.history) == 0:
					for fp in matches:
						self.result.add( self.category, None, None, fp, weight=1)

class DiscoverUrlLess(object):
	def __init__(self, data):
		self.cache = data['cache']
		self.fps = data['fingerprints'].get_url_less()
		self.results = data['results']
		self.matcher = data['matcher']

	def run(self):
		# find matches for all the responses in the cache
		for response in self.cache.get_responses():
			matches = self.matcher.get_result(self.fps, response)
			for fp in matches:
				if 'name' in fp:	name = fp['name']
				elif 'cms' in fp:	name = fp['cms']
				else:				name = ''

				self.results.add(fp['category'], name, fp['output'], fingerprint=fp, weight=1)<commit_msg>added hack for ASP.NET detection :(<commit_after>import re, hashlib, pprint
from collections import Counter
from html.parser import HTMLParser
from classes.fingerprints import Fingerprints
from classes.requester2 import Requester
from classes.matcher import Match
from classes.request import PageFetcher, Request

class DiscoverRedirect(object):

	def __init__(self, options):
		self.org = options['host']
		self.url = options['host']

		fetcher = PageFetcher(self.url)
		try:
			r = fetcher.get()
			request_url = r.get_url()

			if not request_url == self.url:
				# ensure that folders and files are removed
				parts = request_url.split('//')
				http, netloc = parts[0:2]

				# remove subfolders and/or files
				# http://example.com/test -> http://example.com/
				if '/' in netloc:
					self.url = http + '//' + netloc.split('/')[0] + '/'
				else:
					self.url = http + '//' + netloc + '/'
		except:
			self.url = None

	# check if the host redirects to another location
	def is_redirected(self):
		return not self.org == self.url

	# return a cleaned URL
	def get_valid_url(self):
		return self.url

class DiscoverErrorPage(object):
	# find error pages on the site
	# the requester has a built-in list of items and patterns
	# to remove before calculating a checksum of pages that
	# should not exists

	def __init__(self, options, data):
		self.host = options['host']
		self.urls = data['fingerprints'].get_error_urls()
		self.error_pages = set()
		self.requester = data['requester']

	def run(self):
		urls = [ [{'host': self.host, 'url': u}] for u in self.urls ]

		self.requester.set_fingerprints(urls)
		self.requester.set_find_404(True)

		results = self.requester.run()
		while results.qsize() > 0:
			response = results.get()
			self.error_pages.add(response)

		self.requester.set_find_404(False)

	def get_error_pages(self):
		return self.error_pages

class DiscoverCMS(object):

	def __init__(self, options, data):
		self.chunk_size = options['chunk_size']
		self.matcher = data['matcher']
		self.requester = data['requester']
		self.fps = data['fingerprints'].get_ordered_list()
		self.fps_iter = iter(self.fps)
		self.index = 0

	def is_done(self):
		return self.index >= len(self.fps)

	def run(self):
		i = self.index
		cs = self.chunk_size

		# extract a chunck
		chunk = self.fps[i:i+cs]
		self.index += cs

		self.requester.set_fingerprints(chunk)
		results = self.requester.run()

		# process the results and find matches
		while results.qsize() > 0:
			fps,response = results.get()
			matches = self.matcher.get_result(fps, response)
			if matches:
				return [cms['cms'] for cms in matches]

		return []

class DiscoverVersion(object):
	def __init__(self, options, data):
		self.chunk_size = options['chunk_size']
		self.result = data['results']
		self.matcher = data['matcher']
		self.requester = data['requester']
		self.fingerprints = data['fingerprints']

	def run(self, cms):
		cs = self.chunk_size
		fps = self.fingerprints.get_fingerprints_for_cms(cms)

		for i in range(0, len(fps), cs):
			chunk = fps[i:i+cs]

			self.requester.set_fingerprints(chunk)
			results = self.requester.run()

			while results.qsize() > 0:
				res_fps,response = results.get()
				for fp in self.matcher.get_result(res_fps, response):
					self.result.add_cms(fp)

class DiscoverOS(object):
	def __init__(self, data):
		self.cache = data['cache']
		self.results = data['results']
		self.fingerprints = data['fingerprints'].get_os_fingerprints()

		self.category = "Operating System"
		self.os = Counter()
		self.packages = Counter()
		self.oss = []
		self.matched_packages = set()

	def find_match(self, response):
		headers = response.headers
		if 'server' in headers:
			line = headers['server']
			if "(" in line:
				os = line[line.find('(')+1:line.find(')')]
				line = line[:line.find('(')-1] + line[line.find(')')+1: ]
			else:
				os = False

			if os: self.oss.append(os.lower())

			for part in line.split(" "):
				try:
					pkg,version = list(map(str.lower, part.split('/')))
					self.packages[pkg] += 1

					os_list = self.fingerprints[pkg][version]

					for i in os_list:
						if len(i) == 2:
							os, os_version = i
							weight = 1
						elif len(i) == 3:
							os, os_version, weight = i

						self.matched_packages.add( (os, os_version, pkg, version) )
						self.os[(os, os_version)] += weight

				except Exception as e:
					continue

	def finalize(self):
		# this might not be stable for items other than php at the moment
		# recheck this process
		# changed in order to eliminate the 'X-Powered-by' check here
		platforms = self.results.get_platform_results()
		for pkg in platforms:
			for version in platforms[pkg]:
				# hack for asp.net
				if pkg == 'ASP.NET':
					version = version[:3] if not version.startswith("4.5") else version[:5]

				score = platforms[pkg][version]
				try:
					for i in self.fingerprints[pkg.lower()][version]:
						if len(i) == 2:
							os, os_version = i
							weight = 1
						elif len(i) == 3:
							os, os_version, weight = i

						self.matched_packages.add( (os, os_version, pkg, version) )
						self.os[(os, os_version)] += score*weight

				except Exception as e:
					pass

		# if an os string 'self.oss' has been found in the header
		# prioritize the identified os's in self.os

		# iterate over the list of os strings found
		for os in self.oss:
			# iterate over the fingerprinted os's
			for key in self.os:
				if os in key[0].lower():
					self.os[key] += 100

		# add OS to results: self.os: {(os, version): weight, ...}
		results = []
		for p in self.os:
			results.append({'version': p[1], 'os': p[0], 'count': self.os[p]})

		if len(results) == 0: return

		prio = sorted(results, key=lambda x:x['count'], reverse=True)
		max_count = prio[0]['count']
		relevant = []
		for i in prio:
			if i['count'] == max_count:
				if len(relevant) > 0  and i[0] == "": continue
				self.results.add(self.category, i['os'], i['version'], weight=i['count'])
			else:
				break

	def run(self):
		headers = set()
		responses = self.cache.get_responses()
		for response in responses:
			self.find_match(response)

		self.finalize()

# Used by the DiscoverMore crawler
# The
class LinkExtractor(HTMLParser):
	def __init__(self, strict):
		super().__init__(strict=strict)
		self.results = set()

	def get_results(self):
		return self.results

	def handle_starttag(self, tag, attrs):
		try:
			url = ''
			if tag == 'script' or tag == 'img':
				for attr in attrs:
					if attr[0] == 'src':  self.results.add(attr[1])
			if tag == 'link':
				for attr in attrs:
					if attr[0] == 'href': self.results.add(attr[1])
		except:
			pass

class DiscoverMore(object):

	def __init__(self, options, data):
		self.host = options['host']
		self.threads = options['threads']
		self.cache = data['cache']
		self.result = data['results']
		self.matcher = data['matcher']
		self.requester = data['requester']
		self.fingerprints = data['fingerprints']

	def _get_urls(self, response):
		# only get urls from elements that use 'src' to avoid
		# fetching resources provided by <a>-tags, as this could
		# lead to the crawling of the whole application
		regexes = [ 'src="(.+?)"', "src='(.+?)'"]

		urls = set()
		for regex in regexes:
			for match in re.findall(regex, response.body):
				urls.add( match )

		return urls

	def run(self):
		resources = set()
		parser = LinkExtractor(strict=False)

		for req in self.cache.get_responses():
			# skip pages that do not set 'content-type'
			# these might be binaries
			if not 'content-type' in req.headers:
				continue

			# only scrape pages that can contain links/references
			if 'text/html' in req.headers['content-type']:
				tmp = self._get_urls(req)

				parser.feed(req.body)
				tmp = tmp.union( parser.get_results())

				for i in tmp:

					# ensure that only resources located on the domain /sub-domain is requested
					if i.startswith('http') or i.startswith('//'):
						parts = i.split('/')
						host = parts[2]

						# if the resource is out side of the domain, skip it
						if not host in self.host.split('/')[2]:
							continue

						# else update the url so that it only contains the relative location
						else:
							i = '/'.join(parts[3:])

					resources.add( i )

		# the items in the resource set should mimic a list of fingerprints:
		# a fingerprint is a dict with at least an URL key
		urls = [ [{'url':i}] for i in resources ]

		# fetch the discovered resources.
		# As this class' purpose only is to fetch the resource (add to cache)
		# there is no return value, or further actions needed
		for i in range(0, len(urls), self.threads):
			self.requester.set_fingerprints( urls[i:i+self.threads] )
			results = self.requester.run()

class DiscoverAllCMS(object):
	# match all fingerprints against all responses
	# this might generate false positives

	def __init__(self, data):
		self.cache = data['cache']
		self.fps = data['fingerprints'].get_all()
		self.results = data['results']
		self.matcher = data['matcher']

	def run(self):
		# find matches for all the responses in the cache
		for response in self.cache.get_responses():
			matches = self.matcher.get_result(self.fps, response)
			for fp in matches:
				self.results.add_cms(fp)

class DiscoverJavaScript(object):
	def __init__(self, data):
		self.cache = data['cache']
		self.fingerprints = data['fingerprints'].get_js_fingerprints()
		self.matcher = data['matcher']
		self.result = data['results']

	def run(self):
		for response in self.cache.get_responses():

			# match only if the response is JavaScript
			#  check content type
			content_type = response.headers['content-type'] if 'content-type' in response.headers else ''
			# and extension
			is_js = 'javascript' in content_type or '.js' in response.url.split('.')[-1]

			# if the response is JavaScript try to match it to the known fingerprints
			if is_js:
				matches = self.matcher.get_result(self.fingerprints, response)
				for fp in matches:
					self.result.add( fp['category'], fp['name'], fp['output'], fingerprint=fp, weight=1)

class DiscoverInteresting(object):
	def __init__(self, options, data):
		self.requester = data['requester']
		self.interesting = data['fingerprints'].get_interesting_fingerprints()
		self.matcher = data['matcher']
		self.result = data['results']
		self.threads = options['threads']
		self.category = "Interesting"

	def run(self):
		cs = self.threads

		for i in range(0, len(self.interesting), cs):
			chunk = self.interesting[i:i+cs]

			self.requester.set_fingerprints(chunk)
			results = self.requester.run()
			while results.qsize() > 0:
				fps,response = results.get()

				matches = self.matcher.get_result(fps, response)

				# their should not have been a redirection
				# when requesting these files
				if len(response.history) == 0:
					for fp in matches:
						self.result.add( self.category, None, None, fp, weight=1)

class DiscoverUrlLess(object):
	def __init__(self, data):
		self.cache = data['cache']
		self.fps = data['fingerprints'].get_url_less()
		self.results = data['results']
		self.matcher = data['matcher']

	def run(self):
		# find matches for all the responses in the cache
		for response in self.cache.get_responses():
			matches = self.matcher.get_result(self.fps, response)
			for fp in matches:
				if 'name' in fp:	name = fp['name']
				elif 'cms' in fp:	name = fp['cms']
				else:				name = ''

				self.results.add(fp['category'], name, fp['output'], fingerprint=fp, weight=1)<|endoftext|>


<commit_before>#!/usr/bin/env python

"""This script applies a set of regular expressions onto a set of files
to automatically identify and fix typographical errors.
"""

# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.

# author: Christian Berendt <berendt@b1-systems.de>

# Based on the idea of 'Topy' written by Marti Raudsepp <marti@juffo.org>.
# Topy is available on Github at https://github.com/intgr/topy.

import argparse
import logging
import os
import shutil
import sys
import urllib2

from bs4 import BeautifulSoup
import glob2
import regex
import six
import yaml

class DownloadRetfListingFailed(Exception):
    """Exception will be raised when the download of the RETF
    listing failed or the destination file could not be written.
    """

    pass

def download_listing(dest):
    """Download the latest RETF listing from Wikipedia."""
    logger = logging.getLogger('retf')
    try:
        url = ('https://en.wikipedia.org/wiki/Wikipedia:AutoWikiBrowser/'
               'Typos?action=raw')
        logger.debug("Downloading latest RETF listing from %s into %s.",
                     url, dest)
        response = urllib2.urlopen(url)
        data = response.read()
        logger.info("Downloading latest RETF listing from %s succeeded.", url)
    except urllib2.HTTPError as ex:
        raise DownloadRetfListingFailed(six.text_type(ex))
    except urllib2.URLError as ex:
        raise DownloadRetfListingFailed(six.text_type(ex))

    try:
        with open(dest, 'w+') as write:
            write.write(data)
        logger.info("Writing RETF listing to file %s succeeded.", dest)
    except IOError as ex:
        raise DownloadRetfListingFailed(six.text_type(ex))

def soupify_listing(src):
    """Parse a RETF listing."""
    return BeautifulSoup(open(src))

def generate_listing(src):
    """Compile all regular expressions in a RETF listing."""
    logger = logging.getLogger('retf')
    result = []

    soup = soupify_listing(src)

    for typo in soup.findAll('typo'):
        try:
            word = typo.attrs.get('word').encode('utf8')
            find = typo.attrs.get('find').encode('utf8')
            replace = typo.attrs.get('replace').encode('utf8')
            replace = replace.replace(b'$', b'\\')
        except AttributeError:
            continue

        # pylint: disable=W0703
        try:
            logger.debug("Compiling regular expression: %s.", find)
            compiled = regex.compile(find, flags=regex.V1)
        except Exception:
            logger.error("Compilation of regular expression %f failed.", find)
            continue
        # pylint: enable=W0703

        entry = {
            'description': word,
            'find': find,
            'replace': replace,
            'regex': compiled
        }

        result.append(entry)

    logger.debug("Compiled %d regular expression(s).", len(result))

    return result

def load_text_from_file(src):
    """Load content from a file."""
    logger = logging.getLogger('retf')
    logger.debug("Loading text from file %s.", src)
    with open(src, 'rb') as fpointer:
        text = fpointer.read()

    return text

def write_text_to_file(dest, text, no_backup, in_place):
    """Write content into a file."""
    logger = logging.getLogger('retf')

    if not no_backup:
        logger.debug("Copying %s to backup file %s.orig.", dest, dest)
        shutil.copy2(dest, "%s.orig" % dest)

    if not in_place:
        dest = "%s.retf" % dest

    logger.debug("Writing text to file %s.", dest)
    with open(dest, 'wb') as fpointer:
        fpointer.write(text)

def initialize_logging(debug, less_verbose):
    """Initialze the Logger."""
    logger = logging.getLogger(name='retf')
    formatter = logging.Formatter('%(asctime)s %(levelname)-8s %(message)s')
    handler = logging.StreamHandler()
    handler.setFormatter(formatter)
    logger.addHandler(handler)

    logger.setLevel(logging.INFO)

    if less_verbose:
        logger.setLevel(logging.WARN)

    if debug:
        logger.setLevel(logging.DEBUG)

    return logging.getLogger('retf')

def parse_command_line_arguments():
    """Parse the command line arguments."""
    parser = argparse.ArgumentParser()
    parser.add_argument("--debug", help="Print debugging messages.",
                        action="store_true", default=False)
    parser.add_argument("--download", help="Download the latest RETF listing.",
                        action="store_true", default=False)
    parser.add_argument("--less-verbose", help="Be less verbose.",
                        action="store_true", default=False)
    parser.add_argument("--no-backup", help="Don't backup files.",
                        action="store_true", default=False)
    parser.add_argument("--in-place", help="Resolve found errors in place.",
                        action="store_true", default=False)
    parser.add_argument("--write-changes", action="store_true", default=False,
                        help="Write resolved findings back to files.")
    parser.add_argument("--disabled", type=str, default=None,
                        help="File containing the disabled rules.")
    parser.add_argument("--listing", help="File containing the RETF listing.",
                        type=str, default=os.path.join(
                            os.path.dirname(os.path.realpath(__file__)),
                            'retf.lst'))
    parser.add_argument("--path", type=str, nargs='*', default=[],
                        help="Path(s) that should be checked.")
    parser.add_argument("--extension", type=str, nargs='*', default=[],
                        help="Only check files with specified extension(s).")
    parser.add_argument("--file", nargs='*', type=str, default=[],
                        help="File(s) to check for typographical errors.")
    return (parser, parser.parse_args())

def load_disabled_rules(src):
    """Load disabled rules from YAML file."""
    logger = logging.getLogger('retf')
    listing = []

    if src:
        try:
            listing = yaml.load(open(src))
            for rule in listing:
                logger.debug("Rule '%s' is disabled.", rule)

        except IOError:
            logger.error("loading disabled rules from file %s failed", src)

    return listing

def get_file_listing(paths, files, extensions):
    """Generate listing with all files that should be check."""
    result = []
    if files:
        result += files

    # pylint: disable=E1101
    for path in paths:
        if extensions:
            for extension in extensions:
                result += glob2.glob("%s/**/*.%s" % (path, extension))
        else:
            result += glob2.glob("%s/**/*" % path)
    # pylint: enable=E1101

    return result

def check_file(src, rules, disabled):
    """Applies a set of rules on a file."""
    logger = logging.getLogger('retf')
    logger.info("Checking file %s for typographical errors.", src)
    content = load_text_from_file(src)
    findings = 0

    for rule in rules:
        if rule.get('description') in disabled:
            continue

        logger.debug("%s: checking rule '%s'.", file,
                     rule.get('description'))
        logger.debug(rule.get('find'))
        newcontent, count = rule.get('regex').subn(
            rule.get('replace'), content
        )

        if count > 0:
            logger.warning("%d match(s) in file %s : %s.", count, file,
                           rule.get('description'))
            findings += count
        content = newcontent

    return (findings, content)

def main():
    """Entry point for this script."""

    parser, args = parse_command_line_arguments()
    logger = initialize_logging(args.debug, args.less_verbose)

    result = 0

    if args.download:
        try:
            download_listing(args.listing)
        except DownloadRetfListingFailed as ex:
            logger.error("Downloading latest RETF listing failed: %s.", ex)
            result = 1

    if not args.path and not args.file and not args.download:
        parser.print_help()
        result = 2

    if not result and not os.path.isfile(args.listing):
        logger.error("RETF listing not found at %s.", args.listing)
        logger.info("Please download the RETF listing first by using the "
                    "parameter --download.")
        result = 1

    if not result:
        files = get_file_listing(args.path, args.file, args.extension)

        rules = generate_listing(args.listing)
        disabled = load_disabled_rules(args.disabled)

        all_findings = 0
        for check in files:

            (findings, content) = check_file(check, rules, disabled)

            if findings > 0:
                all_findings += findings
                logger.warning("%s finding(s) in file %s.", findings, check)

            if findings > 0 and args.write_changes:
                write_text_to_file(check, content, args.no_backup,
                                   args.in_place)

        if all_findings > 0:
            logger.warning("%s finding(s) in all checked files.", all_findings)
            result = 1

    return result

if __name__ == "__main__":
    sys.exit(main())
<commit_msg>make sure that source is not a directory<commit_after>#!/usr/bin/env python

"""This script applies a set of regular expressions onto a set of files
to automatically identify and fix typographical errors.
"""

# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.

# author: Christian Berendt <berendt@b1-systems.de>

# Based on the idea of 'Topy' written by Marti Raudsepp <marti@juffo.org>.
# Topy is available on Github at https://github.com/intgr/topy.

import argparse
import logging
import os
import shutil
import sys
import urllib2

from bs4 import BeautifulSoup
import glob2
import regex
import six
import yaml

class DownloadRetfListingFailed(Exception):
    """Exception will be raised when the download of the RETF
    listing failed or the destination file could not be written.
    """

    pass

def download_listing(dest):
    """Download the latest RETF listing from Wikipedia."""
    logger = logging.getLogger('retf')
    try:
        url = ('https://en.wikipedia.org/wiki/Wikipedia:AutoWikiBrowser/'
               'Typos?action=raw')
        logger.debug("Downloading latest RETF listing from %s into %s.",
                     url, dest)
        response = urllib2.urlopen(url)
        data = response.read()
        logger.info("Downloading latest RETF listing from %s succeeded.", url)
    except urllib2.HTTPError as ex:
        raise DownloadRetfListingFailed(six.text_type(ex))
    except urllib2.URLError as ex:
        raise DownloadRetfListingFailed(six.text_type(ex))

    try:
        with open(dest, 'w+') as write:
            write.write(data)
        logger.info("Writing RETF listing to file %s succeeded.", dest)
    except IOError as ex:
        raise DownloadRetfListingFailed(six.text_type(ex))

def soupify_listing(src):
    """Parse a RETF listing."""
    return BeautifulSoup(open(src))

def generate_listing(src):
    """Compile all regular expressions in a RETF listing."""
    logger = logging.getLogger('retf')
    result = []

    soup = soupify_listing(src)

    for typo in soup.findAll('typo'):
        try:
            word = typo.attrs.get('word').encode('utf8')
            find = typo.attrs.get('find').encode('utf8')
            replace = typo.attrs.get('replace').encode('utf8')
            replace = replace.replace(b'$', b'\\')
        except AttributeError:
            continue

        # pylint: disable=W0703
        try:
            logger.debug("Compiling regular expression: %s.", find)
            compiled = regex.compile(find, flags=regex.V1)
        except Exception:
            logger.error("Compilation of regular expression %f failed.", find)
            continue
        # pylint: enable=W0703

        entry = {
            'description': word,
            'find': find,
            'replace': replace,
            'regex': compiled
        }

        result.append(entry)

    logger.debug("Compiled %d regular expression(s).", len(result))

    return result

def load_text_from_file(src):
    """Load content from a file."""
    logger = logging.getLogger('retf')
    logger.debug("Loading text from file %s.", src)
    with open(src, 'rb') as fpointer:
        text = fpointer.read()

    return text

def write_text_to_file(dest, text, no_backup, in_place):
    """Write content into a file."""
    logger = logging.getLogger('retf')

    if not no_backup:
        logger.debug("Copying %s to backup file %s.orig.", dest, dest)
        shutil.copy2(dest, "%s.orig" % dest)

    if not in_place:
        dest = "%s.retf" % dest

    logger.debug("Writing text to file %s.", dest)
    with open(dest, 'wb') as fpointer:
        fpointer.write(text)

def initialize_logging(debug, less_verbose):
    """Initialze the Logger."""
    logger = logging.getLogger(name='retf')
    formatter = logging.Formatter('%(asctime)s %(levelname)-8s %(message)s')
    handler = logging.StreamHandler()
    handler.setFormatter(formatter)
    logger.addHandler(handler)

    logger.setLevel(logging.INFO)

    if less_verbose:
        logger.setLevel(logging.WARN)

    if debug:
        logger.setLevel(logging.DEBUG)

    return logging.getLogger('retf')

def parse_command_line_arguments():
    """Parse the command line arguments."""
    parser = argparse.ArgumentParser()
    parser.add_argument("--debug", help="Print debugging messages.",
                        action="store_true", default=False)
    parser.add_argument("--download", help="Download the latest RETF listing.",
                        action="store_true", default=False)
    parser.add_argument("--less-verbose", help="Be less verbose.",
                        action="store_true", default=False)
    parser.add_argument("--no-backup", help="Don't backup files.",
                        action="store_true", default=False)
    parser.add_argument("--in-place", help="Resolve found errors in place.",
                        action="store_true", default=False)
    parser.add_argument("--write-changes", action="store_true", default=False,
                        help="Write resolved findings back to files.")
    parser.add_argument("--disabled", type=str, default=None,
                        help="File containing the disabled rules.")
    parser.add_argument("--listing", help="File containing the RETF listing.",
                        type=str, default=os.path.join(
                            os.path.dirname(os.path.realpath(__file__)),
                            'retf.lst'))
    parser.add_argument("--path", type=str, nargs='*', default=[],
                        help="Path(s) that should be checked.")
    parser.add_argument("--extension", type=str, nargs='*', default=[],
                        help="Only check files with specified extension(s).")
    parser.add_argument("--file", nargs='*', type=str, default=[],
                        help="File(s) to check for typographical errors.")
    return (parser, parser.parse_args())

def load_disabled_rules(src):
    """Load disabled rules from YAML file."""
    logger = logging.getLogger('retf')
    listing = []

    if src:
        try:
            listing = yaml.load(open(src))
            for rule in listing:
                logger.debug("Rule '%s' is disabled.", rule)

        except IOError:
            logger.error("loading disabled rules from file %s failed", src)

    return listing

def get_file_listing(paths, files, extensions):
    """Generate listing with all files that should be check."""
    result = []
    if files:
        result += files

    # pylint: disable=E1101
    for path in paths:
        if extensions:
            for extension in extensions:
                result += glob2.glob("%s/**/*.%s" % (path, extension))
        else:
            result += glob2.glob("%s/**/*" % path)
    # pylint: enable=E1101

    return result

def check_file(src, rules, disabled):
    """Applies a set of rules on a file."""
    logger = logging.getLogger('retf')
    logger.info("Checking file %s for typographical errors.", src)
    content = load_text_from_file(src)
    findings = 0

    for rule in rules:
        if rule.get('description') in disabled:
            continue

        logger.debug("%s: checking rule '%s'.", src,
                     rule.get('description'))
        logger.debug(rule.get('find'))
        newcontent, count = rule.get('regex').subn(
            rule.get('replace'), content
        )

        if count > 0:
            logger.warning("%d match(s) in file %s : %s.", count, src,
                           rule.get('description'))
            findings += count
        content = newcontent

    return (findings, content)

def main():
    """Entry point for this script."""

    parser, args = parse_command_line_arguments()
    logger = initialize_logging(args.debug, args.less_verbose)

    result = 0

    if args.download:
        try:
            download_listing(args.listing)
        except DownloadRetfListingFailed as ex:
            logger.error("Downloading latest RETF listing failed: %s.", ex)
            result = 1

    if not args.path and not args.file and not args.download:
        parser.print_help()
        result = 2

    if not result and not os.path.isfile(args.listing):
        logger.error("RETF listing not found at %s.", args.listing)
        logger.info("Please download the RETF listing first by using the "
                    "parameter --download.")
        result = 1

    if not result:
        files = get_file_listing(args.path, args.file, args.extension)

        rules = generate_listing(args.listing)
        disabled = load_disabled_rules(args.disabled)

        all_findings = 0
        for check in files:
            if not os.path.isfile(check):
                continue

            (findings, content) = check_file(check, rules, disabled)

            if findings > 0:
                all_findings += findings
                logger.warning("%s finding(s) in file %s.", findings, check)

            if findings > 0 and args.write_changes:
                write_text_to_file(check, content, args.no_backup,
                                   args.in_place)

        if all_findings > 0:
            logger.warning("%s finding(s) in all checked files.", all_findings)
            result = 1

    return result

if __name__ == "__main__":
    sys.exit(main())
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-

"""
clint.textui.progress
~~~~~~~~~~~~~~~~~

This module provides the progressbar functionality.

"""

from __future__ import absolute_import

import sys
import time

STREAM = sys.stderr
# Only show bar in terminals by default (better for piping, logging etc.)
try:
    HIDE_DEFAULT = not STREAM.isatty()
except AttributeError:  # output does not support isatty()
    HIDE_DEFAULT = True

BAR_TEMPLATE = '%s[%s%s] %i/%i - %s\r'
MILL_TEMPLATE = '%s %s %i/%i\r'

DOTS_CHAR = '.'
BAR_FILLED_CHAR = '#'
BAR_EMPTY_CHAR = ' '
MILL_CHARS = ['|', '/', '-', '\\']

#How long to wait before recalculating the ETA
ETA_INTERVAL = 1
#How many intervals (excluding the current one) to calculate the simple moving average
ETA_SMA_WINDOW = 9

class Bar(object):
    def __enter__(self):
        return self

    def __exit__(self, exc_type, exc_val, exc_tb):
        self.done()
        return False # we're not surpressing exceptions

    def __init__(self, label='', width=32, hide=None, empty_char=BAR_EMPTY_CHAR,
                 filled_char=BAR_FILLED_CHAR, expected_size=None, every=1):
        self.label = label
        self.width = width
        self.hide = hide
        if hide is None:
            try:
                self.hide = not STREAM.isatty()
            except AttributeError:  # output does not support isatty()
                self.hide = True
        self.empty_char =    empty_char
        self.filled_char =   filled_char
        self.expected_size = expected_size
        self.every =         every
        self.start =         time.time()
        self.ittimes =       []
        self.eta =           0
        self.etadelta =      time.time()
        self.etadisp =       time.strftime('%H:%M:%S', time.gmtime(self.eta))
        if (self.expected_size):
            self.show(0)

    def show(self, progress, count=None):
        if count is not None:
            self.expected_size = count
        if self.expected_size is None:
            raise Exception("expected_size not initialized")
        if (time.time() - self.etadelta) > ETA_INTERVAL:
            self.etadelta = time.time()
            self.ittimes = \
                self.ittimes[-ETA_SMA_WINDOW:]+\
                    [-(self.start-time.time())/(progress+1)]
            self.eta = \
                sum(self.ittimes)/float(len(self.ittimes)) * \
                (self.expected_size-progress)
            self.etadisp = time.strftime('%H:%M:%S', time.gmtime(self.eta))
        x = int(self.width*progress/self.expected_size)
        if not self.hide:
            if ((progress % self.every)==0 or      # True every "every" updates
                (progress == self.expected_size)): # And when we're done
                STREAM.write(BAR_TEMPLATE % (
                    self.label, self.filled_char*x,
                    self.empty_char*(self.width-x), progress,
                    self.expected_size, self.etadisp))
                STREAM.flush()

    def done(self):
        if not self.hide:
            STREAM.write('\n')
            STREAM.flush()

def bar(it, label='', width=32, hide=HIDE_DEFAULT, empty_char=BAR_EMPTY_CHAR, filled_char=BAR_FILLED_CHAR, expected_size=None, every=1):
    """Progress iterator. Wrap your iterables with it."""

    count = len(it) if expected_size is None else expected_size

    with Bar(label=label, width=width, hide=hide, empty_char=BAR_EMPTY_CHAR,
              filled_char=BAR_FILLED_CHAR, expected_size=count, every=every) \
              as bar:

        for i, item in enumerate(it):

            yield item
            bar.show(i+1)

def dots(it, label='', hide=HIDE_DEFAULT, every=1):
    """Progress iterator. Prints a dot for each item being iterated"""

    count = 0

    if not hide:
        STREAM.write(label)

    for (i, item) in enumerate(it):
        if not hide:
            if (i % every)==0:         # True every "every" updates
                STREAM.write(DOTS_CHAR)
                sys.stderr.flush()

        count += 1

        yield item

    STREAM.write('\n')
    STREAM.flush()

def mill(it, label='', hide=HIDE_DEFAULT, expected_size=None, every=1):
    """Progress iterator. Prints a mill while iterating over the items."""

    def _mill_char(_i):
        if _i >= count:
            return ' '
        else:
            return MILL_CHARS[(_i / every) % len(MILL_CHARS)]

    def _show(_i):
        if not hide:
            if ((_i % every)==0 or         # True every "every" updates
                (_i == count)):            # And when we're done

                STREAM.write(MILL_TEMPLATE % (
                    label, _mill_char(_i), _i, count))
                STREAM.flush()

    count = len(it) if expected_size is None else expected_size

    if count:
        _show(0)

    for i, item in enumerate(it):

        yield item
        _show(i+1)

    if not hide:
        STREAM.write('\n')
        STREAM.flush()
<commit_msg>Display elapsed time when the progress bar is complete<commit_after># -*- coding: utf-8 -*-

"""
clint.textui.progress
~~~~~~~~~~~~~~~~~

This module provides the progressbar functionality.

"""

from __future__ import absolute_import

import sys
import time

STREAM = sys.stderr
# Only show bar in terminals by default (better for piping, logging etc.)
try:
    HIDE_DEFAULT = not STREAM.isatty()
except AttributeError:  # output does not support isatty()
    HIDE_DEFAULT = True

BAR_TEMPLATE = '%s[%s%s] %i/%i - %s\r'
MILL_TEMPLATE = '%s %s %i/%i\r'

DOTS_CHAR = '.'
BAR_FILLED_CHAR = '#'
BAR_EMPTY_CHAR = ' '
MILL_CHARS = ['|', '/', '-', '\\']

# How long to wait before recalculating the ETA
ETA_INTERVAL = 1
# How many intervals (excluding the current one) to calculate the simple moving
# average
ETA_SMA_WINDOW = 9

class Bar(object):
    def __enter__(self):
        return self

    def __exit__(self, exc_type, exc_val, exc_tb):
        self.done()
        return False  # we're not suppressing exceptions

    def __init__(self, label='', width=32, hide=None, empty_char=BAR_EMPTY_CHAR,
                 filled_char=BAR_FILLED_CHAR, expected_size=None, every=1):
        self.label = label
        self.width = width
        self.hide = hide
        if hide is None:
            try:
                self.hide = not STREAM.isatty()
            except AttributeError:  # output does not support isatty()
                self.hide = True
        self.empty_char =    empty_char
        self.filled_char =   filled_char
        self.expected_size = expected_size
        self.every =         every
        self.start =         time.time()
        self.ittimes =       []
        self.eta =           0
        self.etadelta =      time.time()
        self.etadisp =       self.format_time(self.eta)
        self.last_progress = 0
        if (self.expected_size):
            self.show(0)

    def show(self, progress, count=None):
        if count is not None:
            self.expected_size = count
        if self.expected_size is None:
            raise Exception("expected_size not initialized")
        self.last_progress = progress
        if (time.time() - self.etadelta) > ETA_INTERVAL:
            self.etadelta = time.time()
            self.ittimes = \
                self.ittimes[-ETA_SMA_WINDOW:] + \
                    [-(self.start - time.time()) / (progress+1)]
            self.eta = \
                sum(self.ittimes) / float(len(self.ittimes)) * \
                (self.expected_size - progress)
            self.etadisp = self.format_time(self.eta)
        x = int(self.width * progress / self.expected_size)
        if not self.hide:
            if ((progress % self.every) == 0 or      # True every "every" updates
                (progress == self.expected_size)):   # And when we're done
                STREAM.write(BAR_TEMPLATE % (
                    self.label, self.filled_char * x,
                    self.empty_char * (self.width - x), progress,
                    self.expected_size, self.etadisp))
                STREAM.flush()

    def done(self):
        self.elapsed = time.time() - self.start
        elapsed_disp = self.format_time(self.elapsed)
        if not self.hide:
            # Print completed bar with elapsed time
            STREAM.write(BAR_TEMPLATE % (
                self.label, self.filled_char * self.width,
                self.empty_char * 0, self.last_progress,
                self.expected_size, elapsed_disp))
            STREAM.write('\n')
            STREAM.flush()

    def format_time(self, seconds):
        return time.strftime('%H:%M:%S', time.gmtime(seconds))

def bar(it, label='', width=32, hide=HIDE_DEFAULT, empty_char=BAR_EMPTY_CHAR,
        filled_char=BAR_FILLED_CHAR, expected_size=None, every=1):
    """Progress iterator. Wrap your iterables with it."""

    count = len(it) if expected_size is None else expected_size

    with Bar(label=label, width=width, hide=hide, empty_char=BAR_EMPTY_CHAR,
             filled_char=BAR_FILLED_CHAR, expected_size=count, every=every) \
            as bar:
        for i, item in enumerate(it):
            yield item
            bar.show(i + 1)

def dots(it, label='', hide=HIDE_DEFAULT, every=1):
    """Progress iterator. Prints a dot for each item being iterated"""

    count = 0

    if not hide:
        STREAM.write(label)

    for i, item in enumerate(it):
        if not hide:
            if i % every == 0:         # True every "every" updates
                STREAM.write(DOTS_CHAR)
                sys.stderr.flush()

        count += 1

        yield item

    STREAM.write('\n')
    STREAM.flush()

def mill(it, label='', hide=HIDE_DEFAULT, expected_size=None, every=1):
    """Progress iterator. Prints a mill while iterating over the items."""

    def _mill_char(_i):
        if _i >= count:
            return ' '
        else:
            return MILL_CHARS[(_i / every) % len(MILL_CHARS)]

    def _show(_i):
        if not hide:
            if ((_i % every) == 0 or         # True every "every" updates
                (_i == count)):            # And when we're done

                STREAM.write(MILL_TEMPLATE % (
                    label, _mill_char(_i), _i, count))
                STREAM.flush()

    count = len(it) if expected_size is None else expected_size

    if count:
        _show(0)

    for i, item in enumerate(it):
        yield item
        _show(i + 1)

    if not hide:
        STREAM.write('\n')
        STREAM.flush()
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-

"""

    Create a test to check if cfg['vars'] does exist in the climatology file,
      to avoid error like have t_mn in one and t_an in the other.
"""

from datetime import timedelta

import numpy as np
from numpy import ma

from pyWOA import WOA

def woa_normbias(data, v, cfg):
    """

        FIXME: Move this procedure into a class to conform with the new system
          and include a limit in minimum ammount of samples to trust it. For
          example, consider as masked all climatologic values estimated from
          less than 5 samples.
    """

    # 3 is the possible minimum to estimate the std, but I shold use higher.
    min_samples = 3
    woa = None

    if ('LATITUDE' in data.keys()) and ('LONGITUDE' in data.keys()):
        if 'datetime' in data.keys():
            d = data['datetime']
        elif ('datetime' in data.attributes):
            d0 = data.attributes['datetime']
            if ('timeS' in data.keys()):
                d = [d0 + timedelta(seconds=s) for s in data['timeS']]
            else:
                d = [data.attributes['datetime']]*len(data['LATITUDE']),

        #woa = woa_track_from_file(
        #        d,
        #        data['LATITUDE'],
        #        data['LONGITUDE'],
        #        cfg['file'],
        #        varnames=cfg['vars'])
        db = WOA()
        if v not in db.keys():
            vtype = v[:-1]
        else:
            vtype = v

        woa = db[vtype].get_track(var=['mn', 'sd'],
                doy=d,
                depth=[0],
                lat=data['LATITUDE'],
                lon=data['LONGITUDE'])

    elif ('LATITUDE' in data.attributes.keys()) and \
            ('LONGITUDE' in data.attributes.keys()) and \
            ('PRES' in data.keys()):
                #woa = woa_profile(v,
                #        data.attributes['datetime'],
                #        data.attributes['LATITUDE'],
                #        data.attributes['LONGITUDE'],
                #        data['PRES'],
                #        cfg)
                db = WOA()
                if v not in db.keys():
                    vtype = v[:-1]
                else:
                    vtype = v

                woa = db[vtype].extract(var=['mn', 'sd', 'dd'],
                        doy=int(data.attributes['datetime'].strftime('%j')),
                        depth=data['PRES'],
                        lat=data.attributes['LATITUDE'],
                        lon=data.attributes['LONGITUDE'])

    if woa is None:
        # self.logger.warn("%s - WOA is not available at this site" %
        # self.name)
        flag = np.zeros(data[v].shape, dtype='i1')
        return flag, {}

    woa_bias = data[v] - woa['mn']
    woa_normbias = woa_bias/woa['sd']

    flag = np.zeros(data[v].shape, dtype='i1')

    ind = np.nonzero((woa['dd'] >= min_samples) &
            (np.absolute(woa_normbias) <= cfg['sigma_threshold']))
    flag[ind] = 1   # cfg['flag_good']
    ind = np.nonzero((woa['dd'] >= min_samples) &
            (np.absolute(woa_normbias) > cfg['sigma_threshold']))
    flag[ind] = 3   # cfg['flag_bad']

    # Flag as 9 any masked input value
    flag[ma.getmaskarray(data[v])] = 9

    features = {'woa_normbias': woa_normbias, 'woa_std': woa['sd'],
            'woa_nsamples': woa['dd']}

    return flag, features
<commit_msg>Adding climatology into features from woa_normbias test.<commit_after># -*- coding: utf-8 -*-

"""

    Create a test to check if cfg['vars'] does exist in the climatology file,
      to avoid error like have t_mn in one and t_an in the other.
"""

from datetime import timedelta

import numpy as np
from numpy import ma

from pyWOA import WOA

def woa_normbias(data, v, cfg):
    """

        FIXME: Move this procedure into a class to conform with the new system
          and include a limit in minimum ammount of samples to trust it. For
          example, consider as masked all climatologic values estimated from
          less than 5 samples.
    """

    # 3 is the possible minimum to estimate the std, but I shold use higher.
    min_samples = 3
    woa = None

    if ('LATITUDE' in data.keys()) and ('LONGITUDE' in data.keys()):
        if 'datetime' in data.keys():
            d = data['datetime']
        elif ('datetime' in data.attributes):
            d0 = data.attributes['datetime']
            if ('timeS' in data.keys()):
                d = [d0 + timedelta(seconds=s) for s in data['timeS']]
            else:
                d = [data.attributes['datetime']]*len(data['LATITUDE']),

        #woa = woa_track_from_file(
        #        d,
        #        data['LATITUDE'],
        #        data['LONGITUDE'],
        #        cfg['file'],
        #        varnames=cfg['vars'])
        db = WOA()
        if v not in db.keys():
            vtype = v[:-1]
        else:
            vtype = v

        woa = db[vtype].get_track(var=['mn', 'sd'],
                doy=d,
                depth=[0],
                lat=data['LATITUDE'],
                lon=data['LONGITUDE'])

    elif ('LATITUDE' in data.attributes.keys()) and \
            ('LONGITUDE' in data.attributes.keys()) and \
            ('PRES' in data.keys()):
                #woa = woa_profile(v,
                #        data.attributes['datetime'],
                #        data.attributes['LATITUDE'],
                #        data.attributes['LONGITUDE'],
                #        data['PRES'],
                #        cfg)
                db = WOA()
                if v not in db.keys():
                    vtype = v[:-1]
                else:
                    vtype = v

                woa = db[vtype].extract(var=['mn', 'sd', 'dd'],
                        doy=int(data.attributes['datetime'].strftime('%j')),
                        depth=data['PRES'],
                        lat=data.attributes['LATITUDE'],
                        lon=data.attributes['LONGITUDE'])

    if woa is None:
        # self.logger.warn("%s - WOA is not available at this site" %
        # self.name)
        flag = np.zeros(data[v].shape, dtype='i1')
        return flag, {}

    woa_bias = data[v] - woa['mn']
    woa_normbias = woa_bias/woa['sd']

    flag = np.zeros(data[v].shape, dtype='i1')

    ind = np.nonzero((woa['dd'] >= min_samples) &
            (np.absolute(woa_normbias) <= cfg['sigma_threshold']))
    flag[ind] = 1   # cfg['flag_good']
    ind = np.nonzero((woa['dd'] >= min_samples) &
            (np.absolute(woa_normbias) > cfg['sigma_threshold']))
    flag[ind] = 3   # cfg['flag_bad']

    # Flag as 9 any masked input value
    flag[ma.getmaskarray(data[v])] = 9

    features = {'woa_normbias': woa_normbias, 'woa_std': woa['sd'],
            'woa_nsamples': woa['dd'], 'woa_mean': woa['mn']}

    return flag, features
<|endoftext|>


<commit_before>"""Cell conversion methods.

Convert/read notebook cells to/from python, R and Rmd cells
"""

import re
from nbformat.v4.nbbase import new_code_cell, new_raw_cell, new_markdown_cell
from .languages import cell_language, is_code
from .cell_metadata import metadata_to_rmd_options, rmd_options_to_metadata, \
    json_options_to_metadata, metadata_to_json_options

def code_to_rmd(source, metadata, default_language):
    """
    Represent a code cell with given source and metadata as a rmd cell
    :param source:
    :param metadata:
    :param default_language:
    :return:
    """
    lines = []
    language = cell_language(source) or default_language
    if 'Rmd' not in re.split('\\.|,', metadata.get('active', 'Rmd')):
        metadata['eval'] = False
    options = metadata_to_rmd_options(language, metadata)
    lines.append(u'```{{{}}}'.format(options))
    lines.extend(source)
    lines.append(u'```')
    return lines

def code_to_text(self,
                 source,
                 metadata,
                 default_language,
                 next_cell_is_code):
    """
    Represent a code cell with given source and metadata as text
    :param self:
    :param source:
    :param metadata:
    :param default_language:
    :param next_cell_is_code:
    :return:
    """

    # Escape jupyter magics
    source = ['# ' + s if _MAGIC.match(s) else s for s in source]

    if self.ext == '.Rmd':
        return code_to_rmd(source, metadata, default_language)
    else:
        lines = []
        language = cell_language(source) or default_language
        if language == default_language:
            if self.ext == '.R':
                options = metadata_to_rmd_options(language, metadata)[2:]
                if options != '':
                    lines.append('#+ ' + options)
            else:
                # Issue #31:  does the cell ends with a blank line?
                # Do we find two blank lines in the cell? In that case
                # we add an end-of-cell marker
                if (len(source) and _BLANK_LINE.match(source[-1])) or \
                        code_to_cell(self, source, False)[1] != len(source):
                    cellend = '-'
                    while True:
                        cellend_re = re.compile('^#( )' + cellend + '\s*$')
                        if len(list(filter(cellend_re.match, source))):
                            cellend = cellend + '-'
                        else:
                            break
                    metadata['cellend'] = cellend

                options = metadata_to_json_options(metadata)
                if options != '{}':
                    lines.append('# + ' + options)
            lines.extend(source)
            if 'cellend' in metadata:
                lines.append('# ' + metadata['cellend'])
        else:
            lines.extend(self.markdown_escape(
                code_to_rmd(source, metadata, default_language)))

        # Two blank lines before next code cell
        if next_cell_is_code:
            lines.append('')

        return lines

def cell_to_text(self,
                 cell,
                 next_cell=None,
                 default_language='python'):
    """
    Represent a notebook cell as a text cell, in either
    py, R or rmd format
    :param self: TextNotebookWriter object
    :param cell: current cell
    :param next_cell: next cell
    :param default_language: default language for the current notebook
    :return:
    """
    source = cell.get('source').splitlines()
    metadata = cell.get('metadata', {})
    skipline = True
    if 'noskipline' in metadata:
        skipline = not metadata['noskipline']
        del metadata['noskipline']

    lines = []
    if is_code(cell):
        lines.extend(code_to_text(self, source, metadata, default_language,
                                  next_cell and is_code(next_cell)))
    else:
        if source == []:
            source = ['']
        lines.extend(self.markdown_escape(source))

        # Two blank lines between consecutive markdown cells in Rmd
        if self.ext == '.Rmd' and next_cell \
                and next_cell.cell_type == 'markdown':
            lines.append('')

    if skipline and next_cell:
        lines.append('')

    return lines

_START_CODE_RMD = re.compile(r"^```\{(.*)\}\s*$")
_END_CODE_MD = re.compile(r"^```\s*$")
_CODE_OPTION_RPY = re.compile(r"^(#|# )\+(.*)$")
_BLANK_LINE = re.compile(r"^\s*$")
_MAGIC = re.compile(r"^(# |#)*%")

def start_code_rmd(line):
    """
    Line indicates that a code cell starts, in a rmd file
    :param line:
    :return:
    """
    return _START_CODE_RMD.match(line)

def start_code_rpy(line):
    """
    A code cell starts here, in a py or R file
    :param line:
    :return:
    """
    return _CODE_OPTION_RPY.match(line)

def next_uncommented_is_code(lines):
    """
    Next non-commented line is code
    :param lines:
    :return:
    """
    for line in lines:
        if line.startswith('#'):
            continue
        return not _BLANK_LINE.match(line)

    return False

# Unescape jupyter magics
def uncomment(line):
    if line.startswith('# '):
        return line[2:]
    elif line.startswith('#'):
        return line[1:]
    else:
        return line

def unescape_magic(source):
    return [uncomment(s) if _MAGIC.match(s) else s for s in source]

def text_to_cell(self, lines):
    """
    Parse text to a cell
    :param self:
    :param lines:
    :return: cell, cursor
    """
    if self.start_code(lines[0]):
        return self.code_to_cell(lines, parse_opt=True)
    elif self.prefix != '' and not lines[0].startswith(self.prefix):
        return self.code_to_cell(lines, parse_opt=False)
    elif self.ext == '.py' and next_uncommented_is_code(lines):
        return self.code_to_cell(lines, parse_opt=False)

    return self.markdown_to_cell(lines)

def parse_code_options(line, ext):
    """
    Parse code options on the given line
    :param line:
    :param ext:
    :return:
    """
    if ext == '.Rmd':
        return rmd_options_to_metadata(_START_CODE_RMD.findall(line)[0])
    elif ext == '.R':
        return rmd_options_to_metadata(_CODE_OPTION_RPY.match(line).group(2))

    return 'python', json_options_to_metadata(
        _CODE_OPTION_RPY.match(line).group(2))

def next_code_is_indented(lines):
    """
    Is next code line indented?
    :param lines:
    :return:
    """
    for line in lines:
        if line.startswith('#'):
            continue
        if _BLANK_LINE.match(line):
            continue
        return line.startswith(' ')

    return False

def no_code_before_next_blank_line(lines):
    """
    Do we find code before next blank line?
    :param lines:
    :return:
    """
    for line in lines:
        if line.startswith('#'):
            continue
        return _BLANK_LINE.match(line)

    return True

def code_or_raw_cell(source, metadata):
    source = unescape_magic(source)
    source = '\n'.join(source)

    if 'ipynb' not in re.split('\\.|,', metadata.get('active', 'ipynb')):
        return new_raw_cell(source=source, metadata=metadata)
    return new_code_cell(source=source, metadata=metadata)

def code_to_cell(self, lines, parse_opt):
    """
    Parse code to a notebook cell
    :param self:
    :param lines:
    :param parse_opt:
    :return: cell, cursor
    """
    # Parse options
    if parse_opt:
        language, metadata = parse_code_options(lines[0], self.ext)
        if self.ext == '.Rmd':
            metadata['language'] = language
    else:
        metadata = {}

    if self.ext == '.Rmd':
        end_cell_re = _END_CODE_MD
    elif 'cellend' in metadata:
        end_cell_re = re.compile('^#( )' + metadata['cellend'] + '\s*$')
        del metadata['cellend']
    else:
        end_cell_re = None

    # Find end of cell and return
    if end_cell_re:
        for pos, line in enumerate(lines):
            if pos > 0 and end_cell_re.match(line):
                next_line_blank = pos + 1 == len(lines) or \
                                  _BLANK_LINE.match(lines[pos + 1])
                if next_line_blank and pos + 2 != len(lines):
                    return code_or_raw_cell(
                        source=lines[1:pos], metadata=metadata), pos + 2
                cell = code_or_raw_cell(
                    source=lines[1:pos], metadata=metadata)
                cell.metadata['noskipline'] = True
                return cell, pos + 1
    else:
        prev_blank = False
        triple = None
        for pos, line in enumerate(lines):
            if parse_opt and pos == 0:
                continue

            if self.ext == '.R' and line.startswith(self.prefix):
                if prev_blank:
                    return code_or_raw_cell(
                        source=lines[parse_opt:(pos - 1)],
                        metadata=metadata), pos
                cell = code_or_raw_cell(
                    source=lines[parse_opt:pos], metadata=metadata)
                cell.metadata['noskipline'] = True
                return cell, pos

            if self.ext == '.py':
                single = None
                for i, char in enumerate(line):
                    if char not in ['"', "'"]:
                        continue

                    if single == char:
                        single = None
                        continue
                    if single is not None:
                        continue
                    if triple == char:
                        if line[i - 2:i + 1] == 3 * char:
                            triple = None
                            continue
                    if triple is not None:
                        continue
                    if line[i - 2:i + 1] == 3 * char:
                        triple = char

                if triple:
                    continue

            if prev_blank:
                if _BLANK_LINE.match(line):
                    # Two blank lines => end of cell
                    # (unless next code is indented)
                    # Two blank lines at the end == empty code cell

                    if self.ext == '.py':
                        if next_code_is_indented(lines[pos:]):
                            continue

                    return code_or_raw_cell(
                        source=lines[parse_opt:(pos - 1)],
                        metadata=metadata), min(pos + 1,
                                                len(lines) - 1)

                # are all the lines from here to next blank
                # escaped with the prefix?
                if self.ext == '.py':
                    if no_code_before_next_blank_line(lines[pos:]):
                        return code_or_raw_cell(
                            source=lines[parse_opt:(pos - 1)],
                            metadata=metadata), pos

            prev_blank = _BLANK_LINE.match(line)

    # Unterminated cell?
    return code_or_raw_cell(
        source=lines[parse_opt:],
        metadata=metadata), len(lines)

def markdown_to_cell(self, lines):
    """
    Parse text and return a markdown cell
    :param self:
    :param lines:
    :return: cell, cursor
    """
    markdown = []
    for pos, line in enumerate(lines):
        # Markdown stops with the end of comments
        if line.startswith(self.prefix):
            markdown.append(self.markdown_unescape(line))
        elif _BLANK_LINE.match(line):
            return new_markdown_cell(source='\n'.join(markdown)), pos + 1
        else:
            cell = new_markdown_cell(source='\n'.join(markdown))
            cell.metadata['noskipline'] = True
            return cell, pos

    # still here => unterminated markdown
    return new_markdown_cell(source='\n'.join(markdown)), len(lines)

def markdown_to_cell_rmd(lines):
    """
    Parse text, in case of a rmd file, and return a markdown cell
    :param lines:
    :return: cell, cursor
    """
    prev_blank = False
    for pos, line in enumerate(lines):
        if start_code_rmd(line):
            if prev_blank and pos > 1:
                return new_markdown_cell(
                    source='\n'.join(lines[:(pos - 1)])), pos
            cell = new_markdown_cell(
                source='\n'.join(lines[:pos]))
            cell.metadata['noskipline'] = True
            return cell, pos

        if _BLANK_LINE.match(line) and prev_blank:
            return new_markdown_cell(
                source='\n'.join(lines[:(pos - 1)])), pos + 1
        prev_blank = _BLANK_LINE.match(line)

    # Unterminated cell?
    return new_markdown_cell(source='\n'.join(lines)), len(lines)
<commit_msg>pylint<commit_after>"""Cell conversion methods.

Convert/read notebook cells to/from python, R and Rmd cells
"""

import re
from nbformat.v4.nbbase import new_code_cell, new_raw_cell, new_markdown_cell
from .languages import cell_language, is_code
from .cell_metadata import metadata_to_rmd_options, rmd_options_to_metadata, \
    json_options_to_metadata, metadata_to_json_options

def code_to_rmd(source, metadata, default_language):
    """
    Represent a code cell with given source and metadata as a rmd cell
    :param source:
    :param metadata:
    :param default_language:
    :return:
    """
    lines = []
    language = cell_language(source) or default_language
    if 'Rmd' not in re.split('\\.|,', metadata.get('active', 'Rmd')):
        metadata['eval'] = False
    options = metadata_to_rmd_options(language, metadata)
    lines.append(u'```{{{}}}'.format(options))
    lines.extend(source)
    lines.append(u'```')
    return lines

def code_to_text(self,
                 source,
                 metadata,
                 default_language,
                 next_cell_is_code):
    """
    Represent a code cell with given source and metadata as text
    :param self:
    :param source:
    :param metadata:
    :param default_language:
    :param next_cell_is_code:
    :return:
    """

    # Escape jupyter magics
    source = ['# ' + s if _MAGIC.match(s) else s for s in source]

    if self.ext == '.Rmd':
        return code_to_rmd(source, metadata, default_language)
    else:
        lines = []
        language = cell_language(source) or default_language
        if language == default_language:
            if self.ext == '.R':
                options = metadata_to_rmd_options(language, metadata)[2:]
                if options != '':
                    lines.append('#+ ' + options)
            else:
                # Issue #31:  does the cell ends with a blank line?
                # Do we find two blank lines in the cell? In that case
                # we add an end-of-cell marker
                if (source and _BLANK_LINE.match(source[-1])) or \
                        code_to_cell(self, source, False)[1] != len(source):
                    cellend = '-'
                    while True:
                        cellend_re = re.compile(r'^#( )' + cellend + r'\s*$')
                        if list(filter(cellend_re.match, source)):
                            cellend = cellend + '-'
                        else:
                            break
                    metadata['cellend'] = cellend

                options = metadata_to_json_options(metadata)
                if options != '{}':
                    lines.append('# + ' + options)
            lines.extend(source)
            if 'cellend' in metadata:
                lines.append('# ' + metadata['cellend'])
        else:
            lines.extend(self.markdown_escape(
                code_to_rmd(source, metadata, default_language)))

        # Two blank lines before next code cell
        if next_cell_is_code:
            lines.append('')

        return lines

def cell_to_text(self,
                 cell,
                 next_cell=None,
                 default_language='python'):
    """
    Represent a notebook cell as a text cell, in either
    py, R or rmd format
    :param self: TextNotebookWriter object
    :param cell: current cell
    :param next_cell: next cell
    :param default_language: default language for the current notebook
    :return:
    """
    source = cell.get('source').splitlines()
    metadata = cell.get('metadata', {})
    skipline = True
    if 'noskipline' in metadata:
        skipline = not metadata['noskipline']
        del metadata['noskipline']

    lines = []
    if is_code(cell):
        lines.extend(code_to_text(self, source, metadata, default_language,
                                  next_cell and is_code(next_cell)))
    else:
        if source == []:
            source = ['']
        lines.extend(self.markdown_escape(source))

        # Two blank lines between consecutive markdown cells in Rmd
        if self.ext == '.Rmd' and next_cell \
                and next_cell.cell_type == 'markdown':
            lines.append('')

    if skipline and next_cell:
        lines.append('')

    return lines

_START_CODE_RMD = re.compile(r"^```\{(.*)\}\s*$")
_END_CODE_MD = re.compile(r"^```\s*$")
_CODE_OPTION_RPY = re.compile(r"^(#|# )\+(.*)$")
_BLANK_LINE = re.compile(r"^\s*$")
_MAGIC = re.compile(r"^(# |#)*%")

def start_code_rmd(line):
    """
    Line indicates that a code cell starts, in a rmd file
    :param line:
    :return:
    """
    return _START_CODE_RMD.match(line)

def start_code_rpy(line):
    """
    A code cell starts here, in a py or R file
    :param line:
    :return:
    """
    return _CODE_OPTION_RPY.match(line)

def next_uncommented_is_code(lines):
    """
    Next non-commented line is code
    :param lines:
    :return:
    """
    for line in lines:
        if line.startswith('#'):
            continue
        return not _BLANK_LINE.match(line)

    return False

# Unescape jupyter magics
def uncomment(line):
    if line.startswith('# '):
        return line[2:]
    elif line.startswith('#'):
        return line[1:]
    else:
        return line

def unescape_magic(source):
    return [uncomment(s) if _MAGIC.match(s) else s for s in source]

def text_to_cell(self, lines):
    """
    Parse text to a cell
    :param self:
    :param lines:
    :return: cell, cursor
    """
    if self.start_code(lines[0]):
        return self.code_to_cell(lines, parse_opt=True)
    elif self.prefix != '' and not lines[0].startswith(self.prefix):
        return self.code_to_cell(lines, parse_opt=False)
    elif self.ext == '.py' and next_uncommented_is_code(lines):
        return self.code_to_cell(lines, parse_opt=False)

    return self.markdown_to_cell(lines)

def parse_code_options(line, ext):
    """
    Parse code options on the given line
    :param line:
    :param ext:
    :return:
    """
    if ext == '.Rmd':
        return rmd_options_to_metadata(_START_CODE_RMD.findall(line)[0])
    elif ext == '.R':
        return rmd_options_to_metadata(_CODE_OPTION_RPY.match(line).group(2))

    return 'python', json_options_to_metadata(
        _CODE_OPTION_RPY.match(line).group(2))

def next_code_is_indented(lines):
    """
    Is next code line indented?
    :param lines:
    :return:
    """
    for line in lines:
        if line.startswith('#'):
            continue
        if _BLANK_LINE.match(line):
            continue
        return line.startswith(' ')

    return False

def no_code_before_next_blank_line(lines):
    """
    Do we find code before next blank line?
    :param lines:
    :return:
    """
    for line in lines:
        if line.startswith('#'):
            continue
        return _BLANK_LINE.match(line)

    return True

def code_or_raw_cell(source, metadata):
    source = unescape_magic(source)
    source = '\n'.join(source)

    if 'ipynb' not in re.split('\\.|,', metadata.get('active', 'ipynb')):
        return new_raw_cell(source=source, metadata=metadata)
    return new_code_cell(source=source, metadata=metadata)

def code_to_cell(self, lines, parse_opt):
    """
    Parse code to a notebook cell
    :param self:
    :param lines:
    :param parse_opt:
    :return: cell, cursor
    """
    # Parse options
    if parse_opt:
        language, metadata = parse_code_options(lines[0], self.ext)
        if self.ext == '.Rmd':
            metadata['language'] = language
    else:
        metadata = {}

    if self.ext == '.Rmd':
        end_cell_re = _END_CODE_MD
    elif 'cellend' in metadata:
        end_cell_re = re.compile(r'^#( )' + metadata['cellend'] + r'\s*$')
        del metadata['cellend']
    else:
        end_cell_re = None

    # Find end of cell and return
    if end_cell_re:
        for pos, line in enumerate(lines):
            if pos > 0 and end_cell_re.match(line):
                next_line_blank = pos + 1 == len(lines) or \
                                  _BLANK_LINE.match(lines[pos + 1])
                if next_line_blank and pos + 2 != len(lines):
                    return code_or_raw_cell(
                        source=lines[1:pos], metadata=metadata), pos + 2
                cell = code_or_raw_cell(
                    source=lines[1:pos], metadata=metadata)
                cell.metadata['noskipline'] = True
                return cell, pos + 1
    else:
        prev_blank = False
        triple = None
        for pos, line in enumerate(lines):
            if parse_opt and pos == 0:
                continue

            if self.ext == '.R' and line.startswith(self.prefix):
                if prev_blank:
                    return code_or_raw_cell(
                        source=lines[parse_opt:(pos - 1)],
                        metadata=metadata), pos
                cell = code_or_raw_cell(
                    source=lines[parse_opt:pos], metadata=metadata)
                cell.metadata['noskipline'] = True
                return cell, pos

            if self.ext == '.py':
                single = None
                for i, char in enumerate(line):
                    if char not in ['"', "'"]:
                        continue

                    if single == char:
                        single = None
                        continue
                    if single is not None:
                        continue
                    if triple == char:
                        if line[i - 2:i + 1] == 3 * char:
                            triple = None
                            continue
                    if triple is not None:
                        continue
                    if line[i - 2:i + 1] == 3 * char:
                        triple = char

                if triple:
                    continue

            if prev_blank:
                if _BLANK_LINE.match(line):
                    # Two blank lines => end of cell
                    # (unless next code is indented)
                    # Two blank lines at the end == empty code cell

                    if self.ext == '.py':
                        if next_code_is_indented(lines[pos:]):
                            continue

                    return code_or_raw_cell(
                        source=lines[parse_opt:(pos - 1)],
                        metadata=metadata), min(pos + 1,
                                                len(lines) - 1)

                # are all the lines from here to next blank
                # escaped with the prefix?
                if self.ext == '.py':
                    if no_code_before_next_blank_line(lines[pos:]):
                        return code_or_raw_cell(
                            source=lines[parse_opt:(pos - 1)],
                            metadata=metadata), pos

            prev_blank = _BLANK_LINE.match(line)

    # Unterminated cell?
    return code_or_raw_cell(
        source=lines[parse_opt:],
        metadata=metadata), len(lines)

def markdown_to_cell(self, lines):
    """
    Parse text and return a markdown cell
    :param self:
    :param lines:
    :return: cell, cursor
    """
    markdown = []
    for pos, line in enumerate(lines):
        # Markdown stops with the end of comments
        if line.startswith(self.prefix):
            markdown.append(self.markdown_unescape(line))
        elif _BLANK_LINE.match(line):
            return new_markdown_cell(source='\n'.join(markdown)), pos + 1
        else:
            cell = new_markdown_cell(source='\n'.join(markdown))
            cell.metadata['noskipline'] = True
            return cell, pos

    # still here => unterminated markdown
    return new_markdown_cell(source='\n'.join(markdown)), len(lines)

def markdown_to_cell_rmd(lines):
    """
    Parse text, in case of a rmd file, and return a markdown cell
    :param lines:
    :return: cell, cursor
    """
    prev_blank = False
    for pos, line in enumerate(lines):
        if start_code_rmd(line):
            if prev_blank and pos > 1:
                return new_markdown_cell(
                    source='\n'.join(lines[:(pos - 1)])), pos
            cell = new_markdown_cell(
                source='\n'.join(lines[:pos]))
            cell.metadata['noskipline'] = True
            return cell, pos

        if _BLANK_LINE.match(line) and prev_blank:
            return new_markdown_cell(
                source='\n'.join(lines[:(pos - 1)])), pos + 1
        prev_blank = _BLANK_LINE.match(line)

    # Unterminated cell?
    return new_markdown_cell(source='\n'.join(lines)), len(lines)
<|endoftext|>