<commit_before>import os
import json
import imp
import mimetypes
import xlrd
import csv
import re
import requests
import time

from httplib import BadStatusLine
from flask import Flask, render_template, send_from_directory, Response
from jinja2 import FileSystemLoader, ChoiceLoader, Markup
from jinja2.loaders import BaseLoader
from jinja2.utils import open_if_exists
from jinja2.exceptions import TemplateNotFound
from jinja2._compat import string_types, iteritems
from pprint import pformat
from slughifi import slughifi
from string import uppercase
from werkzeug.wsgi import FileWrapper
from utils import filter_files
from clint.textui import puts

from .oauth import get_drive_api

# in seconds
SPREADSHEET_CACHE_TTL = 4

# pass template variables to files with these mimetypes
TEMPLATE_TYPES = [
    "text/html",
    "text/css",
    "application/javascript",
]

def split_template_path(template):
    """Split a path into segments and perform a sanity check.  If it detects
    '..' in the path it will raise a `TemplateNotFound` error.
    """
    pieces = []
    for piece in template.split('/'):
        if os.path.sep in piece \
           or (os.path.altsep and os.path.altsep in piece) or \
           piece == os.path.pardir:
            raise TemplateNotFound(template)
        elif piece and piece != '.':
            pieces.append(piece)
    return pieces

class TarbellFileSystemLoader(BaseLoader):
    def __init__(self, searchpath, encoding='utf-8'):
        if isinstance(searchpath, string_types):
            searchpath = [searchpath]
        self.searchpath = list(searchpath)
        self.encoding = encoding

    def get_source(self, environment, template):
        pieces = split_template_path(template)
        for searchpath in self.searchpath:
            filename = os.path.join(searchpath, *pieces)
            f = open_if_exists(filename)
            if f is None:
                continue
            try:
                contents = f.read().decode(self.encoding)
            finally:
                f.close()

            mtime = os.path.getmtime(filename)
            def uptodate():
                try:
                    return os.path.getmtime(filename) == mtime
                except OSError:
                    return False
            return contents, filename, uptodate
        raise TemplateNotFound(template)

    def list_templates(self):
        found = set()
        for searchpath in self.searchpath:
            for dirpath, dirnames, filenames in filter_files(searchpath):
                for filename in filenames:
                    template = os.path.join(dirpath, filename) \
                        [len(searchpath):].strip(os.path.sep) \
                                          .replace(os.path.sep, '/')
                    if template[:2] == './':
                        template = template[2:]
                    if template not in found:
                        found.add(template)
        return sorted(found)

def silent_none(value):
    if value is None:
        return ''
    return value

def pprint_lines(value):
    pformatted = pformat(value, width=1, indent=4)
    formatted = "{0}\n {1}\n{2}".format(
        pformatted[0],
        pformatted[1:-1],
        pformatted[-1]
    )
    return Markup(formatted)

def process_xlsx(content):
    """Turn Excel file contents into Tarbell worksheet data"""
    data = {}
    workbook = xlrd.open_workbook(file_contents=content)
    worksheets = workbook.sheet_names()
    for worksheet_name in worksheets:
        worksheet = workbook.sheet_by_name(worksheet_name)
        worksheet.name = slughifi(worksheet.name)
        headers = make_headers(worksheet)
        worksheet_data = make_worksheet_data(headers, worksheet)
        data[worksheet.name] = worksheet_data
    return data

def copy_global_values(data):
    """Copy values worksheet into global namespace."""
    for k, v in data['values'].items():
        if not data.get(k):
            data[k] = v
        else:
            puts("There is both a worksheet and a "
                 "value named '{0}'. The worksheet data "
                 "will be preserved.".format(k))
    data.pop("values", None)
    return data

def make_headers(worksheet):
    """Make headers"""
    headers = {}
    cell_idx = 0
    while cell_idx < worksheet.ncols:
        cell_type = worksheet.cell_type(0, cell_idx)
        if cell_type == 1:
            header = slughifi(worksheet.cell_value(0, cell_idx))
            if not header.startswith("_"):
                headers[cell_idx] = header
        cell_idx += 1
    return headers

def make_worksheet_data(headers, worksheet):
    # Make data
    data = []
    row_idx = 1
    while row_idx < worksheet.nrows:
        cell_idx = 0
        row_dict = {}
        while cell_idx < worksheet.ncols:
            cell_type = worksheet.cell_type(row_idx, cell_idx)
            if cell_type > 0 and cell_type < 5:
                cell_value = worksheet.cell_value(row_idx, cell_idx)
                try:
                    row_dict[headers[cell_idx]] = cell_value
                except KeyError:
                    try:
                        column = uppercase[cell_idx]
                    except IndexError:
                        column = cell_idx
                        puts("There is no header for cell with value '{0}' in column '{1}' of '{2}'" .format(
                            cell_value, column, worksheet.name
                        ))
            cell_idx += 1
        data.append(row_dict)
        row_idx += 1

    # Magic key handling
    if 'key' in headers.values():
        keyed_data = {}
        for row in data:
            if 'key' in row.keys():
                key = slughifi(row['key'])
                if keyed_data.get(key):
                    puts("There is already a key named '{0}' with value "
                           "'{1}' in '{2}'. It is being overwritten with "
                           "value '{3}'.".format(key,
                                   keyed_data.get(key),
                                   worksheet.name,
                                   row))

                # Magic values worksheet
                if worksheet.name == "values":
                    value = row.get('value')
                    if value:
                        keyed_data[key] = value
                else:
                    keyed_data[key] = row

        data = keyed_data

    return data

class TarbellSite:
    def __init__(self, path, client_secrets_path=None):
        self.app = Flask(__name__)

        self.app.jinja_env.finalize = silent_none  # Don't print "None"
        self.app.debug = True  # Always debug

        self.path = path
        self.project, self.base = self.load_project(path)

        self.data = {}
        self.expires = 0

        self.app.add_url_rule('/', view_func=self.preview)
        self.app.add_url_rule('/<path:path>', view_func=self.preview)
        self.app.add_template_filter(slughifi, 'slugify')
        self.app.add_template_filter(pprint_lines, 'pprint_lines')

    def load_project(self, path):
        base = None
        base_dir = os.path.join(path, "_base/")

        # Get the base as register it as a blueprint
        if os.path.exists(os.path.join(base_dir, "base.py")):
            filename, pathname, description = imp.find_module('base', [base_dir])
            base = imp.load_module('base', filename, pathname, description)
            self.app.register_blueprint(base.blueprint)
        else:
            puts("No _base/base.py file found")

        filename, pathname, description = imp.find_module('tarbell_config', [path])
        project = imp.load_module('project', filename, pathname, description)

        try:
            self.key = project.SPREADSHEET_KEY
            self.client = get_drive_api(self.path)
        except AttributeError:
            self.key = None
            self.client = None

        try:
            project.CREATE_JSON
        except AttributeError:
            project.CREATE_JSON = True

        try:
            project.DEFAULT_CONTEXT
        except AttributeError:
            project.DEFAULT_CONTEXT = {}

        try:
            self.app.register_blueprint(project.blueprint)
        except AttributeError:
            pass

        # Set up template loaders
        template_dirs = [path]
        if os.path.isdir(base_dir):
            template_dirs.append(base_dir)

        self.app.jinja_loader = TarbellFileSystemLoader(template_dirs)

        return project, base

    def preview(self, path=None, extra_context=None, publish=False):
        """ Preview a project path """
        if path is None:
            path = 'index.html'

        ## Serve JSON
        if self.project.CREATE_JSON and path == 'data.json':
            context = self.get_context(publish)
            return Response(json.dumps(context), mimetype="application/json")

        ## Detect files
        filepath = None
        for root, dirs, files in filter_files(self.path):
            # Does it exist under _base?
            basepath = os.path.join(root, "_base", path)
            try:
                with open(basepath):
                    mimetype, encoding = mimetypes.guess_type(basepath)
                    filepath = basepath
            except IOError:
                pass

            # Does it exist under regular path?
            fullpath = os.path.join(root, path)
            try:
                with open(fullpath):
                    mimetype, encoding = mimetypes.guess_type(fullpath)
                    filepath = fullpath
            except IOError:
                pass

        if filepath and mimetype and mimetype in TEMPLATE_TYPES:
            context = self.get_context(publish)
            # Mix in defaults
            context.update({
                "PROJECT_PATH": self.path,
                "PREVIEW_SERVER": not publish,
                "ROOT_URL": "127.0.0.1:5000",
                "PATH": path,
            })
            if extra_context:
                context.update(extra_context)
            rendered = render_template(path, **context)
            return Response(rendered, mimetype=mimetype)

        elif filepath:
            dir, filename = os.path.split(filepath)
            return send_from_directory(dir, filename)

        return Response(status=404)

    def get_context(self, publish=False):
        """
        Use optional CONTEXT_SOURCE_FILE setting to determine data source.
        Return the parsed data.

        Can be an http|https url or local file. Supports csv and excel files.
        """
        context = self.project.DEFAULT_CONTEXT
        try:
            file = self.project.CONTEXT_SOURCE_FILE
            # CSV
            if re.search(r'(csv|CSV)$', file):
                context.update(self.get_context_from_csv())
            # Excel
            if re.search(r'(xlsx|XLSX|xls|XLS)$', file):
                pass
        except AttributeError:
            context.update(self.get_context_from_gdoc())

        return context

    def get_context_from_csv(self):
        """
        Open CONTEXT_SOURCE_FILE, parse and return a context
        """
        if re.search('^(http|https)://', self.project.CONTEXT_SOURCE_FILE):
            data = requests.get(self.project.CONTEXT_SOURCE_FILE)
            reader = csv.reader(
                data.iter_lines(), delimiter=',', quotechar='"')
            ret = {rows[0]: rows[1] for rows in reader}
        else:
            try:
                with open(self.project.CONTEXT_SOURCE_FILE) as csvfile:
                    reader = csv.reader(csvfile, delimiter=',', quotechar='"')
                    ret = {rows[0]: rows[1] for rows in reader}
            except IOError:
                file = "%s/%s" % (
                    os.path.abspath(self.path),
                    self.project.CONTEXT_SOURCE_FILE)
                with open(file) as csvfile:
                    reader = csv.reader(csvfile, delimiter=',', quotechar='"')
                    ret = {rows[0]: rows[1] for rows in reader}
        ret.update({
            "CONTEXT_SOURCE_FILE": self.project.CONTEXT_SOURCE_FILE,
        })
        return ret

    def get_context_from_gdoc(self):
        """Wrap getting context in a simple caching mechanism."""
        try:
            start = int(time.time())
            if not self.data or start > self.expires:
                self.data = self._get_context_from_gdoc(self.project.SPREADSHEET_KEY)
                end = int(time.time())
                self.expires = end + SPREADSHEET_CACHE_TTL
            return self.data
        except AttributeError:
            return {}

    def _get_context_from_gdoc(self, key):
        """Create a Jinja2 context from a Google spreadsheet."""
        try:
            content = self.export_xlsx(key)
            data = process_xlsx(content)
            if 'values' in data:
                data = copy_global_values(data)
            data.update({
                "SPREADSHEET_KEY": key,
            })
            return data
        except BadStatusLine:
            # Stale connection, reset project and data
            self.project, self.base = self.load_project(path)
            self.data = {}
            return self._get_context_from_gdoc(key)

    def export_xlsx(self, key):
        """Download xlsx version of spreadsheet"""
        spreadsheet_file = self.client.files().get(fileId=key).execute()
        links = spreadsheet_file.get('exportLinks')
        downloadurl = links.get('application/vnd.openxmlformats-officedocument.spreadsheetml.sheet')
        resp, content = self.client._http.request(downloadurl)
        return content

    def generate_static_site(self, output_root, extra_context):
        base_dir = os.path.join(self.path, "_base/")

        for root, dirs, files in filter_files(base_dir):
            for filename in files:
                self._copy_file(root.replace("_base/", ""), filename, output_root, extra_context)

        for root, dirs, files in filter_files(self.path):
            for filename in files:
                self._copy_file(root, filename, output_root, extra_context)

    def _copy_file(self, root, filename, output_root, extra_context=None):
        # Strip out full filesystem paths
        path = os.path.join(root, filename)
        rel_path = os.path.join(root.replace(self.path, ""), filename)
        if rel_path.startswith("/"):
            rel_path = rel_path[1:]
        output_path = os.path.join(output_root, rel_path)
        output_dir = os.path.dirname(output_path)

        puts("Writing {0}".format(output_path))
        with self.app.test_request_context():
            preview = self.preview(rel_path, extra_context=extra_context, publish=True)
            if not os.path.exists(output_dir):
                os.makedirs(output_dir)
            with open(output_path, "wb") as f:
                if isinstance(preview.response, FileWrapper):
                    f.write(preview.response.file.read())
                else:
                    f.write(preview.data)
<commit_msg>catch template syntax errors, fixes #40<commit_after>import os
import json
import imp
import mimetypes
import xlrd
import csv
import re
import requests
import time

from httplib import BadStatusLine
from flask import Flask, render_template, send_from_directory, Response
from jinja2 import FileSystemLoader, ChoiceLoader, Markup, TemplateSyntaxError
from jinja2.loaders import BaseLoader
from jinja2.utils import open_if_exists
from jinja2.exceptions import TemplateNotFound
from jinja2._compat import string_types, iteritems
from pprint import pformat
from slughifi import slughifi
from string import uppercase
from werkzeug.wsgi import FileWrapper
from utils import filter_files
from clint.textui import puts, colored

from .oauth import get_drive_api

# in seconds
SPREADSHEET_CACHE_TTL = 4

# pass template variables to files with these mimetypes
TEMPLATE_TYPES = [
    "text/html",
    "text/css",
    "application/javascript",
]

def split_template_path(template):
    """Split a path into segments and perform a sanity check.  If it detects
    '..' in the path it will raise a `TemplateNotFound` error.
    """
    pieces = []
    for piece in template.split('/'):
        if os.path.sep in piece \
           or (os.path.altsep and os.path.altsep in piece) or \
           piece == os.path.pardir:
            raise TemplateNotFound(template)
        elif piece and piece != '.':
            pieces.append(piece)
    return pieces

class TarbellFileSystemLoader(BaseLoader):
    def __init__(self, searchpath, encoding='utf-8'):
        if isinstance(searchpath, string_types):
            searchpath = [searchpath]
        self.searchpath = list(searchpath)
        self.encoding = encoding

    def get_source(self, environment, template):
        pieces = split_template_path(template)
        for searchpath in self.searchpath:
            filename = os.path.join(searchpath, *pieces)
            f = open_if_exists(filename)
            if f is None:
                continue
            try:
                contents = f.read().decode(self.encoding)
            finally:
                f.close()

            mtime = os.path.getmtime(filename)
            def uptodate():
                try:
                    return os.path.getmtime(filename) == mtime
                except OSError:
                    return False
            return contents, filename, uptodate
        raise TemplateNotFound(template)

    def list_templates(self):
        found = set()
        for searchpath in self.searchpath:
            for dirpath, dirnames, filenames in filter_files(searchpath):
                for filename in filenames:
                    template = os.path.join(dirpath, filename) \
                        [len(searchpath):].strip(os.path.sep) \
                                          .replace(os.path.sep, '/')
                    if template[:2] == './':
                        template = template[2:]
                    if template not in found:
                        found.add(template)
        return sorted(found)

def silent_none(value):
    if value is None:
        return ''
    return value

def pprint_lines(value):
    pformatted = pformat(value, width=1, indent=4)
    formatted = "{0}\n {1}\n{2}".format(
        pformatted[0],
        pformatted[1:-1],
        pformatted[-1]
    )
    return Markup(formatted)

def process_xlsx(content):
    """Turn Excel file contents into Tarbell worksheet data"""
    data = {}
    workbook = xlrd.open_workbook(file_contents=content)
    worksheets = workbook.sheet_names()
    for worksheet_name in worksheets:
        worksheet = workbook.sheet_by_name(worksheet_name)
        worksheet.name = slughifi(worksheet.name)
        headers = make_headers(worksheet)
        worksheet_data = make_worksheet_data(headers, worksheet)
        data[worksheet.name] = worksheet_data
    return data

def copy_global_values(data):
    """Copy values worksheet into global namespace."""
    for k, v in data['values'].items():
        if not data.get(k):
            data[k] = v
        else:
            puts("There is both a worksheet and a "
                 "value named '{0}'. The worksheet data "
                 "will be preserved.".format(k))
    data.pop("values", None)
    return data

def make_headers(worksheet):
    """Make headers"""
    headers = {}
    cell_idx = 0
    while cell_idx < worksheet.ncols:
        cell_type = worksheet.cell_type(0, cell_idx)
        if cell_type == 1:
            header = slughifi(worksheet.cell_value(0, cell_idx))
            if not header.startswith("_"):
                headers[cell_idx] = header
        cell_idx += 1
    return headers

def make_worksheet_data(headers, worksheet):
    # Make data
    data = []
    row_idx = 1
    while row_idx < worksheet.nrows:
        cell_idx = 0
        row_dict = {}
        while cell_idx < worksheet.ncols:
            cell_type = worksheet.cell_type(row_idx, cell_idx)
            if cell_type > 0 and cell_type < 5:
                cell_value = worksheet.cell_value(row_idx, cell_idx)
                try:
                    row_dict[headers[cell_idx]] = cell_value
                except KeyError:
                    try:
                        column = uppercase[cell_idx]
                    except IndexError:
                        column = cell_idx
                        puts("There is no header for cell with value '{0}' in column '{1}' of '{2}'" .format(
                            cell_value, column, worksheet.name
                        ))
            cell_idx += 1
        data.append(row_dict)
        row_idx += 1

    # Magic key handling
    if 'key' in headers.values():
        keyed_data = {}
        for row in data:
            if 'key' in row.keys():
                key = slughifi(row['key'])
                if keyed_data.get(key):
                    puts("There is already a key named '{0}' with value "
                           "'{1}' in '{2}'. It is being overwritten with "
                           "value '{3}'.".format(key,
                                   keyed_data.get(key),
                                   worksheet.name,
                                   row))

                # Magic values worksheet
                if worksheet.name == "values":
                    value = row.get('value')
                    if value:
                        keyed_data[key] = value
                else:
                    keyed_data[key] = row

        data = keyed_data

    return data

class TarbellSite:
    def __init__(self, path, client_secrets_path=None):
        self.app = Flask(__name__)

        self.app.jinja_env.finalize = silent_none  # Don't print "None"
        self.app.debug = True  # Always debug

        self.path = path
        self.project, self.base = self.load_project(path)

        self.data = {}
        self.expires = 0

        self.app.add_url_rule('/', view_func=self.preview)
        self.app.add_url_rule('/<path:path>', view_func=self.preview)
        self.app.add_template_filter(slughifi, 'slugify')
        self.app.add_template_filter(pprint_lines, 'pprint_lines')

    def load_project(self, path):
        base = None
        base_dir = os.path.join(path, "_base/")

        # Get the base as register it as a blueprint
        if os.path.exists(os.path.join(base_dir, "base.py")):
            filename, pathname, description = imp.find_module('base', [base_dir])
            base = imp.load_module('base', filename, pathname, description)
            self.app.register_blueprint(base.blueprint)
        else:
            puts("No _base/base.py file found")

        filename, pathname, description = imp.find_module('tarbell_config', [path])
        project = imp.load_module('project', filename, pathname, description)

        try:
            self.key = project.SPREADSHEET_KEY
            self.client = get_drive_api(self.path)
        except AttributeError:
            self.key = None
            self.client = None

        try:
            project.CREATE_JSON
        except AttributeError:
            project.CREATE_JSON = True

        try:
            project.DEFAULT_CONTEXT
        except AttributeError:
            project.DEFAULT_CONTEXT = {}

        try:
            self.app.register_blueprint(project.blueprint)
        except AttributeError:
            pass

        # Set up template loaders
        template_dirs = [path]
        if os.path.isdir(base_dir):
            template_dirs.append(base_dir)

        self.app.jinja_loader = TarbellFileSystemLoader(template_dirs)

        return project, base

    def preview(self, path=None, extra_context=None, publish=False):
        """ Preview a project path """
        if path is None:
            path = 'index.html'

        ## Serve JSON
        if self.project.CREATE_JSON and path == 'data.json':
            context = self.get_context(publish)
            return Response(json.dumps(context), mimetype="application/json")

        ## Detect files
        filepath = None
        for root, dirs, files in filter_files(self.path):
            # Does it exist under _base?
            basepath = os.path.join(root, "_base", path)
            try:
                with open(basepath):
                    mimetype, encoding = mimetypes.guess_type(basepath)
                    filepath = basepath
            except IOError:
                pass

            # Does it exist under regular path?
            fullpath = os.path.join(root, path)
            try:
                with open(fullpath):
                    mimetype, encoding = mimetypes.guess_type(fullpath)
                    filepath = fullpath
            except IOError:
                pass

        if filepath and mimetype and mimetype in TEMPLATE_TYPES:
            context = self.get_context(publish)
            # Mix in defaults
            context.update({
                "PROJECT_PATH": self.path,
                "PREVIEW_SERVER": not publish,
                "ROOT_URL": "127.0.0.1:5000",
                "PATH": path,
            })
            if extra_context:
                context.update(extra_context)
            try:
                rendered = render_template(path, **context)
                return Response(rendered, mimetype=mimetype)
            except TemplateSyntaxError:
                puts("{0} can't be parsed by Jinja, serving static".format(colored.yellow(filepath)))

        if filepath:
            dir, filename = os.path.split(filepath)
            return send_from_directory(dir, filename)

        return Response(status=404)

    def get_context(self, publish=False):
        """
        Use optional CONTEXT_SOURCE_FILE setting to determine data source.
        Return the parsed data.

        Can be an http|https url or local file. Supports csv and excel files.
        """
        context = self.project.DEFAULT_CONTEXT
        try:
            file = self.project.CONTEXT_SOURCE_FILE
            # CSV
            if re.search(r'(csv|CSV)$', file):
                context.update(self.get_context_from_csv())
            # Excel
            if re.search(r'(xlsx|XLSX|xls|XLS)$', file):
                pass
        except AttributeError:
            context.update(self.get_context_from_gdoc())

        return context

    def get_context_from_csv(self):
        """
        Open CONTEXT_SOURCE_FILE, parse and return a context
        """
        if re.search('^(http|https)://', self.project.CONTEXT_SOURCE_FILE):
            data = requests.get(self.project.CONTEXT_SOURCE_FILE)
            reader = csv.reader(
                data.iter_lines(), delimiter=',', quotechar='"')
            ret = {rows[0]: rows[1] for rows in reader}
        else:
            try:
                with open(self.project.CONTEXT_SOURCE_FILE) as csvfile:
                    reader = csv.reader(csvfile, delimiter=',', quotechar='"')
                    ret = {rows[0]: rows[1] for rows in reader}
            except IOError:
                file = "%s/%s" % (
                    os.path.abspath(self.path),
                    self.project.CONTEXT_SOURCE_FILE)
                with open(file) as csvfile:
                    reader = csv.reader(csvfile, delimiter=',', quotechar='"')
                    ret = {rows[0]: rows[1] for rows in reader}
        ret.update({
            "CONTEXT_SOURCE_FILE": self.project.CONTEXT_SOURCE_FILE,
        })
        return ret

    def get_context_from_gdoc(self):
        """Wrap getting context in a simple caching mechanism."""
        try:
            start = int(time.time())
            if not self.data or start > self.expires:
                self.data = self._get_context_from_gdoc(self.project.SPREADSHEET_KEY)
                end = int(time.time())
                self.expires = end + SPREADSHEET_CACHE_TTL
            return self.data
        except AttributeError:
            return {}

    def _get_context_from_gdoc(self, key):
        """Create a Jinja2 context from a Google spreadsheet."""
        try:
            content = self.export_xlsx(key)
            data = process_xlsx(content)
            if 'values' in data:
                data = copy_global_values(data)
            data.update({
                "SPREADSHEET_KEY": key,
            })
            return data
        except BadStatusLine:
            # Stale connection, reset project and data
            self.project, self.base = self.load_project(path)
            self.data = {}
            return self._get_context_from_gdoc(key)

    def export_xlsx(self, key):
        """Download xlsx version of spreadsheet"""
        spreadsheet_file = self.client.files().get(fileId=key).execute()
        links = spreadsheet_file.get('exportLinks')
        downloadurl = links.get('application/vnd.openxmlformats-officedocument.spreadsheetml.sheet')
        resp, content = self.client._http.request(downloadurl)
        return content

    def generate_static_site(self, output_root, extra_context):
        base_dir = os.path.join(self.path, "_base/")

        for root, dirs, files in filter_files(base_dir):
            for filename in files:
                self._copy_file(root.replace("_base/", ""), filename, output_root, extra_context)

        for root, dirs, files in filter_files(self.path):
            for filename in files:
                self._copy_file(root, filename, output_root, extra_context)

    def _copy_file(self, root, filename, output_root, extra_context=None):
        # Strip out full filesystem paths
        path = os.path.join(root, filename)
        rel_path = os.path.join(root.replace(self.path, ""), filename)
        if rel_path.startswith("/"):
            rel_path = rel_path[1:]
        output_path = os.path.join(output_root, rel_path)
        output_dir = os.path.dirname(output_path)

        puts("Writing {0}".format(output_path))
        with self.app.test_request_context():
            preview = self.preview(rel_path, extra_context=extra_context, publish=True)
            if not os.path.exists(output_dir):
                os.makedirs(output_dir)
            with open(output_path, "wb") as f:
                if isinstance(preview.response, FileWrapper):
                    f.write(preview.response.file.read())
                else:
                    f.write(preview.data)
<|endoftext|>


<commit_before>from __future__ import division
from percept.tasks.base import Task
from percept.tasks.train import Train
from percept.fields.base import Complex, List, Dict, Float
from inputs.inputs import SimpsonsFormats
from percept.utils.models import RegistryCategories, get_namespace
import logging
from percept.tests.framework import Tester
from percept.conf.base import settings
import re
import os
from matplotlib import pyplot
import numpy as np
from scikits.audiolab import Sndfile
from scikits.audiolab import oggread
import pandas as pd
from multiprocessing import Pool
from sklearn.ensemble import RandomForestClassifier
import math
import random
from itertools import chain

log = logging.getLogger(__name__)

class LoadAudioFiles(Task):
    data = Complex()
    all_files = List()
    cv = Complex()
    res = Complex()

    data_format = SimpsonsFormats.dataframe

    category = RegistryCategories.preprocessors
    namespace = get_namespace(__module__)

    help_text = "Cleanup simpsons scripts."

    args = {
        'audio_dir' : os.path.abspath(os.path.join(settings.AUDIO_BASE_PATH, "audio")),
    }

    def train(self, data, target, **kwargs):
        """
        Used in the training phase.  Override.
        """
        self.data = self.predict(data, **kwargs)

    def extract_season(self,name):
        match1 = re.search('\[(\d+)[x\.](\d+)\]',name)
        if match1 is not None:
            season = match1.group(1)
            episode = match1.group(2)
            return int(season),int(episode)

        match2 = re.search('S(\d+)E(\d+)',name)
        if match2 is not None:
            season = match2.group(1)
            episode = match2.group(2)
            return int(season),int(episode)

        return None, None

    def predict(self, data, **kwargs):
        """
        Used in the predict phase, after training.  Override
        """
        audio_dir = kwargs['audio_dir']
        all_files = []
        for ad in os.listdir(audio_dir):
            ad_path = os.path.abspath(os.path.join(audio_dir,ad))
            if os.path.isdir(ad_path):
                files = os.listdir(ad_path)
                all_files += [os.path.abspath(os.path.join(ad_path,f)) for f in files]
            else:
                all_files += [ad_path]
        self.all_files = [f for f in all_files if f.endswith(".ogg")]
        frames = []
        counter = 0
        for f in self.all_files:
            season,episode = self.extract_season(f)
            if season is None:
                continue
            subtitle_frame = data[((data['season']==season) & (data['episode']==episode))]
            if subtitle_frame.shape[0]==0:
                continue

            #To cause loop to end early, remove if needed
            label_frame = subtitle_frame[(subtitle_frame['label']!="")]
            if label_frame.shape[0]==0:
                continue
            if counter>5:
                break

            counter+=1
            print "On file {0}".format(counter)
            f_data, fs, enc  = oggread(f)
            subtitle_frame = subtitle_frame.sort('start')
            subtitle_frame.index = range(subtitle_frame.shape[0])
            samps = []
            for i in xrange(0,subtitle_frame.shape[0]):
                start = subtitle_frame['start'].iloc[i]
                end = subtitle_frame['end'].iloc[i]
                samp = f_data[(start*fs):(end*fs),:]
                samps.append({'samp' : samp, 'fs' : fs})
            p = Pool(4)
            results = p.map(process_subtitle, samps)
            good_rows = [i for i in xrange(0,len(results)) if results[i]!=None]
            audio_features = [i for i in results if i!=None]
            df = pd.concat([subtitle_frame.iloc[good_rows],pd.DataFrame(audio_features)],axis=1)
            df = df.fillna(-1)
            df.index = range(df.shape[0])
            frames.append(df)
        data = pd.concat(frames,axis=0)
        data.index = range(data.shape[0])
        label_codes = {k:i for (i,k) in enumerate(set(data['label']))}
        reverse_label_codes = {label_codes[k]:k for k in label_codes}
        data['label_code'] = [label_codes[k] for k in data['label']]
        for c in list(data.columns):
            data[c] = data[c].real
        self.cv = CrossValidate()

        cv_frame = data[data['label']!=""]
        self.cv.train(cv_frame,"",**self.cv.args)
        self.res = self.cv.results
        self.res = self.res[['line', 'label','label_code','result']]
        self.res['actual_result'] = [reverse_label_codes[i] for i in self.res['result']]
        return data

def calc_slope(x,y):
    x_mean = np.mean(x)
    y_mean = np.mean(y)
    x_dev = np.sum(np.abs(np.subtract(x,x_mean)))
    y_dev = np.sum(np.abs(np.subtract(y,y_mean)))

    slope = (x_dev*y_dev)/(x_dev*x_dev)
    return slope

def get_indicators(vec):
    mean = sum(vec)
    slope = calc_slope(np.arange(len(vec)),vec)
    std = np.std(vec)
    return mean, slope, std

def calc_u(vec):
    fft = np.fft.fft(vec)
    return np.sum(np.multiply(fft,vec))/np.sum(vec)

def calc_features(vec,freq):
    #bin count
    bc = 10
    bincount = list(range(bc))
    #framesize
    fsize = 512
    #mean
    m = np.mean(vec)
    #spectral flux
    sf = np.mean(vec-np.roll(vec,fsize))
    mx = np.max(vec)
    mi = np.min(vec)
    sdev = np.std(vec)
    binwidth = len(vec)/bc
    bins = []
    for i in xrange(0,bc):
        bins.append(vec[(i*binwidth):(binwidth*i + binwidth)])
    peaks = [np.max(i) for i in bins]
    mins = [np.min(i) for i in bins]
    amin,smin,stmin = get_indicators(mins)
    apeak, speak, stpeak = get_indicators(peaks)
    fft = np.fft.fft(vec)
    bin_fft = []
    for i in xrange(0,bc):
        bin_fft.append(fft[(i*binwidth):(binwidth*i + binwidth)])

    cepstrums = [np.fft.ifft(np.log(np.abs(i))) for i in bin_fft]
    inter = [get_indicators(i) for i in cepstrums]
    acep,scep, stcep = get_indicators([i[0] for i in inter])
    aacep,sscep, stsscep = get_indicators([i[1] for i in inter])

    zero_crossings = np.where(np.diff(np.sign(vec)))[0]
    zcc = len(zero_crossings)
    zccn = zcc/freq

    u = [calc_u(i) for i in bins]
    spread = np.sqrt(u[-1] - u[0]**2)
    skewness = (u[0]**3 - 3*u[0]*u[5] + u[-1])/spread**3

    #Spectral slope
    ss = calc_slope(np.arange(len(fft)),fft)
    avss = calc_slope(bincount,[calc_slope(np.arange(len(i)),i) for i in bin_fft])

    return [m,sf,mx,mi,sdev,amin,smin,stmin,apeak,speak,stpeak,acep,scep,stcep,aacep,sscep,stsscep,zcc,zccn,spread,skewness,ss,avss]

def extract_features(sample,freq):
    left = calc_features(sample[:,0],freq)
    right = calc_features(sample[:,1],freq)
    return left+right

def process_subtitle(d):
    samp = d['samp']
    fs = d['fs']
    try:
        features = extract_features(samp,fs)
    except Exception:
        log.exception("Cannot generate features")
        return None

    return features

class RandomForestTrain(Train):
    """
    A class to train a random forest
    """
    colnames = List()
    clf = Complex()
    category = RegistryCategories.algorithms
    namespace = get_namespace(__module__)
    algorithm = RandomForestClassifier
    args = {'n_estimators' : 300, 'min_samples_leaf' : 4, 'compute_importances' : True}

    help_text = "Train and predict with Random Forest."

class CrossValidate(Task):
    data = Complex()
    results = Complex()
    error = Float()
    importances = Complex()
    importance = Complex()
    column_names = List()

    data_format = SimpsonsFormats.dataframe

    category = RegistryCategories.preprocessors
    namespace = get_namespace(__module__)
    args = {
        'nfolds' : 3,
        'algo' : RandomForestTrain,
        'target_name' : 'label_code',
        'non_predictors' : ["label","line","label_code"]
    }

    help_text = "Cross validate simpsons data."

    def cross_validate(self, data, **kwargs):
        nfolds = kwargs.get('nfolds', 3)
        algo = kwargs.get('algo')
        seed = kwargs.get('seed', 1)
        target_name = kwargs.get('target_name')
        non_predictors = kwargs.get('non_predictors')

        self.column_names = [l for l in list(data.columns) if l not in non_predictors]
        data_len = data.shape[0]
        counter = 0
        fold_length = int(math.floor(data_len/nfolds))
        folds = []
        data_seq = list(xrange(0,data_len))
        random.seed(seed)
        random.shuffle(data_seq)

        for fold in xrange(0, nfolds):
            start = counter

            end = counter + fold_length
            if fold == (nfolds-1):
                end = data_len
            folds.append(data_seq[start:end])
            counter += fold_length

        results = []
        data.index = range(data.shape[0])
        self.importances = []
        for (i,fold) in enumerate(folds):
            predict_data = data.iloc[fold,:]
            out_indices = list(chain.from_iterable(folds[:i] + folds[(i + 1):]))
            train_data = data.iloc[out_indices,:]
            alg = algo()
            target = train_data[target_name]
            train_data = train_data[[l for l in list(train_data.columns) if l not in non_predictors]]
            predict_data = predict_data[[l for l in list(predict_data.columns) if l not in non_predictors]]
            clf = alg.train(train_data,target,**algo.args)
            results.append(alg.predict(predict_data))
            self.importances.append(clf.feature_importances_)
        return results, folds

    def train(self, data, target, **kwargs):
        """
        Used in the training phase.  Override.
        """
        self.target_name = kwargs.get('target_name')
        results, folds = self.cross_validate(data, **kwargs)
        self.gather_results(results, folds, data)

    def gather_results(self, results, folds, data):
        full_results = list(chain.from_iterable(results))
        full_indices = list(chain.from_iterable(folds))
        partial_result_df = make_df([full_results, full_indices], ["result", "index"])
        partial_result_df = partial_result_df.sort(["index"])
        partial_result_df.index = range(partial_result_df.shape[0])
        result_df = pd.concat([partial_result_df, data], axis=1)
        self.results = result_df
        self.calc_error(result_df)
        self.calc_importance(self.importances, self.column_names)

    def calc_error(self, result_df):
        self.error = np.mean(np.abs(result_df['result'] - result_df[self.target_name]))

    def calc_importance(self, importances, col_names):
        importance_frame = pd.DataFrame(importances)
        importance_frame.columns = col_names
        self.importance = importance_frame.mean(axis=0)
        self.importance.sort(0)

    def predict(self, data, **kwargs):
        """
        Used in the predict phase, after training.  Override
        """
        pass

def make_df(datalist, labels, name_prefix=""):
    df = pd.DataFrame(datalist).T
    if name_prefix!="":
        labels = [name_prefix + "_" + l for l in labels]
    labels = [l.replace(" ", "_").lower() for l in labels]
    df.columns = labels
    df.index = range(df.shape[0])
    return df<commit_msg>Faster calcs with numpy<commit_after>from __future__ import division
from percept.tasks.base import Task
from percept.tasks.train import Train
from percept.fields.base import Complex, List, Dict, Float
from inputs.inputs import SimpsonsFormats
from percept.utils.models import RegistryCategories, get_namespace
import logging
from percept.tests.framework import Tester
from percept.conf.base import settings
import re
import os
from matplotlib import pyplot
import numpy as np
from scikits.audiolab import Sndfile
from scikits.audiolab import oggread
import pandas as pd
from multiprocessing import Pool
from sklearn.ensemble import RandomForestClassifier
import math
import random
from itertools import chain

log = logging.getLogger(__name__)

class LoadAudioFiles(Task):
    data = Complex()
    all_files = List()
    cv = Complex()
    res = Complex()

    data_format = SimpsonsFormats.dataframe

    category = RegistryCategories.preprocessors
    namespace = get_namespace(__module__)

    help_text = "Cleanup simpsons scripts."

    args = {
        'audio_dir' : os.path.abspath(os.path.join(settings.AUDIO_BASE_PATH, "audio")),
    }

    def train(self, data, target, **kwargs):
        """
        Used in the training phase.  Override.
        """
        self.data = self.predict(data, **kwargs)

    def extract_season(self,name):
        match1 = re.search('\[(\d+)[x\.](\d+)\]',name)
        if match1 is not None:
            season = match1.group(1)
            episode = match1.group(2)
            return int(season),int(episode)

        match2 = re.search('S(\d+)E(\d+)',name)
        if match2 is not None:
            season = match2.group(1)
            episode = match2.group(2)
            return int(season),int(episode)

        return None, None

    def predict(self, data, **kwargs):
        """
        Used in the predict phase, after training.  Override
        """
        audio_dir = kwargs['audio_dir']
        all_files = []
        for ad in os.listdir(audio_dir):
            ad_path = os.path.abspath(os.path.join(audio_dir,ad))
            if os.path.isdir(ad_path):
                files = os.listdir(ad_path)
                all_files += [os.path.abspath(os.path.join(ad_path,f)) for f in files]
            else:
                all_files += [ad_path]
        self.all_files = [f for f in all_files if f.endswith(".ogg")]
        frames = []
        counter = 0
        for f in self.all_files:
            season,episode = self.extract_season(f)
            if season is None:
                continue
            subtitle_frame = data[((data['season']==season) & (data['episode']==episode))]
            if subtitle_frame.shape[0]==0:
                continue

            #To cause loop to end early, remove if needed
            label_frame = subtitle_frame[(subtitle_frame['label']!="")]
            if label_frame.shape[0]==0:
                continue
            if counter>5:
                break

            counter+=1
            print "On file {0}".format(counter)
            f_data, fs, enc  = oggread(f)
            subtitle_frame = subtitle_frame.sort('start')
            subtitle_frame.index = range(subtitle_frame.shape[0])
            samps = []
            for i in xrange(0,subtitle_frame.shape[0]):
                start = subtitle_frame['start'].iloc[i]
                end = subtitle_frame['end'].iloc[i]
                samp = f_data[(start*fs):(end*fs),:]
                samps.append({'samp' : samp, 'fs' : fs})
            p = Pool(4)
            results = p.map(process_subtitle, samps)
            good_rows = [i for i in xrange(0,len(results)) if results[i]!=None]
            audio_features = [i for i in results if i!=None]
            df = pd.concat([subtitle_frame.iloc[good_rows],pd.DataFrame(audio_features)],axis=1)
            df = df.fillna(-1)
            df.index = range(df.shape[0])
            frames.append(df)
        data = pd.concat(frames,axis=0)
        data.index = range(data.shape[0])
        label_codes = {k:i for (i,k) in enumerate(set(data['label']))}
        reverse_label_codes = {label_codes[k]:k for k in label_codes}
        data['label_code'] = [label_codes[k] for k in data['label']]
        for c in list(data.columns):
            data[c] = data[c].real
        self.cv = CrossValidate()

        cv_frame = data[data['label']!=""]
        self.cv.train(cv_frame,"",**self.cv.args)
        self.res = self.cv.results
        self.res = self.res[['line', 'label','label_code','result']]
        self.res['actual_result'] = [reverse_label_codes[i] for i in self.res['result']]
        return data

def calc_slope(x,y):
    x_mean = np.mean(x)
    y_mean = np.mean(y)
    x_dev = np.sum(np.abs(np.subtract(x,x_mean)))
    y_dev = np.sum(np.abs(np.subtract(y,y_mean)))

    slope = (x_dev*y_dev)/(x_dev*x_dev)
    return slope

def get_indicators(vec):
    mean = np.sum(vec)
    slope = calc_slope(np.arange(len(vec)),vec)
    std = np.std(vec)
    return mean, slope, std

def calc_u(vec):
    fft = np.fft.fft(vec)
    return np.sum(np.multiply(fft,vec))/np.sum(vec)

def calc_features(vec,freq):
    #bin count
    bc = 10
    bincount = list(range(bc))
    #framesize
    fsize = 512
    #mean
    m = np.mean(vec)
    #spectral flux
    sf = np.mean(vec-np.roll(vec,fsize))
    mx = np.max(vec)
    mi = np.min(vec)
    sdev = np.std(vec)
    binwidth = len(vec)/bc
    bins = []
    for i in xrange(0,bc):
        bins.append(vec[(i*binwidth):(binwidth*i + binwidth)])
    peaks = [np.max(i) for i in bins]
    mins = [np.min(i) for i in bins]
    amin,smin,stmin = get_indicators(mins)
    apeak, speak, stpeak = get_indicators(peaks)
    fft = np.fft.fft(vec)
    bin_fft = []
    for i in xrange(0,bc):
        bin_fft.append(fft[(i*binwidth):(binwidth*i + binwidth)])

    cepstrums = [np.fft.ifft(np.log(np.abs(i))) for i in bin_fft]
    inter = [get_indicators(i) for i in cepstrums]
    acep,scep, stcep = get_indicators([i[0] for i in inter])
    aacep,sscep, stsscep = get_indicators([i[1] for i in inter])

    zero_crossings = np.where(np.diff(np.sign(vec)))[0]
    zcc = len(zero_crossings)
    zccn = zcc/freq

    u = [calc_u(i) for i in bins]
    spread = np.sqrt(u[-1] - u[0]**2)
    skewness = (u[0]**3 - 3*u[0]*u[5] + u[-1])/spread**3

    #Spectral slope
    ss = calc_slope(np.arange(len(fft)),fft)
    avss = calc_slope(bincount,[calc_slope(np.arange(len(i)),i) for i in bin_fft])

    return [m,sf,mx,mi,sdev,amin,smin,stmin,apeak,speak,stpeak,acep,scep,stcep,aacep,sscep,stsscep,zcc,zccn,spread,skewness,ss,avss]

def extract_features(sample,freq):
    left = calc_features(sample[:,0],freq)
    right = calc_features(sample[:,1],freq)
    return left+right

def process_subtitle(d):
    samp = d['samp']
    fs = d['fs']
    try:
        features = extract_features(samp,fs)
    except Exception:
        log.exception("Cannot generate features")
        return None

    return features

class RandomForestTrain(Train):
    """
    A class to train a random forest
    """
    colnames = List()
    clf = Complex()
    category = RegistryCategories.algorithms
    namespace = get_namespace(__module__)
    algorithm = RandomForestClassifier
    args = {'n_estimators' : 300, 'min_samples_leaf' : 4, 'compute_importances' : True}

    help_text = "Train and predict with Random Forest."

class CrossValidate(Task):
    data = Complex()
    results = Complex()
    error = Float()
    importances = Complex()
    importance = Complex()
    column_names = List()

    data_format = SimpsonsFormats.dataframe

    category = RegistryCategories.preprocessors
    namespace = get_namespace(__module__)
    args = {
        'nfolds' : 3,
        'algo' : RandomForestTrain,
        'target_name' : 'label_code',
        'non_predictors' : ["label","line","label_code"]
    }

    help_text = "Cross validate simpsons data."

    def cross_validate(self, data, **kwargs):
        nfolds = kwargs.get('nfolds', 3)
        algo = kwargs.get('algo')
        seed = kwargs.get('seed', 1)
        target_name = kwargs.get('target_name')
        non_predictors = kwargs.get('non_predictors')

        self.column_names = [l for l in list(data.columns) if l not in non_predictors]
        data_len = data.shape[0]
        counter = 0
        fold_length = int(math.floor(data_len/nfolds))
        folds = []
        data_seq = list(xrange(0,data_len))
        random.seed(seed)
        random.shuffle(data_seq)

        for fold in xrange(0, nfolds):
            start = counter

            end = counter + fold_length
            if fold == (nfolds-1):
                end = data_len
            folds.append(data_seq[start:end])
            counter += fold_length

        results = []
        data.index = range(data.shape[0])
        self.importances = []
        for (i,fold) in enumerate(folds):
            predict_data = data.iloc[fold,:]
            out_indices = list(chain.from_iterable(folds[:i] + folds[(i + 1):]))
            train_data = data.iloc[out_indices,:]
            alg = algo()
            target = train_data[target_name]
            train_data = train_data[[l for l in list(train_data.columns) if l not in non_predictors]]
            predict_data = predict_data[[l for l in list(predict_data.columns) if l not in non_predictors]]
            clf = alg.train(train_data,target,**algo.args)
            results.append(alg.predict(predict_data))
            self.importances.append(clf.feature_importances_)
        return results, folds

    def train(self, data, target, **kwargs):
        """
        Used in the training phase.  Override.
        """
        self.target_name = kwargs.get('target_name')
        results, folds = self.cross_validate(data, **kwargs)
        self.gather_results(results, folds, data)

    def gather_results(self, results, folds, data):
        full_results = list(chain.from_iterable(results))
        full_indices = list(chain.from_iterable(folds))
        partial_result_df = make_df([full_results, full_indices], ["result", "index"])
        partial_result_df = partial_result_df.sort(["index"])
        partial_result_df.index = range(partial_result_df.shape[0])
        result_df = pd.concat([partial_result_df, data], axis=1)
        self.results = result_df
        self.calc_error(result_df)
        self.calc_importance(self.importances, self.column_names)

    def calc_error(self, result_df):
        self.error = np.mean(np.abs(result_df['result'] - result_df[self.target_name]))

    def calc_importance(self, importances, col_names):
        importance_frame = pd.DataFrame(importances)
        importance_frame.columns = col_names
        self.importance = importance_frame.mean(axis=0)
        self.importance.sort(0)

    def predict(self, data, **kwargs):
        """
        Used in the predict phase, after training.  Override
        """
        pass

def make_df(datalist, labels, name_prefix=""):
    df = pd.DataFrame(datalist).T
    if name_prefix!="":
        labels = [name_prefix + "_" + l for l in labels]
    labels = [l.replace(" ", "_").lower() for l in labels]
    df.columns = labels
    df.index = range(df.shape[0])
    return df<|endoftext|>


<commit_before>def main():
    import sys

    from optparse import OptionParser
    parser = OptionParser(
            usage="usage: %prog [options] SCRIPT-TO-RUN [SCRIPT-ARGUMENTS]")

    parser.add_option("-s", "--steal-output", action="store_true"),
    parser.add_option("--pre-run", metavar="COMMAND",
            help="Run command before each program run")
    parser.disable_interspersed_args()
    options, args = parser.parse_args()

    if len(args) < 1:
        parser.print_help()
        sys.exit(2)

    mainpyfile =  args[0]
    from os.path import exists, dirname
    if not exists(mainpyfile):
        print 'Error:', mainpyfile, 'does not exist'
        sys.exit(1)

    sys.argv = args

    # Replace pudb's dir with script's dir in front of module search path.
    sys.path[0] = dirname(mainpyfile)

    # Note on saving/restoring sys.argv: it's a good idea when sys.argv was
    # modified by the script being debugged. It's a bad idea when it was
    # changed by the user from the command line. The best approach would be to
    # have a "restart" command which would allow explicit specification of
    # command line arguments.

    from pudb.debugger import Debugger
    dbg = Debugger(steal_output=options.steal_output)

    while True:
        if options.pre_run:
            from subprocess import call
            retcode = call(options.pre_run, close_fds=True, shell=True)
            if retcode:
                print "*** WARNING: pre-run process exited with code %d." % retcode
            pre_run_msg = ("\n\nIf you decide to restart, '%s' will be run prior to "
                    "actually restarting." % options.pre_run)
        else:
            pre_run_msg = ""

        status_msg = ""

        try:
            dbg._runscript(mainpyfile)
        except SystemExit, se:
            status_msg = "The debuggee exited normally with status code was %d.\n\n" % se.code
        except:
            dbg.post_mortem = True
            dbg.interaction(None, sys.exc_info())

        def quit_debugger(w, size, key):
            dbg.ui.quit_event_loop = ["quit"]

        import urwid
        result = dbg.ui.call_with_ui(dbg.ui.dialog,
            urwid.ListBox([urwid.Text(
                ("Your PuDB session has ended.\n\n%s"
                "Would you like to quit PuDB or restart your program?\n"
                "You may hit 'q' to quit."
                % status_msg)+pre_run_msg)]),
            [
                ("Restart", "restart"),
                ("Quit", "quit"),
                ],
            focus_buttons=True,
            bind_enter_esc=False,
            title="Finished",
            extra_bindings=[("q", quit_debugger)])

        if result == "quit":
            return

        dbg.restart()

if __name__=='__main__':
    main()

<commit_msg>Make pre-run command editable before restarting.<commit_after>def main():
    import sys

    from optparse import OptionParser
    parser = OptionParser(
            usage="usage: %prog [options] SCRIPT-TO-RUN [SCRIPT-ARGUMENTS]")

    parser.add_option("-s", "--steal-output", action="store_true"),
    parser.add_option("--pre-run", metavar="COMMAND",
            help="Run command before each program run",
            default="")
    parser.disable_interspersed_args()
    options, args = parser.parse_args()

    if len(args) < 1:
        parser.print_help()
        sys.exit(2)

    mainpyfile =  args[0]
    from os.path import exists, dirname
    if not exists(mainpyfile):
        print 'Error:', mainpyfile, 'does not exist'
        sys.exit(1)

    sys.argv = args

    # Replace pudb's dir with script's dir in front of module search path.
    sys.path[0] = dirname(mainpyfile)

    # Note on saving/restoring sys.argv: it's a good idea when sys.argv was
    # modified by the script being debugged. It's a bad idea when it was
    # changed by the user from the command line. The best approach would be to
    # have a "restart" command which would allow explicit specification of
    # command line arguments.

    from pudb.debugger import Debugger
    dbg = Debugger(steal_output=options.steal_output)

    while True:
        if options.pre_run:
            from subprocess import call
            retcode = call(options.pre_run, close_fds=True, shell=True)
            if retcode:
                print "*** WARNING: pre-run process exited with code %d." % retcode
                raw_input("[Hit Enter]")

        status_msg = ""

        try:
            dbg._runscript(mainpyfile)
        except SystemExit, se:
            status_msg = "The debuggee exited normally with status code was %d.\n\n" % se.code
        except:
            dbg.post_mortem = True
            dbg.interaction(None, sys.exc_info())

        def quit_debugger(w, size, key):
            dbg.ui.quit_event_loop = ["quit"]

        import urwid
        pre_run_edit = urwid.Edit("", options.pre_run)

        result = dbg.ui.call_with_ui(dbg.ui.dialog,
            urwid.ListBox([urwid.Text(
                "Your PuDB session has ended.\n\n%s"
                "Would you like to quit PuDB or restart your program?\n"
                "You may hit 'q' to quit."
                % status_msg),
                urwid.Text("\n\nIf you decide to restart, this command will be run prior to "
                "actually restarting:"),
                urwid.AttrWrap(pre_run_edit, "value")
                ]),
            [
                ("Restart", "restart"),
                ("Quit", "quit"),
                ],
            focus_buttons=True,
            bind_enter_esc=False,
            title="Finished",
            extra_bindings=[("q", quit_debugger)])

        if result == "quit":
            return

        options.pre_run = pre_run_edit.get_edit_text()

        dbg.restart()

if __name__=='__main__':
    main()

<|endoftext|>


<commit_before>"""
Sonogram
Author: Theo Brown (tab53)
"""

from numpy import fft
import numpy as np
import matplotlib.pyplot as plt
from mpl_toolkits.mplot3d import Axes3D
from matplotlib import cm
from scipy import signal

# Function source
def func_1(t, w, x):
    return np.exp((1j*w - x) * t)

def func_2(t, w):
    return np.sin(w*t)

def function_generator(t):
    """
    f1 = func_1(t, 50*2*np.pi, 5)
    f2 = func_1(t, 80*2*np.pi, 3)
    f1_echo = f1/3
    f2_echo = f2/4
    result = f1 + f2
    echo = f1_echo[:f1_echo.size/2] + f2_echo[:f2_echo.size/2]
    result[result.size/2:] += echo
    """
    f1 = func_2(t, 50*2*np.pi)
    f2 = func_2(t, 80*2*np.pi)
    result = f1 + 0.5*f2
    result[result.size/4:result.size/2] *= 0
    result[result.size*3/4:] *= 0
    return result

def dB(x):
    return 20*np.log10(x)

# Sampling information:
# Sampling frequency
f_s = 44.1e3
# Sampling interval
dt_s = 1/f_s

## Create time series
# Length of time series
len_t = 2
# Number of points
#N_t = 4e3
# Interval between points
dt_t = 1e-3
# Create time vector
t = np.arange(0.0, len_t, dt_t)
# Create data
y = function_generator(t)

# Plot data
fig_y = plt.figure()
plt.plot(t, y)

## Fourier Transform
# Take FFT of data
sp = fft.fft(y)
# Find associated frequencies
f = fft.fftfreq(y.size, dt_s) * dt_s * 1e3
# Take only positive part of FFT (?)
sp = sp[:sp.size/2]
f = f[:f.size/2]

# Plot FFT

fig_sp = plt.figure()
plt.plot(f, np.abs(sp))
plt.xlabel('Freq (Hz)')
plt.ylabel('Amplitude')

##################
# SONOGRAM       #
##################
# Set up variables for window of fft
width_W = 256
step_W = 32
N_W = int(y.size/step_W)

# Create data structure to store spectrogram in
sp = np.zeros((t.size, width_W))
f = fft.fftfreq(width_W, dt_s) * dt_s * 1e3

i = 0
lower_W = 0
upper_W = lower_W + width_W

# Take FFT of sliding window
# Repeat until number of spectra reached
while i < N_W:
    print("Taking fft of window {}".format(i))
    # Take FFT of window
    sp[lower_W] = fft.fft(y[lower_W:upper_W], n=sp[lower_W].size)
    # Next spectrum
    i += 1
    # Increment lower bound of window
    lower_W += step_W
    # Increment uppper bound of window
    upper_W += step_W

# Take only positive part of FFT (?)
sp = sp[:, :sp.shape[1]/2]
f = f[:len(f)/2]

# Create grid of data points
F, T = np.meshgrid(f, t)

sp = np.abs(sp)

# Plot sonogram
fig_sonogram_cont = plt.figure()
contours = plt.contour(F, T, sp)

fig_sonogram_surf = plt.figure()
ax = fig_sonogram_surf.gca(projection='3d')
surf = ax.plot_surface(F, T, sp, linewidth=0, cmap=cm.coolwarm)
"""
fig_sonogram_color = plt.figure()
plt.pcolormesh(t, f, sp)
"""

# Built in functions:
fig_spectrogram = plt.figure()
f, t, Sxx = signal.spectrogram(y, f_s)
plt.pcolormesh(t, f, Sxx)
plt.ylabel('Frequency [Hz]')
plt.xlabel('Time [sec]')

# Display graphs
plt.show()
<commit_msg>Added comments, labels to graphs, general tidy-up<commit_after>"""
Sonogram
Author: Theo Brown (tab53)
Date: July 2017
"""

from numpy import fft
import numpy as np
import matplotlib.pyplot as plt
from mpl_toolkits.mplot3d import Axes3D
from matplotlib import cm
from scipy import signal

# Function source
def func_1(t, w, x):
    """
    A simple function (decaying exponential)
    """
    return np.exp((1j*w - x) * t)

def function_generator(t):
    """
    Creates a test function with echoes
    """
    f1 = func_1(t, 50*2*np.pi, 5)
    f2 = func_1(t, 243*2*np.pi, 3)
    result = f1 + f2
    echo1 = 0.5*(f1[:f1.size/2] + f2[:f2.size/2])
    result[result.size/2:] += echo1
    echo2 = 0.3*(f1[:f1.size/4] + f2[:f2.size/4])
    result[result.size*3/4:] += echo2
    return result

# Sampling information:
# Sampling frequency
f_s = 4096
# Sampling interval
dt_s = 1/f_s

## Create time series
# Length of time series
len_t = 2
# Number of points
# N_t = 4e3
# Interval between points
dt_t = 1e-4
# Create time vector
t = np.arange(0.0, len_t, dt_t)
# Create data
y = function_generator(t)

# Plot data
fig_y = plt.figure()
plt.plot(t, y)

"""
## Fourier Transform
# Take FFT of data
sp = fft.fft(y)
# Find associated frequencies
f = fft.fftfreq(y.size, dt_s) * dt_s * 1e3
# Take only positive part of FFT (?)
sp = sp[:sp.size/2]
f = f[:f.size/2]

# Plot FFT
fig_sp = plt.figure()
plt.plot(f, np.abs(sp))
plt.xlabel('Freq (Hz)')
plt.ylabel('Amplitude')
"""

##################
# SONOGRAM       #
##################
# Set up variables for window of fft
width_W = 256
step_W = 4
N_W = int(y.size/step_W)

# Create data structure to store spectrogram in
t = np.linspace(0, len_t, N_W)
sp = np.zeros((t.size, width_W))
f = fft.fftfreq(width_W, dt_s) * dt_s / dt_t

# Set window properties
lower_W = 0
upper_W = lower_W + width_W

i = 0

# Take FFT of sliding window
# Repeat until number of spectra reached
while i < N_W:
    print("Taking fft of window {}".format(i))
    # Take FFT of window
    sp[i] = fft.fft(y[lower_W:upper_W], n=sp[i].size)
    # Next spectrum
    i += 1
    # Increment lower bound of window
    lower_W += step_W
    # Increment uppper bound of window
    upper_W += step_W

# Take only positive part of FFT (?)
sp = sp[:, :sp.shape[1]/2]
f = f[:len(f)/2]

# Create grid of data points
F, T = np.meshgrid(f, t)

# Find magnitude of freqency components
sp = np.abs(sp)

# Plot sonogram
# Contours:
fig_sonogram_cont = plt.figure()
contours = plt.contour(F, T, sp)
plt.xlabel('Freq (Hz)')
plt.xlim(0, 1000)
plt.ylabel('Time (s)')

# Surface:
fig_sonogram_surf = plt.figure()
ax = fig_sonogram_surf.gca(projection='3d')
surf = ax.plot_surface(F, T, sp, linewidth=0, cmap=cm.coolwarm)
ax.set_xlabel('Freq (Hz)')
ax.set_xlim(0, 1000)
ax.set_ylabel('Time (s)')
ax.set_zlabel('Amplitude')

# Colourmap:
fig_sonogram_color = plt.figure()
plt.pcolormesh(F, T, sp)
plt.xlabel('Freq (Hz)')
plt.xlim(0, 1000)
plt.ylabel('Time (s)')

# Display graphs
plt.show()
<|endoftext|>


<commit_before># coding:utf-8
import time
from logger.log import crawler
from tasks.workers import app
from page_parse.user import public
from page_get.basic import get_page
from db.seed_ids import get_home_ids, get_ids_by_home_flag_random
from config.conf import get_max_home_page
from page_parse.weibo import get_weibo_list
# new import for wb_pic
from db.wb_pic import insert_weibo_pics
from db.seed_ids import set_seed_home_crawled
# from db.db_proxy import set_proxy_score
from page_parse.proxy import get_a_random_proxy
from page_parse.proxy import proxy_handler, get_proxy_to_db, proxy_init

# import for exception test
# from page_parse.weibo import check_no_bottom, check_no_bottom, parse_json_to_dict
# end
ori_wb_temp_url = 'http://m.weibo.cn/api/container/getIndex?containerid={}_-_WEIBO_SECOND_PROFILE_WEIBO_ORI&luicode={}&lfid={}&featurecode={}&type=uid&value={}&page_type={}&page={}'

@app.task(ignore_result=True)
def crawl_weibo(uid):

    limit = get_max_home_page()
    cur_page = 1

    pic_count = 0
    max_pic_count = 150

    max_retry_cnt = 2
    cur_retry_cnt = 0

    containerid = '230413' + uid
    luicode = '10000011'
    lfid = '230283' + uid
    featurecode = '20000180'
    value = uid
    page_type = '03'
    page = cur_page

    # print for proxy
    proxy = get_a_random_proxy()
    print(proxy)
    # end

    url = ori_wb_temp_url.format(containerid, luicode, lfid, featurecode, value, page_type, page)
    html = get_page(url, user_verify=False, need_login=False, proxys=proxy)

    # proxy_test
    # proxy = {}
    # url = 'https://www.icanhazip.com'
    # html = get_page(url, user_verify=False, need_login=False, proxys=proxy)
    # print(html)
    # end

    # html为空也有可能是其他原因，但是代理问题应该是大概率，因此对代理进行扣分。
    # 如果重试还是返回空html，那么两个proxy均不扣分，记录uid异常后直接return，如果返回非空但无效的html，则在后面流程进行扣分
    if html == '':
        if cur_retry_cnt < max_retry_cnt:
            cur_retry_cnt = cur_retry_cnt + 1
            proxy_handler(proxy, -1)
            proxy = get_a_random_proxy()
            html = get_page(url, user_verify=False, need_login=False, proxys=proxy)
            if html == '':
                exception_uid_handler(uid, 5, proxy)
                return
        else:
            exception_uid_handler(uid, 1, proxy)
            return
    # end

    weibo_pics = get_weibo_list(html)

    if weibo_pics == '':
        crawler.warning('请求过于频繁')
        proxy_handler(proxy, -1)
        return

    if weibo_pics == None:
        exception_uid_handler(uid, 2, proxy, html)
        return
    elif weibo_pics == False:
        finish_uid_handler(uid, proxy)
        return
    elif weibo_pics:
        insert_weibo_pics(weibo_pics)

    pic_count = pic_count + len(weibo_pics)

    cur_page += 1

    while cur_page <= limit and pic_count < max_pic_count:

        page = cur_page
        url = ori_wb_temp_url.format(containerid, luicode, lfid, featurecode, value, page_type, page)
        html = get_page(url, user_verify=False, need_login=False, proxys=proxy)

        # html为空也有可能是其他原因，但是代理问题应该是大概率，因此对代理进行扣分。
        if html == '':
            if cur_retry_cnt < max_retry_cnt:
                cur_retry_cnt = cur_retry_cnt + 1
                proxy_handler(proxy, -1)
                proxy = get_a_random_proxy()
                html = get_page(url, user_verify=False, need_login=False, proxys=proxy)
                if html == '':
                    exception_uid_handler(uid, 6, proxy)
                    return
            else:
                exception_uid_handler(uid, 3, proxy)
                return
        # end

        weibo_pics = get_weibo_list(html)

        # 如果通过当前代理所获取到的页面是被封锁页面，则将当前代理降分并直接return
        if weibo_pics == '':
            crawler.warning('请求过于频繁')
            proxy_handler(proxy, -1)
            return

        if weibo_pics == None:
            exception_uid_handler(uid, 4, proxy, html)
            return
        elif weibo_pics == False:
            finish_uid_handler(uid, proxy)
            return
        elif weibo_pics:
            insert_weibo_pics(weibo_pics)

        pic_count = pic_count + len(weibo_pics)

        cur_page += 1

    finish_uid_handler(uid, proxy)
    return

@app.task
def excute_weibo_task():
    id_objs = get_ids_by_home_flag_random(0, 100)
    proxy_init()
    for id_obj in id_objs:
        app.send_task('tasks.weibo.crawl_weibo', args=(id_obj.uid,), queue='weibo_crawler',
                      routing_key='weibo_info')
        # crawl_weibo(id_obj.uid)

def finish_uid_handler(uid, proxy):
    crawler.warning('用户id为{}的相册采集完成'.format(uid))
    set_seed_home_crawled(uid, 4)
    proxy_handler(proxy, 1)

def exception_uid_handler(uid, err_code, proxy ={}, html=''):
    crawler.warning('用户id为{}的相册采集出错，这一请求接收到的内容为{}，状态码{}'.format(uid, html, err_code))
    set_seed_home_crawled(uid, 3)
    if proxy:
        proxy_handler(proxy, -1)

<commit_msg>no celery mode<commit_after># coding:utf-8
import time
from logger.log import crawler
from tasks.workers import app
from page_parse.user import public
from page_get.basic import get_page
from db.seed_ids import get_home_ids, get_ids_by_home_flag_random
from config.conf import get_max_home_page
from page_parse.weibo import get_weibo_list
# new import for wb_pic
from db.wb_pic import insert_weibo_pics
from db.seed_ids import set_seed_home_crawled
# from db.db_proxy import set_proxy_score
from page_parse.proxy import get_a_random_proxy
from page_parse.proxy import proxy_handler, get_proxy_to_db, proxy_init

# import for exception test
# from page_parse.weibo import check_no_bottom, check_no_bottom, parse_json_to_dict
# end
ori_wb_temp_url = 'http://m.weibo.cn/api/container/getIndex?containerid={}_-_WEIBO_SECOND_PROFILE_WEIBO_ORI&luicode={}&lfid={}&featurecode={}&type=uid&value={}&page_type={}&page={}'

# @app.task(ignore_result=True)
def crawl_weibo(uid):

    limit = get_max_home_page()
    cur_page = 1

    pic_count = 0
    max_pic_count = 150

    max_retry_cnt = 2
    cur_retry_cnt = 0

    containerid = '230413' + uid
    luicode = '10000011'
    lfid = '230283' + uid
    featurecode = '20000180'
    value = uid
    page_type = '03'
    page = cur_page

    # print for proxy
    proxy = get_a_random_proxy()
    print(proxy)
    # end

    url = ori_wb_temp_url.format(containerid, luicode, lfid, featurecode, value, page_type, page)
    html = get_page(url, user_verify=False, need_login=False, proxys=proxy)

    # proxy_test
    # proxy = {}
    # url = 'https://www.icanhazip.com'
    # html = get_page(url, user_verify=False, need_login=False, proxys=proxy)
    # print(html)
    # end

    # html为空也有可能是其他原因，但是代理问题应该是大概率，因此对代理进行扣分。
    # 如果重试还是返回空html，那么两个proxy均不扣分，记录uid异常后直接return，如果返回非空但无效的html，则在后面流程进行扣分
    if html == '':
        if cur_retry_cnt < max_retry_cnt:
            cur_retry_cnt = cur_retry_cnt + 1
            proxy_handler(proxy, -1)
            proxy = get_a_random_proxy()
            html = get_page(url, user_verify=False, need_login=False, proxys=proxy)
            if html == '':
                exception_uid_handler(uid, 5, proxy)
                return
        else:
            exception_uid_handler(uid, 1, proxy)
            return
    # end

    weibo_pics = get_weibo_list(html)

    if weibo_pics == '':
        crawler.warning('请求过于频繁')
        proxy_handler(proxy, -1)
        return

    if weibo_pics == None:
        exception_uid_handler(uid, 2, proxy, html)
        return
    elif weibo_pics == False:
        finish_uid_handler(uid, proxy)
        return
    elif weibo_pics:
        insert_weibo_pics(weibo_pics)

    pic_count = pic_count + len(weibo_pics)

    cur_page += 1

    while cur_page <= limit and pic_count < max_pic_count:

        page = cur_page
        url = ori_wb_temp_url.format(containerid, luicode, lfid, featurecode, value, page_type, page)
        html = get_page(url, user_verify=False, need_login=False, proxys=proxy)

        # html为空也有可能是其他原因，但是代理问题应该是大概率，因此对代理进行扣分。
        if html == '':
            if cur_retry_cnt < max_retry_cnt:
                cur_retry_cnt = cur_retry_cnt + 1
                proxy_handler(proxy, -1)
                proxy = get_a_random_proxy()
                html = get_page(url, user_verify=False, need_login=False, proxys=proxy)
                if html == '':
                    exception_uid_handler(uid, 6, proxy)
                    return
            else:
                exception_uid_handler(uid, 3, proxy)
                return
        # end

        weibo_pics = get_weibo_list(html)

        # 如果通过当前代理所获取到的页面是被封锁页面，则将当前代理降分并直接return
        if weibo_pics == '':
            crawler.warning('请求过于频繁')
            proxy_handler(proxy, -1)
            return

        if weibo_pics == None:
            exception_uid_handler(uid, 4, proxy, html)
            return
        elif weibo_pics == False:
            finish_uid_handler(uid, proxy)
            return
        elif weibo_pics:
            insert_weibo_pics(weibo_pics)

        pic_count = pic_count + len(weibo_pics)

        cur_page += 1

    finish_uid_handler(uid, proxy)
    return

# @app.task
def excute_weibo_task():
    id_objs = get_ids_by_home_flag_random(0, 200)
    proxy_init()
    for id_obj in id_objs:
        # app.send_task('tasks.weibo.crawl_weibo', args=(id_obj.uid,), queue='weibo_crawler',
        #               routing_key='weibo_info')
        crawl_weibo(id_obj.uid)

def finish_uid_handler(uid, proxy):
    crawler.warning('用户id为{}的相册采集完成'.format(uid))
    set_seed_home_crawled(uid, 4)
    proxy_handler(proxy, 1)

def exception_uid_handler(uid, err_code, proxy ={}, html=''):
    crawler.warning('用户id为{}的相册采集出错，这一请求接收到的内容为{}，状态码{}'.format(uid, html, err_code))
    set_seed_home_crawled(uid, 3)
    if proxy:
        proxy_handler(proxy, -1)

<|endoftext|>


<commit_before>import json
import xml.etree.cElementTree as et
import urllib.request
import re
import os

reRule = re.compile(r'[http://]+')
def openxml(site):
	response = urllib.request.urlopen('http://api.t.sina.com.cn/short_url/shorten.xml?source=3271760578&url_long=%s' % (site))
	html = response.read().decode('utf8')
	sitesplit = reRule.split(site)
	write = open('%s.xml' % sitesplit[1], 'w+').write(html)
	return sitesplit[1]+'.xml'

def openjson(site):
	response = urllib.request.urlopen('http://api.t.sina.com.cn/short_url/shorten.json?source=3271760578&url_long=%s' % (site))
	html = response.read().decode('utf8')
	sitesplit = reRule.split(site)
	write = open('%s.json' % sitesplit[1], 'w+').write(html)
	return sitesplit[1]+'.json'

def readjson(jsonfile):
	openJson = open(jsonfile, 'r').read()
	Json2dict = json.loads(openJson)
	return (Json2dict[0]['url_short'], Json2dict[0]['url_long'])

def readxml(xmlfile):
	openxml = et.parse(xmlfile).getroot()
	return (openxml[0][0].text, openxml[0][1].text)

def urlReturn(xorjtotuple):
	deftuple = ('short_url', 'long_url')
	dictsu = dict(zip(deftuple, xorjtotuple))
	dicttojson = json.dumps(dictsu)
	save = open('tcn.json', 'w+')
	save.write(dicttojson)
	save.close()

print('tcn-analysis by iPixelOldC (Blog:http://hoc117.top )')
inputurl = input('Please enter url: ')
if 'http://' in inputurl:
	pass
else:
	inputurl = 'http://'+inputurl
inputJorX = input('xml(x) or json(j): ')
if 'x' == inputJorX:
	demo = readxml(openxml(inputurl))
	print('[!]OK!')
	print('longurl: {0}\nshorturl: {1}'.format(demo[1], demo[0]))
	os.remove(openxml(inputurl))
	needsave = input('Are you need save to json?(tcn.json)[Y]: ').lower()
	if needsave == 'y':
		urlReturn(demo)
	else:
		pass

if 'j' == inputJorX:
	demo = readjson(openjson(inputurl))
	print('[!]OK!')
	print('longurl: {0}\nshorturl: {1}'.format(demo[1], demo[0]))
	os.remove(openjson(inputurl))
	needsave = input('Are you need save to json?(tcn.json)[Y]: ').lower()
	if needsave == 'y':
		urlReturn(demo)
	else:
		pass
print('Exit!')
<commit_msg>rewrite<commit_after><|endoftext|>


<commit_before>#!/usr/bin/python
# -*- coding: utf-8 -*-
#--------------------------------------
#
#              ds18b20.py
#  Read DS18B20 1-wire temperature sensor
#
# in /boot/config.txt:
#dtoverlay=w1-gpio,gpiopin=4
#
# Author : Matt Hawkins
# Date   : 10/02/2015
#
# http://www.raspberrypi-spy.co.uk/
# http://www.raspberrypi-spy.co.uk/2013/03/raspberry-pi-1-wire-digital-thermometer-sensor/
#
#--------------------------------------

from __future__ import print_function
import os

def gettemp(id):
  try:
    mytemp = ''
    filename = 'w1_slave'
    f = open('/sys/bus/w1/devices/' + id + '/' + filename, 'r')
    line = f.readline() # read 1st line
    crc = line.rsplit(' ',1)
    crc = crc[1].replace('\n', '')
    if crc=='YES':
      line = f.readline() # read 2nd line
      mytemp = line.rsplit('t=',1)
    else:
      mytemp = 99999
    f.close()
    return int(mytemp[1])

  except:
    return 99999

def get_ids():
    p = '/sys/bus/w1/devices/'
    ids = list()
    d = os.listdir(p)
    for i in d:
        if "_bus_master" not in i:
            ids.append(os.path.basename(i))
    return ids

def write_entry(time, temp):
    line = "%s,%s\n" %(time, temp)
    f = open('/var/tmp/temp', 'a')
    f.write(line)
    f.close()

if __name__ == '__main__':
  import time
  # Script has been called directly
  ids = get_ids()
  while True:
      for i in ids:
        temp = gettemp(i)/float(1000)
        now = time.time()
        print("Temp of sensor %s : %.2f C" %(i, temp))
        write_entry(now, temp)
        time.sleep(30)
<commit_msg>add openweathermap.org outside temperature logging<commit_after>#!/usr/bin/python
# -*- coding: utf-8 -*-
#--------------------------------------
#
#              ds18b20.py
#  Read DS18B20 1-wire temperature sensor
#
# in /boot/config.txt:
#dtoverlay=w1-gpio,gpiopin=4
#
# Author : Matt Hawkins
# Date   : 10/02/2015
#
# http://www.raspberrypi-spy.co.uk/
# http://www.raspberrypi-spy.co.uk/2013/03/raspberry-pi-1-wire-digital-thermometer-sensor/
#
#--------------------------------------

from __future__ import print_function
import requests, json
import os

csv_path = '/var/tmp/temp.csv'
interval_s = 60
NOTEMP = 99999

def gettemp_outside():
    try:
        p = {
            'lat': '48.819598',
            'lon': '2.374334699999963',
            'units': 'metric',
        }
        url = "http://api.openweathermap.org/data/2.5/weather"
        r = requests.get(url, params=p).json()['main']['temp']
        return r
    except Exception as e:
        print('Failed to fetch current outside temperature:\n%s' %e)
        return NOTEMP

def gettemp(id):
  try:
    mytemp = ''
    filename = 'w1_slave'
    f = open('/sys/bus/w1/devices/' + id + '/' + filename, 'r')
    line = f.readline() # read 1st line
    crc = line.rsplit(' ',1)
    crc = crc[1].replace('\n', '')
    if crc=='YES':
      line = f.readline() # read 2nd line
      mytemp = line.rsplit('t=',1)
    else:
      mytemp = NOTEMP
    f.close()
    return int(mytemp[1])

  except:
    return NOTEMP

def get_ids():
    p = '/sys/bus/w1/devices/'
    if os.path.isdir(p):
        ids = list()
        d = os.listdir(p)
        for i in d:
            if "_bus_master" not in i:
                ids.append(os.path.basename(i))
        return ids
    return list()

def write_entry(time, temp, temp_yahoo):
    line = "%s,%s,%s\n" %(time, temp, temp_yahoo)
    f = open(csv_path, 'a')
    f.write(line)
    f.close()

if __name__ == '__main__':
  import time
  ids = get_ids()
  sensor_id = ids[0]
  while True:
    now = time.time()
    temp = gettemp(sensor_id)/float(1000)
    temp_outside = gettemp_outside()
    print("Temp of sensor %s : %.2f C, Outside: %s C" %(sensor_id, temp, temp_outside))
    write_entry(now, temp, temp_outside)
    time.sleep(interval_s)
<|endoftext|>


<commit_before>from collections import namedtuple

_Term = namedtuple("Term", "tag data args")
class Term(_Term):
    def __new__(cls, tag, data, args):
        #XXX AstroTag tracks (name, tag_code) and source span
        if data and not isinstance(data, (str, unicode, int, long, float)):
            raise ValueError("Term data can't be of type %r" % (type(data),))
        if data and args:
            raise ValueError("Term %s can't have both data and children" % (tag,))
        if args is None:
            args = ()
        return _Term.__new__(cls, tag, data, tuple(args))

    def __iter__(self):
        #and now I feel a bit silly subclassing namedtuple
        raise NotImplementedError()

    def __eq__(self, other):
        return (     self.tag, self.data, self.args
               ) == (other.tag, other.data, other.args)

    def __repr__(self):
        return "term('%s')" % (self._unparse(4).replace("'", "\\'"))

    def _unparse(self, indentLevel=0):
        newlineAndIndent = '\n' + (' ' * indentLevel)
        if self.data is not None:
            if self.tag.name == '.String.':
                return '"%s"' % repr(self.data)[1:-1].replace("\\'", "'").replace('"', '\\\\"')
            elif self.tag.name == '.char.':
                return "'%s'" % repr(self.data)[1:-1].replace("'", "\\'").replace('\\"', '"')
            else:
                return str(self.data)
        args = ', '.join([a._unparse() for a in self.args])
        if self.tag.name == '.tuple.':
            return "[%s]" % (args,)
        elif self.tag.name == '.attr.':
            return "%s: %s" % (self.args[0]._unparse(indentLevel),
                               self.args[1]._unparse(indentLevel))
        elif self.tag.name == '.bag.':
            return "{%s}" % (args,)
        elif len(self.args) == 1 and self.args[0].tag.name == '.bag.':
            return "%s%s" % (self.tag._unparse(indentLevel), args)
        else:
            if len(self.args) == 0:
                return self.tag._unparse(indentLevel)
            return "%s(%s)" % (self.tag._unparse(indentLevel), args)

    def build(self, builder):
        if self.data is None:
            f = builder.leafTag(self.tag)
        else:
            f = builder.leafData(self.data)

        return builder.term(f, [arg.build(builder) for arg in self.args])

    def __cmp__(self, other):
        tagc = cmp(self.tag, other.tag)
        if tagc:
            return tagc
        datac = cmp(self.data, other.data)
        if datac:
            return datac
        return cmp(self.args, other.args)

    def __int__(self):
        return int(self.data)

    def __float__(self):
        return float(self.data)

    def withoutArgs(self):
        return Term(self.tag, self.data, ())

    def asFunctor(self):
        if self.args:
            raise ValueError("Terms with args can't be used as functors")
        else:
            return self.tag

class Tag(object):
    def __init__(self, name):
        if name[0] == '':
            raise ValueError("Tags must have names")
        self.name = name

    def __eq__(self, other):
        return other.__class__ == self.__class__ and self.name == other.name

    def __ne__(self, other):
        return not self == other

    def __repr__(self):
        return "Tag(%r)" % (self.name,)

    def _unparse(self, indentLevel=0):
        return self.name

def coerceToTerm(val):
    from ometa.runtime import character, unicodeCharacter
    if isinstance(val, Term):
        return val
    if val is None:
        return Term(Tag("null"), None, None)
    if val is True:
        return Term(Tag("true"), None, None)
    if val is False:
        return Term(Tag("false"), None, None)
    if isinstance(val, (int, long)):
        return Term(Tag(".int."), val, None)
    if isinstance(val, float):
        return Term(Tag(".float64."), val, None)
    if isinstance(val, (character, unicodeCharacter)):
        return Term(Tag(".char."), val, None)
    if isinstance(val, basestring):
        return Term(Tag(".String."), val, None)
    if isinstance(val, (list, tuple)):
        return Term(Tag(".tuple."), None, tuple(coerceToTerm(item) for item in val))
    if isinstance(val, set):
        return Term(Tag('.bag.'), None, tuple(coerceToTerm(item) for item in val))
    if isinstance(val, dict):
        return Term(Tag('.bag.'), None, tuple(Term(Tag('.attr.'), None,
                                                   (coerceToTerm(k), coerceToTerm(v)))
                                              for (k, v) in val.iteritems()))
    raise ValueError("Could not coerce %r to Term" % (val,))

class TermMaker(object):
    def __getattr__(self, name):
        def mkterm(*args, **kwargs):
            return Term(Tag(name), None,
                        tuple([coerceToTerm(a) for a in args]))
        return mkterm

termMaker = TermMaker()

<commit_msg>terml/nodes: Fix __eq__().<commit_after>from collections import namedtuple

_Term = namedtuple("Term", "tag data args")
class Term(_Term):
    def __new__(cls, tag, data, args):
        #XXX AstroTag tracks (name, tag_code) and source span
        if data and not isinstance(data, (str, unicode, int, long, float)):
            raise ValueError("Term data can't be of type %r" % (type(data),))
        if data and args:
            raise ValueError("Term %s can't have both data and children" % (tag,))
        if args is None:
            args = ()
        return _Term.__new__(cls, tag, data, tuple(args))

    def __iter__(self):
        #and now I feel a bit silly subclassing namedtuple
        raise NotImplementedError()

    def __eq__(self, other):
        try:
            return (     self.tag, self.data, self.args
                   ) == (other.tag, other.data, other.args)
        except AttributeError:
            return False

    def __repr__(self):
        return "term('%s')" % (self._unparse(4).replace("'", "\\'"))

    def _unparse(self, indentLevel=0):
        newlineAndIndent = '\n' + (' ' * indentLevel)
        if self.data is not None:
            if self.tag.name == '.String.':
                return '"%s"' % repr(self.data)[1:-1].replace("\\'", "'").replace('"', '\\\\"')
            elif self.tag.name == '.char.':
                return "'%s'" % repr(self.data)[1:-1].replace("'", "\\'").replace('\\"', '"')
            else:
                return str(self.data)
        args = ', '.join([a._unparse() for a in self.args])
        if self.tag.name == '.tuple.':
            return "[%s]" % (args,)
        elif self.tag.name == '.attr.':
            return "%s: %s" % (self.args[0]._unparse(indentLevel),
                               self.args[1]._unparse(indentLevel))
        elif self.tag.name == '.bag.':
            return "{%s}" % (args,)
        elif len(self.args) == 1 and self.args[0].tag.name == '.bag.':
            return "%s%s" % (self.tag._unparse(indentLevel), args)
        else:
            if len(self.args) == 0:
                return self.tag._unparse(indentLevel)
            return "%s(%s)" % (self.tag._unparse(indentLevel), args)

    def build(self, builder):
        if self.data is None:
            f = builder.leafTag(self.tag)
        else:
            f = builder.leafData(self.data)

        return builder.term(f, [arg.build(builder) for arg in self.args])

    def __cmp__(self, other):
        tagc = cmp(self.tag, other.tag)
        if tagc:
            return tagc
        datac = cmp(self.data, other.data)
        if datac:
            return datac
        return cmp(self.args, other.args)

    def __int__(self):
        return int(self.data)

    def __float__(self):
        return float(self.data)

    def withoutArgs(self):
        return Term(self.tag, self.data, ())

    def asFunctor(self):
        if self.args:
            raise ValueError("Terms with args can't be used as functors")
        else:
            return self.tag

class Tag(object):
    def __init__(self, name):
        if name[0] == '':
            raise ValueError("Tags must have names")
        self.name = name

    def __eq__(self, other):
        return other.__class__ == self.__class__ and self.name == other.name

    def __ne__(self, other):
        return not self == other

    def __repr__(self):
        return "Tag(%r)" % (self.name,)

    def _unparse(self, indentLevel=0):
        return self.name

def coerceToTerm(val):
    from ometa.runtime import character, unicodeCharacter
    if isinstance(val, Term):
        return val
    if val is None:
        return Term(Tag("null"), None, None)
    if val is True:
        return Term(Tag("true"), None, None)
    if val is False:
        return Term(Tag("false"), None, None)
    if isinstance(val, (int, long)):
        return Term(Tag(".int."), val, None)
    if isinstance(val, float):
        return Term(Tag(".float64."), val, None)
    if isinstance(val, (character, unicodeCharacter)):
        return Term(Tag(".char."), val, None)
    if isinstance(val, basestring):
        return Term(Tag(".String."), val, None)
    if isinstance(val, (list, tuple)):
        return Term(Tag(".tuple."), None, tuple(coerceToTerm(item) for item in val))
    if isinstance(val, set):
        return Term(Tag('.bag.'), None, tuple(coerceToTerm(item) for item in val))
    if isinstance(val, dict):
        return Term(Tag('.bag.'), None, tuple(Term(Tag('.attr.'), None,
                                                   (coerceToTerm(k), coerceToTerm(v)))
                                              for (k, v) in val.iteritems()))
    raise ValueError("Could not coerce %r to Term" % (val,))

class TermMaker(object):
    def __getattr__(self, name):
        def mkterm(*args, **kwargs):
            return Term(Tag(name), None,
                        tuple([coerceToTerm(a) for a in args]))
        return mkterm

termMaker = TermMaker()

<|endoftext|>