<commit_before>
from __future__ import absolute_import, division

import os
import re

import pandas as pd

import pyproteome as pyp

def _prep_csv(data=None, postfix="table", folder_name=None, csv_name=None):
    if csv_name is None:
        csv_name = "{}.csv".format(
            postfix,
        )

    folder_name = pyp.utils.make_folder(
        data=data,
        folder_name=folder_name,
        sub="Tables",
    )

    return os.path.join(folder_name, csv_name)

def _get_table_title(f=None, running_title=None):
    if running_title is None:
        running_title = []

    if f is not None:
        if 'asym_fold' in f:
            running_title.append(
                "Upregulated" if f["asym_fold"] > 1 else "Downregulated"
            )

        if "p" in f:
            running_title.append(
                "p-{:.3e}".format(f["p"])
            )

        if "group_a" in f or "group_b" in f:
            running_title.append(
                "{}vs{}".format(
                    f.get("group_a", ""),
                    f.get("group_b", ""),
                )
            )

    return "-".join(running_title)

def motif_table(
    data, f,
    p=0.05,
    sort="p-value",
    folder_name=None,
    csv_name=None,
    **kwargs
):
    csv_name = _prep_csv(
        data=data,
        folder_name=folder_name,
        csv_name=csv_name,
        postfix=_get_table_title(f=f, running_title=["motifs"]),
    )

    hits = pyp.motifs.motif.run_motif_enrichment(
        data, f,
        **kwargs
    )[0]

    hits = hits.sort_values(
        sort,
        ascending=True if sort == "p-value" else False,
    )

    if csv_name:
        hits.to_csv(csv_name)

    hits = hits[
        hits[
            "pp-value" if kwargs.get("pp_value", False) else "p-value"
        ] < p
    ]

    return hits.style.set_table_styles([
        {"selector": "*", "props": [("font-family", "monospace")]},
        {"selector": "th:first-child", "props": [("display", "none")]},
    ])

def changes_table(
    data,
    sort="p-value",
    folder_name=None,
    csv_name=None,
):
    """
    Show a table of fold changes and p-values.

    Parameters
    ----------
    data : :class:`DataSet<pyproteome.data_sets.DataSet>`
    sort : str, optional
    folder_name : str, optional
    csv_name : str, optional
    """
    csv_name = _prep_csv(
        data=data,
        folder_name=folder_name,
        csv_name=csv_name,
        postfix=_get_table_title(running_title=["changes"]),
    )
    psms = getattr(data, "psms", data)

    psms = psms[
        [
            "Proteins", "Sequence", "Modifications",
            "Fold Change", "p-value", "Validated",
        ]
    ].copy()
    psms["Sequence"] = psms["Sequence"].apply(
        lambda x: "{} ({})".format(x, x.modifications)
    )
    psms["Uniprot Accessions"] = psms["Proteins"].apply(
        lambda x: "; ".join(x.accessions)
    )

    if sort == "Fold Change":
        psms["Fold Change-Sort"] = psms["Fold Change"].apply(
            lambda x: max([x, 1 / x])
        )
        psms.sort_values("Fold Change-Sort", inplace=True, ascending=False)
        psms.drop("Fold Change-Sort", axis=1, inplace=True)
    else:
        psms.sort_values(sort, inplace=True, ascending=True)

    psms.drop("Modifications", axis=1, inplace=True)

    if csv_name:
        psms.to_csv(csv_name)

    # back_colors = {
    #     True: "#BBFFBB",  # light green
    #     False: "#FFBBBB",  # light red
    # }

    if psms.empty:
        return psms

    # return psms.style.apply(  # Color validated rows
    #     lambda row: [
    #         "background-color: " + back_colors[row["Validated"]]
    #         for _ in row
    #     ],
    #     axis=1,
    # )
    return psms.style.set_table_styles(  # Hide index and "Validated" columns
        [
            {"selector": "th:first-child", "props": [("display", "none")]},
            {"selector": "td:last-child", "props": [("display", "none")]},
            {"selector": "th:last-child", "props": [("display", "none")]},
            {"selector": "*", "props": [("text-align", "left")]},
        ]
    )

def ptmsigdb_changes_table(
    data,
    sort="p-value",
    folder_name=None,
    csv_name=None,
):
    """
    Show a table of fold changes and p-values for PTMSigDB.

    Parameters
    ----------
    data : :class:`DataSet<pyproteome.data_sets.DataSet>`
    sort : str, optional
    folder_name : str, optional
    csv_name : str, optional
    """
    csv_name = _prep_csv(
        data=data,
        folder_name=folder_name,
        csv_name=csv_name,
        postfix=_get_table_title(running_title=["ptmsigdb"]),
    )

    psms = getattr(data, "psms", data).copy()
    psms = psms.dropna(subset=("Fold Change",))

    psms["Protein Description"] = psms["Proteins"].apply(
        lambda x: x.proteins[0].description
    )
    psms["Gene"] = psms["Proteins"].apply(
        lambda x: x.genes[0]
    )
    psms["Uniprot Accession"] = psms["Proteins"].apply(
        lambda x: x.accessions[0]
    )
    psms["All Modifications"] = psms["Modifications"]
    psms["Phospho Modifications"] = psms["Modifications"].apply(
        lambda x: x.get_mods([(None, "Phospho")]).__str__(prot_index=0)
    )

    psms = psms[
        [
            "Protein Description",
            "Gene",
            "Uniprot Accession",
            "Sequence",
            "All Modifications",
            "Phospho Modifications",
            "Fold Change",
        ]
    ].copy()
    psms.sort_values("Fold Change", inplace=True, ascending=False)

    if csv_name:
        psms.to_csv(
            csv_name,
            index=False,
        )

    if psms.empty:
        return psms

    return psms.style.set_table_styles(  # Hide index and "Validated" columns
        [
            {"selector": "th:first-child", "props": [("display", "none")]},
            {"selector": "*", "props": [("text-align", "left")]},
        ]
    )

def write_full_tables(datas, folder_name=None, out_name="Full Data.xlsx"):
    """
    Write a full list of data sets to a single .xlsx file.

    Parameters
    ----------
    datas : list of :class:`DataSet<pyproteome.data_sets.DataSet>`
    folder_name : str, optional
    out_name : str, optional
    """

    out_name = _prep_csv(
        data=None,
        folder_name=folder_name,
        csv_name=out_name,
    )

    writer = pd.ExcelWriter(out_name, engine="xlsxwriter")

    for data in datas:
        channels = [
            (name, data.channels[name])
            for group in data.groups.values()
            for name in group
            if name in data.channels and
            data.channels[name] in data.psms.columns
        ]
        df = data.psms[
            [
                "Proteins", "Sequence",
                "Fold Change", "p-value", "Validated",
            ] + [
                chan
                for _, chan in channels
            ]
        ].copy()

        df.rename(
            columns={
                chan: name
                for name, chan in channels
            },
            inplace=True,
        )
        df.insert(
            2, "Modifications",
            df["Sequence"].apply(
                lambda x: str(x.modifications)
            ),
        )
        df["Sequence"] = df["Sequence"].apply(str)
        df.sort_values("p-value", inplace=True, ascending=True)

        ws_name = re.sub(
            "/",
            "+",
            data.name,
        )
        df.to_excel(
            writer,
            sheet_name=ws_name,
            index=False,
        )

        ws = writer.sheets[ws_name]
        ws.freeze_panes(1, 0)
        ws.set_column(0, 0, 60)
        ws.set_column(1, 1, 30)
        ws.set_column(2, 2, 20)
        ws.set_column(3, 3, 12)
        ws.set_column(4, 4, 12)

    writer.save()
<commit_msg>Added docstring<commit_after>
from __future__ import absolute_import, division

import os
import re

import pandas as pd

import pyproteome as pyp

def _prep_csv(data=None, postfix="table", folder_name=None, csv_name=None):
    if csv_name is None:
        csv_name = "{}.csv".format(
            postfix,
        )

    folder_name = pyp.utils.make_folder(
        data=data,
        folder_name=folder_name,
        sub="Tables",
    )

    return os.path.join(folder_name, csv_name)

def _get_table_title(f=None, running_title=None):
    if running_title is None:
        running_title = []

    if f is not None:
        if 'asym_fold' in f:
            running_title.append(
                "Upregulated" if f["asym_fold"] > 1 else "Downregulated"
            )

        if "p" in f:
            running_title.append(
                "p-{:.3e}".format(f["p"])
            )

        if "group_a" in f or "group_b" in f:
            running_title.append(
                "{}vs{}".format(
                    f.get("group_a", ""),
                    f.get("group_b", ""),
                )
            )

    return "-".join(running_title)

def motif_table(
    data, f,
    p=0.05,
    sort="p-value",
    folder_name=None,
    csv_name=None,
    **kwargs
):
    """
    Run a motif enrichment algorithm on a data set and display the
    significantly enriched motifs.

    Parameters
    ----------
    data : :class:`DataSet<pyproteome.data_sets.DataSet>`
    f : dict or list of dict
    p : float, optional
    sort : str, optional
    folder_name : str, optional
    csv_name : str, optional

    Returns
    -------
    :class:`pandas.DataFrame`

    See Also
    --------
    :func:`run_motif_enrichment<pyproteome.motifs.motif.run_motif_enrichment>`
    """
    csv_name = _prep_csv(
        data=data,
        folder_name=folder_name,
        csv_name=csv_name,
        postfix=_get_table_title(f=f, running_title=["motifs"]),
    )

    hits = pyp.motifs.motif.run_motif_enrichment(
        data, f,
        **kwargs
    )[0]

    hits = hits.sort_values(
        sort,
        ascending=True if sort == "p-value" else False,
    )

    if csv_name:
        hits.to_csv(csv_name)

    hits = hits[
        hits[
            "pp-value" if kwargs.get("pp_value", False) else "p-value"
        ] < p
    ]

    return hits.style.set_table_styles([
        {"selector": "*", "props": [("font-family", "monospace")]},
        {"selector": "th:first-child", "props": [("display", "none")]},
    ])

def changes_table(
    data,
    sort="p-value",
    folder_name=None,
    csv_name=None,
):
    """
    Show a table of fold changes and p-values.

    Parameters
    ----------
    data : :class:`DataSet<pyproteome.data_sets.DataSet>`
    sort : str, optional
    folder_name : str, optional
    csv_name : str, optional
    """
    csv_name = _prep_csv(
        data=data,
        folder_name=folder_name,
        csv_name=csv_name,
        postfix=_get_table_title(running_title=["changes"]),
    )
    psms = getattr(data, "psms", data)

    psms = psms[
        [
            "Proteins", "Sequence", "Modifications",
            "Fold Change", "p-value", "Validated",
        ]
    ].copy()
    psms["Sequence"] = psms["Sequence"].apply(
        lambda x: "{} ({})".format(x, x.modifications)
    )
    psms["Uniprot Accessions"] = psms["Proteins"].apply(
        lambda x: "; ".join(x.accessions)
    )

    if sort == "Fold Change":
        psms["Fold Change-Sort"] = psms["Fold Change"].apply(
            lambda x: max([x, 1 / x])
        )
        psms.sort_values("Fold Change-Sort", inplace=True, ascending=False)
        psms.drop("Fold Change-Sort", axis=1, inplace=True)
    else:
        psms.sort_values(sort, inplace=True, ascending=True)

    psms.drop("Modifications", axis=1, inplace=True)

    if csv_name:
        psms.to_csv(csv_name)

    # back_colors = {
    #     True: "#BBFFBB",  # light green
    #     False: "#FFBBBB",  # light red
    # }

    if psms.empty:
        return psms

    # return psms.style.apply(  # Color validated rows
    #     lambda row: [
    #         "background-color: " + back_colors[row["Validated"]]
    #         for _ in row
    #     ],
    #     axis=1,
    # )
    return psms.style.set_table_styles(  # Hide index and "Validated" columns
        [
            {"selector": "th:first-child", "props": [("display", "none")]},
            {"selector": "td:last-child", "props": [("display", "none")]},
            {"selector": "th:last-child", "props": [("display", "none")]},
            {"selector": "*", "props": [("text-align", "left")]},
        ]
    )

def ptmsigdb_changes_table(
    data,
    sort="p-value",
    folder_name=None,
    csv_name=None,
):
    """
    Show a table of fold changes and p-values for PTMSigDB.

    Parameters
    ----------
    data : :class:`DataSet<pyproteome.data_sets.DataSet>`
    sort : str, optional
    folder_name : str, optional
    csv_name : str, optional
    """
    csv_name = _prep_csv(
        data=data,
        folder_name=folder_name,
        csv_name=csv_name,
        postfix=_get_table_title(running_title=["ptmsigdb"]),
    )

    psms = getattr(data, "psms", data).copy()
    psms = psms.dropna(subset=("Fold Change",))

    psms["Protein Description"] = psms["Proteins"].apply(
        lambda x: x.proteins[0].description
    )
    psms["Gene"] = psms["Proteins"].apply(
        lambda x: x.genes[0]
    )
    psms["Uniprot Accession"] = psms["Proteins"].apply(
        lambda x: x.accessions[0]
    )
    psms["All Modifications"] = psms["Modifications"]
    psms["Phospho Modifications"] = psms["Modifications"].apply(
        lambda x: x.get_mods([(None, "Phospho")]).__str__(prot_index=0)
    )

    psms = psms[
        [
            "Protein Description",
            "Gene",
            "Uniprot Accession",
            "Sequence",
            "All Modifications",
            "Phospho Modifications",
            "Fold Change",
        ]
    ].copy()
    psms.sort_values("Fold Change", inplace=True, ascending=False)

    if csv_name:
        psms.to_csv(
            csv_name,
            index=False,
        )

    if psms.empty:
        return psms

    return psms.style.set_table_styles(  # Hide index and "Validated" columns
        [
            {"selector": "th:first-child", "props": [("display", "none")]},
            {"selector": "*", "props": [("text-align", "left")]},
        ]
    )

def write_full_tables(datas, folder_name=None, out_name="Full Data.xlsx"):
    """
    Write a full list of data sets to a single .xlsx file.

    Parameters
    ----------
    datas : list of :class:`DataSet<pyproteome.data_sets.DataSet>`
    folder_name : str, optional
    out_name : str, optional
    """

    out_name = _prep_csv(
        data=None,
        folder_name=folder_name,
        csv_name=out_name,
    )

    writer = pd.ExcelWriter(out_name, engine="xlsxwriter")

    for data in datas:
        channels = [
            (name, data.channels[name])
            for group in data.groups.values()
            for name in group
            if name in data.channels and
            data.channels[name] in data.psms.columns
        ]
        df = data.psms[
            [
                "Proteins", "Sequence",
                "Fold Change", "p-value", "Validated",
            ] + [
                chan
                for _, chan in channels
            ]
        ].copy()

        df.rename(
            columns={
                chan: name
                for name, chan in channels
            },
            inplace=True,
        )
        df.insert(
            2, "Modifications",
            df["Sequence"].apply(
                lambda x: str(x.modifications)
            ),
        )
        df["Sequence"] = df["Sequence"].apply(str)
        df.sort_values("p-value", inplace=True, ascending=True)

        ws_name = re.sub(
            "/",
            "+",
            data.name,
        )
        df.to_excel(
            writer,
            sheet_name=ws_name,
            index=False,
        )

        ws = writer.sheets[ws_name]
        ws.freeze_panes(1, 0)
        ws.set_column(0, 0, 60)
        ws.set_column(1, 1, 30)
        ws.set_column(2, 2, 20)
        ws.set_column(3, 3, 12)
        ws.set_column(4, 4, 12)

    writer.save()
<|endoftext|>


<commit_before>from app import app, db
import datetime
import re
from flask import Markup

from werkzeug.security import generate_password_hash, \
     check_password_hash

def markdown_filter(data):
    from markdown import markdown
    from smartypants import smartypants
    return smartypants(markdown(data, extensions=['codehilite']))

def plain_text_filter(plain):
    plain = re.sub(r'(?<!href=.)https?://([a-zA-Z0-9/\.\-_:%?@$#&=]+)', r'<a href="\g<0>">\g<1>', plain)
    plain = re.sub(r'@([a-zA-Z0-9_]+)', r'<a href="http://twitter.com/\g<1>">\g<0>', plain)
    return plain.replace('\n', '
')

def repost_preview_filter(url):
    m = re.match(r'https?://(?:www.)?youtube.com/watch\?v=(\w+)', url)
    if m:
        return """"""\
            .format(m.group(1))

    m = re.match(r'https?://instagram.com/p/(\w+)/?#?', url)
    if m:
        return """"""\
            .format(m.group(1)

def get_md5_hash(inp):
    result = get_md5_hash.cache.get(inp)
    if not result:
        import hashlib
        result = hashlib.md5(inp.encode()).hexdigest()
        get_md5_hash.cache[inp] = result
    return result

get_md5_hash.cache = {}

class User(db.Model):
    id = db.Column(db.Integer, primary_key=True)
    login = db.Column(db.String(80), unique=True)
    email = db.Column(db.String(120), unique=True)
    pw_hash = db.Column(db.String(256))
    display_name = db.Column(db.String(80))
    twitter_username = db.Column(db.String(80))

    def set_password(self, plaintext):
        self.pw_hash = generate_password_hash(plaintext)

    def check_password(self, plaintext):
        return check_password_hash(self.pw_hash, plaintext)

    # Flask-Login integration
    def is_authenticated(self):
        return True

    def is_active(self):
        return True

    def is_anonymous(self):
        return False

    def get_id(self):
        return self.login

    @property
    def image_url(self):
        return "http://gravatar.com/avatar/{}?s=64".format(get_md5_hash(self.email))

    def __init__(self, login, email, password):
        self.login = login
        self.email = email
        self.set_password(password)

    def __repr__(self):
        return 'user:{}'.format(self.login)

class Tag(db.Model):
    id = db.Column(db.Integer, primary_key=True)
    name = db.Column(db.String(256))

    def __init__(self, name):
        self.name = name

    def __repr__(self):
        return 'tag:{}'.format(self.name)

tags_to_posts = db.Table('tags_to_posts', db.Model.metadata,
                         db.Column('tag_id', db.Integer, db.ForeignKey('tag.id')),
                         db.Column('post_id', db.Integer, db.ForeignKey('post.id')),
                         db.Column('position', db.Integer))

class Post(db.Model):
    id = db.Column(db.Integer, primary_key=True)
    pub_date = db.Column(db.DateTime)
    author_id = db.Column(db.Integer, db.ForeignKey('user.id'))
    author = db.relationship('User',
                             backref=db.backref('posts', lazy='dynamic'))
    title = db.Column(db.String(256))
    content = db.Column(db.Text)
    post_type = db.Column(db.String(64)) # note/article/etc.
    content_format = db.Column(db.String(64)) # markdown/html/plain
    in_reply_to = db.Column(db.String(256))
    repost_source = db.Column(db.String(256))
    twitter_status_id = db.Column(db.String(64))
    slug = db.Column(db.String(256))
    tags = db.relationship('Tag', secondary=tags_to_posts,
                           order_by=tags_to_posts.columns.position,
                           backref='posts')

    def __init__(self, title, slug, content, post_type, content_format, author, pub_date=None):
        self.title = title
        self.slug = slug
        self.content = content
        self.post_type = post_type
        self.content_format = content_format
        self.author = author
        if pub_date is None:
            self.pub_date = datetime.datetime.utcnow()
        else:
            self.pub_date = pub_date

    def format_text(self, text):
        if self.content_format == 'markdown':
            return markdown_filter(text)
        elif self.content_format == 'plain':
            return plain_text_filter(text)
        else:
            return text

    @property
    def html_content(self):
        text = self.format_text(self.content)
        if self.repost_source:
            preview = repost_preview_filter(self.repost_source)
            if preview: text += preview
        return Markup(text)

    @property
    def html_excerpt(self):
        text = self.html_content
        split = text.split('', 1)
        text = split[0]
        if self.repost_source:
            preview = repost_preview_filter(self.repost_source)
            if preview: text += preview
        if len(split) > 1:
            text += "Keep Reading...".format(self.permalink_url)
        return Markup(text)

    @property
    def permalink_url(self):
        site_url = app.config.get('SITE_URL') or 'http://localhost'
        path_components = [site_url, self.post_type, str(self.pub_date.year), str(self.id)]
        if self.slug:
            path_components.append(self.slug)
        return '/'.join(path_components)

    @property
    def permalink_short_url(self):
        site_url = app.config.get('SITE_URL') or 'http://localhost'
        path_components = [site_url, self.post_type, str(self.pub_date.year), str(self.id)]
        return '/'.join(path_components)

    @property
    def twitter_url(self):
        if self.twitter_status_id:
            return "https://twitter.com/{}/status/{}".format(
                self.author.twitter_username,
                self.twitter_status_id)

    def __repr__(self):
        if self.title:
            return 'post:{}'.format(self.title)
        else:
            return 'post:{}'.format(self.content[:20])
<commit_msg>Fix typo<commit_after>from app import app, db
import datetime
import re
from flask import Markup

from werkzeug.security import generate_password_hash, \
     check_password_hash

def markdown_filter(data):
    from markdown import markdown
    from smartypants import smartypants
    return smartypants(markdown(data, extensions=['codehilite']))

def plain_text_filter(plain):
    plain = re.sub(r'(?<!href=.)https?://([a-zA-Z0-9/\.\-_:%?@$#&=]+)', r'<a href="\g<0>">\g<1>', plain)
    plain = re.sub(r'@([a-zA-Z0-9_]+)', r'<a href="http://twitter\.com/\g<1>">\g<0>', plain)
    return plain.replace('\n', '
')

def repost_preview_filter(url):
    m = re.match(r'https?://(?:www.)?youtube\.com/watch\?v=(\w+)', url)
    if m:
        return """"""\
            .format(m.group(1))

    m = re.match(r'https?://instagram\.com/p/(\w+)/?#?', url)
    if m:
        return """"""\
            .format(m.group(1))

def get_md5_hash(inp):
    result = get_md5_hash.cache.get(inp)
    if not result:
        import hashlib
        result = hashlib.md5(inp.encode()).hexdigest()
        get_md5_hash.cache[inp] = result
    return result

get_md5_hash.cache = {}

class User(db.Model):
    id = db.Column(db.Integer, primary_key=True)
    login = db.Column(db.String(80), unique=True)
    email = db.Column(db.String(120), unique=True)
    pw_hash = db.Column(db.String(256))
    display_name = db.Column(db.String(80))
    twitter_username = db.Column(db.String(80))

    def set_password(self, plaintext):
        self.pw_hash = generate_password_hash(plaintext)

    def check_password(self, plaintext):
        return check_password_hash(self.pw_hash, plaintext)

    # Flask-Login integration
    def is_authenticated(self):
        return True

    def is_active(self):
        return True

    def is_anonymous(self):
        return False

    def get_id(self):
        return self.login

    @property
    def image_url(self):
        return "http://gravatar.com/avatar/{}?s=64".format(get_md5_hash(self.email))

    def __init__(self, login, email, password):
        self.login = login
        self.email = email
        self.set_password(password)

    def __repr__(self):
        return 'user:{}'.format(self.login)

class Tag(db.Model):
    id = db.Column(db.Integer, primary_key=True)
    name = db.Column(db.String(256))

    def __init__(self, name):
        self.name = name

    def __repr__(self):
        return 'tag:{}'.format(self.name)

tags_to_posts = db.Table('tags_to_posts', db.Model.metadata,
                         db.Column('tag_id', db.Integer, db.ForeignKey('tag.id')),
                         db.Column('post_id', db.Integer, db.ForeignKey('post.id')),
                         db.Column('position', db.Integer))

class Post(db.Model):
    id = db.Column(db.Integer, primary_key=True)
    pub_date = db.Column(db.DateTime)
    author_id = db.Column(db.Integer, db.ForeignKey('user.id'))
    author = db.relationship('User',
                             backref=db.backref('posts', lazy='dynamic'))
    title = db.Column(db.String(256))
    content = db.Column(db.Text)
    post_type = db.Column(db.String(64)) # note/article/etc.
    content_format = db.Column(db.String(64)) # markdown/html/plain
    in_reply_to = db.Column(db.String(256))
    repost_source = db.Column(db.String(256))
    twitter_status_id = db.Column(db.String(64))
    slug = db.Column(db.String(256))
    tags = db.relationship('Tag', secondary=tags_to_posts,
                           order_by=tags_to_posts.columns.position,
                           backref='posts')

    def __init__(self, title, slug, content, post_type, content_format, author, pub_date=None):
        self.title = title
        self.slug = slug
        self.content = content
        self.post_type = post_type
        self.content_format = content_format
        self.author = author
        if pub_date is None:
            self.pub_date = datetime.datetime.utcnow()
        else:
            self.pub_date = pub_date

    def format_text(self, text):
        if self.content_format == 'markdown':
            return markdown_filter(text)
        elif self.content_format == 'plain':
            return plain_text_filter(text)
        else:
            return text

    @property
    def html_content(self):
        text = self.format_text(self.content)
        if self.repost_source:
            preview = repost_preview_filter(self.repost_source)
            if preview: text += preview
        return Markup(text)

    @property
    def html_excerpt(self):
        text = self.html_content
        split = text.split('', 1)
        text = split[0]
        if self.repost_source:
            preview = repost_preview_filter(self.repost_source)
            if preview: text += preview
        if len(split) > 1:
            text += "Keep Reading...".format(self.permalink_url)
        return Markup(text)

    @property
    def permalink_url(self):
        site_url = app.config.get('SITE_URL') or 'http://localhost'
        path_components = [site_url, self.post_type, str(self.pub_date.year), str(self.id)]
        if self.slug:
            path_components.append(self.slug)
        return '/'.join(path_components)

    @property
    def permalink_short_url(self):
        site_url = app.config.get('SITE_URL') or 'http://localhost'
        path_components = [site_url, self.post_type, str(self.pub_date.year), str(self.id)]
        return '/'.join(path_components)

    @property
    def twitter_url(self):
        if self.twitter_status_id:
            return "https://twitter.com/{}/status/{}".format(
                self.author.twitter_username,
                self.twitter_status_id)

    def __repr__(self):
        if self.title:
            return 'post:{}'.format(self.title)
        else:
            return 'post:{}'.format(self.content[:20])
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
"""
Contains the default indenter.
"""
import logging
from pyqode.core.api import TextHelper
from pyqode.core.api.mode import Mode
from pyqode.qt import QtGui

def _logger():
    return logging.getLogger(__name__)

def debug(msg, *args):
    return _logger().log(5, msg, *args)

class IndenterMode(Mode):
    """ Implements classic indentation/tabulation (Tab/Shift+Tab)

    It inserts/removes tabulations (a series of spaces defined by the
    tabLength settings) at the cursor position if there is no selection,
    otherwise it fully indents/un-indents selected lines.

    To trigger an indentation/un-indentation programatically, you must emit
    :attr:`pyqode.core.api.CodeEdit.indent_requested` or
    :attr:`pyqode.core.api.CodeEdit.unindent_requested`.
    """
    def __init__(self):
        super(IndenterMode, self).__init__()

    def on_state_changed(self, state):
        if state:
            self.editor.indent_requested.connect(self.indent)
            self.editor.unindent_requested.connect(self.unindent)
        else:
            self.editor.indent_requested.disconnect(self.indent)
            self.editor.unindent_requested.disconnect(self.unindent)

    def indent_selection(self, cursor):
        """
        Indent selected text

        :param cursor: QTextCursor
        """
        doc = self.editor.document()
        tab_len = self.editor.tab_length
        cursor.beginEditBlock()
        nb_lines = len(cursor.selection().toPlainText().splitlines())
        c = self.editor.textCursor()
        if c.atBlockStart() and c.position() == c.selectionEnd():
            nb_lines += 1
        block = doc.findBlock(cursor.selectionStart())
        i = 0
        # indent every lines
        while i < nb_lines:
            nb_space_to_add = tab_len
            cursor = QtGui.QTextCursor(block)
            cursor.movePosition(cursor.StartOfLine, cursor.MoveAnchor)
            if self.editor.use_spaces_instead_of_tabs:
                for _ in range(nb_space_to_add):
                    cursor.insertText(" ")
            else:
                cursor.insertText('\t')
            block = block.next()
            i += 1
        cursor.endEditBlock()

    def unindent_selection(self, cursor):
        """
        Un-indents selected text

        :param cursor: QTextCursor
        """
        doc = self.editor.document()
        tab_len = self.editor.tab_length
        nb_lines = len(cursor.selection().toPlainText().splitlines())
        if nb_lines == 0:
            nb_lines = 1
        block = doc.findBlock(cursor.selectionStart())
        assert isinstance(block, QtGui.QTextBlock)
        i = 0
        debug('unindent selection: %d lines', nb_lines)
        while i < nb_lines:
            txt = block.text()
            debug('line to unindent: %s', txt)
            debug('self.editor.use_spaces_instead_of_tabs: %r',
                  self.editor.use_spaces_instead_of_tabs)
            if self.editor.use_spaces_instead_of_tabs:
                indentation = (len(txt) - len(txt.lstrip()))
            else:
                indentation = len(txt) - len(txt.replace('\t', ''))
            debug('unindent line %d: %d spaces', i, indentation)
            if indentation > 0:
                c = QtGui.QTextCursor(block)
                c.movePosition(c.StartOfLine, cursor.MoveAnchor)
                for _ in range(tab_len):
                    txt = block.text()
                    if len(txt) and txt[0] == ' ':
                        c.deleteChar()
            block = block.next()
            i += 1
        return cursor

    def indent(self):
        """
        Indents text at cursor position.
        """
        cursor = self.editor.textCursor()
        assert isinstance(cursor, QtGui.QTextCursor)
        if cursor.hasSelection():
            self.indent_selection(cursor)
        else:
            # simply insert indentation at the cursor position
            tab_len = self.editor.tab_length
            cursor.beginEditBlock()
            if self.editor.use_spaces_instead_of_tabs:
                nb_space_to_add = tab_len - cursor.positionInBlock() % tab_len
                cursor.insertText(nb_space_to_add * " ")
            else:
                cursor.insertText('\t')
            cursor.endEditBlock()

    def count_deletable_spaces(self, cursor, max_spaces):
        # count the number of spaces deletable, stop at tab len
        max_spaces = abs(max_spaces)
        if max_spaces > self.editor.tab_length:
            max_spaces = self.editor.tab_length
        spaces = 0
        trav_cursor = QtGui.QTextCursor(cursor)
        while spaces < max_spaces or trav_cursor.atBlockStart():
            pos = trav_cursor.position()
            trav_cursor.movePosition(cursor.Left, cursor.KeepAnchor)
            char = trav_cursor.selectedText()
            if char == " ":
                spaces += 1
            else:
                break
            trav_cursor.setPosition(pos - 1)
        return spaces

    def unindent(self):
        """
        Un-indents text at cursor position.
        """

        debug('unindent')
        cursor = self.editor.textCursor()
        debug('cursor has selection %r', cursor.hasSelection())
        if cursor.hasSelection():
            cursor.beginEditBlock()
            self.unindent_selection(cursor)
            cursor.endEditBlock()
            self.editor.setTextCursor(cursor)
        else:
            tab_len = self.editor.tab_length
            indentation = cursor.positionInBlock()
            max_spaces = tab_len - (indentation - (indentation % tab_len))
            spaces = self.count_deletable_spaces(cursor, max_spaces)
            debug('deleting %d space before cursor' % spaces)
            cursor.beginEditBlock()
            if spaces:
                # delete spaces before cursor
                for _ in range(spaces):
                    cursor.deletePreviousChar()
            else:
                # un-indent whole line
                debug('un-indent whole line')
                cursor = self.unindent_selection(cursor)
            cursor.endEditBlock()
            self.editor.setTextCursor(cursor)
            debug(cursor.block().text())
<commit_msg>IndenterMode: fix tab-based indentation<commit_after># -*- coding: utf-8 -*-
"""
Contains the default indenter.
"""
import logging
from pyqode.core.api import TextHelper
from pyqode.core.api.mode import Mode
from pyqode.qt import QtGui

def _logger():
    return logging.getLogger(__name__)

def debug(msg, *args):
    return _logger().log(5, msg, *args)

class IndenterMode(Mode):
    """ Implements classic indentation/tabulation (Tab/Shift+Tab)

    It inserts/removes tabulations (a series of spaces defined by the
    tabLength settings) at the cursor position if there is no selection,
    otherwise it fully indents/un-indents selected lines.

    To trigger an indentation/un-indentation programatically, you must emit
    :attr:`pyqode.core.api.CodeEdit.indent_requested` or
    :attr:`pyqode.core.api.CodeEdit.unindent_requested`.
    """
    def __init__(self):
        super(IndenterMode, self).__init__()

    @property
    def _indent_char(self):

        if self.editor.use_spaces_instead_of_tabs:
            return ' '
        return '\t'

    @property
    def _single_indent(self):

        if self.editor.use_spaces_instead_of_tabs:
            return self.editor.tab_length * ' '
        return '\t'

    def on_state_changed(self, state):
        if state:
            self.editor.indent_requested.connect(self.indent)
            self.editor.unindent_requested.connect(self.unindent)
        else:
            self.editor.indent_requested.disconnect(self.indent)
            self.editor.unindent_requested.disconnect(self.unindent)

    def indent_selection(self, cursor):
        """
        Indent selected text

        :param cursor: QTextCursor
        """
        doc = self.editor.document()
        cursor.beginEditBlock()
        nb_lines = len(cursor.selection().toPlainText().splitlines())
        c = self.editor.textCursor()
        if c.atBlockStart() and c.position() == c.selectionEnd():
            nb_lines += 1
        block = doc.findBlock(cursor.selectionStart())
        i = 0
        # indent every lines
        while i < nb_lines:
            cursor = QtGui.QTextCursor(block)
            cursor.movePosition(cursor.StartOfLine, cursor.MoveAnchor)
            cursor.insertText(self._single_indent)
            block = block.next()
            i += 1
        cursor.endEditBlock()

    def unindent_selection(self, cursor):
        """
        Un-indents selected text

        :param cursor: QTextCursor
        """
        doc = self.editor.document()
        nb_lines = len(cursor.selection().toPlainText().splitlines())
        if nb_lines == 0:
            nb_lines = 1
        block = doc.findBlock(cursor.selectionStart())
        assert isinstance(block, QtGui.QTextBlock)
        i = 0
        debug('unindent selection: %d lines', nb_lines)
        while i < nb_lines:
            cursor.setPosition(block.position(), cursor.MoveAnchor)
            txt = block.text()
            debug('line to unindent: %s', txt)
            debug('self.editor.use_spaces_instead_of_tabs: %r',
                  self.editor.use_spaces_instead_of_tabs)
            indentation = len(txt) - len(txt.lstrip(self._indent_char))
            debug('unindent line %d: %d characters', i, indentation)
            cursor.movePosition(cursor.StartOfLine, cursor.MoveAnchor)
            for _ in range(len(self._single_indent)):
                txt = block.text()
                if len(txt) and txt[0] == self._indent_char:
                    cursor.deleteChar()
            block = block.next()
            i += 1
        return cursor

    def indent(self):
        """
        Indents text at cursor position.
        """
        cursor = self.editor.textCursor()
        assert isinstance(cursor, QtGui.QTextCursor)
        if cursor.hasSelection():
            self.indent_selection(cursor)
        else:
            # simply insert indentation at the cursor position
            tab_len = self.editor.tab_length
            cursor.beginEditBlock()
            if self.editor.use_spaces_instead_of_tabs:
                nb_space_to_add = tab_len - cursor.positionInBlock() % tab_len
                cursor.insertText(nb_space_to_add * " ")
            else:
                cursor.insertText('\t')
            cursor.endEditBlock()

    def count_deletable_spaces(self, cursor, max_spaces):
        # count the number of spaces deletable, stop at tab len
        max_spaces = abs(max_spaces)
        if max_spaces > self.editor.tab_length:
            max_spaces = self.editor.tab_length
        spaces = 0
        trav_cursor = QtGui.QTextCursor(cursor)
        while spaces < max_spaces or trav_cursor.atBlockStart():
            pos = trav_cursor.position()
            trav_cursor.movePosition(cursor.Left, cursor.KeepAnchor)
            char = trav_cursor.selectedText()
            if char == self._indent_char:
                spaces += 1
            else:
                break
            trav_cursor.setPosition(pos - 1)
        return spaces

    def unindent(self):
        """
        Un-indents text at cursor position.
        """

        debug('unindent')
        cursor = self.editor.textCursor()
        debug('cursor has selection %r', cursor.hasSelection())
        if cursor.hasSelection():
            cursor.beginEditBlock()
            self.unindent_selection(cursor)
            cursor.endEditBlock()
            self.editor.setTextCursor(cursor)
        else:
            tab_len = self.editor.tab_length
            indentation = cursor.positionInBlock()
            max_spaces = tab_len - (indentation - (indentation % tab_len))
            spaces = self.count_deletable_spaces(cursor, max_spaces)
            debug('deleting %d space before cursor' % spaces)
            cursor.beginEditBlock()
            if spaces:
                # delete spaces before cursor
                for _ in range(spaces):
                    cursor.deletePreviousChar()
            else:
                # un-indent whole line
                debug('un-indent whole line')
                cursor = self.unindent_selection(cursor)
            cursor.endEditBlock()
            self.editor.setTextCursor(cursor)
            debug(cursor.block().text())
<|endoftext|>


<commit_before>import os
import shutil
import pytest
import numpy as np
from osgeo import ogr
from pyroSAR import identify
from pyroSAR.spatial import crsConvert, haversine, Raster, stack, ogr2ogr, gdal_translate, gdal_rasterize
from pyroSAR.ancillary import finder

def test_crsConvert():
    assert crsConvert(crsConvert(4326, 'wkt'), 'proj4') == '+proj=longlat +datum=WGS84 +no_defs '
    assert crsConvert(crsConvert(4326, 'prettyWkt'), 'opengis') == 'http://www.opengis.net/def/crs/EPSG/0/4326'
    assert crsConvert('+proj=longlat +datum=WGS84 +no_defs ', 'epsg') == 4326
    assert crsConvert('http://www.opengis.net/def/crs/EPSG/0/4326', 'epsg') == 4326
    assert crsConvert(crsConvert('http://www.opengis.net/def/crs/EPSG/0/4326', 'osr'), 'epsg') == 4326
    with pytest.raises(TypeError):
        crsConvert('xyz', 'epsg')
    with pytest.raises(ValueError):
        crsConvert(4326, 'xyz')

def test_haversine():
    assert haversine(50, 10, 51, 10) == 111194.92664455889

def test_Vector():
    scene = identify('pyroSAR/tests/data/S1A_IW_GRDH_1SDV_20150222T170750_20150222T170815_004739_005DD8_3768.zip')
    bbox = scene.bbox()
    assert bbox.getArea() == 7.573045244595988
    assert bbox.extent == {'ymax': 52.183979, 'ymin': 50.295261, 'xmin': 8.017178, 'xmax': 12.0268}
    assert bbox.nlayers == 1
    assert bbox.getProjection('epsg') == 4326
    assert bbox.proj4 == '+proj=longlat +datum=WGS84 +no_defs'
    assert isinstance(bbox.getFeatureByIndex(0), ogr.Feature)
    with pytest.raises(IndexError):
        bbox.getFeatureByIndex(1)
    bbox.reproject(32633)
    assert bbox.proj4 == '+proj=utm +zone=33 +datum=WGS84 +units=m +no_defs'

def test_Raster():
    with pytest.raises(OSError):
        ras = Raster('foobar')
    ras = Raster('pyroSAR/tests/data/S1A__IW___A_20150309T173017_VV_grd_mli_geo_norm_db.tif')
    assert ras.bands == 1
    assert ras.proj4 == '+proj=utm +zone=31 +datum=WGS84 +units=m +no_defs '
    assert ras.cols == 268
    assert ras.rows == 217
    assert ras.dim == [217, 268, 1]
    assert ras.dtype == 'Float32'
    assert ras.dtypes(ras.dtype) == 6
    assert ras.epsg == 32631
    assert ras.format == 'GTiff'
    assert ras.geo == {'ymax': 4830114.70107, 'rotation_y': 0.0, 'rotation_x': 0.0, 'xmax': 625408.241204, 'xres': 20.0,
                       'xmin': 620048.241204, 'ymin': 4825774.70107, 'yres': -20.0}
    assert ras.typemap() == {'int32': 5, 'int16': 3, 'float64': 7, 'complex128': 11, 'uint8': 1, 'uint16': 2,
                             'complex64': 10, 'uint32': 4, 'int8': 1, 'float32': 6}
    assert ras.geogcs == 'WGS 84'
    assert ras.is_valid() is True
    assert ras.proj4args == {'units': 'm', 'no_defs': None, 'datum': 'WGS84', 'proj': 'utm', 'zone': '31'}
    assert ras.allstats == [[-26.65471076965332, 1.4325850009918213, -12.124929534450377, 4.738273594738293]]
    assert ras.bbox().getArea() == 23262400.0
    assert len(ras.layers()) == 1
    ras.load()
    mat = ras.matrix()
    assert isinstance(mat, np.ndarray)
    ras.assign(mat)
    # ras.reduce()
    ras.rescale(lambda x: 10 * x)
    ras.write('pyroSAR/tests/data/S1A__IW___A_20150309T173017_VV_grd_mli_geo_norm_db_new.tif')
    for item in finder('pyroSAR/tests/data', ['S1A*_new*']):
        os.remove(item)

def test_stack():
    name = 'pyroSAR/tests/data/S1A__IW___A_20150309T173017_VV_grd_mli_geo_norm_db.tif'
    outname = 'pyroSAR/tests/data/S1A__IW___A_20150309T173017_VV_grd_mli_geo_norm_db_sub'
    tr = (30, 30)
    with pytest.raises(IOError):
        stack(srcfiles=[], resampling='near', targetres=tr,
              srcnodata=-99, dstnodata=-99, dstfile=outname)

    with pytest.raises(IOError):
        stack(srcfiles=[name, name], resampling='near', targetres=tr,
              srcnodata=-99, dstnodata=-99, dstfile=outname, layernames=['a'])

    with pytest.raises(RuntimeError):
        stack(srcfiles=[name, name], resampling='near', targetres=30,
              srcnodata=-99, dstnodata=-99, dstfile=outname)

    with pytest.raises(RuntimeError):
        stack(srcfiles=[name, name], resampling='near', targetres=(30, 30, 30),
              srcnodata=-99, dstnodata=-99, dstfile=outname)

    with pytest.raises(IOError):
        stack(srcfiles=[name, name], resampling='foobar', targetres=tr,
              srcnodata=-99, dstnodata=-99, dstfile=outname)

    with pytest.raises(OSError):
        stack(srcfiles=[name, 'foobar'], resampling='near', targetres=tr,
              srcnodata=-99, dstnodata=-99, dstfile=outname)

    stack(srcfiles=[name, name], resampling='near', targetres=tr, overwrite=True,
          srcnodata=-99, dstnodata=-99, dstfile=outname)
    for item in finder('pyroSAR/tests/data', ['S1A*_sub*']):
        os.remove(item)

def test_auxil():
    dir = 'pyroSAR/tests/data/subdir'
    if not os.path.exists(dir):
        os.makedirs(dir)
    ras = Raster('pyroSAR/tests/data/S1A__IW___A_20150309T173017_VV_grd_mli_geo_norm_db.tif')
    bbox = os.path.join(dir, 'bbox.shp')
    ras.bbox(bbox)
    ogr2ogr(bbox, os.path.join(dir, 'bbox.gml'), {'format': 'GML'})
    gdal_translate(ras.raster, os.path.join(dir, 'test'), {'format': 'ENVI'})
    gdal_rasterize(bbox, os.path.join(dir, 'test2'), {'format': 'GTiff', 'xRes': 20, 'yRes': 20})
    shutil.rmtree(dir)
<commit_msg>additional tests for module vector<commit_after>import os
import shutil
import pytest
import numpy as np
from osgeo import ogr
from pyroSAR import identify
from pyroSAR.spatial import crsConvert, haversine, Raster, stack, ogr2ogr, gdal_translate, gdal_rasterize, Vector, bbox, intersect
from pyroSAR.spatial.vector import feature2vector, dissolve
from pyroSAR.ancillary import finder

def test_crsConvert():
    assert crsConvert(crsConvert(4326, 'wkt'), 'proj4') == '+proj=longlat +datum=WGS84 +no_defs '
    assert crsConvert(crsConvert(4326, 'prettyWkt'), 'opengis') == 'http://www.opengis.net/def/crs/EPSG/0/4326'
    assert crsConvert('+proj=longlat +datum=WGS84 +no_defs ', 'epsg') == 4326
    assert crsConvert('http://www.opengis.net/def/crs/EPSG/0/4326', 'epsg') == 4326
    assert crsConvert(crsConvert('http://www.opengis.net/def/crs/EPSG/0/4326', 'osr'), 'epsg') == 4326
    with pytest.raises(TypeError):
        crsConvert('xyz', 'epsg')
    with pytest.raises(ValueError):
        crsConvert(4326, 'xyz')

def test_haversine():
    assert haversine(50, 10, 51, 10) == 111194.92664455889

def test_Vector(tmpdir):
    scene = identify('pyroSAR/tests/data/S1A_IW_GRDH_1SDV_20150222T170750_20150222T170815_004739_005DD8_3768.zip')
    bbox1 = scene.bbox()
    assert bbox1.getArea() == 7.573045244595988
    assert bbox1.extent == {'ymax': 52.183979, 'ymin': 50.295261, 'xmin': 8.017178, 'xmax': 12.0268}
    assert bbox1.nlayers == 1
    assert bbox1.getProjection('epsg') == 4326
    assert bbox1.proj4 == '+proj=longlat +datum=WGS84 +no_defs'
    assert isinstance(bbox1.wkt, str)
    assert isinstance(bbox1.getFeatureByIndex(0), ogr.Feature)
    with pytest.raises(IndexError):
        bbox1.getFeatureByIndex(1)
    bbox1.reproject(32633)
    assert bbox1.proj4 == '+proj=utm +zone=33 +datum=WGS84 +units=m +no_defs'
    assert isinstance(bbox1['id=1'], Vector)
    with pytest.raises(RuntimeError):
        test = bbox1[0.1]
    assert bbox1.fieldnames == ['id']
    assert bbox1.getUniqueAttributes('id') == [1]
    feat = bbox1.getFeatureByAttribute('id', 1)
    assert isinstance(feat, ogr.Feature)
    bbox2 = feature2vector(feat, ref=bbox1)
    bbox2.close()
    feat.Destroy()
    with pytest.raises(KeyError):
        select = bbox1.getFeatureByAttribute('foo', 'bar')
    ext = bbox1.extent
    for key in ext.keys():
        ext[key] += 1
    bbox2_name = os.path.join(str(tmpdir), 'bbox2.shp')
    bbox(ext, bbox1.srs, bbox2_name)
    with Vector(bbox2_name) as bbox2:
        assert intersect(bbox1, bbox2) is not None
        bbox1.addvector(bbox2)
    bbox3_name = os.path.join(str(tmpdir), 'bbox3.shp')
    bbox1.write(bbox3_name)
    bbox1.close()
    bbox4_name = os.path.join(str(tmpdir), 'bbox4.shp')
    dissolve(bbox3_name, bbox4_name, field='id')
    assert os.path.isfile(bbox4_name)
    with pytest.raises(RuntimeError):
        vec = Vector(driver='foobar')

def test_Raster():
    with pytest.raises(OSError):
        ras = Raster('foobar')
    ras = Raster('pyroSAR/tests/data/S1A__IW___A_20150309T173017_VV_grd_mli_geo_norm_db.tif')
    assert ras.bands == 1
    assert ras.proj4 == '+proj=utm +zone=31 +datum=WGS84 +units=m +no_defs '
    assert ras.cols == 268
    assert ras.rows == 217
    assert ras.dim == [217, 268, 1]
    assert ras.dtype == 'Float32'
    assert ras.dtypes(ras.dtype) == 6
    assert ras.epsg == 32631
    assert ras.format == 'GTiff'
    assert ras.geo == {'ymax': 4830114.70107, 'rotation_y': 0.0, 'rotation_x': 0.0, 'xmax': 625408.241204, 'xres': 20.0,
                       'xmin': 620048.241204, 'ymin': 4825774.70107, 'yres': -20.0}
    assert ras.typemap() == {'int32': 5, 'int16': 3, 'float64': 7, 'complex128': 11, 'uint8': 1, 'uint16': 2,
                             'complex64': 10, 'uint32': 4, 'int8': 1, 'float32': 6}
    assert ras.geogcs == 'WGS 84'
    assert ras.is_valid() is True
    assert ras.proj4args == {'units': 'm', 'no_defs': None, 'datum': 'WGS84', 'proj': 'utm', 'zone': '31'}
    assert ras.allstats == [[-26.65471076965332, 1.4325850009918213, -12.124929534450377, 4.738273594738293]]
    assert ras.bbox().getArea() == 23262400.0
    assert len(ras.layers()) == 1
    ras.load()
    mat = ras.matrix()
    assert isinstance(mat, np.ndarray)
    ras.assign(mat)
    # ras.reduce()
    ras.rescale(lambda x: 10 * x)
    ras.write('pyroSAR/tests/data/S1A__IW___A_20150309T173017_VV_grd_mli_geo_norm_db_new.tif')
    for item in finder('pyroSAR/tests/data', ['S1A*_new*']):
        os.remove(item)

def test_stack():
    name = 'pyroSAR/tests/data/S1A__IW___A_20150309T173017_VV_grd_mli_geo_norm_db.tif'
    outname = 'pyroSAR/tests/data/S1A__IW___A_20150309T173017_VV_grd_mli_geo_norm_db_sub'
    tr = (30, 30)
    with pytest.raises(IOError):
        stack(srcfiles=[], resampling='near', targetres=tr,
              srcnodata=-99, dstnodata=-99, dstfile=outname)

    with pytest.raises(IOError):
        stack(srcfiles=[name, name], resampling='near', targetres=tr,
              srcnodata=-99, dstnodata=-99, dstfile=outname, layernames=['a'])

    with pytest.raises(RuntimeError):
        stack(srcfiles=[name, name], resampling='near', targetres=30,
              srcnodata=-99, dstnodata=-99, dstfile=outname)

    with pytest.raises(RuntimeError):
        stack(srcfiles=[name, name], resampling='near', targetres=(30, 30, 30),
              srcnodata=-99, dstnodata=-99, dstfile=outname)

    with pytest.raises(IOError):
        stack(srcfiles=[name, name], resampling='foobar', targetres=tr,
              srcnodata=-99, dstnodata=-99, dstfile=outname)

    with pytest.raises(OSError):
        stack(srcfiles=[name, 'foobar'], resampling='near', targetres=tr,
              srcnodata=-99, dstnodata=-99, dstfile=outname)

    stack(srcfiles=[name, name], resampling='near', targetres=tr, overwrite=True,
          srcnodata=-99, dstnodata=-99, dstfile=outname)
    for item in finder('pyroSAR/tests/data', ['S1A*_sub*']):
        os.remove(item)

def test_auxil():
    dir = 'pyroSAR/tests/data/subdir'
    if not os.path.exists(dir):
        os.makedirs(dir)
    ras = Raster('pyroSAR/tests/data/S1A__IW___A_20150309T173017_VV_grd_mli_geo_norm_db.tif')
    bbox = os.path.join(dir, 'bbox.shp')
    ras.bbox(bbox)
    ogr2ogr(bbox, os.path.join(dir, 'bbox.gml'), {'format': 'GML'})
    gdal_translate(ras.raster, os.path.join(dir, 'test'), {'format': 'ENVI'})
    gdal_rasterize(bbox, os.path.join(dir, 'test2'), {'format': 'GTiff', 'xRes': 20, 'yRes': 20})
    shutil.rmtree(dir)
<|endoftext|>


<commit_before>#!/usr/bin/env python
import os
import sys
from memrise import Course, COURSE_URL
from gtts import gTTS

def dump_tts(*, course_url : str):
    """
    :course_url:   course URL
    """

    course = Course(course_url=course_url)
    os.makedirs(course.name, exist_ok=True)

    for level_url, title in course.levels:
        print("*** %s (%s)" % (title, level_url), flush=True)
        output_dir = os.path.join(course.name, title)
        os.makedirs(output_dir, exist_ok=True)

        for word, *_ in course.cards(level_url=level_url):
            file_name = os.path.join(output_dir, word) + ".mp3"
            if not os.path.exists(file_name):
                tts = gTTS(word, lang="ko")
                tts.save(file_name)

if __name__ == "__main__":
    course_url = COURSE_URL if len(sys.argv) < 2 else sys.argv[1]
    dump_tts(course_url=course_url)
<commit_msg>storing card data in TSV files<commit_after>#!/usr/bin/env python
import os
import sys
from memrise import Course, COURSE_URL
from gtts import gTTS

def dump_tts(*, course_url : str):
    """
    :course_url:   course URL
    """

    course = Course(course_url=course_url)
    os.makedirs(course.name, exist_ok=True)

    for level_url, title in course.levels:
        print("*** %s (%s)" % (title, level_url), flush=True)
        output_dir = os.path.join(course.name, title)
        if not no_audio:
            os.makedirs(output_dir, exist_ok=True)

        with open(os.path.join(course.name, title) + ".csv", "w") as level_file:
            for card in course.cards(level_url=level_url):
                card = list(card)
                word = card[0]
                level_file.write('\t'.join(card))
                level_file.write('\n')
                file_name = os.path.join(output_dir, word) + ".mp3"
                if not no_audio and not os.path.exists(file_name):
                    tts = gTTS(word, lang="ko")
                    tts.save(file_name)

if __name__ == "__main__":
    no_audio = ("--no-audio" in sys.argv)
    if no_audio:
        sys.argv.pop(sys.argv.index("--no-audio"))
    course_url = COURSE_URL if len(sys.argv) < 2 else sys.argv[1]
    dump_tts(course_url=course_url)
<|endoftext|>


<commit_before>from keras import backend as K
from keras.models import (
    Sequential,
    Model
)
from keras.layers import (
    Activation,
    AveragePooling2D,
    BatchNormalization,
    Convolution2D,
    Dense,
    Dropout,
    Flatten,
    GlobalAveragePooling2D,
    Input,
    MaxPooling2D,
    merge,
)
from keras.regularizers import l2

def channel_axis():
    if K.image_dim_ordering() == 'tf':
        return 3
    else:
        return 1

def load_simple_cnn(input_shape, n_classes, weight_decay):
    model = Sequential()

    model.add(Convolution2D(32, 3, 3, border_mode='same',
                            input_shape=input_shape,
                            W_regularizer=l2(weight_decay)))
    model.add(Activation('relu'))
    model.add(Convolution2D(32, 3, 3))
    model.add(Activation('relu'))
    model.add(MaxPooling2D(pool_size=(2, 2)))
    model.add(Dropout(0.25))

    model.add(Convolution2D(64, 3, 3, border_mode='same',
                            W_regularizer=l2(weight_decay)))
    model.add(Activation('relu'))
    model.add(Convolution2D(64, 3, 3,
                            W_regularizer=l2(weight_decay)))
    model.add(Activation('relu'))
    model.add(MaxPooling2D(pool_size=(2, 2)))
    model.add(Dropout(0.25))

    model.add(Flatten())
    model.add(Dense(512, W_regularizer=l2(weight_decay)))
    model.add(Activation('relu'))
    model.add(Dropout(0.5))
    model.add(Dense(n_classes, W_regularizer=l2(weight_decay)))
    model.add(Activation('softmax'))

    return model

def load_resnet(input_shape, n_classes, depth, weight_decay):

    wd = weight_decay

    def bnrelu(x_input):
        x_output = BatchNormalization(mode=0,
                                      axis=channel_axis(),
                                      gamma_regularizer=l2(wd),
                                      beta_regularizer=l2(wd))(x_input)
        x_output = Activation('relu')(x_output)
        return x_output

    def basic_block(x, n_input, n_output, stride):

        resmap = bnrelu(x)

        if n_input != n_output:
            x = resmap

        resmap = Convolution2D(n_output, 3, 3,
                               init='he_normal',
                               border_mode='same',
                               subsample=(stride, stride),
                               bias=False,
                               W_regularizer=l2(wd))(resmap)

        resmap = bnrelu(resmap)

        resmap = Convolution2D(n_output, 3, 3,
                               init='he_normal',
                               border_mode='same',
                               subsample=(1,1),
                               bias=False,
                               W_regularizer=l2(wd))(resmap)

        if n_input == n_output:
            skip = x # Identity skip connection
        else:
            skip = Convolution2D(n_output, 1, 1,
                                 init='he_normal',
                                 border_mode='same',
                                 subsample=(stride, stride),
                                 bias=False,
                                 W_regularizer=l2(wd))(x)

        return merge([resmap, skip], mode='sum')

    def stage(x, n_input, n_output, n_block, stride):
        x = basic_block(x, n_input, n_output, stride)
        for i in range(n_block - 1):
            x = basic_block(x, n_input, n_output, 1)
        return x

    stages = [16, 16, 32, 64]
    if (depth - 4) % 6 != 0:
        error(Exception("depth must be 6n+4."))
    n_block = int((depth - 4) / 6)

    # Create model
    x_input = Input(shape=input_shape)
    x = Convolution2D(stages[0], 3, 3,
                      init='he_normal',
                      border_mode='same',
                      subsample=(1,1),
                      bias=False,
                      W_regularizer=l2(wd))(x_input) # spatial size 32x32
    x = stage(x, stages[0], stages[1], n_block, 1) # spatial size 32x32
    x = stage(x, stages[1], stages[2], n_block, 2) # spatial size 16x16
    x = stage(x, stages[2], stages[3], n_block, 2) # spatial size 8x8
    x = bnrelu(x)
    x = GlobalAveragePooling2D()(x)
    x = Dense(n_classes,
              init='he_normal',
              W_regularizer=l2(wd),
              b_regularizer=l2(wd))(x)
    x_output = Activation('softmax')(x)
    model = Model(input=x_input, output=x_output)

    return model

def load_model(net_type, input_shape, n_classes, depth, weight_decay):
    if net_type == 'simple':
        model = load_simple_cnn(input_shape, n_classes, weight_decay)
    elif net_type == 'resnet':
        model = load_resnet(input_shape, n_classes, depth, weight_decay)
    else:
        raise("Invalid net_type.")
    return model
<commit_msg>bug fix in resnet<commit_after>from keras import backend as K
from keras.models import (
    Sequential,
    Model
)
from keras.layers import (
    Activation,
    AveragePooling2D,
    BatchNormalization,
    Convolution2D,
    Dense,
    Dropout,
    Flatten,
    GlobalAveragePooling2D,
    Input,
    MaxPooling2D,
    merge,
)
from keras.regularizers import l2

def channel_axis():
    if K.image_dim_ordering() == 'tf':
        return 3
    else:
        return 1

def load_simple_cnn(input_shape, n_classes, weight_decay):
    model = Sequential()

    model.add(Convolution2D(32, 3, 3, border_mode='same',
                            input_shape=input_shape,
                            W_regularizer=l2(weight_decay)))
    model.add(Activation('relu'))
    model.add(Convolution2D(32, 3, 3))
    model.add(Activation('relu'))
    model.add(MaxPooling2D(pool_size=(2, 2)))
    model.add(Dropout(0.25))

    model.add(Convolution2D(64, 3, 3, border_mode='same',
                            W_regularizer=l2(weight_decay)))
    model.add(Activation('relu'))
    model.add(Convolution2D(64, 3, 3,
                            W_regularizer=l2(weight_decay)))
    model.add(Activation('relu'))
    model.add(MaxPooling2D(pool_size=(2, 2)))
    model.add(Dropout(0.25))

    model.add(Flatten())
    model.add(Dense(512, W_regularizer=l2(weight_decay)))
    model.add(Activation('relu'))
    model.add(Dropout(0.5))
    model.add(Dense(n_classes, W_regularizer=l2(weight_decay)))
    model.add(Activation('softmax'))

    return model

def load_resnet(input_shape, n_classes, depth, weight_decay):

    wd = weight_decay

    def bnrelu(x_input):
        x_output = BatchNormalization(mode=0,
                                      axis=channel_axis(),
                                      gamma_regularizer=l2(wd),
                                      beta_regularizer=l2(wd))(x_input)
        x_output = Activation('relu')(x_output)
        return x_output

    def basic_block(x, n_input, n_output, stride):

        resmap = bnrelu(x)

        if n_input != n_output:
            x = resmap

        resmap = Convolution2D(n_output, 3, 3,
                               init='he_normal',
                               border_mode='same',
                               subsample=(stride, stride),
                               bias=False,
                               W_regularizer=l2(wd))(resmap)

        resmap = bnrelu(resmap)

        resmap = Convolution2D(n_output, 3, 3,
                               init='he_normal',
                               border_mode='same',
                               subsample=(1,1),
                               bias=False,
                               W_regularizer=l2(wd))(resmap)

        if n_input == n_output:
            skip = x # Identity skip connection
        else:
            skip = Convolution2D(n_output, 1, 1,
                                 init='he_normal',
                                 border_mode='same',
                                 subsample=(stride, stride),
                                 bias=False,
                                 W_regularizer=l2(wd))(x)

        return merge([resmap, skip], mode='sum')

    def stage(x, n_input, n_output, n_block, stride):
        x = basic_block(x, n_input, n_output, stride)
        for i in range(n_block - 1):
            x = basic_block(x, n_output, n_output, 1)
        return x

    stages = [16, 16, 32, 64]
    if (depth - 4) % 6 != 0:
        error(Exception("depth must be 6n+4."))
    n_block = int((depth - 4) / 6)

    # Create model
    x_input = Input(shape=input_shape)
    x = Convolution2D(stages[0], 3, 3,
                      init='he_normal',
                      border_mode='same',
                      subsample=(1,1),
                      bias=False,
                      W_regularizer=l2(wd))(x_input) # spatial size 32x32
    x = stage(x, stages[0], stages[1], n_block, 1) # spatial size 32x32
    x = stage(x, stages[1], stages[2], n_block, 2) # spatial size 16x16
    x = stage(x, stages[2], stages[3], n_block, 2) # spatial size 8x8
    x = bnrelu(x)
    x = GlobalAveragePooling2D()(x)
    x = Dense(n_classes,
              init='he_normal',
              W_regularizer=l2(wd),
              b_regularizer=l2(wd))(x)
    x_output = Activation('softmax')(x)
    model = Model(input=x_input, output=x_output)

    return model

def load_model(net_type, input_shape, n_classes, depth, weight_decay):
    if net_type == 'simple':
        model = load_simple_cnn(input_shape, n_classes, weight_decay)
    elif net_type == 'resnet':
        model = load_resnet(input_shape, n_classes, depth, weight_decay)
    else:
        raise("Invalid net_type.")
    return model
<|endoftext|>


<commit_before>import pysal
import datetime
import struct
import itertools

class DBF(pysal.core.Tables.DataTable):
    """ """
    FORMATS = ['dbf']
    MODES = ['r','w']
    def __init__(self,*args,**kwargs):
        pysal.core.Tables.DataTable.__init__(self,*args,**kwargs)
        if self.mode == 'r':
            self.f = f = open(self.dataPath,'rb')
            numrec, lenheader = struct.unpack('<xxxxLH22x', f.read(32))
            numfields = (lenheader - 33) // 32
            self.n_records = numrec
            self.n_fields = numfields
            self.field_info = [('DeletionFlag', 'C', 1, 0)]
            record_size = 1
            fmt = 's'
            for fieldno in xrange(numfields):
                name, typ, size, deci = struct.unpack('<11sc4xBB14x', f.read(32))
                fmt+='%ds'%size
                record_size+=size
                name = name.replace('\0', '')       # eliminate NULs from string
                self.field_info.append((name, typ, size, deci))
            terminator = f.read(1)
            assert terminator == '\r'
            self.header_size = self.f.tell()
            self.record_size = record_size
            self.record_fmt = fmt
            self.pos = 0
            self.header = [fInfo[0] for fInfo in self.field_info[1:]]
            field_spec = []
            for fname,ftype,flen,fpre in self.field_info[1:]:
                field_spec.append((ftype,flen,fpre))
            self.field_spec = field_spec
            #self.spec = [types[fInfo[0]] for fInfo in self.field_info]
        elif self.mode == 'w':
            self.f = open(self.dataPath,'wb')
            self.header = None
            self.field_spec = None
            self.numrec = 0
            self.FIRST_WRITE = True
    def __len__(self):
        if self.mode != 'r': raise IOError, "Invalid operation, Cannot read from a file opened in 'w' mode."
        return self.n_records
    def seek(self,i):
        self.f.seek(self.header_size + (self.record_size*i))
        self.pos = i
    def read_record(self,i):
        self.seek(i)
        rec = list(struct.unpack(self.record_fmt, self.f.read(self.record_size)))
        if rec[0] != ' ':
            return self.read_record(i+1)
        result = []
        for (name, typ, size, deci), value in itertools.izip(self.field_info, rec):
            if name == 'DeletionFlag':
                continue
            if typ == 'N':
                value = value.replace('\0', '').lstrip()
                if value == '':
                    value = 0
                elif deci:
                    value = float(value)
                else:
                    value = int(value)
            elif typ == 'D':
                y, m, d = int(value[:4]), int(value[4:6]), int(value[6:8])
                value = datetime.date(y, m, d)
            elif typ == 'L':
                value = (value in 'YyTt' and 'T') or (value in 'NnFf' and 'F') or '?'
            elif typ == 'F':
                value = float(value)
            if isinstance(value, str):
                value = value.rstrip()
            result.append(value)
        return result
    def _read(self):
        if self.mode != 'r': raise IOError, "Invalid operation, Cannot read from a file opened in 'w' mode."
        if self.pos < len(self):
            rec = self.read_record(self.pos)
            self.pos+=1
            return rec
        else:
            return None
    def list_dbf(self):
        if self.mode != 'r': raise IOError, "Invalid operation, Cannot read from a file opened in 'w' mode."
        dbf=self.f
        print "%d records, %d fields" % (dbf.record_count(), dbf.field_count())
        format = ""
        for i in range(dbf.field_count()):
            type, name, len, decc = dbf.field_info(i)
            print type,name,len,decc
            if type == 0: #string
                format = format + " %%(%s)%ds" % (name, len)
            elif type == 1: #int
                format = format + " %%(%s)%dd" % (name, len)
            elif type == 2: #float
                format = format + " %%(%s)%dg" % (name, len)
        print format
        for i in range(dbf.record_count()):
            print format % dbf.read_record(i)
    def write(self,obj):
        self._complain_ifclosed(self.closed)
        if self.mode != 'w': raise IOError, "Invalid operation, Cannot write to a file opened in 'r' mode."
        if self.FIRST_WRITE:
            self._firstWrite(obj)
        if len(obj) != len(self.header):
            raise TypeError, "Rows must contains %d fields"%len(self.header)
        self.numrec+=1
        self.f.write(' ')                        # deletion flag
        for (typ, size, deci), value in itertools.izip(self.field_spec, obj):
            if typ == "N":
                value = str(value).rjust(size, ' ')
            elif typ == 'D':
                value = value.strftime('%Y%m%d')
            elif typ == 'L':
                value = str(value)[0].upper()
            else:
                value = str(value)[:size].ljust(size, ' ')
            try:
                assert len(value) == size
            except:
                print value,len(value),size
                raise
            self.f.write(value)
            self.pos+=1
    def flush(self):
        self._complain_ifclosed(self.closed)
        self._writeHeader()
        self.f.flush()
    def close(self):
        if self.mode == 'w':
            self.flush()
            # End of file
            self.f.write('\x1A')
            self.f.close()
        pysal.core.Tables.DataTable.close(self)
    def _firstWrite(self,obj):
        if not self.header: raise IOError, "No header, DBF files require a header."
        if not self.field_spec: raise IOError, "No field_spec, DBF files require a specification."
        self._writeHeader()
        self.FIRST_WRITE = False
    def _writeHeader(self):
        """ Modified from: http://aspn.activestate.com/ASPN/Cookbook/Python/Recipe/362715 """
        POS = self.f.tell()
        self.f.seek(0)
        ver = 3
        now = datetime.datetime.now()
        yr, mon, day = now.year-1900, now.month, now.day
        numrec = self.numrec
        numfields = len(self.header)
        lenheader = numfields * 32 + 33
        lenrecord = sum(field[1] for field in self.field_spec) + 1
        hdr = struct.pack('<BBBBLHH20x', ver, yr, mon, day, numrec, lenheader, lenrecord)
        self.f.write(hdr)
        # field specs
        for name, (typ, size, deci) in itertools.izip(self.header, self.field_spec):
            name = name.ljust(11, '\x00')
            fld = struct.pack('<11sc4xBB14x', name, typ, size, deci)
            self.f.write(fld)
        # terminator
        self.f.write('\r')
        if self.f.tell() != POS and not self.FIRST_WRITE:
            self.f.seek(POS)

if __name__ == '__main__':
    file_name = "../../examples/10740.dbf"
    f=pysal.open(file_name,'r')
    newDB = pysal.open('copy.dbf','w')
    newDB.header = f.header
    newDB.field_spec = f.field_spec
    print f.header
    for row in f:
        print row
        newDB.write(row)
    newDB.close()
    copy = pysal.open('copy.dbf','r')
    f.seek(0)
    print "HEADER: ", copy.header == f.header
    print "SPEC: ", copy.field_spec == f.field_spec
    print "DATA: ",list(copy) == list(f)
<commit_msg>commenting DBF handler<commit_after>import pysal
import datetime
import struct
import itertools

class DBF(pysal.core.Tables.DataTable):
    """
    PySAL DBF Reader/Writer

    This DBF handler implements the PySAL DataTable interface.

    Attributes:
    header -- list -- A list of field names.
    field_spec -- list -- A list of field specs.

    Heeader:
        The header is a python list of strings.  Each string is a field name and field name must not be longer than 10 characters.
        Eg.
        ['field1','field2','field3']
    Field Spec:
        The field_spec describes the data types of each field. It is comprised of a list of tuples, each tuple describing a field.
        The format for the tuples is ("Type",len,precision).  Valid Types are 'C' for characters, 'L' for bool, 'D' for data, 'N' or 'F' for number.
        [('C',20,0),('N',20,10),('L',1,0)]
    """
    FORMATS = ['dbf']
    MODES = ['r','w']
    def __init__(self,*args,**kwargs):
        """
        Initializes an instance of the pysal's DBF handler.

        Arguments:
        dataPath -- str -- Path to file, including file.
        mode -- str -- 'r' or 'w'
        """
        pysal.core.Tables.DataTable.__init__(self,*args,**kwargs)
        if self.mode == 'r':
            self.f = f = open(self.dataPath,'rb')
            numrec, lenheader = struct.unpack('<xxxxLH22x', f.read(32))
            numfields = (lenheader - 33) // 32
            self.n_records = numrec
            self.n_fields = numfields
            self.field_info = [('DeletionFlag', 'C', 1, 0)]
            record_size = 1
            fmt = 's'
            for fieldno in xrange(numfields):
                name, typ, size, deci = struct.unpack('<11sc4xBB14x', f.read(32))
                fmt+='%ds'%size
                record_size+=size
                name = name.replace('\0', '')       # eliminate NULs from string
                self.field_info.append((name, typ, size, deci))
            terminator = f.read(1)
            assert terminator == '\r'
            self.header_size = self.f.tell()
            self.record_size = record_size
            self.record_fmt = fmt
            self.pos = 0
            self.header = [fInfo[0] for fInfo in self.field_info[1:]]
            field_spec = []
            for fname,ftype,flen,fpre in self.field_info[1:]:
                field_spec.append((ftype,flen,fpre))
            self.field_spec = field_spec
            #self.spec = [types[fInfo[0]] for fInfo in self.field_info]
        elif self.mode == 'w':
            self.f = open(self.dataPath,'wb')
            self.header = None
            self.field_spec = None
            self.numrec = 0
            self.FIRST_WRITE = True
    def __len__(self):
        if self.mode != 'r': raise IOError, "Invalid operation, Cannot read from a file opened in 'w' mode."
        return self.n_records
    def seek(self,i):
        self.f.seek(self.header_size + (self.record_size*i))
        self.pos = i
    def read_record(self,i):
        self.seek(i)
        rec = list(struct.unpack(self.record_fmt, self.f.read(self.record_size)))
        if rec[0] != ' ':
            return self.read_record(i+1)
        result = []
        for (name, typ, size, deci), value in itertools.izip(self.field_info, rec):
            if name == 'DeletionFlag':
                continue
            if typ == 'N':
                value = value.replace('\0', '').lstrip()
                if value == '':
                    value = 0
                elif deci:
                    value = float(value)
                else:
                    value = int(value)
            elif typ == 'D':
                y, m, d = int(value[:4]), int(value[4:6]), int(value[6:8])
                value = datetime.date(y, m, d)
            elif typ == 'L':
                value = (value in 'YyTt' and 'T') or (value in 'NnFf' and 'F') or '?'
            elif typ == 'F':
                value = float(value)
            if isinstance(value, str):
                value = value.rstrip()
            result.append(value)
        return result
    def _read(self):
        if self.mode != 'r': raise IOError, "Invalid operation, Cannot read from a file opened in 'w' mode."
        if self.pos < len(self):
            rec = self.read_record(self.pos)
            self.pos+=1
            return rec
        else:
            return None
    def write(self,obj):
        self._complain_ifclosed(self.closed)
        if self.mode != 'w': raise IOError, "Invalid operation, Cannot write to a file opened in 'r' mode."
        if self.FIRST_WRITE:
            self._firstWrite(obj)
        if len(obj) != len(self.header):
            raise TypeError, "Rows must contains %d fields"%len(self.header)
        self.numrec+=1
        self.f.write(' ')                        # deletion flag
        for (typ, size, deci), value in itertools.izip(self.field_spec, obj):
            if typ == "N":
                value = str(value).rjust(size, ' ')
            elif typ == 'D':
                value = value.strftime('%Y%m%d')
            elif typ == 'L':
                value = str(value)[0].upper()
            else:
                value = str(value)[:size].ljust(size, ' ')
            try:
                assert len(value) == size
            except:
                print value,len(value),size
                raise
            self.f.write(value)
            self.pos+=1
    def flush(self):
        self._complain_ifclosed(self.closed)
        self._writeHeader()
        self.f.flush()
    def close(self):
        if self.mode == 'w':
            self.flush()
            # End of file
            self.f.write('\x1A')
            self.f.close()
        pysal.core.Tables.DataTable.close(self)
    def _firstWrite(self,obj):
        if not self.header: raise IOError, "No header, DBF files require a header."
        if not self.field_spec: raise IOError, "No field_spec, DBF files require a specification."
        self._writeHeader()
        self.FIRST_WRITE = False
    def _writeHeader(self):
        """ Modified from: http://aspn.activestate.com/ASPN/Cookbook/Python/Recipe/362715 """
        POS = self.f.tell()
        self.f.seek(0)
        ver = 3
        now = datetime.datetime.now()
        yr, mon, day = now.year-1900, now.month, now.day
        numrec = self.numrec
        numfields = len(self.header)
        lenheader = numfields * 32 + 33
        lenrecord = sum(field[1] for field in self.field_spec) + 1
        hdr = struct.pack('<BBBBLHH20x', ver, yr, mon, day, numrec, lenheader, lenrecord)
        self.f.write(hdr)
        # field specs
        for name, (typ, size, deci) in itertools.izip(self.header, self.field_spec):
            name = name.ljust(11, '\x00')
            fld = struct.pack('<11sc4xBB14x', name, typ, size, deci)
            self.f.write(fld)
        # terminator
        self.f.write('\r')
        if self.f.tell() != POS and not self.FIRST_WRITE:
            self.f.seek(POS)

if __name__ == '__main__':
    file_name = "../../examples/10740.dbf"
    f=pysal.open(file_name,'r')
    newDB = pysal.open('copy.dbf','w')
    newDB.header = f.header
    newDB.field_spec = f.field_spec
    print f.header
    for row in f:
        print row
        newDB.write(row)
    newDB.close()
    copy = pysal.open('copy.dbf','r')
    f.seek(0)
    print "HEADER: ", copy.header == f.header
    print "SPEC: ", copy.field_spec == f.field_spec
    print "DATA: ",list(copy) == list(f)
<|endoftext|>


<commit_before>"""
PySAL ShapeFile Reader and Writer based on pure python shapefile module.

Authors:
Charles R Schmidt <Charles.R.Schmidt@asu.edu>

Not to be used without permission of the authors.
"""

__author__ = "Charles R. Schmidt"
__credits__ = "Copyright (c) 2009 Charles R. Schmidt"

import pysal
import pysal.core.FileIO as FileIO
from _pyShpIO import shp_file
import pysal.cg as cg
from warnings import warn
import unittest

STRING_TO_TYPE = {'POLYGON':cg.Polygon,'POINT':cg.Point,'POINTM':cg.Point,'POINTZ':cg.Point,'ARC':cg.Chain}
TYPE_TO_STRING = {cg.Polygon:'POLYGON',cg.Point:'POINT',cg.Chain:'ARC'} #build the reverse map
for key,value in STRING_TO_TYPE.iteritems():
    TYPE_TO_STRING[value] = key

class PurePyShpWrapper(pysal.core.FileIO.FileIO):
    """
    FileIO handeler for ESRI ShapeFiles.

    Attributes:
    FORMATS -- list -- A list of support file extensions.
    MODES -- list -- A list of support file modes.

    Notes:
    This class wraps _pyShpIO's shp_file class with the PySAL FileIO API. shp_file can be used without PySAL.

    Example:
    >>> import tempfile
    >>> f = tempfile.NamedTemporaryFile(suffix='.shp',delete=False); fname = f.name; f.close()
    >>> import pysal
    >>> i = pysal.open('../../examples/10740.shp','r')
    >>> o = pysal.open(fname,'w')
    >>> for shp in i:
    ...     o.write(shp)
    >>> o.close()
    >>> open('../../examples/10740.shp','rb').read() == open(fname,'rb').read()
    True
    >>> open('../../examples/10740.shx','rb').read() == open(fname[:-1]+'x','rb').read()
    True
    >>> import os
    >>> os.remove(fname); os.remove(fname.replace('.shp','.shx'))
    """
    FORMATS = ['shp','shx']
    MODES = ['w','r','wb','rb']
    def __init__(self,*args,**kwargs):
        pysal.core.FileIO.FileIO.__init__(self,*args,**kwargs)
        self.dataObj = None
        if self.mode == 'r' or self.mode == 'rb':
            self.__open()
        elif self.mode == 'w' or self.mode == 'wb':
            self.__create()
    def __len__(self):
        if self.dataObj: return len(self.dataObj)
        else: return 0
    def __open(self):
        self.dataObj = shp_file(self.dataPath)
        self.header = self.dataObj.header
        self.bbox = self.dataObj.bbox
        try:
            self.type = STRING_TO_TYPE[self.dataObj.type()]
        except KeyError:
            raise TypeError,'%s does not support shapes of type: %s'\
                        %(self.__class__.__name__,self.dataObj.type())
    def __create(self):
        self.write = self.__firstWrite
    def __firstWrite(self,shape):
        self.type = TYPE_TO_STRING[type(shape)]
        if self.type == 'POINT':
            if len(shape) == 3:
                self.type = 'POINTM'
            if len(shape) == 4:
                self.type = 'POINTZ'
        self.dataObj = shp_file(self.dataPath,'w',self.type)
        self.write = self.__writer
        self.write(shape)
    def __writer(self,shape):
        if TYPE_TO_STRING[type(shape)] != self.type:
            raise TypeError, "This file only supports %s type shapes"%self.type
        rec = {}
        rec['Shape Type'] = shp_file.SHAPE_TYPES[self.type]
        if self.type == 'POINT':
            rec['X'] = shape[0]
            rec['Y'] = shape[1]
            if len(shape) > 2:
                rec['M'] = shape[2]
            if len(shape) > 3:
                rec['Z'] = shape[3]
            shape = rec
        else:
            rec['BBOX Xmin'] = shape.bounding_box.left
            rec['BBOX Ymin'] = shape.bounding_box.lower
            rec['BBOX Xmax'] = shape.bounding_box.right
            rec['BBOX Ymax'] = shape.bounding_box.upper
            if self.type == 'POLYGON':
                holes = [hole[::-1] for hole in shape.holes if hole] #holes should be in CCW order
                rec['NumParts'] = len(shape.parts) + len(holes)
                all_parts = shape.parts+holes
            else:
                rec['NumParts'] = len(shape.parts)
                all_parts = shape.parts
            partsIndex = [0]
            for l in map(len,all_parts)[:-1]:
                partsIndex.append(partsIndex[-1]+l)
            rec['Parts Index'] = partsIndex
            verts = sum(all_parts,[])
            verts = [(x,y) for x,y in verts]
            rec['NumPoints'] = len(verts)
            rec['Vertices'] = verts
        self.dataObj.add_shape(rec)
        self.pos+=1
    def _read(self):
        try:
            rec = self.dataObj.get_shape(self.pos)
        except IndexError:
            return None
        self.pos+=1
        if self.dataObj.type() == 'POINT':
            shp = self.type((rec['X'],rec['Y']))
            if self.ids:
                shp.id = self.rIds[self.pos-1] # shp IDs start at 1.
            else:
                shp.id = self.pos # shp IDs start at 1.
            return shp
        else:
            if rec['NumParts'] > 1:
                partsIndex = list(rec['Parts Index'])
                partsIndex.append(None)
                parts = [rec['Vertices'][partsIndex[i]:partsIndex[i+1]] for i in xrange(rec['NumParts'])]
                if self.dataObj.type() == 'POLYGON':
                    is_cw =  map(pysal.cg.is_clockwise,parts)
                    vertices = [part for part,cw in zip(parts,is_cw) if cw]
                    holes = [part for part,cw in zip(parts,is_cw) if not cw]
                    if not holes:
                        holes = None
                    return self.type(vertices,holes)
                else:
                    vertices = parts
            else:
                vertices = rec['Vertices']
                if not pysal.cg.is_clockwise(vertices):
                    ### SHAPEFILE WARNING: Polygon %d topology has been fixed. (ccw -> cw)
                    warn("SHAPEFILE WARNING: Polygon %d topology has been fixed. (ccw -> cw)"%(self.pos),RuntimeWarning)
                    print "SHAPEFILE WARNING: Polygon %d topology has been fixed. (ccw -> cw)"%(self.pos)

            shp = self.type(vertices)
            if self.ids:
                shp.id = self.rIds[self.pos-1] # shp IDs start at 1.
            else:
                shp.id = self.pos # shp IDs start at 1.
            return shp
            return shp

    def close(self):
        self.dataObj.close()
        FileIO.FileIO.close(self)

def _test():
    import doctest
    doctest.testmod(verbose=True)
    unittest.main()

if __name__=='__main__':
    _test()
<commit_msg>fixing issue #39, problem cause by the multi-part polygons, they were returning before the id property was set.  Since id's are NOT part of the Polygon API, this should be removed in a future version.  weights should manage it's own ids.<commit_after>"""
PySAL ShapeFile Reader and Writer based on pure python shapefile module.

Authors:
Charles R Schmidt <Charles.R.Schmidt@asu.edu>

Not to be used without permission of the authors.
"""

__author__ = "Charles R. Schmidt"
__credits__ = "Copyright (c) 2009 Charles R. Schmidt"

import pysal
import pysal.core.FileIO as FileIO
from _pyShpIO import shp_file
import pysal.cg as cg
from warnings import warn
import unittest

STRING_TO_TYPE = {'POLYGON':cg.Polygon,'POINT':cg.Point,'POINTM':cg.Point,'POINTZ':cg.Point,'ARC':cg.Chain}
TYPE_TO_STRING = {cg.Polygon:'POLYGON',cg.Point:'POINT',cg.Chain:'ARC'} #build the reverse map
for key,value in STRING_TO_TYPE.iteritems():
    TYPE_TO_STRING[value] = key

class PurePyShpWrapper(pysal.core.FileIO.FileIO):
    """
    FileIO handeler for ESRI ShapeFiles.

    Attributes:
    FORMATS -- list -- A list of support file extensions.
    MODES -- list -- A list of support file modes.

    Notes:
    This class wraps _pyShpIO's shp_file class with the PySAL FileIO API. shp_file can be used without PySAL.

    Example:
    >>> import tempfile
    >>> f = tempfile.NamedTemporaryFile(suffix='.shp',delete=False); fname = f.name; f.close()
    >>> import pysal
    >>> i = pysal.open('../../examples/10740.shp','r')
    >>> o = pysal.open(fname,'w')
    >>> for shp in i:
    ...     o.write(shp)
    >>> o.close()
    >>> open('../../examples/10740.shp','rb').read() == open(fname,'rb').read()
    True
    >>> open('../../examples/10740.shx','rb').read() == open(fname[:-1]+'x','rb').read()
    True
    >>> import os
    >>> os.remove(fname); os.remove(fname.replace('.shp','.shx'))
    """
    FORMATS = ['shp','shx']
    MODES = ['w','r','wb','rb']
    def __init__(self,*args,**kwargs):
        pysal.core.FileIO.FileIO.__init__(self,*args,**kwargs)
        self.dataObj = None
        if self.mode == 'r' or self.mode == 'rb':
            self.__open()
        elif self.mode == 'w' or self.mode == 'wb':
            self.__create()
    def __len__(self):
        if self.dataObj: return len(self.dataObj)
        else: return 0
    def __open(self):
        self.dataObj = shp_file(self.dataPath)
        self.header = self.dataObj.header
        self.bbox = self.dataObj.bbox
        try:
            self.type = STRING_TO_TYPE[self.dataObj.type()]
        except KeyError:
            raise TypeError,'%s does not support shapes of type: %s'\
                        %(self.__class__.__name__,self.dataObj.type())
    def __create(self):
        self.write = self.__firstWrite
    def __firstWrite(self,shape):
        self.type = TYPE_TO_STRING[type(shape)]
        if self.type == 'POINT':
            if len(shape) == 3:
                self.type = 'POINTM'
            if len(shape) == 4:
                self.type = 'POINTZ'
        self.dataObj = shp_file(self.dataPath,'w',self.type)
        self.write = self.__writer
        self.write(shape)
    def __writer(self,shape):
        if TYPE_TO_STRING[type(shape)] != self.type:
            raise TypeError, "This file only supports %s type shapes"%self.type
        rec = {}
        rec['Shape Type'] = shp_file.SHAPE_TYPES[self.type]
        if self.type == 'POINT':
            rec['X'] = shape[0]
            rec['Y'] = shape[1]
            if len(shape) > 2:
                rec['M'] = shape[2]
            if len(shape) > 3:
                rec['Z'] = shape[3]
            shape = rec
        else:
            rec['BBOX Xmin'] = shape.bounding_box.left
            rec['BBOX Ymin'] = shape.bounding_box.lower
            rec['BBOX Xmax'] = shape.bounding_box.right
            rec['BBOX Ymax'] = shape.bounding_box.upper
            if self.type == 'POLYGON':
                holes = [hole[::-1] for hole in shape.holes if hole] #holes should be in CCW order
                rec['NumParts'] = len(shape.parts) + len(holes)
                all_parts = shape.parts+holes
            else:
                rec['NumParts'] = len(shape.parts)
                all_parts = shape.parts
            partsIndex = [0]
            for l in map(len,all_parts)[:-1]:
                partsIndex.append(partsIndex[-1]+l)
            rec['Parts Index'] = partsIndex
            verts = sum(all_parts,[])
            verts = [(x,y) for x,y in verts]
            rec['NumPoints'] = len(verts)
            rec['Vertices'] = verts
        self.dataObj.add_shape(rec)
        self.pos+=1
    def _read(self):
        try:
            rec = self.dataObj.get_shape(self.pos)
        except IndexError:
            return None
        self.pos+=1
        if self.dataObj.type() == 'POINT':
            shp = self.type((rec['X'],rec['Y']))
        else:
            if rec['NumParts'] > 1:
                partsIndex = list(rec['Parts Index'])
                partsIndex.append(None)
                parts = [rec['Vertices'][partsIndex[i]:partsIndex[i+1]] for i in xrange(rec['NumParts'])]
                if self.dataObj.type() == 'POLYGON':
                    is_cw =  map(pysal.cg.is_clockwise,parts)
                    vertices = [part for part,cw in zip(parts,is_cw) if cw]
                    holes = [part for part,cw in zip(parts,is_cw) if not cw]
                    if not holes:
                        holes = None
                    shp = self.type(vertices,holes)
                else:
                    vertices = parts
                    shp = self.type(vertices)
            else:
                vertices = rec['Vertices']
                if not pysal.cg.is_clockwise(vertices):
                    ### SHAPEFILE WARNING: Polygon %d topology has been fixed. (ccw -> cw)
                    warn("SHAPEFILE WARNING: Polygon %d topology has been fixed. (ccw -> cw)"%(self.pos),RuntimeWarning)
                    print "SHAPEFILE WARNING: Polygon %d topology has been fixed. (ccw -> cw)"%(self.pos)

                shp = self.type(vertices)
        if self.ids:
            shp.id = self.rIds[self.pos-1] # shp IDs start at 1.
        else:
            shp.id = self.pos # shp IDs start at 1.
        return shp

    def close(self):
        self.dataObj.close()
        FileIO.FileIO.close(self)

def _test():
    import doctest
    doctest.testmod(verbose=True)
    unittest.main()

if __name__=='__main__':
    _test()
<|endoftext|>