<commit_before>import datetime
import docker.errors
from django.conf import settings
from django.db import models
from django.contrib.postgres.fields import ArrayField, JSONField
from django.core.files.base import ContentFile
from django.core.files.storage import default_storage
from django.urls import reverse
import os
import requests
from ..scraper.query import query_single_paper
from .processor import process_render
from .renderer import render_paper, create_client

class DownloadError(Exception):
    """Error downloading a paper."""

class RenderError(Exception):
    pass

class PaperIsNotRenderableError(RenderError):
    """Paper cannot be rendered."""

class RenderAlreadyStartedError(RenderError):
    """Render started when render has already been started."""

class RenderWrongStateError(RenderError):
    """The state of a render is being updated when it has not been started."""

def guess_extension_from_headers(h):
    """
    Given headers from an ArXiV e-print response, try and guess what the file
    extension should be.

    Based on: https://arxiv.org/help/mimetypes
    """
    if h.get('content-type') == 'application/pdf':
        return '.pdf'
    if h.get('content-encoding') == 'x-gzip' and h.get('content-type') == 'application/postscript':
        return '.ps.gz'
    if h.get('content-encoding') == 'x-gzip' and h.get('content-type') == 'application/x-eprint-tar':
        return '.tar.gz'
    # content-encoding is x-gzip but this appears to normally be a lie - it's
    # just plain text
    if h.get('content-type') == 'application/x-eprint':
        return '.tex'
    if h.get('content-encoding') == 'x-gzip' and h.get('content-type') == 'application/x-dvi':
        return '.dvi.gz'
    return None

class PaperQuerySet(models.QuerySet):
    def _with_has_successful_render_annotation(self):
        renders = Render.objects.filter(paper=models.OuterRef('pk'),
                                        state=Render.STATE_SUCCESS)
        return self.annotate(has_successful_render=models.Exists(renders))

    def has_successful_render(self):
        qs = self._with_has_successful_render_annotation()
        return qs.filter(has_successful_render=True)

    def has_no_successful_render(self):
        qs = self._with_has_successful_render_annotation()
        return qs.filter(has_successful_render=False)

    def _with_has_not_expired_render_annotation(self):
        expired_delta = datetime.timedelta(days=settings.PAPERS_EXPIRED_DAYS)
        expired_date = datetime.datetime.now() - expired_delta
        renders = Render.objects.filter(paper=models.OuterRef('pk'),
                                        created_at__gte=expired_date)
        return self.annotate(has_not_expired_render=models.Exists(renders))

    def has_not_expired_render(self):
        qs = self._with_has_not_expired_render_annotation()
        return qs.filter(has_not_expired_render=True)

    def downloaded(self):
        return self.filter(source_file__isnull=False)

    def not_downloaded(self):
        return self.filter(source_file__isnull=True)

    def update_or_create_from_api(self, result):
        return self.update_or_create(arxiv_id=result['arxiv_id'],
                                     defaults=result)

    def update_or_create_from_arxiv_id(self, arxiv_id):
        """
        Query the Arxiv API and create a Paper from it.

        Raises:
            `arxiv_vanity.scraper.query.PaperNotFoundError`: If paper does not exist on arxiv.
        """
        return self.update_or_create_from_api(query_single_paper(arxiv_id))

    def machine_learning(self):
        """
        Return only machine learning papers.
        """
        return self.filter(categories__overlap=settings.PAPERS_MACHINE_LEARNING_CATEGORIES)

class Paper(models.Model):
    # ArXiV fields
    arxiv_id = models.CharField(max_length=50, unique=True)
    title = models.TextField()
    published = models.DateTimeField()
    updated = models.DateTimeField()
    summary = models.TextField()
    authors = JSONField()
    arxiv_url = models.URLField()
    pdf_url = models.URLField()
    primary_category = models.CharField(max_length=100)
    categories = ArrayField(models.CharField(max_length=100))
    comment = models.TextField(null=True, blank=True)
    doi = models.CharField(null=True, blank=True, max_length=100)
    journal_ref = models.TextField(null=True, blank=True, max_length=100)

    # ASS fields
    source_file = models.FileField(upload_to='paper-sources/', null=True, blank=True)

    objects = PaperQuerySet.as_manager()

    class Meta:
        get_latest_by = 'updated'
        ordering = ['-updated']

    def __str__(self):
        return self.title

    def get_absolute_url(self):
        return reverse('paper_detail', args=(self.arxiv_id,))

    def get_source_url(self):
        # This URL is normally a tarball, but sometimes something else.
        # ArXiV provides a /src/ URL which always serves up a tarball,
        # but if we used this, we'd have to untar the file to figure out
        # whether it's renderable or not. By using the /e-print/ endpoint
        # we can figure out straight away whether we should bother rendering
        # it or not.
        # https://arxiv.org/help/mimetypes has more info
        return 'https://arxiv.org/e-print/' + self.arxiv_id

    def get_https_arxiv_url(self):
        url = self.arxiv_url
        if url.startswith('http://arxiv.org'):
            url = 'https' + url.lstrip('http')
        return url

    def get_https_pdf_url(self):
        url = self.pdf_url
        if url.startswith('http://arxiv.org'):
            url = 'https' + url.lstrip('http')
        return url

    def download(self):
        """
        Download the LaTeX source of this paper and save to storage. It will
        save the model.
        """
        # Delete an existing file if it exists so we don't clutter up storage
        # with dupes. This might mean we lose a file the download fails,
        # but we can just download it again.
        if self.source_file.name:
            self.source_file.delete()

        res = requests.get(self.get_source_url())
        res.raise_for_status()
        extension = guess_extension_from_headers(res.headers)
        if not extension:
            raise DownloadError("Could not determine file extension from "
                                "headers: Content-Type: {}; "
                                "Content-Encoding: {}".format(
                                    res.headers.get('content-type'),
                                    res.headers.get('content-encoding')))
        content = ContentFile(res.content)
        self.source_file.save(self.arxiv_id + extension, content)

    def is_renderable(self):
        """
        Returns whether it is possible to render this paper.
        """
        return self.source_file.name is not None and self.source_file.name.endswith('.tar.gz')

    def render(self):
        """
        Make a new render of this paper. Will download the source file and save
        itself if it hasn't already.
        """
        if not self.source_file.name:
            self.download()
        if not self.is_renderable():
            raise PaperIsNotRenderableError("This paper is not renderable.")
        render = Render.objects.create(paper=self)
        render.run()
        return render

class RenderQuerySet(models.QuerySet):
    def running(self):
        return self.filter(state=Render.STATE_RUNNING)

    def succeeded(self):
        return self.filter(state=Render.STATE_SUCCESS)

    def failed(self):
        return self.filter(state=Render.STATE_FAILURE)

    def not_expired(self):
        """
        Renders which have occurred in the last PAPERS_EXPIRED_DAYS days.
        """
        expired_delta = datetime.timedelta(days=settings.PAPERS_EXPIRED_DAYS)
        expired_date = datetime.datetime.now() - expired_delta
        return self.filter(created_at__gte=expired_date)

    def update_state(self):
        """
        Update the state of renders that have a container.
        """
        for render in self.exclude(state=Render.STATE_UNSTARTED).filter(container_is_removed=False):
            try:
                render.update_state()
            except docker.errors.NotFound:
                # TODO: logging
                print(f"Could not update render {render.id}: Container ID {render.container_id} does not exist")
        return self

class Render(models.Model):
    STATE_UNSTARTED = 'unstarted'
    STATE_RUNNING = 'running'
    STATE_SUCCESS = 'success'
    STATE_FAILURE = 'failure'

    paper = models.ForeignKey(Paper, on_delete=models.CASCADE, related_name='renders')
    created_at = models.DateTimeField(auto_now_add=True)
    state = models.CharField(max_length=20, default=STATE_UNSTARTED, choices=(
        (STATE_UNSTARTED, 'Unstarted'),
        (STATE_RUNNING, 'Running'),
        (STATE_SUCCESS, 'Success'),
        (STATE_FAILURE, 'Failure'),
    ))
    container_id = models.CharField(max_length=64, null=True, blank=True)
    container_inspect = JSONField(null=True, blank=True)
    container_logs = models.TextField(null=True, blank=True)
    container_is_removed = models.BooleanField(default=False)

    objects = RenderQuerySet.as_manager()

    class Meta:
        get_latest_by = 'created_at'

    def __str__(self):
        return self.paper.title

    def save(self, *args, **kwargs):
        super(Render, self).save(*args, **kwargs)

    def get_output_path(self):
        """
        Path to the directory that this render is in.
        """
        return os.path.join('render-output', str(self.id))

    def get_html_path(self):
        """
        Path to the HTML file of this render.
        """
        return os.path.join(self.get_output_path(), "index.html")

    def get_output_url(self):
        """
        Returns the URL to the output path.
        """
        return settings.MEDIA_URL + self.get_output_path()

    def short_container_id(self):
        if self.container_id:
            return self.container_id[:12]

    def get_webhook_url(self):
        """
        Get the webhook to call when the Engrafo job ends to update render
        state.
        """
        path = reverse('render_update_state', args=(self.id,))
        return settings.ENGRAFO_WEBHOOK_URL_PREFIX + path

    def run(self):
        """
        Start running this render.
        """
        if self.state != Render.STATE_UNSTARTED:
            raise RenderAlreadyStartedError(f"Render {self.id} has already been started")
        self.container_id = render_paper(
            self.paper.source_file.name,
            self.get_output_path(),
            webhook_url=self.get_webhook_url()
        )
        self.state = Render.STATE_RUNNING
        self.save()

    def update_state(self, exit_code=None):
        """
        Update state of this render from the container.

        This is used for two purposes:

        1. Called by the webhook to update the state. In this case, exit_code
            is passed through instead of using the containers exit code
            because the container is still running (it's sending the webhook!).

        2. Called by the update_render_state cron job. In this case, it'll
            sync all of the details about the containers that exist, and remove
            any containers that have stopped. This sweeps up containers
            that have reported their state in (1).
        """
        if self.state == Render.STATE_UNSTARTED:
            raise RenderWrongStateError(f"Render {self.id} has not been started")
        client = create_client()

        try:
            container = client.containers.get(self.container_id)
            self.container_inspect = container.attrs
            self.container_logs = container.logs()
        except docker.errors.NotFound:
            # Container has been removed for some reason, so mark it as
            # removed so we don't try to update its state again
            self.container_is_removed = True
            self.save()
            return

        if exit_code is None and container.status == 'exited':
            exit_code = container.attrs['State']['ExitCode']

        if exit_code is not None:
            # Safer to convert int to str than other way round
            if str(exit_code) == '0':
                self.state = Render.STATE_SUCCESS
            else:
                self.state = Render.STATE_FAILURE

        if container.status == 'exited':
            try:
                container.remove()
            except docker.errors.NotFound:
                # Somebody got in there before us. Oh well.
                pass
            self.container_is_removed = True

        self.save()

    def get_processed_render(self):
        """
        Do final processing on this render and returns it as a dictionary of
        {"body", "script", "styles"}.
        """
        context = {
            'render': self,
            'paper': self.paper,
        }
        with default_storage.open(self.get_html_path()) as fh:
            return process_render(fh, self.get_output_url(), context=context)
<commit_msg>Update name<commit_after>import datetime
import docker.errors
from django.conf import settings
from django.db import models
from django.contrib.postgres.fields import ArrayField, JSONField
from django.core.files.base import ContentFile
from django.core.files.storage import default_storage
from django.urls import reverse
import os
import requests
from ..scraper.query import query_single_paper
from .processor import process_render
from .renderer import render_paper, create_client

class DownloadError(Exception):
    """Error downloading a paper."""

class RenderError(Exception):
    pass

class PaperIsNotRenderableError(RenderError):
    """Paper cannot be rendered."""

class RenderAlreadyStartedError(RenderError):
    """Render started when render has already been started."""

class RenderWrongStateError(RenderError):
    """The state of a render is being updated when it has not been started."""

def guess_extension_from_headers(h):
    """
    Given headers from an ArXiV e-print response, try and guess what the file
    extension should be.

    Based on: https://arxiv.org/help/mimetypes
    """
    if h.get('content-type') == 'application/pdf':
        return '.pdf'
    if h.get('content-encoding') == 'x-gzip' and h.get('content-type') == 'application/postscript':
        return '.ps.gz'
    if h.get('content-encoding') == 'x-gzip' and h.get('content-type') == 'application/x-eprint-tar':
        return '.tar.gz'
    # content-encoding is x-gzip but this appears to normally be a lie - it's
    # just plain text
    if h.get('content-type') == 'application/x-eprint':
        return '.tex'
    if h.get('content-encoding') == 'x-gzip' and h.get('content-type') == 'application/x-dvi':
        return '.dvi.gz'
    return None

class PaperQuerySet(models.QuerySet):
    def _with_has_successful_render_annotation(self):
        renders = Render.objects.filter(paper=models.OuterRef('pk'),
                                        state=Render.STATE_SUCCESS)
        return self.annotate(has_successful_render=models.Exists(renders))

    def has_successful_render(self):
        qs = self._with_has_successful_render_annotation()
        return qs.filter(has_successful_render=True)

    def has_no_successful_render(self):
        qs = self._with_has_successful_render_annotation()
        return qs.filter(has_successful_render=False)

    def _with_has_not_expired_render_annotation(self):
        expired_delta = datetime.timedelta(days=settings.PAPERS_EXPIRED_DAYS)
        expired_date = datetime.datetime.now() - expired_delta
        renders = Render.objects.filter(paper=models.OuterRef('pk'),
                                        created_at__gte=expired_date)
        return self.annotate(has_not_expired_render=models.Exists(renders))

    def has_not_expired_render(self):
        qs = self._with_has_not_expired_render_annotation()
        return qs.filter(has_not_expired_render=True)

    def downloaded(self):
        return self.filter(source_file__isnull=False)

    def not_downloaded(self):
        return self.filter(source_file__isnull=True)

    def update_or_create_from_api(self, result):
        return self.update_or_create(arxiv_id=result['arxiv_id'],
                                     defaults=result)

    def update_or_create_from_arxiv_id(self, arxiv_id):
        """
        Query the Arxiv API and create a Paper from it.

        Raises:
            `arxiv_vanity.scraper.query.PaperNotFoundError`: If paper does not exist on arxiv.
        """
        return self.update_or_create_from_api(query_single_paper(arxiv_id))

    def machine_learning(self):
        """
        Return only machine learning papers.
        """
        return self.filter(categories__overlap=settings.PAPERS_MACHINE_LEARNING_CATEGORIES)

class Paper(models.Model):
    # ArXiV fields
    arxiv_id = models.CharField(max_length=50, unique=True)
    title = models.TextField()
    published = models.DateTimeField()
    updated = models.DateTimeField()
    summary = models.TextField()
    authors = JSONField()
    arxiv_url = models.URLField()
    pdf_url = models.URLField()
    primary_category = models.CharField(max_length=100)
    categories = ArrayField(models.CharField(max_length=100))
    comment = models.TextField(null=True, blank=True)
    doi = models.CharField(null=True, blank=True, max_length=100)
    journal_ref = models.TextField(null=True, blank=True, max_length=100)

    # Arxiv Vanity fields
    source_file = models.FileField(upload_to='paper-sources/', null=True, blank=True)

    objects = PaperQuerySet.as_manager()

    class Meta:
        get_latest_by = 'updated'
        ordering = ['-updated']

    def __str__(self):
        return self.title

    def get_absolute_url(self):
        return reverse('paper_detail', args=(self.arxiv_id,))

    def get_source_url(self):
        # This URL is normally a tarball, but sometimes something else.
        # ArXiV provides a /src/ URL which always serves up a tarball,
        # but if we used this, we'd have to untar the file to figure out
        # whether it's renderable or not. By using the /e-print/ endpoint
        # we can figure out straight away whether we should bother rendering
        # it or not.
        # https://arxiv.org/help/mimetypes has more info
        return 'https://arxiv.org/e-print/' + self.arxiv_id

    def get_https_arxiv_url(self):
        url = self.arxiv_url
        if url.startswith('http://arxiv.org'):
            url = 'https' + url.lstrip('http')
        return url

    def get_https_pdf_url(self):
        url = self.pdf_url
        if url.startswith('http://arxiv.org'):
            url = 'https' + url.lstrip('http')
        return url

    def download(self):
        """
        Download the LaTeX source of this paper and save to storage. It will
        save the model.
        """
        # Delete an existing file if it exists so we don't clutter up storage
        # with dupes. This might mean we lose a file the download fails,
        # but we can just download it again.
        if self.source_file.name:
            self.source_file.delete()

        res = requests.get(self.get_source_url())
        res.raise_for_status()
        extension = guess_extension_from_headers(res.headers)
        if not extension:
            raise DownloadError("Could not determine file extension from "
                                "headers: Content-Type: {}; "
                                "Content-Encoding: {}".format(
                                    res.headers.get('content-type'),
                                    res.headers.get('content-encoding')))
        content = ContentFile(res.content)
        self.source_file.save(self.arxiv_id + extension, content)

    def is_renderable(self):
        """
        Returns whether it is possible to render this paper.
        """
        return self.source_file.name is not None and self.source_file.name.endswith('.tar.gz')

    def render(self):
        """
        Make a new render of this paper. Will download the source file and save
        itself if it hasn't already.
        """
        if not self.source_file.name:
            self.download()
        if not self.is_renderable():
            raise PaperIsNotRenderableError("This paper is not renderable.")
        render = Render.objects.create(paper=self)
        render.run()
        return render

class RenderQuerySet(models.QuerySet):
    def running(self):
        return self.filter(state=Render.STATE_RUNNING)

    def succeeded(self):
        return self.filter(state=Render.STATE_SUCCESS)

    def failed(self):
        return self.filter(state=Render.STATE_FAILURE)

    def not_expired(self):
        """
        Renders which have occurred in the last PAPERS_EXPIRED_DAYS days.
        """
        expired_delta = datetime.timedelta(days=settings.PAPERS_EXPIRED_DAYS)
        expired_date = datetime.datetime.now() - expired_delta
        return self.filter(created_at__gte=expired_date)

    def update_state(self):
        """
        Update the state of renders that have a container.
        """
        for render in self.exclude(state=Render.STATE_UNSTARTED).filter(container_is_removed=False):
            try:
                render.update_state()
            except docker.errors.NotFound:
                # TODO: logging
                print(f"Could not update render {render.id}: Container ID {render.container_id} does not exist")
        return self

class Render(models.Model):
    STATE_UNSTARTED = 'unstarted'
    STATE_RUNNING = 'running'
    STATE_SUCCESS = 'success'
    STATE_FAILURE = 'failure'

    paper = models.ForeignKey(Paper, on_delete=models.CASCADE, related_name='renders')
    created_at = models.DateTimeField(auto_now_add=True)
    state = models.CharField(max_length=20, default=STATE_UNSTARTED, choices=(
        (STATE_UNSTARTED, 'Unstarted'),
        (STATE_RUNNING, 'Running'),
        (STATE_SUCCESS, 'Success'),
        (STATE_FAILURE, 'Failure'),
    ))
    container_id = models.CharField(max_length=64, null=True, blank=True)
    container_inspect = JSONField(null=True, blank=True)
    container_logs = models.TextField(null=True, blank=True)
    container_is_removed = models.BooleanField(default=False)

    objects = RenderQuerySet.as_manager()

    class Meta:
        get_latest_by = 'created_at'

    def __str__(self):
        return self.paper.title

    def save(self, *args, **kwargs):
        super(Render, self).save(*args, **kwargs)

    def get_output_path(self):
        """
        Path to the directory that this render is in.
        """
        return os.path.join('render-output', str(self.id))

    def get_html_path(self):
        """
        Path to the HTML file of this render.
        """
        return os.path.join(self.get_output_path(), "index.html")

    def get_output_url(self):
        """
        Returns the URL to the output path.
        """
        return settings.MEDIA_URL + self.get_output_path()

    def short_container_id(self):
        if self.container_id:
            return self.container_id[:12]

    def get_webhook_url(self):
        """
        Get the webhook to call when the Engrafo job ends to update render
        state.
        """
        path = reverse('render_update_state', args=(self.id,))
        return settings.ENGRAFO_WEBHOOK_URL_PREFIX + path

    def run(self):
        """
        Start running this render.
        """
        if self.state != Render.STATE_UNSTARTED:
            raise RenderAlreadyStartedError(f"Render {self.id} has already been started")
        self.container_id = render_paper(
            self.paper.source_file.name,
            self.get_output_path(),
            webhook_url=self.get_webhook_url()
        )
        self.state = Render.STATE_RUNNING
        self.save()

    def update_state(self, exit_code=None):
        """
        Update state of this render from the container.

        This is used for two purposes:

        1. Called by the webhook to update the state. In this case, exit_code
            is passed through instead of using the containers exit code
            because the container is still running (it's sending the webhook!).

        2. Called by the update_render_state cron job. In this case, it'll
            sync all of the details about the containers that exist, and remove
            any containers that have stopped. This sweeps up containers
            that have reported their state in (1).
        """
        if self.state == Render.STATE_UNSTARTED:
            raise RenderWrongStateError(f"Render {self.id} has not been started")
        client = create_client()

        try:
            container = client.containers.get(self.container_id)
            self.container_inspect = container.attrs
            self.container_logs = container.logs()
        except docker.errors.NotFound:
            # Container has been removed for some reason, so mark it as
            # removed so we don't try to update its state again
            self.container_is_removed = True
            self.save()
            return

        if exit_code is None and container.status == 'exited':
            exit_code = container.attrs['State']['ExitCode']

        if exit_code is not None:
            # Safer to convert int to str than other way round
            if str(exit_code) == '0':
                self.state = Render.STATE_SUCCESS
            else:
                self.state = Render.STATE_FAILURE

        if container.status == 'exited':
            try:
                container.remove()
            except docker.errors.NotFound:
                # Somebody got in there before us. Oh well.
                pass
            self.container_is_removed = True

        self.save()

    def get_processed_render(self):
        """
        Do final processing on this render and returns it as a dictionary of
        {"body", "script", "styles"}.
        """
        context = {
            'render': self,
            'paper': self.paper,
        }
        with default_storage.open(self.get_html_path()) as fh:
            return process_render(fh, self.get_output_url(), context=context)
<|endoftext|>


<commit_before>import pandas as pd
import geopandas as gpd
from shared import compute_area
import time

# this file reads the split_parcels.csv and the
# buildings_linked_to_parcels.csv and splits up the attribuets
# from parcels to buildings.  Since it also removes attributes
# from the parcel tables it writes out both moved_attribute_parcels.csv
# and moved_attribute_buildings.csv

parcels = gpd.read_geocsv("cache/split_parcels.csv", index_col="apn")
buildings_linked_to_parcels = gpd.read_geocsv(
    "cache/buildings_linked_to_parcels.csv", low_memory=False,
    index_col="building_id")

# this file contains mapping of blocks to mazs to tazs, but we want
# the maz to taz mapping
maz_to_taz = pd.read_csv("GeogXWalk2010_Blocks_MAZ_TAZ.csv").\
    drop_duplicates(subset=["MAZ_ORIGINAL"]).\
    set_index("MAZ_ORIGINAL").TAZ_ORIGINAL

parcels["taz_id"] = parcels.maz_id.map(maz_to_taz)

buildings_linked_to_parcels['building:levels'] = pd.to_numeric(
    buildings_linked_to_parcels['building:levels'], errors='coerce')

def drop_parcel_attributes(parcels):
    # used in two place so make a function
    return parcels[['county_id', 'geometry', 'maz_id', 'taz_id', 'orig_apn']]

def assign_parcel_attributes_to_buildings(buildings, parcels):
    # attributes of the first parcel get applied to the buildings - if
    # we did this right, the attributes of all subparcels will be the
    # same - we pass in the parcels in order to set the schema for all
    # the parcels.  test this a little bit:
    for col in ['bldg_sqft', 'res_units', 'nres_sqft']:
        assert len(parcels) == 1 or \
            parcels[col].fillna(0).describe()['std'] == 0
    # take attributes of the first parcel
    parcel = parcels.iloc[0]

    # drop address and amenity - they're great columns but infrequently used
    buildings = buildings[[
        'name', 'geometry', 'apn', 'building:levels', 'building']]
    buildings = buildings.rename(columns={
        'building:levels': 'stories',
        'building': 'osm_building_type'})
    buildings['calc_area'] = compute_area(buildings).round()

    # we call a building a shed if it's less than 50 meters large and it
    # doesn't get any of the parcel data
    small_buildings = buildings[buildings.calc_area < 80].copy()
    small_buildings["small_building"] = True
    large_buildings = buildings[buildings.calc_area >= 80].copy()
    large_buildings["small_building"] = False

    large_buildings["stories"] = large_buildings.stories.fillna(
        parcel.stories).fillna(1)
    large_buildings["year_built"] = parcel.year_built
    large_buildings["building_type"] = parcel.dev_type

    # account for height
    built_area = large_buildings.calc_area * \
        large_buildings.stories.astype('float')
    # get built area proportion in each building footprint
    proportion_built_area = built_area / built_area.sum()

    large_buildings["building_sqft"] = \
        (proportion_built_area * parcel.bldg_sqft).round()
    large_buildings["residential_units"] = \
        (proportion_built_area * parcel.res_units).round()
    large_buildings["non_residential_sqft"] = \
        (proportion_built_area * parcel.nres_sqft).round()

    return pd.concat([small_buildings, large_buildings]),\
        drop_parcel_attributes(parcels)

def make_dummy_building(parcels):
    # when there's more than one parcel, we put the dummy building on the
    # biggest sub parcel
    parcel = parcels.sort_values(by="calc_area", ascending=False).head(1)

    if parcels.bldg_sqft.fillna(0).sum() == 0 and \
            parcels.res_units.fillna(0).sum() == 0:
        # there's no reason to make a dummy building if the attributes aren't
        # there
        return pd.DataFrame(), drop_parcel_attributes(parcels)

    parcel.crs = {'init': 'epsg:4326'}
    parcel = parcel.to_crs(epsg=3857)  # switch to meters
    circle = parcel.centroid.buffer(15).values[0]  # buffer a circle in meters
    parcel = parcel.to_crs(epsg=4326)  # back to lat-lng
    building = gpd.GeoDataFrame({
        'name': ['Generated from parcel centroid'],
        'geometry': [circle],
        'apn': [parcel.index[0]],
        'building:levels': [1],
        'building': ['yes']
    })
    building.crs = {'init': 'epsg:3857'}
    building = building.to_crs(epsg=4326)
    return assign_parcel_attributes_to_buildings(building, parcels)

new_buildings_list = []
new_parcel_list = []

cnt = 0

grps = parcels.groupby("orig_apn")
total_cnt = len(grps)
# iterate over parcels (not sub-parcels)
for index, shared_apn_parcels in grps:
    # get all buildngs that are on any subparcel of this parcel
    buildings = buildings_linked_to_parcels[
        buildings_linked_to_parcels.apn.isin(shared_apn_parcels.index)]

    if len(buildings) == 0:
        new_buildings, new_parcels = make_dummy_building(shared_apn_parcels)
    else:
        new_buildings, new_parcels = assign_parcel_attributes_to_buildings(
            gpd.GeoDataFrame(buildings), shared_apn_parcels)

    new_buildings_list.append(new_buildings)
    new_parcel_list.append(new_parcels)

    cnt += 1
    if cnt % 250 == 0:
        print "Finished %d of %d" % (cnt, total_cnt)

new_parcels = pd.concat(new_parcel_list)
new_buildings = pd.concat(new_buildings_list)

new_parcels.to_csv("cache/moved_attribute_parcels.csv")
new_buildings.to_csv("cache/moved_attribute_buildings.csv")
<commit_msg>clip stories so it's at least 1<commit_after>import pandas as pd
import geopandas as gpd
from shared import compute_area
import time

# this file reads the split_parcels.csv and the
# buildings_linked_to_parcels.csv and splits up the attribuets
# from parcels to buildings.  Since it also removes attributes
# from the parcel tables it writes out both moved_attribute_parcels.csv
# and moved_attribute_buildings.csv

parcels = gpd.read_geocsv("cache/split_parcels.csv", index_col="apn")
buildings_linked_to_parcels = gpd.read_geocsv(
    "cache/buildings_linked_to_parcels.csv", low_memory=False,
    index_col="building_id")

# this file contains mapping of blocks to mazs to tazs, but we want
# the maz to taz mapping
maz_to_taz = pd.read_csv("GeogXWalk2010_Blocks_MAZ_TAZ.csv").\
    drop_duplicates(subset=["MAZ_ORIGINAL"]).\
    set_index("MAZ_ORIGINAL").TAZ_ORIGINAL

parcels["taz_id"] = parcels.maz_id.map(maz_to_taz)

buildings_linked_to_parcels['building:levels'] = pd.to_numeric(
    buildings_linked_to_parcels['building:levels'], errors='coerce')

def drop_parcel_attributes(parcels):
    # used in two place so make a function
    return parcels[['county_id', 'geometry', 'maz_id', 'taz_id', 'orig_apn']]

def assign_parcel_attributes_to_buildings(buildings, parcels):
    # attributes of the first parcel get applied to the buildings - if
    # we did this right, the attributes of all subparcels will be the
    # same - we pass in the parcels in order to set the schema for all
    # the parcels.  test this a little bit:
    for col in ['bldg_sqft', 'res_units', 'nres_sqft']:
        assert len(parcels) == 1 or \
            parcels[col].fillna(0).describe()['std'] == 0
    # take attributes of the first parcel
    parcel = parcels.iloc[0]

    # drop address and amenity - they're great columns but infrequently used
    buildings = buildings[[
        'name', 'geometry', 'apn', 'building:levels', 'building']]
    buildings = buildings.rename(columns={
        'building:levels': 'stories',
        'building': 'osm_building_type'})
    buildings['calc_area'] = compute_area(buildings).round()

    # we call a building a shed if it's less than 50 meters large and it
    # doesn't get any of the parcel data
    small_buildings = buildings[buildings.calc_area < 80].copy()
    small_buildings["small_building"] = True
    large_buildings = buildings[buildings.calc_area >= 80].copy()
    large_buildings["small_building"] = False

    large_buildings["stories"] = large_buildings.stories.fillna(
        parcel.stories).fillna(1)
    large_buildings["year_built"] = parcel.year_built
    large_buildings["building_type"] = parcel.dev_type

    # account for height
    built_area = large_buildings.calc_area * \
        large_buildings.stories.astype('float').clip(1)
    # get built area proportion in each building footprint
    proportion_built_area = built_area / built_area.sum()

    large_buildings["building_sqft"] = \
        (proportion_built_area * parcel.bldg_sqft).round()
    large_buildings["residential_units"] = \
        (proportion_built_area * parcel.res_units).round()
    large_buildings["non_residential_sqft"] = \
        (proportion_built_area * parcel.nres_sqft).round()

    return pd.concat([small_buildings, large_buildings]),\
        drop_parcel_attributes(parcels)

def make_dummy_building(parcels):
    # when there's more than one parcel, we put the dummy building on the
    # biggest sub parcel
    parcel = parcels.sort_values(by="calc_area", ascending=False).head(1)

    if parcels.bldg_sqft.fillna(0).sum() == 0 and \
            parcels.res_units.fillna(0).sum() == 0:
        # there's no reason to make a dummy building if the attributes aren't
        # there
        return pd.DataFrame(), drop_parcel_attributes(parcels)

    parcel.crs = {'init': 'epsg:4326'}
    parcel = parcel.to_crs(epsg=3857)  # switch to meters
    circle = parcel.centroid.buffer(15).values[0]  # buffer a circle in meters
    parcel = parcel.to_crs(epsg=4326)  # back to lat-lng
    building = gpd.GeoDataFrame({
        'name': ['Generated from parcel centroid'],
        'geometry': [circle],
        'apn': [parcel.index[0]],
        'building:levels': [1],
        'building': ['yes']
    })
    building.crs = {'init': 'epsg:3857'}
    building = building.to_crs(epsg=4326)
    return assign_parcel_attributes_to_buildings(building, parcels)

new_buildings_list = []
new_parcel_list = []

cnt = 0

grps = parcels.groupby("orig_apn")
total_cnt = len(grps)
# iterate over parcels (not sub-parcels)
for index, shared_apn_parcels in grps:
    # get all buildngs that are on any subparcel of this parcel
    buildings = buildings_linked_to_parcels[
        buildings_linked_to_parcels.apn.isin(shared_apn_parcels.index)]

    if len(buildings) == 0:
        new_buildings, new_parcels = make_dummy_building(shared_apn_parcels)
    else:
        new_buildings, new_parcels = assign_parcel_attributes_to_buildings(
            gpd.GeoDataFrame(buildings), shared_apn_parcels)

    new_buildings_list.append(new_buildings)
    new_parcel_list.append(new_parcels)

    cnt += 1
    if cnt % 250 == 0:
        print "Finished %d of %d" % (cnt, total_cnt)

new_parcels = pd.concat(new_parcel_list)
new_buildings = pd.concat(new_buildings_list)
new_buildings.index.name = "building_id"

new_parcels.to_csv("cache/moved_attribute_parcels.csv")
new_buildings.to_csv("cache/moved_attribute_buildings.csv")
<|endoftext|>


<commit_before>from datetime import datetime

from django.template.defaultfilters import timesince
from django.utils.translation import ugettext as _
from django.template import Library
from django.conf import settings
from django.core.paginator import Paginator, EmptyPage, PageNotAnInteger
from django.contrib.auth.models import User

from pybb.models import Topic, Post

from astrobin.gear import *

register = Library()

@register.filter
def startswith(x, y):
    return x.startswith(y)

@register.simple_tag
def current(request, pattern):
    import re
    if re.search(pattern, request.path):
        return 'active'
    return ''

@register.simple_tag
def in_gallery(request):
    import re
    if re.search('/users/[\w.@+-]+/(?:collections.*)?$', request.path):
        return 'visible'
    return 'hidden'

@register.simple_tag
def in_collection(request):
    import re
    if re.search('/users/[\w.@+-]+/collections/(\d+)/$', request.path):
        return 'visible'
    return 'hidden'

@register.inclusion_tag('inclusion_tags/related_images.html')
def related_images(request, object_list, type):
    paginator = Paginator(object_list, 12)

    page = request.GET.get('p')
    try:
        images = paginator.page(page)
    except (TypeError, PageNotAnInteger):
        images = paginator.page(1)
    except EmptyPage:
        images = paginator.page(paginator.num_pages)

    return {
            'request': request,
            'images': images,
            'related_type': type,
           }

@register.filter
def ago(date_time):
    date_time = date_time.replace(tzinfo = None)
    diff = abs(date_time - datetime.today())
    if diff.days <= 0:
        span = timesince(date_time)
        span = span.split(",")[0] # just the most significant digit
        if span == "0 " + _("minutes"):
            return _("seconds ago")
        return _("%s ago") % span
    return datetime.date(date_time)

@register.filter
def string_to_date(date):
    try:
        return datetime.strptime(date, "%Y-%m-%d")
    except:
        return datetime.now()

def image_list(context, object_list, **kwargs):
    alias = kwargs.get('alias', 'gallery')
    nav_ctx = kwargs.get('nav_ctx', 'all')
    nav_ctx_extra = kwargs.get('nav_ctx_extra', None)

    view = kwargs.get('view')
    if view is None and 'view' in context:
      view = context['view']
    if view is None:
      view = context['request'].GET.get('view', 'default')

    return {
        'image_list': object_list,
        'request': context['request'],
        'alias': alias,
        'view': view,
        'nav_ctx': nav_ctx,
        'nav_ctx_extra': nav_ctx_extra,
    }
register.inclusion_tag('inclusion_tags/image_list.html', takes_context=True)(image_list)

@register.inclusion_tag('inclusion_tags/search_image_list.html', takes_context = True)
def search_image_list(context, paginate = True, **kwargs):
    context.update({
        'paginate': paginate,
        'search_domain': context['request'].GET.get('d'),
        'sort': context['request'].GET.get('sort'),
    })

    return context

@register.filter
def seconds_to_hours(value):
    try:
        value = int(value)
    except ValueError:
        return "0"

    return "%.1f" % (int(value) / 3600.0)

@register.filter
def cut_decimals(value, places):
    try:
        value = float(value)
    except ValueError:
        return value

    return '{0:.{1}f}'.format(value, places)

@register.filter
def is_checkbox(value):
    from django.forms.fields import CheckboxInput
    return isinstance(value, CheckboxInput)

@register.inclusion_tag('inclusion_tags/list_gear.html')
def list_gear(label, gear, klass, user):
    return {
        'label': label,
        'gear': gear,
        'klass': klass,
        'user': user,
    }

@register.simple_tag
def gear_complete_class(gear):
    return 'incomplete' if not is_gear_complete(gear.id) else ''

@register.simple_tag
def gear_alias(gear, user):
    default = _("no alias")

    try:
        gear_user_info = GearUserInfo.objects.get(gear = gear, user = user)
        if gear_user_info.alias:
            return gear_user_info.alias
    except GearUserInfo.DoesNotExist:
        pass

    return default

@register.simple_tag
def gear_name_iriencoded(gear):
    from django.template.defaultfilters import iriencode
    name = unicode(gear)
    return iriencode(name)

@register.simple_tag
def gear_owners(gear):
    gear, gear_type = get_correct_gear(gear.id)
    return UserProfile.objects.filter(
        **{UserProfile.GEAR_ATTR_LOOKUP[gear_type]: gear}).count()

@register.simple_tag
def gear_set_images(gear_set):
    images = 0
    for g in gear_set.all():
        images += gear_images(g)

    return images

@register.simple_tag
def gear_set_owners(gear_set):
    owners = 0
    for g in gear_set.all():
        owners += gear_owners(g)

    return owners

@register.simple_tag
def gear_images(gear):
    return Image.by_gear(gear).count()

@register.simple_tag
def gear_type(gear):
    from astrobin.gear import get_correct_gear, TYPES_LOOKUP
    real_gear, gear_type = get_correct_gear(gear.id)

    if real_gear and gear_type and hasattr(real_gear, 'type') and real_gear.type:
        try:
            t = TYPES_LOOKUP[gear_type][real_gear.type][1]
            return t
        except KeyError:
            pass

    return '-'

@register.filter
def show_ads(user):
    from astrobin_apps_donations.templatetags.astrobin_apps_donations_tags import is_donor
    from astrobin_apps_premium.templatetags.astrobin_apps_premium_tags import is_premium, is_lite

    if not settings.ADS_ENABLED:
        return False

    if is_donor(user):
        return False

    if settings.PREMIUM_ENABLED and (is_lite(user) or is_premium(user)):
        return False

    return True;

@register.filter
def valid_subscriptions(user):
    from subscription.models import UserSubscription

    if user.is_anonymous():
        return []

    us = UserSubscription.active_objects.filter(user = user)
    subs = [x.subscription for x in us if x.valid()]
    return subs

@register.filter
def inactive_subscriptions(user):
    from subscription.models import UserSubscription

    if user.is_anonymous():
        return []

    return [x.subscription
            for x
            in UserSubscription.objects.filter(user = user)
            if not x.valid() or not x.active]

@register.filter
def has_valid_subscription(user, subscription_pk):
    from subscription.models import UserSubscription

    if user.is_anonymous():
        return False

    us = UserSubscription.active_objects.filter(
        user = user, subscription__pk = subscription_pk)

    if us.count() == 0:
        return False

    return us[0].valid()

@register.filter
def has_valid_subscription_in_category(user, category):
    from subscription.models import UserSubscription

    if user.is_anonymous():
        return False

    us = UserSubscription.active_objects.filter(
        user = user, subscription__category = category)

    if us.count() == 0:
        return False

    return us[0].valid()

@register.filter
def get_premium_subscription_expiration(user):
    from subscription.models import UserSubscription

    if user.is_anonymous():
        return None

    us = UserSubscription.active_objects.filter(
        user = user,
        subscription__group__name__in = ['astrobin_premium', 'astrobin_lite'])

    if us.count() == 0:
        return None

    return us[0].expires

@register.filter
def has_subscription_by_name(user, name):
    from subscription.models import UserSubscription

    if user.is_anonymous():
        return False

    return UserSubscription.objects.filter(
        user = user, subscription__name = name).count() > 0

@register.filter
def get_subscription_by_name(user, name):
    from subscription.models import UserSubscription

    if user.is_anonymous():
        return None

    return UserSubscription.objects.get(
        user = user, subscription__name = name)

@register.filter
def is_content_moderator(user):
    if not user.is_authenticated():
        return False

    return user.groups.filter(name='content_moderators').count() > 0

@register.filter
def is_image_moderator(user):
    if not user.is_authenticated():
        return False

    return user.groups.filter(name='image_moderators').count() > 0

@register.filter
def to_user_timezone(value, user):
    from astrobin.utils import to_user_timezone as tut
    if user.is_authenticated():
        return tut(value, user.userprofile)
    return value

@register.filter
def can_like(user, image):
    from astrobin_apps_premium.templatetags.astrobin_apps_premium_tags import is_free

    user_scores_index = 0
    min_index_to_like = settings.MIN_INDEX_TO_LIKE

    if user.is_authenticated():
        user_scores_index = user.userprofile.get_scores()['user_scores_index']

    if user.is_superuser:
        return True

    if user == image.user:
        return False

    if is_free(user) and user_scores_index < min_index_to_like:
        return False

    return True
<commit_msg>(quality) Remove one more unnecessary semicolon<commit_after>from datetime import datetime

from django.template.defaultfilters import timesince
from django.utils.translation import ugettext as _
from django.template import Library
from django.conf import settings
from django.core.paginator import Paginator, EmptyPage, PageNotAnInteger
from django.contrib.auth.models import User

from pybb.models import Topic, Post

from astrobin.gear import *

register = Library()

@register.filter
def startswith(x, y):
    return x.startswith(y)

@register.simple_tag
def current(request, pattern):
    import re
    if re.search(pattern, request.path):
        return 'active'
    return ''

@register.simple_tag
def in_gallery(request):
    import re
    if re.search('/users/[\w.@+-]+/(?:collections.*)?$', request.path):
        return 'visible'
    return 'hidden'

@register.simple_tag
def in_collection(request):
    import re
    if re.search('/users/[\w.@+-]+/collections/(\d+)/$', request.path):
        return 'visible'
    return 'hidden'

@register.inclusion_tag('inclusion_tags/related_images.html')
def related_images(request, object_list, type):
    paginator = Paginator(object_list, 12)

    page = request.GET.get('p')
    try:
        images = paginator.page(page)
    except (TypeError, PageNotAnInteger):
        images = paginator.page(1)
    except EmptyPage:
        images = paginator.page(paginator.num_pages)

    return {
            'request': request,
            'images': images,
            'related_type': type,
           }

@register.filter
def ago(date_time):
    date_time = date_time.replace(tzinfo = None)
    diff = abs(date_time - datetime.today())
    if diff.days <= 0:
        span = timesince(date_time)
        span = span.split(",")[0] # just the most significant digit
        if span == "0 " + _("minutes"):
            return _("seconds ago")
        return _("%s ago") % span
    return datetime.date(date_time)

@register.filter
def string_to_date(date):
    try:
        return datetime.strptime(date, "%Y-%m-%d")
    except:
        return datetime.now()

def image_list(context, object_list, **kwargs):
    alias = kwargs.get('alias', 'gallery')
    nav_ctx = kwargs.get('nav_ctx', 'all')
    nav_ctx_extra = kwargs.get('nav_ctx_extra', None)

    view = kwargs.get('view')
    if view is None and 'view' in context:
      view = context['view']
    if view is None:
      view = context['request'].GET.get('view', 'default')

    return {
        'image_list': object_list,
        'request': context['request'],
        'alias': alias,
        'view': view,
        'nav_ctx': nav_ctx,
        'nav_ctx_extra': nav_ctx_extra,
    }
register.inclusion_tag('inclusion_tags/image_list.html', takes_context=True)(image_list)

@register.inclusion_tag('inclusion_tags/search_image_list.html', takes_context = True)
def search_image_list(context, paginate = True, **kwargs):
    context.update({
        'paginate': paginate,
        'search_domain': context['request'].GET.get('d'),
        'sort': context['request'].GET.get('sort'),
    })

    return context

@register.filter
def seconds_to_hours(value):
    try:
        value = int(value)
    except ValueError:
        return "0"

    return "%.1f" % (int(value) / 3600.0)

@register.filter
def cut_decimals(value, places):
    try:
        value = float(value)
    except ValueError:
        return value

    return '{0:.{1}f}'.format(value, places)

@register.filter
def is_checkbox(value):
    from django.forms.fields import CheckboxInput
    return isinstance(value, CheckboxInput)

@register.inclusion_tag('inclusion_tags/list_gear.html')
def list_gear(label, gear, klass, user):
    return {
        'label': label,
        'gear': gear,
        'klass': klass,
        'user': user,
    }

@register.simple_tag
def gear_complete_class(gear):
    return 'incomplete' if not is_gear_complete(gear.id) else ''

@register.simple_tag
def gear_alias(gear, user):
    default = _("no alias")

    try:
        gear_user_info = GearUserInfo.objects.get(gear = gear, user = user)
        if gear_user_info.alias:
            return gear_user_info.alias
    except GearUserInfo.DoesNotExist:
        pass

    return default

@register.simple_tag
def gear_name_iriencoded(gear):
    from django.template.defaultfilters import iriencode
    name = unicode(gear)
    return iriencode(name)

@register.simple_tag
def gear_owners(gear):
    gear, gear_type = get_correct_gear(gear.id)
    return UserProfile.objects.filter(
        **{UserProfile.GEAR_ATTR_LOOKUP[gear_type]: gear}).count()

@register.simple_tag
def gear_set_images(gear_set):
    images = 0
    for g in gear_set.all():
        images += gear_images(g)

    return images

@register.simple_tag
def gear_set_owners(gear_set):
    owners = 0
    for g in gear_set.all():
        owners += gear_owners(g)

    return owners

@register.simple_tag
def gear_images(gear):
    return Image.by_gear(gear).count()

@register.simple_tag
def gear_type(gear):
    from astrobin.gear import get_correct_gear, TYPES_LOOKUP
    real_gear, gear_type = get_correct_gear(gear.id)

    if real_gear and gear_type and hasattr(real_gear, 'type') and real_gear.type:
        try:
            t = TYPES_LOOKUP[gear_type][real_gear.type][1]
            return t
        except KeyError:
            pass

    return '-'

@register.filter
def show_ads(user):
    from astrobin_apps_donations.templatetags.astrobin_apps_donations_tags import is_donor
    from astrobin_apps_premium.templatetags.astrobin_apps_premium_tags import is_premium, is_lite

    if not settings.ADS_ENABLED:
        return False

    if is_donor(user):
        return False

    if settings.PREMIUM_ENABLED and (is_lite(user) or is_premium(user)):
        return False

    return True

@register.filter
def valid_subscriptions(user):
    from subscription.models import UserSubscription

    if user.is_anonymous():
        return []

    us = UserSubscription.active_objects.filter(user = user)
    subs = [x.subscription for x in us if x.valid()]
    return subs

@register.filter
def inactive_subscriptions(user):
    from subscription.models import UserSubscription

    if user.is_anonymous():
        return []

    return [x.subscription
            for x
            in UserSubscription.objects.filter(user = user)
            if not x.valid() or not x.active]

@register.filter
def has_valid_subscription(user, subscription_pk):
    from subscription.models import UserSubscription

    if user.is_anonymous():
        return False

    us = UserSubscription.active_objects.filter(
        user = user, subscription__pk = subscription_pk)

    if us.count() == 0:
        return False

    return us[0].valid()

@register.filter
def has_valid_subscription_in_category(user, category):
    from subscription.models import UserSubscription

    if user.is_anonymous():
        return False

    us = UserSubscription.active_objects.filter(
        user = user, subscription__category = category)

    if us.count() == 0:
        return False

    return us[0].valid()

@register.filter
def get_premium_subscription_expiration(user):
    from subscription.models import UserSubscription

    if user.is_anonymous():
        return None

    us = UserSubscription.active_objects.filter(
        user = user,
        subscription__group__name__in = ['astrobin_premium', 'astrobin_lite'])

    if us.count() == 0:
        return None

    return us[0].expires

@register.filter
def has_subscription_by_name(user, name):
    from subscription.models import UserSubscription

    if user.is_anonymous():
        return False

    return UserSubscription.objects.filter(
        user = user, subscription__name = name).count() > 0

@register.filter
def get_subscription_by_name(user, name):
    from subscription.models import UserSubscription

    if user.is_anonymous():
        return None

    return UserSubscription.objects.get(
        user = user, subscription__name = name)

@register.filter
def is_content_moderator(user):
    if not user.is_authenticated():
        return False

    return user.groups.filter(name='content_moderators').count() > 0

@register.filter
def is_image_moderator(user):
    if not user.is_authenticated():
        return False

    return user.groups.filter(name='image_moderators').count() > 0

@register.filter
def to_user_timezone(value, user):
    from astrobin.utils import to_user_timezone as tut
    if user.is_authenticated():
        return tut(value, user.userprofile)
    return value

@register.filter
def can_like(user, image):
    from astrobin_apps_premium.templatetags.astrobin_apps_premium_tags import is_free

    user_scores_index = 0
    min_index_to_like = settings.MIN_INDEX_TO_LIKE

    if user.is_authenticated():
        user_scores_index = user.userprofile.get_scores()['user_scores_index']

    if user.is_superuser:
        return True

    if user == image.user:
        return False

    if is_free(user) and user_scores_index < min_index_to_like:
        return False

    return True
<|endoftext|>


<commit_before># Licensed under a 3-clause BSD style license - see LICENSE.rst
"""
Contains astronomical and physical constants for use in Astropy or other
places.

A typical use case might be::

    >>> from astropy.constants import c, m_e
    >>> # ... define the mass of something you want the rest energy of as m ...
    >>> m = m_e
    >>> E = m * c**2
    >>> E.to('MeV')  # doctest: +FLOAT_CMP
    <Quantity 0.510998927603161 MeV>

"""
from contextlib import contextmanager

from astropy.utils import find_current_module

# Hack to make circular imports with units work
try:
    from astropy import units
    del units
except ImportError:
    pass

from .constant import Constant, EMConstant  # noqa
from . import si  # noqa
from . import cgs  # noqa
from . import codata2014, iau2015  # noqa
from . import utils as _utils

# for updating the constants module docstring
_lines = [
    'The following constants are available:\n',
    '========== ============== ================ =========================',
    '   Name        Value            Unit       Description',
    '========== ============== ================ =========================',
]

# NOTE: Update this when default changes.
_utils._set_c(codata2014, iau2015, find_current_module(),
              not_in_module_only=True, doclines=_lines, set_class=True)

_lines.append(_lines[1])

if __doc__ is not None:
    __doc__ += '\n'.join(_lines)

# TODO: Re-implement in a way that is more consistent with astropy.units.
#       See https://github.com/astropy/astropy/pull/7008 discussions.
@contextmanager
def set_enabled_constants(modname):
    """
    Context manager to temporarily set values in the ``constants``
    namespace to an older version.
    See :ref:`astropy-constants-prior` for usage.

    Parameters
    ----------
    modname : {'astropyconst13'}
        Name of the module containing an older version.

    """

    # Re-import here because these were deleted from namespace on init.
    import warnings
    from . import utils as _utils

    # NOTE: Update this when default changes.
    if modname == 'astropyconst13':
        from .astropyconst13 import codata2010 as codata
        from .astropyconst13 import iau2012 as iaudata
    else:
        raise ValueError(
            'Context manager does not currently handle {}'.format(modname))

    module = find_current_module()

    # Ignore warnings about "Constant xxx already has a definition..."
    with warnings.catch_warnings():
        warnings.simplefilter('ignore')
        _utils._set_c(codata, iaudata, module,
                      not_in_module_only=False, set_class=True)

    try:
        yield
    finally:
        with warnings.catch_warnings():
            warnings.simplefilter('ignore')
            # NOTE: Update this when default changes.
            _utils._set_c(codata2014, iau2015, module,
                          not_in_module_only=False, set_class=True)

# Clean up namespace
del find_current_module
del contextmanager
del _utils
del _lines
<commit_msg>Fix missing import<commit_after># Licensed under a 3-clause BSD style license - see LICENSE.rst
"""
Contains astronomical and physical constants for use in Astropy or other
places.

A typical use case might be::

    >>> from astropy.constants import c, m_e
    >>> # ... define the mass of something you want the rest energy of as m ...
    >>> m = m_e
    >>> E = m * c**2
    >>> E.to('MeV')  # doctest: +FLOAT_CMP
    <Quantity 0.510998927603161 MeV>

"""
from contextlib import contextmanager

from astropy.utils import find_current_module

# Hack to make circular imports with units work
try:
    from astropy import units
    del units
except ImportError:
    pass

from .constant import Constant, EMConstant  # noqa
from . import si  # noqa
from . import cgs  # noqa
from . import codata2014, iau2015  # noqa
from . import utils as _utils

# for updating the constants module docstring
_lines = [
    'The following constants are available:\n',
    '========== ============== ================ =========================',
    '   Name        Value            Unit       Description',
    '========== ============== ================ =========================',
]

# NOTE: Update this when default changes.
_utils._set_c(codata2014, iau2015, find_current_module(),
              not_in_module_only=True, doclines=_lines, set_class=True)

_lines.append(_lines[1])

if __doc__ is not None:
    __doc__ += '\n'.join(_lines)

# TODO: Re-implement in a way that is more consistent with astropy.units.
#       See https://github.com/astropy/astropy/pull/7008 discussions.
@contextmanager
def set_enabled_constants(modname):
    """
    Context manager to temporarily set values in the ``constants``
    namespace to an older version.
    See :ref:`astropy-constants-prior` for usage.

    Parameters
    ----------
    modname : {'astropyconst13'}
        Name of the module containing an older version.

    """

    # Re-import here because these were deleted from namespace on init.
    import warnings
    from astropy.utils import find_current_module
    from . import utils as _utils

    # NOTE: Update this when default changes.
    if modname == 'astropyconst13':
        from .astropyconst13 import codata2010 as codata
        from .astropyconst13 import iau2012 as iaudata
    else:
        raise ValueError(
            'Context manager does not currently handle {}'.format(modname))

    module = find_current_module()

    # Ignore warnings about "Constant xxx already has a definition..."
    with warnings.catch_warnings():
        warnings.simplefilter('ignore')
        _utils._set_c(codata, iaudata, module,
                      not_in_module_only=False, set_class=True)

    try:
        yield
    finally:
        with warnings.catch_warnings():
            warnings.simplefilter('ignore')
            # NOTE: Update this when default changes.
            _utils._set_c(codata2014, iau2015, module,
                          not_in_module_only=False, set_class=True)

# Clean up namespace
del find_current_module
del contextmanager
del _utils
del _lines
<|endoftext|>


<commit_before># Copyright 2021 Pants project contributors (see CONTRIBUTORS.md).
# Licensed under the Apache License, Version 2.0 (see LICENSE).

from __future__ import annotations

from collections import defaultdict
from dataclasses import dataclass
from textwrap import dedent
from typing import Any, Iterable, Sequence

import toml
from pkg_resources import Requirement

from pants.backend.python.subsystems.python_tool_base import PythonToolRequirementsBase
from pants.backend.python.util_rules.interpreter_constraints import InterpreterConstraints
from pants.engine.fs import FileContent

# ----------------------------------------------------------------------------------------
# Subsystem
# ----------------------------------------------------------------------------------------

class PoetrySubsystem(PythonToolRequirementsBase):
    options_scope = "poetry"
    help = "Used to generate lockfiles for third-party Python dependencies."

    default_version = "poetry==1.1.8"

    register_interpreter_constraints = True
    default_interpreter_constraints = ["CPython>=3.7,<4"]

# We must monkeypatch Poetry to include `setuptools` and `wheel` in the lockfile. This was fixed
# in Poetry 1.2. See https://github.com/python-poetry/poetry/issues/1584.
# WONTFIX(#12314): only use this custom launcher if using Poetry 1.1..
POETRY_LAUNCHER = FileContent(
    "__pants_poetry_launcher.py",
    dedent(
        """\
        from poetry.console import main
        from poetry.puzzle.provider import Provider

        Provider.UNSAFE_PACKAGES = set()
        main()
        """
    ).encode(),
)

# ----------------------------------------------------------------------------------------
# Parsing
# ----------------------------------------------------------------------------------------

_HEADER = {
    "name": "pants-lockfile-generation",
    "version": "0.1.0",
    "description": "",
    "authors": ["pantsbuild"],
}

def create_pyproject_toml(
    requirements: Iterable[str], interpreter_constraints: InterpreterConstraints
) -> str:
    return toml.dumps(create_pyproject_toml_as_dict(requirements, interpreter_constraints))

def create_pyproject_toml_as_dict(
    raw_requirements: Iterable[str], interpreter_constraints: InterpreterConstraints
) -> dict:
    python_constraint = {"python": interpreter_constraints.to_poetry_constraint()}
    project_name_to_poetry_deps = defaultdict(list)
    for raw_req in raw_requirements:
        # WONTFIX(#12314): add error handling.
        req = Requirement.parse(raw_req)
        poetry_dep = PoetryDependency.from_requirement(req)
        project_name_to_poetry_deps[req.project_name].append(poetry_dep)

    deps = {
        project_name: PoetryDependency.to_pyproject_toml_metadata(poetry_deps)
        for project_name, poetry_deps in project_name_to_poetry_deps.items()
    }
    return {"tool": {"poetry": {**_HEADER, "dependencies": {**python_constraint, **deps}}}}

@dataclass(frozen=True)
class PoetryDependency:
    name: str
    version: str | None
    extras: tuple[str, ...] = ()
    markers: str | None = None

    @classmethod
    def from_requirement(cls, requirement: Requirement) -> PoetryDependency:
        return PoetryDependency(
            requirement.project_name,
            version=str(requirement.specifier) or None,  # type: ignore[attr-defined]
            extras=tuple(sorted(requirement.extras)),
            markers=str(requirement.marker) if requirement.marker else None,
        )

    @classmethod
    def to_pyproject_toml_metadata(
        cls, deps: Sequence[PoetryDependency]
    ) -> dict[str, Any] | list[dict[str, Any]]:
        def convert_dep(dep: PoetryDependency) -> dict[str, Any]:
            metadata: dict[str, Any] = {"version": dep.version or "*"}
            if dep.extras:
                metadata["extras"] = dep.extras
            if dep.markers:
                metadata["markers"] = dep.markers
            return metadata

        if not deps:
            raise AssertionError("Must have at least one element!")
        if len(deps) == 1:
            return convert_dep(deps[0])

        entries = []
        name = deps[0].name
        for dep in deps:
            if dep.name != name:
                raise AssertionError(f"All elements must have the same project name. Given: {deps}")
            entries.append(convert_dep(dep))
        return entries
<commit_msg>Fix poetry locks missing hashes. (#16112)<commit_after># Copyright 2021 Pants project contributors (see CONTRIBUTORS.md).
# Licensed under the Apache License, Version 2.0 (see LICENSE).

from __future__ import annotations

from collections import defaultdict
from dataclasses import dataclass
from textwrap import dedent
from typing import Any, Iterable, Sequence

import toml
from pkg_resources import Requirement

from pants.backend.python.subsystems.python_tool_base import PythonToolRequirementsBase
from pants.backend.python.util_rules.interpreter_constraints import InterpreterConstraints
from pants.engine.fs import FileContent

# ----------------------------------------------------------------------------------------
# Subsystem
# ----------------------------------------------------------------------------------------

class PoetrySubsystem(PythonToolRequirementsBase):
    options_scope = "poetry"
    help = "Used to generate lockfiles for third-party Python dependencies."

    default_version = "poetry==1.1.14"

    register_interpreter_constraints = True
    default_interpreter_constraints = ["CPython>=3.7,<4"]

# We must monkeypatch Poetry to include `setuptools` and `wheel` in the lockfile. This was fixed
# in Poetry 1.2. See https://github.com/python-poetry/poetry/issues/1584.
# WONTFIX(#12314): only use this custom launcher if using Poetry 1.1..
POETRY_LAUNCHER = FileContent(
    "__pants_poetry_launcher.py",
    dedent(
        """\
        from poetry.console import main
        from poetry.puzzle.provider import Provider

        Provider.UNSAFE_PACKAGES = set()
        main()
        """
    ).encode(),
)

# ----------------------------------------------------------------------------------------
# Parsing
# ----------------------------------------------------------------------------------------

_HEADER = {
    "name": "pants-lockfile-generation",
    "version": "0.1.0",
    "description": "",
    "authors": ["pantsbuild"],
}

def create_pyproject_toml(
    requirements: Iterable[str], interpreter_constraints: InterpreterConstraints
) -> str:
    return toml.dumps(create_pyproject_toml_as_dict(requirements, interpreter_constraints))

def create_pyproject_toml_as_dict(
    raw_requirements: Iterable[str], interpreter_constraints: InterpreterConstraints
) -> dict:
    python_constraint = {"python": interpreter_constraints.to_poetry_constraint()}
    project_name_to_poetry_deps = defaultdict(list)
    for raw_req in raw_requirements:
        # WONTFIX(#12314): add error handling.
        req = Requirement.parse(raw_req)
        poetry_dep = PoetryDependency.from_requirement(req)
        project_name_to_poetry_deps[req.project_name].append(poetry_dep)

    deps = {
        project_name: PoetryDependency.to_pyproject_toml_metadata(poetry_deps)
        for project_name, poetry_deps in project_name_to_poetry_deps.items()
    }
    return {"tool": {"poetry": {**_HEADER, "dependencies": {**python_constraint, **deps}}}}

@dataclass(frozen=True)
class PoetryDependency:
    name: str
    version: str | None
    extras: tuple[str, ...] = ()
    markers: str | None = None

    @classmethod
    def from_requirement(cls, requirement: Requirement) -> PoetryDependency:
        return PoetryDependency(
            requirement.project_name,
            version=str(requirement.specifier) or None,  # type: ignore[attr-defined]
            extras=tuple(sorted(requirement.extras)),
            markers=str(requirement.marker) if requirement.marker else None,
        )

    @classmethod
    def to_pyproject_toml_metadata(
        cls, deps: Sequence[PoetryDependency]
    ) -> dict[str, Any] | list[dict[str, Any]]:
        def convert_dep(dep: PoetryDependency) -> dict[str, Any]:
            metadata: dict[str, Any] = {"version": dep.version or "*"}
            if dep.extras:
                metadata["extras"] = dep.extras
            if dep.markers:
                metadata["markers"] = dep.markers
            return metadata

        if not deps:
            raise AssertionError("Must have at least one element!")
        if len(deps) == 1:
            return convert_dep(deps[0])

        entries = []
        name = deps[0].name
        for dep in deps:
            if dep.name != name:
                raise AssertionError(f"All elements must have the same project name. Given: {deps}")
            entries.append(convert_dep(dep))
        return entries
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
import gnupg
import shutil
import smtplib
import tarfile
from cStringIO import StringIO as BIO
from email.mime.multipart import MIMEMultipart
from itsdangerous import URLSafeTimedSerializer
from json import load, dumps
from os import mkdir, chmod, listdir
from os.path import exists, join, splitext
from pyramid.settings import asbool, aslist
from random import SystemRandom

allchars = '23456qwertasdfgzxcvbQWERTASDFGZXCVB789yuiophjknmYUIPHJKLNM'

def generate_drop_id(length=8):
    rng = SystemRandom()
    drop_id = ""
    for i in range(length):
        drop_id += rng.choice(allchars)
    return drop_id

def generate_post_token(secret):
    """ returns a URL safe, signed token that contains a UUID"""
    return URLSafeTimedSerializer(secret, salt=u'post').dumps(generate_drop_id())

def parse_post_token(token, secret, max_age=300):
    return URLSafeTimedSerializer(secret, salt=u'post').loads(token, max_age=max_age)

def sanitize_filename(filename):
    """preserve the file ending, but replace the name with a random token """
    # TODO: fix broken splitext (it reveals everything of the filename after the first `.` - doh!)
    token = generate_drop_id()
    name, extension = splitext(filename)
    if extension:
        return '%s%s' % (token, extension)
    else:
        return token

class DropboxContainer(object):

    settings = None

    def __init__(self, settings=None):
        if settings is not None:
            self.init(settings)

    def init(self, settings):
        self.settings = settings
        self.fs_path = settings['fs_dropbox_root']
        from os import makedirs
        if not exists(self.fs_path):
            makedirs(self.fs_path)

    def add_dropbox(self, drop_id, message=None, attachments=None):
        return Dropbox(self, drop_id, message=message, attachments=attachments)

    def get_dropbox(self, drop_id):
        """ returns the dropbox with the given id, if it does not exist an empty dropbox
        will be created and returned"""
        return Dropbox(self, drop_id=drop_id)

    def destroy(self):
        shutil.rmtree(self.fs_path)

    def __contains__(self, drop_id):
        return exists(join(self.fs_path, drop_id))

def checkRecipient(gpg_context, r):
    uid = '<' + r + '>'
    valid_keys = [ k for k in gpg_context.list_keys() if uid in ', '.join(k['uids']) and k['trust'] in 'ofqmu-' ]
#   pprint(valid_keys)
    return bool(valid_keys)

def sendMultiPart(sender, to, subject, attachments):
    msg = MIMEMultipart()

    msg['From'] = sender
    msg['To'] = to
    msg['Subject'] = subject

    for textfile in attachments:
        with open(textfile, 'rb') as fp:
            attach = MIMEText(fp.read())
            msg.attach(attach)

    s = smtplib.SMTP('localhost')
    s.sendmail(sender, to, msg.as_string())
    s.quit()

class Dropbox(object):

    def __init__(self, container, drop_id, message=None, attachments=None):
        """
        the attachments are expected to conform to what the deform library serializes for a file widget,
        namely a dictionary containing:
            - a file handle under the key `fp`
            - the name of the file under `filename`
        """
        self.drop_id = drop_id
        self.container = container
        self.paths_created = []
        self.fs_path = fs_dropbox_path = join(container.fs_path, drop_id)

        if not exists(fs_dropbox_path):
            mkdir(fs_dropbox_path)
            chmod(fs_dropbox_path, 0770)
            self.paths_created.append(fs_dropbox_path)
            self.status = u'010 created'
            # create an editor token
            self.editor_token = editor_token = generate_drop_id()
            self._write_message(fs_dropbox_path, 'editor_token', editor_token)
        else:
            self.editor_token = open(join(self.fs_path, 'editor_token')).readline()

        if message is not None:
            # write the message into a file
            self._write_message(fs_dropbox_path, 'message', message)
            self.message = message

        # write the attachment into a file
        if attachments is not None:
            for attachment in attachments:
                if attachment is None:
                    continue
                self.add_attachment(attachment)

        self.fs_replies_path = join(self.fs_path, 'replies')

    @property
    def settings(self):
        return self.container.settings

    @property
    def fs_attachment_container(self):
        return join(self.fs_path, 'attach')

    def update_message(self, newtext):
        """ overwrite the message text. this also updates the corresponding file. """
        self._write_message(self.fs_path, 'message', newtext)

    def add_attachment(self, attachment):
        fs_attachment_container = self.fs_attachment_container
        if not exists(fs_attachment_container):
            mkdir(fs_attachment_container)
            chmod(fs_attachment_container, 0770)
            self.paths_created.append(fs_attachment_container)
        sanitized = sanitize_filename(attachment.filename)
        fs_attachment_path = join(fs_attachment_container, sanitized)
        with open(fs_attachment_path, 'w') as fs_attachment:
            shutil.copyfileobj(attachment.file , fs_attachment)
        fs_attachment.close()
        chmod(fs_attachment_path, 0660)
        self.paths_created.append(fs_attachment_path)
        return sanitized

    def process(self, purge_meta_data=True, testing=False):
        """ Calls the external helper scripts to (optionally) purge the meta data and then
            send the contents of the dropbox via email.
        """
        self.status = u'020 submitted'
        gpg_context = gnupg.GPG(gnupghome=self.settings['fs_pgp_pubkeys'])
        editors = aslist(self.settings['editors'])
        admins = aslist(self.settings['admins'])

        # create initial backup if we can't clean
        backup_recipients = [ r for r in editors + admins if checkRecipient(gpg_context, r) ]

        # this will be handled by watchdog, no need to send for each drop
        if not backup_recipients:
            self.status = u'500 no valid keys at all'
            return self.status

        if asbool(self.settings.get('debug', False)): #  use bool helper
            file_out = BIO()
            with tarfile.open(mode = 'w|', fileobj = file_out) as tar:
                import pdb; pdb.set_trace()
                tar.add(join(self.fs_path, 'message'))
                tar.add(join(self.fs_path, 'attach'))
            gpg_context.encrypt(
                file_out.getvalue(),
                backup_recipients,
                always_trust=True,
                output=join(self.fs_path, 'backup.tar.gpg')
            )

        # TODO: do the actual processing, erdgeist!
        return self.status

    def add_reply(self, reply):
        """ Add an editorial reply to the drop box.

            :param reply: the message, must conform to  :class:`views.DropboxReplySchema`

        """
        self._write_message(self.fs_replies_path, 'message_001.txt', dumps(reply))

    def _write_message(self, fs_container, fs_name, message):
        if not exists(fs_container):
            mkdir(fs_container)
            chmod(fs_container, 0770)
        fs_reply_path = join(fs_container, fs_name)
        with open(fs_reply_path, 'w') as fs_reply:
            fs_reply.write(message.encode('utf-8'))
        chmod(fs_reply_path, 0660)
        self.paths_created.append(fs_reply_path)

    @property
    def num_attachments(self):
        """returns the current number of uploaded attachments in the filesystem"""
        if exists(self.fs_attachment_container):
            return len(listdir(self.fs_attachment_container))
        else:
            return 0

    @property
    def replies(self):
        """ returns a list of strings """
        fs_reply_path = join(self.fs_replies_path, 'message_001.txt')
        if exists(fs_reply_path):
            return [load(open(fs_reply_path, 'r'))]
        else:
            return []

    @property
    def status(self):
        """ returns either 'created', 'quarantined', 'success' or 'failure'
        """
        try:
            with open(join(self.fs_path, u'status')) as status_file:
                return status_file.readline()
        except IOError:
            return u'000 no status file'

    @property
    def status_int(self):
        return int(self.status.split()[0])

    @status.setter
    def status(self, state):
        with open(join(self.fs_path, u'status'), 'w') as status_file:
            status_file.write(state)
        if self.status_int >= 500:
            self.wipe()

    def sanitize(self):
        """ removes all unencrypted user input """

    def wipe(self):
        """ removes all data except the status file"""
        # TODO: erdgeist :)
        pass
<commit_msg>remove debug cruft<commit_after># -*- coding: utf-8 -*-
import gnupg
import shutil
import smtplib
import tarfile
from cStringIO import StringIO as BIO
from email.mime.multipart import MIMEMultipart
from itsdangerous import URLSafeTimedSerializer
from json import load, dumps
from os import mkdir, chmod, listdir
from os.path import exists, join, splitext
from pyramid.settings import asbool, aslist
from random import SystemRandom

allchars = '23456qwertasdfgzxcvbQWERTASDFGZXCVB789yuiophjknmYUIPHJKLNM'

def generate_drop_id(length=8):
    rng = SystemRandom()
    drop_id = ""
    for i in range(length):
        drop_id += rng.choice(allchars)
    return drop_id

def generate_post_token(secret):
    """ returns a URL safe, signed token that contains a UUID"""
    return URLSafeTimedSerializer(secret, salt=u'post').dumps(generate_drop_id())

def parse_post_token(token, secret, max_age=300):
    return URLSafeTimedSerializer(secret, salt=u'post').loads(token, max_age=max_age)

def sanitize_filename(filename):
    """preserve the file ending, but replace the name with a random token """
    # TODO: fix broken splitext (it reveals everything of the filename after the first `.` - doh!)
    token = generate_drop_id()
    name, extension = splitext(filename)
    if extension:
        return '%s%s' % (token, extension)
    else:
        return token

class DropboxContainer(object):

    settings = None

    def __init__(self, settings=None):
        if settings is not None:
            self.init(settings)

    def init(self, settings):
        self.settings = settings
        self.fs_path = settings['fs_dropbox_root']
        from os import makedirs
        if not exists(self.fs_path):
            makedirs(self.fs_path)

    def add_dropbox(self, drop_id, message=None, attachments=None):
        return Dropbox(self, drop_id, message=message, attachments=attachments)

    def get_dropbox(self, drop_id):
        """ returns the dropbox with the given id, if it does not exist an empty dropbox
        will be created and returned"""
        return Dropbox(self, drop_id=drop_id)

    def destroy(self):
        shutil.rmtree(self.fs_path)

    def __contains__(self, drop_id):
        return exists(join(self.fs_path, drop_id))

def checkRecipient(gpg_context, r):
    uid = '<' + r + '>'
    valid_keys = [ k for k in gpg_context.list_keys() if uid in ', '.join(k['uids']) and k['trust'] in 'ofqmu-' ]
#   pprint(valid_keys)
    return bool(valid_keys)

def sendMultiPart(sender, to, subject, attachments):
    msg = MIMEMultipart()

    msg['From'] = sender
    msg['To'] = to
    msg['Subject'] = subject

    for textfile in attachments:
        with open(textfile, 'rb') as fp:
            attach = MIMEText(fp.read())
            msg.attach(attach)

    s = smtplib.SMTP('localhost')
    s.sendmail(sender, to, msg.as_string())
    s.quit()

class Dropbox(object):

    def __init__(self, container, drop_id, message=None, attachments=None):
        """
        the attachments are expected to conform to what the deform library serializes for a file widget,
        namely a dictionary containing:
            - a file handle under the key `fp`
            - the name of the file under `filename`
        """
        self.drop_id = drop_id
        self.container = container
        self.paths_created = []
        self.fs_path = fs_dropbox_path = join(container.fs_path, drop_id)

        if not exists(fs_dropbox_path):
            mkdir(fs_dropbox_path)
            chmod(fs_dropbox_path, 0770)
            self.paths_created.append(fs_dropbox_path)
            self.status = u'010 created'
            # create an editor token
            self.editor_token = editor_token = generate_drop_id()
            self._write_message(fs_dropbox_path, 'editor_token', editor_token)
        else:
            self.editor_token = open(join(self.fs_path, 'editor_token')).readline()

        if message is not None:
            # write the message into a file
            self._write_message(fs_dropbox_path, 'message', message)
            self.message = message

        # write the attachment into a file
        if attachments is not None:
            for attachment in attachments:
                if attachment is None:
                    continue
                self.add_attachment(attachment)

        self.fs_replies_path = join(self.fs_path, 'replies')

    @property
    def settings(self):
        return self.container.settings

    @property
    def fs_attachment_container(self):
        return join(self.fs_path, 'attach')

    def update_message(self, newtext):
        """ overwrite the message text. this also updates the corresponding file. """
        self._write_message(self.fs_path, 'message', newtext)

    def add_attachment(self, attachment):
        fs_attachment_container = self.fs_attachment_container
        if not exists(fs_attachment_container):
            mkdir(fs_attachment_container)
            chmod(fs_attachment_container, 0770)
            self.paths_created.append(fs_attachment_container)
        sanitized = sanitize_filename(attachment.filename)
        fs_attachment_path = join(fs_attachment_container, sanitized)
        with open(fs_attachment_path, 'w') as fs_attachment:
            shutil.copyfileobj(attachment.file , fs_attachment)
        fs_attachment.close()
        chmod(fs_attachment_path, 0660)
        self.paths_created.append(fs_attachment_path)
        return sanitized

    def process(self, purge_meta_data=True, testing=False):
        """ Calls the external helper scripts to (optionally) purge the meta data and then
            send the contents of the dropbox via email.
        """
        self.status = u'020 submitted'
        gpg_context = gnupg.GPG(gnupghome=self.settings['fs_pgp_pubkeys'])
        editors = aslist(self.settings['editors'])
        admins = aslist(self.settings['admins'])

        # create initial backup if we can't clean
        backup_recipients = [ r for r in editors + admins if checkRecipient(gpg_context, r) ]

        # this will be handled by watchdog, no need to send for each drop
        if not backup_recipients:
            self.status = u'500 no valid keys at all'
            return self.status

        if asbool(self.settings.get('debug', False)): #  use bool helper
            file_out = BIO()
            with tarfile.open(mode = 'w|', fileobj = file_out) as tar:
                tar.add(join(self.fs_path, 'message'))
                tar.add(join(self.fs_path, 'attach'))
            gpg_context.encrypt(
                file_out.getvalue(),
                backup_recipients,
                always_trust=True,
                output=join(self.fs_path, 'backup.tar.gpg')
            )

        # TODO: do the actual processing, erdgeist!
        return self.status

    def add_reply(self, reply):
        """ Add an editorial reply to the drop box.

            :param reply: the message, must conform to  :class:`views.DropboxReplySchema`

        """
        self._write_message(self.fs_replies_path, 'message_001.txt', dumps(reply))

    def _write_message(self, fs_container, fs_name, message):
        if not exists(fs_container):
            mkdir(fs_container)
            chmod(fs_container, 0770)
        fs_reply_path = join(fs_container, fs_name)
        with open(fs_reply_path, 'w') as fs_reply:
            fs_reply.write(message.encode('utf-8'))
        chmod(fs_reply_path, 0660)
        self.paths_created.append(fs_reply_path)

    @property
    def num_attachments(self):
        """returns the current number of uploaded attachments in the filesystem"""
        if exists(self.fs_attachment_container):
            return len(listdir(self.fs_attachment_container))
        else:
            return 0

    @property
    def replies(self):
        """ returns a list of strings """
        fs_reply_path = join(self.fs_replies_path, 'message_001.txt')
        if exists(fs_reply_path):
            return [load(open(fs_reply_path, 'r'))]
        else:
            return []

    @property
    def status(self):
        """ returns either 'created', 'quarantined', 'success' or 'failure'
        """
        try:
            with open(join(self.fs_path, u'status')) as status_file:
                return status_file.readline()
        except IOError:
            return u'000 no status file'

    @property
    def status_int(self):
        return int(self.status.split()[0])

    @status.setter
    def status(self, state):
        with open(join(self.fs_path, u'status'), 'w') as status_file:
            status_file.write(state)
        if self.status_int >= 500:
            self.wipe()

    def sanitize(self):
        """ removes all unencrypted user input """

    def wipe(self):
        """ removes all data except the status file"""
        # TODO: erdgeist :)
        pass
<|endoftext|>


<commit_before>import hashlib

def serialize_user(user):
    """
    Helper to serialize a user
    and adding a gravatar url
    """
    if isinstance(user, dict):
        out = user
        if 'email' not in out:
            out['email'] = out['name'] # weird case of uplift authors
    else:
        out = {
            'email' : user,
            'real_name' : user,
        }

    # Add gravatar
    email = out['email'].strip().lower()
    h = hashlib.md5(email.encode('utf-8'))
    out['avatar'] = 'https://www.gravatar.com/avatar/{}'.format(h.hexdigest())

    return out

def serialize_bug(bug):
    """
    Helper to serialize a bug from its payload
    """
    payload = bug.payload_data
    if not payload:
        raise Exception('Missing payload')
    bug_data = payload.get('bug')
    analysis = payload.get('analysis')
    if not bug_data or not analysis:
        raise Exception('Missing Bug data or Analysis')

    # Build uplift
    uplift = None
    if analysis.get('uplift_comment') and analysis.get('uplift_author'):
        author = analysis['uplift_author']
        comment = analysis['uplift_comment']
        uplift = {
            'id' : comment['id'],
            'author' : serialize_user(author),
            'comment' : comment['html'].replace('&', '&'),
        }

    # Build versions
    # TODO: check structure with sylvestre
    approval_base_flag = 'approval-mozilla-'
    versions = {}
    for a in bug_data.get('attachments', []):
        for flag in a['flags']:
            if not flag['name'].startswith(approval_base_flag):
                continue
            base_name = flag['name'].replace(approval_base_flag, '')
            name = '{} {}'.format(base_name, flag['status'])
            if name not in versions:
                versions[name] = {
                    'name' : flag['name'],
                    'attachments' : [],
                    'status' : flag['status'],
                }
            versions[name]['attachments'].append(str(a['id']))

    # Build flags
    status_base_flag = 'cf_status_'
    tracking_base_flag = 'cf_tracking_'

    def _filter_flags(base):
        out = [(k.replace(base, '', 1), v) for k,v in bug_data.items() if k.startswith(base + 'firefox')]
        return dict(out)

    return {
        # Base
        'id': bug.id,
        'bugzilla_id': bug.bugzilla_id,
        'url': payload.get('url', 'https://bugzil.la/{}'.format(bug.bugzilla_id)),
        'summary' : bug_data['summary'],
        'keywords' : bug_data['keywords'],
        'flags_status' : _filter_flags(status_base_flag),
        'flags_tracking' : _filter_flags(tracking_base_flag),

        # Contributor structures
        'creator' : serialize_user(analysis['users']['creator']),
        'assignee' : serialize_user(analysis['users']['assignee']),
        'reviewers' : [serialize_user(r) for r in analysis['users']['reviewers']],

        # Stats
        'changes_size' : analysis.get('changes_size', 0),

        # Uplift request
        'uplift' : uplift,

        # Patches
        'patches' : analysis['patches'],

        # Versions
        'versions' : versions,
    }

def serialize_analysis(analysis, full=True):
    """
    Helper to serialize an analysis
    """
    out = {
        'id': analysis.id,
        'name': analysis.name,
        'count' : len(analysis.bugs),
        'parameters' : analysis.parameters,
    }

    if full:
        # Add bugs
        out['bugs'] = [serialize_bug(b) for b in analysis.bugs if b.payload]
    else:
        out['bugs'] = []

    return out

<commit_msg>Shipit: support both html & text comments<commit_after>import hashlib

def serialize_user(user):
    """
    Helper to serialize a user
    and adding a gravatar url
    """
    if isinstance(user, dict):
        out = user
        if 'email' not in out:
            out['email'] = out['name'] # weird case of uplift authors
    else:
        out = {
            'email' : user,
            'real_name' : user,
        }

    # Add gravatar
    email = out['email'].strip().lower()
    h = hashlib.md5(email.encode('utf-8'))
    out['avatar'] = 'https://www.gravatar.com/avatar/{}'.format(h.hexdigest())

    return out

def serialize_bug(bug):
    """
    Helper to serialize a bug from its payload
    """
    payload = bug.payload_data
    if not payload:
        raise Exception('Missing payload')
    bug_data = payload.get('bug')
    analysis = payload.get('analysis')
    if not bug_data or not analysis:
        raise Exception('Missing Bug data or Analysis')

    # Build uplift
    uplift = None
    if analysis.get('uplift_comment') and analysis.get('uplift_author'):
        author = analysis['uplift_author']
        comment = analysis['uplift_comment']
        if 'html' in comment:
            # Weird bug in elm-html-parser
            # does not support direct &
            comment_html = comment['html'].replace('&', '&')
        else:
            comment_html = comment.get('text', 'No comment.')
        uplift = {
            'id' : comment['id'],
            'author' : serialize_user(author),
            'comment' : comment_html,
        }

    # Build versions
    # TODO: check structure with sylvestre
    approval_base_flag = 'approval-mozilla-'
    versions = {}
    for a in bug_data.get('attachments', []):
        for flag in a['flags']:
            if not flag['name'].startswith(approval_base_flag):
                continue
            base_name = flag['name'].replace(approval_base_flag, '')
            name = '{} {}'.format(base_name, flag['status'])
            if name not in versions:
                versions[name] = {
                    'name' : flag['name'],
                    'attachments' : [],
                    'status' : flag['status'],
                }
            versions[name]['attachments'].append(str(a['id']))

    # Build flags
    status_base_flag = 'cf_status_'
    tracking_base_flag = 'cf_tracking_'

    def _filter_flags(base):
        out = [(k.replace(base, '', 1), v) for k,v in bug_data.items() if k.startswith(base + 'firefox')]
        return dict(out)

    return {
        # Base
        'id': bug.id,
        'bugzilla_id': bug.bugzilla_id,
        'url': payload.get('url', 'https://bugzil.la/{}'.format(bug.bugzilla_id)),
        'summary' : bug_data['summary'],
        'keywords' : bug_data['keywords'],
        'flags_status' : _filter_flags(status_base_flag),
        'flags_tracking' : _filter_flags(tracking_base_flag),

        # Contributor structures
        'creator' : serialize_user(analysis['users']['creator']),
        'assignee' : serialize_user(analysis['users']['assignee']),
        'reviewers' : [serialize_user(r) for r in analysis['users']['reviewers']],

        # Stats
        'changes_size' : analysis.get('changes_size', 0),

        # Uplift request
        'uplift' : uplift,

        # Patches
        'patches' : analysis['patches'],

        # Versions
        'versions' : versions,
    }

def serialize_analysis(analysis, full=True):
    """
    Helper to serialize an analysis
    """
    out = {
        'id': analysis.id,
        'name': analysis.name,
        'count' : len(analysis.bugs),
        'parameters' : analysis.parameters,
    }

    if full:
        # Add bugs
        out['bugs'] = [serialize_bug(b) for b in analysis.bugs if b.payload]
    else:
        out['bugs'] = []

    return out

<|endoftext|>


<commit_before>from application import app
from .models import *
import sendgrid
import time
from itsdangerous import URLSafeTimedSerializer

sg = sendgrid.SendGridClient(app.config["SENDGRID_API_KEY"])
ts = URLSafeTimedSerializer(app.config["SECRET_KEY"])

def validate_email(entry):
	token = ts.dumps(entry.email, salt = app.config["EMAIL_TOKENIZER_SALT"])
	message = sendgrid.Mail()
	message.add_to(entry.email)
	message.set_from("noreply@hackbca.com")
	message.set_subject("hackBCA III - Mailing List Subscription Confirmation")
	message.set_html("

")

	message.add_filter("templates", "enable", "1")
	message.add_filter("templates", "template_id", "47174e8a-880e-4bde-978b-6b42caae71af")
	message.add_substitution("token", token)

	status, msg = sg.send(message)

def confirm_email(token):
	email = ts.loads(token, salt = "email-confirm-key", max_age = 86400)
	entry = MailingListEntry.objects(email = email)[0]
	entry.verified = True
	entry.save()

def add_email(email):
	num_entries = MailingListEntry.objects(email=email).count()
	if num_entries > 0:
		raise Exception("EmailExistsError", "Email already exists in database.")
	new_entry = MailingListEntry(email = email)
	new_entry.save()

	validate_email(new_entry)
<commit_msg>Referencing app.config as CONFIG . Removed hardcoded salt.<commit_after>from application import CONFIG
from .models import *
import sendgrid
import time
from itsdangerous import URLSafeTimedSerializer

sg = sendgrid.SendGridClient(CONFIG["SENDGRID_API_KEY"])
ts = URLSafeTimedSerializer(CONFIG["SECRET_KEY"])

def validate_email(entry):
	token = ts.dumps(entry.email, salt = CONFIG["EMAIL_TOKENIZER_SALT"])
	message = sendgrid.Mail()
	message.add_to(entry.email)
	message.set_from("noreply@hackbca.com")
	message.set_subject("hackBCA III - Mailing List Subscription Confirmation")
	message.set_html("

")

	message.add_filter("templates", "enable", "1")
	message.add_filter("templates", "template_id", "47174e8a-880e-4bde-978b-6b42caae71af")
	message.add_substitution("token", token)

	status, msg = sg.send(message)

def confirm_email(token):
	email = ts.loads(token, salt = CONFIG["EMAIL_TOKENIZER_SALT"], max_age = 86400)
	entry = MailingListEntry.objects(email = email)[0]
	entry.verified = True
	entry.save()

def add_email(email):
	num_entries = MailingListEntry.objects(email=email).count()
	if num_entries > 0:
		raise Exception("EmailExistsError", "Email already exists in database.")
	new_entry = MailingListEntry(email = email)
	new_entry.save()

	validate_email(new_entry)
<|endoftext|>