<commit_before>from django.db import models
from django.utils.html import escape
from madrona.features import register
from madrona.features.models import PolygonFeature, FeatureCollection, MultiPolygonFeature, SpatialFeature, Feature
from madrona.common.utils import LargestPolyFromMulti, clean_geometry
from general.utils import sq_meters_to_sq_miles, format_precision
from ofr_manipulators import clip_to_grid, intersecting_cells
from reports import get_summary_reports, get_drawing_summary_reports
import settings
from django.contrib.gis.db.models import GeometryField, PointField, MultiPolygonField, GeoManager
import simplejson

class GeometryFeature(SpatialFeature):
    geometry_orig = GeometryField(srid=settings.GEOMETRY_DB_SRID,
            null=True, blank=True, verbose_name="Original Polygon Geometry")
    geometry_final = GeometryField(srid=settings.GEOMETRY_DB_SRID,
            null=True, blank=True, verbose_name="Final Polygon Geometry")

    @property
    def centroid_kml(self):
        """
        KML geometry representation of the centroid of the polygon
        """
        geom = self.geometry_final.point_on_surface.transform(settings.GEOMETRY_CLIENT_SRID, clone=True)
        return geom.kml

    class Meta(Feature.Meta):
        abstract = True

@register
class AOI(GeometryFeature):
    import settings
    ACTION_CHOICES = (
        ('close', 'Close'),
        ('reopen', 'Reopen'),
        ('other', 'Other'),
        ('none', 'None')
    )
    description = models.TextField(null=True,blank=True)
    reg_action = models.CharField(max_length=30, blank=True, null=True, choices=ACTION_CHOICES, default='none')
    summary = models.TextField(blank=True, null=True, default=settings.SUMMARY_DEFAULT)

    @property
    def is_loading(self):
        if self.summary == settings.SUMMARY_DEFAULT:
            return True
        return False

    @property
    def true_area_m2(self):
        return self.geometry_orig.transform(2163, clone=True).area

    @property
    def true_area_final_m2(self):
        return self.geometry_final.transform(2163, clone=True).area

    @property
    def formatted_area(self):
        return int((self.area_in_sq_miles * 10) +.5) / 10.

    @property
    def area_in_sq_miles(self):
        return sq_meters_to_sq_miles(self.true_area_final_m2)

    def summary_reports(self, attributes):
        from ofr_manipulators import intersecting_drawing_cells
        # Call get_summary_reports with intersecting Grid Cells
        attributes.append({'title': 'Total Area', 'data': str(format_precision(float(self.true_area_m2) / 2590000.0, 0)) + ' sq mi'})
        grid_cells = intersecting_cells(self.geometry_orig)
        get_summary_reports(grid_cells, attributes)
        drawing_grid_cells = intersecting_drawing_cells(self.geometry_orig)
        get_drawing_summary_reports(drawing_grid_cells, attributes)

    @property
    def serialize_attributes(self):
        attributes = []
        if self.description:
            attributes.append({'title': 'Description', 'data': self.description})
        attributes += simplejson.loads(self.summary)
        return { 'event': 'click', 'attributes': attributes }

    @classmethod
    def fill_color(self):
        return '776BAEFD'

    @classmethod
    def outline_color(self):
        return '776BAEFD'

    def clip_to_grid(self, drawing=False):
        geom = self.geometry_orig
        clipped_shape = clip_to_grid(geom, drawing)
        if clipped_shape:
            return LargestPolyFromMulti(clipped_shape)
        else:
            return clipped_shape

    def save(self, *args, **kwargs):
        # TODO Toggle clipping drawings to grid in settings
        # self.geometry_final = self.clip_to_grid()

        self.geometry_final = self.geometry_orig
        if self.geometry_final:
            self.geometry_final = clean_geometry(self.geometry_final)

        if not self.id:  #first save - if part of a large import, now is not a good time to calculate the summary
            super(AOI, self).save(*args, **kwargs) # Call the "real" save() method
        else:
            attributes = []
            self.summary_reports(attributes)
            self.summary = simplejson.dumps(attributes)
            super(AOI, self).save(*args, **kwargs) # Call the "real" save() method

    class Options:
        verbose_name = 'Area of Interest'
        icon_url = 'img/aoi.png'
        export_png = False
        manipulators = []
        # manipulators = ['drawing.manipulators.ClipToPlanningGrid']
        # optional_manipulators = ['clipping.manipulators.ClipToShoreManipulator']
        form = 'drawing.forms.AOIForm'
        form_template = 'aoi/form.html'
        show_template = 'aoi/show.html'

@register
class Collection(FeatureCollection):
    description = models.TextField(null=True,blank=True)

    class Options:
        verbose_name = 'Collection of Proposed Scenarios'
        form = 'drawing.forms.CollectionForm'
        form_template = 'collection/form.html'
        show_template = 'collection/show.html'
        valid_children = (
            'drawing.models.AOI',
            'scenarios.models.Scenario'
        )

    @property
    def is_loading(self):
        ret_val = False
        for feature in self.feature_set():
            if feature.is_loading:
                feature.save()
                ret_val = True
        return False

    def clean_summary_value(self, field, field_map, weight=1):
        import re
        method = field_map['aggregate']
        ret_type = field_map['type']
        num_val = re.findall(r'\d+',field['data'])
        if method in ['sum', 'min', 'max']:
            return ret_type(num_val[0])
        elif method in ['mean', 'minmax']:
            if method == 'mean':
                return (ret_type(num_val[0]), weight)
            if method == 'minmax':
                return (ret_type(num_val[0]),ret_type(num_val[1]))
        else:
            return (ret_type(num_val),False)

    def aggregate_values(self, val_list, method, text, unit_type):
        if method == 'sum':
            return "%s%s" % (str(sum(val_list)), text)
        if method == 'min':
            return "%s%s" % (str(min(val_list)), text)
        if method == 'max':
            return "%s%s" % (str(max(val_list)), text)
        if method == 'minmax':
            min_list = [x for (x,y) in val_list]
            max_list = [y for (x,y) in val_list]
            if len(min_list) > 0 and len(max_list) > 0:
                return "%s%s%s%s" % (str(min(min_list)), text[0], str(max(max_list)), text[1])
            else:
                return False
        if method == 'mean':
            total_area = sum([area for (value, area) in val_list])
            numerator_list = [value*area for (value, area) in val_list]
            if len(numerator_list) > 0:
                return "%s%s" % (str(unit_type(sum(numerator_list)/total_area)), text)
            else:
                return False

    def generate_summary_value(self, attributes, name, field):
        data = self.aggregate_values(field['values'], field['aggregate'], field['unit'], field['type'])
        if data:
            attributes.append({'title': name,'data': data})

    def summary_reports(self, attributes):
        if not self.is_loading:
            from datetime import datetime
            feature_set = list(enumerate(self.feature_set(), start=1))
            count = len(feature_set)
            summary_dict = {
                'name': self.name,
                'description': self.description
            }
            val_collector = {}
            for field in settings.COMPARISON_FIELD_LOOKUP:
                val_collector[field['name']] = field
                val_collector[field['name']]['values'] = []
            for (index, feature) in feature_set:
                feature_summary = eval(feature.summary)
                feature_area = feature.true_area_m2
                for field in feature_summary:
                    clean_val = self.clean_summary_value(field, val_collector[field['title']], feature_area)
                    val_collector[field['title']]['values'].append(clean_val)
            for key in [x['name'] for x in settings.COMPARISON_FIELD_LOOKUP]:
                self.generate_summary_value(attributes,key,val_collector[key])
        else:
            attributes.append(eval(settings.SUMMARY_DEFAULT))

    @property
    def serialize_attributes(self):
        attributes = []
        if self.description:
            attributes.append({'title': 'Description', 'data': self.description})
        self.summary_reports(attributes)
        return { 'event': 'click', 'attributes': attributes }

    def feature_set(self, recurse=False, feature_classes=None):
        return super(Collection, self).feature_set(recurse, feature_classes)

    def save(self, rerun=True, *args, **kwargs):
        # Foothold to manage save logic outside of Madrona - may not be necessary with overriding the form post url/view
        super(Collection, self).save(*args, **kwargs) # Call the "real" save() method

class GridCell(models.Model):

    gridcode = models.IntegerField(null=True, blank=True)
    sq_mi = models.DecimalField(null=True, blank=True, decimal_places=11, max_digits=16)
    # Substrate unit: sq miles
    hrd_sub_m2 = models.DecimalField(null=True, blank=True, decimal_places=11, max_digits=16)
    mix_sub_m2 = models.DecimalField(null=True, blank=True, decimal_places=11, max_digits=16)
    sft_sub_m2 = models.DecimalField(null=True, blank=True, decimal_places=11, max_digits=16)
    rck_sub_m2 = models.DecimalField(null=True, blank=True, decimal_places=11, max_digits=16)
    # Technically this is the depth of the centroid of the unit, but small enough to treat planning units as discrete
    depth = models.DecimalField(null=True, blank=True, decimal_places=11, max_digits=16)
    # PHS unit: sq miles
    hsall1_m2 = models.DecimalField(null=True, blank=True, decimal_places=11, max_digits=16)
    hsall2_m2 = models.DecimalField(null=True, blank=True, decimal_places=11, max_digits=16)
    hsall3_m2 = models.DecimalField(null=True, blank=True, decimal_places=11, max_digits=16)
    hsall4_m2 = models.DecimalField(null=True, blank=True, decimal_places=11, max_digits=16)
    hssclr1_m2 = models.DecimalField(null=True, blank=True, decimal_places=11, max_digits=16)
    hssclr2_m2 = models.DecimalField(null=True, blank=True, decimal_places=11, max_digits=16)
    hssclr3_m2 = models.DecimalField(null=True, blank=True, decimal_places=11, max_digits=16)
    unique_id = models.IntegerField(null=True, blank=True)

    centroid = PointField(
        srid=settings.GEOMETRY_DB_SRID,
        null=True,
        blank=True
    )

    geometry = MultiPolygonField(
        srid=settings.GEOMETRY_DB_SRID,
        null=True, blank=True,
        verbose_name="Grid Cell Geometry"
    )
    objects = GeoManager()
<commit_msg>hacky fix for bad summary bug on new shapes<commit_after>from django.db import models
from django.utils.html import escape
from madrona.features import register
from madrona.features.models import PolygonFeature, FeatureCollection, MultiPolygonFeature, SpatialFeature, Feature
from madrona.common.utils import LargestPolyFromMulti, clean_geometry
from general.utils import sq_meters_to_sq_miles, format_precision
from ofr_manipulators import clip_to_grid, intersecting_cells
from reports import get_summary_reports, get_drawing_summary_reports
import settings
from django.contrib.gis.db.models import GeometryField, PointField, MultiPolygonField, GeoManager
import simplejson

class GeometryFeature(SpatialFeature):
    geometry_orig = GeometryField(srid=settings.GEOMETRY_DB_SRID,
            null=True, blank=True, verbose_name="Original Polygon Geometry")
    geometry_final = GeometryField(srid=settings.GEOMETRY_DB_SRID,
            null=True, blank=True, verbose_name="Final Polygon Geometry")

    @property
    def centroid_kml(self):
        """
        KML geometry representation of the centroid of the polygon
        """
        geom = self.geometry_final.point_on_surface.transform(settings.GEOMETRY_CLIENT_SRID, clone=True)
        return geom.kml

    class Meta(Feature.Meta):
        abstract = True

@register
class AOI(GeometryFeature):
    import settings
    ACTION_CHOICES = (
        ('close', 'Close'),
        ('reopen', 'Reopen'),
        ('other', 'Other'),
        ('none', 'None')
    )
    description = models.TextField(null=True,blank=True)
    reg_action = models.CharField(max_length=30, blank=True, null=True, choices=ACTION_CHOICES, default='none')
    summary = models.TextField(blank=True, null=True, default=settings.SUMMARY_DEFAULT)

    @property
    def is_loading(self):
        if self.summary == settings.SUMMARY_DEFAULT:
            return True
        return False

    @property
    def true_area_m2(self):
        return self.geometry_orig.transform(2163, clone=True).area

    @property
    def true_area_final_m2(self):
        return self.geometry_final.transform(2163, clone=True).area

    @property
    def formatted_area(self):
        return int((self.area_in_sq_miles * 10) +.5) / 10.

    @property
    def area_in_sq_miles(self):
        return sq_meters_to_sq_miles(self.true_area_final_m2)

    def summary_reports(self, attributes):
        from ofr_manipulators import intersecting_drawing_cells
        # Call get_summary_reports with intersecting Grid Cells
        attributes.append({'title': 'Total Area', 'data': str(format_precision(float(self.true_area_m2) / 2590000.0, 0)) + ' sq mi'})
        grid_cells = intersecting_cells(self.geometry_orig)
        get_summary_reports(grid_cells, attributes)
        drawing_grid_cells = intersecting_drawing_cells(self.geometry_orig)
        get_drawing_summary_reports(drawing_grid_cells, attributes)

    @property
    def serialize_attributes(self):
        attributes = []
        if self.description:
            attributes.append({'title': 'Description', 'data': self.description})
        try:
            attributes += simplejson.loads(self.summary)
            return { 'event': 'click', 'attributes': attributes }
        except simplejson.JSONDecodeError as e:
            print(str(e))
            self.save()
            return self.serialize_attributes

    @classmethod
    def fill_color(self):
        return '776BAEFD'

    @classmethod
    def outline_color(self):
        return '776BAEFD'

    def clip_to_grid(self, drawing=False):
        geom = self.geometry_orig
        clipped_shape = clip_to_grid(geom, drawing)
        if clipped_shape:
            return LargestPolyFromMulti(clipped_shape)
        else:
            return clipped_shape

    def save(self, *args, **kwargs):
        # TODO Toggle clipping drawings to grid in settings
        # self.geometry_final = self.clip_to_grid()

        self.geometry_final = self.geometry_orig
        if self.geometry_final:
            self.geometry_final = clean_geometry(self.geometry_final)

        if not self.id:  #first save - if part of a large import, now is not a good time to calculate the summary
            super(AOI, self).save(*args, **kwargs) # Call the "real" save() method
        else:
            attributes = []
            self.summary_reports(attributes)
            self.summary = simplejson.dumps(attributes)
            super(AOI, self).save(*args, **kwargs) # Call the "real" save() method

    class Options:
        verbose_name = 'Area of Interest'
        icon_url = 'img/aoi.png'
        export_png = False
        manipulators = []
        # manipulators = ['drawing.manipulators.ClipToPlanningGrid']
        # optional_manipulators = ['clipping.manipulators.ClipToShoreManipulator']
        form = 'drawing.forms.AOIForm'
        form_template = 'aoi/form.html'
        show_template = 'aoi/show.html'

@register
class Collection(FeatureCollection):
    description = models.TextField(null=True,blank=True)

    class Options:
        verbose_name = 'Collection of Proposed Scenarios'
        form = 'drawing.forms.CollectionForm'
        form_template = 'collection/form.html'
        show_template = 'collection/show.html'
        valid_children = (
            'drawing.models.AOI',
            'scenarios.models.Scenario'
        )

    @property
    def is_loading(self):
        ret_val = False
        for feature in self.feature_set():
            if feature.is_loading:
                feature.save()
                ret_val = True
        return False

    def clean_summary_value(self, field, field_map, weight=1):
        import re
        method = field_map['aggregate']
        ret_type = field_map['type']
        num_val = re.findall(r'\d+',field['data'])
        if method in ['sum', 'min', 'max']:
            return ret_type(num_val[0])
        elif method in ['mean', 'minmax']:
            if method == 'mean':
                return (ret_type(num_val[0]), weight)
            if method == 'minmax':
                return (ret_type(num_val[0]),ret_type(num_val[1]))
        else:
            return (ret_type(num_val),False)

    def aggregate_values(self, val_list, method, text, unit_type):
        if method == 'sum':
            return "%s%s" % (str(sum(val_list)), text)
        if method == 'min':
            return "%s%s" % (str(min(val_list)), text)
        if method == 'max':
            return "%s%s" % (str(max(val_list)), text)
        if method == 'minmax':
            min_list = [x for (x,y) in val_list]
            max_list = [y for (x,y) in val_list]
            if len(min_list) > 0 and len(max_list) > 0:
                return "%s%s%s%s" % (str(min(min_list)), text[0], str(max(max_list)), text[1])
            else:
                return False
        if method == 'mean':
            total_area = sum([area for (value, area) in val_list])
            numerator_list = [value*area for (value, area) in val_list]
            if len(numerator_list) > 0:
                return "%s%s" % (str(unit_type(sum(numerator_list)/total_area)), text)
            else:
                return False

    def generate_summary_value(self, attributes, name, field):
        data = self.aggregate_values(field['values'], field['aggregate'], field['unit'], field['type'])
        if data:
            attributes.append({'title': name,'data': data})

    def summary_reports(self, attributes):
        if not self.is_loading:
            from datetime import datetime
            feature_set = list(enumerate(self.feature_set(), start=1))
            count = len(feature_set)
            summary_dict = {
                'name': self.name,
                'description': self.description
            }
            val_collector = {}
            for field in settings.COMPARISON_FIELD_LOOKUP:
                val_collector[field['name']] = field
                val_collector[field['name']]['values'] = []
            for (index, feature) in feature_set:
                feature_summary = eval(feature.summary)
                feature_area = feature.true_area_m2
                for field in feature_summary:
                    clean_val = self.clean_summary_value(field, val_collector[field['title']], feature_area)
                    val_collector[field['title']]['values'].append(clean_val)
            for key in [x['name'] for x in settings.COMPARISON_FIELD_LOOKUP]:
                self.generate_summary_value(attributes,key,val_collector[key])
        else:
            attributes.append(eval(settings.SUMMARY_DEFAULT))

    @property
    def serialize_attributes(self):
        attributes = []
        if self.description:
            attributes.append({'title': 'Description', 'data': self.description})
        self.summary_reports(attributes)
        return { 'event': 'click', 'attributes': attributes }

    def feature_set(self, recurse=False, feature_classes=None):
        return super(Collection, self).feature_set(recurse, feature_classes)

    def save(self, rerun=True, *args, **kwargs):
        # Foothold to manage save logic outside of Madrona - may not be necessary with overriding the form post url/view
        super(Collection, self).save(*args, **kwargs) # Call the "real" save() method

class GridCell(models.Model):

    gridcode = models.IntegerField(null=True, blank=True)
    sq_mi = models.DecimalField(null=True, blank=True, decimal_places=11, max_digits=16)
    # Substrate unit: sq miles
    hrd_sub_m2 = models.DecimalField(null=True, blank=True, decimal_places=11, max_digits=16)
    mix_sub_m2 = models.DecimalField(null=True, blank=True, decimal_places=11, max_digits=16)
    sft_sub_m2 = models.DecimalField(null=True, blank=True, decimal_places=11, max_digits=16)
    rck_sub_m2 = models.DecimalField(null=True, blank=True, decimal_places=11, max_digits=16)
    # Technically this is the depth of the centroid of the unit, but small enough to treat planning units as discrete
    depth = models.DecimalField(null=True, blank=True, decimal_places=11, max_digits=16)
    # PHS unit: sq miles
    hsall1_m2 = models.DecimalField(null=True, blank=True, decimal_places=11, max_digits=16)
    hsall2_m2 = models.DecimalField(null=True, blank=True, decimal_places=11, max_digits=16)
    hsall3_m2 = models.DecimalField(null=True, blank=True, decimal_places=11, max_digits=16)
    hsall4_m2 = models.DecimalField(null=True, blank=True, decimal_places=11, max_digits=16)
    hssclr1_m2 = models.DecimalField(null=True, blank=True, decimal_places=11, max_digits=16)
    hssclr2_m2 = models.DecimalField(null=True, blank=True, decimal_places=11, max_digits=16)
    hssclr3_m2 = models.DecimalField(null=True, blank=True, decimal_places=11, max_digits=16)
    unique_id = models.IntegerField(null=True, blank=True)

    centroid = PointField(
        srid=settings.GEOMETRY_DB_SRID,
        null=True,
        blank=True
    )

    geometry = MultiPolygonField(
        srid=settings.GEOMETRY_DB_SRID,
        null=True, blank=True,
        verbose_name="Grid Cell Geometry"
    )
    objects = GeoManager()
<|endoftext|>


<commit_before>#!/usr/bin/python
#Python script to rename mp3 files according to the format
#"Track-number Track-name.mp3", for example: 02 Self Control.mp3
#Note: Tracks must have valid ID3 data for this to work - python-mutagen is required.
#By Charles Bos

import os
import sys
from mutagen.id3 import ID3, ID3NoHeaderError

def usage() :
    print('''Usage:
  mp3-file-renamer.py <path to music>''')

#Get music directory
args = sys.argv
if (len(args) != 2) or (args[1] == '-h') or (args[1] == '--help') :
    usage()
    os._exit(0)
else :
    if os.path.exists(args[1]) : musicDir = args[1]
    else :
        usage()
        os._exit(0)

#Get titles and track numbers for songs
musicFiles = []
convList = []
ignore = []

for root, dirs, files in os.walk(musicDir, topdown=False):
    for name in files:
        if name.find(".mp3") != -1 :
            musicFiles.append((root, os.path.join(root, name)))

for x in musicFiles :
    try :
        audio = ID3(x[1])
        title = str(audio["TIT2"].text[0])
        trackNum = str(audio["TRCK"].text[0])
        convList.append([trackNum, title])
    except (ID3NoHeaderError, KeyError, PermissionError) :
        ignore.append(x)

#Remove problem files from list
for x in ignore : musicFiles.remove(x)

#Add leading 0 if missing
#Remove number of tracks per album if it exists
for x in convList :
    if len(x[0].split("/")[0]) == 1 : convList[convList.index(x)][0] = "0" + x[0]
    convList[convList.index(x)][0] = x[0].split("/")[0]

#If no files to convert, report problem files and exit
if len(convList) == 0 :
    if len(convList) == 0 and len(ignore) == 0:
        print("No music files were found.")
    elif len(convList) == 0 and len(ignore) != 0 :
        print("Some music files were found but none were renamable.\n\nCheck the tags and permissions on the following files:\n-----")
        for x in ignore : print(x[1])
    os._exit(0)
else :
    #Start renaming
    counter = 0
    for x in musicFiles :
        os.rename(x[1], os.path.join(x[0], convList[counter][0] + " " + convList[counter][1] + ".mp3"))
        counter += 1
    print(str(len(convList)) + " files were successfully renamed.")
    if len(ignore) > 0 :
        print("\nSome files could not be renamed.\nCheck the tags and permissions on the following files:\n-----")
        for x in ignore : print(x[1])<commit_msg>mp3-files-renamer.py - add fix for illegal characters<commit_after>#!/usr/bin/python
#Python script to rename mp3 files according to the format
#"Track-number Track-name.mp3", for example: 02 Self Control.mp3
#Note: Tracks must have valid ID3 data for this to work - python-mutagen is required.
#By Charles Bos

import os
import sys
from mutagen.id3 import ID3, ID3NoHeaderError

def usage() :
    print('''Usage:
  mp3-file-renamer.py <path to music>''')

#Get music directory
args = sys.argv
if (len(args) != 2) or (args[1] == '-h') or (args[1] == '--help') :
    usage()
    os._exit(0)
else :
    if os.path.exists(args[1]) : musicDir = args[1]
    else :
        usage()
        os._exit(0)

#Get titles and track numbers for songs
musicFiles = []
convList = []
ignore = []

for root, dirs, files in os.walk(musicDir, topdown=False):
    for name in files:
        if name.find(".mp3") != -1 :
            musicFiles.append((root, name))

for x in musicFiles :
    try :
        audio = ID3(os.path.join(x[0], x[1]))
        title = str(audio["TIT2"].text[0])
        trackNum = str(audio["TRCK"].text[0])
        convList.append([trackNum, title])
    except (ID3NoHeaderError, KeyError, PermissionError) :
        ignore.append(x)

#Remove problem files from list
for x in ignore : musicFiles.remove(x)

#If no files to convert, report problem files and exit
if len(convList) == 0 :
    if len(convList) == 0 and len(ignore) == 0:
        print("No music files were found.")
    elif len(convList) == 0 and len(ignore) != 0 :
        print("Some music files were found but none were renamable.\n\nCheck the tags and permissions on the following files:\n-----")
        for x in ignore : print(x[1])
    os._exit(0)
else :
    #Add leading 0 if missing
    #Remove number of tracks per album if it exists
    for x in convList :
        if len(x[0].split("/")[0]) == 1 : convList[convList.index(x)][0] = "0" + x[0]
        convList[convList.index(x)][0] = x[0].split("/")[0]

    #Fix illegal characters
    counter = 0
    while counter < len(musicFiles) :
        if sys.platform == "win32" :
            winIllegals = ["<", ">", ":", '"', "/", "\\", "|", "?", "*"]
            for x in convList[counter][1] :
                if x in winIllegals : convList[counter][1].replace(x, "_")
        else :
            if convList[counter][1].find("/") != -1 : convList[counter][1] = convList[counter][1].replace("/", "_")
        counter += 1

    #Rename files
    counter = 0
    for x in musicFiles :
        os.rename(os.path.join(x[0], x[1]), os.path.join(x[0], convList[counter][0] + " " + convList[counter][1] + ".mp3"))
        counter += 1
    print(str(len(convList)) + " files were successfully renamed.")
    if len(ignore) > 0 :
        print("\nSome files could not be renamed.\nCheck the tags and permissions on the following files:\n-----")
        for x in ignore : print(x[1])<|endoftext|>


<commit_before>from __future__ import absolute_import
import os

from sh import git

class Git(object):
  def __init__(self, repo):
    self.repo = repo

  def commit(self, objects, message):
    # validate commit message
    if not message or not isinstance(message, basestring):
      raise ValueError("Commit message should not be empty or not string")

    os.chdir(self.repo)

    # pull and push from and to the remote
    git.pull()

    for obj in objects:
      git.add("-A", obj)

    git.commit("-m", message)

    git.push()
<commit_msg>Use _env when you do some git operations<commit_after>from __future__ import absolute_import
import os

from sh import git

class Git(object):
  def __init__(self, repo):
    self.repo = repo

  def commit(self, objects, message):
    # validate commit message
    if not message or not isinstance(message, basestring):
      raise ValueError("Commit message should not be empty or not string")

    env = os.environ.copy()
    env.update({
        'GIT_WORK_TREE': self.repo,
        'GIT_DIR': '%s/.git' % self.repo,
    })

    git.gc("--prune", _env=env)
    git.checkout("HEAD", _env=env)

    # pull and push from and to the remote
    git.pull("origin", "master", _env=env)

    for obj in objects:
      git.add("-A", obj, _env=env)

    git.commit("-m", message, _env=env)

    git.push(_env=env)
<|endoftext|>


<commit_before>import getpass
import logging
import os

from pyp2rpm.convertor import Convertor
from pyp2rpm import settings
from pyp2rpm import utils
from pyp2rpm.logger import register_file_log_handler, register_console_log_handler

import click

CONTEXT_SETTINGS = dict(help_option_names=['-h', '--help'])

@click.command(context_settings=CONTEXT_SETTINGS)
@click.option('-t',
              help='Template file (jinja2 format) to render (default: "{0}").'
              'Search order is 1) filesystem, 2) default templates.'.format(
                  settings.DEFAULT_TEMPLATE),
              metavar='TEMPLATE')
@click.option('-o',
              help='Default distro whose conversion rules to use (default:"{0}").'
              'Default templates have their rules associated and ignore this.'.format(
                  settings.DEFAULT_DISTRO),
              type=click.Choice(settings.KNOWN_DISTROS),
              default=settings.DEFAULT_DISTRO)
@click.option('-b',
              help='Base Python version to package for (default: "{0}").'.format(
                  settings.DEFAULT_PYTHON_VERSION),
              default=None,
              metavar='BASE_PYTHON')
@click.option('-p',
              help='Additional Python versions to include in the specfile (e.g -p3 for %{0}).'
              'Can be specified multiple times (default: "{1}"). Specify additional version '
              'or use -b explicitly to disable default.'.format(
                  '{?with_python3}', settings.DEFAULT_ADDITIONAL_VERSION),
              default=[],
              multiple=True,
              metavar='PYTHON_VERSIONS')
@click.option('-s',
              help='Spec file ~/rpmbuild/SPECS/python-package_name.spec will be created (default: '
              'prints spec file to stdout).',
              is_flag=True)
@click.option('--srpm',
              help='When used pyp2rpm will produce srpm instead of printing specfile into stdout.',
              is_flag=True)
@click.option('--proxy',
              help='Specify proxy in the form proxy.server:port.',
              default=None,
              metavar='PROXY')
@click.option('-r',
              help='Name of rpm package (overrides calculated name).',
              default=None,
              metavar='RPM_NAME')
@click.option('-d',
              help='Specify where to save package file, specfile and generated SRPM (default: "{0}").'.format(
                  settings.DEFAULT_PKG_SAVE_PATH),
              default=settings.DEFAULT_PKG_SAVE_PATH,
              metavar='SAVE_PATH')
@click.option('-v', help='Version of the package to download (ignored for local files).',
              metavar='VERSION')
@click.option('--venv / --no-venv',
              default=True,
              help='Enable / disable metadata extraction from virtualenv (default: enabled).')
@click.argument('package', nargs=1)
def main(package, v, d, s, r, proxy, srpm, p, b, o, t, venv):
    """Convert PyPI package to RPM specfile or SRPM.

    \b
    \b\bArguments:
    PACKAGE             Provide PyPI name of the package or path to compressed source file."""
    register_file_log_handler('/tmp/pyp2rpm-{0}.log'.format(getpass.getuser()))

    if srpm or s:
        settings.CONSOLE_LOGGING = True
        register_console_log_handler()

    distro = o
    if t in settings.KNOWN_DISTROS:
        distro = t

    logger = logging.getLogger(__name__)

    logger.info('Pyp2rpm initialized.')

    convertor = Convertor(package=package,
                          version=v,
                          save_dir=d,
                          template=t or settings.DEFAULT_TEMPLATE,
                          distro=distro,
                          base_python_version=b,
                          python_versions=p,
                          rpm_name=r,
                          proxy=proxy,
                          venv=venv)

    logger.debug('Convertor: {0} created. Trying to convert.'.format(convertor))
    converted = convertor.convert()
    logger.debug('Convertor: {0} succesfully converted.'.format(convertor))

    if srpm or s:
        if r:
            spec_name = r + '.spec'
        else:
            prefix = 'python-' if not convertor.name.startswith('python-') else ''
            spec_name = prefix + convertor.name + '.spec'
        logger.info('Using name: {0} for specfile.'.format(spec_name))
        if d == settings.DEFAULT_PKG_SAVE_PATH:
            # default save_path is rpmbuild tree so we want to save spec
            # in  rpmbuild/SPECS/
            spec_path = d + '/SPECS/' + spec_name
        else:
            # if user provide save_path then save spec in provided path
            spec_path = d + '/' + spec_name
        spec_dir = os.path.dirname(spec_path)
        if not os.path.exists(spec_dir):
            os.makedirs(spec_dir)
        logger.debug('Opening specfile: {0}.'.format(spec_path))

        if not utils.PY3:
            converted = converted.encode('utf-8')
        with open(spec_path, 'w') as f:
            f.write(converted)
            logger.info('Specfile saved at: {0}.'.format(spec_path))

        if srpm:
            msg = utils.build_srpm(spec_path, d)
            if utils.PY3:
                logger.info(msg.decode('utf-8'))
            else:
                logger.info(msg)

    else:
        logger.debug('Printing specfile to stdout.')
        if utils.PY3:
            print(converted)
        else:
            print(converted.encode('utf-8'))
        logger.debug('Specfile printed.')
    logger.info("That's all folks!")
<commit_msg>Fix python3 str decode error<commit_after>import getpass
import logging
import os

from pyp2rpm.convertor import Convertor
from pyp2rpm import settings
from pyp2rpm import utils
from pyp2rpm.logger import register_file_log_handler, register_console_log_handler

import click

CONTEXT_SETTINGS = dict(help_option_names=['-h', '--help'])

@click.command(context_settings=CONTEXT_SETTINGS)
@click.option('-t',
              help='Template file (jinja2 format) to render (default: "{0}").'
              'Search order is 1) filesystem, 2) default templates.'.format(
                  settings.DEFAULT_TEMPLATE),
              metavar='TEMPLATE')
@click.option('-o',
              help='Default distro whose conversion rules to use (default:"{0}").'
              'Default templates have their rules associated and ignore this.'.format(
                  settings.DEFAULT_DISTRO),
              type=click.Choice(settings.KNOWN_DISTROS),
              default=settings.DEFAULT_DISTRO)
@click.option('-b',
              help='Base Python version to package for (default: "{0}").'.format(
                  settings.DEFAULT_PYTHON_VERSION),
              default=None,
              metavar='BASE_PYTHON')
@click.option('-p',
              help='Additional Python versions to include in the specfile (e.g -p3 for %{0}).'
              'Can be specified multiple times (default: "{1}"). Specify additional version '
              'or use -b explicitly to disable default.'.format(
                  '{?with_python3}', settings.DEFAULT_ADDITIONAL_VERSION),
              default=[],
              multiple=True,
              metavar='PYTHON_VERSIONS')
@click.option('-s',
              help='Spec file ~/rpmbuild/SPECS/python-package_name.spec will be created (default: '
              'prints spec file to stdout).',
              is_flag=True)
@click.option('--srpm',
              help='When used pyp2rpm will produce srpm instead of printing specfile into stdout.',
              is_flag=True)
@click.option('--proxy',
              help='Specify proxy in the form proxy.server:port.',
              default=None,
              metavar='PROXY')
@click.option('-r',
              help='Name of rpm package (overrides calculated name).',
              default=None,
              metavar='RPM_NAME')
@click.option('-d',
              help='Specify where to save package file, specfile and generated SRPM (default: "{0}").'.format(
                  settings.DEFAULT_PKG_SAVE_PATH),
              default=settings.DEFAULT_PKG_SAVE_PATH,
              metavar='SAVE_PATH')
@click.option('-v', help='Version of the package to download (ignored for local files).',
              metavar='VERSION')
@click.option('--venv / --no-venv',
              default=True,
              help='Enable / disable metadata extraction from virtualenv (default: enabled).')
@click.argument('package', nargs=1)
def main(package, v, d, s, r, proxy, srpm, p, b, o, t, venv):
    """Convert PyPI package to RPM specfile or SRPM.

    \b
    \b\bArguments:
    PACKAGE             Provide PyPI name of the package or path to compressed source file."""
    register_file_log_handler('/tmp/pyp2rpm-{0}.log'.format(getpass.getuser()))

    if srpm or s:
        settings.CONSOLE_LOGGING = True
        register_console_log_handler()

    distro = o
    if t in settings.KNOWN_DISTROS:
        distro = t

    logger = logging.getLogger(__name__)

    logger.info('Pyp2rpm initialized.')

    convertor = Convertor(package=package,
                          version=v,
                          save_dir=d,
                          template=t or settings.DEFAULT_TEMPLATE,
                          distro=distro,
                          base_python_version=b,
                          python_versions=p,
                          rpm_name=r,
                          proxy=proxy,
                          venv=venv)

    logger.debug('Convertor: {0} created. Trying to convert.'.format(convertor))
    converted = convertor.convert()
    logger.debug('Convertor: {0} succesfully converted.'.format(convertor))

    if srpm or s:
        if r:
            spec_name = r + '.spec'
        else:
            prefix = 'python-' if not convertor.name.startswith('python-') else ''
            spec_name = prefix + convertor.name + '.spec'
        logger.info('Using name: {0} for specfile.'.format(spec_name))
        if d == settings.DEFAULT_PKG_SAVE_PATH:
            # default save_path is rpmbuild tree so we want to save spec
            # in  rpmbuild/SPECS/
            spec_path = d + '/SPECS/' + spec_name
        else:
            # if user provide save_path then save spec in provided path
            spec_path = d + '/' + spec_name
        spec_dir = os.path.dirname(spec_path)
        if not os.path.exists(spec_dir):
            os.makedirs(spec_dir)
        logger.debug('Opening specfile: {0}.'.format(spec_path))

        if not utils.PY3:
            converted = converted.encode('utf-8')
        with open(spec_path, 'w') as f:
            f.write(converted)
            logger.info('Specfile saved at: {0}.'.format(spec_path))

        if srpm:
            msg = utils.build_srpm(spec_path, d)
            logger.info(msg)

    else:
        logger.debug('Printing specfile to stdout.')
        if utils.PY3:
            print(converted)
        else:
            print(converted.encode('utf-8'))
        logger.debug('Specfile printed.')
    logger.info("That's all folks!")
<|endoftext|>


<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-
# @Author: oesteban
# @Date:   2016-03-16 11:28:27
# @Last Modified by:   oesteban
# @Last Modified time: 2016-03-16 15:12:13

"""
Compares pandas dataframes by columns

"""
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
from __future__ import unicode_literals

import sys
from argparse import ArgumentParser, RawTextHelpFormatter
import numpy as np
import pandas as pd

def main():
    """Entry point"""
    from mriqc.classifier.data import read_iqms
    parser = ArgumentParser(description='compare two pandas dataframes',
                            formatter_class=RawTextHelpFormatter)
    g_input = parser.add_argument_group('Inputs')
    g_input.add_argument('-i', '--input-csv', action='store',
                         required=True, help='input data frame')
    g_input.add_argument('-r', '--reference-csv', action='store',
                         required=True, help='reference dataframe')

    opts = parser.parse_args()

    ref_df, ref_names, ref_bids = read_iqms(opts.reference_csv)
    tst_df, tst_names, tst_bids = read_iqms(opts.input_csv)

    if sorted(ref_bids) != sorted(tst_bids):
        sys.exit('Dataset has different BIDS bits w.r.t. reference')

    if sorted(ref_names) != sorted(tst_names):
        sys.exit('Output CSV file changed number of columns')

    ref_df = ref_df.sort_values(by=ref_bids)
    tst_df = tst_df.sort_values(by=tst_bids)

    if not np.all(ref_df[ref_names].values == tst_df[tst_names].values):
        sys.exit('Output CSV file changed one or more values')

    sys.exit(0)

if __name__ == '__main__':
    main()
<commit_msg>add printing of changing values<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-
# @Author: oesteban
# @Date:   2016-03-16 11:28:27
# @Last Modified by:   oesteban
# @Last Modified time: 2016-03-16 15:12:13

"""
Compares pandas dataframes by columns

"""
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
from __future__ import unicode_literals

import sys
from argparse import ArgumentParser, RawTextHelpFormatter
import numpy as np
import pandas as pd

def main():
    """Entry point"""
    from mriqc.classifier.data import read_iqms
    parser = ArgumentParser(description='compare two pandas dataframes',
                            formatter_class=RawTextHelpFormatter)
    g_input = parser.add_argument_group('Inputs')
    g_input.add_argument('-i', '--input-csv', action='store',
                         required=True, help='input data frame')
    g_input.add_argument('-r', '--reference-csv', action='store',
                         required=True, help='reference dataframe')

    opts = parser.parse_args()

    ref_df, ref_names, ref_bids = read_iqms(opts.reference_csv)
    tst_df, tst_names, tst_bids = read_iqms(opts.input_csv)

    ref_df.set_index(ref_bids)
    tst_df.set_index(tst_bids)

    if sorted(ref_bids) != sorted(tst_bids):
        sys.exit('Dataset has different BIDS bits w.r.t. reference')

    if sorted(ref_names) != sorted(tst_names):
        sys.exit('Output CSV file changed number of columns')

    ref_df = ref_df.sort_values(by=ref_bids)
    tst_df = tst_df.sort_values(by=tst_bids)

    diff = ref_df[ref_names].values != tst_df[tst_names].values
    if np.any(diff):
        ne_stacked = (ref_df[ref_names] != tst_df[ref_names]).stack()
        changed = ne_stacked[ne_stacked]
        # changed.set_index(ref_bids)
        difference_locations = np.where(diff)
        changed_from = ref_df[ref_names].values[difference_locations]
        changed_to = tst_df[ref_names].values[difference_locations]
        cols = [ref_names[v] for v in difference_locations[1]]
        bids_df = ref_df.loc[difference_locations[0], ref_bids].reset_index()
        chng_df = pd.DataFrame({'iqm': cols, 'from': changed_from, 'to': changed_to})
        table = pd.concat([bids_df, chng_df], axis=1)
        print(table[ref_bids + ['iqm', 'from', 'to']].to_string(index=False))
        sys.exit('Output CSV file changed one or more values')

    sys.exit(0)

if __name__ == '__main__':
    main()
<|endoftext|>


<commit_before>#! /usr/bin/env python
# -*- coding: utf-8 -*-

import numpy as np
import re
import sys

def extract(arq):
    '''
    INPUT
    ----------------
    arq: opened csv file

    OUTPUT
    ----------------
    curves:
    Store the "Rapid Light Curves"
    pulses:
    saturated light pulses
    '''
    #f = open(sys.argv[1],'r')
    f = open(arq,'r')
    t = f.readlines()
    f.close()

    pre_dados = re.compile("[0-9]{2}[A-Z]{3}[0-9]{4}")
    header = t[0].split(',')
    dicts = []
    new_dict = {}
    keys = []
    for line in t:
        line = line[:-1].strip()
        if not line:
            pass
        if "Date,Time,No.," in line:
            keys = line.split(',')
            for k in keys:
                new_dict[k] = []
        elif re.match(pre_dados,line): #
            if line.split(',')[2] == str(1):
                #print line
                if new_dict:
                    dicts.append(new_dict)
                new_dict = {}
                for k in keys:
                    new_dict[k] = []
            data = line.split(',')
            values = zip(keys, data)
            for k, v in values:
                new_dict[k].append(v)
    dicts.append(new_dict)

    # FIXME Insert some functionalities to exclude
    #       duplicate measures
    curves = [l for l in dicts if len(l['No.']) >= 19]
    del curves[16] #retira a medida duplicada do arquivo

    pulses = []
    for k in dicts:
        if k['No.']:
            if len(k['No.']) < 19:
                if len(k['No.']) <= 3:
                    pulses.append(k)
    del pulses[16] # retira medida duplicada do arquivo.

    return curves, pulses

#extract(sys.argv[1])
<commit_msg>docstring<commit_after>#! /usr/bin/env python
# -*- coding: utf-8 -*-

import numpy as np
import re
import sys

def extract(arq):
    '''
    Parse a csv processed file by PhytoWIN.
    PhytoWIN is a software by WALZ, to proccess PhytoPAM data.

    Parameters
    ----------
    arq: opened csv file

    Returns
    -------
    curves : Store the "Rapid Light Curves"
    pulses : Saturated light pulses
    '''
    #f = open(sys.argv[1],'r')
    f = open(arq,'r')
    t = f.readlines()
    f.close()

    pre_dados = re.compile("[0-9]{2}[A-Z]{3}[0-9]{4}")
    header = t[0].split(',')
    dicts = []
    new_dict = {}
    keys = []
    for line in t:
        line = line[:-1].strip()
        if not line:
            pass
        if "Date,Time,No.," in line:
            keys = line.split(',')
            for k in keys:
                new_dict[k] = []
        elif re.match(pre_dados,line): #
            if line.split(',')[2] == str(1):
                #print line
                if new_dict:
                    dicts.append(new_dict)
                new_dict = {}
                for k in keys:
                    new_dict[k] = []
            data = line.split(',')
            values = zip(keys, data)
            for k, v in values:
                new_dict[k].append(v)
    dicts.append(new_dict)

    # FIXME Insert some functionalities to exclude
    #       duplicate measures
    curves = [l for l in dicts if len(l['No.']) >= 19]
    del curves[16] #retira a medida duplicada do arquivo

    pulses = []
    for k in dicts:
        if k['No.']:
            if len(k['No.']) < 19:
                if len(k['No.']) <= 3:
                    pulses.append(k)
    del pulses[16] # retira medida duplicada do arquivo.

    return curves, pulses

#extract(sys.argv[1])
<|endoftext|>


<commit_before>from decimal import ROUND_UP, Decimal

import click
from binary import TEBIBYTE, convert_units

from pypinfo.core import (
    add_percentages, build_query, create_client, create_config, format_json,
    parse_query_result, tabulate
)
from pypinfo.db import get_credentials, set_credentials
from pypinfo.fields import (
    Project, Date, Month, Year, Country, Version, PythonVersion, Percent3,
    Percent2, Installer, InstallerVersion, SetuptoolsVersion, System,
    SystemRelease, Implementation, ImplementationVersion, OpenSSLVersion,
    Distro, DistroVersion, CPU
)

CONTEXT_SETTINGS = {
    'max_content_width': 300
}
FIELD_MAP = {
    'project': Project,
    'version': Version,
    'pyversion': PythonVersion,
    'percent3': Percent3,
    'percent2': Percent2,
    'impl': Implementation,
    'impl-version': ImplementationVersion,
    'openssl': OpenSSLVersion,
    'date': Date,
    'month': Month,
    'year': Year,
    'country': Country,
    'installer': Installer,
    'installer-version': InstallerVersion,
    'setuptools-version': SetuptoolsVersion,
    'system': System,
    'system-release': SystemRelease,
    'distro': Distro,
    'distro-version': DistroVersion,
    'cpu': CPU,
}
TIER_COST = 5
TB = Decimal(TEBIBYTE)
TO_CENTS = Decimal('0.00')

@click.group(invoke_without_command=True, context_settings=CONTEXT_SETTINGS)
@click.argument('project', required=False)
@click.argument('fields', nargs=-1, required=False)
@click.option('--auth', '-a', help='Path to Google credentials JSON file.')
@click.option('--run/--test', default=True, help='--test simply prints the query.')
@click.option('--json', '-j', is_flag=True, help='Print data as JSON.')
@click.option('--timeout', '-t', type=int, default=120000,
              help='Milliseconds. Default: 120000 (2 minutes)')
@click.option('--limit', '-l', help='Maximum number of query results. Default: 10')
@click.option('--days', '-d', help='Number of days in the past to include. Default: 30')
@click.option('--start-date', '-sd', help='Must be negative. Default: -31')
@click.option('--end-date', '-ed', help='Must be negative. Default: -1')
@click.option('--where', '-w', help='WHERE conditional. Default: file.project = "project"')
@click.option('--order', '-o', help='Field to order by. Default: download_count')
@click.option('--pip', '-p', is_flag=True, help='Only show installs by pip.')
@click.option('--percent', '-pc', is_flag=True, help='Print percentages.')
@click.option('--markdown', '-md', is_flag=True, help='Output as Markdown.')
@click.version_option()
@click.pass_context
def pypinfo(ctx, project, fields, auth, run, json, timeout, limit, days,
            start_date, end_date, where, order, pip, percent, markdown):
    """Valid fields are:\n
    project | version | pyversion | percent3 | percent2 | impl | impl-version |\n
    openssl | date | month | year | country | installer | installer-version |\n
    setuptools-version | system | system-release | distro | distro-version | cpu
    """
    if auth:
        set_credentials(auth)
        click.echo('Credentials location set to "{}".'.format(get_credentials()))
        return

    if project is None and not fields:
        click.echo(ctx.get_help())
        return

    parsed_fields = []
    for field in fields:
        parsed = FIELD_MAP.get(field)
        if parsed is None:
            raise ValueError('"{}" is an unsupported field.'.format(field))
        parsed_fields.append(parsed)

    order_name = order
    order = FIELD_MAP.get(order)
    if order:
        order_name = order.name
        parsed_fields.insert(0, order)

    built_query = build_query(
        project, parsed_fields, limit=limit, days=days, start_date=start_date,
        end_date=end_date, where=where, order=order_name, pip=pip
    )

    if run:
        client = create_client(get_credentials())
        query_job = client.query(built_query, job_config=create_config())
        query_rows = query_job.result(timeout=timeout // 1000)

        # Cached
        click.echo('Served from cache: {}'.format(not not query_job.cache_hit))

        # Processed
        amount, unit = convert_units(query_job.total_bytes_processed or 0)
        click.echo('Data processed: {:.2f} {}'.format(amount, unit))

        # Billed
        bytes_billed = query_job.total_bytes_billed
        amount, unit = convert_units(bytes_billed or 0)
        click.echo('Data billed: {:.2f} {}'.format(amount, unit))

        # Cost
        billing_tier = query_job.billing_tier or 1
        estimated_cost = Decimal(TIER_COST * billing_tier) / TB * Decimal(bytes_billed)
        estimated_cost = estimated_cost.quantize(TO_CENTS, rounding=ROUND_UP)
        click.echo('Estimated cost: ${}'.format(estimated_cost))

        click.echo()

        rows = parse_query_result(query_job, query_rows)

        if percent:
            rows = add_percentages(rows, include_sign=not json)

        if not json:
            click.echo(tabulate(rows, markdown))
        else:
            click.echo(format_json(rows))
    else:
        click.echo(built_query)
<commit_msg>fix json<commit_after>from decimal import ROUND_UP, Decimal

import click
from binary import TEBIBYTE, convert_units

from pypinfo.core import (
    add_percentages, build_query, create_client, create_config, format_json,
    parse_query_result, tabulate
)
from pypinfo.db import get_credentials, set_credentials
from pypinfo.fields import (
    Project, Date, Month, Year, Country, Version, PythonVersion, Percent3,
    Percent2, Installer, InstallerVersion, SetuptoolsVersion, System,
    SystemRelease, Implementation, ImplementationVersion, OpenSSLVersion,
    Distro, DistroVersion, CPU
)

CONTEXT_SETTINGS = {
    'max_content_width': 300
}
FIELD_MAP = {
    'project': Project,
    'version': Version,
    'pyversion': PythonVersion,
    'percent3': Percent3,
    'percent2': Percent2,
    'impl': Implementation,
    'impl-version': ImplementationVersion,
    'openssl': OpenSSLVersion,
    'date': Date,
    'month': Month,
    'year': Year,
    'country': Country,
    'installer': Installer,
    'installer-version': InstallerVersion,
    'setuptools-version': SetuptoolsVersion,
    'system': System,
    'system-release': SystemRelease,
    'distro': Distro,
    'distro-version': DistroVersion,
    'cpu': CPU,
}
TIER_COST = 5
TB = Decimal(TEBIBYTE)
TO_CENTS = Decimal('0.00')

@click.group(invoke_without_command=True, context_settings=CONTEXT_SETTINGS)
@click.argument('project', required=False)
@click.argument('fields', nargs=-1, required=False)
@click.option('--auth', '-a', help='Path to Google credentials JSON file.')
@click.option('--run/--test', default=True, help='--test simply prints the query.')
@click.option('--json', '-j', is_flag=True, help='Print data as JSON.')
@click.option('--timeout', '-t', type=int, default=120000,
              help='Milliseconds. Default: 120000 (2 minutes)')
@click.option('--limit', '-l', help='Maximum number of query results. Default: 10')
@click.option('--days', '-d', help='Number of days in the past to include. Default: 30')
@click.option('--start-date', '-sd', help='Must be negative. Default: -31')
@click.option('--end-date', '-ed', help='Must be negative. Default: -1')
@click.option('--where', '-w', help='WHERE conditional. Default: file.project = "project"')
@click.option('--order', '-o', help='Field to order by. Default: download_count')
@click.option('--pip', '-p', is_flag=True, help='Only show installs by pip.')
@click.option('--percent', '-pc', is_flag=True, help='Print percentages.')
@click.option('--markdown', '-md', is_flag=True, help='Output as Markdown.')
@click.version_option()
@click.pass_context
def pypinfo(ctx, project, fields, auth, run, json, timeout, limit, days,
            start_date, end_date, where, order, pip, percent, markdown):
    """Valid fields are:\n
    project | version | pyversion | percent3 | percent2 | impl | impl-version |\n
    openssl | date | month | year | country | installer | installer-version |\n
    setuptools-version | system | system-release | distro | distro-version | cpu
    """
    if auth:
        set_credentials(auth)
        click.echo('Credentials location set to "{}".'.format(get_credentials()))
        return

    if project is None and not fields:
        click.echo(ctx.get_help())
        return

    parsed_fields = []
    for field in fields:
        parsed = FIELD_MAP.get(field)
        if parsed is None:
            raise ValueError('"{}" is an unsupported field.'.format(field))
        parsed_fields.append(parsed)

    order_name = order
    order = FIELD_MAP.get(order)
    if order:
        order_name = order.name
        parsed_fields.insert(0, order)

    built_query = build_query(
        project, parsed_fields, limit=limit, days=days, start_date=start_date,
        end_date=end_date, where=where, order=order_name, pip=pip
    )

    if run:
        client = create_client(get_credentials())
        query_job = client.query(built_query, job_config=create_config())
        query_rows = query_job.result(timeout=timeout // 1000)

        # Cached
        from_cache = not not query_job.cache_hit

        # Processed
        bytes_processed = query_job.total_bytes_processed
        processed_amount, processed_unit = convert_units(bytes_processed or 0)

        # Billed
        bytes_billed = query_job.total_bytes_billed
        billed_amount, billed_unit = convert_units(bytes_billed or 0)

        # Cost
        billing_tier = query_job.billing_tier or 1
        estimated_cost = Decimal(TIER_COST * billing_tier) / TB * Decimal(bytes_billed)
        estimated_cost = estimated_cost.quantize(TO_CENTS, rounding=ROUND_UP)

        rows = parse_query_result(query_job, query_rows)

        if percent:
            rows = add_percentages(rows, include_sign=not json)

        if not json:
            click.echo('Served from cache: {}'.format(from_cache))
            click.echo('Data processed: {:.2f} {}'.format(processed_amount, processed_unit))
            click.echo('Data billed: {:.2f} {}'.format(billed_amount, billed_unit))
            click.echo('Estimated cost: ${}'.format(estimated_cost))

            click.echo()
            click.echo(tabulate(rows, markdown))
        else:
            click.echo(format_json(rows))
    else:
        click.echo(built_query)
<|endoftext|>


<commit_before>import logging
logger = logging.getLogger(name=__name__)

def getmodel(modeltype):
    try:
        return globals()[modeltype]
    except KeyError:
        # try to find a similar model with lowercase spelling
        for k in globals():
            if k.lower() == modeltype.lower():
                return globals()[k]
        logger.error("Model %s not found in model definition file %s",
                     modeltype, __file__)

class Model(object):
    " generic model object that will make smart use of its inputs and outputs"
    state = {'theoretical': {},
             'experimental': {},
             'set': {}}
    inputs = {}
    outputs = {}

    def __init__(self, parent=None):
        if parent is None:
            self._parent = self
        else:
            self._parent = parent
        self.logger = logging.getLogger(__name__)

    def setup(self):
        pass

    def search(self, *args, **kwargs):
        pass

    def lock(self, *args, **kwargs):
        pass

class Interferometer(Model):
    pass

class FabryPerot(Model):
    pass

<commit_msg>started model but bug appeared<commit_after>import logging
logger = logging.getLogger(name=__name__)

def getmodel(modeltype):
    try:
        return globals()[modeltype]
    except KeyError:
        # try to find a similar model with lowercase spelling
        for k in globals():
            if k.lower() == modeltype.lower():
                return globals()[k]
        logger.error("Model %s not found in model definition file %s",
                     modeltype, __file__)

class Model(object):
    " generic model object that will make smart use of its inputs and outputs"
    state = {'theoretical': {},
             'experimental': {},
             'set': {}}
    inputs = {}
    outputs = {}

    def __init__(self, parent=None):
        self.logger = logging.getLogger(__name__)
        if parent is None:
            self._parent = self
        else:
            self._parent = parent
        self._config = self._parent.c.model

    def setup(self):
        pass

    def search(self, *args, **kwargs):
        pass

    def lock(self, *args, **kwargs):
        pass

class Interferometer(Model):
    def reflection(self, variable):
        return variable

    pass

class FabryPerot(Model):
    pass

<|endoftext|>


<commit_before>import codecs
import grp
import os
import pwd
import shutil

from django.contrib.admin.views.decorators import staff_member_required
from django.shortcuts import render
from .common import *
from .constants import *
from zipfile import ZipFile
from .models import *

from internetarchive import upload

try:
    import zookeeper
    HAS_ZOOKEEPER = True
except ImportError:
    HAS_ZOOKEEPER = False

@staff_member_required
def add_livestream(request, pk):
    """ Returns page to add a livestream VOD article """
    data = {
        "title": "Tools",
        "file": File.objects.get(pk=pk),
        "today": str(datetime.now())[:10]
    }
    print(SITE_ROOT)

    if request.POST.get("action"):
        if request.POST.get("pk"):
            a = Article.objects.get(pk=int(request.POST["pk"]))
        else:
            a = Article()

        # Convert video URL
        url = request.POST.get("url")
        if request.POST.get("url").startswith("http"):
            url = url.replace("https://youtu.be/", "")
            url = url.replace("https://www.youtube.com/watch?v=", "")
            url = url.replace("https://studio.youtube.com/video/", "")
            url = url.replace("/edit", "")
            if "&" in url:
                url = url[:url.find("&")]
        data["video_id"] = url

        a.title = "Livestream - " + request.POST.get("title")
        a.author = request.POST.get("author")
        a.category = "Livestream"
        a.schema = "django"
        a.date = request.POST.get("date")
        a.published = PUBLISHED_ARTICLE
        a.summary = request.POST.get("summary")
        a.allow_comments = True

        # Open the template
        file_path = os.path.join(SITE_ROOT, "tools", "data", "youtube_template.html")
        with open(file_path) as fh:
            template = fh.read()

        # Process the description
        final_desc = request.POST.get("desc")
        final_desc = final_desc[:final_desc.find("Download:")]
        final_desc = "
" + final_desc.replace("\r\n", "
\n
")
        final_desc = final_desc[:-3]
        final_desc = final_desc.replace("

", "")

        a.content = template.format(video_id=data["video_id"], desc=final_desc)

        a.save()
        data["article_pk"] = str(a.id)

        # Upload the preview
        if request.FILES.get("preview"):
            # Save the file to the uploaded folder
            file_path = os.path.join(SITE_ROOT, "museum_site", "static", "images", "articles", "streams", (str(a.id) + ".png"))
            with open(file_path, 'wb+') as fh:
                for chunk in request.FILES["preview"].chunks():
                    fh.write(chunk)

            a.preview = "articles/streams/" + str(a.id) + ".png"
            a.save()

        # Associate the article with the relevant file
        data["file"].articles.add(a)
        data["file"].save()

    return render(request, "museum_site/tools/add_livestream.html", data)

@staff_member_required
def audit_zeta_config(request):
    data = {
        "title": "Zeta Config Audit",
    }
    data["special"] = File.objects.filter(details__in=[DETAIL_ZZT, DETAIL_SZZT]).exclude(zeta_config_id__in=[1, 4]).order_by("zeta_config")
    return render(request, "museum_site/tools/audit_zeta_config.html", data)

@staff_member_required
def calculate(request, field, pk):
    f = File.objects.get(pk=pk)
    data = {
        "title": "Calculate " + field.title(),
    }
    return render(request, "museum_site/tools/calculate.html", data)

@staff_member_required
def mirror(request, pk):
    """ Returns page to publish file on Archive.org """
    f = File.objects.get(pk=pk)
    data = {
        "title": "Archive.org Mirror",
        "file": f,
        "ret": None,
        "packages": PACKAGE_PROFILES,
        "collection": ARCHIVE_COLLECTION,
    }

    package = int(request.GET.get("package", 0))
    data["package"] = PACKAGE_PROFILES[package]
    data["split"] = math.ceil(len(data["packages"]) // 2)

    zip_file = zipfile.ZipFile(os.path.join(SITE_ROOT, f.download_url()[1:]))

    file_list = zip_file.namelist()
    file_list.sort(key=str.lower)
    data["file_list"] = file_list

    # Mirror the file
    if request.POST.get("mirror"):
        if request.POST.get("package") != "NONE":
            package = PACKAGE_PROFILES[int(request.POST.get("package", 0))]

            # Advanced settings
            if request.POST.get("upload_name"):
                upload_name = request.POST["upload_name"]
                zip_name = upload_name + ".zip"
            else:
                zip_name = package["prefix"] + f.filename
                upload_name = zip_name[:-4]

            # Copy the base package zip
            shutil.copy(
                SITE_ROOT + f.download_url(),
                os.path.join(TEMP_PATH, zip_name)
            )

        # Handle alternative Zip upload
        if request.FILES.get("alt_src"):
            with open(os.path.join(TEMP_PATH, zip_name), "wb") as fh:
                fh.write(request.FILES["alt_src"].read())

        temp_zip = os.path.join(TEMP_PATH, zip_name)

        # Open the WIP zip
        with ZipFile(temp_zip, "a") as z:
            # Insert the base files
            to_add = glob.glob(
                os.path.join(BASE_PATH, package["directory"], "*")
            )
            for a in to_add:
                z.write(a, arcname=os.path.basename(a))

            # Create ZZT.CFG if needed
            if package.get("use_cfg"):
                config_content = request.POST.get("launch")[:-4].upper()  # Remove .ZZT extension
                if package["registered"]:
                    config_content += "\r\nREGISTERED"
                z.writestr("ZZT.CFG", config_content)

        # Create description
        description = "{}\n\n{}".format(package["auto_desc"], request.POST.get("description", ""))

        # Determine the launch command
        if request.POST.get("alt_launch"):
            launch_command = request.POST["alt_launch"]
        else:
            launch_command = package["executable"] + " " + request.POST.get("launch", "").upper()

        # Zip file is completed, prepare the upload
        meta = {
            "title": request.POST.get("title"),
            "mediatype": "software",
            "collection": ARCHIVE_COLLECTION,
            "emulator": "dosbox",
            "emulator_ext": "zip",
            "emulator_start": launch_command,
            "year": str(f.release_date)[:4],
            "subject": [package["engine"]] + f.genre.split("/"),
            "creator": f.author.split("/"),
            "description": description
        }

        if DEBUG:
            upload_name = "test-" + upload_name

        print("I'm gonna upload:", os.path.join(TEMP_PATH, zip_name))
        file_path = os.path.join(TEMP_PATH, zip_name)

        r = upload(
            upload_name,
            files=[file_path],
            metadata=meta,
            access_key=IA_ACCESS,
            secret_key=IA_SECRET,
        )

        if r[0].status_code == 200:
            data["status"] = "SUCCESS"
            f.archive_name = upload_name
            f.save()
            os.remove(os.path.join(TEMP_PATH, zip_name))
        else:
            data["status"] = "FAILURE"

        data["archive_resp"] = r

    return render(request, "museum_site/tools/mirror.html", data)

@staff_member_required
def publish(request, pk):
    """ Returns page to publish a file marked as uploaded """
    data = {
        "title": "Publish",
        "file": File.objects.get(pk=pk),
        "file_list": []
    }

    if request.POST.get("publish"):
        # Move the file
        src = SITE_ROOT + data["file"].download_url()
        dst = SITE_ROOT + "/zgames/" + data["file"].letter + "/" + data["file"].filename
        shutil.move(src, dst)

        # Adjust the details
        data["file"].details.remove(Detail.objects.get(pk=DETAIL_UPLOADED))
        for detail in request.POST.getlist("details"):
            data["file"].details.add(Detail.objects.get(pk=detail))

        # Save
        data["file"].publish_date = datetime.now()
        data["file"].save()

        # Redirect
        return redirect("tool_list", pk=pk)

    with ZipFile(SITE_ROOT + data["file"].download_url(), "r") as zf:
        data["file_list"] = zf.namelist()
    data["file_list"].sort()
    data["extension_list"] = []

    for f in data["file_list"]:
        data["extension_list"].append(f.split(".")[-1].upper())

    return render(request, "museum_site/tools/publish.html", data)

@staff_member_required
def replace_zip(request, pk):
    """ Returns page with latest Museum scan results"""
    data = {"title": "Replace Zip"}
    data["file"] = File.objects.get(pk=pk)

    # Original file info
    data["stat"] = os.stat(data["file"].phys_path())
    data["mtime"] = datetime.fromtimestamp(data["stat"].st_mtime)
    data["file_user"] = pwd.getpwuid(data["stat"].st_uid)
    data["file_group"] = grp.getgrgid(data["stat"].st_gid)

    if request.POST.get("action") == "replace-zip":
        file_path = data["file"].phys_path()
        print(request.FILES)
        with open(file_path, 'wb+') as fh:
            for chunk in request.FILES["replacement"].chunks():
                fh.write(chunk)

        data["new_file"] = File.objects.get(pk=pk)

        # Update checksum
        if request.POST.get("update-checksum"):
            data["new_file"].calculate_checksum()
        if request.POST.get("update-board-count"):
            data["new_file"].calculate_boards()
        if request.POST.get("update-size"):
            data["new_file"].calculate_size()
        data["new_file"].save()

        data["new_stat"] = os.stat(data["file"].phys_path())
        data["new_mtime"] = datetime.fromtimestamp(data["stat"].st_mtime)
        data["new_file_user"] = pwd.getpwuid(data["stat"].st_uid)
        data["new_file_group"] = grp.getgrgid(data["stat"].st_gid)

    return render(request, "museum_site/tools/replace_zip.html", data)

@staff_member_required
def scan(request):
    """ Returns page with latest Museum scan results"""
    data = {"title": "Museum Scan"}
    try:
        with codecs.open(os.path.join(STATIC_PATH, "data", "scan.log"), "r", "utf-8") as fh:
            data["scan"] = fh.read()
    except FileNotFoundError:
        data["scan"] = ""
    return render(request, "museum_site/tools/scan.html", data)

@staff_member_required
def tool_index(request):
    data = {"title": "Tool Index"}
    data["file"] = {"id": 1}
    return render(request, "museum_site/tools/tool_index.html", data)

@staff_member_required
def tool_list(request, pk):
    """ Returns page listing tools available for a file and file information """
    data = {
        "title": "Tools",
        "file": File.objects.get(pk=pk)
    }

    # Simple validation tools
    data["valid_letter"] = True if data["file"].letter in LETTERS else False
    data["valid_filename"] = True if data["file"].phys_path() else False

    if request.GET.get("recalculate"):
        field = request.GET["recalculate"]
        if field == "sort-title":
            data["file"].sorted_title()
            data["new"] = data["file"].sort_title
        elif field == "size":
            data["file"].calculate_size()
            data["new"] =  data["file"].size
        elif field == "reviews":
            data["file"].calculate_reviews()
            data["new"] =  "{}/{}".format(data["file"].review_count, data["file"].rating)
        elif field == "articles":
            data["file"].calculate_article_count()
            data["new"] =  data["file"].article_count
        elif field == "checksum":
            data["file"].calculate_checksum()
            data["new"] = data["file"].checksum
        elif field == "boards":
            data["file"].calculate_boards()
            data["new"] = "{}/{}".format(data["file"].playable_boards, data["file"].total_boards)

        data["file"].basic_save()

    return render(request, "museum_site/tools/list.html", data)

@staff_member_required
def set_screenshot(request, pk):
    """ Returns page to generate and set a file's screenshot """
    data = {
        "title": "Set Screenshot",
    }
    file = File.objects.get(pk=pk)
    data["file"] = file
    data["file_list"] = []

    if not HAS_ZOOKEEPER:
        return HttpResponse("Zookeeper library not found.")

    with ZipFile(SITE_ROOT + file.download_url(), "r") as zf:
        all_files = zf.namelist()
        for f in all_files:
            if f.lower().endswith(".zzt"):
                data["file_list"].append(f)
    data["file_list"].sort()

    if request.GET.get("file"):
        with ZipFile(SITE_ROOT + file.download_url(), "r") as zf:
            zf.extract(request.GET["file"], path=SITE_ROOT + "/museum_site/static/data/")

        z = zookeeper.Zookeeper(SITE_ROOT + "/museum_site/static/data/" + request.GET["file"])
        data["board_list"] = []
        for board in z.boards:
            print(board.title)
            data["board_list"].append(board.title)

    if request.GET.get("board"):
        data["board_num"] = int(request.GET["board"])

        if data["board_num"] != 0:
            z.boards[data["board_num"]].screenshot(SITE_ROOT + "/museum_site/static/data/temp")
        else:
            z.boards[data["board_num"]].screenshot(SITE_ROOT + "/museum_site/static/data/temp", title_screen=True)
        data["show_preview"] = True

    if request.POST.get("save"):
        src = SITE_ROOT + "/museum_site/static/data/temp.png"
        dst = SITE_ROOT + "/museum_site/static/images/screenshots/" + file.letter + "/" + file.filename[:-4] + ".png"
        shutil.copyfile(src, dst)

        file.screenshot = file.filename[:-4] + ".png"
        file.save()
    elif request.POST.get("b64img"):
        raw = request.POST.get("b64img").replace("data:image/png;base64,", "", 1)
        from PIL import Image
        from io import BytesIO
        import base64

        image = Image.open(BytesIO(base64.b64decode(raw)))
        image = image.crop((0, 0, 480, 350))
        image.save(SITE_ROOT + "/museum_site/static/images/screenshots/" + file.letter + "/" + file.filename[:-4] + ".png")

    if os.path.isfile(SITE_ROOT + "/museum_site/static/data/" + request.GET.get("file", "")):
        os.remove(SITE_ROOT + "/museum_site/static/data/" + request.GET["file"])
    return render(request, "museum_site/tools/set_screenshot.html", data)
<commit_msg>Record error message when archive.org upload fails<commit_after>import codecs
import grp
import os
import pwd
import shutil

from django.contrib.admin.views.decorators import staff_member_required
from django.shortcuts import render
from .common import *
from .constants import *
from zipfile import ZipFile
from .models import *

from internetarchive import upload

try:
    import zookeeper
    HAS_ZOOKEEPER = True
except ImportError:
    HAS_ZOOKEEPER = False

@staff_member_required
def add_livestream(request, pk):
    """ Returns page to add a livestream VOD article """
    data = {
        "title": "Tools",
        "file": File.objects.get(pk=pk),
        "today": str(datetime.now())[:10]
    }
    print(SITE_ROOT)

    if request.POST.get("action"):
        if request.POST.get("pk"):
            a = Article.objects.get(pk=int(request.POST["pk"]))
        else:
            a = Article()

        # Convert video URL
        url = request.POST.get("url")
        if request.POST.get("url").startswith("http"):
            url = url.replace("https://youtu.be/", "")
            url = url.replace("https://www.youtube.com/watch?v=", "")
            url = url.replace("https://studio.youtube.com/video/", "")
            url = url.replace("/edit", "")
            if "&" in url:
                url = url[:url.find("&")]
        data["video_id"] = url

        a.title = "Livestream - " + request.POST.get("title")
        a.author = request.POST.get("author")
        a.category = "Livestream"
        a.schema = "django"
        a.date = request.POST.get("date")
        a.published = PUBLISHED_ARTICLE
        a.summary = request.POST.get("summary")
        a.allow_comments = True

        # Open the template
        file_path = os.path.join(SITE_ROOT, "tools", "data", "youtube_template.html")
        with open(file_path) as fh:
            template = fh.read()

        # Process the description
        final_desc = request.POST.get("desc")
        final_desc = final_desc[:final_desc.find("Download:")]
        final_desc = "
" + final_desc.replace("\r\n", "
\n
")
        final_desc = final_desc[:-3]
        final_desc = final_desc.replace("

", "")

        a.content = template.format(video_id=data["video_id"], desc=final_desc)

        a.save()
        data["article_pk"] = str(a.id)

        # Upload the preview
        if request.FILES.get("preview"):
            # Save the file to the uploaded folder
            file_path = os.path.join(SITE_ROOT, "museum_site", "static", "images", "articles", "streams", (str(a.id) + ".png"))
            with open(file_path, 'wb+') as fh:
                for chunk in request.FILES["preview"].chunks():
                    fh.write(chunk)

            a.preview = "articles/streams/" + str(a.id) + ".png"
            a.save()

        # Associate the article with the relevant file
        data["file"].articles.add(a)
        data["file"].save()

    return render(request, "museum_site/tools/add_livestream.html", data)

@staff_member_required
def audit_zeta_config(request):
    data = {
        "title": "Zeta Config Audit",
    }
    data["special"] = File.objects.filter(details__in=[DETAIL_ZZT, DETAIL_SZZT]).exclude(zeta_config_id__in=[1, 4]).order_by("zeta_config")
    return render(request, "museum_site/tools/audit_zeta_config.html", data)

@staff_member_required
def calculate(request, field, pk):
    f = File.objects.get(pk=pk)
    data = {
        "title": "Calculate " + field.title(),
    }
    return render(request, "museum_site/tools/calculate.html", data)

@staff_member_required
def mirror(request, pk):
    """ Returns page to publish file on Archive.org """
    f = File.objects.get(pk=pk)
    data = {
        "title": "Archive.org Mirror",
        "file": f,
        "ret": None,
        "packages": PACKAGE_PROFILES,
        "collection": ARCHIVE_COLLECTION,
    }

    package = int(request.GET.get("package", 0))
    data["package"] = PACKAGE_PROFILES[package]
    data["split"] = math.ceil(len(data["packages"]) // 2)

    zip_file = zipfile.ZipFile(os.path.join(SITE_ROOT, f.download_url()[1:]))

    file_list = zip_file.namelist()
    file_list.sort(key=str.lower)
    data["file_list"] = file_list

    # Mirror the file
    if request.POST.get("mirror"):
        if request.POST.get("package") != "NONE":
            package = PACKAGE_PROFILES[int(request.POST.get("package", 0))]

            # Advanced settings
            if request.POST.get("upload_name"):
                upload_name = request.POST["upload_name"]
                zip_name = upload_name + ".zip"
            else:
                zip_name = package["prefix"] + f.filename
                upload_name = zip_name[:-4]

            # Copy the base package zip
            shutil.copy(
                SITE_ROOT + f.download_url(),
                os.path.join(TEMP_PATH, zip_name)
            )

        # Handle alternative Zip upload
        if request.FILES.get("alt_src"):
            with open(os.path.join(TEMP_PATH, zip_name), "wb") as fh:
                fh.write(request.FILES["alt_src"].read())

        temp_zip = os.path.join(TEMP_PATH, zip_name)

        # Open the WIP zip
        with ZipFile(temp_zip, "a") as z:
            # Insert the base files
            to_add = glob.glob(
                os.path.join(BASE_PATH, package["directory"], "*")
            )
            for a in to_add:
                z.write(a, arcname=os.path.basename(a))

            # Create ZZT.CFG if needed
            if package.get("use_cfg"):
                config_content = request.POST.get("launch")[:-4].upper()  # Remove .ZZT extension
                if package["registered"]:
                    config_content += "\r\nREGISTERED"
                z.writestr("ZZT.CFG", config_content)

        # Create description
        description = "{}\n\n{}".format(package["auto_desc"], request.POST.get("description", ""))

        # Determine the launch command
        if request.POST.get("alt_launch"):
            launch_command = request.POST["alt_launch"]
        else:
            launch_command = package["executable"] + " " + request.POST.get("launch", "").upper()

        # Zip file is completed, prepare the upload
        meta = {
            "title": request.POST.get("title"),
            "mediatype": "software",
            "collection": ARCHIVE_COLLECTION,
            "emulator": "dosbox",
            "emulator_ext": "zip",
            "emulator_start": launch_command,
            "year": str(f.release_date)[:4],
            "subject": [package["engine"]] + f.genre.split("/"),
            "creator": f.author.split("/"),
            "description": description
        }

        if DEBUG:
            upload_name = "test-" + upload_name

        file_path = os.path.join(TEMP_PATH, zip_name)

        try:
            r = upload(
                upload_name,
                files=[file_path],
                metadata=meta,
                access_key=IA_ACCESS,
                secret_key=IA_SECRET,
            )
        except HTTPError as e:
            data["status"] = "FAILURE"
            data["archive_resp"] = r

        if r[0].status_code == 200:
            data["status"] = "SUCCESS"
            f.archive_name = upload_name
            f.save()
            os.remove(os.path.join(TEMP_PATH, zip_name))
        else:
            data["status"] = "FAILURE"

        data["archive_resp"] = r

    return render(request, "museum_site/tools/mirror.html", data)

@staff_member_required
def publish(request, pk):
    """ Returns page to publish a file marked as uploaded """
    data = {
        "title": "Publish",
        "file": File.objects.get(pk=pk),
        "file_list": []
    }

    if request.POST.get("publish"):
        # Move the file
        src = SITE_ROOT + data["file"].download_url()
        dst = SITE_ROOT + "/zgames/" + data["file"].letter + "/" + data["file"].filename
        shutil.move(src, dst)

        # Adjust the details
        data["file"].details.remove(Detail.objects.get(pk=DETAIL_UPLOADED))
        for detail in request.POST.getlist("details"):
            data["file"].details.add(Detail.objects.get(pk=detail))

        # Save
        data["file"].publish_date = datetime.now()
        data["file"].save()

        # Redirect
        return redirect("tool_list", pk=pk)

    with ZipFile(SITE_ROOT + data["file"].download_url(), "r") as zf:
        data["file_list"] = zf.namelist()
    data["file_list"].sort()
    data["extension_list"] = []

    for f in data["file_list"]:
        data["extension_list"].append(f.split(".")[-1].upper())

    return render(request, "museum_site/tools/publish.html", data)

@staff_member_required
def replace_zip(request, pk):
    """ Returns page with latest Museum scan results"""
    data = {"title": "Replace Zip"}
    data["file"] = File.objects.get(pk=pk)

    # Original file info
    data["stat"] = os.stat(data["file"].phys_path())
    data["mtime"] = datetime.fromtimestamp(data["stat"].st_mtime)
    data["file_user"] = pwd.getpwuid(data["stat"].st_uid)
    data["file_group"] = grp.getgrgid(data["stat"].st_gid)

    if request.POST.get("action") == "replace-zip":
        file_path = data["file"].phys_path()
        print(request.FILES)
        with open(file_path, 'wb+') as fh:
            for chunk in request.FILES["replacement"].chunks():
                fh.write(chunk)

        data["new_file"] = File.objects.get(pk=pk)

        # Update checksum
        if request.POST.get("update-checksum"):
            data["new_file"].calculate_checksum()
        if request.POST.get("update-board-count"):
            data["new_file"].calculate_boards()
        if request.POST.get("update-size"):
            data["new_file"].calculate_size()
        data["new_file"].save()

        data["new_stat"] = os.stat(data["file"].phys_path())
        data["new_mtime"] = datetime.fromtimestamp(data["stat"].st_mtime)
        data["new_file_user"] = pwd.getpwuid(data["stat"].st_uid)
        data["new_file_group"] = grp.getgrgid(data["stat"].st_gid)

    return render(request, "museum_site/tools/replace_zip.html", data)

@staff_member_required
def scan(request):
    """ Returns page with latest Museum scan results"""
    data = {"title": "Museum Scan"}
    try:
        with codecs.open(os.path.join(STATIC_PATH, "data", "scan.log"), "r", "utf-8") as fh:
            data["scan"] = fh.read()
    except FileNotFoundError:
        data["scan"] = ""
    return render(request, "museum_site/tools/scan.html", data)

@staff_member_required
def tool_index(request):
    data = {"title": "Tool Index"}
    data["file"] = {"id": 1}
    return render(request, "museum_site/tools/tool_index.html", data)

@staff_member_required
def tool_list(request, pk):
    """ Returns page listing tools available for a file and file information """
    data = {
        "title": "Tools",
        "file": File.objects.get(pk=pk)
    }

    # Simple validation tools
    data["valid_letter"] = True if data["file"].letter in LETTERS else False
    data["valid_filename"] = True if data["file"].phys_path() else False

    if request.GET.get("recalculate"):
        field = request.GET["recalculate"]
        if field == "sort-title":
            data["file"].sorted_title()
            data["new"] = data["file"].sort_title
        elif field == "size":
            data["file"].calculate_size()
            data["new"] =  data["file"].size
        elif field == "reviews":
            data["file"].calculate_reviews()
            data["new"] =  "{}/{}".format(data["file"].review_count, data["file"].rating)
        elif field == "articles":
            data["file"].calculate_article_count()
            data["new"] =  data["file"].article_count
        elif field == "checksum":
            data["file"].calculate_checksum()
            data["new"] = data["file"].checksum
        elif field == "boards":
            data["file"].calculate_boards()
            data["new"] = "{}/{}".format(data["file"].playable_boards, data["file"].total_boards)

        data["file"].basic_save()

    return render(request, "museum_site/tools/list.html", data)

@staff_member_required
def set_screenshot(request, pk):
    """ Returns page to generate and set a file's screenshot """
    data = {
        "title": "Set Screenshot",
    }
    file = File.objects.get(pk=pk)
    data["file"] = file
    data["file_list"] = []

    if not HAS_ZOOKEEPER:
        return HttpResponse("Zookeeper library not found.")

    with ZipFile(SITE_ROOT + file.download_url(), "r") as zf:
        all_files = zf.namelist()
        for f in all_files:
            if f.lower().endswith(".zzt"):
                data["file_list"].append(f)
    data["file_list"].sort()

    if request.GET.get("file"):
        with ZipFile(SITE_ROOT + file.download_url(), "r") as zf:
            zf.extract(request.GET["file"], path=SITE_ROOT + "/museum_site/static/data/")

        z = zookeeper.Zookeeper(SITE_ROOT + "/museum_site/static/data/" + request.GET["file"])
        data["board_list"] = []
        for board in z.boards:
            print(board.title)
            data["board_list"].append(board.title)

    if request.GET.get("board"):
        data["board_num"] = int(request.GET["board"])

        if data["board_num"] != 0:
            z.boards[data["board_num"]].screenshot(SITE_ROOT + "/museum_site/static/data/temp")
        else:
            z.boards[data["board_num"]].screenshot(SITE_ROOT + "/museum_site/static/data/temp", title_screen=True)
        data["show_preview"] = True

    if request.POST.get("save"):
        src = SITE_ROOT + "/museum_site/static/data/temp.png"
        dst = SITE_ROOT + "/museum_site/static/images/screenshots/" + file.letter + "/" + file.filename[:-4] + ".png"
        shutil.copyfile(src, dst)

        file.screenshot = file.filename[:-4] + ".png"
        file.save()
    elif request.POST.get("b64img"):
        raw = request.POST.get("b64img").replace("data:image/png;base64,", "", 1)
        from PIL import Image
        from io import BytesIO
        import base64

        image = Image.open(BytesIO(base64.b64decode(raw)))
        image = image.crop((0, 0, 480, 350))
        image.save(SITE_ROOT + "/museum_site/static/images/screenshots/" + file.letter + "/" + file.filename[:-4] + ".png")

    if os.path.isfile(SITE_ROOT + "/museum_site/static/data/" + request.GET.get("file", "")):
        os.remove(SITE_ROOT + "/museum_site/static/data/" + request.GET["file"])
    return render(request, "museum_site/tools/set_screenshot.html", data)
<|endoftext|>


<commit_before>import ctypes

stringtools = ctypes.CDLL("../target/debug/libstringtools-4149b7695a4035ac.so")
print(stringtools.count_substrings(b"banana", b"na"))
<commit_msg>Don't hardcore library name in Python example.<commit_after>import ctypes
import glob
import sys

try:
    library_name = glob.glob("../target/debug/libstringtools-*.so")[0]
except IndexError:
    print("Couldn't find dynamic library")
    sys.exit(1)
else:
    stringtools = ctypes.CDLL(library_name)
    print(stringtools.count_substrings(b"banana", b"na"))
<|endoftext|>