<commit_before>DOMAIN_INDEX="hqdomains_3bda1774c383ce88b8d0d00cdd331592"
DOMAIN_MAPPING={'date_formats': ['yyyy-MM-dd', "yyyy-MM-dd'T'HH:mm:ssZZ", "yyyy-MM-dd'T'HH:mm:ss.SSSSSS", "yyyy-MM-dd'T'HH:mm:ss.SSSSSS'Z'", "yyyy-MM-dd'T'HH:mm:ss'Z'", "yyyy-MM-dd'T'HH:mm:ssZ", "yyyy-MM-dd'T'HH:mm:ssZZ'Z'", "yyyy-MM-dd'T'HH:mm:ss.SSSZZ", "yyyy-MM-dd'T'HH:mm:ss", "yyyy-MM-dd' 'HH:mm:ss", "yyyy-MM-dd' 'HH:mm:ss.SSSSSS", "mm/dd/yy' 'HH:mm:ss"], 'dynamic': False, '_meta': {'comment': 'Autogenerated [corehq.apps.domain.models.Domain] mapping from ptop_generate_mapping 07/11/2013', 'created': None}, 'date_detection': False, 'properties': {'cda': {'dynamic': False, 'type': 'object', 'properties': {'doc_type': {'index': 'not_analyzed', 'type': 'string'}, 'user_id': {'type': 'string'}, 'signed': {'type': 'boolean'}, 'user_ip': {'type': 'string'}, 'date': {'type': 'date', 'format': "yyyy-MM-dd||yyyy-MM-dd'T'HH:mm:ssZZ||yyyy-MM-dd'T'HH:mm:ss.SSSSSS||yyyy-MM-dd'T'HH:mm:ss.SSSSSS'Z'||yyyy-MM-dd'T'HH:mm:ss'Z'||yyyy-MM-dd'T'HH:mm:ssZ||yyyy-MM-dd'T'HH:mm:ssZZ'Z'||yyyy-MM-dd'T'HH:mm:ss.SSSZZ||yyyy-MM-dd'T'HH:mm:ss||yyyy-MM-dd' 'HH:mm:ss||yyyy-MM-dd' 'HH:mm:ss.SSSSSS||mm/dd/yy' 'HH:mm:ss"}, 'type': {'index': 'not_analyzed', 'type': 'string'}}}, 'sub_area': {'type': 'string'}, 'cp_n_inactive_cases': {'type': 'long'}, 'cp_n_cases': {'type': 'long'}, 'cp_n_active_cases': {'type': 'long'}, 'default_sms_backend_id': {'type': 'string'}, 'sms_case_registration_owner_id': {'type': 'string'}, 'multimedia_included': {'type': 'boolean'}, 'phone_model': {'type': 'string'}, 'copy_history': {'type': 'string'}, 'title': {'fields': {'exact': {'index': 'not_analyzed', 'type': 'string'}, 'title': {'index': 'analyzed', 'type': 'string'}}, 'type': 'multi_field'}, 'image_type': {'type': 'string'}, 'description': {'type': 'string'}, 'name': {'fields': {'exact': {'index': 'not_analyzed', 'type': 'string'}, 'name': {'index': 'analyzed', 'type': 'string'}}, 'type': 'multi_field'}, 'cp_n_cc_users': {'type': 'long'}, 'is_snapshot': {'type': 'boolean'}, 'sms_case_registration_user_id': {'type': 'string'}, 'cp_n_web_users': {'type': 'long'}, 'is_shared': {'type': 'boolean'}, 'billing_address': {'dynamic': False, 'type': 'object', 'properties': {'state_province': {'type': 'string'}, 'city': {'type': 'string'}, 'name': {'type': 'string'}, 'doc_type': {'index': 'not_analyzed', 'type': 'string'}, 'country': {'type': 'string'}, 'postal_code': {'type': 'string'}}}, 'is_active': {'type': 'boolean'}, 'default_timezone': {'type': 'string'}, 'date_created': {'type': 'date', 'format': "yyyy-MM-dd||yyyy-MM-dd'T'HH:mm:ssZZ||yyyy-MM-dd'T'HH:mm:ss.SSSSSS||yyyy-MM-dd'T'HH:mm:ss.SSSSSS'Z'||yyyy-MM-dd'T'HH:mm:ss'Z'||yyyy-MM-dd'T'HH:mm:ssZ||yyyy-MM-dd'T'HH:mm:ssZZ'Z'||yyyy-MM-dd'T'HH:mm:ss.SSSZZ||yyyy-MM-dd'T'HH:mm:ss||yyyy-MM-dd' 'HH:mm:ss||yyyy-MM-dd' 'HH:mm:ss.SSSSSS||mm/dd/yy' 'HH:mm:ss"}, 'deployment': {'dynamic': False, 'type': 'object', 'properties': {'city': {'fields': {'city': {'index': 'analyzed', 'type': 'string'}, 'exact': {'index': 'not_analyzed', 'type': 'string'}}, 'type': 'multi_field'}, 'description': {'fields': {'exact': {'index': 'not_analyzed', 'type': 'string'}, 'description': {'index': 'analyzed', 'type': 'string'}}, 'type': 'multi_field'}, 'doc_type': {'index': 'not_analyzed', 'type': 'string'}, 'country': {'fields': {'country': {'index': 'analyzed', 'type': 'string'}, 'exact': {'index': 'not_analyzed', 'type': 'string'}}, 'type': 'multi_field'}, 'region': {'fields': {'region': {'index': 'analyzed', 'type': 'string'}, 'exact': {'index': 'not_analyzed', 'type': 'string'}}, 'type': 'multi_field'}, 'date': {'type': 'date', 'format': "yyyy-MM-dd||yyyy-MM-dd'T'HH:mm:ssZZ||yyyy-MM-dd'T'HH:mm:ss.SSSSSS||yyyy-MM-dd'T'HH:mm:ss.SSSSSS'Z'||yyyy-MM-dd'T'HH:mm:ss'Z'||yyyy-MM-dd'T'HH:mm:ssZ||yyyy-MM-dd'T'HH:mm:ssZZ'Z'||yyyy-MM-dd'T'HH:mm:ss.SSSZZ||yyyy-MM-dd'T'HH:mm:ss||yyyy-MM-dd' 'HH:mm:ss||yyyy-MM-dd' 'HH:mm:ss.SSSSSS||mm/dd/yy' 'HH:mm:ss"}, 'public': {'type': 'boolean'}}}, 'is_public': {'type': 'boolean'}, 'cp_n_60_day_cases': {'type': 'long'}, 'doc_type': {'index': 'not_analyzed', 'type': 'string'}, 'creating_user': {'type': 'string'}, 'is_sms_billable': {'type': 'boolean'}, 'case_display': {'dynamic': False, 'type': 'object', 'properties': {'doc_type': {'index': 'not_analyzed', 'type': 'string'}}}, 'yt_id': {'type': 'string'}, 'cp_n_active_cc_users': {'type': 'long'}, 'customer_type': {'type': 'string'}, 'case_sharing': {'type': 'boolean'}, 'currency_code': {'type': 'string'}, 'hr_name': {'type': 'string'}, 'cp_first_form': {'type': 'date', 'format': "yyyy-MM-dd||yyyy-MM-dd'T'HH:mm:ssZZ||yyyy-MM-dd'T'HH:mm:ss.SSSSSS||yyyy-MM-dd'T'HH:mm:ss.SSSSSS'Z'||yyyy-MM-dd'T'HH:mm:ss'Z'||yyyy-MM-dd'T'HH:mm:ssZ||yyyy-MM-dd'T'HH:mm:ssZZ'Z'||yyyy-MM-dd'T'HH:mm:ss.SSSZZ||yyyy-MM-dd'T'HH:mm:ss||yyyy-MM-dd' 'HH:mm:ss||yyyy-MM-dd' 'HH:mm:ss.SSSSSS||mm/dd/yy' 'HH:mm:ss"}, 'eula': {'dynamic': False, 'type': 'object', 'properties': {'doc_type': {'index': 'not_analyzed', 'type': 'string'}, 'user_id': {'type': 'string'}, 'signed': {'type': 'boolean'}, 'user_ip': {'type': 'string'}, 'date': {'type': 'date', 'format': "yyyy-MM-dd||yyyy-MM-dd'T'HH:mm:ssZZ||yyyy-MM-dd'T'HH:mm:ss.SSSSSS||yyyy-MM-dd'T'HH:mm:ss.SSSSSS'Z'||yyyy-MM-dd'T'HH:mm:ss'Z'||yyyy-MM-dd'T'HH:mm:ssZ||yyyy-MM-dd'T'HH:mm:ssZZ'Z'||yyyy-MM-dd'T'HH:mm:ss.SSSZZ||yyyy-MM-dd'T'HH:mm:ss||yyyy-MM-dd' 'HH:mm:ss||yyyy-MM-dd' 'HH:mm:ss.SSSSSS||mm/dd/yy' 'HH:mm:ss"}, 'type': {'type': 'string'}}}, 'project_type': {'type': 'string', 'analyzer': 'comma'}, 'restrict_superusers': {'type': 'boolean'}, 'is_test': {'type': 'boolean'}, 'cp_last_form': {'type': 'date', 'format': "yyyy-MM-dd||yyyy-MM-dd'T'HH:mm:ssZZ||yyyy-MM-dd'T'HH:mm:ss.SSSSSS||yyyy-MM-dd'T'HH:mm:ss.SSSSSS'Z'||yyyy-MM-dd'T'HH:mm:ss'Z'||yyyy-MM-dd'T'HH:mm:ssZ||yyyy-MM-dd'T'HH:mm:ssZZ'Z'||yyyy-MM-dd'T'HH:mm:ss.SSSZZ||yyyy-MM-dd'T'HH:mm:ss||yyyy-MM-dd' 'HH:mm:ss||yyyy-MM-dd' 'HH:mm:ss.SSSSSS||mm/dd/yy' 'HH:mm:ss"}, 'image_path': {'type': 'string'}, 'sms_case_registration_enabled': {'type': 'boolean'}, 'is_approved': {'type': 'boolean'}, 'author': {'fields': {'exact': {'index': 'not_analyzed', 'type': 'string'}, 'author': {'index': 'analyzed', 'type': 'string'}}, 'type': 'multi_field'}, 'migrations': {'dynamic': False, 'type': 'object', 'properties': {'doc_type': {'index': 'not_analyzed', 'type': 'string'}, 'has_migrated_permissions': {'type': 'boolean'}}}, 'internal': {'dynamic': False, 'type': 'object', 'properties': {'using_call_center': {'type': 'boolean'}, 'sf_contract_id': {'type': 'string'}, 'self_started': {'type': 'boolean'}, 'can_use_data': {'type': 'boolean'}, 'sub_area': {'fields': {'exact': {'index': 'not_analyzed', 'type': 'string'}, 'sub_area': {'index': 'analyzed', 'type': 'string'}}, 'type': 'multi_field'}, 'area': {'fields': {'exact': {'index': 'not_analyzed', 'type': 'string'}, 'area': {'index': 'analyzed', 'type': 'string'}}, 'type': 'multi_field'}, 'using_adm': {'type': 'boolean'}, 'notes': {'type': 'string'}, 'project_state': {'type': 'string'}, 'organization_name': {'type': 'string'}, 'commcare_edition': {'type': 'string'}, 'custom_eula': {'type': 'boolean'}, 'initiative': {'fields': {'exact': {'index': 'not_analyzed', 'type': 'string'}, 'initiative': {'index': 'analyzed', 'type': 'string'}}, 'type': 'multi_field'}, 'platform': {'type': 'string'}, 'services': {'type': 'string'}, 'doc_type': {'index': 'not_analyzed', 'type': 'string'}, 'sf_account_id': {'type': 'string'}}}, 'short_description': {'fields': {'short_description': {'index': 'analyzed', 'type': 'string'}, 'exact': {'index': 'not_analyzed', 'type': 'string'}}, 'type': 'multi_field'}, 'survey_management_enabled': {'type': 'boolean'}, 'snapshot_time': {'type': 'date', 'format': "yyyy-MM-dd||yyyy-MM-dd'T'HH:mm:ssZZ||yyyy-MM-dd'T'HH:mm:ss.SSSSSS||yyyy-MM-dd'T'HH:mm:ss.SSSSSS'Z'||yyyy-MM-dd'T'HH:mm:ss'Z'||yyyy-MM-dd'T'HH:mm:ssZ||yyyy-MM-dd'T'HH:mm:ssZZ'Z'||yyyy-MM-dd'T'HH:mm:ss.SSSZZ||yyyy-MM-dd'T'HH:mm:ss||yyyy-MM-dd' 'HH:mm:ss||yyyy-MM-dd' 'HH:mm:ss.SSSSSS||mm/dd/yy' 'HH:mm:ss"}, 'tags': {'type': 'string'}, 'downloads': {'type': 'long'}, 'call_center_config': {'dynamic': False, 'type': 'object', 'properties': {'doc_type': {'index': 'not_analyzed', 'type': 'string'}, 'case_type': {'type': 'string'}, 'enabled': {'type': 'boolean'}, 'case_owner_id': {'type': 'string'}}}, 'cp_n_forms': {'type': 'long'}, 'cp_has_app': {'type': 'boolean'}, 'sms_mobile_worker_registration_enabled': {'type': 'boolean'}, 'sms_case_registration_type': {'type': 'string'}, 'publisher': {'type': 'string'}, 'license': {'type': 'string'}, 'billing_number': {'type': 'string'}, 'area': {'type': 'string'}, 'attribution_notes': {'type': 'string'}, 'commtrack_enabled': {'type': 'boolean'}, 'published': {'type': 'boolean'}, 'billable_client': {'type': 'string'}, 'organization': {'type': 'string'}, 'full_downloads': {'type': 'long'}, 'cp_is_active': {'type': 'boolean'}}}<commit_msg>remake domain mapping<commit_after>DOMAIN_INDEX="hqdomains_d81313a0b53830714fab352d590054b8"
DOMAIN_MAPPING={'date_formats': ['yyyy-MM-dd', "yyyy-MM-dd'T'HH:mm:ssZZ", "yyyy-MM-dd'T'HH:mm:ss.SSSSSS", "yyyy-MM-dd'T'HH:mm:ss.SSSSSS'Z'", "yyyy-MM-dd'T'HH:mm:ss'Z'", "yyyy-MM-dd'T'HH:mm:ssZ", "yyyy-MM-dd'T'HH:mm:ssZZ'Z'", "yyyy-MM-dd'T'HH:mm:ss.SSSZZ", "yyyy-MM-dd'T'HH:mm:ss", "yyyy-MM-dd' 'HH:mm:ss", "yyyy-MM-dd' 'HH:mm:ss.SSSSSS", "mm/dd/yy' 'HH:mm:ss"], 'dynamic': False, '_meta': {'comment': 'Autogenerated [corehq.apps.domain.models.Domain] mapping from ptop_generate_mapping 07/12/2013', 'created': None}, 'date_detection': False, 'properties': {'cda': {'dynamic': False, 'type': 'object', 'properties': {'doc_type': {'index': 'not_analyzed', 'type': 'string'}, 'user_id': {'type': 'string'}, 'signed': {'type': 'boolean'}, 'user_ip': {'type': 'string'}, 'date': {'type': 'date', 'format': "yyyy-MM-dd||yyyy-MM-dd'T'HH:mm:ssZZ||yyyy-MM-dd'T'HH:mm:ss.SSSSSS||yyyy-MM-dd'T'HH:mm:ss.SSSSSS'Z'||yyyy-MM-dd'T'HH:mm:ss'Z'||yyyy-MM-dd'T'HH:mm:ssZ||yyyy-MM-dd'T'HH:mm:ssZZ'Z'||yyyy-MM-dd'T'HH:mm:ss.SSSZZ||yyyy-MM-dd'T'HH:mm:ss||yyyy-MM-dd' 'HH:mm:ss||yyyy-MM-dd' 'HH:mm:ss.SSSSSS||mm/dd/yy' 'HH:mm:ss"}, 'type': {'index': 'not_analyzed', 'type': 'string'}}}, 'sub_area': {'type': 'string'}, 'cp_n_inactive_cases': {'type': 'long'}, 'cp_n_cases': {'type': 'long'}, 'cp_n_active_cases': {'type': 'long'}, 'default_sms_backend_id': {'type': 'string'}, 'sms_case_registration_owner_id': {'type': 'string'}, 'multimedia_included': {'type': 'boolean'}, 'phone_model': {'type': 'string'}, 'copy_history': {'type': 'string'}, 'title': {'fields': {'exact': {'index': 'not_analyzed', 'type': 'string'}, 'title': {'index': 'analyzed', 'type': 'string'}}, 'type': 'multi_field'}, 'image_type': {'type': 'string'}, 'description': {'type': 'string'}, 'name': {'fields': {'exact': {'index': 'not_analyzed', 'type': 'string'}, 'name': {'index': 'analyzed', 'type': 'string'}}, 'type': 'multi_field'}, 'cp_n_cc_users': {'type': 'long'}, 'is_snapshot': {'type': 'boolean'}, 'sms_case_registration_user_id': {'type': 'string'}, 'cp_n_web_users': {'type': 'long'}, 'is_shared': {'type': 'boolean'}, 'billing_address': {'dynamic': False, 'type': 'object', 'properties': {'state_province': {'type': 'string'}, 'city': {'type': 'string'}, 'name': {'type': 'string'}, 'doc_type': {'index': 'not_analyzed', 'type': 'string'}, 'country': {'type': 'string'}, 'postal_code': {'type': 'string'}}}, 'is_active': {'type': 'boolean'}, 'default_timezone': {'type': 'string'}, 'date_created': {'type': 'date', 'format': "yyyy-MM-dd||yyyy-MM-dd'T'HH:mm:ssZZ||yyyy-MM-dd'T'HH:mm:ss.SSSSSS||yyyy-MM-dd'T'HH:mm:ss.SSSSSS'Z'||yyyy-MM-dd'T'HH:mm:ss'Z'||yyyy-MM-dd'T'HH:mm:ssZ||yyyy-MM-dd'T'HH:mm:ssZZ'Z'||yyyy-MM-dd'T'HH:mm:ss.SSSZZ||yyyy-MM-dd'T'HH:mm:ss||yyyy-MM-dd' 'HH:mm:ss||yyyy-MM-dd' 'HH:mm:ss.SSSSSS||mm/dd/yy' 'HH:mm:ss"}, 'deployment': {'dynamic': False, 'type': 'object', 'properties': {'city': {'fields': {'city': {'index': 'analyzed', 'type': 'string'}, 'exact': {'index': 'not_analyzed', 'type': 'string'}}, 'type': 'multi_field'}, 'description': {'fields': {'exact': {'index': 'not_analyzed', 'type': 'string'}, 'description': {'index': 'analyzed', 'type': 'string'}}, 'type': 'multi_field'}, 'doc_type': {'index': 'not_analyzed', 'type': 'string'}, 'country': {'fields': {'country': {'index': 'analyzed', 'type': 'string'}, 'exact': {'index': 'not_analyzed', 'type': 'string'}}, 'type': 'multi_field'}, 'region': {'fields': {'region': {'index': 'analyzed', 'type': 'string'}, 'exact': {'index': 'not_analyzed', 'type': 'string'}}, 'type': 'multi_field'}, 'date': {'type': 'date', 'format': "yyyy-MM-dd||yyyy-MM-dd'T'HH:mm:ssZZ||yyyy-MM-dd'T'HH:mm:ss.SSSSSS||yyyy-MM-dd'T'HH:mm:ss.SSSSSS'Z'||yyyy-MM-dd'T'HH:mm:ss'Z'||yyyy-MM-dd'T'HH:mm:ssZ||yyyy-MM-dd'T'HH:mm:ssZZ'Z'||yyyy-MM-dd'T'HH:mm:ss.SSSZZ||yyyy-MM-dd'T'HH:mm:ss||yyyy-MM-dd' 'HH:mm:ss||yyyy-MM-dd' 'HH:mm:ss.SSSSSS||mm/dd/yy' 'HH:mm:ss"}, 'public': {'type': 'boolean'}}}, 'is_public': {'type': 'boolean'}, 'cp_n_60_day_cases': {'type': 'long'}, 'doc_type': {'index': 'not_analyzed', 'type': 'string'}, 'creating_user': {'type': 'string'}, 'is_sms_billable': {'type': 'boolean'}, 'case_display': {'dynamic': False, 'type': 'object', 'properties': {'doc_type': {'index': 'not_analyzed', 'type': 'string'}}}, 'yt_id': {'type': 'string'}, 'cp_n_active_cc_users': {'type': 'long'}, 'customer_type': {'type': 'string'}, 'case_sharing': {'type': 'boolean'}, 'currency_code': {'type': 'string'}, 'hr_name': {'type': 'string'}, 'cp_first_form': {'type': 'date', 'format': "yyyy-MM-dd||yyyy-MM-dd'T'HH:mm:ssZZ||yyyy-MM-dd'T'HH:mm:ss.SSSSSS||yyyy-MM-dd'T'HH:mm:ss.SSSSSS'Z'||yyyy-MM-dd'T'HH:mm:ss'Z'||yyyy-MM-dd'T'HH:mm:ssZ||yyyy-MM-dd'T'HH:mm:ssZZ'Z'||yyyy-MM-dd'T'HH:mm:ss.SSSZZ||yyyy-MM-dd'T'HH:mm:ss||yyyy-MM-dd' 'HH:mm:ss||yyyy-MM-dd' 'HH:mm:ss.SSSSSS||mm/dd/yy' 'HH:mm:ss"}, 'eula': {'dynamic': False, 'type': 'object', 'properties': {'doc_type': {'index': 'not_analyzed', 'type': 'string'}, 'user_id': {'type': 'string'}, 'signed': {'type': 'boolean'}, 'user_ip': {'type': 'string'}, 'date': {'type': 'date', 'format': "yyyy-MM-dd||yyyy-MM-dd'T'HH:mm:ssZZ||yyyy-MM-dd'T'HH:mm:ss.SSSSSS||yyyy-MM-dd'T'HH:mm:ss.SSSSSS'Z'||yyyy-MM-dd'T'HH:mm:ss'Z'||yyyy-MM-dd'T'HH:mm:ssZ||yyyy-MM-dd'T'HH:mm:ssZZ'Z'||yyyy-MM-dd'T'HH:mm:ss.SSSZZ||yyyy-MM-dd'T'HH:mm:ss||yyyy-MM-dd' 'HH:mm:ss||yyyy-MM-dd' 'HH:mm:ss.SSSSSS||mm/dd/yy' 'HH:mm:ss"}, 'type': {'type': 'string'}}}, 'project_type': {'type': 'string', 'analyzer': 'comma'}, 'restrict_superusers': {'type': 'boolean'}, 'is_test': {'type': 'boolean'}, 'cp_last_form': {'type': 'date', 'format': "yyyy-MM-dd||yyyy-MM-dd'T'HH:mm:ssZZ||yyyy-MM-dd'T'HH:mm:ss.SSSSSS||yyyy-MM-dd'T'HH:mm:ss.SSSSSS'Z'||yyyy-MM-dd'T'HH:mm:ss'Z'||yyyy-MM-dd'T'HH:mm:ssZ||yyyy-MM-dd'T'HH:mm:ssZZ'Z'||yyyy-MM-dd'T'HH:mm:ss.SSSZZ||yyyy-MM-dd'T'HH:mm:ss||yyyy-MM-dd' 'HH:mm:ss||yyyy-MM-dd' 'HH:mm:ss.SSSSSS||mm/dd/yy' 'HH:mm:ss"}, 'image_path': {'type': 'string'}, 'sms_case_registration_enabled': {'type': 'boolean'}, 'is_approved': {'type': 'boolean'}, 'author': {'fields': {'exact': {'index': 'not_analyzed', 'type': 'string'}, 'author': {'index': 'analyzed', 'type': 'string'}}, 'type': 'multi_field'}, 'migrations': {'dynamic': False, 'type': 'object', 'properties': {'doc_type': {'index': 'not_analyzed', 'type': 'string'}, 'has_migrated_permissions': {'type': 'boolean'}}}, 'internal': {'dynamic': False, 'type': 'object', 'properties': {'using_call_center': {'type': 'boolean'}, 'sf_contract_id': {'type': 'string'}, 'self_started': {'type': 'boolean'}, 'can_use_data': {'type': 'boolean'}, 'sub_area': {'fields': {'exact': {'index': 'not_analyzed', 'type': 'string'}, 'sub_area': {'index': 'analyzed', 'type': 'string'}}, 'type': 'multi_field'}, 'area': {'fields': {'exact': {'index': 'not_analyzed', 'type': 'string'}, 'area': {'index': 'analyzed', 'type': 'string'}}, 'type': 'multi_field'}, 'using_adm': {'type': 'boolean'}, 'notes': {'type': 'string'}, 'project_state': {'type': 'string'}, 'organization_name': {'type': 'string'}, 'commcare_edition': {'type': 'string'}, 'custom_eula': {'type': 'boolean'}, 'initiative': {'fields': {'exact': {'index': 'not_analyzed', 'type': 'string'}, 'initiative': {'index': 'analyzed', 'type': 'string'}}, 'type': 'multi_field'}, 'platform': {'type': 'string'}, 'services': {'type': 'string'}, 'doc_type': {'index': 'not_analyzed', 'type': 'string'}, 'sf_account_id': {'type': 'string'}}}, 'short_description': {'fields': {'short_description': {'index': 'analyzed', 'type': 'string'}, 'exact': {'index': 'not_analyzed', 'type': 'string'}}, 'type': 'multi_field'}, 'survey_management_enabled': {'type': 'boolean'}, 'snapshot_time': {'type': 'date', 'format': "yyyy-MM-dd||yyyy-MM-dd'T'HH:mm:ssZZ||yyyy-MM-dd'T'HH:mm:ss.SSSSSS||yyyy-MM-dd'T'HH:mm:ss.SSSSSS'Z'||yyyy-MM-dd'T'HH:mm:ss'Z'||yyyy-MM-dd'T'HH:mm:ssZ||yyyy-MM-dd'T'HH:mm:ssZZ'Z'||yyyy-MM-dd'T'HH:mm:ss.SSSZZ||yyyy-MM-dd'T'HH:mm:ss||yyyy-MM-dd' 'HH:mm:ss||yyyy-MM-dd' 'HH:mm:ss.SSSSSS||mm/dd/yy' 'HH:mm:ss"}, 'tags': {'type': 'string'}, 'downloads': {'type': 'long'}, 'call_center_config': {'dynamic': False, 'type': 'object', 'properties': {'doc_type': {'index': 'not_analyzed', 'type': 'string'}, 'case_type': {'type': 'string'}, 'enabled': {'type': 'boolean'}, 'case_owner_id': {'type': 'string'}}}, 'cp_n_forms': {'type': 'long'}, 'cp_has_app': {'type': 'boolean'}, 'sms_mobile_worker_registration_enabled': {'type': 'boolean'}, 'sms_case_registration_type': {'type': 'string'}, 'publisher': {'type': 'string'}, 'license': {'type': 'string'}, 'billing_number': {'type': 'string'}, 'area': {'type': 'string'}, 'attribution_notes': {'type': 'string'}, 'commtrack_enabled': {'type': 'boolean'}, 'published': {'type': 'boolean'}, 'billable_client': {'type': 'string'}, 'organization': {'type': 'string'}, 'full_downloads': {'type': 'long'}, 'cp_is_active': {'type': 'boolean'}}}<|endoftext|>


<commit_before>#!/usr/bin/env python
#
# NumPy version of Rolf Poulsens code for pricing american options.
#
# R-code source: http://www.math.ku.dk/~rolf/FAMOES/UsualBinomAmrPut.R
#
"""
source("UsualBinomAmrPut.R")

S0<-100;r<-0.03; alpha<-0.07; sigma<-0.20;

expiry<-1; strike<-100

n<-expiry*252; dt<-expiry/n
u<-exp(alpha*dt+sigma*sqrt(dt)); d<-exp(alpha*dt-sigma*sqrt(dt))
R<-exp(r*dt)

q<-(R-d)/(u-d)

put<-matrix(0,nrow=(n+1),ncol=(n+1))

put[,n+1]<-pmax(strike-S0*u^(0:n)*d^(n:0),0)

for (i in n:1) {
  for (j in 1:i){
     S<-S0*u^(j-1)* d^(i-j)
     put[j,i]<-max(max(strike-S,0),(q*put[j+1,i+1]+(1-q)*put[j,i+1])/R)
   }
}

print(put[1,1])
"""
import cphvbnumpy as np

S0  = 100
r   = 0.03
alpha   = 0.07
sigma   = 0.20
expiry  = 1
strike  = 1

n   = expiry * 252
dt  = expiry / n
u   = np.exp( alpha*dt + sigma*np.sqrt(dt) )
d   = np.exp( alpha*dt - sigma*np.sqrt(dt) )
R   = np.exp( r*dt )
q   = (R-d)/(u-d)

put = np.array( (n+1, n+1) )
# put[,n+1] = ...

for i in xrange(0, n):
    for j in xrange(1, i):
        S = S0*u*(j-1)*d^(i-j)
        put[j][j] = max(max(strike-S,0), (q*put[j+1,][i+1]+(1-q)*put[j][i+1])/R)

<commit_msg>Numpyfication of american option-pricing code with a off-by-one error somewhere.<commit_after>#!/usr/bin/env python
#
# NumPy version of Rolf Poulsens code for pricing american options.
#
# R-code source: http://www.math.ku.dk/~rolf/FAMOES/UsualBinomAmrPut.R
#
"""
source("UsualBinomAmrPut.R")

S0<-100;r<-0.03; alpha<-0.07; sigma<-0.20;

expiry<-1; strike<-100

n<-expiry*252; dt<-expiry/n
u<-exp(alpha*dt+sigma*sqrt(dt)); d<-exp(alpha*dt-sigma*sqrt(dt))
R<-exp(r*dt)

q<-(R-d)/(u-d)

put<-matrix(0,nrow=(n+1),ncol=(n+1))

put[,n+1]<-pmax(strike-S0*u^(0:n)*d^(n:0),0)

for (i in n:1) {
  for (j in 1:i){
     S<-S0*u^(j-1)* d^(i-j)
     put[j,i]<-max(max(strike-S,0),(q*put[j+1,i+1]+(1-q)*put[j,i+1])/R)
   }
}

print(put[1,1])
"""
import cphvbnumpy as np
import math

S0  = 100.0         # Pricing parameters
r   = 0.03
alpha   = 0.07
sigma   = 0.20
expiry  = 1.0
strike  = 100.0

n   = int(expiry * 252)
dt  = expiry / n
u   = math.exp( alpha*dt + sigma*math.sqrt(dt) )
d   = math.exp( alpha*dt - sigma*math.sqrt(dt) )
R   = math.exp( r*dt )
q   = (R-d)/(u-d)

asc     = np.array( range(0, n+1) )
desc    = np.array( range(n,-1,-1) )

put = np.zeros( (n+1, n+1) )
put[:,n] = np.maximum( strike - S0* (u**asc) * (d**desc), 0 )

for i in xrange(n-1, -1, -1):
    for j in xrange(0, i+1):
        S = S0*u*(j-1)* (d**(i-j))
        put[j,i] = max(max(strike-S, 0.0), (q*put[j+1,i+1]+(1-q) * put[j,i+1])/R)

print put[0,0], put[1,1]
<|endoftext|>


<commit_before>"""
Codes for correcting and estimating various radar and meteorological parameters.

@title: radar_codes
@author: Valentin Louf <valentin.louf@monash.edu>
@institutions: Monash University and the Australian Bureau of Meteorology
@creation: 04/04/2017
@date: 14/11/2017

.. autosummary::
    :toctree: generated/

    _my_snr_from_reflectivity
    _nearest
    check_azimuth
    check_reflectivity
    correct_rhohv
    correct_zdr
    get_field_names
    get_radiosoundings
    read_radar
    rename_radar_fields
    snr_and_sounding
"""
# Python Standard Library
import os
import re
import glob
import time
import fnmatch
import datetime

from copy import deepcopy

# Other Libraries
import pyart
import scipy
import netCDF4
import numpy as np

def _my_snr_from_reflectivity(radar, refl_field='DBZ'):
    """
    Just in case pyart.retrieve.calculate_snr_from_reflectivity, I can calculate
    it 'by hand'.
    Parameter:
    ===========
    radar:
        Py-ART radar structure.
    refl_field_name: str
        Name of the reflectivity field.

    Return:
    =======
    snr: dict
        Signal to noise ratio.
    """
    range_grid, azi_grid = np.meshgrid(radar.range['data'], radar.azimuth['data'])
    range_grid += 1  # Cause of 0

    # remove range scale.. This is basically the radar constant scaled dBm
    pseudo_power = (radar.fields[refl_field]['data'] - 20.0 * np.log10(range_grid / 1000.0))
    # The noise_floor_estimate can fail sometimes in pyart, that's the reason
    # why this whole function exists.
    noise_floor_estimate = -40

    snr_field = pyart.config.get_field_name('signal_to_noise_ratio')
    snr_dict = pyart.config.get_metadata(snr_field)
    snr_dict['data'] = pseudo_power - noise_floor_estimate

    return snr_dict

def _nearest(items, pivot):
    """
    Find the nearest item.

    Parameters:
    ===========
        items:
            List of item.
        pivot:
            Item we're looking for.

    Returns:
    ========
        item:
            Value of the nearest item found.
    """
    return min(items, key=lambda x: abs(x - pivot))

def check_azimuth(radar, refl_field_name='DBZ'):
    """
    Checking if radar has a proper reflectivity field.  It's a minor problem
    concerning a few days in 2011 for CPOL.

    Parameters:
    ===========
        radar:
            Py-ART radar structure.
        refl_field_name: str
            Name of the reflectivity field.

    Return:
    =======
        is_good: bool
            True if radar has a proper azimuth field.
    """
    is_good = True
    dbz = radar.fields[refl_field_name]['data']

    if dbz.shape[0] < 360:
        is_good = False

    return is_good

def check_reflectivity(radar, refl_field_name='DBZ'):
    """
    Checking if radar has a proper reflectivity field.  It's a minor problem
    concerning a few days in 2011 for CPOL.

    Parameters:
    ===========
        radar:
            Py-ART radar structure.
        refl_field_name: str
            Name of the reflectivity field.

    Return:
    =======
        is_good: bool
            True if radar has a proper azimuth field.
    """
    is_good = True
    dbz = radar.fields[refl_field_name]['data']

    if np.ma.isMaskedArray(dbz):
        if dbz.count() == 0:
            # Reflectivity field is empty.
            is_good = False

    return is_good

def correct_azimuth(radar):
    """
    Check if the azimuth is right.

    Parameters:
    ===========
    radar: Py-ART radar structure

    Returns:
    ========
    azimuth: ndarray
        Corrected azimuth
    has_changed: bool
        Is there any change?
    """
    has_changed = False
    azimuth = radar.azimuth['data']
    for sl in range(radar.nsweeps):
        azi = azimuth[radar.get_slice(sl)]
        if np.sum(azi == 0) <= 2:
            continue

        azi_zero = azi[-1]
        for na in range(len(azi) - 2, -1, -1):
            if azi[na] != azi_zero - 1:
                if azi_zero == 0 and azi[na] == 359:
                    azi_zero = azi[na]
                    continue
                else:
                    has_changed = True
                    azi[na] = azi_zero - 1
            azi_zero = azi[na]

        azimuth[azimuth < 0] += 360
        azimuth[radar.get_slice(sl)] = azi

    return azimuth, has_changed

def correct_rhohv(radar, rhohv_name='RHOHV', snr_name='SNR'):
    """
    Correct cross correlation ratio (RHOHV) from noise. From the Schuur et al.
    2003 NOAA report (p7 eq 5)

    Parameters:
    ===========
        radar:
            Py-ART radar structure.
        rhohv_name: str
            Cross correlation field name.
        snr_name: str
            Signal to noise ratio field name.

    Returns:
    ========
        rho_corr: array
            Corrected cross correlation ratio.
    """
    rhohv = radar.fields[rhohv_name]['data'].copy()
    snr = radar.fields[snr_name]['data'].copy()

    natural_snr = 10**(0.1 * snr)
    natural_snr = natural_snr.filled(-9999)
    rho_corr = rhohv * (1 + 1 / natural_snr)

    # Not allowing the corrected RHOHV to be lower than the raw rhohv
    # pos = rho_corr < rhohv
    # rho_corr[pos] = rhohv[pos]
    rho_corr[np.isnan(rho_corr) | (rho_corr < 0) | (rho_corr > 1)] = 1
    try:
        rho_corr = rho_corr.filled(1)
    except Exception:
        pass

    return rho_corr

def correct_zdr(radar, zdr_name='ZDR', snr_name='SNR'):
    """
    Correct differential reflectivity (ZDR) from noise. From the Schuur et al.
    2003 NOAA report (p7 eq 6)

    Parameters:
    ===========
        radar:
            Py-ART radar structure.
        zdr_name: str
            Differential reflectivity field name.
        snr_name: str
            Signal to noise ratio field name.

    Returns:
    ========
        corr_zdr: array
            Corrected differential reflectivity.
    """
    zdr = radar.fields[zdr_name]['data']
    snr = radar.fields[snr_name]['data']
    alpha = 1.48
    natural_zdr = 10**(0.1 * zdr)
    natural_snr = 10**(0.1 * snr)
    corr_zdr = 10 * np.log10((alpha * natural_snr * natural_zdr) / (alpha * natural_snr + alpha - natural_zdr))

    return corr_zdr

def get_field_names():
    """
    Fields name definition.

    Returns:
    ========
        fields_names: array
            Containing [(old key, new key), ...]
    """
    fields_names = [('VEL', 'velocity'),
                    ('VEL_CORR', 'corrected_velocity'),
                    ('VEL_UNFOLDED', 'region_dealias_velocity'),
                    ("RAW_VEL", "raw_velocity"),
                    ('TVEL', "velocity_texture"),
                    ('TPHI', "differential_phase_texture"),
                    ('DBZ', 'total_power'),
                    ('DBZ_CORR', 'reflectivity'),
                    ('RHOHV_CORR', 'RHOHV'),
                    ('RHOHV', 'cross_correlation_ratio'),
                    ('ZDR', 'differential_reflectivity'),
                    ('ZDR_CORR', 'corrected_differential_reflectivity'),
                    ('PHIDP', 'differential_phase'),
                    ('PHIDP_BRINGI', 'corrected_differential_phase'),
                    ('PHIDP_GG', 'giangrande_differential_phase'),
                    ('KDP', 'specific_differential_phase'),
                    ('KDP_BRINGI', 'corrected_specific_differential_phase'),
                    ('KDP_GG', 'giangrande_specific_differential_phase'),
                    ('WIDTH', 'spectrum_width'),
                    ('SNR', 'signal_to_noise_ratio'),
                    ('NCP', 'normalized_coherent_power')]

    return fields_names

def get_radiosoundings(sound_dir, radar_start_date):
    """
    Find the radiosoundings
    """
    def _fdate(flist):
        rslt = [None] * len(flist)
        for cnt, f in enumerate(flist):
            try:
                rslt[cnt] = datetime.datetime.strptime(re.findall("[0-9]{8}", f)[0], "%Y%m%d")
            except Exception:
                continue
        return rslt
    # Looking for radiosoundings:
    all_sonde_files = sorted(os.listdir(sound_dir))

    pos = [cnt for cnt, f in enumerate(all_sonde_files) if fnmatch.fnmatch(f, "*" + radar_start_date.strftime("%Y%m%d") + "*")]
    if len(pos) > 0:
        # Looking for the exact date.
        sonde_name = all_sonde_files[pos[0]]
        sonde_name = os.path.join(sound_dir, sonde_name)
    else:
        # Looking for the closest date.
        dtime_none = _fdate(all_sonde_files)
        dtime = [d for d in dtime_none if d is not None]
        closest_date = _nearest(dtime, radar_start_date)
        sonde_name = [e for e in all_sonde_files if closest_date.strftime("%Y%m%d") in e]
        if len(sonde_name) == 0:
            sonde_name = os.path.join(sound_dir, all_sonde_files[-1])
        elif type(sonde_name) is list:
            sonde_name = os.path.join(sound_dir, sonde_name[0])
        else:
            sonde_name = os.path.join(sound_dir, sonde_name)

    return sonde_name

def read_radar(radar_file_name):
    """
    Read the input radar file.

    Parameter:
    ==========
        radar_file_name: str
            Radar file name.

    Return:
    =======
        radar: struct
            Py-ART radar structure.
    """
    # Read the input radar file.
    try:
        if ".h5" in radar_file_name:
            radar = pyart.aux_io.read_odim_h5(radar_file_name, file_field_names=True)
        elif ".hdf" in radar_file_name:
            radar = pyart.aux_io.read_odim_h5(radar_file_name, file_field_names=True)
        else:
            radar = pyart.io.read(radar_file_name)
    except Exception:
        raise

    # SEAPOL hack change fields key.
    try:
        radar.fields['DBZ']
    except KeyError:
        myfields = [('SQIH', 'NCP'),
                    ('NCPH', "NCP"),
                    ('SNRH', 'SNR'),
                    ('normalized_coherent_power', "NCP"),
                    ('DBZH', "DBZ"),
                    ("DBZH_CLEAN", "DBZ"),
                    ('reflectivity', "DBZ"),
                    ('WRADH', "WIDTH"),
                    ('WIDTHH', "WIDTH"),
                    ('sprectrum_width', "WIDTH"),
                    ('UH', "DBZ"),
                    ('total_power', "DBZ"),
                    ("differential_reflectivity", "ZDR"),
                    ("VRADH", "VEL"),
                    ('VELH', "VEL"),
                    ('velocity', "VEL"),
                    ("cross_correlation_ratio", "RHOHV"),
                    ("differential_phase", "PHIDP"),
                    ("specific_differential_phase", "KDP")]
        for mykey, newkey in myfields:
            try:
                radar.add_field(newkey, radar.fields.pop(mykey))
            except Exception:
                continue

    return radar

def rename_radar_fields(radar):
    """
    Rename radar fields from their old name to the Py-ART default name.

    Parameter:
    ==========
        radar:
            Py-ART radar structure.

    Returns:
    ========
        radar:
            Py-ART radar structure.
    """
    fields_names = get_field_names()

    # Try to remove occasional fields.
    try:
        vdop_art = radar.fields['PHIDP_CORR']
        radar.add_field('PHIDP', radar.fields.pop('PHIDP_CORR'), replace_existing=True)
    except KeyError:
        pass

    # Parse array old_key, new_key
    for old_key, new_key in fields_names:
        try:
            radar.add_field(new_key, radar.fields.pop(old_key), replace_existing=True)
        except KeyError:
            continue

    return radar

def snr_and_sounding(radar, sonde_name, refl_field_name='DBZ', temp_field_name="temp"):
    """
    Compute the signal-to-noise ratio as well as interpolating the radiosounding
    temperature on to the radar grid. The function looks for the radiosoundings
    that happened at the closest time from the radar. There is no time
    difference limit.

    Parameters:
    ===========
        radar:
        sonde_name: str
            Path to the radiosoundings.
        refl_field_name: str
            Name of the reflectivity field.

    Returns:
    ========
        z_dict: dict
            Altitude in m, interpolated at each radar gates.
        temp_info_dict: dict
            Temperature in Celsius, interpolated at each radar gates.
        snr: dict
            Signal to noise ratio.
    """
    radar_start_date = netCDF4.num2date(radar.time['data'][0], radar.time['units'])
    # Altitude hack.
    true_alt = radar.altitude['data'].copy()
    radar.altitude['data'] = np.array([0])

    # print("Reading radiosounding %s" % (sonde_name))
    interp_sonde = netCDF4.Dataset(sonde_name)
    temperatures = interp_sonde.variables[temp_field_name][:]
    temperatures[(temperatures < -100) | (temperatures > 100)] = np.NaN
    try:
        temperatures = temperatures.filled(np.NaN)
    except AttributeError:
        pass
    times = interp_sonde.variables['time'][:]
    heights = interp_sonde.variables['height'][:]

    # Height profile corresponding to radar.
    my_profile = pyart.retrieve.fetch_radar_time_profile(interp_sonde, radar)

    # CPOL altitude is 50 m.
    good_altitude = my_profile['height'] >= 0
    # Getting the temperature
    z_dict, temp_dict = pyart.retrieve.map_profile_to_gates(temperatures[good_altitude],
                                                            my_profile['height'][good_altitude],
                                                            radar)

    temp_info_dict = {'data': temp_dict['data'],
                      'long_name': 'Sounding temperature at gate',
                      'standard_name': 'temperature',
                      'valid_min': -100, 'valid_max': 100,
                      'units': 'degrees Celsius',
                      'comment': 'Radiosounding date: %s' % (radar_start_date.strftime("%Y/%m/%d"))}

    # Altitude hack
    radar.altitude['data'] = true_alt

    # Calculate SNR
    snr = pyart.retrieve.calculate_snr_from_reflectivity(radar, refl_field=refl_field_name)
    # Sometimes the SNR is an empty array, this is due to the toa parameter.
    # Here we try to recalculate the SNR with a lower value for toa (top of atm).
    if snr['data'].count() == 0:
        snr = pyart.retrieve.calculate_snr_from_reflectivity(radar, refl_field=refl_field_name, toa=20000)

    if snr['data'].count() == 0:
        # If it fails again, then we compute the SNR with the noise value
        # given by the CPOL radar manufacturer.
        snr = _my_snr_from_reflectivity(radar, refl_field=refl_field_name)

    return z_dict, temp_info_dict, snr
<commit_msg>Update radar codes<commit_after>"""
Codes for correcting and estimating various radar and meteorological parameters.

@title: radar_codes
@author: Valentin Louf <valentin.louf@monash.edu>
@institutions: Monash University and the Australian Bureau of Meteorology
@creation: 04/04/2017
@date: 14/11/2017

.. autosummary::
    :toctree: generated/

    _my_snr_from_reflectivity
    _nearest
    check_azimuth
    check_reflectivity
    correct_rhohv
    correct_zdr
    get_field_names
    get_radiosoundings
    read_radar
    rename_radar_fields
    snr_and_sounding
"""
# Python Standard Library
import os
import re
import glob
import time
import fnmatch
import datetime

from copy import deepcopy

# Other Libraries
import pyart
import scipy
import netCDF4
import numpy as np

def _my_snr_from_reflectivity(radar, refl_field='DBZ'):
    """
    Just in case pyart.retrieve.calculate_snr_from_reflectivity, I can calculate
    it 'by hand'.
    Parameter:
    ===========
    radar:
        Py-ART radar structure.
    refl_field_name: str
        Name of the reflectivity field.

    Return:
    =======
    snr: dict
        Signal to noise ratio.
    """
    range_grid, azi_grid = np.meshgrid(radar.range['data'], radar.azimuth['data'])
    range_grid += 1  # Cause of 0

    # remove range scale.. This is basically the radar constant scaled dBm
    pseudo_power = (radar.fields[refl_field]['data'] - 20.0 * np.log10(range_grid / 1000.0))
    # The noise_floor_estimate can fail sometimes in pyart, that's the reason
    # why this whole function exists.
    noise_floor_estimate = -40

    snr_field = pyart.config.get_field_name('signal_to_noise_ratio')
    snr_dict = pyart.config.get_metadata(snr_field)
    snr_dict['data'] = pseudo_power - noise_floor_estimate

    return snr_dict

def _nearest(items, pivot):
    """
    Find the nearest item.

    Parameters:
    ===========
        items:
            List of item.
        pivot:
            Item we're looking for.

    Returns:
    ========
        item:
            Value of the nearest item found.
    """
    return min(items, key=lambda x: abs(x - pivot))

def check_azimuth(radar, refl_field_name='DBZ'):
    """
    Checking if radar has a proper reflectivity field.  It's a minor problem
    concerning a few days in 2011 for CPOL.

    Parameters:
    ===========
        radar:
            Py-ART radar structure.
        refl_field_name: str
            Name of the reflectivity field.

    Return:
    =======
        True if radar has a proper azimuth field.
    """
    if radar.fields[refl_field_name]['data'].shape[0] < 360:
        return False

    return True

def check_reflectivity(radar, refl_field_name='DBZ'):
    """
    Checking if radar has a proper reflectivity field.  It's a minor problem
    concerning a few days in 2011 for CPOL.

    Parameters:
    ===========
        radar:
            Py-ART radar structure.
        refl_field_name: str
            Name of the reflectivity field.

    Return:
    =======
    True if radar has a non-empty reflectivity field.
    """
    dbz = radar.fields[refl_field_name]['data']

    if np.ma.isMaskedArray(dbz):
        if dbz.count() == 0:
            # Reflectivity field is empty.
            return False

    return True

def correct_azimuth(radar):
    """
    Check if the azimuth is right.

    Parameters:
    ===========
    radar: Py-ART radar structure

    Returns:
    ========
    azimuth: ndarray
        Corrected azimuth
    has_changed: bool
        Is there any change?
    """
    has_changed = False
    azimuth = radar.azimuth['data']
    for sl in range(radar.nsweeps):
        azi = azimuth[radar.get_slice(sl)]
        if np.sum(azi == 0) <= 2:
            continue

        azi_zero = azi[-1]
        for na in range(len(azi) - 2, -1, -1):
            if azi[na] != azi_zero - 1:
                if azi_zero == 0 and azi[na] == 359:
                    azi_zero = azi[na]
                    continue
                else:
                    has_changed = True
                    azi[na] = azi_zero - 1
            azi_zero = azi[na]

        azimuth[azimuth < 0] += 360
        azimuth[radar.get_slice(sl)] = azi

    return azimuth, has_changed

def correct_rhohv(radar, rhohv_name='RHOHV', snr_name='SNR'):
    """
    Correct cross correlation ratio (RHOHV) from noise. From the Schuur et al.
    2003 NOAA report (p7 eq 5)

    Parameters:
    ===========
        radar:
            Py-ART radar structure.
        rhohv_name: str
            Cross correlation field name.
        snr_name: str
            Signal to noise ratio field name.

    Returns:
    ========
        rho_corr: array
            Corrected cross correlation ratio.
    """
    rhohv = radar.fields[rhohv_name]['data'].copy()
    snr = radar.fields[snr_name]['data'].copy()

    natural_snr = 10**(0.1 * snr)
    natural_snr = natural_snr.filled(-9999)
    rho_corr = rhohv * (1 + 1 / natural_snr)

    # Not allowing the corrected RHOHV to be lower than the raw rhohv
    rho_corr[np.isnan(rho_corr) | (rho_corr < 0) | (rho_corr > 1)] = 1
    try:
        rho_corr = rho_corr.filled(1)
    except Exception:
        pass

    return rho_corr

def correct_zdr(radar, zdr_name='ZDR', snr_name='SNR'):
    """
    Correct differential reflectivity (ZDR) from noise. From the Schuur et al.
    2003 NOAA report (p7 eq 6)

    Parameters:
    ===========
        radar:
            Py-ART radar structure.
        zdr_name: str
            Differential reflectivity field name.
        snr_name: str
            Signal to noise ratio field name.

    Returns:
    ========
        corr_zdr: array
            Corrected differential reflectivity.
    """
    zdr = radar.fields[zdr_name]['data']
    snr = radar.fields[snr_name]['data']
    alpha = 1.48
    natural_zdr = 10**(0.1 * zdr)
    natural_snr = 10**(0.1 * snr)
    corr_zdr = 10 * np.log10((alpha * natural_snr * natural_zdr) / (alpha * natural_snr + alpha - natural_zdr))

    return corr_zdr

def get_field_names():
    """
    Fields name definition.

    Returns:
    ========
        fields_names: array
            Containing [(old key, new key), ...]
    """
    fields_names = [('VEL', 'velocity'),
                    ('VEL_CORR', 'corrected_velocity'),
                    ('VEL_UNFOLDED', 'region_dealias_velocity'),
                    ("RAW_VEL", "raw_velocity"),
                    ('TVEL', "velocity_texture"),
                    ('TPHI', "differential_phase_texture"),
                    ('DBZ', 'total_power'),
                    ('DBZ_CORR', 'reflectivity'),
                    ('RHOHV_CORR', 'RHOHV'),
                    ('RHOHV', 'cross_correlation_ratio'),
                    ('ZDR', 'differential_reflectivity'),
                    ('ZDR_CORR', 'corrected_differential_reflectivity'),
                    ('PHIDP', 'differential_phase'),
                    ('PHIDP_BRINGI', 'corrected_differential_phase'),
                    ('PHIDP_GG', 'giangrande_differential_phase'),
                    ('KDP', 'specific_differential_phase'),
                    ('KDP_BRINGI', 'corrected_specific_differential_phase'),
                    ('KDP_GG', 'giangrande_specific_differential_phase'),
                    ('WIDTH', 'spectrum_width'),
                    ('SNR', 'signal_to_noise_ratio'),
                    ('NCP', 'normalized_coherent_power')]

    return fields_names

def get_radiosoundings(sound_dir, radar_start_date):
    """
    Find the radiosoundings
    """
    def _fdate(flist):
        rslt = [None] * len(flist)
        for cnt, f in enumerate(flist):
            try:
                rslt[cnt] = datetime.datetime.strptime(re.findall("[0-9]{8}", f)[0], "%Y%m%d")
            except Exception:
                continue
        return rslt
    # Looking for radiosoundings:
    all_sonde_files = sorted(os.listdir(sound_dir))

    pos = [cnt for cnt, f in enumerate(all_sonde_files) if fnmatch.fnmatch(f, "*" + radar_start_date.strftime("%Y%m%d") + "*")]
    if len(pos) > 0:
        # Looking for the exact date.
        sonde_name = all_sonde_files[pos[0]]
        sonde_name = os.path.join(sound_dir, sonde_name)
    else:
        # Looking for the closest date.
        dtime_none = _fdate(all_sonde_files)
        dtime = [d for d in dtime_none if d is not None]
        closest_date = _nearest(dtime, radar_start_date)
        sonde_name = [e for e in all_sonde_files if closest_date.strftime("%Y%m%d") in e]
        if len(sonde_name) == 0:
            sonde_name = os.path.join(sound_dir, all_sonde_files[-1])
        elif type(sonde_name) is list:
            sonde_name = os.path.join(sound_dir, sonde_name[0])
        else:
            sonde_name = os.path.join(sound_dir, sonde_name)

    return sonde_name

def read_radar(radar_file_name):
    """
    Read the input radar file.

    Parameter:
    ==========
        radar_file_name: str
            Radar file name.

    Return:
    =======
        radar: struct
            Py-ART radar structure.
    """
    # Read the input radar file.
    try:
        if ".h5" in radar_file_name:
            radar = pyart.aux_io.read_odim_h5(radar_file_name, file_field_names=True)
        elif ".hdf" in radar_file_name:
            radar = pyart.aux_io.read_odim_h5(radar_file_name, file_field_names=True)
        else:
            radar = pyart.io.read(radar_file_name)
    except Exception:
        raise

    # SEAPOL hack change fields key.
    try:
        radar.fields['DBZ']
    except KeyError:
        myfields = [('SQIH', 'NCP'),
                    ('NCPH', "NCP"),
                    ('SNRH', 'SNR'),
                    ('normalized_coherent_power', "NCP"),
                    ('DBZH', "DBZ"),
                    ("DBZH_CLEAN", "DBZ"),
                    ('reflectivity', "DBZ"),
                    ('WRADH', "WIDTH"),
                    ('WIDTHH', "WIDTH"),
                    ('sprectrum_width', "WIDTH"),
                    ('UH', "DBZ"),
                    ('total_power', "DBZ"),
                    ("differential_reflectivity", "ZDR"),
                    ("VRADH", "VEL"),
                    ('VELH', "VEL"),
                    ('velocity', "VEL"),
                    ("cross_correlation_ratio", "RHOHV"),
                    ("differential_phase", "PHIDP"),
                    ("specific_differential_phase", "KDP")]
        for mykey, newkey in myfields:
            try:
                radar.add_field(newkey, radar.fields.pop(mykey))
            except Exception:
                continue

    return radar

def rename_radar_fields(radar):
    """
    Rename radar fields from their old name to the Py-ART default name.

    Parameter:
    ==========
        radar:
            Py-ART radar structure.

    Returns:
    ========
        radar:
            Py-ART radar structure.
    """
    fields_names = get_field_names()

    # Try to remove occasional fields.
    try:
        vdop_art = radar.fields['PHIDP_CORR']
        radar.add_field('PHIDP', radar.fields.pop('PHIDP_CORR'), replace_existing=True)
    except KeyError:
        pass

    # Parse array old_key, new_key
    for old_key, new_key in fields_names:
        try:
            radar.add_field(new_key, radar.fields.pop(old_key), replace_existing=True)
        except KeyError:
            continue

    return radar

def snr_and_sounding(radar, sonde_name, refl_field_name='DBZ', temp_field_name="temp"):
    """
    Compute the signal-to-noise ratio as well as interpolating the radiosounding
    temperature on to the radar grid. The function looks for the radiosoundings
    that happened at the closest time from the radar. There is no time
    difference limit.

    Parameters:
    ===========
        radar:
        sonde_name: str
            Path to the radiosoundings.
        refl_field_name: str
            Name of the reflectivity field.

    Returns:
    ========
        z_dict: dict
            Altitude in m, interpolated at each radar gates.
        temp_info_dict: dict
            Temperature in Celsius, interpolated at each radar gates.
        snr: dict
            Signal to noise ratio.
    """
    radar_start_date = netCDF4.num2date(radar.time['data'][0], radar.time['units'])
    # Altitude hack.
    true_alt = radar.altitude['data'].copy()
    radar.altitude['data'] = np.array([0])

    # print("Reading radiosounding %s" % (sonde_name))
    interp_sonde = netCDF4.Dataset(sonde_name)
    temperatures = interp_sonde.variables[temp_field_name][:]
    temperatures[(temperatures < -100) | (temperatures > 100)] = np.NaN
    try:
        temperatures = temperatures.filled(np.NaN)
    except AttributeError:
        pass
    # times = interp_sonde.variables['time'][:]
    # heights = interp_sonde.variables['height'][:]

    # Height profile corresponding to radar.
    my_profile = pyart.retrieve.fetch_radar_time_profile(interp_sonde, radar)

    # CPOL altitude is 50 m.
    good_altitude = my_profile['height'] >= 0
    # Getting the temperature
    z_dict, temp_dict = pyart.retrieve.map_profile_to_gates(temperatures[good_altitude],
                                                            my_profile['height'][good_altitude],
                                                            radar)

    temp_info_dict = {'data': temp_dict['data'],
                      'long_name': 'Sounding temperature at gate',
                      'standard_name': 'temperature',
                      'valid_min': -100, 'valid_max': 100,
                      'units': 'degrees Celsius',
                      'comment': 'Radiosounding date: %s' % (radar_start_date.strftime("%Y/%m/%d"))}

    # Altitude hack
    radar.altitude['data'] = true_alt

    # Calculate SNR
    snr = pyart.retrieve.calculate_snr_from_reflectivity(radar, refl_field=refl_field_name)
    # Sometimes the SNR is an empty array, this is due to the toa parameter.
    # Here we try to recalculate the SNR with a lower value for toa (top of atm).
    if snr['data'].count() == 0:
        snr = pyart.retrieve.calculate_snr_from_reflectivity(radar, refl_field=refl_field_name, toa=20000)

    if snr['data'].count() == 0:
        # If it fails again, then we compute the SNR with the noise value
        # given by the CPOL radar manufacturer.
        snr = _my_snr_from_reflectivity(radar, refl_field=refl_field_name)

    return z_dict, temp_info_dict, snr
<|endoftext|>


<commit_before>#http://legacy.python.org/dev/peps/pep-0440/
version_info = ('0', '3', '5')
<commit_msg>Bump version<commit_after>#http://legacy.python.org/dev/peps/pep-0440/
version_info = ('0', '3', '6')
<|endoftext|>


<commit_before>#!/usr/bin/env python
# Copyright 2013 The Chromium Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

"""Accesses an isolateserver instance via remote_api."""

import code
import getpass
import logging
import optparse
import os
import sys

APP_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))

sys.path.insert(0, APP_DIR)

# pylint doesn't know where the AppEngine SDK is, so silence these errors.
# F0401: Unable to import 'XXX'
# E0611: No name 'XXX' in module 'YYY'
# pylint: disable=E0611,F0401

def find_gae_sdk(search_dir):
  """Returns the path to GAE SDK if found, else None."""
  # First search up the directories up to root.
  while True:
    attempt = os.path.join(search_dir, 'google_appengine')
    if os.path.isfile(os.path.join(attempt, 'dev_appserver.py')):
      return attempt
    prev_dir = search_dir
    search_dir = os.path.dirname(search_dir)
    if search_dir == prev_dir:
      break

  # Next search PATH.
  for item in os.environ['PATH'].split(os.pathsep):
    if not item:
      continue
    item = os.path.normpath(os.path.abspath(item))
    if os.path.isfile(os.path.join(item, 'dev_appserver.py')):
      return item

def setup_gae_sdk(sdk_path):
  """Sets up App Engine environment.

  Then any AppEngine included module can be imported. The change is global and
  permanent.
  """
  sys.path.insert(0, sdk_path)

  import dev_appserver
  dev_appserver.fix_sys_path()

def default_auth_func():
  user = os.environ.get('EMAIL_ADDRESS')
  if user:
    result = raw_input('Username (default: %s): ' % user)
    if result:
      user = result
  else:
    user = raw_input('Username: ')
  return user, getpass.getpass('Password: ')

def load_context(sdk_path, app_dir, host, app_id, version):
  """Returns a closure where the GAE SDK is initialized."""
  setup_gae_sdk(sdk_path)

  # Import GAE's SDK modules as needed.
  from google.appengine.ext.remote_api import remote_api_stub
  import yaml

  def default_app_id():
    """Returns the application name."""
    return yaml.load(open(os.path.join(app_dir, 'app.yaml')))['application']

  def setup_env(host):
    """Setup remote access to a GAE instance."""
    # Unused variable 'XXX'; they are accessed via locals().
    # pylint: disable=W0612
    from google.appengine.api import memcache
    from google.appengine.api.users import User
    from google.appengine.ext import db

    remote_api_stub.ConfigureRemoteDatastore(
        None, '/_ah/remote_api', default_auth_func, host,
        save_cookies=True, secure=True)
    remote_api_stub.MaybeInvokeAuthentication()

    os.environ['SERVER_SOFTWARE'] = 'Development (remote_api_shell)/1.0'

    # Create shortcuts.
    import acl
    import main
    from main import ContentNamespace, ContentEntry

    # Symbols presented to the user.
    predefined_vars = locals().copy()
    return predefined_vars

  app_id = app_id or default_app_id()
  if not host:
    if version:
      host = '%s-dot-%s.appspot.com' % (version, app_id)
    else:
      host = '%s..appspot.com' % (app_id)
  return setup_env(host), app_id

def Main():
  parser = optparse.OptionParser(description=sys.modules[__name__].__doc__)
  parser.add_option('-v', '--verbose', action='store_true')
  parser.add_option('-A', '--app-id', help='Defaults to name in app.yaml')
  parser.add_option(
      '-H', '--host', help='Only necessary if not hosted on .appspot.com')
  parser.add_option(
      '-V', '--version',
      help='Defaults to the default active instance. Override to connect to a '
           'non-default instance.')
  parser.add_option(
      '-s', '--sdk-path',
      help='Path to AppEngine SDK. Will try to find by itself.')
  options, args = parser.parse_args()
  logging.basicConfig(level=logging.DEBUG if options.verbose else logging.ERROR)

  if args:
    parser.error('Unknown arguments, %s' % args)
  options.sdk_path = options.sdk_path or find_gae_sdk(APP_DIR)
  if not options.sdk_path:
    parser.error('Failed to find the AppEngine SDK. Pass --sdk-path argument.')

  predefined_vars, app_id = load_context(options.sdk_path, APP_DIR,
      options.host, options.app_id, options.version)
  prompt = (
      'App Engine interactive console for "%s".\n'
      'Available symbols:\n'
      '  %s\n') % (app_id, ', '.join(sorted(predefined_vars)))
  code.interact(prompt, None, predefined_vars)

if __name__ == '__main__':
  sys.exit(Main())
<commit_msg>Trap exception and fix typo in remote_api.py.<commit_after>#!/usr/bin/env python
# Copyright 2013 The Chromium Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

"""Accesses an isolateserver instance via remote_api."""

import code
import getpass
import logging
import optparse
import os
import sys
import urllib2

APP_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))

sys.path.insert(0, APP_DIR)

# pylint doesn't know where the AppEngine SDK is, so silence these errors.
# F0401: Unable to import 'XXX'
# E0611: No name 'XXX' in module 'YYY'
# pylint: disable=E0611,F0401

def find_gae_sdk(search_dir):
  """Returns the path to GAE SDK if found, else None."""
  # First search up the directories up to root.
  while True:
    attempt = os.path.join(search_dir, 'google_appengine')
    if os.path.isfile(os.path.join(attempt, 'dev_appserver.py')):
      return attempt
    prev_dir = search_dir
    search_dir = os.path.dirname(search_dir)
    if search_dir == prev_dir:
      break

  # Next search PATH.
  for item in os.environ['PATH'].split(os.pathsep):
    if not item:
      continue
    item = os.path.normpath(os.path.abspath(item))
    if os.path.isfile(os.path.join(item, 'dev_appserver.py')):
      return item

def setup_gae_sdk(sdk_path):
  """Sets up App Engine environment.

  Then any AppEngine included module can be imported. The change is global and
  permanent.
  """
  sys.path.insert(0, sdk_path)

  import dev_appserver
  dev_appserver.fix_sys_path()

def default_auth_func():
  user = os.environ.get('EMAIL_ADDRESS')
  if user:
    result = raw_input('Username (default: %s): ' % user)
    if result:
      user = result
  else:
    user = raw_input('Username: ')
  return user, getpass.getpass('Password: ')

def load_context(sdk_path, app_dir, host, app_id, version):
  """Returns a closure where the GAE SDK is initialized."""
  setup_gae_sdk(sdk_path)

  # Import GAE's SDK modules as needed.
  from google.appengine.ext.remote_api import remote_api_stub
  import yaml

  def default_app_id():
    """Returns the application name."""
    return yaml.load(open(os.path.join(app_dir, 'app.yaml')))['application']

  def setup_env(host):
    """Setup remote access to a GAE instance."""
    # Unused variable 'XXX'; they are accessed via locals().
    # pylint: disable=W0612
    from google.appengine.api import memcache
    from google.appengine.api.users import User
    from google.appengine.ext import db

    try:
      remote_api_stub.ConfigureRemoteDatastore(
          None, '/_ah/remote_api', default_auth_func, host,
          save_cookies=True, secure=True)
    except urllib2.URLError:
      print >> sys.stderr, 'Failed to access %s' % host
      return None
    remote_api_stub.MaybeInvokeAuthentication()

    os.environ['SERVER_SOFTWARE'] = 'Development (remote_api_shell)/1.0'

    # Create shortcuts.
    import acl
    import main
    from main import ContentNamespace, ContentEntry

    # Symbols presented to the user.
    predefined_vars = locals().copy()
    return predefined_vars

  app_id = app_id or default_app_id()
  if not host:
    if version:
      host = '%s-dot-%s.appspot.com' % (version, app_id)
    else:
      host = '%s.appspot.com' % (app_id)
  return setup_env(host), app_id

def Main():
  parser = optparse.OptionParser(description=sys.modules[__name__].__doc__)
  parser.add_option('-v', '--verbose', action='store_true')
  parser.add_option('-A', '--app-id', help='Defaults to name in app.yaml')
  parser.add_option(
      '-H', '--host', help='Only necessary if not hosted on .appspot.com')
  parser.add_option(
      '-V', '--version',
      help='Defaults to the default active instance. Override to connect to a '
           'non-default instance.')
  parser.add_option(
      '-s', '--sdk-path',
      help='Path to AppEngine SDK. Will try to find by itself.')
  options, args = parser.parse_args()
  logging.basicConfig(level=logging.DEBUG if options.verbose else logging.ERROR)

  if args:
    parser.error('Unknown arguments, %s' % args)
  options.sdk_path = options.sdk_path or find_gae_sdk(APP_DIR)
  if not options.sdk_path:
    parser.error('Failed to find the AppEngine SDK. Pass --sdk-path argument.')

  predefined_vars, app_id = load_context(options.sdk_path, APP_DIR,
      options.host, options.app_id, options.version)
  if not predefined_vars:
    return 1
  prompt = (
      'App Engine interactive console for "%s".\n'
      'Available symbols:\n'
      '  %s\n') % (app_id, ', '.join(sorted(predefined_vars)))
  code.interact(prompt, None, predefined_vars)

if __name__ == '__main__':
  sys.exit(Main())
<|endoftext|>


<commit_before>#!/usr/bin/python
#
# wakeuptime    Summarize sleep to wakeup time by waker kernel stack
#               For Linux, uses BCC, eBPF.
#
# USAGE: wakeuptime [-h] [-u] [-p PID] [-v] [-f] [duration]
#
# Copyright 2016 Netflix, Inc.
# Licensed under the Apache License, Version 2.0 (the "License")
#
# 14-Jan-2016	Brendan Gregg	Created this.

from __future__ import print_function
from bcc import BPF
from bcc.utils import printb
from time import sleep, strftime
import argparse
import signal
import errno
from sys import stderr

# arg validation
def positive_int(val):
    try:
        ival = int(val)
    except ValueError:
        raise argparse.ArgumentTypeError("must be an integer")

    if ival < 0:
        raise argparse.ArgumentTypeError("must be positive")
    return ival

def positive_nonzero_int(val):
    ival = positive_int(val)
    if ival == 0:
        raise argparse.ArgumentTypeError("must be nonzero")
    return ival

# arguments
examples = """examples:
    ./wakeuptime             # trace blocked time with waker stacks
    ./wakeuptime 5           # trace for 5 seconds only
    ./wakeuptime -f 5        # 5 seconds, and output in folded format
    ./wakeuptime -u          # don't include kernel threads (user only)
    ./wakeuptime -p 185      # trace fo PID 185 only
"""
parser = argparse.ArgumentParser(
    description="Summarize sleep to wakeup time by waker kernel stack",
    formatter_class=argparse.RawDescriptionHelpFormatter,
    epilog=examples)
parser.add_argument("-u", "--useronly", action="store_true",
    help="user threads only (no kernel threads)")
parser.add_argument("-p", "--pid",
    type=positive_int,
    help="trace this PID only")
parser.add_argument("-v", "--verbose", action="store_true",
    help="show raw addresses")
parser.add_argument("-f", "--folded", action="store_true",
    help="output folded format")
parser.add_argument("--stack-storage-size", default=1024,
    type=positive_nonzero_int,
    help="the number of unique stack traces that can be stored and "
         "displayed (default 1024)")
parser.add_argument("duration", nargs="?", default=99999999,
    type=positive_nonzero_int,
    help="duration of trace, in seconds")
parser.add_argument("-m", "--min-block-time", default=1,
    type=positive_nonzero_int,
    help="the amount of time in microseconds over which we " +
         "store traces (default 1)")
parser.add_argument("-M", "--max-block-time", default=(1 << 64) - 1,
    type=positive_nonzero_int,
    help="the amount of time in microseconds under which we " +
         "store traces (default U64_MAX)")
parser.add_argument("--ebpf", action="store_true",
    help=argparse.SUPPRESS)
args = parser.parse_args()
folded = args.folded
duration = int(args.duration)
debug = 0
if args.pid and args.useronly:
    parser.error("use either -p or -u.")

# signal handler
def signal_ignore(signal, frame):
    print()

# define BPF program
bpf_text = """
#include <uapi/linux/ptrace.h>
#include <linux/sched.h>

#define MINBLOCK_US    MINBLOCK_US_VALUEULL
#define MAXBLOCK_US    MAXBLOCK_US_VALUEULL

struct key_t {
    int  w_k_stack_id;
    char waker[TASK_COMM_LEN];
    char target[TASK_COMM_LEN];
};
BPF_HASH(counts, struct key_t);
BPF_HASH(start, u32);
BPF_STACK_TRACE(stack_traces, STACK_STORAGE_SIZE);

int offcpu(struct pt_regs *ctx) {
    u32 pid = bpf_get_current_pid_tgid();
    struct task_struct *p = (struct task_struct *) bpf_get_current_task();
    u64 ts;

    if (FILTER)
        return 0;

    ts = bpf_ktime_get_ns();
    start.update(&pid, &ts);
    return 0;
}

int waker(struct pt_regs *ctx, struct task_struct *p) {
    u32 pid = p->pid;
    u64 delta, *tsp, ts;

    tsp = start.lookup(&pid);
    if (tsp == 0)
        return 0;        // missed start
    start.delete(&pid);

    if (FILTER)
        return 0;

    // calculate delta time
    delta = bpf_ktime_get_ns() - *tsp;
    delta = delta / 1000;
    if ((delta < MINBLOCK_US) || (delta > MAXBLOCK_US))
        return 0;

    struct key_t key = {};

    key.w_k_stack_id = stack_traces.get_stackid(ctx, BPF_F_REUSE_STACKID);
    bpf_probe_read(&key.target, sizeof(key.target), p->comm);
    bpf_get_current_comm(&key.waker, sizeof(key.waker));

    counts.increment(key, delta);
    return 0;
}
"""
if args.pid:
    filter = 'pid != %s' % args.pid
elif args.useronly:
    filter = 'p->flags & PF_KTHREAD'
else:
    filter = '0'
bpf_text = bpf_text.replace('FILTER', filter)

# set stack storage size
bpf_text = bpf_text.replace('STACK_STORAGE_SIZE', str(args.stack_storage_size))
bpf_text = bpf_text.replace('MINBLOCK_US_VALUE', str(args.min_block_time))
bpf_text = bpf_text.replace('MAXBLOCK_US_VALUE', str(args.max_block_time))

if debug or args.ebpf:
    print(bpf_text)
    if args.ebpf:
        exit()

# initialize BPF
b = BPF(text=bpf_text)
b.attach_kprobe(event="schedule", fn_name="offcpu")
b.attach_kprobe(event="try_to_wake_up", fn_name="waker")
matched = b.num_open_kprobes()
if matched == 0:
    print("0 functions traced. Exiting.")
    exit()

# header
if not folded:
    print("Tracing blocked time (us) by kernel stack", end="")
    if duration < 99999999:
        print(" for %d secs." % duration)
    else:
        print("... Hit Ctrl-C to end.")

# output
while (1):
    try:
        sleep(duration)
    except KeyboardInterrupt:
        # as cleanup can take many seconds, trap Ctrl-C:
        signal.signal(signal.SIGINT, signal_ignore)

    if not folded:
        print()
    missing_stacks = 0
    has_enomem = False
    counts = b.get_table("counts")
    stack_traces = b.get_table("stack_traces")
    for k, v in sorted(counts.items(), key=lambda counts: counts[1].value):
        # handle get_stackid errors
        # check for an ENOMEM error
        if k.w_k_stack_id == -errno.ENOMEM:
            missing_stacks += 1
            continue

        waker_kernel_stack = [] if k.w_k_stack_id < 1 else \
            reversed(list(stack_traces.walk(k.w_k_stack_id))[1:])

        if folded:
            # print folded stack output
            line = \
                [k.waker] + \
                [b.ksym(addr)
                    for addr in reversed(list(waker_kernel_stack))] + \
                [k.target]
            printb(b"%s %d" % (b";".join(line), v.value))
        else:
            # print default multi-line stack output
            printb(b"    %-16s %s" % (b"target:", k.target))
            for addr in waker_kernel_stack:
                printb(b"    %-16x %s" % (addr, b.ksym(addr)))
            printb(b"    %-16s %s" % (b"waker:", k.waker))
            print("        %d\n" % v.value)
    counts.clear()

    if missing_stacks > 0:
        enomem_str = " Consider increasing --stack-storage-size."
        print("WARNING: %d stack traces could not be displayed.%s" %
            (missing_stacks, enomem_str),
            file=stderr)

    if not folded:
        print("Detaching...")
    exit()
<commit_msg>Update wakeuptime.py (#1957)<commit_after>#!/usr/bin/python
#
# wakeuptime    Summarize sleep to wakeup time by waker kernel stack
#               For Linux, uses BCC, eBPF.
#
# USAGE: wakeuptime [-h] [-u] [-p PID] [-v] [-f] [duration]
#
# Copyright 2016 Netflix, Inc.
# Licensed under the Apache License, Version 2.0 (the "License")
#
# 14-Jan-2016	Brendan Gregg	Created this.

from __future__ import print_function
from bcc import BPF
from bcc.utils import printb
from time import sleep, strftime
import argparse
import signal
import errno
from sys import stderr

# arg validation
def positive_int(val):
    try:
        ival = int(val)
    except ValueError:
        raise argparse.ArgumentTypeError("must be an integer")

    if ival < 0:
        raise argparse.ArgumentTypeError("must be positive")
    return ival

def positive_nonzero_int(val):
    ival = positive_int(val)
    if ival == 0:
        raise argparse.ArgumentTypeError("must be nonzero")
    return ival

# arguments
examples = """examples:
    ./wakeuptime             # trace blocked time with waker stacks
    ./wakeuptime 5           # trace for 5 seconds only
    ./wakeuptime -f 5        # 5 seconds, and output in folded format
    ./wakeuptime -u          # don't include kernel threads (user only)
    ./wakeuptime -p 185      # trace for PID 185 only
"""
parser = argparse.ArgumentParser(
    description="Summarize sleep to wakeup time by waker kernel stack",
    formatter_class=argparse.RawDescriptionHelpFormatter,
    epilog=examples)
parser.add_argument("-u", "--useronly", action="store_true",
    help="user threads only (no kernel threads)")
parser.add_argument("-p", "--pid",
    type=positive_int,
    help="trace this PID only")
parser.add_argument("-v", "--verbose", action="store_true",
    help="show raw addresses")
parser.add_argument("-f", "--folded", action="store_true",
    help="output folded format")
parser.add_argument("--stack-storage-size", default=1024,
    type=positive_nonzero_int,
    help="the number of unique stack traces that can be stored and "
         "displayed (default 1024)")
parser.add_argument("duration", nargs="?", default=99999999,
    type=positive_nonzero_int,
    help="duration of trace, in seconds")
parser.add_argument("-m", "--min-block-time", default=1,
    type=positive_nonzero_int,
    help="the amount of time in microseconds over which we " +
         "store traces (default 1)")
parser.add_argument("-M", "--max-block-time", default=(1 << 64) - 1,
    type=positive_nonzero_int,
    help="the amount of time in microseconds under which we " +
         "store traces (default U64_MAX)")
parser.add_argument("--ebpf", action="store_true",
    help=argparse.SUPPRESS)
args = parser.parse_args()
folded = args.folded
duration = int(args.duration)
debug = 0
if args.pid and args.useronly:
    parser.error("use either -p or -u.")

# signal handler
def signal_ignore(signal, frame):
    print()

# define BPF program
bpf_text = """
#include <uapi/linux/ptrace.h>
#include <linux/sched.h>

#define MINBLOCK_US    MINBLOCK_US_VALUEULL
#define MAXBLOCK_US    MAXBLOCK_US_VALUEULL

struct key_t {
    int  w_k_stack_id;
    char waker[TASK_COMM_LEN];
    char target[TASK_COMM_LEN];
};
BPF_HASH(counts, struct key_t);
BPF_HASH(start, u32);
BPF_STACK_TRACE(stack_traces, STACK_STORAGE_SIZE);

int offcpu(struct pt_regs *ctx) {
    u32 pid = bpf_get_current_pid_tgid();
    struct task_struct *p = (struct task_struct *) bpf_get_current_task();
    u64 ts;

    if (FILTER)
        return 0;

    ts = bpf_ktime_get_ns();
    start.update(&pid, &ts);
    return 0;
}

int waker(struct pt_regs *ctx, struct task_struct *p) {
    u32 pid = p->pid;
    u64 delta, *tsp, ts;

    tsp = start.lookup(&pid);
    if (tsp == 0)
        return 0;        // missed start
    start.delete(&pid);

    if (FILTER)
        return 0;

    // calculate delta time
    delta = bpf_ktime_get_ns() - *tsp;
    delta = delta / 1000;
    if ((delta < MINBLOCK_US) || (delta > MAXBLOCK_US))
        return 0;

    struct key_t key = {};

    key.w_k_stack_id = stack_traces.get_stackid(ctx, BPF_F_REUSE_STACKID);
    bpf_probe_read(&key.target, sizeof(key.target), p->comm);
    bpf_get_current_comm(&key.waker, sizeof(key.waker));

    counts.increment(key, delta);
    return 0;
}
"""
if args.pid:
    filter = 'pid != %s' % args.pid
elif args.useronly:
    filter = 'p->flags & PF_KTHREAD'
else:
    filter = '0'
bpf_text = bpf_text.replace('FILTER', filter)

# set stack storage size
bpf_text = bpf_text.replace('STACK_STORAGE_SIZE', str(args.stack_storage_size))
bpf_text = bpf_text.replace('MINBLOCK_US_VALUE', str(args.min_block_time))
bpf_text = bpf_text.replace('MAXBLOCK_US_VALUE', str(args.max_block_time))

if debug or args.ebpf:
    print(bpf_text)
    if args.ebpf:
        exit()

# initialize BPF
b = BPF(text=bpf_text)
b.attach_kprobe(event="schedule", fn_name="offcpu")
b.attach_kprobe(event="try_to_wake_up", fn_name="waker")
matched = b.num_open_kprobes()
if matched == 0:
    print("0 functions traced. Exiting.")
    exit()

# header
if not folded:
    print("Tracing blocked time (us) by kernel stack", end="")
    if duration < 99999999:
        print(" for %d secs." % duration)
    else:
        print("... Hit Ctrl-C to end.")

# output
while (1):
    try:
        sleep(duration)
    except KeyboardInterrupt:
        # as cleanup can take many seconds, trap Ctrl-C:
        signal.signal(signal.SIGINT, signal_ignore)

    if not folded:
        print()
    missing_stacks = 0
    has_enomem = False
    counts = b.get_table("counts")
    stack_traces = b.get_table("stack_traces")
    for k, v in sorted(counts.items(), key=lambda counts: counts[1].value):
        # handle get_stackid errors
        # check for an ENOMEM error
        if k.w_k_stack_id == -errno.ENOMEM:
            missing_stacks += 1
            continue

        waker_kernel_stack = [] if k.w_k_stack_id < 1 else \
            reversed(list(stack_traces.walk(k.w_k_stack_id))[1:])

        if folded:
            # print folded stack output
            line = \
                [k.waker] + \
                [b.ksym(addr)
                    for addr in reversed(list(waker_kernel_stack))] + \
                [k.target]
            printb(b"%s %d" % (b";".join(line), v.value))
        else:
            # print default multi-line stack output
            printb(b"    %-16s %s" % (b"target:", k.target))
            for addr in waker_kernel_stack:
                printb(b"    %-16x %s" % (addr, b.ksym(addr)))
            printb(b"    %-16s %s" % (b"waker:", k.waker))
            print("        %d\n" % v.value)
    counts.clear()

    if missing_stacks > 0:
        enomem_str = " Consider increasing --stack-storage-size."
        print("WARNING: %d stack traces could not be displayed.%s" %
            (missing_stacks, enomem_str),
            file=stderr)

    if not folded:
        print("Detaching...")
    exit()
<|endoftext|>


<commit_before>import datetime
import optparse
import os

import cloudfiles

from django.conf import settings
from django.core.management.base import BaseCommand
from cumulus.settings import CUMULUS
from cumulus.storage import sync_headers

class Command(BaseCommand):
    help = "Synchronizes static media to cloud files."

    option_list = BaseCommand.option_list + (
        optparse.make_option('-w', '--wipe',
            action='store_true', dest='wipe', default=False,
            help="Wipes out entire contents of container first."),
        optparse.make_option('-t', '--test-run',
            action='store_true', dest='test_run', default=False,
            help="Performs a test run of the sync."),
        optparse.make_option('-c', '--container',
            dest='container', help="Override STATIC_CONTAINER."),
    )

    # settings from cumulus.settings
    USERNAME         = CUMULUS['USERNAME']
    API_KEY          = CUMULUS['API_KEY']
    STATIC_CONTAINER = CUMULUS['STATIC_CONTAINER']
    USE_SERVICENET   = CUMULUS['SERVICENET']
    FILTER_LIST      = CUMULUS['FILTER_LIST']
    AUTH_URL         = CUMULUS['AUTH_URL']

    # paths
    DIRECTORY        = os.path.abspath(settings.STATIC_ROOT)
    STATIC_URL       = settings.STATIC_URL

    if not DIRECTORY.endswith('/'):
        DIRECTORY = DIRECTORY + '/'

    if STATIC_URL.startswith('/'):
        STATIC_URL = STATIC_URL[1:]

    local_object_names = []
    create_count = 0
    upload_count = 0
    update_count = 0
    skip_count = 0
    delete_count = 0
    conn = None
    container = None

    def handle(self, *args, **options):
        self.wipe = options.get('wipe')
        self.test_run = options.get('test_run')
        self.verbosity = int(options.get('verbosity'))
        self.STATIC_CONTAINER = options.get('container', self.STATIC_CONTAINER)
        self.sync_files()

    def sync_files(self):
        self.conn = cloudfiles.get_connection(username = self.USERNAME,
                                              api_key = self.API_KEY,
                                              authurl = self.AUTH_URL,
                                              servicenet=self.USE_SERVICENET)

        try:
            self.container = self.conn.get_container(self.STATIC_CONTAINER)
        except cloudfiles.errors.NoSuchContainer:
            self.container = self.conn.create_container(self.STATIC_CONTAINER)

        if not self.container.is_public():
            self.container.make_public()

        # if -w option is provided, wipe out the contents of the container
        if self.wipe:
            if self.test_run:
                print "Wipe would delete %d objects." % self.container.object_count
            else:
                print "Deleting %d objects..." % self.container.object_count
                for cloud_obj in self.container.get_objects():
                    self.container.delete_object(cloud_obj.name)

        # walk through the directory, creating or updating files on the cloud
        os.path.walk(self.DIRECTORY, self.upload_files, "foo")

        # remove any files on remote that don't exist locally
        self.delete_files()

        # print out the final tally to the cmd line
        self.update_count = self.upload_count - self.create_count
        print
        if self.test_run:
            print "Test run complete with the following results:"
        print "Skipped %d. Created %d. Updated %d. Deleted %d." % (
            self.skip_count, self.create_count, self.update_count, self.delete_count)

    def upload_files(self, arg, dirname, names):
        # upload or skip items
        for item in names:
            if item in self.FILTER_LIST:
                continue # Skip files we don't want to sync

            file_path = os.path.join(dirname, item)
            if os.path.isdir(file_path):
                continue # Don't try to upload directories

            object_name = self.STATIC_URL + file_path.split(self.DIRECTORY)[1]
            self.local_object_names.append(object_name)

            try:
                cloud_obj = self.container.get_object(object_name)
            except cloudfiles.errors.NoSuchObject:
                cloud_obj = self.container.create_object(object_name)
                self.create_count += 1

            cloud_datetime = (cloud_obj.last_modified and
                              datetime.datetime.strptime(
                                cloud_obj.last_modified,
                                "%a, %d %b %Y %H:%M:%S %Z"
                              ) or None)
            local_datetime = datetime.datetime.utcfromtimestamp(
                                               os.stat(file_path).st_mtime)
            if cloud_datetime and local_datetime < cloud_datetime:
                self.skip_count += 1
                if self.verbosity > 1:
                    print "Skipped %s: not modified." % object_name
                continue

            if not self.test_run:
                cloud_obj.load_from_filename(file_path)
                sync_headers(cloud_obj)
            self.upload_count += 1
            if self.verbosity > 1:
                print "Uploaded", cloud_obj.name

    def delete_files(self):
        # remove any objects on the cloud that don't exist locally
        for cloud_name in self.container.list_objects():
            if cloud_name not in self.local_object_names:
                self.delete_count += 1
                if self.verbosity > 1:
                    print "Deleted %s" % cloud_name
                if not self.test_run:
                    self.container.delete_object(cloud_name)
<commit_msg>Update cumulus/management/commands/syncstatic.py<commit_after>import datetime
import optparse
import os

import cloudfiles

from django.conf import settings
from django.core.management.base import BaseCommand
from cumulus.settings import CUMULUS
from cumulus.storage import sync_headers

class Command(BaseCommand):
    help = "Synchronizes static media to cloud files."

    option_list = BaseCommand.option_list + (
        optparse.make_option('-w', '--wipe',
            action='store_true', dest='wipe', default=False,
            help="Wipes out entire contents of container first."),
        optparse.make_option('-t', '--test-run',
            action='store_true', dest='test_run', default=False,
            help="Performs a test run of the sync."),
        optparse.make_option('-c', '--container',
            dest='container', help="Override STATIC_CONTAINER."),
    )

    # settings from cumulus.settings
    USERNAME         = CUMULUS['USERNAME']
    API_KEY          = CUMULUS['API_KEY']
    STATIC_CONTAINER = CUMULUS['STATIC_CONTAINER']
    USE_SERVICENET   = CUMULUS['SERVICENET']
    FILTER_LIST      = CUMULUS['FILTER_LIST']
    AUTH_URL         = CUMULUS['AUTH_URL']

    # paths
    DIRECTORY        = os.path.abspath(settings.STATIC_ROOT)
    STATIC_URL       = settings.STATIC_URL

    if not DIRECTORY.endswith('/'):
        DIRECTORY = DIRECTORY + '/'

    if STATIC_URL.startswith('/'):
        STATIC_URL = STATIC_URL[1:]

    local_object_names = []
    create_count = 0
    upload_count = 0
    update_count = 0
    skip_count = 0
    delete_count = 0
    conn = None
    container = None

    def handle(self, *args, **options):
        self.wipe = options.get('wipe')
        self.test_run = options.get('test_run')
        self.verbosity = int(options.get('verbosity'))
        if hasattr(options, 'container'):
            self.STATIC_CONTAINER = options.get('container')
        self.sync_files()

    def sync_files(self):
        self.conn = cloudfiles.get_connection(username = self.USERNAME,
                                              api_key = self.API_KEY,
                                              authurl = self.AUTH_URL,
                                              servicenet=self.USE_SERVICENET)

        try:
            self.container = self.conn.get_container(self.STATIC_CONTAINER)
        except cloudfiles.errors.NoSuchContainer:
            self.container = self.conn.create_container(self.STATIC_CONTAINER)

        if not self.container.is_public():
            self.container.make_public()

        # if -w option is provided, wipe out the contents of the container
        if self.wipe:
            if self.test_run:
                print "Wipe would delete %d objects." % self.container.object_count
            else:
                print "Deleting %d objects..." % self.container.object_count
                for cloud_obj in self.container.get_objects():
                    self.container.delete_object(cloud_obj.name)

        # walk through the directory, creating or updating files on the cloud
        os.path.walk(self.DIRECTORY, self.upload_files, "foo")

        # remove any files on remote that don't exist locally
        self.delete_files()

        # print out the final tally to the cmd line
        self.update_count = self.upload_count - self.create_count
        print
        if self.test_run:
            print "Test run complete with the following results:"
        print "Skipped %d. Created %d. Updated %d. Deleted %d." % (
            self.skip_count, self.create_count, self.update_count, self.delete_count)

    def upload_files(self, arg, dirname, names):
        # upload or skip items
        for item in names:
            if item in self.FILTER_LIST:
                continue # Skip files we don't want to sync

            file_path = os.path.join(dirname, item)
            if os.path.isdir(file_path):
                continue # Don't try to upload directories

            object_name = self.STATIC_URL + file_path.split(self.DIRECTORY)[1]
            self.local_object_names.append(object_name)

            try:
                cloud_obj = self.container.get_object(object_name)
            except cloudfiles.errors.NoSuchObject:
                cloud_obj = self.container.create_object(object_name)
                self.create_count += 1

            cloud_datetime = (cloud_obj.last_modified and
                              datetime.datetime.strptime(
                                cloud_obj.last_modified,
                                "%a, %d %b %Y %H:%M:%S %Z"
                              ) or None)
            local_datetime = datetime.datetime.utcfromtimestamp(
                                               os.stat(file_path).st_mtime)
            if cloud_datetime and local_datetime < cloud_datetime:
                self.skip_count += 1
                if self.verbosity > 1:
                    print "Skipped %s: not modified." % object_name
                continue

            if not self.test_run:
                cloud_obj.load_from_filename(file_path)
                sync_headers(cloud_obj)
            self.upload_count += 1
            if self.verbosity > 1:
                print "Uploaded", cloud_obj.name

    def delete_files(self):
        # remove any objects on the cloud that don't exist locally
        for cloud_name in self.container.list_objects():
            if cloud_name not in self.local_object_names:
                self.delete_count += 1
                if self.verbosity > 1:
                    print "Deleted %s" % cloud_name
                if not self.test_run:
                    self.container.delete_object(cloud_name)
<|endoftext|>


<commit_before>#! coding:utf-8
# compatible for win32 / python 2 & 3
# TODO clean_url; frequency_tester; frequency_checker; string_converter; regex_mappers

import logging
from requests.utils import urlparse

dummy_logger = logging.getLogger('torequests.dummy')
main_logger = logging.getLogger('torequests.main')

def init_logger(name='', handler_path_levels=None,
                level=logging.INFO, formatter=None,
                formatter_str=None, datefmt="%Y-%m-%d %H:%M:%S"):
    '''Args:
    name = '' or logger obj.
    handler_path_levels = [['loggerfile.log',13],['','DEBUG'],['','info'],['','notSet']] # [[path,level]]
    level : the least level for the logger.
    formatter = logging.Formatter(
            '%(levelname)-6s  %(asctime)s  %(name)s (%(filename)s: %(lineno)s): %(message)s',
             "%Y-%m-%d %H:%M:%S")
    formatter_str = '%(levelname)-6s  %(asctime)s  %(name)s (%(funcName)s: %(lineno)s): %(message)s'

    custom formatter:
        %(asctime)s  %(created)f  %(filename)s  %(funcName)s  %(levelname)s  %(levelno)s  %(lineno)s   %(message)s   %(module)s    %(name)s   %(pathname)s   %(process)s   %(relativeCreated)s   %(thread)s  %(threadName)s
    '''
    levels = {'NOTSET': logging.NOTSET, 'DEBUG': logging.DEBUG, 'INFO': logging.INFO,
              'WARNING': logging.WARNING, 'ERROR': logging.ERROR, 'CRITICAL': logging.CRITICAL}
    if not formatter:
        if formatter_str:
            formatter_str = formatter_str
        else:
            formatter_str = '%(levelname)-6s  %(asctime)s  %(name)s (%(filename)s: %(lineno)s): %(message)s'
        formatter = logging.Formatter(formatter_str, datefmt=datefmt)
    logger = name if isinstance(
        name, logging.Logger) else logging.getLogger(str(name))
    logger.setLevel(level)
    handler_path_levels = handler_path_levels or [['', 'INFO']]
    # ---------------------------------------
    for each_handler in handler_path_levels:
        path, handler_level = each_handler
        handler = logging.FileHandler(
            path) if path else logging.StreamHandler()
        handler.setLevel(levels.get(handler_level.upper(), 1) if isinstance(
            handler_level, str) else handler_level)
        handler.setFormatter(formatter)
        logger.addHandler(handler)
    return logger

class FailureException(Exception):
    '''This class mainly used for __bool__,
        self.error for reviewing the source exception.'''

    def __init__(self, error):
        self.__dict__ = error.__dict__
        self.error = error
        self.ok = False

    def __bool__(self):
        return False

    def __str__(self):
        return repr(self)
<commit_msg>wip tool kits for crawler<commit_after>#! coding:utf-8
# compatible for win32 / python 2 & 3
# TODO clean_url; frequency_tester; frequency_checker; string_converter;
# regex_mappers

import logging
import re
from requests.utils import urlparse

dummy_logger = logging.getLogger('torequests.dummy')
main_logger = logging.getLogger('torequests.main')

def init_logger(name='', handler_path_levels=None,
                level=logging.INFO, formatter=None,
                formatter_str=None, datefmt="%Y-%m-%d %H:%M:%S"):
    '''Args:
    name = '' or logger obj.
    handler_path_levels = [['loggerfile.log',13],['','DEBUG'],['','info'],['','notSet']] # [[path,level]]
    level : the least level for the logger.
    formatter = logging.Formatter(
            '%(levelname)-6s  %(asctime)s  %(name)s (%(filename)s: %(lineno)s): %(message)s',
             "%Y-%m-%d %H:%M:%S")
    formatter_str = '%(levelname)-6s  %(asctime)s  %(name)s (%(funcName)s: %(lineno)s): %(message)s'

    custom formatter:
        %(asctime)s  %(created)f  %(filename)s  %(funcName)s  %(levelname)s  %(levelno)s  %(lineno)s   %(message)s   %(module)s    %(name)s   %(pathname)s   %(process)s   %(relativeCreated)s   %(thread)s  %(threadName)s
    '''
    levels = {'NOTSET': logging.NOTSET, 'DEBUG': logging.DEBUG, 'INFO': logging.INFO,
              'WARNING': logging.WARNING, 'ERROR': logging.ERROR, 'CRITICAL': logging.CRITICAL}
    if not formatter:
        if formatter_str:
            formatter_str = formatter_str
        else:
            formatter_str = '%(levelname)-6s  %(asctime)s  %(name)s (%(filename)s: %(lineno)s): %(message)s'
        formatter = logging.Formatter(formatter_str, datefmt=datefmt)
    logger = name if isinstance(
        name, logging.Logger) else logging.getLogger(str(name))
    logger.setLevel(level)
    handler_path_levels = handler_path_levels or [['', 'INFO']]
    # ---------------------------------------
    for each_handler in handler_path_levels:
        path, handler_level = each_handler
        handler = logging.FileHandler(
            path) if path else logging.StreamHandler()
        handler.setLevel(levels.get(handler_level.upper(), 1) if isinstance(
            handler_level, str) else handler_level)
        handler.setFormatter(formatter)
        logger.addHandler(handler)
    return logger

class FailureException(Exception):
    '''This class mainly used for __bool__,
        self.error for reviewing the source exception.'''

    def __init__(self, error):
        self.__dict__ = error.__dict__
        self.error = error
        self.ok = False

    def __bool__(self):
        return False

    def __str__(self):
        return repr(self)

class String(object):
    '''Tool kits for string converter'''
    pass

class Time(object):
    '''Tool kits for time converter'''
    pass

class Frequency(object):
    '''guess the anti-crawling frequency'''
    pass

class RegexMapper(object):
    '''dict-like for regex mapper: input string, output obj.
    support persistence rules;
    flags: set multi flags combined with '|', FLAG1 | FLAG2 | FLAG3
    '''

    def __init__(self, name=None, file_path=None, match_mode='search', flags=None):
        # init regex from string to compiled regex
        self.match_mode = match_mode
        self.rules = []  # list of (str, obj)
        self.rules_compile = []
        pass

    def init_regex(self):
        pass

    def find(self, string):
        'return only one matching obj'
        pass

    def findall(self, string):
        'return all matching obj'
        pass

    def regex_walker(self):
        'return generator'
        pass

    def save(self, file_path):
        pass

    def load(self, file_path):
        pass

    def __getitem__(self, key):
        pass

    def __setitem__(self, key, value):
        pass

<|endoftext|>


<commit_before>from random import choice

from tota.utils import adjacent_positions

AUTHOR = 'fisa'

def create():

    def noob_hero_logic(self, things, t):
        actions = 'move', 'attack', 'fireball', 'heal', 'stun'
        positions = adjacent_positions(self.position)

        return 'stun', self.position
        return choice(actions), choice(positions)

    return noob_hero_logic
<commit_msg>Fix debug noob<commit_after>from random import choice

from tota.utils import adjacent_positions

AUTHOR = 'fisa'

def create():

    def noob_hero_logic(self, things, t):
        actions = 'move', 'attack', 'fireball', 'heal', 'stun'
        positions = adjacent_positions(self.position)

        return choice(actions), choice(positions)

    return noob_hero_logic
<|endoftext|>


<commit_before>#!/usr/bin/env python

__version__ = "1.1.0"
__author__ = "Deren Eaton"

from .Toytree import ToyTree as tree
from .Toytree import RawTree as _rawtree
from .Randomtree import RandomTree as rtree
from .Multitree import MultiTree as mtree
from .PCM import PCM as pcm

# make a color palette easily accessible and an iter cycling version
from .TreeStyle import COLORS1 as colors
from .TreeStyle import COLORS2 as darkcolors
import itertools as _itertools
icolors1 = _itertools.cycle(colors)
icolors2 = _itertools.cycle(darkcolors)
<commit_msg>tag to 0.1.1 with PCM and update cookbooks<commit_after>#!/usr/bin/env python

__version__ = "1.1.1"
__author__ = "Deren Eaton"

from .Toytree import ToyTree as tree
from .Toytree import RawTree as _rawtree
from .Randomtree import RandomTree as rtree
from .Multitree import MultiTree as mtree
from .PCM import PCM as pcm

# make a color palette easily accessible and an iter cycling version
from .TreeStyle import COLORS1 as colors
from .TreeStyle import COLORS2 as darkcolors
import itertools as _itertools
icolors1 = _itertools.cycle(colors)
icolors2 = _itertools.cycle(darkcolors)
<|endoftext|>