etalage/pois.py

   1 # -*- coding: utf-8 -*-
   2
   3
   4 # Etalage -- Open Data POIs portal
   5 # By: Emmanuel Raviart <eraviart@easter-eggs.com>
   6 #
   7 # Copyright (C) 2011, 2012 Easter-eggs
   8 # http://gitorious.org/infos-pratiques/etalage
   9 #
  10 # This file is part of Etalage.
  11 #
  12 # Etalage is free software; you can redistribute it and/or modify
  13 # it under the terms of the GNU Affero General Public License as
  14 # published by the Free Software Foundation, either version 3 of the
  15 # License, or (at your option) any later version.
  16 #
  17 # Etalage is distributed in the hope that it will be useful,
  18 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  19 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  20 # GNU Affero General Public License for more details.
  21 #
  22 # You should have received a copy of the GNU Affero General Public License
  23 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
  24
  25
  26 """Objects for POIs"""
  27
  28
  29 from copy import copy
  30 import datetime
  31 import itertools
  32 import logging
  33 import math
  34 import sys
  35 import urlparse
  36 import urllib
  37
  38 import bson
  39 from biryani import strings
  40 from suq import representations
  41 import webob.multidict
  42
  43 from . import conf, conv, ramdb, urls
  44
  45
  46 __all__ = ['Cluster', 'Field', 'get_first_field', 'iter_fields', 'Poi', 'pop_first_field']
  47
  48 log = logging.getLogger(__name__)
  49
  50
  51 class Cluster(representations.UserRepresentable):
  52     bottom = None  # South latitude of rectangle enclosing all POIs of cluster
  53     center_latitude = None  # Copy of center_pois[*].geo[0] for quick access
  54     center_longitude = None  # Copy of center_pois[*].geo[1] for quick access
  55     center_pois = None  # POIs at the center of cluster, sharing the same coordinates
  56      # False = Not competent for current territory, None = Competent for any territory or unknown territory,
  57      # True = Competent for current territory
  58     competent = False
  59     count = None  # Number of POIs in cluster
  60     left = None  # West longitude of rectangle enclosing all POIs of cluster
  61     right = None  # East longitude of rectangle enclosing all POIs of cluster
  62     top = None  # North latitude of rectangle enclosing all POIs of cluster
  63
  64
  65 class Field(representations.UserRepresentable):
  66     id = None  # Petitpois id = format of value
  67     kind = None
  68     label = None
  69     relation = None
  70     title = None
  71     type = None
  72     value = None
  73
  74     def __init__(self, **attributes):
  75         if attributes:
  76             self.set_attributes(**attributes)
  77
  78     def get_first_field(self, id, label = None):
  79         # Note: Only for composite fields.
  80         return get_first_field(self.value, id, label = label)
  81
  82     @property
  83     def is_composite(self):
  84         return self.id in ('adr', 'date-range', 'source')
  85
  86     def iter_csv_fields(self, ctx, counts_by_label, parent_ref = None):
  87         """Iter fields, entering inside composite fields."""
  88         if self.value is not None:
  89             if self.is_composite:
  90                 same_label_index = counts_by_label.get(self.label, 0)
  91                 ref = (parent_ref or []) + [self.label, same_label_index]
  92                 field_counts_by_label = {}
  93                 for field in self.value:
  94                     for subfield_ref, subfield in field.iter_csv_fields(ctx, field_counts_by_label,
  95                             parent_ref = ref):
  96                         yield subfield_ref, subfield
  97                 if field_counts_by_label:
  98                     # Some subfields were not empty, so increment number of exported fields having the same label.
  99                     counts_by_label[self.label] = same_label_index + 1
 100             elif self.id in ('autocompleters', 'checkboxes'):
 101                 field_attributes = self.__dict__.copy()
 102                 field_attributes['value'] = u'\n'.join(self.value)
 103                 field = Field(**field_attributes)
 104                 same_label_index = counts_by_label.get(field.label, 0)
 105                 yield (parent_ref or []) + [field.label, same_label_index], field
 106                 counts_by_label[field.label] = same_label_index + 1
 107             elif self.id == 'commune':
 108                 field_attributes = self.__dict__.copy()
 109                 field_attributes['label'] = u'Code Insee commune'  # Better than "Commune"
 110                 field = Field(**field_attributes)
 111                 same_label_index = counts_by_label.get(field.label, 0)
 112                 yield (parent_ref or []) + [field.label, same_label_index], field
 113                 counts_by_label[field.label] = same_label_index + 1
 114             elif self.id == 'geo':
 115                 for field in (
 116                         Field(id = 'float', value = self.value[0], label = u'Latitude'),
 117                         Field(id = 'float', value = self.value[1], label = u'Longitude'),
 118                         Field(id = 'int', value = self.value[2], label = u'Précision'),
 119                         ):
 120                     for subfield_ref, subfield in field.iter_csv_fields(ctx, counts_by_label, parent_ref = parent_ref):
 121                         yield subfield_ref, subfield
 122             elif self.id == 'links':
 123                 field_attributes = self.__dict__.copy()
 124                 field_attributes['value'] = u'\n'.join(
 125                     unicode(object_id)
 126                     for object_id in self.value
 127                     )
 128                 field = Field(**field_attributes)
 129                 same_label_index = counts_by_label.get(field.label, 0)
 130                 yield (parent_ref or []) + [field.label, same_label_index], field
 131                 counts_by_label[field.label] = same_label_index + 1
 132             elif self.id == 'poi-last-update':
 133                 last_update_field = copy(self)
 134                 last_update_field.value = last_update_field.value.strftime('%d/%m/%Y')
 135                 last_update_label_index = counts_by_label.get(self.label, 0)
 136                 yield (parent_ref or []) + [self.label, last_update_label_index], last_update_field
 137                 counts_by_label[self.label] = last_update_label_index + 1
 138             elif self.id == 'postal-distribution':
 139                 postal_code, postal_routing = conv.check(conv.split_postal_distribution)(self.value, state = ctx)
 140                 for field in (
 141                         Field(id = 'postal-code', value = postal_code, label = u'Code postal'),
 142                         Field(id = 'postal-routing', value = postal_routing, label = u'Localité'),
 143                         ):
 144                     for subfield_ref, subfield in field.iter_csv_fields(ctx, counts_by_label, parent_ref = parent_ref):
 145                         yield subfield_ref, subfield
 146             elif self.id == 'street-address':
 147                 for item_value in self.value.split('\n'):
 148                     item_value = item_value.strip()
 149                     item_field_attributes = self.__dict__.copy()
 150                     item_field_attributes['id'] = 'street-address-lines'  # Change ID to avoid infinite recursion.
 151                     # item_field_attributes['label'] = u'Adresse'  # Better than "N° et libellé de voie"?
 152                     item_field_attributes['value'] = item_value
 153                     item_field = Field(**item_field_attributes)
 154                     for subfield_ref, subfield in item_field.iter_csv_fields(ctx, counts_by_label,
 155                             parent_ref = parent_ref):
 156                         yield subfield_ref, subfield
 157             elif self.id == 'territories':
 158                 territories = [
 159                     territory
 160                     for territory in (
 161                         ramdb.territory_by_id.get(territory_id)
 162                         for territory_id in self.value
 163                         )
 164                     if territory is not None
 165                     ]
 166                 if territories:
 167                     field_attributes = self.__dict__.copy()
 168                     field_attributes['value'] = u'\n'.join(
 169                         territory.main_postal_distribution_str
 170                         for territory in territories
 171                         )
 172                     field = Field(**field_attributes)
 173                     same_label_index = counts_by_label.get(field.label, 0)
 174                     yield (parent_ref or []) + [field.label, same_label_index], field
 175                     counts_by_label[field.label] = same_label_index + 1
 176             elif self.id == 'territory':
 177                 territory = ramdb.territory_by_id.get(self.value)
 178                 if territory is not None:
 179                     field_attributes = self.__dict__.copy()
 180                     field_attributes['value'] = territory.main_postal_distribution_str
 181                     field = Field(**field_attributes)
 182                     same_label_index = counts_by_label.get(field.label, 0)
 183                     yield (parent_ref or []) + [field.label, same_label_index], field
 184                     counts_by_label[field.label] = same_label_index + 1
 185             elif isinstance(self.value, list):
 186                 for item_value in self.value:
 187                     item_field_attributes = self.__dict__.copy()
 188                     item_field_attributes['value'] = item_value
 189                     item_field = Field(**item_field_attributes)
 190                     for subfield_ref, subfield in item_field.iter_csv_fields(ctx, counts_by_label,
 191                             parent_ref = parent_ref):
 192                         yield subfield_ref, subfield
 193             else:
 194                 # Note: self.value is now always a single value, not a list.
 195                 same_label_index = counts_by_label.get(self.label, 0)
 196                 yield (parent_ref or []) + [self.label, same_label_index], self
 197                 counts_by_label[self.label] = same_label_index + 1
 198
 199     @property
 200     def linked_pois_id(self):
 201         if self.id not in ('link', 'links'):
 202             return None
 203         if self.value is None:
 204             return None
 205         if isinstance(self.value, list):
 206             return self.value
 207         if isinstance(self.value, basestring):
 208             # When field is a CSV field, links are a linefeed-separated list of IDs
 209             return [
 210                 bson.objectid.ObjectId(id_str)
 211                 for id_str in self.value.split()
 212                 ]
 213         return [self.value]
 214
 215     @classmethod
 216     def load(cls, id, metadata, value):
 217         if len(metadata) != (1 if 'kind' in metadata else 0) \
 218                 + (1 if 'label' in metadata else 0) \
 219                 + (1 if 'relation' in metadata else 0) \
 220                 + (1 if 'type' in metadata else 0) \
 221                 + (1 + len(metadata['positions']) if 'positions' in metadata else 0):
 222             log.warning('Unexpected attributes in field {0}, metadata {1}, value {2}'.format(id, metadata, value))
 223         if 'positions' in metadata:
 224             fields_position = {}
 225             fields = []
 226             for field_id in metadata['positions']:
 227                 field_position = fields_position.get(field_id, 0)
 228                 fields_position[field_id] = field_position + 1
 229                 field_metadata = metadata[field_id][field_position]
 230                 field_value = value[field_id][field_position]
 231                 fields.append(cls.load(field_id, field_metadata, field_value))
 232             value = fields or None
 233         elif id == 'territories':
 234             # Replace each kind-code with the corresponding territory ID.
 235             if value is not None:
 236                 value = [
 237                     territory_id
 238                     for territory_id in (
 239                         ramdb.territory_id_by_kind_code.get((territory_kind_code['kind'],
 240                             territory_kind_code['code']))
 241                         for territory_kind_code in value
 242                         )
 243                     if territory_id is not None
 244                     ]
 245         return cls(
 246             id = id,
 247             kind = metadata.get('kind'),
 248             label = metadata['label'],
 249             relation = metadata.get('relation'),
 250             type = metadata.get('type'),
 251             value = value,
 252             )
 253
 254     def set_attributes(self, **attributes):
 255         """Set given attributes and return a boolean stating whether existing attributes have changed."""
 256         changed = False
 257         for name, value in attributes.iteritems():
 258             if value is getattr(self.__class__, name, UnboundLocalError):
 259                 if value is not getattr(self, name, UnboundLocalError):
 260                     delattr(self, name)
 261                     changed = True
 262             elif value is not getattr(self, name, UnboundLocalError):
 263                 setattr(self, name, value)
 264                 changed = True
 265         return changed
 266
 267
 268 class Poi(representations.UserRepresentable):
 269     _id = None
 270     # IDs of territories for which POI is fully competent. None when POI has no notion of competence territory
 271     competence_territories_id = None
 272     fields = None
 273     geo = None
 274     ids_by_category_slug = {}
 275     ids_by_competence_territory_id = {}
 276     ids_by_begin_datetime = []
 277     ids_by_end_datetime = []
 278     ids_by_last_update_datetime = []
 279     ids_by_parent_id = {}  # class attribute
 280     ids_by_presence_territory_id = {}
 281     ids_by_word = {}
 282     indexed_ids = set()
 283     instance_by_id = {}
 284     last_update_datetime = None
 285     last_update_organization = None
 286     name = None
 287     parent_id = None
 288     petitpois_url = None  # class attribute defined in subclass. URL of Petitpois site
 289     postal_distribution_str = None
 290     schema_name = None
 291     slug_by_id = {}
 292     street_address = None
 293     subclass_by_database_and_schema_name = {}
 294     theme_slug = None
 295
 296     def __init__(self, **attributes):
 297         if attributes:
 298             self.set_attributes(**attributes)
 299
 300     @classmethod
 301     def clear_indexes(cls):
 302         cls.indexed_ids.clear()
 303         cls.instance_by_id.clear()
 304         cls.ids_by_parent_id.clear()
 305         cls.ids_by_category_slug.clear()
 306         cls.ids_by_competence_territory_id.clear()
 307         cls.ids_by_presence_territory_id.clear()
 308         cls.ids_by_word.clear()
 309         cls.slug_by_id.clear()
 310         cls.subclass_by_database_and_schema_name.clear()
 311
 312     @classmethod
 313     def extract_non_territorial_search_data(cls, ctx, data):
 314         return dict(
 315             categories_slug = data['categories_slug'],
 316             term = data['term'],
 317             )
 318
 319     @classmethod
 320     def extract_search_inputs_from_params(cls, ctx, params):
 321         return dict(
 322             categories_slug = params.getall('category'),
 323             term = params.get('term'),
 324             territory = params.get('territory'),
 325             )
 326
 327     def generate_all_fields(self):
 328         """Return all fields of POI including dynamic ones (ie linked fields, etc)."""
 329         fields = self.fields[:] if self.fields is not None else []
 330
 331         # Add children POIs as linked fields.
 332         children = sorted(
 333             (
 334                 self.instance_by_id[child_id]
 335                 for child_id in self.ids_by_parent_id.get(self._id, set())
 336                 ),
 337             key = lambda child: (child.schema_name, child.name),
 338             )
 339         for child in children:
 340             fields.append(
 341                 Field(
 342                     id = 'link',
 343                     label = ramdb.schema_title_by_name[child.schema_name],
 344                     relation = 'child',
 345                     name = child.name,
 346                     value = child._id,
 347                     ))
 348
 349         # Add last-update field.
 350         fields.append(Field(id = 'last-update', label = u"Dernière mise à jour", value = u' par '.join(
 351             unicode(fragment)
 352             for fragment in (
 353                 self.last_update_datetime.strftime('%Y-%m-%d %H:%M') if self.last_update_datetime is not None else None,
 354                 self.last_update_organization,
 355                 )
 356             if fragment
 357             )))
 358         return fields
 359
 360     def get_first_field(self, id, label = None):
 361         return get_first_field(self.fields, id, label = label)
 362
 363     def get_full_url(self, ctx, params_prefix = 'cmq_'):
 364         if ctx.container_base_url is None:
 365             return urls.get_full_url(ctx, 'organismes', self.slug, self._id)
 366         else:
 367             parsed_container_base_url = urlparse.urlparse(ctx.container_base_url)
 368             params = dict([
 369                 ('{0}path'.format(params_prefix), urls.get_url(ctx, 'organismes', self.slug, self._id))
 370                 ])
 371             params.update(dict(urlparse.parse_qsl(parsed_container_base_url.query)))
 372             return urlparse.urljoin(
 373                 '{0}://{1}{2}'.format(
 374                     parsed_container_base_url.scheme,
 375                     parsed_container_base_url.netloc,
 376                     parsed_container_base_url.path
 377                     ),
 378                 '?{0}#{0}'.format(urllib.urlencode(params)),
 379                 )
 380
 381     @classmethod
 382     def get_search_params_name(cls, ctx):
 383         return set(
 384             cls.rename_input_to_param(name)
 385             for name in cls.extract_search_inputs_from_params(ctx, webob.multidict.MultiDict()).iterkeys()
 386             )
 387
 388     @classmethod
 389     def get_visibility_params_names(cls, ctx):
 390         visibility_params = list(cls.get_search_params_name(ctx))
 391         visibility_params.extend(['checkboxes', 'directory', 'export', 'gadget', 'legend', 'list', 'map', 'minisite'])
 392         return [
 393             'hide_{0}'.format(visibility_param)
 394             for visibility_param in visibility_params
 395             ]
 396
 397     def index(self, indexed_poi_id):
 398         poi_bson = self.bson
 399         metadata = poi_bson['metadata']
 400         for category_slug in (metadata.get('categories-index') or set()):
 401             self.ids_by_category_slug.setdefault(category_slug, set()).add(indexed_poi_id)
 402
 403         if conf['index.date.field']:
 404             for date_range_index, date_range_metadata in enumerate(metadata.get('date-range') or []):
 405                 if date_range_metadata['label'] == conf['index.date.field']:
 406                     date_range_values = poi_bson['date-range'][date_range_index]
 407                     date_range_begin = date_range_values.get('date-range-begin', [None])[0]
 408                     date_range_end = date_range_values.get('date-range-end', [None])[0]
 409
 410                     if date_range_begin is not None:
 411                         for index, (begin_datetime, poi_id) in enumerate(self.ids_by_begin_datetime):
 412                             if begin_datetime is not None and begin_datetime > date_range_begin:
 413                                 break
 414                     else:
 415                         index = 0
 416                     self.ids_by_begin_datetime.insert(index, (date_range_begin, indexed_poi_id))
 417                     if date_range_end is not None:
 418                         for index, (end_datetime, poi_id) in enumerate(self.ids_by_end_datetime):
 419                             if end_datetime is not None and end_datetime < date_range_end:
 420                                 break
 421                     else:
 422                         index = 0
 423                     self.ids_by_end_datetime.insert(index, (date_range_end, indexed_poi_id))
 424
 425             if not metadata.get('date-range'):
 426                 self.ids_by_begin_datetime.append((None, indexed_poi_id))
 427                 self.ids_by_end_datetime.append((None, indexed_poi_id))
 428         self.ids_by_last_update_datetime.append((self.last_update_datetime, indexed_poi_id))
 429
 430         for i, territory_metadata in enumerate(metadata.get('territories') or []):
 431             # Note: Don't fail when territory doesn't exist, because Etalage can be configured to ignore some kinds
 432             # of territories (cf conf['territories_kinds']).
 433             self.competence_territories_id = set(
 434                 territory_id
 435                 for territory_id in (
 436                     ramdb.territory_id_by_kind_code.get((territory_kind_code['kind'], territory_kind_code['code']))
 437                     for territory_kind_code in poi_bson['territories'][i]
 438                     )
 439                 if territory_id is not None
 440                 )
 441             for territory_id in self.competence_territories_id:
 442                 self.ids_by_competence_territory_id.setdefault(territory_id, set()).add(indexed_poi_id)
 443             break
 444         if not self.competence_territories_id:
 445             self.ids_by_competence_territory_id.setdefault(None, set()).add(indexed_poi_id)
 446
 447         poi_territories_id = set(
 448             territory_id
 449             for territory_id in (
 450                 ramdb.territory_id_by_kind_code.get((territory_kind_code['kind'], territory_kind_code['code']))
 451                 for territory_kind_code in metadata['territories-index']
 452                 if territory_kind_code['kind'] not in (u'Country', u'InternationalOrganization')
 453                 )
 454             if territory_id is not None
 455             ) if metadata.get('territories-index') is not None else None
 456         for territory_id in (poi_territories_id or set()):
 457             self.ids_by_presence_territory_id.setdefault(territory_id, set()).add(indexed_poi_id)
 458
 459         for word in strings.slugify(self.name).split(u'-'):
 460             self.ids_by_word.setdefault(word, set()).add(indexed_poi_id)
 461         self.slug_by_id[indexed_poi_id] = strings.slugify(self.name)
 462
 463     @classmethod
 464     def index_pois(cls):
 465         for self in cls.instance_by_id.itervalues():
 466             # Note: self._id is not added to cls.indexed_ids by method self.index(self._id) to allow
 467             # customizations where not all POIs are indexed (Passim for example).
 468             cls.indexed_ids.add(self._id)
 469             self.index(self._id)
 470             del self.bson
 471         if conf['index.date.field']:
 472             cls.ids_by_begin_datetime = sorted(
 473                 cls.ids_by_begin_datetime,
 474                 key = lambda t: t[0] or datetime.datetime(datetime.MINYEAR, 1, 1),
 475                 )
 476             cls.ids_by_end_datetime = sorted(
 477                 cls.ids_by_end_datetime,
 478                 key = lambda t: t[0] or datetime.datetime(datetime.MAXYEAR, 1, 1),
 479                 reverse = True
 480                 )
 481
 482         cls.ids_by_last_update_datetime = sorted(cls.ids_by_last_update_datetime, key = lambda t: t[0], reverse = True)
 483
 484     @classmethod
 485     def is_search_param_visible(cls, ctx, name):
 486         param_visibility_name = 'hide_{0}'.format(name)
 487         return getattr(ctx, param_visibility_name, False) \
 488             if param_visibility_name.startswith('show_') \
 489             else not getattr(ctx, param_visibility_name, False)
 490
 491     def iter_csv_fields(self, ctx):
 492         counts_by_label = {}
 493
 494         id_field = Field(id = 'poi-id', value = self._id, label = u'Identifiant')
 495         for subfield_ref, subfield in id_field.iter_csv_fields(ctx, counts_by_label):
 496             yield subfield_ref, subfield
 497
 498         last_update_field = Field(
 499             id = 'poi-last-update',
 500             value = self.last_update_datetime,
 501             label = u'Date de dernière modification'
 502             )
 503         for subfield_ref, subfield in last_update_field.iter_csv_fields(ctx, counts_by_label):
 504             yield subfield_ref, subfield
 505
 506         if self.fields is not None:
 507             for field in self.fields:
 508                 for subfield_ref, subfield in field.iter_csv_fields(ctx, counts_by_label):
 509                     yield subfield_ref, subfield
 510
 511     @classmethod
 512     def iter_ids(cls, ctx, categories_slug = None, competence_territories_id = None, competence_type = None,
 513             presence_territory = None, term = None):
 514         intersected_sets = []
 515
 516         if competence_territories_id is not None:
 517             competence_territories_sets = []
 518             if competence_type in (None, 'by_territory'):
 519                 competence_territories_sets.extend(
 520                     cls.ids_by_competence_territory_id.get(competence_territory_id)
 521                     for competence_territory_id in competence_territories_id
 522                     )
 523             if competence_type in (None, 'by_nature'):
 524                 competence_territories_sets.append(cls.ids_by_competence_territory_id.get(None))
 525             territory_competent_pois_id = ramdb.union_set(competence_territories_sets)
 526             if not territory_competent_pois_id:
 527                 return set()
 528             intersected_sets.append(territory_competent_pois_id)
 529
 530         if presence_territory is not None:
 531             territory_present_pois_id = cls.ids_by_presence_territory_id.get(presence_territory._id)
 532             if not territory_present_pois_id:
 533                 return set()
 534             intersected_sets.append(territory_present_pois_id)
 535
 536         if ctx.base_categories_slug is not None:
 537             base_categories_sets = []
 538             base_categories_slug = copy(ctx.base_categories_slug or [])
 539             for category_slug in set(base_categories_slug or []):
 540                 if category_slug is not None:
 541                     category_pois_id = cls.ids_by_category_slug.get(category_slug)
 542                     if category_pois_id:
 543                         base_categories_sets.append(category_pois_id)
 544             intersected_sets.append(ramdb.union_set(base_categories_sets))
 545
 546         for category_slug in set(categories_slug or []):
 547             if category_slug is not None:
 548                 category_pois_id = cls.ids_by_category_slug.get(category_slug)
 549                 if not category_pois_id:
 550                     return set()
 551                 intersected_sets.append(category_pois_id)
 552
 553         if conf['index.date.field']:
 554             current_datetime = datetime.datetime.utcnow()
 555             ids_by_begin_datetime_set = set()
 556             for poi_begin_datetime, poi_id in cls.ids_by_begin_datetime:
 557                 if poi_begin_datetime is None or current_datetime >= poi_begin_datetime:
 558                     ids_by_begin_datetime_set.add(poi_id)
 559                 else:
 560                     break
 561             ids_by_end_datetime_set = set()
 562             for poi_end_datetime, poi_id in cls.ids_by_end_datetime:
 563                 if poi_end_datetime is None or current_datetime <= poi_end_datetime:
 564                     ids_by_end_datetime_set.add(poi_id)
 565                 else:
 566                     break
 567             intersected_sets.append(ramdb.intersection_set([ids_by_begin_datetime_set, ids_by_end_datetime_set]))
 568
 569         # We should filter on term *after* having looked for competent organizations. Otherwise, when no organization
 570         # matching term is found, the nearest organizations will be used even when there are competent organizations
 571         # (that don't match the term).
 572         if term:
 573             prefixes = strings.slugify(term).split(u'-')
 574             pois_id_by_prefix = {}
 575             for prefix in prefixes:
 576                 if prefix in pois_id_by_prefix:
 577                     # TODO? Handle pois with several words sharing the same prefix?
 578                     continue
 579                 pois_id_by_prefix[prefix] = ramdb.union_set(
 580                     pois_id
 581                     for word, pois_id in cls.ids_by_word.iteritems()
 582                     if word.startswith(prefix)
 583                     ) or set()
 584             intersected_sets.extend(pois_id_by_prefix.itervalues())
 585
 586         found_pois_id = ramdb.intersection_set(intersected_sets)
 587         if found_pois_id is None:
 588             return cls.indexed_ids
 589         return found_pois_id
 590
 591     @classmethod
 592     def load(cls, poi_bson):
 593         metadata = poi_bson['metadata']
 594         last_update = metadata['last-update']
 595         if poi_bson.get('geo') is None:
 596             geo = None
 597         else:
 598             geo = poi_bson['geo'][0]
 599             if len(geo) > 2 and geo[2] == 0:
 600                 # Don't use geographical coordinates with a 0 accuracy because their coordinates may be None.
 601                 geo = None
 602         self = cls(
 603             _id = poi_bson['_id'],
 604             geo = geo,
 605             last_update_datetime = last_update['date'],
 606             last_update_organization = last_update['organization'],
 607             name = metadata['title'],
 608             schema_name = metadata['schema-name'],
 609             )
 610
 611         if conf['theme_field'] is None:
 612             theme_field_id = None
 613             theme_field_name = None
 614         else:
 615             theme_field_id = conf['theme_field']['id']
 616             theme_field_name = conf['theme_field'].get('name')
 617         fields_position = {}
 618         fields = []
 619         for field_id in metadata['positions']:
 620             field_position = fields_position.get(field_id, 0)
 621             fields_position[field_id] = field_position + 1
 622             field_metadata = metadata[field_id][field_position]
 623             field_value = poi_bson[field_id][field_position]
 624             field = Field.load(field_id, field_metadata, field_value)
 625             if field.id == u'adr' and self.postal_distribution_str is None:
 626                 for sub_field in (field.value or []):
 627                     if sub_field.id == u'postal-distribution':
 628                         self.postal_distribution_str = sub_field.value
 629                     elif sub_field.id == u'street-address':
 630                         self.street_address = sub_field.value
 631             elif field.id == u'link' and field.relation == u'parent':
 632                 assert self.parent is None, str(self)
 633                 self.parent_id = field.value
 634
 635             if field_id == theme_field_id and (
 636                     theme_field_name is None or theme_field_name == strings.slugify(field.label)):
 637                 if field.id == u'organism-type':
 638                     organism_type_slug = ramdb.category_slug_by_pivot_code.get(field.value)
 639                     if organism_type_slug is None:
 640                         log.warning('Ignoring organism type "{0}" without matching category.'.format(field.value))
 641                     else:
 642                         self.theme_slug = organism_type_slug
 643                 else:
 644                     theme_slug = strings.slugify(field.value)
 645                     if theme_slug in ramdb.category_by_slug:
 646                         self.theme_slug = theme_slug
 647                     else:
 648                         log.warning('Ignoring theme "{0}" without matching category.'.format(field.value))
 649
 650             fields.append(field)
 651         if fields:
 652             self.fields = fields
 653
 654         # Temporarily store bson in poi because it is needed by index_pois.
 655         self.bson = poi_bson
 656
 657         cls.instance_by_id[self._id] = self
 658         if self.parent_id is not None:
 659             cls.ids_by_parent_id.setdefault(self.parent_id, set()).add(self._id)
 660         return self
 661
 662     @classmethod
 663     def load_pois(cls):
 664         from . import model
 665         for db, petitpois_url in zip(model.dbs, conf['petitpois_url']):
 666             for poi_bson in db.pois.find({'metadata.deleted': {'$exists': False}}):
 667                 if (db.name, poi_bson['metadata']['schema-name']) not in cls.subclass_by_database_and_schema_name:
 668                     schema = db.schemas.find_one({'name': poi_bson['metadata']['schema-name']})
 669                     cls.subclass_by_database_and_schema_name[(db.name, poi_bson['metadata']['schema-name'])] = type(
 670                         'PoiWithPetitpois',
 671                         (cls,),
 672                         dict(
 673                             icon_url = schema.get('icon_url') if schema is not None else None,
 674                             petitpois_url = petitpois_url,
 675                             ),
 676                         )
 677                 cls.subclass_by_database_and_schema_name[(db.name, poi_bson['metadata']['schema-name'])].load(poi_bson)
 678             # Create subclasses for schema with no POI.
 679             for schema in db.schemas.find({}, ['name', 'icon_url']):
 680                 if (db.name, schema['name']) not in cls.subclass_by_database_and_schema_name:
 681                     cls.subclass_by_database_and_schema_name[(db.name, schema['name'])] = type(
 682                         'PoiWithPetitpois',
 683                         (cls,),
 684                         dict(
 685                             icon_url = schema.get('icon_url') if schema is not None else None,
 686                             petitpois_url = petitpois_url,
 687                             ),
 688                         )
 689
 690     @classmethod
 691     def make_inputs_to_search_data(cls):
 692         return conv.pipe(
 693             conv.struct(
 694                 dict(
 695                     base_territory = conv.input_to_postal_distribution_to_geolocated_territory,
 696                     categories_slug = conv.uniform_sequence(conv.input_to_category_slug),
 697                     term = conv.input_to_slug,
 698                     territory = conv.input_to_postal_distribution_to_geolocated_territory,
 699                     ),
 700                 default = 'drop',
 701                 drop_none_values = False,
 702                 ),
 703             conv.test_territory_in_base_territory,
 704             )
 705
 706     @property
 707     def parent(self):
 708         if self.parent_id is None:
 709             return None
 710         return self.instance_by_id.get(self.parent_id)
 711
 712     @classmethod
 713     def rename_input_to_param(cls, input_name):
 714         return dict(
 715             categories_slug = u'category',
 716             ).get(input_name, input_name)
 717
 718     def set_attributes(self, **attributes):
 719         """Set given attributes and return a boolean stating whether existing attributes have changed."""
 720         changed = False
 721         for name, value in attributes.iteritems():
 722             if value is getattr(self.__class__, name, UnboundLocalError):
 723                 if value is not getattr(self, name, UnboundLocalError):
 724                     delattr(self, name)
 725                     changed = True
 726             elif value is not getattr(self, name, UnboundLocalError):
 727                 setattr(self, name, value)
 728                 changed = True
 729         return changed
 730
 731     @property
 732     def slug(self):
 733         return strings.slugify(self.name)
 734
 735     @classmethod
 736     def sort_and_paginate_pois_list(cls, ctx, pager, poi_by_id, related_territories_id = None, reverse = False,
 737             territory = None, sort_key = None, **other_search_data):
 738         if territory is None:
 739             if sort_key is not None and sort_key == 'organism-type':
 740                 key = lambda poi: ([
 741                     ramdb.category_by_slug.get(ramdb.category_slug_by_pivot_code.get(field.value)) or field.value
 742                     for field in poi.fields
 743                     if field.id == 'organism-type'
 744                     ] or [''])[0]
 745             elif sort_key is not None and sort_key == 'last_update_datetime':
 746                 key = lambda poi: getattr(poi, sort_key, poi.name) if sort_key is not None else poi.name
 747                 reverse = True
 748             elif sort_key == 'name':
 749                 key = lambda poi: Poi.slug_by_id.get(poi._id)
 750             else:
 751                 key = lambda poi: getattr(poi, sort_key, Poi.slug_by_id.get(poi._id)) \
 752                         if sort_key is not None else Poi.slug_by_id.get(poi._id)
 753             pois = sorted(poi_by_id.itervalues(), key = key, reverse = reverse)
 754             return [
 755                 poi
 756                 for poi in itertools.islice(pois, pager.first_item_index, pager.last_item_number)
 757                 ]
 758         territory_latitude_cos = math.cos(math.radians(territory.geo[0]))
 759         territory_latitude_sin = math.sin(math.radians(territory.geo[0]))
 760
 761         if sort_key is None:
 762             key = lambda incompetence_distance_and_poi_triple: incompetence_distance_and_poi_triple[:2]
 763         elif sort_key == 'organism-type':
 764             key = lambda incompetence_distance_and_poi_triple: (
 765                 ([
 766                     ramdb.category_by_slug.get(ramdb.category_slug_by_pivot_code.get(field.value)) or field.value
 767                     for field in incompetence_distance_and_poi_triple[2].fields
 768                     if field.id == 'organism-type'
 769                     ] or [''])[0],
 770                 incompetence_distance_and_poi_triple[1],
 771                 )
 772         else:
 773             key = lambda incompetence_distance_and_poi_triple: (
 774                 getattr(incompetence_distance_and_poi_triple[2], sort_key, None),
 775                 incompetence_distance_and_poi_triple[1],
 776                 )
 777         incompetence_distance_and_poi_triples = sorted(
 778             (
 779                 (
 780                     # is not competent
 781                     poi.competence_territories_id is not None
 782                         and related_territories_id is not None
 783                         and related_territories_id.isdisjoint(poi.competence_territories_id),
 784                     # distance
 785                     6372.8 * math.acos(
 786                         round(
 787                             math.sin(math.radians(poi.geo[0])) * territory_latitude_sin
 788                             + math.cos(math.radians(poi.geo[0])) * territory_latitude_cos
 789                             * math.cos(math.radians(poi.geo[1] - territory.geo[1])),
 790                             13,
 791                         )) if poi.geo is not None else (sys.float_info.max, poi),
 792                     # POI
 793                     poi,
 794                     )
 795                 for poi in poi_by_id.itervalues()
 796                 ),
 797             key = key,
 798             reverse = reverse,
 799             )
 800         return [
 801             poi
 802             for incompetence, distance, poi in itertools.islice(incompetence_distance_and_poi_triples,
 803                 pager.first_item_index, pager.last_item_number)
 804             ]
 805
 806
 807 def get_first_field(fields, id, label = None):
 808     for field in iter_fields(fields, id, label = label):
 809         return field
 810     return None
 811
 812
 813 def iter_fields(fields, id, label = None):
 814     if fields is not None:
 815         for field in fields:
 816             if field.id == id and (label is None or field.label == label):
 817                 yield field
 818
 819
 820 def pop_first_field(fields, id, label = None):
 821     for field in iter_fields(fields, id, label = label):
 822         fields.remove(field)
 823         return field
 824     return None