Fix 500 error when "related_territories_id" is None
[infos-pratiques:etalage.git] / etalage / pois.py
1 # -*- coding: utf-8 -*-
2
3
4 # Etalage -- Open Data POIs portal
5 # By: Emmanuel Raviart <eraviart@easter-eggs.com>
6 #
7 # Copyright (C) 2011, 2012 Easter-eggs
8 # http://gitorious.org/infos-pratiques/etalage
9 #
10 # This file is part of Etalage.
11 #
12 # Etalage is free software; you can redistribute it and/or modify
13 # it under the terms of the GNU Affero General Public License as
14 # published by the Free Software Foundation, either version 3 of the
15 # License, or (at your option) any later version.
16 #
17 # Etalage is distributed in the hope that it will be useful,
18 # but WITHOUT ANY WARRANTY; without even the implied warranty of
19 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
20 # GNU Affero General Public License for more details.
21 #
22 # You should have received a copy of the GNU Affero General Public License
23 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
24
25
26 """Objects for POIs"""
27
28
29 from copy import copy
30 import datetime
31 import itertools
32 import logging
33 import math
34 import sys
35 import urlparse
36 import urllib
37
38 import bson
39 from biryani import strings
40 from suq import representations
41 import webob.multidict
42
43 from . import conf, conv, ramdb, urls
44
45
46 __all__ = ['Cluster', 'Field', 'get_first_field', 'iter_fields', 'Poi', 'pop_first_field']
47
48 log = logging.getLogger(__name__)
49
50
51 class Cluster(representations.UserRepresentable):
52     bottom = None  # South latitude of rectangle enclosing all POIs of cluster
53     center_latitude = None  # Copy of center_pois[*].geo[0] for quick access
54     center_longitude = None  # Copy of center_pois[*].geo[1] for quick access
55     center_pois = None  # POIs at the center of cluster, sharing the same coordinates
56      # False = Not competent for current territory, None = Competent for any territory or unknown territory,
57      # True = Competent for current territory
58     competent = False
59     count = None  # Number of POIs in cluster
60     left = None  # West longitude of rectangle enclosing all POIs of cluster
61     right = None  # East longitude of rectangle enclosing all POIs of cluster
62     top = None  # North latitude of rectangle enclosing all POIs of cluster
63
64
65 class Field(representations.UserRepresentable):
66     id = None  # Petitpois id = format of value
67     kind = None
68     label = None
69     relation = None
70     title = None
71     type = None
72     value = None
73
74     def __init__(self, **attributes):
75         if attributes:
76             self.set_attributes(**attributes)
77
78     def get_first_field(self, id, label = None):
79         # Note: Only for composite fields.
80         return get_first_field(self.value, id, label = label)
81
82     @property
83     def is_composite(self):
84         return self.id in ('adr', 'date-range', 'source')
85
86     def iter_csv_fields(self, ctx, counts_by_label, parent_ref = None):
87         """Iter fields, entering inside composite fields."""
88         if self.value is not None:
89             if self.is_composite:
90                 same_label_index = counts_by_label.get(self.label, 0)
91                 ref = (parent_ref or []) + [self.label, same_label_index]
92                 field_counts_by_label = {}
93                 for field in self.value:
94                     for subfield_ref, subfield in field.iter_csv_fields(ctx, field_counts_by_label,
95                             parent_ref = ref):
96                         yield subfield_ref, subfield
97                 if field_counts_by_label:
98                     # Some subfields were not empty, so increment number of exported fields having the same label.
99                     counts_by_label[self.label] = same_label_index + 1
100             elif self.id in ('autocompleters', 'checkboxes'):
101                 field_attributes = self.__dict__.copy()
102                 field_attributes['value'] = u'\n'.join(self.value)
103                 field = Field(**field_attributes)
104                 same_label_index = counts_by_label.get(field.label, 0)
105                 yield (parent_ref or []) + [field.label, same_label_index], field
106                 counts_by_label[field.label] = same_label_index + 1
107             elif self.id == 'commune':
108                 field_attributes = self.__dict__.copy()
109                 field_attributes['label'] = u'Code Insee commune'  # Better than "Commune"
110                 field = Field(**field_attributes)
111                 same_label_index = counts_by_label.get(field.label, 0)
112                 yield (parent_ref or []) + [field.label, same_label_index], field
113                 counts_by_label[field.label] = same_label_index + 1
114             elif self.id == 'geo':
115                 for field in (
116                         Field(id = 'float', value = self.value[0], label = u'Latitude'),
117                         Field(id = 'float', value = self.value[1], label = u'Longitude'),
118                         Field(id = 'int', value = self.value[2], label = u'Précision'),
119                         ):
120                     for subfield_ref, subfield in field.iter_csv_fields(ctx, counts_by_label, parent_ref = parent_ref):
121                         yield subfield_ref, subfield
122             elif self.id == 'links':
123                 field_attributes = self.__dict__.copy()
124                 field_attributes['value'] = u'\n'.join(
125                     unicode(object_id)
126                     for object_id in self.value
127                     )
128                 field = Field(**field_attributes)
129                 same_label_index = counts_by_label.get(field.label, 0)
130                 yield (parent_ref or []) + [field.label, same_label_index], field
131                 counts_by_label[field.label] = same_label_index + 1
132             elif self.id == 'poi-last-update':
133                 last_update_field = copy(self)
134                 last_update_field.value = last_update_field.value.strftime('%d/%m/%Y')
135                 last_update_label_index = counts_by_label.get(self.label, 0)
136                 yield (parent_ref or []) + [self.label, last_update_label_index], last_update_field
137                 counts_by_label[self.label] = last_update_label_index + 1
138             elif self.id == 'postal-distribution':
139                 postal_code, postal_routing = conv.check(conv.split_postal_distribution)(self.value, state = ctx)
140                 for field in (
141                         Field(id = 'postal-code', value = postal_code, label = u'Code postal'),
142                         Field(id = 'postal-routing', value = postal_routing, label = u'Localité'),
143                         ):
144                     for subfield_ref, subfield in field.iter_csv_fields(ctx, counts_by_label, parent_ref = parent_ref):
145                         yield subfield_ref, subfield
146             elif self.id == 'street-address':
147                 for item_value in self.value.split('\n'):
148                     item_value = item_value.strip()
149                     item_field_attributes = self.__dict__.copy()
150                     item_field_attributes['id'] = 'street-address-lines'  # Change ID to avoid infinite recursion.
151                     # item_field_attributes['label'] = u'Adresse'  # Better than "N° et libellé de voie"?
152                     item_field_attributes['value'] = item_value
153                     item_field = Field(**item_field_attributes)
154                     for subfield_ref, subfield in item_field.iter_csv_fields(ctx, counts_by_label,
155                             parent_ref = parent_ref):
156                         yield subfield_ref, subfield
157             elif self.id == 'territories':
158                 territories = [
159                     territory
160                     for territory in (
161                         ramdb.territory_by_id.get(territory_id)
162                         for territory_id in self.value
163                         )
164                     if territory is not None
165                     ]
166                 if territories:
167                     field_attributes = self.__dict__.copy()
168                     field_attributes['value'] = u'\n'.join(
169                         territory.main_postal_distribution_str
170                         for territory in territories
171                         )
172                     field = Field(**field_attributes)
173                     same_label_index = counts_by_label.get(field.label, 0)
174                     yield (parent_ref or []) + [field.label, same_label_index], field
175                     counts_by_label[field.label] = same_label_index + 1
176             elif self.id == 'territory':
177                 territory = ramdb.territory_by_id.get(self.value)
178                 if territory is not None:
179                     field_attributes = self.__dict__.copy()
180                     field_attributes['value'] = territory.main_postal_distribution_str
181                     field = Field(**field_attributes)
182                     same_label_index = counts_by_label.get(field.label, 0)
183                     yield (parent_ref or []) + [field.label, same_label_index], field
184                     counts_by_label[field.label] = same_label_index + 1
185             elif isinstance(self.value, list):
186                 for item_value in self.value:
187                     item_field_attributes = self.__dict__.copy()
188                     item_field_attributes['value'] = item_value
189                     item_field = Field(**item_field_attributes)
190                     for subfield_ref, subfield in item_field.iter_csv_fields(ctx, counts_by_label,
191                             parent_ref = parent_ref):
192                         yield subfield_ref, subfield
193             else:
194                 # Note: self.value is now always a single value, not a list.
195                 same_label_index = counts_by_label.get(self.label, 0)
196                 yield (parent_ref or []) + [self.label, same_label_index], self
197                 counts_by_label[self.label] = same_label_index + 1
198
199     @property
200     def linked_pois_id(self):
201         if self.id not in ('link', 'links'):
202             return None
203         if self.value is None:
204             return None
205         if isinstance(self.value, list):
206             return self.value
207         if isinstance(self.value, basestring):
208             # When field is a CSV field, links are a linefeed-separated list of IDs
209             return [
210                 bson.objectid.ObjectId(id_str)
211                 for id_str in self.value.split()
212                 ]
213         return [self.value]
214
215     @classmethod
216     def load(cls, id, metadata, value):
217         if len(metadata) != (1 if 'kind' in metadata else 0) \
218                 + (1 if 'label' in metadata else 0) \
219                 + (1 if 'relation' in metadata else 0) \
220                 + (1 if 'type' in metadata else 0) \
221                 + (1 + len(metadata['positions']) if 'positions' in metadata else 0):
222             log.warning('Unexpected attributes in field {0}, metadata {1}, value {2}'.format(id, metadata, value))
223         if 'positions' in metadata:
224             fields_position = {}
225             fields = []
226             for field_id in metadata['positions']:
227                 field_position = fields_position.get(field_id, 0)
228                 fields_position[field_id] = field_position + 1
229                 field_metadata = metadata[field_id][field_position]
230                 field_value = value[field_id][field_position]
231                 fields.append(cls.load(field_id, field_metadata, field_value))
232             value = fields or None
233         elif id == 'territories':
234             # Replace each kind-code with the corresponding territory ID.
235             if value is not None:
236                 value = [
237                     territory_id
238                     for territory_id in (
239                         ramdb.territory_id_by_kind_code.get((territory_kind_code['kind'],
240                             territory_kind_code['code']))
241                         for territory_kind_code in value
242                         )
243                     if territory_id is not None
244                     ]
245         return cls(
246             id = id,
247             kind = metadata.get('kind'),
248             label = metadata['label'],
249             relation = metadata.get('relation'),
250             type = metadata.get('type'),
251             value = value,
252             )
253
254     def set_attributes(self, **attributes):
255         """Set given attributes and return a boolean stating whether existing attributes have changed."""
256         changed = False
257         for name, value in attributes.iteritems():
258             if value is getattr(self.__class__, name, UnboundLocalError):
259                 if value is not getattr(self, name, UnboundLocalError):
260                     delattr(self, name)
261                     changed = True
262             elif value is not getattr(self, name, UnboundLocalError):
263                 setattr(self, name, value)
264                 changed = True
265         return changed
266
267
268 class Poi(representations.UserRepresentable):
269     _id = None
270     # IDs of territories for which POI is fully competent. None when POI has no notion of competence territory
271     competence_territories_id = None
272     fields = None
273     geo = None
274     ids_by_category_slug = {}
275     ids_by_competence_territory_id = {}
276     ids_by_begin_datetime = []
277     ids_by_end_datetime = []
278     ids_by_last_update_datetime = []
279     ids_by_parent_id = {}  # class attribute
280     ids_by_presence_territory_id = {}
281     ids_by_word = {}
282     indexed_ids = set()
283     instance_by_id = {}
284     last_update_datetime = None
285     last_update_organization = None
286     name = None
287     parent_id = None
288     petitpois_url = None  # class attribute defined in subclass. URL of Petitpois site
289     postal_distribution_str = None
290     schema_name = None
291     slug_by_id = {}
292     street_address = None
293     subclass_by_database_and_schema_name = {}
294     theme_slug = None
295
296     def __init__(self, **attributes):
297         if attributes:
298             self.set_attributes(**attributes)
299
300     @classmethod
301     def clear_indexes(cls):
302         cls.indexed_ids.clear()
303         cls.instance_by_id.clear()
304         cls.ids_by_parent_id.clear()
305         cls.ids_by_category_slug.clear()
306         cls.ids_by_competence_territory_id.clear()
307         cls.ids_by_presence_territory_id.clear()
308         cls.ids_by_word.clear()
309         cls.slug_by_id.clear()
310         cls.subclass_by_database_and_schema_name.clear()
311
312     @classmethod
313     def extract_non_territorial_search_data(cls, ctx, data):
314         return dict(
315             categories_slug = data['categories_slug'],
316             term = data['term'],
317             )
318
319     @classmethod
320     def extract_search_inputs_from_params(cls, ctx, params):
321         return dict(
322             categories_slug = params.getall('category'),
323             term = params.get('term'),
324             territory = params.get('territory'),
325             )
326
327     def generate_all_fields(self):
328         """Return all fields of POI including dynamic ones (ie linked fields, etc)."""
329         fields = self.fields[:] if self.fields is not None else []
330
331         # Add children POIs as linked fields.
332         children = sorted(
333             (
334                 self.instance_by_id[child_id]
335                 for child_id in self.ids_by_parent_id.get(self._id, set())
336                 ),
337             key = lambda child: (child.schema_name, child.name),
338             )
339         for child in children:
340             fields.append(
341                 Field(
342                     id = 'link',
343                     label = ramdb.schema_title_by_name[child.schema_name],
344                     relation = 'child',
345                     name = child.name,
346                     value = child._id,
347                     ))
348
349         # Add last-update field.
350         fields.append(Field(id = 'last-update', label = u"Dernière mise à jour", value = u' par '.join(
351             unicode(fragment)
352             for fragment in (
353                 self.last_update_datetime.strftime('%Y-%m-%d %H:%M') if self.last_update_datetime is not None else None,
354                 self.last_update_organization,
355                 )
356             if fragment
357             )))
358         return fields
359
360     def get_first_field(self, id, label = None):
361         return get_first_field(self.fields, id, label = label)
362
363     def get_full_url(self, ctx, params_prefix = 'cmq_'):
364         if ctx.container_base_url is None:
365             return urls.get_full_url(ctx, 'organismes', self.slug, self._id)
366         else:
367             parsed_container_base_url = urlparse.urlparse(ctx.container_base_url)
368             params = dict([
369                 ('{0}path'.format(params_prefix), urls.get_url(ctx, 'organismes', self.slug, self._id))
370                 ])
371             params.update(dict(urlparse.parse_qsl(parsed_container_base_url.query)))
372             return urlparse.urljoin(
373                 '{0}://{1}{2}'.format(
374                     parsed_container_base_url.scheme,
375                     parsed_container_base_url.netloc,
376                     parsed_container_base_url.path
377                     ),
378                 '?{0}#{0}'.format(urllib.urlencode(params)),
379                 )
380
381     @classmethod
382     def get_search_params_name(cls, ctx):
383         return set(
384             cls.rename_input_to_param(name)
385             for name in cls.extract_search_inputs_from_params(ctx, webob.multidict.MultiDict()).iterkeys()
386             )
387
388     @classmethod
389     def get_visibility_params_names(cls, ctx):
390         visibility_params = list(cls.get_search_params_name(ctx))
391         visibility_params.extend(['checkboxes', 'directory', 'export', 'gadget', 'legend', 'list', 'map', 'minisite'])
392         return [
393             'hide_{0}'.format(visibility_param)
394             for visibility_param in visibility_params
395             ]
396
397     def index(self, indexed_poi_id):
398         poi_bson = self.bson
399         metadata = poi_bson['metadata']
400         for category_slug in (metadata.get('categories-index') or set()):
401             self.ids_by_category_slug.setdefault(category_slug, set()).add(indexed_poi_id)
402
403         if conf['index.date.field']:
404             for date_range_index, date_range_metadata in enumerate(metadata.get('date-range') or []):
405                 if date_range_metadata['label'] == conf['index.date.field']:
406                     date_range_values = poi_bson['date-range'][date_range_index]
407                     date_range_begin = date_range_values.get('date-range-begin', [None])[0]
408                     date_range_end = date_range_values.get('date-range-end', [None])[0]
409
410                     if date_range_begin is not None:
411                         for index, (begin_datetime, poi_id) in enumerate(self.ids_by_begin_datetime):
412                             if begin_datetime is not None and begin_datetime > date_range_begin:
413                                 break
414                     else:
415                         index = 0
416                     self.ids_by_begin_datetime.insert(index, (date_range_begin, indexed_poi_id))
417                     if date_range_end is not None:
418                         for index, (end_datetime, poi_id) in enumerate(self.ids_by_end_datetime):
419                             if end_datetime is not None and end_datetime < date_range_end:
420                                 break
421                     else:
422                         index = 0
423                     self.ids_by_end_datetime.insert(index, (date_range_end, indexed_poi_id))
424
425             if not metadata.get('date-range'):
426                 self.ids_by_begin_datetime.append((None, indexed_poi_id))
427                 self.ids_by_end_datetime.append((None, indexed_poi_id))
428         self.ids_by_last_update_datetime.append((self.last_update_datetime, indexed_poi_id))
429
430         for i, territory_metadata in enumerate(metadata.get('territories') or []):
431             # Note: Don't fail when territory doesn't exist, because Etalage can be configured to ignore some kinds
432             # of territories (cf conf['territories_kinds']).
433             self.competence_territories_id = set(
434                 territory_id
435                 for territory_id in (
436                     ramdb.territory_id_by_kind_code.get((territory_kind_code['kind'], territory_kind_code['code']))
437                     for territory_kind_code in poi_bson['territories'][i]
438                     )
439                 if territory_id is not None
440                 )
441             for territory_id in self.competence_territories_id:
442                 self.ids_by_competence_territory_id.setdefault(territory_id, set()).add(indexed_poi_id)
443             break
444         if not self.competence_territories_id:
445             self.ids_by_competence_territory_id.setdefault(None, set()).add(indexed_poi_id)
446
447         poi_territories_id = set(
448             territory_id
449             for territory_id in (
450                 ramdb.territory_id_by_kind_code.get((territory_kind_code['kind'], territory_kind_code['code']))
451                 for territory_kind_code in metadata['territories-index']
452                 if territory_kind_code['kind'] not in (u'Country', u'InternationalOrganization')
453                 )
454             if territory_id is not None
455             ) if metadata.get('territories-index') is not None else None
456         for territory_id in (poi_territories_id or set()):
457             self.ids_by_presence_territory_id.setdefault(territory_id, set()).add(indexed_poi_id)
458
459         for word in strings.slugify(self.name).split(u'-'):
460             self.ids_by_word.setdefault(word, set()).add(indexed_poi_id)
461         self.slug_by_id[indexed_poi_id] = strings.slugify(self.name)
462
463     @classmethod
464     def index_pois(cls):
465         for self in cls.instance_by_id.itervalues():
466             # Note: self._id is not added to cls.indexed_ids by method self.index(self._id) to allow
467             # customizations where not all POIs are indexed (Passim for example).
468             cls.indexed_ids.add(self._id)
469             self.index(self._id)
470             del self.bson
471         if conf['index.date.field']:
472             cls.ids_by_begin_datetime = sorted(
473                 cls.ids_by_begin_datetime,
474                 key = lambda t: t[0] or datetime.datetime(datetime.MINYEAR, 1, 1),
475                 )
476             cls.ids_by_end_datetime = sorted(
477                 cls.ids_by_end_datetime,
478                 key = lambda t: t[0] or datetime.datetime(datetime.MAXYEAR, 1, 1),
479                 reverse = True
480                 )
481
482         cls.ids_by_last_update_datetime = sorted(cls.ids_by_last_update_datetime, key = lambda t: t[0], reverse = True)
483
484     @classmethod
485     def is_search_param_visible(cls, ctx, name):
486         param_visibility_name = 'hide_{0}'.format(name)
487         return getattr(ctx, param_visibility_name, False) \
488             if param_visibility_name.startswith('show_') \
489             else not getattr(ctx, param_visibility_name, False)
490
491     def iter_csv_fields(self, ctx):
492         counts_by_label = {}
493
494         id_field = Field(id = 'poi-id', value = self._id, label = u'Identifiant')
495         for subfield_ref, subfield in id_field.iter_csv_fields(ctx, counts_by_label):
496             yield subfield_ref, subfield
497
498         last_update_field = Field(
499             id = 'poi-last-update',
500             value = self.last_update_datetime,
501             label = u'Date de dernière modification'
502             )
503         for subfield_ref, subfield in last_update_field.iter_csv_fields(ctx, counts_by_label):
504             yield subfield_ref, subfield
505
506         if self.fields is not None:
507             for field in self.fields:
508                 for subfield_ref, subfield in field.iter_csv_fields(ctx, counts_by_label):
509                     yield subfield_ref, subfield
510
511     @classmethod
512     def iter_ids(cls, ctx, categories_slug = None, competence_territories_id = None, competence_type = None,
513             presence_territory = None, term = None):
514         intersected_sets = []
515
516         if competence_territories_id is not None:
517             competence_territories_sets = []
518             if competence_type in (None, 'by_territory'):
519                 competence_territories_sets.extend(
520                     cls.ids_by_competence_territory_id.get(competence_territory_id)
521                     for competence_territory_id in competence_territories_id
522                     )
523             if competence_type in (None, 'by_nature'):
524                 competence_territories_sets.append(cls.ids_by_competence_territory_id.get(None))
525             territory_competent_pois_id = ramdb.union_set(competence_territories_sets)
526             if not territory_competent_pois_id:
527                 return set()
528             intersected_sets.append(territory_competent_pois_id)
529
530         if presence_territory is not None:
531             territory_present_pois_id = cls.ids_by_presence_territory_id.get(presence_territory._id)
532             if not territory_present_pois_id:
533                 return set()
534             intersected_sets.append(territory_present_pois_id)
535
536         if ctx.base_categories_slug is not None:
537             base_categories_sets = []
538             base_categories_slug = copy(ctx.base_categories_slug or [])
539             for category_slug in set(base_categories_slug or []):
540                 if category_slug is not None:
541                     category_pois_id = cls.ids_by_category_slug.get(category_slug)
542                     if category_pois_id:
543                         base_categories_sets.append(category_pois_id)
544             intersected_sets.append(ramdb.union_set(base_categories_sets))
545
546         for category_slug in set(categories_slug or []):
547             if category_slug is not None:
548                 category_pois_id = cls.ids_by_category_slug.get(category_slug)
549                 if not category_pois_id:
550                     return set()
551                 intersected_sets.append(category_pois_id)
552
553         if conf['index.date.field']:
554             current_datetime = datetime.datetime.utcnow()
555             ids_by_begin_datetime_set = set()
556             for poi_begin_datetime, poi_id in cls.ids_by_begin_datetime:
557                 if poi_begin_datetime is None or current_datetime >= poi_begin_datetime:
558                     ids_by_begin_datetime_set.add(poi_id)
559                 else:
560                     break
561             ids_by_end_datetime_set = set()
562             for poi_end_datetime, poi_id in cls.ids_by_end_datetime:
563                 if poi_end_datetime is None or current_datetime <= poi_end_datetime:
564                     ids_by_end_datetime_set.add(poi_id)
565                 else:
566                     break
567             intersected_sets.append(ramdb.intersection_set([ids_by_begin_datetime_set, ids_by_end_datetime_set]))
568
569         # We should filter on term *after* having looked for competent organizations. Otherwise, when no organization
570         # matching term is found, the nearest organizations will be used even when there are competent organizations
571         # (that don't match the term).
572         if term:
573             prefixes = strings.slugify(term).split(u'-')
574             pois_id_by_prefix = {}
575             for prefix in prefixes:
576                 if prefix in pois_id_by_prefix:
577                     # TODO? Handle pois with several words sharing the same prefix?
578                     continue
579                 pois_id_by_prefix[prefix] = ramdb.union_set(
580                     pois_id
581                     for word, pois_id in cls.ids_by_word.iteritems()
582                     if word.startswith(prefix)
583                     ) or set()
584             intersected_sets.extend(pois_id_by_prefix.itervalues())
585
586         found_pois_id = ramdb.intersection_set(intersected_sets)
587         if found_pois_id is None:
588             return cls.indexed_ids
589         return found_pois_id
590
591     @classmethod
592     def load(cls, poi_bson):
593         metadata = poi_bson['metadata']
594         last_update = metadata['last-update']
595         if poi_bson.get('geo') is None:
596             geo = None
597         else:
598             geo = poi_bson['geo'][0]
599             if len(geo) > 2 and geo[2] == 0:
600                 # Don't use geographical coordinates with a 0 accuracy because their coordinates may be None.
601                 geo = None
602         self = cls(
603             _id = poi_bson['_id'],
604             geo = geo,
605             last_update_datetime = last_update['date'],
606             last_update_organization = last_update['organization'],
607             name = metadata['title'],
608             schema_name = metadata['schema-name'],
609             )
610
611         if conf['theme_field'] is None:
612             theme_field_id = None
613             theme_field_name = None
614         else:
615             theme_field_id = conf['theme_field']['id']
616             theme_field_name = conf['theme_field'].get('name')
617         fields_position = {}
618         fields = []
619         for field_id in metadata['positions']:
620             field_position = fields_position.get(field_id, 0)
621             fields_position[field_id] = field_position + 1
622             field_metadata = metadata[field_id][field_position]
623             field_value = poi_bson[field_id][field_position]
624             field = Field.load(field_id, field_metadata, field_value)
625             if field.id == u'adr' and self.postal_distribution_str is None:
626                 for sub_field in (field.value or []):
627                     if sub_field.id == u'postal-distribution':
628                         self.postal_distribution_str = sub_field.value
629                     elif sub_field.id == u'street-address':
630                         self.street_address = sub_field.value
631             elif field.id == u'link' and field.relation == u'parent':
632                 assert self.parent is None, str(self)
633                 self.parent_id = field.value
634
635             if field_id == theme_field_id and (
636                     theme_field_name is None or theme_field_name == strings.slugify(field.label)):
637                 if field.id == u'organism-type':
638                     organism_type_slug = ramdb.category_slug_by_pivot_code.get(field.value)
639                     if organism_type_slug is None:
640                         log.warning('Ignoring organism type "{0}" without matching category.'.format(field.value))
641                     else:
642                         self.theme_slug = organism_type_slug
643                 else:
644                     theme_slug = strings.slugify(field.value)
645                     if theme_slug in ramdb.category_by_slug:
646                         self.theme_slug = theme_slug
647                     else:
648                         log.warning('Ignoring theme "{0}" without matching category.'.format(field.value))
649
650             fields.append(field)
651         if fields:
652             self.fields = fields
653
654         # Temporarily store bson in poi because it is needed by index_pois.
655         self.bson = poi_bson
656
657         cls.instance_by_id[self._id] = self
658         if self.parent_id is not None:
659             cls.ids_by_parent_id.setdefault(self.parent_id, set()).add(self._id)
660         return self
661
662     @classmethod
663     def load_pois(cls):
664         from . import model
665         for db, petitpois_url in zip(model.dbs, conf['petitpois_url']):
666             for poi_bson in db.pois.find({'metadata.deleted': {'$exists': False}}):
667                 if (db.name, poi_bson['metadata']['schema-name']) not in cls.subclass_by_database_and_schema_name:
668                     schema = db.schemas.find_one({'name': poi_bson['metadata']['schema-name']})
669                     cls.subclass_by_database_and_schema_name[(db.name, poi_bson['metadata']['schema-name'])] = type(
670                         'PoiWithPetitpois',
671                         (cls,),
672                         dict(
673                             icon_url = schema.get('icon_url') if schema is not None else None,
674                             petitpois_url = petitpois_url,
675                             ),
676                         )
677                 cls.subclass_by_database_and_schema_name[(db.name, poi_bson['metadata']['schema-name'])].load(poi_bson)
678             # Create subclasses for schema with no POI.
679             for schema in db.schemas.find({}, ['name', 'icon_url']):
680                 if (db.name, schema['name']) not in cls.subclass_by_database_and_schema_name:
681                     cls.subclass_by_database_and_schema_name[(db.name, schema['name'])] = type(
682                         'PoiWithPetitpois',
683                         (cls,),
684                         dict(
685                             icon_url = schema.get('icon_url') if schema is not None else None,
686                             petitpois_url = petitpois_url,
687                             ),
688                         )
689
690     @classmethod
691     def make_inputs_to_search_data(cls):
692         return conv.pipe(
693             conv.struct(
694                 dict(
695                     base_territory = conv.input_to_postal_distribution_to_geolocated_territory,
696                     categories_slug = conv.uniform_sequence(conv.input_to_category_slug),
697                     term = conv.input_to_slug,
698                     territory = conv.input_to_postal_distribution_to_geolocated_territory,
699                     ),
700                 default = 'drop',
701                 drop_none_values = False,
702                 ),
703             conv.test_territory_in_base_territory,
704             )
705
706     @property
707     def parent(self):
708         if self.parent_id is None:
709             return None
710         return self.instance_by_id.get(self.parent_id)
711
712     @classmethod
713     def rename_input_to_param(cls, input_name):
714         return dict(
715             categories_slug = u'category',
716             ).get(input_name, input_name)
717
718     def set_attributes(self, **attributes):
719         """Set given attributes and return a boolean stating whether existing attributes have changed."""
720         changed = False
721         for name, value in attributes.iteritems():
722             if value is getattr(self.__class__, name, UnboundLocalError):
723                 if value is not getattr(self, name, UnboundLocalError):
724                     delattr(self, name)
725                     changed = True
726             elif value is not getattr(self, name, UnboundLocalError):
727                 setattr(self, name, value)
728                 changed = True
729         return changed
730
731     @property
732     def slug(self):
733         return strings.slugify(self.name)
734
735     @classmethod
736     def sort_and_paginate_pois_list(cls, ctx, pager, poi_by_id, related_territories_id = None, reverse = False,
737             territory = None, sort_key = None, **other_search_data):
738         if territory is None:
739             if sort_key is not None and sort_key == 'organism-type':
740                 key = lambda poi: ([
741                     ramdb.category_by_slug.get(ramdb.category_slug_by_pivot_code.get(field.value)) or field.value
742                     for field in poi.fields
743                     if field.id == 'organism-type'
744                     ] or [''])[0]
745             elif sort_key is not None and sort_key == 'last_update_datetime':
746                 key = lambda poi: getattr(poi, sort_key, poi.name) if sort_key is not None else poi.name
747                 reverse = True
748             elif sort_key == 'name':
749                 key = lambda poi: Poi.slug_by_id.get(poi._id)
750             else:
751                 key = lambda poi: getattr(poi, sort_key, Poi.slug_by_id.get(poi._id)) \
752                         if sort_key is not None else Poi.slug_by_id.get(poi._id)
753             pois = sorted(poi_by_id.itervalues(), key = key, reverse = reverse)
754             return [
755                 poi
756                 for poi in itertools.islice(pois, pager.first_item_index, pager.last_item_number)
757                 ]
758         territory_latitude_cos = math.cos(math.radians(territory.geo[0]))
759         territory_latitude_sin = math.sin(math.radians(territory.geo[0]))
760
761         if sort_key is None:
762             key = lambda incompetence_distance_and_poi_triple: incompetence_distance_and_poi_triple[:2]
763         elif sort_key == 'organism-type':
764             key = lambda incompetence_distance_and_poi_triple: (
765                 ([
766                     ramdb.category_by_slug.get(ramdb.category_slug_by_pivot_code.get(field.value)) or field.value
767                     for field in incompetence_distance_and_poi_triple[2].fields
768                     if field.id == 'organism-type'
769                     ] or [''])[0],
770                 incompetence_distance_and_poi_triple[1],
771                 )
772         else:
773             key = lambda incompetence_distance_and_poi_triple: (
774                 getattr(incompetence_distance_and_poi_triple[2], sort_key, None),
775                 incompetence_distance_and_poi_triple[1],
776                 )
777         incompetence_distance_and_poi_triples = sorted(
778             (
779                 (
780                     # is not competent
781                     poi.competence_territories_id is not None
782                         and related_territories_id is not None
783                         and related_territories_id.isdisjoint(poi.competence_territories_id),
784                     # distance
785                     6372.8 * math.acos(
786                         round(
787                             math.sin(math.radians(poi.geo[0])) * territory_latitude_sin
788                             + math.cos(math.radians(poi.geo[0])) * territory_latitude_cos
789                             * math.cos(math.radians(poi.geo[1] - territory.geo[1])),
790                             13,
791                         )) if poi.geo is not None else (sys.float_info.max, poi),
792                     # POI
793                     poi,
794                     )
795                 for poi in poi_by_id.itervalues()
796                 ),
797             key = key,
798             reverse = reverse,
799             )
800         return [
801             poi
802             for incompetence, distance, poi in itertools.islice(incompetence_distance_and_poi_triples,
803                 pager.first_item_index, pager.last_item_number)
804             ]
805
806
807 def get_first_field(fields, id, label = None):
808     for field in iter_fields(fields, id, label = label):
809         return field
810     return None
811
812
813 def iter_fields(fields, id, label = None):
814     if fields is not None:
815         for field in fields:
816             if field.id == id and (label is None or field.label == label):
817                 yield field
818
819
820 def pop_first_field(fields, id, label = None):
821     for field in iter_fields(fields, id, label = label):
822         fields.remove(field)
823         return field
824     return None