Always use territories related to base_territory as competent territories.
[infos-pratiques:etalage.git] / etalage / conv.py
1 # -*- coding: utf-8 -*-
2
3
4 # Etalage -- Open Data POIs portal
5 # By: Emmanuel Raviart <eraviart@easter-eggs.com>
6 #
7 # Copyright (C) 2011, 2012 Easter-eggs
8 # http://gitorious.org/infos-pratiques/etalage
9 #
10 # This file is part of Etalage.
11 #
12 # Etalage is free software; you can redistribute it and/or modify
13 # it under the terms of the GNU Affero General Public License as
14 # published by the Free Software Foundation, either version 3 of the
15 # License, or (at your option) any later version.
16 #
17 # Etalage is distributed in the hope that it will be useful,
18 # but WITHOUT ANY WARRANTY; without even the implied warranty of
19 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
20 # GNU Affero General Public License for more details.
21 #
22 # You should have received a copy of the GNU Affero General Public License
23 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
24
25
26 """Conversion functions"""
27
28
29 from cStringIO import StringIO
30 import csv
31 import math
32
33 from biryani.baseconv import *
34 from biryani.bsonconv import *
35 from biryani.objectconv import *
36 from biryani.frconv import *
37 from biryani import states, strings
38 import bson
39 import xlwt
40 from territoria2.conv import split_postal_distribution, input_to_postal_distribution
41
42
43 default_state = states.default_state
44 N_ = lambda message: message
45
46
47 # Level-1 Converters
48
49
50 def bson_to_site(bson, state = None):
51     from . import model
52     if state is None:
53         state = default_state
54     return pipe(
55         struct(
56             dict(
57                 subscriptions = uniform_sequence(function(model.Subscription.from_bson)),
58                 ),
59             default = noop,
60             ),
61         make_dict_to_object(model.Site),
62         )(bson, state = state)
63
64
65 def bson_to_subscriber(bson, state = None):
66     from . import model
67     if state is None:
68         state = default_state
69     return pipe(
70         struct(
71             dict(
72                 sites = uniform_sequence(function(model.Site.from_bson)),
73                 users = uniform_sequence(function(model.User.from_bson)),
74                 ),
75             default = noop,
76             ),
77         make_dict_to_object(model.Subscriber),
78         )(bson, state = state)
79
80
81 def bson_to_subscription(bson, state = None):
82     from . import model
83     if state is None:
84         state = default_state
85     return make_dict_to_object(model.Subscription)(bson, state = state)
86
87
88 def bson_to_user(bson, state = None):
89     from . import model
90     if state is None:
91         state = default_state
92     return make_dict_to_object(model.User)(bson, state = state)
93
94
95 def csv_infos_to_csv_bytes(csv_infos_by_schema_name, state = None):
96     from . import ramdb
97     if csv_infos_by_schema_name is None:
98         return None, None
99     if state is None:
100         state = default_state
101     csv_bytes_by_name = {}
102     for schema_name, csv_infos in csv_infos_by_schema_name.iteritems():
103         csv_file = StringIO()
104         writer = csv.writer(csv_file, delimiter = ',', quotechar = '"', quoting = csv.QUOTE_MINIMAL)
105         writer.writerow([
106             (label or u'').encode("utf-8")
107             for label in csv_infos['columns_label']
108             ])
109         for row in csv_infos['rows']:
110             writer.writerow([
111                 unicode(cell).encode('utf-8') if cell is not None else None
112                 for cell in row
113                 ])
114         csv_filename = '{0}.csv'.format(strings.slugify(ramdb.schema_title_by_name.get(schema_name, schema_name)))
115         csv_bytes_by_name[csv_filename] = csv_file.getvalue()
116     return csv_bytes_by_name or None, None
117
118
119 def csv_infos_to_excel_bytes(csv_infos_by_schema_name, state = None):
120     from . import ramdb
121     if csv_infos_by_schema_name is None:
122         return None, None
123     if state is None:
124         state = default_state
125     book = xlwt.Workbook(encoding = 'utf-8')
126     for schema_name, csv_infos in csv_infos_by_schema_name.iteritems():
127         sheet = book.add_sheet(ramdb.schema_title_by_name.get(schema_name, schema_name)[:31])
128         sheet_row = sheet.row(0)
129         for column_index, label in enumerate(csv_infos['columns_label']):
130             sheet_row.write(column_index, label or u'')
131         for row_index, row in enumerate(csv_infos['rows'], 1):
132             if row_index % 1000 == 0:
133                 sheet.flush_row_data()
134             sheet_row = sheet.row(row_index)
135             for column_index, cell in enumerate(row):
136                 if cell is not None:
137                     sheet_row.write(column_index,
138                         unicode(cell) if isinstance(cell, bson.objectid.ObjectId) else cell,
139                         )
140         sheet.flush_row_data()
141     excel_file = StringIO()
142     book.save(excel_file)
143     return excel_file.getvalue(), None
144
145
146 def default_pois_layer_data_bbox(data, state = None):
147     """Compute bounding box and add it when it is missing from data. Return modified data."""
148     from . import model, ramdb
149     if data is None:
150         return data, None
151     if state is None:
152         state = default_state
153     if data['bbox'] is not None:
154         return data, None
155     data = data.copy()
156     filter = data['filter']
157     territory = data['territory']
158     poi_by_id = model.Poi.instance_by_id
159     if territory is None:
160         presence_territory = None
161         pois_id_iter = model.Poi.iter_ids(state,
162             competence_territories_id = ramdb.get_territory_related_territories_id(
163                 data['base_territory'],
164                 ) if data.get('base_territory') is not None else None,
165             presence_territory = presence_territory,
166             **model.Poi.extract_non_territorial_search_data(state, data))
167         pois = [
168             poi
169             for poi in (
170                 poi_by_id[poi_id]
171                 for poi_id in pois_id_iter
172                 )
173             if poi.geo is not None
174             ]
175         if not pois:
176             data['bbox'] = [-180.0, -90.0, 180.0, 90.0]
177             return data, None
178         bottom = top = pois[0].geo[0]
179         left = right = pois[0].geo[1]
180     else:
181         center_latitude = territory.geo[0]
182         center_longitude = territory.geo[1]
183         bottom = center_latitude
184         left = center_longitude
185         right = center_longitude
186         top = center_latitude
187         if filter == 'competence':
188             competence_territories_id = ramdb.get_territory_related_territories_id(territory)
189             presence_territory = None
190             pois_id_iter = model.Poi.iter_ids(state,
191                 competence_territories_id = competence_territories_id or (ramdb.get_territory_related_territories_id(
192                     data['base_territory'],
193                     ) if data.get('base_territory') is not None else None),
194                 presence_territory = presence_territory,
195                 **model.Poi.extract_non_territorial_search_data(state, data))
196             pois = [
197                 poi
198                 for poi in (
199                     poi_by_id[poi_id]
200                     for poi_id in pois_id_iter
201                     )
202                 if poi.geo is not None
203                 ]
204         elif filter == 'presence':
205             presence_territory = territory
206             pois_id_iter = model.Poi.iter_ids(state,
207                 competence_territories_id = ramdb.get_territory_related_territories_id(
208                     data['base_territory'],
209                     ) if data.get('base_territory') is not None else None,
210                 presence_territory = presence_territory,
211                 **model.Poi.extract_non_territorial_search_data(state, data))
212             pois = [
213                 poi
214                 for poi in (
215                     poi_by_id[poi_id]
216                     for poi_id in pois_id_iter
217                     )
218                 if poi.geo is not None
219                 ]
220         else:
221             # When no filter is given, use the bounding box of the territory (ie the bounding box enclosing every POI
222             # present in the territory).
223             presence_territory = territory
224             pois_id_iter = model.Poi.iter_ids(state,
225                 competence_territories_id = ramdb.get_territory_related_territories_id(
226                     data['base_territory'],
227                     ) if data.get('base_territory') is not None else None,
228                 presence_territory = presence_territory,
229                 **model.Poi.extract_non_territorial_search_data(state, data))
230             pois = [
231                 poi
232                 for poi in (
233                     poi_by_id[poi_id]
234                     for poi_id in pois_id_iter
235                     )
236                 if poi.geo is not None
237                 ]
238             if not pois:
239                 # When no POI has been found in territory, use the bounding box enclosing every competent POI.
240                 competence_territories_id = ramdb.get_territory_related_territories_id(territory)
241                 presence_territory = None
242                 pois_id_iter = model.Poi.iter_ids(state,
243                     competence_territories_id = competence_territories_id or (
244                         ramdb.get_territory_related_territories_id(data['base_territory'])\
245                                 if data.get('base_territory') is not None else None),
246                     presence_territory = presence_territory,
247                     **model.Poi.extract_non_territorial_search_data(state, data))
248                 pois = [
249                     poi
250                     for poi in (
251                         poi_by_id[poi_id]
252                         for poi_id in pois_id_iter
253                         )
254                     if poi.geo is not None
255                     ]
256                 if not pois:
257                     # When no present nor competent POI has been found, compute bounding box using given distance.
258                     delta = math.degrees(state.distance / 6372.8)
259                     data['bbox'] = [
260                         center_longitude - delta,  # left
261                         center_latitude - delta,  # bottom
262                         center_longitude + delta,  # left
263                         center_latitude + delta,  # top
264                         ]
265                     return data, None
266     for poi in pois:
267         poi_latitude = poi.geo[0]
268         if poi_latitude < bottom:
269             bottom = poi_latitude
270         elif poi_latitude > top:
271             top = poi_latitude
272         poi_longitude = poi.geo[1]
273         if poi_longitude < left:
274             left = poi_longitude
275         elif poi_longitude > right:
276             right = poi_longitude
277     data['bbox'] = [left, bottom, right, top]
278     return data, None
279
280
281 def id_name_dict_list_to_ignored_fields(value, state = None):
282     if not value:
283         return None, None
284     if state is None:
285         state = default_state
286     ignored_fields = {}
287     for id_name_dict in value:
288         id = id_name_dict['id']
289         name = id_name_dict.get('name')
290         if id in ignored_fields:
291             ignored_field = ignored_fields[id]
292             if ignored_field is not None:
293                 ignored_field.add(name)
294         else:
295             if name is None:
296                 ignored_fields[id] = None
297             else:
298                 ignored_fields[id] = set([name])
299     return ignored_fields, None
300
301
302 def id_to_poi(poi_id, state = None):
303     import model
304     if poi_id is None:
305         return poi_id, None
306     if state is None:
307         state = default_state
308     poi = model.Poi.instance_by_id.get(poi_id)
309     if poi is None:
310         return poi_id, state._("POI {0} doesn't exist").format(poi_id)
311     return poi, None
312
313
314 input_to_filter = pipe(
315     input_to_slug,
316     test_in(['competence', 'presence']),
317     )
318
319
320 def input_to_category_slug(value, state = None):
321     from . import ramdb
322     if state is None:
323         state = default_state
324     return pipe(
325         input_to_tag_slug,
326         function(lambda slug: ramdb.category_by_slug[slug]),
327         test(lambda category: (category.tags_slug or set()).issuperset(state.category_tags_slug or []),
328             error = N_(u'Invalid category')),
329         function(lambda category: category.slug),
330         )(value, state = state)
331
332
333 def input_to_tag_slug(value, state = None):
334     from . import ramdb
335     if state is None:
336         state = default_state
337     return pipe(
338         input_to_slug,
339         test(lambda slug: slug in ramdb.category_by_slug, error = N_(u'Invalid category')),
340         )(value, state = state)
341
342
343 def inputs_to_geographical_coverage_csv_infos(inputs, state = None):
344     from . import model, ramdb
345     if state is None:
346         state = default_state
347     data, errors = model.Poi.make_inputs_to_search_data()(inputs, state = state)
348     if errors is not None:
349         return data, errors
350
351     territory = data['territory']
352     competence_territories_id = ramdb.get_territory_related_territories_id(territory) if territory is not None else None
353     if competence_territories_id is None:
354         competence_territories_id = ramdb.get_territory_related_territories_id(
355             data['base_territory'],
356             ) if data.get('base_territory') is not None else None
357     if competence_territories_id is None:
358         competence_territories_id = set(ramdb.territory_by_id.iterkeys())
359     pois_id = set(model.Poi.iter_ids(state, **model.Poi.extract_non_territorial_search_data(state, data)))
360     pois_id_by_commune_id = {}
361     rows_count = 0
362     if pois_id:
363         pois_id_by_competence_territory_id = {}
364         for commune_id in competence_territories_id:
365             commune = ramdb.territory_by_id.get(commune_id)
366             if commune is None:
367                 continue
368             if commune.__class__.__name__ in (u'ArrondissementOfCommuneOfFrance', u'CommuneOfFrance') \
369                     and commune.code not in (u'13055', u'69123', u'75056'):
370                 commune_pois_id = set()
371                 for related_territory_id in ramdb.get_territory_related_territories_id(commune):
372                     if related_territory_id not in pois_id_by_competence_territory_id:
373                         related_territory_pois_id = model.Poi.ids_by_competence_territory_id.get(related_territory_id)
374                         pois_id_by_competence_territory_id[related_territory_id] = pois_id.intersection(
375                             related_territory_pois_id) if related_territory_pois_id is not None else set()
376                     commune_pois_id.update(pois_id_by_competence_territory_id[related_territory_id])
377                 if commune_pois_id:
378                     pois_id_by_commune_id[commune_id] = commune_pois_id
379                     rows_count += len(commune_pois_id)
380                     if rows_count > 65535:
381                         # Excel doesn't support sheets with more than 65535 rows.
382                         return None, state._(u'Export is too big. Restrict some search criteria and try again.')
383     return pois_id_by_commune_id_to_csv_infos(pois_id_by_commune_id, state = state)
384
385
386 def inputs_to_pois_csv_infos(inputs, state = None):
387     from . import conf, model, ramdb
388     if state is None:
389         state = default_state
390     data, errors = pipe(
391         model.Poi.make_inputs_to_search_data(),
392         struct(
393             dict(
394                 # By default, when no default_filter is given, export only POIs present on given territory.
395                 filter = default(conf['default_filter'] or 'presence'),
396                 ),
397             default = noop,
398             keep_none_values = True,
399             ),
400         )(inputs, state = state)
401     if errors is not None:
402         return data, errors
403
404     filter = data['filter']
405     territory = data['territory']
406     related_territories_id = ramdb.get_territory_related_territories_id(territory) if territory is not None else None
407     if filter == 'competence':
408         competence_territories_id = related_territories_id
409         presence_territory = None
410     elif filter == 'presence':
411         competence_territories_id = None
412         presence_territory = territory
413     else:
414         competence_territories_id = None
415         presence_territory = None
416     pois_id = set(model.Poi.iter_ids(state,
417         competence_territories_id = competence_territories_id or (ramdb.get_territory_related_territories_id(
418             data['base_territory'],
419             ) if data.get('base_territory') is not None else None),
420         presence_territory = presence_territory,
421         **model.Poi.extract_non_territorial_search_data(state, data)))
422     if len(pois_id) > 65535:
423         # Excel doesn't support sheets with more than 65535 rows.
424         return None, state._(u'Export is too big. Restrict some search criteria and try again.')
425
426     # Add sub-...-children of found POIs.
427     def add_children_id(poi_id, pois_id):
428         for child_id in (model.Poi.ids_by_parent_id.get(poi_id) or set()):
429             if child_id not in pois_id:
430                 pois_id.add(child_id)
431                 add_children_id(child_id, pois_id)
432     for poi_id in pois_id.copy():
433         add_children_id(poi_id, pois_id)
434         if len(pois_id) > 65535:
435             # Excel doesn't support sheets with more than 65535 rows.
436             return None, state._(u'Export is too big. Restrict some search criteria and try again.')
437
438     return pois_id_to_csv_infos(pois_id, state = state)
439
440
441 def inputs_to_pois_directory_data(inputs, state = None):
442     from . import model
443     if state is None:
444         state = default_state
445     return pipe(
446         model.Poi.make_inputs_to_search_data(),
447         struct(
448             dict(
449                 territory = pipe(
450 #                    test(lambda territory: territory.__class__.__name__ in model.communes_kinds,
451 #                        error = N_(u'In "directory" mode, territory must be a commune')),
452                     test_not_none(error = N_(u'In "directory" mode, a commune is required')),
453                     ),
454                 ),
455             default = noop,
456             keep_none_values = True,
457             ),
458         set_default_filter,
459         )(inputs, state = state)
460
461
462 def inputs_to_pois_layer_data(inputs, state = None):
463     from . import model
464     if state is None:
465         state = default_state
466     return pipe(
467         merge(
468             model.Poi.make_inputs_to_search_data(),
469             struct(
470                 dict(
471                     bbox = pipe(
472                         function(lambda bbox: bbox.split(u',')),
473                         struct(
474                             [
475                                 # West longitude
476                                 pipe(
477                                     input_to_float,
478                                     test_between(-180, 180),
479                                     not_none,
480                                     ),
481                                 # South latitude
482                                 pipe(
483                                     input_to_float,
484                                     test_between(-90, 90),
485                                     not_none,
486                                     ),
487                                 # East longitude
488                                 pipe(
489                                     input_to_float,
490                                     test_between(-180, 180),
491                                     not_none,
492                                     ),
493                                 # North latitude
494                                 pipe(
495                                     input_to_float,
496                                     test_between(-90, 90),
497                                     not_none,
498                                     ),
499                                 ],
500                             ),
501                         ),
502                     current = pipe(
503                         input_to_object_id,
504                         id_to_poi,
505                         test(lambda poi: poi.geo is not None, error = N_('POI has no geographical coordinates')),
506                         ),
507                     enable_cluster = pipe(
508                         guess_bool,
509                         default(True),
510                         ),
511                     ),
512                 default = 'drop',
513                 keep_none_values = True,
514                 ),
515             ),
516         set_default_filter,
517         )(inputs, state = state)
518
519
520 def inputs_to_pois_list_data(inputs, state = None):
521     from . import model
522     if state is None:
523         state = default_state
524     return pipe(
525         merge(
526             model.Poi.make_inputs_to_search_data(),
527             struct(
528                 dict(
529                     page = pipe(
530                         input_to_int,
531                         test_greater_or_equal(1),
532                         default(1),
533                         ),
534                     sort_key = pipe(
535                         cleanup_line,
536                         test_in(['name', 'organism-type', 'postal_distribution_str', 'schema_name', 'street_address']),
537                         ),
538                     ),
539                 default = 'drop',
540                 keep_none_values = True,
541                 ),
542             ),
543         set_default_filter,
544         rename_item('page', 'page_number'),
545         )(inputs, state = state)
546
547
548 def layer_data_to_clusters(data, state = None):
549     from . import model, ramdb
550     if data is None:
551         return None, None
552     if state is None:
553         state = default_state
554     left, bottom, right, top = data['bbox']
555     center_latitude = (bottom + top) / 2.0
556     center_latitude_cos = math.cos(math.radians(center_latitude))
557     center_latitude_sin = math.sin(math.radians(center_latitude))
558     center_longitude = (left + right) / 2.0
559     filter = data['filter']
560     territory = data['territory']
561     related_territories_id = ramdb.get_territory_related_territories_id(territory) if territory is not None else None
562     if filter == 'competence':
563         competence_territories_id = related_territories_id
564         presence_territory = None
565     elif filter == 'presence':
566         competence_territories_id = None
567         presence_territory = territory
568     else:
569         competence_territories_id = None
570         presence_territory = None
571     pois_id_iter = model.Poi.iter_ids(state,
572         competence_territories_id = competence_territories_id or (ramdb.get_territory_related_territories_id(
573             data['base_territory'],
574             ) if data.get('base_territory') is not None else None),
575         presence_territory = presence_territory,
576         **model.Poi.extract_non_territorial_search_data(state, data))
577     poi_by_id = model.Poi.instance_by_id
578     current = data['current']
579     pois_iter = (
580         poi
581         for poi in (
582             poi_by_id[poi_id]
583             for poi_id in pois_id_iter
584             )
585         if poi.geo is not None and bottom <= poi.geo[0] <= top and left <= poi.geo[1] <= right and (
586             current is None or poi._id != current._id)
587         )
588     distance_and_poi_couples = sorted(
589         (
590             (
591                 # distance from center of map
592                 6372.8 * math.acos(
593                     round(
594                         math.sin(math.radians(poi.geo[0])) * center_latitude_sin
595                         + math.cos(math.radians(poi.geo[0])) * center_latitude_cos
596                             * math.cos(math.radians(poi.geo[1] - center_longitude)),
597                         13,
598                     )) if poi.geo is not None else (sys.float_info.max, poi),
599                 # POI
600                 poi,
601                 )
602             for poi in pois_iter
603             ),
604         key = lambda distance_and_poi_couple: distance_and_poi_couple[0],
605         )
606     pois = [
607         poi
608         for distance, poi in distance_and_poi_couples
609         ]
610     if current is not None:
611         pois.insert(0, current)
612     horizontal_iota = (right - left) / 20.0
613     vertical_iota = (top - bottom) / 15.0
614 #    vertical_iota = horizontal_iota = (right - left) / 30.0
615     clusters = []
616     for poi in pois:
617         poi_latitude = poi.geo[0]
618         poi_longitude = poi.geo[1]
619         for cluster in clusters:
620             if abs(poi_latitude - cluster.center_latitude) <= vertical_iota \
621                     and abs(poi_longitude - cluster.center_longitude) <= horizontal_iota:
622                 cluster.count += 1
623                 if poi_latitude == cluster.center_latitude and poi_longitude == cluster.center_longitude:
624                     cluster.center_pois.append(poi)
625                 if poi_latitude < cluster.bottom:
626                     cluster.bottom = poi_latitude
627                 elif poi_latitude > cluster.top:
628                     cluster.top = poi_latitude
629                 if poi_longitude < cluster.left:
630                     cluster.left = poi_longitude
631                 elif poi_longitude > cluster.right:
632                     cluster.right = poi_longitude
633                 break
634         else:
635             cluster = model.Cluster()
636             cluster.competent = False  # changed below
637             cluster.count = 1
638             cluster.bottom = cluster.top = cluster.center_latitude = poi_latitude
639             cluster.left = cluster.right = cluster.center_longitude = poi_longitude
640             cluster.center_pois = [poi]
641             clusters.append(cluster)
642         if cluster.competent is False:
643             if related_territories_id is None or poi.competence_territories_id is None:
644                 cluster.competent = None
645             elif not related_territories_id.isdisjoint(poi.competence_territories_id):
646                 cluster.competent = True
647         elif cluster.competent is None and related_territories_id is not None \
648                 and poi.competence_territories_id is not None \
649                 and not related_territories_id.isdisjoint(poi.competence_territories_id):
650             cluster.competent = True
651     return clusters, None
652
653
654 def pois_id_by_commune_id_to_csv_infos(pois_id_by_commune_id, state = None):
655     from . import model, ramdb
656     if pois_id_by_commune_id is None:
657         return None, None
658     if state is None:
659         state = default_state
660     csv_infos_by_schema_name = {}
661     for commune_id, commune_pois_id in pois_id_by_commune_id.iteritems():
662         commune = ramdb.territory_by_id.get(commune_id)
663         if commune is None:
664             continue
665         for poi_id in commune_pois_id:
666             poi = model.Poi.instance_by_id.get(poi_id)
667             if poi is None:
668                 continue
669             csv_infos = csv_infos_by_schema_name.get(poi.schema_name)
670             if csv_infos is None:
671                 csv_infos_by_schema_name[poi.schema_name] = csv_infos = dict(
672                     columns_label = [u'Code commune', u'Nom commune'],
673                     columns_ref = [None, None],
674                     rows = [],
675                     )
676             columns_label = csv_infos['columns_label']
677             columns_index = {}
678             columns_ref = csv_infos['columns_ref']
679             row = [commune.code, commune.name] + [None] * len(columns_ref)
680             for field_ref, field in poi.iter_csv_fields(state):
681                 # Detect column number to use for field. Create a new column if needed.
682                 column_ref = tuple(field_ref[:-1])
683                 same_ref_columns_count = field_ref[-1]
684                 if columns_ref.count(column_ref) == same_ref_columns_count:
685                     column_index = len(columns_ref)
686                     columns_label.append(field.label)  # or u' - '.join(label for label in field_ref[::2])
687                     columns_ref.append(column_ref)
688                     row.append(None)
689                 else:
690                     column_index = columns_ref.index(column_ref, columns_index.get(column_ref, -1) + 1)
691                 columns_index[column_ref] = column_index
692                 row[column_index] = field.value
693             csv_infos['rows'].append(row)
694
695     # Sort rows by commune code and POI ID.
696     for csv_infos in csv_infos_by_schema_name.itervalues():
697         csv_infos['rows'].sort(key = lambda row: (row[0], row[2]))
698
699     return csv_infos_by_schema_name or None, None
700
701
702 def pois_id_to_csv_infos(pois_id, state = None):
703     from . import model
704     if pois_id is None:
705         return None, None
706     if state is None:
707         state = default_state
708     csv_infos_by_schema_name = {}
709     visited_pois_id = set(pois_id)
710     while pois_id:
711         remaining_pois_id = []
712         for poi_id in pois_id:
713             poi = model.Poi.instance_by_id.get(poi_id)
714             if poi is None:
715                 continue
716             csv_infos = csv_infos_by_schema_name.get(poi.schema_name)
717             if csv_infos is None:
718                 csv_infos_by_schema_name[poi.schema_name] = csv_infos = dict(
719                     columns_label = [],
720                     columns_ref = [],
721                     rows = [],
722                     )
723             columns_label = csv_infos['columns_label']
724             columns_index = {}
725             columns_ref = csv_infos['columns_ref']
726             row = [None] * len(columns_ref)
727             for field_ref, field in poi.iter_csv_fields(state):
728                 # Detect column number to use for field. Create a new column if needed.
729                 column_ref = tuple(field_ref[:-1])
730                 same_ref_columns_count = field_ref[-1]
731                 if columns_ref.count(column_ref) == same_ref_columns_count:
732                     column_index = len(columns_ref)
733                     columns_label.append(field.label)  # or u' - '.join(label for label in field_ref[::2])
734                     columns_ref.append(column_ref)
735                     row.append(None)
736                 else:
737                     column_index = columns_ref.index(column_ref, columns_index.get(column_ref, -1) + 1)
738                 columns_index[column_ref] = column_index
739                 row[column_index] = field.value
740                 for linked_poi_id in (field.linked_pois_id or []):
741                     if linked_poi_id not in visited_pois_id:
742                         visited_pois_id.add(linked_poi_id)
743                         remaining_pois_id.append(linked_poi_id)
744             csv_infos['rows'].append(row)
745         pois_id = remaining_pois_id
746     return csv_infos_by_schema_name or None, None
747
748
749 def postal_distribution_to_territory(postal_distribution, state = None):
750     from . import ramdb
751     if postal_distribution is None:
752         return postal_distribution, None
753     if state is None:
754         state = default_state
755     territory_id = ramdb.territories_id_by_postal_distribution.get(postal_distribution)
756     if territory_id is None:
757         return postal_distribution, state._(u'Unknown territory')
758     territory = ramdb.territory_by_id.get(territory_id)
759     if territory is None:
760         return postal_distribution, state._(u'Unknown territory')
761     return territory, None
762
763
764 def set_default_filter(data, state = None):
765     if data is None:
766         return None, None
767
768     from . import conf, model
769
770     if state is None:
771         state = default_state
772
773     if data.get('filter') is None and conf['default_filter'] is not None:
774         data['filter'] = conf['default_filter']
775     elif data.get('filter') is None and data['territory'] is not None \
776             and data['territory'].__class__.__name__ not in model.communes_kinds:
777         # When no filter is given and territory is not a commune, search only for POIs present on territory instead of
778         # POIs near the territory.
779         data['filter'] = u'presence'
780     return data, None
781
782
783 def site_to_bson(subscriber, state = None):
784     if state is None:
785         state = default_state
786     return pipe(
787         object_to_clean_dict,
788         struct(
789             dict(
790                 subscriptions = uniform_sequence(function(lambda subscription: subscription.to_bson())),
791                 ),
792             default = noop,
793             ),
794         )(session, state = state)
795
796
797 def subscriber_to_bson(subscriber, state = None):
798     if state is None:
799         state = default_state
800     return pipe(
801         object_to_clean_dict,
802         struct(
803             dict(
804                 sites = uniform_sequence(function(lambda site: site.to_bson())),
805                 users = uniform_sequence(function(lambda user: user.to_bson())),
806                 ),
807             default = noop,
808             ),
809         )(session, state = state)
810
811
812 subscription_to_bson = object_to_clean_dict
813
814
815 def test_territory_in_base_territory(data, state = None):
816     if state is None:
817         state = default_state
818     if not data.get('base_territory') or \
819         data.get('territory') and data['base_territory']._id in data['territory'].ancestors_id:
820         return data, None
821     if not data.get('territory'):
822         return data, None
823     return data, {'territory': state._(u'Searched territory not located in base territory {0}').format(
824         data['base_territory'].main_postal_distribution['postal_routing']
825         )}
826
827
828 user_to_bson = object_to_clean_dict
829
830
831 # Level-2 Converters
832
833
834 input_to_postal_distribution_to_geolocated_territory = pipe(
835     input_to_postal_distribution,
836     postal_distribution_to_territory,
837     test(lambda territory: territory.geo is not None, error = N_(u'Territory has no geographical coordinates')),
838     )