etalage/conv.py

   1 # -*- coding: utf-8 -*-
   2
   3
   4 # Etalage -- Open Data POIs portal
   5 # By: Emmanuel Raviart <eraviart@easter-eggs.com>
   6 #
   7 # Copyright (C) 2011, 2012 Easter-eggs
   8 # http://gitorious.org/infos-pratiques/etalage
   9 #
  10 # This file is part of Etalage.
  11 #
  12 # Etalage is free software; you can redistribute it and/or modify
  13 # it under the terms of the GNU Affero General Public License as
  14 # published by the Free Software Foundation, either version 3 of the
  15 # License, or (at your option) any later version.
  16 #
  17 # Etalage is distributed in the hope that it will be useful,
  18 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  19 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  20 # GNU Affero General Public License for more details.
  21 #
  22 # You should have received a copy of the GNU Affero General Public License
  23 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
  24
  25
  26 """Conversion functions"""
  27
  28
  29 from cStringIO import StringIO
  30 import csv
  31 import math
  32
  33 from biryani.baseconv import *
  34 from biryani.bsonconv import *
  35 from biryani.objectconv import *
  36 from biryani.frconv import *
  37 from biryani import states, strings
  38 import bson
  39 import xlwt
  40 from territoria2.conv import split_postal_distribution, input_to_postal_distribution
  41
  42
  43 default_state = states.default_state
  44 N_ = lambda message: message
  45
  46
  47 # Level-1 Converters
  48
  49
  50 def bson_to_site(bson, state = None):
  51     from . import model
  52     if state is None:
  53         state = default_state
  54     return pipe(
  55         struct(
  56             dict(
  57                 subscriptions = uniform_sequence(function(model.Subscription.from_bson)),
  58                 ),
  59             default = noop,
  60             ),
  61         make_dict_to_object(model.Site),
  62         )(bson, state = state)
  63
  64
  65 def bson_to_subscriber(bson, state = None):
  66     from . import model
  67     if state is None:
  68         state = default_state
  69     return pipe(
  70         struct(
  71             dict(
  72                 sites = uniform_sequence(function(model.Site.from_bson)),
  73                 users = uniform_sequence(function(model.User.from_bson)),
  74                 ),
  75             default = noop,
  76             ),
  77         make_dict_to_object(model.Subscriber),
  78         )(bson, state = state)
  79
  80
  81 def bson_to_subscription(bson, state = None):
  82     from . import model
  83     if state is None:
  84         state = default_state
  85     return make_dict_to_object(model.Subscription)(bson, state = state)
  86
  87
  88 def bson_to_user(bson, state = None):
  89     from . import model
  90     if state is None:
  91         state = default_state
  92     return make_dict_to_object(model.User)(bson, state = state)
  93
  94
  95 def csv_infos_to_csv_bytes(csv_infos_by_schema_name, state = None):
  96     from . import ramdb
  97     if csv_infos_by_schema_name is None:
  98         return None, None
  99     if state is None:
 100         state = default_state
 101     csv_bytes_by_name = {}
 102     for schema_name, csv_infos in csv_infos_by_schema_name.iteritems():
 103         csv_file = StringIO()
 104         writer = csv.writer(csv_file, delimiter = ',', quotechar = '"', quoting = csv.QUOTE_MINIMAL)
 105         writer.writerow([
 106             (label or u'').encode("utf-8")
 107             for label in csv_infos['columns_label']
 108             ])
 109         for row in csv_infos['rows']:
 110             writer.writerow([
 111                 unicode(cell).encode('utf-8') if cell is not None else None
 112                 for cell in row
 113                 ])
 114         csv_filename = '{0}.csv'.format(strings.slugify(ramdb.schema_title_by_name.get(schema_name, schema_name)))
 115         csv_bytes_by_name[csv_filename] = csv_file.getvalue()
 116     return csv_bytes_by_name or None, None
 117
 118
 119 def csv_infos_to_excel_bytes(csv_infos_by_schema_name, state = None):
 120     from . import ramdb
 121     if csv_infos_by_schema_name is None:
 122         return None, None
 123     if state is None:
 124         state = default_state
 125     book = xlwt.Workbook(encoding = 'utf-8')
 126     for schema_name, csv_infos in csv_infos_by_schema_name.iteritems():
 127         sheet = book.add_sheet(ramdb.schema_title_by_name.get(schema_name, schema_name)[:31])
 128         sheet_row = sheet.row(0)
 129         for column_index, label in enumerate(csv_infos['columns_label']):
 130             sheet_row.write(column_index, label or u'')
 131         for row_index, row in enumerate(csv_infos['rows'], 1):
 132             if row_index % 1000 == 0:
 133                 sheet.flush_row_data()
 134             sheet_row = sheet.row(row_index)
 135             for column_index, cell in enumerate(row):
 136                 if cell is not None:
 137                     sheet_row.write(column_index,
 138                         unicode(cell) if isinstance(cell, bson.objectid.ObjectId) else cell,
 139                         )
 140         sheet.flush_row_data()
 141     excel_file = StringIO()
 142     book.save(excel_file)
 143     return excel_file.getvalue(), None
 144
 145
 146 def default_pois_layer_data_bbox(data, state = None):
 147     """Compute bounding box and add it when it is missing from data. Return modified data."""
 148     from . import model, ramdb
 149     if data is None:
 150         return data, None
 151     if state is None:
 152         state = default_state
 153     if data['bbox'] is not None:
 154         return data, None
 155     data = data.copy()
 156     filter = data['filter']
 157     territory = data['territory']
 158     poi_by_id = model.Poi.instance_by_id
 159     if territory is None:
 160         presence_territory = None
 161         pois_id_iter = model.Poi.iter_ids(state,
 162             competence_territories_id = ramdb.get_territory_related_territories_id(
 163                 data['base_territory'],
 164                 ) if data.get('base_territory') is not None else None,
 165             presence_territory = presence_territory,
 166             **model.Poi.extract_non_territorial_search_data(state, data))
 167         pois = [
 168             poi
 169             for poi in (
 170                 poi_by_id[poi_id]
 171                 for poi_id in pois_id_iter
 172                 )
 173             if poi.geo is not None
 174             ]
 175         if not pois:
 176             data['bbox'] = [-180.0, -90.0, 180.0, 90.0]
 177             return data, None
 178         bottom = top = pois[0].geo[0]
 179         left = right = pois[0].geo[1]
 180     else:
 181         center_latitude = territory.geo[0]
 182         center_longitude = territory.geo[1]
 183         bottom = center_latitude
 184         left = center_longitude
 185         right = center_longitude
 186         top = center_latitude
 187         if filter == 'competence':
 188             competence_territories_id = ramdb.get_territory_related_territories_id(territory)
 189             presence_territory = None
 190             pois_id_iter = model.Poi.iter_ids(state,
 191                 competence_territories_id = competence_territories_id or (ramdb.get_territory_related_territories_id(
 192                     data['base_territory'],
 193                     ) if data.get('base_territory') is not None else None),
 194                 presence_territory = presence_territory,
 195                 **model.Poi.extract_non_territorial_search_data(state, data))
 196             pois = [
 197                 poi
 198                 for poi in (
 199                     poi_by_id[poi_id]
 200                     for poi_id in pois_id_iter
 201                     )
 202                 if poi.geo is not None
 203                 ]
 204         elif filter == 'presence':
 205             presence_territory = territory
 206             pois_id_iter = model.Poi.iter_ids(state,
 207                 competence_territories_id = ramdb.get_territory_related_territories_id(
 208                     data['base_territory'],
 209                     ) if data.get('base_territory') is not None else None,
 210                 presence_territory = presence_territory,
 211                 **model.Poi.extract_non_territorial_search_data(state, data))
 212             pois = [
 213                 poi
 214                 for poi in (
 215                     poi_by_id[poi_id]
 216                     for poi_id in pois_id_iter
 217                     )
 218                 if poi.geo is not None
 219                 ]
 220         else:
 221             # When no filter is given, use the bounding box of the territory (ie the bounding box enclosing every POI
 222             # present in the territory).
 223             presence_territory = territory
 224             pois_id_iter = model.Poi.iter_ids(state,
 225                 competence_territories_id = ramdb.get_territory_related_territories_id(
 226                     data['base_territory'],
 227                     ) if data.get('base_territory') is not None else None,
 228                 presence_territory = presence_territory,
 229                 **model.Poi.extract_non_territorial_search_data(state, data))
 230             pois = [
 231                 poi
 232                 for poi in (
 233                     poi_by_id[poi_id]
 234                     for poi_id in pois_id_iter
 235                     )
 236                 if poi.geo is not None
 237                 ]
 238             if not pois:
 239                 # When no POI has been found in territory, use the bounding box enclosing every competent POI.
 240                 competence_territories_id = ramdb.get_territory_related_territories_id(territory)
 241                 presence_territory = None
 242                 pois_id_iter = model.Poi.iter_ids(state,
 243                     competence_territories_id = competence_territories_id or (
 244                         ramdb.get_territory_related_territories_id(data['base_territory'])
 245                         if data.get('base_territory') is not None else None
 246                         ),
 247                     presence_territory = presence_territory,
 248                     **model.Poi.extract_non_territorial_search_data(state, data))
 249                 pois = [
 250                     poi
 251                     for poi in (
 252                         poi_by_id[poi_id]
 253                         for poi_id in pois_id_iter
 254                         )
 255                     if poi.geo is not None
 256                     ]
 257                 if not pois:
 258                     # When no present nor competent POI has been found, compute bounding box using given distance.
 259                     delta = math.degrees(state.distance / 6372.8)
 260                     data['bbox'] = [
 261                         center_longitude - delta,  # left
 262                         center_latitude - delta,  # bottom
 263                         center_longitude + delta,  # left
 264                         center_latitude + delta,  # top
 265                         ]
 266                     return data, None
 267     for poi in pois:
 268         poi_latitude = poi.geo[0]
 269         if poi_latitude < bottom:
 270             bottom = poi_latitude
 271         elif poi_latitude > top:
 272             top = poi_latitude
 273         poi_longitude = poi.geo[1]
 274         if poi_longitude < left:
 275             left = poi_longitude
 276         elif poi_longitude > right:
 277             right = poi_longitude
 278     data['bbox'] = [left, bottom, right, top]
 279     return data, None
 280
 281
 282 def id_name_dict_list_to_ignored_fields(value, state = None):
 283     if not value:
 284         return None, None
 285     if state is None:
 286         state = default_state
 287     ignored_fields = {}
 288     for id_name_dict in value:
 289         id = id_name_dict['id']
 290         name = id_name_dict.get('name')
 291         if id in ignored_fields:
 292             ignored_field = ignored_fields[id]
 293             if ignored_field is not None:
 294                 ignored_field.add(name)
 295         else:
 296             if name is None:
 297                 ignored_fields[id] = None
 298             else:
 299                 ignored_fields[id] = set([name])
 300     return ignored_fields, None
 301
 302
 303 def id_to_poi(poi_id, state = None):
 304     import model
 305     if poi_id is None:
 306         return poi_id, None
 307     if state is None:
 308         state = default_state
 309     poi = model.Poi.instance_by_id.get(poi_id)
 310     if poi is None:
 311         return poi_id, state._("POI {0} doesn't exist").format(poi_id)
 312     return poi, None
 313
 314
 315 input_to_filter = pipe(
 316     input_to_slug,
 317     test_in(['competence', 'presence']),
 318     )
 319
 320
 321 def input_to_category_slug(value, state = None):
 322     from . import ramdb
 323     if state is None:
 324         state = default_state
 325     return pipe(
 326         input_to_tag_slug,
 327         function(lambda slug: ramdb.category_by_slug[slug]),
 328         test(lambda category: (category.tags_slug or set()).issuperset(state.category_tags_slug or []),
 329             error = N_(u'Invalid category')),
 330         function(lambda category: category.slug),
 331         )(value, state = state)
 332
 333
 334 def input_to_tag_slug(value, state = None):
 335     from . import ramdb
 336     if state is None:
 337         state = default_state
 338     return pipe(
 339         input_to_slug,
 340         test(lambda slug: slug in ramdb.category_by_slug, error = N_(u'Invalid category')),
 341         )(value, state = state)
 342
 343
 344 def inputs_to_geographical_coverage_csv_infos(inputs, state = None):
 345     from . import model, ramdb
 346     if state is None:
 347         state = default_state
 348     data, errors = model.Poi.make_inputs_to_search_data()(inputs, state = state)
 349     if errors is not None:
 350         return data, errors
 351
 352     territory = data['territory']
 353     competence_territories_id = ramdb.get_territory_related_territories_id(territory) if territory is not None else None
 354     if competence_territories_id is None:
 355         competence_territories_id = ramdb.get_territory_related_territories_id(
 356             data['base_territory'],
 357             ) if data.get('base_territory') is not None else None
 358     if competence_territories_id is None:
 359         competence_territories_id = set(ramdb.territory_by_id.iterkeys())
 360     pois_id = set(model.Poi.iter_ids(state, **model.Poi.extract_non_territorial_search_data(state, data)))
 361     pois_id_by_commune_id = {}
 362     rows_count = 0
 363     if pois_id:
 364         pois_id_by_competence_territory_id = {}
 365         for commune_id in competence_territories_id:
 366             commune = ramdb.territory_by_id.get(commune_id)
 367             if commune is None:
 368                 continue
 369             if commune.__class__.__name__ in (u'ArrondissementOfCommuneOfFrance', u'CommuneOfFrance') \
 370                     and commune.code not in (u'13055', u'69123', u'75056'):
 371                 commune_pois_id = set()
 372                 for related_territory_id in ramdb.get_territory_related_territories_id(commune):
 373                     if related_territory_id not in pois_id_by_competence_territory_id:
 374                         related_territory_pois_id = model.Poi.ids_by_competence_territory_id.get(related_territory_id)
 375                         pois_id_by_competence_territory_id[related_territory_id] = pois_id.intersection(
 376                             related_territory_pois_id) if related_territory_pois_id is not None else set()
 377                     commune_pois_id.update(pois_id_by_competence_territory_id[related_territory_id])
 378                 if commune_pois_id:
 379                     pois_id_by_commune_id[commune_id] = commune_pois_id
 380                     rows_count += len(commune_pois_id)
 381                     if rows_count > 65535:
 382                         # Excel doesn't support sheets with more than 65535 rows.
 383                         return None, state._(u'Export is too big. Restrict some search criteria and try again.')
 384     return pois_id_by_commune_id_to_csv_infos(pois_id_by_commune_id, state = state)
 385
 386
 387 def inputs_to_pois_csv_infos(inputs, state = None):
 388     from . import conf, model, ramdb
 389     if state is None:
 390         state = default_state
 391     data, errors = pipe(
 392         model.Poi.make_inputs_to_search_data(),
 393         struct(
 394             dict(
 395                 # By default, when no default_filter is given, export only POIs present on given territory.
 396                 filter = default(conf['default_filter'] or 'presence'),
 397                 ),
 398             default = noop,
 399             keep_none_values = True,
 400             ),
 401         )(inputs, state = state)
 402     if errors is not None:
 403         return data, errors
 404
 405     filter = data['filter']
 406     territory = data['territory']
 407     related_territories_id = ramdb.get_territory_related_territories_id(territory) if territory is not None else None
 408     if filter == 'competence':
 409         competence_territories_id = related_territories_id
 410         presence_territory = None
 411     elif filter == 'presence':
 412         competence_territories_id = None
 413         presence_territory = territory
 414     else:
 415         competence_territories_id = None
 416         presence_territory = None
 417     pois_id = set(model.Poi.iter_ids(state,
 418         competence_territories_id = competence_territories_id or (ramdb.get_territory_related_territories_id(
 419             data['base_territory'],
 420             ) if data.get('base_territory') is not None else None),
 421         presence_territory = presence_territory,
 422         **model.Poi.extract_non_territorial_search_data(state, data)))
 423     if len(pois_id) > 65535:
 424         # Excel doesn't support sheets with more than 65535 rows.
 425         return None, state._(u'Export is too big. Restrict some search criteria and try again.')
 426
 427     # Add sub-...-children of found POIs.
 428     def add_children_id(poi_id, pois_id):
 429         for child_id in (model.Poi.ids_by_parent_id.get(poi_id) or set()):
 430             if child_id not in pois_id:
 431                 pois_id.add(child_id)
 432                 add_children_id(child_id, pois_id)
 433     for poi_id in pois_id.copy():
 434         add_children_id(poi_id, pois_id)
 435         if len(pois_id) > 65535:
 436             # Excel doesn't support sheets with more than 65535 rows.
 437             return None, state._(u'Export is too big. Restrict some search criteria and try again.')
 438
 439     return pois_id_to_csv_infos(pois_id, state = state)
 440
 441
 442 def inputs_to_pois_directory_data(inputs, state = None):
 443     from . import model
 444     if state is None:
 445         state = default_state
 446     return pipe(
 447         model.Poi.make_inputs_to_search_data(),
 448         struct(
 449             dict(
 450                 territory = pipe(
 451 #                    test(lambda territory: territory.__class__.__name__ in model.communes_kinds,
 452 #                        error = N_(u'In "directory" mode, territory must be a commune')),
 453                     test_not_none(error = N_(u'In "directory" mode, a commune is required')),
 454                     ),
 455                 ),
 456             default = noop,
 457             keep_none_values = True,
 458             ),
 459         set_default_filter,
 460         )(inputs, state = state)
 461
 462
 463 def inputs_to_pois_layer_data(inputs, state = None):
 464     from . import model
 465     if state is None:
 466         state = default_state
 467     return pipe(
 468         merge(
 469             model.Poi.make_inputs_to_search_data(),
 470             struct(
 471                 dict(
 472                     bbox = pipe(
 473                         function(lambda bbox: bbox.split(u',')),
 474                         struct(
 475                             [
 476                                 # West longitude
 477                                 pipe(
 478                                     input_to_float,
 479                                     test_between(-180, 180),
 480                                     not_none,
 481                                     ),
 482                                 # South latitude
 483                                 pipe(
 484                                     input_to_float,
 485                                     test_between(-90, 90),
 486                                     not_none,
 487                                     ),
 488                                 # East longitude
 489                                 pipe(
 490                                     input_to_float,
 491                                     test_between(-180, 180),
 492                                     not_none,
 493                                     ),
 494                                 # North latitude
 495                                 pipe(
 496                                     input_to_float,
 497                                     test_between(-90, 90),
 498                                     not_none,
 499                                     ),
 500                                 ],
 501                             ),
 502                         ),
 503                     current = pipe(
 504                         input_to_object_id,
 505                         id_to_poi,
 506                         test(lambda poi: poi.geo is not None, error = N_('POI has no geographical coordinates')),
 507                         ),
 508                     enable_cluster = pipe(
 509                         guess_bool,
 510                         default(True),
 511                         ),
 512                     ),
 513                 default = 'drop',
 514                 keep_none_values = True,
 515                 ),
 516             ),
 517         set_default_filter,
 518         )(inputs, state = state)
 519
 520
 521 def inputs_to_pois_list_data(inputs, state = None):
 522     from . import model
 523     if state is None:
 524         state = default_state
 525     return pipe(
 526         merge(
 527             model.Poi.make_inputs_to_search_data(),
 528             struct(
 529                 dict(
 530                     page = pipe(
 531                         input_to_int,
 532                         test_greater_or_equal(1),
 533                         default(1),
 534                         ),
 535                     sort_key = pipe(
 536                         cleanup_line,
 537                         test_in(['name', 'organism-type', 'postal_distribution_str', 'schema_name', 'street_address']),
 538                         ),
 539                     ),
 540                 default = 'drop',
 541                 keep_none_values = True,
 542                 ),
 543             ),
 544         set_default_filter,
 545         rename_item('page', 'page_number'),
 546         )(inputs, state = state)
 547
 548
 549 def layer_data_to_clusters(data, state = None):
 550     from . import model, ramdb
 551     if data is None:
 552         return None, None
 553     if state is None:
 554         state = default_state
 555     left, bottom, right, top = data['bbox']
 556     center_latitude = (bottom + top) / 2.0
 557     center_latitude_cos = math.cos(math.radians(center_latitude))
 558     center_latitude_sin = math.sin(math.radians(center_latitude))
 559     center_longitude = (left + right) / 2.0
 560     filter = data['filter']
 561     territory = data['territory']
 562     related_territories_id = ramdb.get_territory_related_territories_id(territory) if territory is not None else None
 563     if filter == 'competence':
 564         competence_territories_id = related_territories_id
 565         presence_territory = None
 566     elif filter == 'presence':
 567         competence_territories_id = None
 568         presence_territory = territory
 569     else:
 570         competence_territories_id = None
 571         presence_territory = None
 572     pois_id_iter = model.Poi.iter_ids(state,
 573         competence_territories_id = competence_territories_id or (ramdb.get_territory_related_territories_id(
 574             data['base_territory'],
 575             ) if data.get('base_territory') is not None else None),
 576         presence_territory = presence_territory,
 577         **model.Poi.extract_non_territorial_search_data(state, data))
 578     poi_by_id = model.Poi.instance_by_id
 579     current = data['current']
 580     pois_iter = (
 581         poi
 582         for poi in (
 583             poi_by_id[poi_id]
 584             for poi_id in pois_id_iter
 585             )
 586         if poi.geo is not None and bottom <= poi.geo[0] <= top and left <= poi.geo[1] <= right and (
 587             current is None or poi._id != current._id)
 588         )
 589     distance_and_poi_couples = sorted(
 590         (
 591             (
 592                 # distance from center of map
 593                 6372.8 * math.acos(
 594                     round(
 595                         math.sin(math.radians(poi.geo[0])) * center_latitude_sin
 596                         + math.cos(math.radians(poi.geo[0])) * center_latitude_cos
 597                             * math.cos(math.radians(poi.geo[1] - center_longitude)),
 598                         13,
 599                     )) if poi.geo is not None else (sys.float_info.max, poi),
 600                 # POI
 601                 poi,
 602                 )
 603             for poi in pois_iter
 604             ),
 605         key = lambda distance_and_poi_couple: distance_and_poi_couple[0],
 606         )
 607     pois = [
 608         poi
 609         for distance, poi in distance_and_poi_couples
 610         ]
 611     if current is not None:
 612         pois.insert(0, current)
 613     horizontal_iota = (right - left) / 20.0
 614     vertical_iota = (top - bottom) / 15.0
 615 #    vertical_iota = horizontal_iota = (right - left) / 30.0
 616     clusters = []
 617     for poi in pois:
 618         poi_latitude = poi.geo[0]
 619         poi_longitude = poi.geo[1]
 620         for cluster in clusters:
 621             if abs(poi_latitude - cluster.center_latitude) <= vertical_iota \
 622                     and abs(poi_longitude - cluster.center_longitude) <= horizontal_iota:
 623                 cluster.count += 1
 624                 if poi_latitude == cluster.center_latitude and poi_longitude == cluster.center_longitude:
 625                     cluster.center_pois.append(poi)
 626                 if poi_latitude < cluster.bottom:
 627                     cluster.bottom = poi_latitude
 628                 elif poi_latitude > cluster.top:
 629                     cluster.top = poi_latitude
 630                 if poi_longitude < cluster.left:
 631                     cluster.left = poi_longitude
 632                 elif poi_longitude > cluster.right:
 633                     cluster.right = poi_longitude
 634                 break
 635         else:
 636             cluster = model.Cluster()
 637             cluster.competent = False  # changed below
 638             cluster.count = 1
 639             cluster.bottom = cluster.top = cluster.center_latitude = poi_latitude
 640             cluster.left = cluster.right = cluster.center_longitude = poi_longitude
 641             cluster.center_pois = [poi]
 642             clusters.append(cluster)
 643         if cluster.competent is False:
 644             if related_territories_id is None or poi.competence_territories_id is None:
 645                 cluster.competent = None
 646             elif not related_territories_id.isdisjoint(poi.competence_territories_id):
 647                 cluster.competent = True
 648         elif cluster.competent is None and related_territories_id is not None \
 649                 and poi.competence_territories_id is not None \
 650                 and not related_territories_id.isdisjoint(poi.competence_territories_id):
 651             cluster.competent = True
 652     return clusters, None
 653
 654
 655 def pois_id_by_commune_id_to_csv_infos(pois_id_by_commune_id, state = None):
 656     from . import model, ramdb
 657     if pois_id_by_commune_id is None:
 658         return None, None
 659     if state is None:
 660         state = default_state
 661     csv_infos_by_schema_name = {}
 662     for commune_id, commune_pois_id in pois_id_by_commune_id.iteritems():
 663         commune = ramdb.territory_by_id.get(commune_id)
 664         if commune is None:
 665             continue
 666         for poi_id in commune_pois_id:
 667             poi = model.Poi.instance_by_id.get(poi_id)
 668             if poi is None:
 669                 continue
 670             csv_infos = csv_infos_by_schema_name.get(poi.schema_name)
 671             if csv_infos is None:
 672                 csv_infos_by_schema_name[poi.schema_name] = csv_infos = dict(
 673                     columns_label = [u'Code commune', u'Nom commune'],
 674                     columns_ref = [None, None],
 675                     rows = [],
 676                     )
 677             columns_label = csv_infos['columns_label']
 678             columns_index = {}
 679             columns_ref = csv_infos['columns_ref']
 680             row = [commune.code, commune.name] + [None] * len(columns_ref)
 681             for field_ref, field in poi.iter_csv_fields(state):
 682                 # Detect column number to use for field. Create a new column if needed.
 683                 column_ref = tuple(field_ref[:-1])
 684                 same_ref_columns_count = field_ref[-1]
 685                 if columns_ref.count(column_ref) == same_ref_columns_count:
 686                     column_index = len(columns_ref)
 687                     columns_label.append(field.label)  # or u' - '.join(label for label in field_ref[::2])
 688                     columns_ref.append(column_ref)
 689                     row.append(None)
 690                 else:
 691                     column_index = columns_ref.index(column_ref, columns_index.get(column_ref, -1) + 1)
 692                 columns_index[column_ref] = column_index
 693                 row[column_index] = field.value
 694             csv_infos['rows'].append(row)
 695
 696     # Sort rows by commune code and POI ID.
 697     for csv_infos in csv_infos_by_schema_name.itervalues():
 698         csv_infos['rows'].sort(key = lambda row: (row[0], row[2]))
 699
 700     return csv_infos_by_schema_name or None, None
 701
 702
 703 def pois_id_to_csv_infos(pois_id, state = None):
 704     from . import model
 705     if pois_id is None:
 706         return None, None
 707     if state is None:
 708         state = default_state
 709     csv_infos_by_schema_name = {}
 710     visited_pois_id = set(pois_id)
 711     while pois_id:
 712         remaining_pois_id = []
 713         for poi_id in pois_id:
 714             poi = model.Poi.instance_by_id.get(poi_id)
 715             if poi is None:
 716                 continue
 717             csv_infos = csv_infos_by_schema_name.get(poi.schema_name)
 718             if csv_infos is None:
 719                 csv_infos_by_schema_name[poi.schema_name] = csv_infos = dict(
 720                     columns_label = [],
 721                     columns_ref = [],
 722                     rows = [],
 723                     )
 724             columns_label = csv_infos['columns_label']
 725             columns_index = {}
 726             columns_ref = csv_infos['columns_ref']
 727             row = [None] * len(columns_ref)
 728             for field_ref, field in poi.iter_csv_fields(state):
 729                 # Detect column number to use for field. Create a new column if needed.
 730                 column_ref = tuple(field_ref[:-1])
 731                 same_ref_columns_count = field_ref[-1]
 732                 if columns_ref.count(column_ref) == same_ref_columns_count:
 733                     column_index = len(columns_ref)
 734                     columns_label.append(field.label)  # or u' - '.join(label for label in field_ref[::2])
 735                     columns_ref.append(column_ref)
 736                     row.append(None)
 737                 else:
 738                     column_index = columns_ref.index(column_ref, columns_index.get(column_ref, -1) + 1)
 739                 columns_index[column_ref] = column_index
 740                 row[column_index] = field.value
 741                 for linked_poi_id in (field.linked_pois_id or []):
 742                     if linked_poi_id not in visited_pois_id:
 743                         visited_pois_id.add(linked_poi_id)
 744                         remaining_pois_id.append(linked_poi_id)
 745             csv_infos['rows'].append(row)
 746         pois_id = remaining_pois_id
 747     return csv_infos_by_schema_name or None, None
 748
 749
 750 def postal_distribution_to_territory(postal_distribution, state = None):
 751     from . import ramdb
 752     if postal_distribution is None:
 753         return postal_distribution, None
 754     if state is None:
 755         state = default_state
 756     territory_id = ramdb.territories_id_by_postal_distribution.get(postal_distribution)
 757     if territory_id is None:
 758         return postal_distribution, state._(u'Unknown territory')
 759     territory = ramdb.territory_by_id.get(territory_id)
 760     if territory is None:
 761         return postal_distribution, state._(u'Unknown territory')
 762     return territory, None
 763
 764
 765 def set_default_filter(data, state = None):
 766     if data is None:
 767         return None, None
 768
 769     from . import conf, model
 770
 771     if state is None:
 772         state = default_state
 773
 774     if data.get('filter') is None and conf['default_filter'] is not None:
 775         data['filter'] = conf['default_filter']
 776     elif data.get('filter') is None and data['territory'] is not None \
 777             and data['territory'].__class__.__name__ not in model.communes_kinds:
 778         # When no filter is given and territory is not a commune, search only for POIs present on territory instead of
 779         # POIs near the territory.
 780         data['filter'] = u'presence'
 781     return data, None
 782
 783
 784 def site_to_bson(subscriber, state = None):
 785     if state is None:
 786         state = default_state
 787     return pipe(
 788         object_to_clean_dict,
 789         struct(
 790             dict(
 791                 subscriptions = uniform_sequence(function(lambda subscription: subscription.to_bson())),
 792                 ),
 793             default = noop,
 794             ),
 795         )(session, state = state)
 796
 797
 798 def subscriber_to_bson(subscriber, state = None):
 799     if state is None:
 800         state = default_state
 801     return pipe(
 802         object_to_clean_dict,
 803         struct(
 804             dict(
 805                 sites = uniform_sequence(function(lambda site: site.to_bson())),
 806                 users = uniform_sequence(function(lambda user: user.to_bson())),
 807                 ),
 808             default = noop,
 809             ),
 810         )(session, state = state)
 811
 812
 813 subscription_to_bson = object_to_clean_dict
 814
 815
 816 def test_territory_in_base_territory(data, state = None):
 817     if state is None:
 818         state = default_state
 819     if not data.get('base_territory') or \
 820         data.get('territory') and data['base_territory']._id in data['territory'].ancestors_id:
 821         return data, None
 822     if not data.get('territory'):
 823         return data, None
 824     return data, {'territory': state._(u'Searched territory not located in base territory {0}').format(
 825         data['base_territory'].main_postal_distribution['postal_routing']
 826         )}
 827
 828
 829 user_to_bson = object_to_clean_dict
 830
 831
 832 # Level-2 Converters
 833
 834
 835 input_to_postal_distribution_to_geolocated_territory = pipe(
 836     input_to_postal_distribution,
 837     postal_distribution_to_territory,
 838     test(lambda territory: territory.geo is not None, error = N_(u'Territory has no geographical coordinates')),
 839     )