Ignore LayoutItemPortals in the details view.
[online-glom:gwt-glom.git] / src / main / java / org / glom / web / server / OnlineGlomServiceImpl.java
1 /*
2  * Copyright (C) 2010, 2011 Openismus GmbH
3  *
4  * This file is part of GWT-Glom.
5  *
6  * GWT-Glom is free software: you can redistribute it and/or modify it
7  * under the terms of the GNU Lesser General Public License as published by the
8  * Free Software Foundation, either version 3 of the License, or (at your
9  * option) any later version.
10  *
11  * GWT-Glom is distributed in the hope that it will be useful, but WITHOUT
12  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
13  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU Lesser General Public License
14  * for more details.
15  *
16  * You should have received a copy of the GNU Lesser General Public License
17  * along with GWT-Glom.  If not, see <http://www.gnu.org/licenses/>.
18  */
19
20 package org.glom.web.server;
21
22 import java.io.File;
23 import java.io.FilenameFilter;
24 import java.io.IOException;
25 import java.io.InputStream;
26 import java.sql.Connection;
27 import java.sql.Date;
28 import java.sql.ResultSet;
29 import java.sql.SQLException;
30 import java.sql.Statement;
31 import java.sql.Time;
32 import java.text.DateFormat;
33 import java.text.NumberFormat;
34 import java.util.ArrayList;
35 import java.util.Currency;
36 import java.util.Hashtable;
37 import java.util.Locale;
38 import java.util.Properties;
39
40 import org.glom.libglom.BakeryDocument.LoadFailureCodes;
41 import org.glom.libglom.Document;
42 import org.glom.libglom.Field;
43 import org.glom.libglom.FieldFormatting;
44 import org.glom.libglom.FieldVector;
45 import org.glom.libglom.Glom;
46 import org.glom.libglom.LayoutFieldVector;
47 import org.glom.libglom.LayoutGroupVector;
48 import org.glom.libglom.LayoutItem;
49 import org.glom.libglom.LayoutItemVector;
50 import org.glom.libglom.LayoutItem_Field;
51 import org.glom.libglom.LayoutItem_Portal;
52 import org.glom.libglom.NumericFormat;
53 import org.glom.libglom.SortClause;
54 import org.glom.libglom.SortFieldPair;
55 import org.glom.libglom.StringVector;
56 import org.glom.web.client.OnlineGlomService;
57 import org.glom.web.shared.Documents;
58 import org.glom.web.shared.GlomDocument;
59 import org.glom.web.shared.GlomField;
60 import org.glom.web.shared.layout.Formatting;
61 import org.glom.web.shared.layout.LayoutGroup;
62 import org.glom.web.shared.layout.LayoutItemField;
63 import org.glom.web.shared.layout.LayoutItemPortal;
64
65 import com.google.gwt.user.server.rpc.RemoteServiceServlet;
66 import com.mchange.v2.c3p0.ComboPooledDataSource;
67 import com.mchange.v2.c3p0.DataSources;
68
69 /**
70  * The servlet for retrieving layout information from libglom and data from the underlying PostgreSQL database.
71  * 
72  * TODO: move methods that that require a glom document object to the ConfiguredDocument class.
73  * 
74  * @author Ben Konrath <ben@bagu.org>
75  */
76 @SuppressWarnings("serial")
77 public class OnlineGlomServiceImpl extends RemoteServiceServlet implements OnlineGlomService {
78
79         // convenience class to for dealing with the Online Glom configuration file
80         private class OnlineGlomProperties extends Properties {
81                 public String getKey(String value) {
82                         for (String key : stringPropertyNames()) {
83                                 if (getProperty(key).trim().equals(value))
84                                         return key;
85                         }
86                         return null;
87                 }
88         }
89
90         private final Hashtable<String, ConfiguredDocument> documentMapping = new Hashtable<String, ConfiguredDocument>();
91         // TODO implement locale
92         private final Locale locale = Locale.ROOT;
93
94         /*
95          * This is called when the servlet is started or restarted.
96          */
97         public OnlineGlomServiceImpl() throws Exception {
98
99                 // Find the configuration file. See this thread for background info:
100                 // http://stackoverflow.com/questions/2161054/where-to-place-properties-files-in-a-jsp-servlet-web-application
101                 OnlineGlomProperties config = new OnlineGlomProperties();
102                 InputStream is = Thread.currentThread().getContextClassLoader().getResourceAsStream("onlineglom.properties");
103                 if (is == null) {
104                         Log.fatal("onlineglom.properties not found.");
105                         throw new IOException();
106                 }
107                 config.load(is);
108
109                 // check if we can read the configured glom file directory
110                 String documentDirName = config.getProperty("glom.document.directory");
111                 File documentDir = new File(documentDirName);
112                 if (!documentDir.isDirectory()) {
113                         Log.fatal(documentDirName + " is not a directory.");
114                         throw new IOException();
115                 }
116                 if (!documentDir.canRead()) {
117                         Log.fatal("Can't read the files in : " + documentDirName);
118                         throw new IOException();
119                 }
120
121                 // get and check the glom files in the specified directory
122                 final String glomFileExtension = ".glom";
123                 File[] glomFiles = documentDir.listFiles(new FilenameFilter() {
124                         @Override
125                         public boolean accept(File dir, String name) {
126                                 return name.endsWith(glomFileExtension);
127                         }
128                 });
129                 Glom.libglom_init();
130                 for (File glomFile : glomFiles) {
131                         Document document = new Document();
132                         document.set_file_uri("file://" + glomFile.getAbsolutePath());
133                         int error = 0;
134                         boolean retval = document.load(error);
135                         if (retval == false) {
136                                 String message;
137                                 if (LoadFailureCodes.LOAD_FAILURE_CODE_NOT_FOUND == LoadFailureCodes.swigToEnum(error)) {
138                                         message = "Could not find file: " + glomFile.getAbsolutePath();
139                                 } else {
140                                         message = "An unknown error occurred when trying to load file: " + glomFile.getAbsolutePath();
141                                 }
142                                 Log.error(message);
143                                 // continue with for loop because there may be other documents in the directory
144                                 continue;
145                         }
146
147                         ConfiguredDocument configuredDocument = new ConfiguredDocument(document);
148                         // check if a username and password have been set and work for the current document
149                         String filename = glomFile.getName();
150                         String key = config.getKey(filename);
151                         if (key != null) {
152                                 String[] keyArray = key.split("\\.");
153                                 if (keyArray.length == 3 && "filename".equals(keyArray[2])) {
154                                         // username/password could be set, let's check to see if it works
155                                         String usernameKey = key.replaceAll(keyArray[2], "username");
156                                         String passwordKey = key.replaceAll(keyArray[2], "password");
157                                         configuredDocument.setUsernameAndPassword(config.getProperty(usernameKey),
158                                                         config.getProperty(passwordKey));
159                                 }
160                         }
161
162                         // check the if the global username and password have been set and work with this document
163                         if (!configuredDocument.isAuthenticated()) {
164                                 configuredDocument.setUsernameAndPassword(config.getProperty("glom.document.username"),
165                                                 config.getProperty("glom.document.password"));
166                         }
167
168                         // The key for the hash table is the file name without the .glom extension and with spaces ( ) replaced with
169                         // pluses (+). The space/plus replacement makes the key more friendly for URLs.
170                         String hashKey = filename.substring(0, glomFile.getName().length() - glomFileExtension.length()).replace(
171                                         ' ', '+');
172                         documentMapping.put(hashKey, configuredDocument);
173
174                 }
175         }
176
177         /*
178          * This is called when the servlet is stopped or restarted.
179          * 
180          * @see javax.servlet.GenericServlet#destroy()
181          */
182         @Override
183         public void destroy() {
184                 Glom.libglom_deinit();
185
186                 for (String documenTitle : documentMapping.keySet()) {
187                         ConfiguredDocument configuredDoc = documentMapping.get(documenTitle);
188                         try {
189                                 DataSources.destroy(configuredDoc.getCpds());
190                         } catch (SQLException e) {
191                                 Log.error(documenTitle, "Error cleaning up the ComboPooledDataSource.", e);
192                         }
193                 }
194
195         }
196
197         public GlomDocument getGlomDocument(String documentID) {
198
199                 Document document = documentMapping.get(documentID).getDocument();
200                 GlomDocument glomDocument = new GlomDocument();
201
202                 // get arrays of table names and titles, and find the default table index
203                 StringVector tablesVec = document.get_table_names();
204
205                 int numTables = safeLongToInt(tablesVec.size());
206                 // we don't know how many tables will be hidden so we'll use half of the number of tables for the default size
207                 // of the ArrayList
208                 ArrayList<String> tableNames = new ArrayList<String>(numTables / 2);
209                 ArrayList<String> tableTitles = new ArrayList<String>(numTables / 2);
210                 boolean foundDefaultTable = false;
211                 int visibleIndex = 0;
212                 for (int i = 0; i < numTables; i++) {
213                         String tableName = tablesVec.get(i);
214                         if (!document.get_table_is_hidden(tableName)) {
215                                 tableNames.add(tableName);
216                                 // JNI is "expensive", the comparison will only be called if we haven't already found the default table
217                                 if (!foundDefaultTable && tableName.equals(document.get_default_table())) {
218                                         glomDocument.setDefaultTableIndex(visibleIndex);
219                                         foundDefaultTable = true;
220                                 }
221                                 tableTitles.add(document.get_table_title(tableName));
222                                 visibleIndex++;
223                         }
224                 }
225
226                 // set everything we need
227                 glomDocument.setTableNames(tableNames);
228                 glomDocument.setTableTitles(tableTitles);
229
230                 return glomDocument;
231         }
232
233         public LayoutGroup getListLayout(String documentID, String tableName) {
234                 ConfiguredDocument configuredDoc = documentMapping.get(documentID);
235                 Document document = configuredDoc.getDocument();
236
237                 // Use the default table if the table name hasn't been set or if the table name isn't in the glom document.
238                 // The last check guards against SQL injection attacks since the table name could come from the URL.
239                 if (tableName == null || tableName.isEmpty() || !document.get_table_is_known(tableName)) {
240                         tableName = document.get_default_table();
241                 }
242
243                 // access the layout list
244                 LayoutGroupVector layoutGroupVec = document.get_data_layout_groups("list", tableName);
245                 int listViewLayoutGroupSize = safeLongToInt(layoutGroupVec.size());
246                 org.glom.libglom.LayoutGroup libglomLayoutGroup = null;
247                 if (listViewLayoutGroupSize > 0) {
248                         // a list layout group is defined; we can use the first group as the list
249                         if (listViewLayoutGroupSize > 1)
250                                 Log.warn(documentID, tableName,
251                                                 "The size of the list layout group is greater than 1. Attempting to use the first item for the layout list view.");
252
253                         libglomLayoutGroup = layoutGroupVec.get(0);
254                 } else {
255                         // a list layout group is *not* defined; we are going make a libglom layout group from the list of fields
256                         Log.info(documentID, tableName,
257                                         "A list layout is not defined for this table. Displaying a list layout based on the field list.");
258
259                         FieldVector fieldsVec = document.get_table_fields(tableName);
260                         libglomLayoutGroup = new org.glom.libglom.LayoutGroup();
261                         for (int i = 0; i < fieldsVec.size(); i++) {
262                                 Field field = fieldsVec.get(i);
263                                 LayoutItem_Field layoutItemField = new LayoutItem_Field();
264                                 layoutItemField.set_full_field_details(field);
265                                 libglomLayoutGroup.add_item(layoutItemField);
266                         }
267                 }
268
269                 // confirm the libglom LayoutGroup is not null as per the getLayoutGroup method precondition
270                 if (libglomLayoutGroup == null) {
271                         Log.error(documentID, tableName, "A LayoutGroup was not found. Returning null.");
272                         return null;
273                 }
274
275                 LayoutGroup layoutGroup = getLayoutGroup(documentID, tableName, libglomLayoutGroup);
276
277                 // use the same fields list as will be used for the query
278                 LayoutFieldVector fieldsToGet = getFieldsToShowForSQLQuery(document, tableName, "list");
279                 layoutGroup.setExpectedResultSize(getResultSizeOfSQLQuery(documentID, tableName, fieldsToGet));
280
281                 // Set the primary key index for the table
282                 int primaryKeyIndex = getPrimaryKeyIndex(fieldsToGet);
283                 if (primaryKeyIndex < 0) {
284                         // Add a LayoutItemField for the primary key to the end of the item list in the LayoutGroup because it
285                         // doesn't already contain a primary key.
286                         LayoutItem_Field libglomLayoutItemField = getPrimaryKeyLayoutItemFromFields(document, tableName);
287                         layoutGroup.addItem(convertToGWTGlomLayoutItemField(libglomLayoutItemField));
288                         layoutGroup.setPrimaryKeyIndex(layoutGroup.getItems().size() - 1);
289                         layoutGroup.setHiddenPrimaryKey(true);
290
291                 } else {
292                         layoutGroup.setPrimaryKeyIndex(primaryKeyIndex);
293                 }
294
295                 layoutGroup.setTableName(tableName);
296
297                 return layoutGroup;
298         }
299
300         private LayoutItem_Field getPrimaryKeyLayoutItemFromFields(Document document, String tableName) {
301                 Field primaryKey = null;
302                 FieldVector fieldVec = document.get_table_fields(tableName);
303                 for (int i = 0; i < fieldVec.size(); i++) {
304                         Field field = fieldVec.get(i);
305                         if (field != null && field.get_primary_key()) {
306                                 primaryKey = field;
307                                 break;
308                         }
309                 }
310                 if (primaryKey == null) {
311                         Log.fatal(document.get_database_title(), tableName,
312                                         "A primary key was not found in the FieldVector for this table.");
313                         // TODO throw exception
314                 }
315
316                 LayoutItem_Field libglomLayoutItemField = new LayoutItem_Field();
317                 libglomLayoutItemField.set_full_field_details(primaryKey);
318                 return libglomLayoutItemField;
319         }
320
321         /*
322          * Gets the primary key index of the LayoutFieldVector.
323          */
324         private int getPrimaryKeyIndex(LayoutFieldVector layoutFieldVec) {
325                 for (int i = 0; i < layoutFieldVec.size(); i++) {
326                         LayoutItem_Field layoutItemField = layoutFieldVec.get(i);
327                         Field field = layoutItemField.get_full_field_details();
328                         if (field != null && field.get_primary_key())
329                                 return i;
330                 }
331                 return -1;
332         }
333
334         /*
335          * Get the number of rows a query with the table name and layout fields would return. This is needed for the /* list
336          * view pager.
337          */
338         private int getResultSizeOfSQLQuery(String documentID, String tableName, LayoutFieldVector fieldsToGet) {
339                 ConfiguredDocument configuredDoc = documentMapping.get(documentID);
340                 if (!configuredDoc.isAuthenticated())
341                         return -1;
342                 Connection conn = null;
343                 Statement st = null;
344                 ResultSet rs = null;
345                 try {
346                         // Setup and execute the count query. Special care needs to be take to ensure that the results will be based
347                         // on a cursor so that large amounts of memory are not consumed when the query retrieve a large amount of
348                         // data. Here's the relevant PostgreSQL documentation:
349                         // http://jdbc.postgresql.org/documentation/83/query.html#query-with-cursor
350                         ComboPooledDataSource cpds = configuredDoc.getCpds();
351                         conn = cpds.getConnection();
352                         conn.setAutoCommit(false);
353                         st = conn.createStatement(ResultSet.TYPE_FORWARD_ONLY, ResultSet.CONCUR_READ_ONLY);
354                         String query = Glom.build_sql_select_count_simple(tableName, fieldsToGet);
355                         // TODO Test execution time of this query with when the number of rows in the table is large (say >
356                         // 1,000,000). Test memory usage at the same time (see the todo item in getTableData()).
357                         rs = st.executeQuery(query);
358
359                         // get the number of rows in the query
360                         rs.next();
361                         return rs.getInt(1);
362
363                 } catch (SQLException e) {
364                         Log.error(documentID, tableName, "Error calculating number of rows in the query.", e);
365                         return -1;
366                 } finally {
367                         // cleanup everything that has been used
368                         try {
369                                 if (rs != null)
370                                         rs.close();
371                                 if (st != null)
372                                         st.close();
373                                 if (conn != null)
374                                         conn.close();
375                         } catch (Exception e) {
376                                 Log.error(documentID, tableName,
377                                                 "Error closing database resources. Subsequent database queries may not work.", e);
378                         }
379                 }
380         }
381
382         // FIXME Check if we can use getFieldsToShowForSQLQuery() in these methods
383         public ArrayList<GlomField[]> getListData(String documentID, String tableName, int start, int length) {
384                 return getListData(documentID, tableName, start, length, false, 0, false);
385         }
386
387         public ArrayList<GlomField[]> getSortedListData(String documentID, String tableName, int start, int length,
388                         int sortColumnIndex, boolean isAscending) {
389                 return getListData(documentID, tableName, start, length, true, sortColumnIndex, isAscending);
390         }
391
392         private ArrayList<GlomField[]> getListData(String documentID, String tableName, int start, int length,
393                         boolean useSortClause, int sortColumnIndex, boolean isAscending) {
394
395                 ConfiguredDocument configuredDoc = documentMapping.get(documentID);
396                 if (!configuredDoc.isAuthenticated()) {
397                         return new ArrayList<GlomField[]>();
398                 }
399
400                 Document document = configuredDoc.getDocument();
401
402                 // Use the default table if the table name hasn't been set or if the table name isn't in the glom document.
403                 // The last check guards against SQL injection attacks since the table name could come from the URL.
404                 if (tableName == null || tableName.isEmpty() || !document.get_table_is_known(tableName)) {
405                         tableName = document.get_default_table();
406                 }
407
408                 // access the layout list using the defined layout list or the table fields if there's no layout list
409                 LayoutGroupVector layoutListVec = document.get_data_layout_groups("list", tableName);
410                 LayoutFieldVector layoutFields = new LayoutFieldVector();
411                 SortClause sortClause = new SortClause();
412                 int listViewLayoutGroupSize = safeLongToInt(layoutListVec.size());
413                 if (layoutListVec.size() > 0) {
414                         // a layout list is defined, we can use it to for the LayoutListTable
415                         if (listViewLayoutGroupSize > 1)
416                                 Log.warn(documentID, tableName, "The size of the list view layout group for table is greater than 1. "
417                                                 + "Attempting to use the first item for the layout list view.");
418                         LayoutItemVector layoutItemsVec = layoutListVec.get(0).get_items();
419
420                         // find the defined layout list fields
421                         int numItems = safeLongToInt(layoutItemsVec.size());
422                         for (int i = 0; i < numItems; i++) {
423                                 // TODO add support for other LayoutItems (Text, Image, Button)
424                                 LayoutItem item = layoutItemsVec.get(i);
425                                 LayoutItem_Field layoutItemfield = LayoutItem_Field.cast_dynamic(item);
426                                 if (layoutItemfield != null) {
427                                         // use this field in the layout
428                                         layoutFields.add(layoutItemfield);
429
430                                         // create a sort clause if it's a primary key and we're not asked to sort a specific column
431                                         if (!useSortClause) {
432                                                 Field details = layoutItemfield.get_full_field_details();
433                                                 if (details != null && details.get_primary_key()) {
434                                                         sortClause.addLast(new SortFieldPair(layoutItemfield, true)); // ascending
435                                                 }
436                                         }
437                                 }
438                         }
439                 } else {
440                         // no layout list is defined, use the table fields as the layout list
441                         FieldVector fieldsVec = document.get_table_fields(tableName);
442
443                         // find the fields to display in the layout list
444                         int numItems = safeLongToInt(fieldsVec.size());
445                         for (int i = 0; i < numItems; i++) {
446                                 Field field = fieldsVec.get(i);
447                                 LayoutItem_Field layoutItemField = new LayoutItem_Field();
448                                 layoutItemField.set_full_field_details(field);
449                                 layoutFields.add(layoutItemField);
450
451                                 // create a sort clause if it's a primary key and we're not asked to sort a specific column
452                                 if (!useSortClause) {
453                                         if (field.get_primary_key()) {
454                                                 sortClause.addLast(new SortFieldPair(layoutItemField, true)); // ascending
455                                         }
456                                 }
457                         }
458                 }
459
460                 // create a sort clause for the column we've been asked to sort
461                 if (useSortClause) {
462                         LayoutItem item = layoutFields.get(sortColumnIndex);
463                         LayoutItem_Field field = LayoutItem_Field.cast_dynamic(item);
464                         if (field != null)
465                                 sortClause.addLast(new SortFieldPair(field, isAscending));
466                         else {
467                                 Log.error(documentID, tableName, "Error getting LayoutItem_Field for column index " + sortColumnIndex
468                                                 + ". Cannot create a sort clause for this column.");
469                         }
470                 }
471
472                 // Add a LayoutItem_Field for the primary key to the end of the LayoutFieldVector if it doesn't already contain
473                 // a primary key.
474                 // TODO Can we use a cached LayoutGroup object to find out if we need to add a LayoutItem_Field object for the
475                 // primary key field?
476                 if (getPrimaryKeyIndex(layoutFields) < 0) {
477                         layoutFields.add(getPrimaryKeyLayoutItemFromFields(document, tableName));
478                 }
479
480                 ArrayList<GlomField[]> rowsList = new ArrayList<GlomField[]>();
481                 Connection conn = null;
482                 Statement st = null;
483                 ResultSet rs = null;
484                 try {
485                         // Setup the JDBC driver and get the query. Special care needs to be take to ensure that the results will be
486                         // based on a cursor so that large amounts of memory are not consumed when the query retrieve a large amount
487                         // of data. Here's the relevant PostgreSQL documentation:
488                         // http://jdbc.postgresql.org/documentation/83/query.html#query-with-cursor
489                         ComboPooledDataSource cpds = configuredDoc.getCpds();
490                         conn = cpds.getConnection();
491                         conn.setAutoCommit(false);
492                         st = conn.createStatement(ResultSet.TYPE_FORWARD_ONLY, ResultSet.CONCUR_READ_ONLY);
493                         st.setFetchSize(length);
494                         String query = Glom.build_sql_select_simple(tableName, layoutFields, sortClause) + " OFFSET " + start;
495                         // TODO Test memory usage before and after we execute the query that would result in a large ResultSet.
496                         // We need to ensure that the JDBC driver is in fact returning a cursor based result set that has a low
497                         // memory footprint. Check the difference between this value before and after the query:
498                         // Runtime.getRuntime().totalMemory() - Runtime.getRuntime().freeMemory()
499                         // Test the execution time at the same time (see the todo item in getLayoutListTable()).
500                         rs = st.executeQuery(query);
501
502                         // get the results from the ResultSet
503                         rowsList = getData(documentID, tableName, length, layoutFields, rs);
504                 } catch (SQLException e) {
505                         Log.error(documentID, tableName, "Error executing database query.", e);
506                         // TODO: somehow notify user of problem
507                 } finally {
508                         // cleanup everything that has been used
509                         try {
510                                 if (rs != null)
511                                         rs.close();
512                                 if (st != null)
513                                         st.close();
514                                 if (conn != null)
515                                         conn.close();
516                         } catch (Exception e) {
517                                 Log.error(documentID, tableName,
518                                                 "Error closing database resources. Subsequent database queries may not work.", e);
519                         }
520                 }
521                 return rowsList;
522         }
523
524         private ArrayList<GlomField[]> getData(String documentID, String tableName, int length,
525                         LayoutFieldVector layoutFields, ResultSet rs) throws SQLException {
526
527                 // get the data we've been asked for
528                 int rowCount = 0;
529                 ArrayList<GlomField[]> rowsList = new ArrayList<GlomField[]>();
530                 while (rs.next() && rowCount <= length) {
531                         int layoutFieldsSize = safeLongToInt(layoutFields.size());
532                         GlomField[] rowArray = new GlomField[layoutFieldsSize];
533                         for (int i = 0; i < layoutFieldsSize; i++) {
534                                 // make a new GlomField to set the text and colours
535                                 rowArray[i] = new GlomField();
536
537                                 // get foreground and background colours
538                                 LayoutItem_Field field = layoutFields.get(i);
539                                 FieldFormatting formatting = field.get_formatting_used();
540                                 String fgcolour = formatting.get_text_format_color_foreground();
541                                 if (!fgcolour.isEmpty())
542                                         rowArray[i].setFGColour(convertGdkColorToHtmlColour(fgcolour));
543                                 String bgcolour = formatting.get_text_format_color_background();
544                                 if (!bgcolour.isEmpty())
545                                         rowArray[i].setBGColour(convertGdkColorToHtmlColour(bgcolour));
546
547                                 // Convert the field value to a string based on the glom type. We're doing the formatting on the
548                                 // server side for now but it might be useful to move this to the client side.
549                                 switch (field.get_glom_type()) {
550                                 case TYPE_TEXT:
551                                         String text = rs.getString(i + 1);
552                                         rowArray[i].setText(text != null ? text : "");
553                                         break;
554                                 case TYPE_BOOLEAN:
555                                         rowArray[i].setBoolean(rs.getBoolean(i + 1));
556                                         break;
557                                 case TYPE_NUMERIC:
558                                         // Take care of the numeric formatting before converting the number to a string.
559                                         NumericFormat numFormatGlom = formatting.getM_numeric_format();
560                                         // There's no isCurrency() method in the glom NumericFormat class so we're assuming that the
561                                         // number should be formatted as a currency if the currency code string is not empty.
562                                         String currencyCode = numFormatGlom.getM_currency_symbol();
563                                         NumberFormat numFormatJava = null;
564                                         boolean useGlomCurrencyCode = false;
565                                         if (currencyCode.length() == 3) {
566                                                 // Try to format the currency using the Java Locales system.
567                                                 try {
568                                                         Currency currency = Currency.getInstance(currencyCode);
569                                                         Log.info(documentID, tableName, "A valid ISO 4217 currency code is being used."
570                                                                         + " Overriding the numeric formatting with information from the locale.");
571                                                         int digits = currency.getDefaultFractionDigits();
572                                                         numFormatJava = NumberFormat.getCurrencyInstance(locale);
573                                                         numFormatJava.setCurrency(currency);
574                                                         numFormatJava.setMinimumFractionDigits(digits);
575                                                         numFormatJava.setMaximumFractionDigits(digits);
576                                                 } catch (IllegalArgumentException e) {
577                                                         Log.warn(documentID, tableName, currencyCode + " is not a valid ISO 4217 code."
578                                                                         + " Manually setting currency code with this value.");
579                                                         // The currency code is not this is not an ISO 4217 currency code.
580                                                         // We're going to manually set the currency code and use the glom numeric formatting.
581                                                         useGlomCurrencyCode = true;
582                                                         numFormatJava = convertToJavaNumberFormat(numFormatGlom);
583                                                 }
584                                         } else if (currencyCode.length() > 0) {
585                                                 Log.warn(documentID, tableName, currencyCode + " is not a valid ISO 4217 code."
586                                                                 + " Manually setting currency code with this value.");
587                                                 // The length of the currency code is > 0 and != 3; this is not an ISO 4217 currency code.
588                                                 // We're going to manually set the currency code and use the glom numeric formatting.
589                                                 useGlomCurrencyCode = true;
590                                                 numFormatJava = convertToJavaNumberFormat(numFormatGlom);
591                                         } else {
592                                                 // The length of the currency code is 0; the number is not a currency.
593                                                 numFormatJava = convertToJavaNumberFormat(numFormatGlom);
594                                         }
595
596                                         // TODO: Do I need to do something with NumericFormat.get_default_precision() from libglom?
597
598                                         double number = rs.getDouble(i + 1);
599                                         if (number < 0) {
600                                                 if (formatting.getM_numeric_format().getM_alt_foreground_color_for_negatives())
601                                                         // overrides the set foreground colour
602                                                         rowArray[i].setFGColour(convertGdkColorToHtmlColour(NumericFormat
603                                                                         .get_alternative_color_for_negatives()));
604                                         }
605
606                                         // Finally convert the number to text using the glom currency string if required.
607                                         if (useGlomCurrencyCode) {
608                                                 rowArray[i].setText(currencyCode + " " + numFormatJava.format(number));
609                                         } else {
610                                                 rowArray[i].setText(numFormatJava.format(number));
611                                         }
612                                         break;
613                                 case TYPE_DATE:
614                                         Date date = rs.getDate(i + 1);
615                                         if (date != null) {
616                                                 DateFormat dateFormat = DateFormat.getDateInstance(DateFormat.MEDIUM, locale);
617                                                 rowArray[i].setText(dateFormat.format(date));
618                                         } else {
619                                                 rowArray[i].setText("");
620                                         }
621                                         break;
622                                 case TYPE_TIME:
623                                         Time time = rs.getTime(i + 1);
624                                         if (time != null) {
625                                                 DateFormat timeFormat = DateFormat.getTimeInstance(DateFormat.MEDIUM, locale);
626                                                 rowArray[i].setText(timeFormat.format(time));
627                                         } else {
628                                                 rowArray[i].setText("");
629                                         }
630                                         break;
631                                 case TYPE_IMAGE:
632                                         byte[] image = rs.getBytes(i + 1);
633                                         if (image != null) {
634                                                 // TODO implement field TYPE_IMAGE
635                                                 rowArray[i].setText("Image (FIXME)");
636                                         } else {
637                                                 rowArray[i].setText("");
638                                         }
639                                         break;
640                                 case TYPE_INVALID:
641                                 default:
642                                         Log.warn(documentID, tableName, "Invalid LayoutItem Field type. Using empty string for value.");
643                                         rowArray[i].setText("");
644                                         break;
645                                 }
646                         }
647
648                         // add the row of GlomFields to the ArrayList we're going to return and update the row count
649                         rowsList.add(rowArray);
650                         rowCount++;
651                 }
652
653                 return rowsList;
654         }
655
656         public Documents getDocuments() {
657                 Documents documents = new Documents();
658                 for (String documentID : documentMapping.keySet()) {
659                         ConfiguredDocument configuredDoc = documentMapping.get(documentID);
660                         documents.addDocument(documentID, configuredDoc.getDocument().get_database_title());
661                 }
662                 return documents;
663         }
664
665         /*
666          * This method safely converts longs from libglom into ints. This method was taken from stackoverflow:
667          * 
668          * http://stackoverflow.com/questions/1590831/safely-casting-long-to-int-in-java
669          */
670         private int safeLongToInt(long value) {
671                 if (value < Integer.MIN_VALUE || value > Integer.MAX_VALUE) {
672                         throw new IllegalArgumentException(value + " cannot be cast to int without changing its value.");
673                 }
674                 return (int) value;
675         }
676
677         private NumberFormat convertToJavaNumberFormat(NumericFormat numFormatGlom) {
678                 NumberFormat numFormatJava = NumberFormat.getInstance(locale);
679                 if (numFormatGlom.getM_decimal_places_restricted()) {
680                         int digits = safeLongToInt(numFormatGlom.getM_decimal_places());
681                         numFormatJava.setMinimumFractionDigits(digits);
682                         numFormatJava.setMaximumFractionDigits(digits);
683                 }
684                 numFormatJava.setGroupingUsed(numFormatGlom.getM_use_thousands_separator());
685                 return numFormatJava;
686         }
687
688         /*
689          * Converts a Gdk::Color (16-bits per channel) to an HTML colour (8-bits per channel) by discarding the least
690          * significant 8-bits in each channel.
691          */
692         private String convertGdkColorToHtmlColour(String gdkColor) {
693                 if (gdkColor.length() == 13)
694                         return gdkColor.substring(0, 3) + gdkColor.substring(5, 7) + gdkColor.substring(9, 11);
695                 else if (gdkColor.length() == 7) {
696                         // This shouldn't happen but let's deal with it if it does.
697                         Log.warn("Expected a 13 character string but received a 7 character string. Returning received string.");
698                         return gdkColor;
699                 } else {
700                         Log.error("Did not receive a 13 or 7 character string. Returning black HTML colour code.");
701                         return "#000000";
702                 }
703         }
704
705         /*
706          * This method converts a FieldFormatting.HorizontalAlignment to the equivalent ColumnInfo.HorizontalAlignment. The
707          * need for this comes from the fact that the GWT HorizontalAlignment classes can't be used with RPC and there's no
708          * easy way to use the java-libglom FieldFormatting.HorizontalAlignment enum with RPC. An enum identical to
709          * FieldFormatting.HorizontalAlignment is included in the ColumnInfo class.
710          */
711         private Formatting.HorizontalAlignment convertToGWTGlomHorizonalAlignment(
712                         FieldFormatting.HorizontalAlignment alignment) {
713                 switch (alignment) {
714                 case HORIZONTAL_ALIGNMENT_AUTO:
715                         return Formatting.HorizontalAlignment.HORIZONTAL_ALIGNMENT_AUTO;
716                 case HORIZONTAL_ALIGNMENT_LEFT:
717                         return Formatting.HorizontalAlignment.HORIZONTAL_ALIGNMENT_LEFT;
718                 case HORIZONTAL_ALIGNMENT_RIGHT:
719                         return Formatting.HorizontalAlignment.HORIZONTAL_ALIGNMENT_RIGHT;
720                 default:
721                         Log.error("Recieved an alignment that I don't know about: "
722                                         + FieldFormatting.HorizontalAlignment.class.getName() + "." + alignment.toString() + ". Returning "
723                                         + Formatting.HorizontalAlignment.HORIZONTAL_ALIGNMENT_RIGHT.toString() + ".");
724                         return Formatting.HorizontalAlignment.HORIZONTAL_ALIGNMENT_RIGHT;
725                 }
726         }
727
728         /*
729          * This method converts a Field.glom_field_type to the equivalent ColumnInfo.FieldType. The need for this comes from
730          * the fact that the GWT FieldType classes can't be used with RPC and there's no easy way to use the java-libglom
731          * Field.glom_field_type enum with RPC. An enum identical to FieldFormatting.glom_field_type is included in the
732          * ColumnInfo class.
733          */
734         private LayoutItemField.GlomFieldType convertToGWTGlomFieldType(Field.glom_field_type type) {
735                 switch (type) {
736                 case TYPE_BOOLEAN:
737                         return LayoutItemField.GlomFieldType.TYPE_BOOLEAN;
738                 case TYPE_DATE:
739                         return LayoutItemField.GlomFieldType.TYPE_DATE;
740                 case TYPE_IMAGE:
741                         return LayoutItemField.GlomFieldType.TYPE_IMAGE;
742                 case TYPE_NUMERIC:
743                         return LayoutItemField.GlomFieldType.TYPE_NUMERIC;
744                 case TYPE_TEXT:
745                         return LayoutItemField.GlomFieldType.TYPE_TEXT;
746                 case TYPE_TIME:
747                         return LayoutItemField.GlomFieldType.TYPE_TIME;
748                 case TYPE_INVALID:
749                         Log.info("Returning TYPE_INVALID.");
750                         return LayoutItemField.GlomFieldType.TYPE_INVALID;
751                 default:
752                         Log.error("Recieved a type that I don't know about: " + Field.glom_field_type.class.getName() + "."
753                                         + type.toString() + ". Returning " + LayoutItemField.GlomFieldType.TYPE_INVALID.toString() + ".");
754                         return LayoutItemField.GlomFieldType.TYPE_INVALID;
755                 }
756         }
757
758         /*
759          * (non-Javadoc)
760          * 
761          * @see org.glom.web.client.OnlineGlomService#isAuthenticated(java.lang.String)
762          */
763         public boolean isAuthenticated(String documentID) {
764                 return documentMapping.get(documentID).isAuthenticated();
765         }
766
767         /*
768          * (non-Javadoc)
769          * 
770          * @see org.glom.web.client.OnlineGlomService#checkAuthentication(java.lang.String, java.lang.String,
771          * java.lang.String)
772          */
773         public boolean checkAuthentication(String documentID, String username, String password) {
774                 ConfiguredDocument configuredDoc = documentMapping.get(documentID);
775                 try {
776                         return configuredDoc.setUsernameAndPassword(username, password);
777                 } catch (SQLException e) {
778                         Log.error(documentID, "Unknown SQL Error checking the database authentication.", e);
779                         return false;
780                 }
781         }
782
783         /*
784          * (non-Javadoc)
785          * 
786          * @see org.glom.web.client.OnlineGlomService#getDetailsLayoutGroup(java.lang.String, java.lang.String)
787          */
788         public ArrayList<LayoutGroup> getDetailsLayout(String documentID, String tableName) {
789                 ConfiguredDocument configuredDoc = documentMapping.get(documentID);
790                 Document document = configuredDoc.getDocument();
791
792                 // Use the default table if the table name hasn't been set or if the table name isn't in the glom document.
793                 // The last check guards against SQL injection attacks since the table name could come from the URL.
794                 if (tableName == null || tableName.isEmpty() || !document.get_table_is_known(tableName)) {
795                         tableName = document.get_default_table();
796                 }
797
798                 LayoutGroupVector layoutGroupVec = document.get_data_layout_groups("details", tableName);
799
800                 ArrayList<LayoutGroup> layoutGroups = new ArrayList<LayoutGroup>();
801                 for (int i = 0; i < layoutGroupVec.size(); i++) {
802                         org.glom.libglom.LayoutGroup libglomLayoutGroup = layoutGroupVec.get(i);
803
804                         if (libglomLayoutGroup == null)
805                                 continue;
806
807                         layoutGroups.add(getLayoutGroup(documentID, tableName, libglomLayoutGroup));
808                 }
809
810                 return layoutGroups;
811         }
812
813         /*
814          * Gets a GWT-Glom LayoutGroup object for the specified libglom LayoutGroup object. This is used for getting layout
815          * information for the list and details views.
816          * 
817          * @param documentID Glom document identifier
818          * 
819          * @param tableName table name in the specified Glom document
820          * 
821          * @param libglomLayoutGroup libglom LayoutGroup to convert
822          * 
823          * @precondition libglomLayoutGroup must not be null
824          * 
825          * @return {@link LayoutGroup} object that represents the layout for the specified {@link
826          * org.glom.libglom.LayoutGroup}
827          */
828         private LayoutGroup getLayoutGroup(String documentID, String tableName,
829                         org.glom.libglom.LayoutGroup libglomLayoutGroup) {
830                 LayoutGroup layoutGroup = new LayoutGroup();
831                 layoutGroup.setColumnCount(safeLongToInt(libglomLayoutGroup.get_columns_count()));
832
833                 // look at each child item
834                 LayoutItemVector layoutItemsVec = libglomLayoutGroup.get_items();
835                 for (int i = 0; i < layoutItemsVec.size(); i++) {
836                         org.glom.libglom.LayoutItem libglomLayoutItem = layoutItemsVec.get(i);
837
838                         // just a safety check
839                         if (libglomLayoutItem == null)
840                                 continue;
841
842                         org.glom.web.shared.layout.LayoutItem layoutItem = null;
843                         org.glom.libglom.LayoutGroup group = org.glom.libglom.LayoutGroup.cast_dynamic(libglomLayoutItem);
844                         if (group != null) {
845                                 // libglomLayoutItem is a LayoutGroup
846                                 LayoutItem_Portal layoutItemPortal = LayoutItem_Portal.cast_dynamic(group);
847                                 if (layoutItemPortal != null) {
848                                         // group is a LayoutItemPortal
849                                         layoutItem = new LayoutItemPortal();
850                                 } else {
851                                         // group is *not* a LayoutItemPortal
852                                         // recurse into child groups
853                                         layoutItem = getLayoutGroup(documentID, tableName, group);
854                                 }
855                         } else {
856                                 // libglomLayoutItem is *not* a LayoutGroup
857                                 // create GWT-Glom LayoutItem types based on the the libglom type
858                                 // TODO add support for other LayoutItems (Text, Image, Button etc.)
859                                 LayoutItem_Field libglomLayoutField = LayoutItem_Field.cast_dynamic(libglomLayoutItem);
860                                 if (libglomLayoutField != null) {
861                                         layoutItem = convertToGWTGlomLayoutItemField(libglomLayoutField);
862                                 } else {
863                                         Log.info(documentID, tableName,
864                                                         "Ignoring unknown LayoutItem of type " + libglomLayoutItem.get_part_type_name() + ".");
865                                         continue;
866                                 }
867
868                         }
869
870                         layoutItem.setTitle(libglomLayoutItem.get_title_or_name());
871                         layoutGroup.addItem(layoutItem);
872                 }
873
874                 return layoutGroup;
875         }
876
877         private LayoutItemField convertToGWTGlomLayoutItemField(LayoutItem_Field libglomLayoutField) {
878                 LayoutItemField layoutItemField = new LayoutItemField();
879
880                 // set type
881                 layoutItemField.setType(convertToGWTGlomFieldType(libglomLayoutField.get_glom_type()));
882
883                 // set formatting
884                 Formatting formatting = new Formatting();
885                 formatting.setHorizontalAlignment(convertToGWTGlomHorizonalAlignment(libglomLayoutField
886                                 .get_formatting_used_horizontal_alignment()));
887                 layoutItemField.setFormatting(formatting);
888
889                 return layoutItemField;
890         }
891
892         public GlomField[] getDetailsData(String documentID, String tableName, String primaryKeyValue) {
893
894                 ConfiguredDocument configuredDoc = documentMapping.get(documentID);
895                 Document document = configuredDoc.getDocument();
896
897                 // Use the default table if the table name hasn't been set or if the table name isn't in the glom document.
898                 // The last check guards against SQL injection attacks since the table name could come from the URL.
899                 if (tableName == null || tableName.isEmpty() || !document.get_table_is_known(tableName)) {
900                         tableName = document.get_default_table();
901                 }
902
903                 LayoutFieldVector fieldsToGet = getFieldsToShowForSQLQuery(document, tableName, "details");
904
905                 if (fieldsToGet == null || fieldsToGet.size() <= 0) {
906                         Log.warn(documentID, tableName, "Didn't find any fields to show. Returning null.");
907                         return null;
908                 }
909
910                 // get primary key for the table to use in the SQL query
911                 Field primaryKey = null;
912                 FieldVector fieldsVec = document.get_table_fields(tableName);
913                 for (int i = 0; i < safeLongToInt(fieldsVec.size()); i++) {
914                         Field field = fieldsVec.get(i);
915                         if (field.get_primary_key()) {
916                                 primaryKey = field;
917                                 break;
918                         }
919                 }
920
921                 if (primaryKey == null) {
922                         Log.error(documentID, tableName, "Couldn't find primary key in table. Returning null.");
923                         return null;
924                 }
925
926                 ArrayList<GlomField[]> rowsList = new ArrayList<GlomField[]>();
927                 Connection conn = null;
928                 Statement st = null;
929                 ResultSet rs = null;
930                 try {
931                         // Setup the JDBC driver and get the query.
932                         ComboPooledDataSource cpds = configuredDoc.getCpds();
933                         conn = cpds.getConnection();
934                         st = conn.createStatement(ResultSet.TYPE_FORWARD_ONLY, ResultSet.CONCUR_READ_ONLY);
935                         String query = Glom.build_sql_select_with_key(tableName, fieldsToGet, primaryKey, primaryKeyValue);
936                         rs = st.executeQuery(query);
937
938                         // get the results from the ResultSet
939                         // using 2 as a length parameter so we can log a warning if appropriate
940                         rowsList = getData(documentID, tableName, 2, fieldsToGet, rs);
941                 } catch (SQLException e) {
942                         Log.error(documentID, tableName, "Error executing database query.", e);
943                         // TODO: somehow notify user of problem
944                         return null;
945                 } finally {
946                         // cleanup everything that has been used
947                         try {
948                                 if (rs != null)
949                                         rs.close();
950                                 if (st != null)
951                                         st.close();
952                                 if (conn != null)
953                                         conn.close();
954                         } catch (Exception e) {
955                                 Log.error(documentID, tableName,
956                                                 "Error closing database resources. Subsequent database queries may not work.", e);
957                         }
958                 }
959
960                 if (rowsList.size() == 0) {
961                         Log.error(documentID, tableName, "The query returned an empty ResultSet. Returning null.");
962                         return null;
963                 } else if (rowsList.size() > 1 && primaryKeyValue != null && !primaryKeyValue.isEmpty()) {
964                         // only log a warning if the result size is greater than 1 and the primaryKeyValue was set
965                         Log.warn(documentID, tableName,
966                                         "The query did not return the expected unique result. Returning the first result in the set.");
967                 }
968
969                 return rowsList.get(0);
970
971         }
972
973         /*
974          * Gets a LayoutFieldVector to use when generating an SQL query.
975          */
976         private LayoutFieldVector getFieldsToShowForSQLQuery(Document document, String tableName, String layoutName) {
977                 LayoutGroupVector layoutGroupVec = document.get_data_layout_groups(layoutName, tableName);
978                 LayoutFieldVector layoutFieldVector = new LayoutFieldVector();
979
980                 // special case for list layouts that don't have a defined layout group
981                 if ("list".equals(layoutName) && layoutGroupVec.size() == 0) {
982                         FieldVector fieldsVec = document.get_table_fields(tableName);
983                         for (int i = 0; i < fieldsVec.size(); i++) {
984                                 Field field = fieldsVec.get(i);
985                                 LayoutItem_Field layoutItemField = new LayoutItem_Field();
986                                 layoutItemField.set_full_field_details(field);
987                                 layoutFieldVector.add(layoutItemField);
988                         }
989                         return layoutFieldVector;
990                 }
991
992                 // We will show the fields that the document says we should:
993                 for (int i = 0; i < layoutGroupVec.size(); i++) {
994                         org.glom.libglom.LayoutGroup layoutGroup = layoutGroupVec.get(i);
995
996                         // Get the fields:
997                         ArrayList<LayoutItem_Field> layoutItemsFields = getFieldsToShowForSQLQueryAddGroup(document, tableName,
998                                         layoutGroup);
999                         for (LayoutItem_Field layoutItem_Field : layoutItemsFields) {
1000                                 layoutFieldVector.add(layoutItem_Field);
1001                         }
1002                 }
1003                 return layoutFieldVector;
1004         }
1005
1006         private ArrayList<LayoutItem_Field> getFieldsToShowForSQLQueryAddGroup(Document document, String tableName,
1007                         org.glom.libglom.LayoutGroup layoutGroup) {
1008
1009                 ArrayList<LayoutItem_Field> layoutItemFields = new ArrayList<LayoutItem_Field>();
1010                 LayoutItemVector items = layoutGroup.get_items();
1011                 for (int i = 0; i < items.size(); i++) {
1012                         LayoutItem layoutItem = items.get(i);
1013
1014                         LayoutItem_Field layoutItemField = LayoutItem_Field.cast_dynamic(layoutItem);
1015                         if (layoutItemField != null) {
1016                                 // the layoutItem is a LayoutItem_Field
1017                                 FieldVector fields;
1018                                 if (layoutItemField.get_has_relationship_name()) {
1019                                         // layoutItemField is a field in a related table
1020                                         fields = document.get_table_fields(layoutItemField.get_table_used(tableName));
1021                                 } else {
1022                                         // layoutItemField is a field in this table
1023                                         fields = document.get_table_fields(tableName);
1024                                 }
1025
1026                                 // set the layoutItemFeild with details from its Field in the document and
1027                                 // add it to the list to be returned
1028                                 for (int j = 0; j < fields.size(); j++) {
1029                                         // check the names to see if they're the same
1030                                         // this works because we're using the field list from the related table if necessary
1031                                         if (layoutItemField.get_name().equals(fields.get(j).get_name())) {
1032                                                 Field field = fields.get(j);
1033                                                 if (field != null) {
1034                                                         layoutItemField.set_full_field_details(field);
1035                                                         layoutItemFields.add(layoutItemField);
1036                                                 } else {
1037                                                         Log.warn(document.get_database_title(), tableName,
1038                                                                         "LayoutItem_Field " + layoutItemField.get_layout_display_name()
1039                                                                                         + " not found in document field list.");
1040                                                 }
1041                                                 break;
1042                                         }
1043                                 }
1044
1045                         } else {
1046                                 // the layoutItem is not a LayoutItem_Field
1047                                 org.glom.libglom.LayoutGroup subLayoutGroup = org.glom.libglom.LayoutGroup.cast_dynamic(layoutItem);
1048                                 if (subLayoutGroup != null) {
1049                                         // the layoutItem is a LayoutGroup
1050                                         LayoutItem_Portal layoutItemPortal = LayoutItem_Portal.cast_dynamic(layoutItem);
1051                                         if (layoutItemPortal == null) {
1052                                                 // The subGroup is not a LayoutItem_Portal.
1053                                                 // We're ignoring portals because they are filled by means of a separate SQL query.
1054                                                 layoutItemFields
1055                                                                 .addAll(getFieldsToShowForSQLQueryAddGroup(document, tableName, subLayoutGroup));
1056                                         }
1057                                 }
1058                         }
1059                 }
1060                 return layoutItemFields;
1061         }
1062
1063 }