Cleanup comments.
[online-glom:gwt-glom.git] / src / main / java / org / glom / web / server / OnlineGlomServiceImpl.java
1 /*
2  * Copyright (C) 2010, 2011 Openismus GmbH
3  *
4  * This file is part of GWT-Glom.
5  *
6  * GWT-Glom is free software: you can redistribute it and/or modify it
7  * under the terms of the GNU Lesser General Public License as published by the
8  * Free Software Foundation, either version 3 of the License, or (at your
9  * option) any later version.
10  *
11  * GWT-Glom is distributed in the hope that it will be useful, but WITHOUT
12  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
13  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU Lesser General Public License
14  * for more details.
15  *
16  * You should have received a copy of the GNU Lesser General Public License
17  * along with GWT-Glom.  If not, see <http://www.gnu.org/licenses/>.
18  */
19
20 package org.glom.web.server;
21
22 import java.io.File;
23 import java.io.FilenameFilter;
24 import java.io.IOException;
25 import java.io.InputStream;
26 import java.sql.Connection;
27 import java.sql.Date;
28 import java.sql.ResultSet;
29 import java.sql.SQLException;
30 import java.sql.Statement;
31 import java.sql.Time;
32 import java.text.DateFormat;
33 import java.text.NumberFormat;
34 import java.util.ArrayList;
35 import java.util.Currency;
36 import java.util.Hashtable;
37 import java.util.Locale;
38 import java.util.Properties;
39
40 import org.glom.libglom.BakeryDocument.LoadFailureCodes;
41 import org.glom.libglom.Document;
42 import org.glom.libglom.Field;
43 import org.glom.libglom.FieldFormatting;
44 import org.glom.libglom.FieldVector;
45 import org.glom.libglom.Glom;
46 import org.glom.libglom.LayoutFieldVector;
47 import org.glom.libglom.LayoutGroupVector;
48 import org.glom.libglom.LayoutItem;
49 import org.glom.libglom.LayoutItemVector;
50 import org.glom.libglom.LayoutItem_Field;
51 import org.glom.libglom.LayoutItem_Portal;
52 import org.glom.libglom.NumericFormat;
53 import org.glom.libglom.SortClause;
54 import org.glom.libglom.SortFieldPair;
55 import org.glom.libglom.StringVector;
56 import org.glom.web.client.OnlineGlomService;
57 import org.glom.web.shared.Documents;
58 import org.glom.web.shared.GlomDocument;
59 import org.glom.web.shared.GlomField;
60 import org.glom.web.shared.layout.Formatting;
61 import org.glom.web.shared.layout.LayoutGroup;
62 import org.glom.web.shared.layout.LayoutItemField;
63
64 import com.google.gwt.user.server.rpc.RemoteServiceServlet;
65 import com.mchange.v2.c3p0.ComboPooledDataSource;
66 import com.mchange.v2.c3p0.DataSources;
67
68 /**
69  * The servlet for retrieving layout information from libglom and data from the underlying PostgreSQL database.
70  * 
71  * TODO: move methods that that require a glom document object to the ConfiguredDocument class.
72  * 
73  * @author Ben Konrath <ben@bagu.org>
74  */
75 @SuppressWarnings("serial")
76 public class OnlineGlomServiceImpl extends RemoteServiceServlet implements OnlineGlomService {
77
78         // convenience class to for dealing with the Online Glom configuration file
79         private class OnlineGlomProperties extends Properties {
80                 public String getKey(String value) {
81                         for (String key : stringPropertyNames()) {
82                                 if (getProperty(key).trim().equals(value))
83                                         return key;
84                         }
85                         return null;
86                 }
87         }
88
89         private final Hashtable<String, ConfiguredDocument> documentMapping = new Hashtable<String, ConfiguredDocument>();
90         // TODO implement locale
91         private final Locale locale = Locale.ROOT;
92
93         /*
94          * This is called when the servlet is started or restarted.
95          */
96         public OnlineGlomServiceImpl() throws Exception {
97
98                 // Find the configuration file. See this thread for background info:
99                 // http://stackoverflow.com/questions/2161054/where-to-place-properties-files-in-a-jsp-servlet-web-application
100                 OnlineGlomProperties config = new OnlineGlomProperties();
101                 InputStream is = Thread.currentThread().getContextClassLoader().getResourceAsStream("onlineglom.properties");
102                 if (is == null) {
103                         Log.fatal("onlineglom.properties not found.");
104                         throw new IOException();
105                 }
106                 config.load(is);
107
108                 // check if we can read the configured glom file directory
109                 String documentDirName = config.getProperty("glom.document.directory");
110                 File documentDir = new File(documentDirName);
111                 if (!documentDir.isDirectory()) {
112                         Log.fatal(documentDirName + " is not a directory.");
113                         throw new IOException();
114                 }
115                 if (!documentDir.canRead()) {
116                         Log.fatal("Can't read the files in : " + documentDirName);
117                         throw new IOException();
118                 }
119
120                 // get and check the glom files in the specified directory
121                 final String glomFileExtension = ".glom";
122                 File[] glomFiles = documentDir.listFiles(new FilenameFilter() {
123                         @Override
124                         public boolean accept(File dir, String name) {
125                                 return name.endsWith(glomFileExtension);
126                         }
127                 });
128                 Glom.libglom_init();
129                 for (File glomFile : glomFiles) {
130                         Document document = new Document();
131                         document.set_file_uri("file://" + glomFile.getAbsolutePath());
132                         int error = 0;
133                         boolean retval = document.load(error);
134                         if (retval == false) {
135                                 String message;
136                                 if (LoadFailureCodes.LOAD_FAILURE_CODE_NOT_FOUND == LoadFailureCodes.swigToEnum(error)) {
137                                         message = "Could not find file: " + glomFile.getAbsolutePath();
138                                 } else {
139                                         message = "An unknown error occurred when trying to load file: " + glomFile.getAbsolutePath();
140                                 }
141                                 Log.error(message);
142                                 // continue with for loop because there may be other documents in the directory
143                                 continue;
144                         }
145
146                         ConfiguredDocument configuredDocument = new ConfiguredDocument(document);
147                         // check if a username and password have been set and work for the current document
148                         String filename = glomFile.getName();
149                         String key = config.getKey(filename);
150                         if (key != null) {
151                                 String[] keyArray = key.split("\\.");
152                                 if (keyArray.length == 3 && "filename".equals(keyArray[2])) {
153                                         // username/password could be set, let's check to see if it works
154                                         String usernameKey = key.replaceAll(keyArray[2], "username");
155                                         String passwordKey = key.replaceAll(keyArray[2], "password");
156                                         configuredDocument.setUsernameAndPassword(config.getProperty(usernameKey),
157                                                         config.getProperty(passwordKey));
158                                 }
159                         }
160
161                         // check the if the global username and password have been set and work with this document
162                         if (!configuredDocument.isAuthenticated()) {
163                                 configuredDocument.setUsernameAndPassword(config.getProperty("glom.document.username"),
164                                                 config.getProperty("glom.document.password"));
165                         }
166
167                         // The key for the hash table is the file name without the .glom extension and with spaces ( ) replaced with
168                         // pluses (+). The space/plus replacement makes the key more friendly for URLs.
169                         String hashKey = filename.substring(0, glomFile.getName().length() - glomFileExtension.length()).replace(
170                                         ' ', '+');
171                         documentMapping.put(hashKey, configuredDocument);
172
173                 }
174         }
175
176         /*
177          * This is called when the servlet is stopped or restarted.
178          * 
179          * @see javax.servlet.GenericServlet#destroy()
180          */
181         @Override
182         public void destroy() {
183                 Glom.libglom_deinit();
184
185                 for (String documenTitle : documentMapping.keySet()) {
186                         ConfiguredDocument configuredDoc = documentMapping.get(documenTitle);
187                         try {
188                                 DataSources.destroy(configuredDoc.getCpds());
189                         } catch (SQLException e) {
190                                 Log.error(documenTitle, "Error cleaning up the ComboPooledDataSource.", e);
191                         }
192                 }
193
194         }
195
196         public GlomDocument getGlomDocument(String documentID) {
197
198                 Document document = documentMapping.get(documentID).getDocument();
199                 GlomDocument glomDocument = new GlomDocument();
200
201                 // get arrays of table names and titles, and find the default table index
202                 StringVector tablesVec = document.get_table_names();
203
204                 int numTables = safeLongToInt(tablesVec.size());
205                 // we don't know how many tables will be hidden so we'll use half of the number of tables for the default size
206                 // of the ArrayList
207                 ArrayList<String> tableNames = new ArrayList<String>(numTables / 2);
208                 ArrayList<String> tableTitles = new ArrayList<String>(numTables / 2);
209                 boolean foundDefaultTable = false;
210                 int visibleIndex = 0;
211                 for (int i = 0; i < numTables; i++) {
212                         String tableName = tablesVec.get(i);
213                         if (!document.get_table_is_hidden(tableName)) {
214                                 tableNames.add(tableName);
215                                 // JNI is "expensive", the comparison will only be called if we haven't already found the default table
216                                 if (!foundDefaultTable && tableName.equals(document.get_default_table())) {
217                                         glomDocument.setDefaultTableIndex(visibleIndex);
218                                         foundDefaultTable = true;
219                                 }
220                                 tableTitles.add(document.get_table_title(tableName));
221                                 visibleIndex++;
222                         }
223                 }
224
225                 // set everything we need
226                 glomDocument.setTableNames(tableNames);
227                 glomDocument.setTableTitles(tableTitles);
228
229                 return glomDocument;
230         }
231
232         public LayoutGroup getListLayout(String documentID, String tableName) {
233                 ConfiguredDocument configuredDoc = documentMapping.get(documentID);
234                 Document document = configuredDoc.getDocument();
235
236                 // Use the default table if the table name hasn't been set or if the table name isn't in the glom document.
237                 // The last check guards against SQL injection attacks since the table name could come from the URL.
238                 if (tableName == null || tableName.isEmpty() || !document.get_table_is_known(tableName)) {
239                         tableName = document.get_default_table();
240                 }
241
242                 // access the layout list
243                 LayoutGroupVector layoutGroupVec = document.get_data_layout_groups("list", tableName);
244                 int listViewLayoutGroupSize = safeLongToInt(layoutGroupVec.size());
245                 org.glom.libglom.LayoutGroup libglomLayoutGroup = null;
246                 if (listViewLayoutGroupSize > 0) {
247                         // a list layout group is defined; we can use the first group as the list
248                         if (listViewLayoutGroupSize > 1)
249                                 Log.warn(documentID, tableName,
250                                                 "The size of the list layout group is greater than 1. Attempting to use the first item for the layout list view.");
251
252                         libglomLayoutGroup = layoutGroupVec.get(0);
253                 } else {
254                         // a list layout group is *not* defined; we are going make a libglom layout group from the list of fields
255                         Log.info(documentID, tableName,
256                                         "A list layout is not defined for this table. Displaying a list layout based on the field list.");
257
258                         FieldVector fieldsVec = document.get_table_fields(tableName);
259                         libglomLayoutGroup = new org.glom.libglom.LayoutGroup();
260                         for (int i = 0; i < fieldsVec.size(); i++) {
261                                 Field field = fieldsVec.get(i);
262                                 LayoutItem_Field layoutItemField = new LayoutItem_Field();
263                                 layoutItemField.set_full_field_details(field);
264                                 libglomLayoutGroup.add_item(layoutItemField);
265                         }
266                 }
267
268                 // confirm the libglom LayoutGroup is not null as per the getLayoutGroup method precondition
269                 if (libglomLayoutGroup == null) {
270                         Log.error(documentID, tableName, "A LayoutGroup was not found. Returning null.");
271                         return null;
272                 }
273
274                 LayoutGroup layoutGroup = getLayoutGroup(documentID, tableName, libglomLayoutGroup);
275
276                 // use the same fields list as will be used for the query
277                 LayoutFieldVector fieldsToGet = getFieldsToShowForSQLQuery(document, tableName, "list");
278                 layoutGroup.setExpectedResultSize(getResultSizeOfSQLQuery(documentID, tableName, fieldsToGet));
279
280                 // Set the primary key index for the table
281                 int primaryKeyIndex = getPrimaryKeyIndex(fieldsToGet);
282                 if (primaryKeyIndex < 0) {
283                         // Add a LayoutItemField for the primary key to the end of the item list in the LayoutGroup because it
284                         // doesn't already contain a primary key.
285                         LayoutItem_Field libglomLayoutItemField = getPrimaryKeyLayoutItemFromFields(document, tableName);
286                         layoutGroup.addItem(convertToGWTGlomLayoutItemField(libglomLayoutItemField));
287                         layoutGroup.setPrimaryKeyIndex(layoutGroup.getItems().size() - 1);
288                         layoutGroup.setHiddenPrimaryKey(true);
289
290                 } else {
291                         layoutGroup.setPrimaryKeyIndex(primaryKeyIndex);
292                 }
293
294                 layoutGroup.setTableName(tableName);
295
296                 return layoutGroup;
297         }
298
299         private LayoutItem_Field getPrimaryKeyLayoutItemFromFields(Document document, String tableName) {
300                 Field primaryKey = null;
301                 FieldVector fieldVec = document.get_table_fields(tableName);
302                 for (int i = 0; i < fieldVec.size(); i++) {
303                         Field field = fieldVec.get(i);
304                         if (field != null && field.get_primary_key()) {
305                                 primaryKey = field;
306                                 break;
307                         }
308                 }
309                 if (primaryKey == null) {
310                         Log.fatal(document.get_database_title(), tableName,
311                                         "A primary key was not found in the FieldVector for this table.");
312                         // TODO throw exception
313                 }
314
315                 LayoutItem_Field libglomLayoutItemField = new LayoutItem_Field();
316                 libglomLayoutItemField.set_full_field_details(primaryKey);
317                 return libglomLayoutItemField;
318         }
319
320         /*
321          * Gets the primary key index of the LayoutFieldVector.
322          */
323         private int getPrimaryKeyIndex(LayoutFieldVector layoutFieldVec) {
324                 for (int i = 0; i < layoutFieldVec.size(); i++) {
325                         LayoutItem_Field layoutItemField = layoutFieldVec.get(i);
326                         Field field = layoutItemField.get_full_field_details();
327                         if (field != null && field.get_primary_key())
328                                 return i;
329                 }
330                 return -1;
331         }
332
333         /*
334          * Get the number of rows a query with the table name and layout fields would return. This is needed for the /* list
335          * view pager.
336          */
337         private int getResultSizeOfSQLQuery(String documentID, String tableName, LayoutFieldVector fieldsToGet) {
338                 ConfiguredDocument configuredDoc = documentMapping.get(documentID);
339                 if (!configuredDoc.isAuthenticated())
340                         return -1;
341                 Connection conn = null;
342                 Statement st = null;
343                 ResultSet rs = null;
344                 try {
345                         // Setup and execute the count query. Special care needs to be take to ensure that the results will be based
346                         // on a cursor so that large amounts of memory are not consumed when the query retrieve a large amount of
347                         // data. Here's the relevant PostgreSQL documentation:
348                         // http://jdbc.postgresql.org/documentation/83/query.html#query-with-cursor
349                         ComboPooledDataSource cpds = configuredDoc.getCpds();
350                         conn = cpds.getConnection();
351                         conn.setAutoCommit(false);
352                         st = conn.createStatement(ResultSet.TYPE_FORWARD_ONLY, ResultSet.CONCUR_READ_ONLY);
353                         String query = Glom.build_sql_select_count_simple(tableName, fieldsToGet);
354                         // TODO Test execution time of this query with when the number of rows in the table is large (say >
355                         // 1,000,000). Test memory usage at the same time (see the todo item in getTableData()).
356                         rs = st.executeQuery(query);
357
358                         // get the number of rows in the query
359                         rs.next();
360                         return rs.getInt(1);
361
362                 } catch (SQLException e) {
363                         Log.error(documentID, tableName, "Error calculating number of rows in the query.", e);
364                         return -1;
365                 } finally {
366                         // cleanup everything that has been used
367                         try {
368                                 if (rs != null)
369                                         rs.close();
370                                 if (st != null)
371                                         st.close();
372                                 if (conn != null)
373                                         conn.close();
374                         } catch (Exception e) {
375                                 Log.error(documentID, tableName,
376                                                 "Error closing database resources. Subsequent database queries may not work.", e);
377                         }
378                 }
379         }
380
381         // FIXME Check if we can use getFieldsToShowForSQLQuery() in these methods
382         public ArrayList<GlomField[]> getListData(String documentID, String tableName, int start, int length) {
383                 return getListData(documentID, tableName, start, length, false, 0, false);
384         }
385
386         public ArrayList<GlomField[]> getSortedListData(String documentID, String tableName, int start, int length,
387                         int sortColumnIndex, boolean isAscending) {
388                 return getListData(documentID, tableName, start, length, true, sortColumnIndex, isAscending);
389         }
390
391         private ArrayList<GlomField[]> getListData(String documentID, String tableName, int start, int length,
392                         boolean useSortClause, int sortColumnIndex, boolean isAscending) {
393
394                 ConfiguredDocument configuredDoc = documentMapping.get(documentID);
395                 if (!configuredDoc.isAuthenticated()) {
396                         return new ArrayList<GlomField[]>();
397                 }
398
399                 Document document = configuredDoc.getDocument();
400
401                 // Use the default table if the table name hasn't been set or if the table name isn't in the glom document.
402                 // The last check guards against SQL injection attacks since the table name could come from the URL.
403                 if (tableName == null || tableName.isEmpty() || !document.get_table_is_known(tableName)) {
404                         tableName = document.get_default_table();
405                 }
406
407                 // access the layout list using the defined layout list or the table fields if there's no layout list
408                 LayoutGroupVector layoutListVec = document.get_data_layout_groups("list", tableName);
409                 LayoutFieldVector layoutFields = new LayoutFieldVector();
410                 SortClause sortClause = new SortClause();
411                 int listViewLayoutGroupSize = safeLongToInt(layoutListVec.size());
412                 if (layoutListVec.size() > 0) {
413                         // a layout list is defined, we can use it to for the LayoutListTable
414                         if (listViewLayoutGroupSize > 1)
415                                 Log.warn(documentID, tableName, "The size of the list view layout group for table is greater than 1. "
416                                                 + "Attempting to use the first item for the layout list view.");
417                         LayoutItemVector layoutItemsVec = layoutListVec.get(0).get_items();
418
419                         // find the defined layout list fields
420                         int numItems = safeLongToInt(layoutItemsVec.size());
421                         for (int i = 0; i < numItems; i++) {
422                                 // TODO add support for other LayoutItems (Text, Image, Button)
423                                 LayoutItem item = layoutItemsVec.get(i);
424                                 LayoutItem_Field layoutItemfield = LayoutItem_Field.cast_dynamic(item);
425                                 if (layoutItemfield != null) {
426                                         // use this field in the layout
427                                         layoutFields.add(layoutItemfield);
428
429                                         // create a sort clause if it's a primary key and we're not asked to sort a specific column
430                                         if (!useSortClause) {
431                                                 Field details = layoutItemfield.get_full_field_details();
432                                                 if (details != null && details.get_primary_key()) {
433                                                         sortClause.addLast(new SortFieldPair(layoutItemfield, true)); // ascending
434                                                 }
435                                         }
436                                 }
437                         }
438                 } else {
439                         // no layout list is defined, use the table fields as the layout list
440                         FieldVector fieldsVec = document.get_table_fields(tableName);
441
442                         // find the fields to display in the layout list
443                         int numItems = safeLongToInt(fieldsVec.size());
444                         for (int i = 0; i < numItems; i++) {
445                                 Field field = fieldsVec.get(i);
446                                 LayoutItem_Field layoutItemField = new LayoutItem_Field();
447                                 layoutItemField.set_full_field_details(field);
448                                 layoutFields.add(layoutItemField);
449
450                                 // create a sort clause if it's a primary key and we're not asked to sort a specific column
451                                 if (!useSortClause) {
452                                         if (field.get_primary_key()) {
453                                                 sortClause.addLast(new SortFieldPair(layoutItemField, true)); // ascending
454                                         }
455                                 }
456                         }
457                 }
458
459                 // create a sort clause for the column we've been asked to sort
460                 if (useSortClause) {
461                         LayoutItem item = layoutFields.get(sortColumnIndex);
462                         LayoutItem_Field field = LayoutItem_Field.cast_dynamic(item);
463                         if (field != null)
464                                 sortClause.addLast(new SortFieldPair(field, isAscending));
465                         else {
466                                 Log.error(documentID, tableName, "Error getting LayoutItem_Field for column index " + sortColumnIndex
467                                                 + ". Cannot create a sort clause for this column.");
468                         }
469                 }
470
471                 // Add a LayoutItem_Field for the primary key to the end of the LayoutFieldVector if it doesn't already contain
472                 // a primary key.
473                 // TODO Can we use a cached LayoutGroup object to find out if we need to add a LayoutItem_Field object for the
474                 // primary key field?
475                 if (getPrimaryKeyIndex(layoutFields) < 0) {
476                         layoutFields.add(getPrimaryKeyLayoutItemFromFields(document, tableName));
477                 }
478
479                 ArrayList<GlomField[]> rowsList = new ArrayList<GlomField[]>();
480                 Connection conn = null;
481                 Statement st = null;
482                 ResultSet rs = null;
483                 try {
484                         // Setup the JDBC driver and get the query. Special care needs to be take to ensure that the results will be
485                         // based on a cursor so that large amounts of memory are not consumed when the query retrieve a large amount
486                         // of data. Here's the relevant PostgreSQL documentation:
487                         // http://jdbc.postgresql.org/documentation/83/query.html#query-with-cursor
488                         ComboPooledDataSource cpds = configuredDoc.getCpds();
489                         conn = cpds.getConnection();
490                         conn.setAutoCommit(false);
491                         st = conn.createStatement(ResultSet.TYPE_FORWARD_ONLY, ResultSet.CONCUR_READ_ONLY);
492                         st.setFetchSize(length);
493                         String query = Glom.build_sql_select_simple(tableName, layoutFields, sortClause) + " OFFSET " + start;
494                         // TODO Test memory usage before and after we execute the query that would result in a large ResultSet.
495                         // We need to ensure that the JDBC driver is in fact returning a cursor based result set that has a low
496                         // memory footprint. Check the difference between this value before and after the query:
497                         // Runtime.getRuntime().totalMemory() - Runtime.getRuntime().freeMemory()
498                         // Test the execution time at the same time (see the todo item in getLayoutListTable()).
499                         rs = st.executeQuery(query);
500
501                         // get the results from the ResultSet
502                         rowsList = getData(documentID, tableName, length, layoutFields, rs);
503                 } catch (SQLException e) {
504                         Log.error(documentID, tableName, "Error executing database query.", e);
505                         // TODO: somehow notify user of problem
506                 } finally {
507                         // cleanup everything that has been used
508                         try {
509                                 if (rs != null)
510                                         rs.close();
511                                 if (st != null)
512                                         st.close();
513                                 if (conn != null)
514                                         conn.close();
515                         } catch (Exception e) {
516                                 Log.error(documentID, tableName,
517                                                 "Error closing database resources. Subsequent database queries may not work.", e);
518                         }
519                 }
520                 return rowsList;
521         }
522
523         private ArrayList<GlomField[]> getData(String documentID, String tableName, int length,
524                         LayoutFieldVector layoutFields, ResultSet rs) throws SQLException {
525
526                 // get the data we've been asked for
527                 int rowCount = 0;
528                 ArrayList<GlomField[]> rowsList = new ArrayList<GlomField[]>();
529                 while (rs.next() && rowCount <= length) {
530                         int layoutFieldsSize = safeLongToInt(layoutFields.size());
531                         GlomField[] rowArray = new GlomField[layoutFieldsSize];
532                         for (int i = 0; i < layoutFieldsSize; i++) {
533                                 // make a new GlomField to set the text and colours
534                                 rowArray[i] = new GlomField();
535
536                                 // get foreground and background colours
537                                 LayoutItem_Field field = layoutFields.get(i);
538                                 FieldFormatting formatting = field.get_formatting_used();
539                                 String fgcolour = formatting.get_text_format_color_foreground();
540                                 if (!fgcolour.isEmpty())
541                                         rowArray[i].setFGColour(convertGdkColorToHtmlColour(fgcolour));
542                                 String bgcolour = formatting.get_text_format_color_background();
543                                 if (!bgcolour.isEmpty())
544                                         rowArray[i].setBGColour(convertGdkColorToHtmlColour(bgcolour));
545
546                                 // Convert the field value to a string based on the glom type. We're doing the formatting on the
547                                 // server side for now but it might be useful to move this to the client side.
548                                 switch (field.get_glom_type()) {
549                                 case TYPE_TEXT:
550                                         String text = rs.getString(i + 1);
551                                         rowArray[i].setText(text != null ? text : "");
552                                         break;
553                                 case TYPE_BOOLEAN:
554                                         rowArray[i].setBoolean(rs.getBoolean(i + 1));
555                                         break;
556                                 case TYPE_NUMERIC:
557                                         // Take care of the numeric formatting before converting the number to a string.
558                                         NumericFormat numFormatGlom = formatting.getM_numeric_format();
559                                         // There's no isCurrency() method in the glom NumericFormat class so we're assuming that the
560                                         // number should be formatted as a currency if the currency code string is not empty.
561                                         String currencyCode = numFormatGlom.getM_currency_symbol();
562                                         NumberFormat numFormatJava = null;
563                                         boolean useGlomCurrencyCode = false;
564                                         if (currencyCode.length() == 3) {
565                                                 // Try to format the currency using the Java Locales system.
566                                                 try {
567                                                         Currency currency = Currency.getInstance(currencyCode);
568                                                         Log.info(documentID, tableName, "A valid ISO 4217 currency code is being used."
569                                                                         + " Overriding the numeric formatting with information from the locale.");
570                                                         int digits = currency.getDefaultFractionDigits();
571                                                         numFormatJava = NumberFormat.getCurrencyInstance(locale);
572                                                         numFormatJava.setCurrency(currency);
573                                                         numFormatJava.setMinimumFractionDigits(digits);
574                                                         numFormatJava.setMaximumFractionDigits(digits);
575                                                 } catch (IllegalArgumentException e) {
576                                                         Log.warn(documentID, tableName, currencyCode + " is not a valid ISO 4217 code."
577                                                                         + " Manually setting currency code with this value.");
578                                                         // The currency code is not this is not an ISO 4217 currency code.
579                                                         // We're going to manually set the currency code and use the glom numeric formatting.
580                                                         useGlomCurrencyCode = true;
581                                                         numFormatJava = convertToJavaNumberFormat(numFormatGlom);
582                                                 }
583                                         } else if (currencyCode.length() > 0) {
584                                                 Log.warn(documentID, tableName, currencyCode + " is not a valid ISO 4217 code."
585                                                                 + " Manually setting currency code with this value.");
586                                                 // The length of the currency code is > 0 and != 3; this is not an ISO 4217 currency code.
587                                                 // We're going to manually set the currency code and use the glom numeric formatting.
588                                                 useGlomCurrencyCode = true;
589                                                 numFormatJava = convertToJavaNumberFormat(numFormatGlom);
590                                         } else {
591                                                 // The length of the currency code is 0; the number is not a currency.
592                                                 numFormatJava = convertToJavaNumberFormat(numFormatGlom);
593                                         }
594
595                                         // TODO: Do I need to do something with NumericFormat.get_default_precision() from libglom?
596
597                                         double number = rs.getDouble(i + 1);
598                                         if (number < 0) {
599                                                 if (formatting.getM_numeric_format().getM_alt_foreground_color_for_negatives())
600                                                         // overrides the set foreground colour
601                                                         rowArray[i].setFGColour(convertGdkColorToHtmlColour(NumericFormat
602                                                                         .get_alternative_color_for_negatives()));
603                                         }
604
605                                         // Finally convert the number to text using the glom currency string if required.
606                                         if (useGlomCurrencyCode) {
607                                                 rowArray[i].setText(currencyCode + " " + numFormatJava.format(number));
608                                         } else {
609                                                 rowArray[i].setText(numFormatJava.format(number));
610                                         }
611                                         break;
612                                 case TYPE_DATE:
613                                         Date date = rs.getDate(i + 1);
614                                         if (date != null) {
615                                                 DateFormat dateFormat = DateFormat.getDateInstance(DateFormat.MEDIUM, locale);
616                                                 rowArray[i].setText(dateFormat.format(date));
617                                         } else {
618                                                 rowArray[i].setText("");
619                                         }
620                                         break;
621                                 case TYPE_TIME:
622                                         Time time = rs.getTime(i + 1);
623                                         if (time != null) {
624                                                 DateFormat timeFormat = DateFormat.getTimeInstance(DateFormat.MEDIUM, locale);
625                                                 rowArray[i].setText(timeFormat.format(time));
626                                         } else {
627                                                 rowArray[i].setText("");
628                                         }
629                                         break;
630                                 case TYPE_IMAGE:
631                                         byte[] image = rs.getBytes(i + 1);
632                                         if (image != null) {
633                                                 // TODO implement field TYPE_IMAGE
634                                                 rowArray[i].setText("Image (FIXME)");
635                                         } else {
636                                                 rowArray[i].setText("");
637                                         }
638                                         break;
639                                 case TYPE_INVALID:
640                                 default:
641                                         Log.warn(documentID, tableName, "Invalid LayoutItem Field type. Using empty string for value.");
642                                         rowArray[i].setText("");
643                                         break;
644                                 }
645                         }
646
647                         // add the row of GlomFields to the ArrayList we're going to return and update the row count
648                         rowsList.add(rowArray);
649                         rowCount++;
650                 }
651
652                 return rowsList;
653         }
654
655         public Documents getDocuments() {
656                 Documents documents = new Documents();
657                 for (String documentID : documentMapping.keySet()) {
658                         ConfiguredDocument configuredDoc = documentMapping.get(documentID);
659                         documents.addDocuemnt(documentID, configuredDoc.getDocument().get_database_title());
660                 }
661                 return documents;
662         }
663
664         /*
665          * This method safely converts longs from libglom into ints. This method was taken from stackoverflow:
666          * 
667          * http://stackoverflow.com/questions/1590831/safely-casting-long-to-int-in-java
668          */
669         private int safeLongToInt(long value) {
670                 if (value < Integer.MIN_VALUE || value > Integer.MAX_VALUE) {
671                         throw new IllegalArgumentException(value + " cannot be cast to int without changing its value.");
672                 }
673                 return (int) value;
674         }
675
676         private NumberFormat convertToJavaNumberFormat(NumericFormat numFormatGlom) {
677                 NumberFormat numFormatJava = NumberFormat.getInstance(locale);
678                 if (numFormatGlom.getM_decimal_places_restricted()) {
679                         int digits = safeLongToInt(numFormatGlom.getM_decimal_places());
680                         numFormatJava.setMinimumFractionDigits(digits);
681                         numFormatJava.setMaximumFractionDigits(digits);
682                 }
683                 numFormatJava.setGroupingUsed(numFormatGlom.getM_use_thousands_separator());
684                 return numFormatJava;
685         }
686
687         /*
688          * Converts a Gdk::Color (16-bits per channel) to an HTML colour (8-bits per channel) by discarding the least
689          * significant 8-bits in each channel.
690          */
691         private String convertGdkColorToHtmlColour(String gdkColor) {
692                 if (gdkColor.length() == 13)
693                         return gdkColor.substring(0, 3) + gdkColor.substring(5, 7) + gdkColor.substring(9, 11);
694                 else if (gdkColor.length() == 7) {
695                         // This shouldn't happen but let's deal with it if it does.
696                         Log.warn("Expected a 13 character string but received a 7 character string. Returning received string.");
697                         return gdkColor;
698                 } else {
699                         Log.error("Did not receive a 13 or 7 character string. Returning black HTML colour code.");
700                         return "#000000";
701                 }
702         }
703
704         /*
705          * This method converts a FieldFormatting.HorizontalAlignment to the equivalent ColumnInfo.HorizontalAlignment. The
706          * need for this comes from the fact that the GWT HorizontalAlignment classes can't be used with RPC and there's no
707          * easy way to use the java-libglom FieldFormatting.HorizontalAlignment enum with RPC. An enum identical to
708          * FieldFormatting.HorizontalAlignment is included in the ColumnInfo class.
709          */
710         private Formatting.HorizontalAlignment convertToGWTGlomHorizonalAlignment(
711                         FieldFormatting.HorizontalAlignment alignment) {
712                 switch (alignment) {
713                 case HORIZONTAL_ALIGNMENT_AUTO:
714                         return Formatting.HorizontalAlignment.HORIZONTAL_ALIGNMENT_AUTO;
715                 case HORIZONTAL_ALIGNMENT_LEFT:
716                         return Formatting.HorizontalAlignment.HORIZONTAL_ALIGNMENT_LEFT;
717                 case HORIZONTAL_ALIGNMENT_RIGHT:
718                         return Formatting.HorizontalAlignment.HORIZONTAL_ALIGNMENT_RIGHT;
719                 default:
720                         Log.error("Recieved an alignment that I don't know about: "
721                                         + FieldFormatting.HorizontalAlignment.class.getName() + "." + alignment.toString() + ". Returning "
722                                         + Formatting.HorizontalAlignment.HORIZONTAL_ALIGNMENT_RIGHT.toString() + ".");
723                         return Formatting.HorizontalAlignment.HORIZONTAL_ALIGNMENT_RIGHT;
724                 }
725         }
726
727         /*
728          * This method converts a Field.glom_field_type to the equivalent ColumnInfo.FieldType. The need for this comes from
729          * the fact that the GWT FieldType classes can't be used with RPC and there's no easy way to use the java-libglom
730          * Field.glom_field_type enum with RPC. An enum identical to FieldFormatting.glom_field_type is included in the
731          * ColumnInfo class.
732          */
733         private LayoutItemField.GlomFieldType convertToGWTGlomFieldType(Field.glom_field_type type) {
734                 switch (type) {
735                 case TYPE_BOOLEAN:
736                         return LayoutItemField.GlomFieldType.TYPE_BOOLEAN;
737                 case TYPE_DATE:
738                         return LayoutItemField.GlomFieldType.TYPE_DATE;
739                 case TYPE_IMAGE:
740                         return LayoutItemField.GlomFieldType.TYPE_IMAGE;
741                 case TYPE_NUMERIC:
742                         return LayoutItemField.GlomFieldType.TYPE_NUMERIC;
743                 case TYPE_TEXT:
744                         return LayoutItemField.GlomFieldType.TYPE_TEXT;
745                 case TYPE_TIME:
746                         return LayoutItemField.GlomFieldType.TYPE_TIME;
747                 case TYPE_INVALID:
748                         Log.info("Returning TYPE_INVALID.");
749                         return LayoutItemField.GlomFieldType.TYPE_INVALID;
750                 default:
751                         Log.error("Recieved a type that I don't know about: " + Field.glom_field_type.class.getName() + "."
752                                         + type.toString() + ". Returning " + LayoutItemField.GlomFieldType.TYPE_INVALID.toString() + ".");
753                         return LayoutItemField.GlomFieldType.TYPE_INVALID;
754                 }
755         }
756
757         /*
758          * (non-Javadoc)
759          * 
760          * @see org.glom.web.client.OnlineGlomService#isAuthenticated(java.lang.String)
761          */
762         public boolean isAuthenticated(String documentID) {
763                 return documentMapping.get(documentID).isAuthenticated();
764         }
765
766         /*
767          * (non-Javadoc)
768          * 
769          * @see org.glom.web.client.OnlineGlomService#checkAuthentication(java.lang.String, java.lang.String,
770          * java.lang.String)
771          */
772         public boolean checkAuthentication(String documentID, String username, String password) {
773                 ConfiguredDocument configuredDoc = documentMapping.get(documentID);
774                 try {
775                         return configuredDoc.setUsernameAndPassword(username, password);
776                 } catch (SQLException e) {
777                         Log.error(documentID, "Unknown SQL Error checking the database authentication.", e);
778                         return false;
779                 }
780         }
781
782         /*
783          * (non-Javadoc)
784          * 
785          * @see org.glom.web.client.OnlineGlomService#getDetailsLayoutGroup(java.lang.String, java.lang.String)
786          */
787         public ArrayList<LayoutGroup> getDetailsLayout(String documentID, String tableName) {
788                 ConfiguredDocument configuredDoc = documentMapping.get(documentID);
789                 Document document = configuredDoc.getDocument();
790
791                 // Use the default table if the table name hasn't been set or if the table name isn't in the glom document.
792                 // The last check guards against SQL injection attacks since the table name could come from the URL.
793                 if (tableName == null || tableName.isEmpty() || !document.get_table_is_known(tableName)) {
794                         tableName = document.get_default_table();
795                 }
796
797                 LayoutGroupVector layoutGroupVec = document.get_data_layout_groups("details", tableName);
798
799                 ArrayList<LayoutGroup> layoutGroups = new ArrayList<LayoutGroup>();
800                 for (int i = 0; i < layoutGroupVec.size(); i++) {
801                         org.glom.libglom.LayoutGroup libglomLayoutGroup = layoutGroupVec.get(i);
802
803                         if (libglomLayoutGroup == null)
804                                 continue;
805
806                         layoutGroups.add(getLayoutGroup(documentID, tableName, libglomLayoutGroup));
807                 }
808
809                 return layoutGroups;
810         }
811
812         /*
813          * Gets a GWT-Glom LayoutGroup object for the specified libglom LayoutGroup object. This is used for getting layout
814          * information for the list and details views.
815          * 
816          * @param documentID Glom document identifier
817          * 
818          * @param tableName table name in the specified Glom document
819          * 
820          * @param libglomLayoutGroup libglom LayoutGroup to convert
821          * 
822          * @precondition libglomLayoutGroup must not be null
823          * 
824          * @return {@link LayoutGroup} object that represents the layout for the specified {@link
825          * org.glom.libglom.LayoutGroup}
826          */
827         private LayoutGroup getLayoutGroup(String documentID, String tableName,
828                         org.glom.libglom.LayoutGroup libglomLayoutGroup) {
829                 LayoutGroup layoutGroup = new LayoutGroup();
830                 layoutGroup.setColumnCount(safeLongToInt(libglomLayoutGroup.get_columns_count()));
831
832                 // look at each child item
833                 LayoutItemVector layoutItemsVec = libglomLayoutGroup.get_items();
834                 for (int i = 0; i < layoutItemsVec.size(); i++) {
835                         org.glom.libglom.LayoutItem libglomLayoutItem = layoutItemsVec.get(i);
836
837                         // just a safety check
838                         if (libglomLayoutItem == null)
839                                 continue;
840
841                         org.glom.web.shared.layout.LayoutItem layoutItem = null;
842                         org.glom.libglom.LayoutGroup group = org.glom.libglom.LayoutGroup.cast_dynamic(libglomLayoutItem);
843                         if (group != null) {
844                                 // recurse into child groups
845                                 layoutItem = getLayoutGroup(documentID, tableName, group);
846                         } else {
847                                 // create GWT-Glom LayoutItem types based on the the libglom type
848                                 // TODO add support for other LayoutItems (Text, Image, Button etc.)
849                                 LayoutItem_Field libglomLayoutField = LayoutItem_Field.cast_dynamic(libglomLayoutItem);
850                                 if (libglomLayoutField != null) {
851                                         layoutItem = convertToGWTGlomLayoutItemField(libglomLayoutField);
852                                 } else {
853                                         Log.info(documentID, tableName,
854                                                         "Ignoring unknown LayoutItem of type " + libglomLayoutItem.get_part_type_name() + ".");
855                                         continue;
856                                 }
857
858                         }
859
860                         layoutItem.setTitle(libglomLayoutItem.get_title_or_name());
861                         layoutGroup.addItem(layoutItem);
862                 }
863
864                 return layoutGroup;
865         }
866
867         private LayoutItemField convertToGWTGlomLayoutItemField(LayoutItem_Field libglomLayoutField) {
868                 LayoutItemField layoutItemField = new LayoutItemField();
869
870                 // set type
871                 layoutItemField.setType(convertToGWTGlomFieldType(libglomLayoutField.get_glom_type()));
872
873                 // set formatting
874                 Formatting formatting = new Formatting();
875                 formatting.setHorizontalAlignment(convertToGWTGlomHorizonalAlignment(libglomLayoutField
876                                 .get_formatting_used_horizontal_alignment()));
877                 layoutItemField.setFormatting(formatting);
878
879                 return layoutItemField;
880         }
881
882         public GlomField[] getDetailsData(String documentID, String tableName, String primaryKeyValue) {
883
884                 ConfiguredDocument configuredDoc = documentMapping.get(documentID);
885                 Document document = configuredDoc.getDocument();
886
887                 // Use the default table if the table name hasn't been set or if the table name isn't in the glom document.
888                 // The last check guards against SQL injection attacks since the table name could come from the URL.
889                 if (tableName == null || tableName.isEmpty() || !document.get_table_is_known(tableName)) {
890                         tableName = document.get_default_table();
891                 }
892
893                 LayoutFieldVector fieldsToGet = getFieldsToShowForSQLQuery(document, tableName, "details");
894
895                 if (fieldsToGet == null || fieldsToGet.size() <= 0) {
896                         Log.warn(documentID, tableName, "Didn't find any fields to show. Returning null.");
897                         return null;
898                 }
899
900                 // get primary key for the table to use in the SQL query
901                 Field primaryKey = null;
902                 FieldVector fieldsVec = document.get_table_fields(tableName);
903                 for (int i = 0; i < safeLongToInt(fieldsVec.size()); i++) {
904                         Field field = fieldsVec.get(i);
905                         if (field.get_primary_key()) {
906                                 primaryKey = field;
907                                 break;
908                         }
909                 }
910
911                 if (primaryKey == null) {
912                         Log.error(documentID, tableName, "Couldn't find primary key in table. Returning null.");
913                         return null;
914                 }
915
916                 ArrayList<GlomField[]> rowsList = new ArrayList<GlomField[]>();
917                 Connection conn = null;
918                 Statement st = null;
919                 ResultSet rs = null;
920                 try {
921                         // Setup the JDBC driver and get the query.
922                         ComboPooledDataSource cpds = configuredDoc.getCpds();
923                         conn = cpds.getConnection();
924                         st = conn.createStatement(ResultSet.TYPE_FORWARD_ONLY, ResultSet.CONCUR_READ_ONLY);
925                         String query = Glom.build_sql_select_with_key(tableName, fieldsToGet, primaryKey, primaryKeyValue);
926                         rs = st.executeQuery(query);
927
928                         // get the results from the ResultSet
929                         // using 2 as a length parameter so we can log a warning if appropriate
930                         rowsList = getData(documentID, tableName, 2, fieldsToGet, rs);
931                 } catch (SQLException e) {
932                         Log.error(documentID, tableName, "Error executing database query.", e);
933                         // TODO: somehow notify user of problem
934                         return null;
935                 } finally {
936                         // cleanup everything that has been used
937                         try {
938                                 if (rs != null)
939                                         rs.close();
940                                 if (st != null)
941                                         st.close();
942                                 if (conn != null)
943                                         conn.close();
944                         } catch (Exception e) {
945                                 Log.error(documentID, tableName,
946                                                 "Error closing database resources. Subsequent database queries may not work.", e);
947                         }
948                 }
949
950                 if (rowsList.size() == 0) {
951                         Log.error(documentID, tableName, "The query returned an empty ResultSet. Returning null.");
952                         return null;
953                 } else if (rowsList.size() > 1 && primaryKeyValue != null && !primaryKeyValue.isEmpty()) {
954                         // only log a warning if the result size is greater than 1 and the primaryKeyValue was set
955                         Log.warn(documentID, tableName,
956                                         "The query did not return the expected unique result. Returning the first result in the set.");
957                 }
958
959                 return rowsList.get(0);
960
961         }
962
963         /*
964          * Gets a LayoutFieldVector to use when generating an SQL query.
965          */
966         private LayoutFieldVector getFieldsToShowForSQLQuery(Document document, String tableName, String layoutName) {
967                 LayoutGroupVector layoutGroupVec = document.get_data_layout_groups(layoutName, tableName);
968                 LayoutFieldVector layoutFieldVector = new LayoutFieldVector();
969
970                 // special case for list layouts that don't have a defined layout group
971                 if ("list".equals(layoutName) && layoutGroupVec.size() == 0) {
972                         FieldVector fieldsVec = document.get_table_fields(tableName);
973                         for (int i = 0; i < fieldsVec.size(); i++) {
974                                 Field field = fieldsVec.get(i);
975                                 LayoutItem_Field layoutItemField = new LayoutItem_Field();
976                                 layoutItemField.set_full_field_details(field);
977                                 layoutFieldVector.add(layoutItemField);
978                         }
979                         return layoutFieldVector;
980                 }
981
982                 // We will show the fields that the document says we should:
983                 for (int i = 0; i < layoutGroupVec.size(); i++) {
984                         org.glom.libglom.LayoutGroup layoutGroup = layoutGroupVec.get(i);
985
986                         // Get the fields:
987                         ArrayList<LayoutItem_Field> layoutItemsFields = getFieldsToShowForSQLQueryAddGroup(document, tableName,
988                                         layoutGroup);
989                         for (LayoutItem_Field layoutItem_Field : layoutItemsFields) {
990                                 layoutFieldVector.add(layoutItem_Field);
991                         }
992                 }
993                 return layoutFieldVector;
994         }
995
996         private ArrayList<LayoutItem_Field> getFieldsToShowForSQLQueryAddGroup(Document document, String tableName,
997                         org.glom.libglom.LayoutGroup layoutGroup) {
998
999                 ArrayList<LayoutItem_Field> layoutItemFields = new ArrayList<LayoutItem_Field>();
1000                 LayoutItemVector items = layoutGroup.get_items();
1001                 for (int i = 0; i < items.size(); i++) {
1002                         LayoutItem layoutItem = items.get(i);
1003
1004                         LayoutItem_Field layoutItemField = LayoutItem_Field.cast_dynamic(layoutItem);
1005                         if (layoutItemField != null) {
1006                                 // the layoutItem is a LayoutItem_Field
1007                                 FieldVector fields;
1008                                 if (layoutItemField.get_has_relationship_name()) {
1009                                         // layoutItemField is a field in a related table
1010                                         fields = document.get_table_fields(layoutItemField.get_table_used(tableName));
1011                                 } else {
1012                                         // layoutItemField is a field in this table
1013                                         fields = document.get_table_fields(tableName);
1014                                 }
1015
1016                                 // set the layoutItemFeild with details from its Field in the document and
1017                                 // add it to the list to be returned
1018                                 for (int j = 0; j < fields.size(); j++) {
1019                                         // check the names to see if they're the same
1020                                         // this works because we're using the field list from the related table if necessary
1021                                         if (layoutItemField.get_name().equals(fields.get(j).get_name())) {
1022                                                 Field field = fields.get(j);
1023                                                 if (field != null) {
1024                                                         layoutItemField.set_full_field_details(field);
1025                                                         layoutItemFields.add(layoutItemField);
1026                                                 } else {
1027                                                         Log.warn(document.get_database_title(), tableName,
1028                                                                         "LayoutItem_Field " + layoutItemField.get_layout_display_name()
1029                                                                                         + " not found in document field list.");
1030                                                 }
1031                                                 break;
1032                                         }
1033                                 }
1034
1035                         } else {
1036                                 // the layoutItem is not a LayoutItem_Field
1037                                 org.glom.libglom.LayoutGroup subLayoutGroup = org.glom.libglom.LayoutGroup.cast_dynamic(layoutItem);
1038                                 if (subLayoutGroup != null) {
1039                                         // the layoutItem is a LayoutGroup
1040                                         LayoutItem_Portal layoutItemPortal = LayoutItem_Portal.cast_dynamic(layoutItem);
1041                                         if (layoutItemPortal == null) {
1042                                                 // The subGroup is not a LayoutItem_Portal.
1043                                                 // We're ignoring portals because they are filled by means of a separate SQL query.
1044                                                 layoutItemFields
1045                                                                 .addAll(getFieldsToShowForSQLQueryAddGroup(document, tableName, subLayoutGroup));
1046                                         }
1047                                 }
1048                         }
1049                 }
1050                 return layoutItemFields;
1051         }
1052
1053 }