Add primary key index to LayoutGroup DTO and add ProvidesKey to CellTable.
[online-glom:gwt-glom.git] / src / main / java / org / glom / web / server / OnlineGlomServiceImpl.java
1 /*
2  * Copyright (C) 2010, 2011 Openismus GmbH
3  *
4  * This file is part of GWT-Glom.
5  *
6  * GWT-Glom is free software: you can redistribute it and/or modify it
7  * under the terms of the GNU Lesser General Public License as published by the
8  * Free Software Foundation, either version 3 of the License, or (at your
9  * option) any later version.
10  *
11  * GWT-Glom is distributed in the hope that it will be useful, but WITHOUT
12  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
13  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU Lesser General Public License
14  * for more details.
15  *
16  * You should have received a copy of the GNU Lesser General Public License
17  * along with GWT-Glom.  If not, see <http://www.gnu.org/licenses/>.
18  */
19
20 package org.glom.web.server;
21
22 import java.io.File;
23 import java.io.FilenameFilter;
24 import java.io.IOException;
25 import java.io.InputStream;
26 import java.sql.Connection;
27 import java.sql.Date;
28 import java.sql.ResultSet;
29 import java.sql.SQLException;
30 import java.sql.Statement;
31 import java.sql.Time;
32 import java.text.DateFormat;
33 import java.text.NumberFormat;
34 import java.util.ArrayList;
35 import java.util.Currency;
36 import java.util.Hashtable;
37 import java.util.Locale;
38 import java.util.Properties;
39
40 import org.glom.libglom.BakeryDocument.LoadFailureCodes;
41 import org.glom.libglom.Document;
42 import org.glom.libglom.Field;
43 import org.glom.libglom.FieldFormatting;
44 import org.glom.libglom.FieldVector;
45 import org.glom.libglom.Glom;
46 import org.glom.libglom.LayoutFieldVector;
47 import org.glom.libglom.LayoutGroupVector;
48 import org.glom.libglom.LayoutItem;
49 import org.glom.libglom.LayoutItemVector;
50 import org.glom.libglom.LayoutItem_Field;
51 import org.glom.libglom.LayoutItem_Portal;
52 import org.glom.libglom.NumericFormat;
53 import org.glom.libglom.SortClause;
54 import org.glom.libglom.SortFieldPair;
55 import org.glom.libglom.StringVector;
56 import org.glom.web.client.OnlineGlomService;
57 import org.glom.web.shared.GlomDocument;
58 import org.glom.web.shared.GlomField;
59 import org.glom.web.shared.layout.Formatting;
60 import org.glom.web.shared.layout.LayoutGroup;
61 import org.glom.web.shared.layout.LayoutItemField;
62
63 import com.google.gwt.user.server.rpc.RemoteServiceServlet;
64 import com.mchange.v2.c3p0.ComboPooledDataSource;
65 import com.mchange.v2.c3p0.DataSources;
66
67 /**
68  * The servlet for retrieving layout information from libglom and data from the underlying PostgreSQL database.
69  * 
70  * TODO: move methods that that require a glom document object to the ConfiguredDocument class.
71  * 
72  * @author Ben Konrath <ben@bagu.org>
73  */
74 @SuppressWarnings("serial")
75 public class OnlineGlomServiceImpl extends RemoteServiceServlet implements OnlineGlomService {
76
77         // convenience class to for dealing with the Online Glom configuration file
78         private class OnlineGlomProperties extends Properties {
79                 public String getKey(String value) {
80                         for (String key : stringPropertyNames()) {
81                                 if (getProperty(key).trim().equals(value))
82                                         return key;
83                         }
84                         return null;
85                 }
86         }
87
88         private final Hashtable<String, ConfiguredDocument> documents = new Hashtable<String, ConfiguredDocument>();
89         // TODO implement locale
90         private final Locale locale = Locale.ROOT;
91
92         /*
93          * This is called when the servlet is started or restarted.
94          */
95         public OnlineGlomServiceImpl() throws Exception {
96
97                 // Find the configuration file. See this thread for background info:
98                 // http://stackoverflow.com/questions/2161054/where-to-place-properties-files-in-a-jsp-servlet-web-application
99                 OnlineGlomProperties config = new OnlineGlomProperties();
100                 InputStream is = Thread.currentThread().getContextClassLoader().getResourceAsStream("onlineglom.properties");
101                 if (is == null) {
102                         Log.fatal("onlineglom.properties not found.");
103                         throw new IOException();
104                 }
105                 config.load(is);
106
107                 // check the configured glom file directory
108                 String documentDirName = config.getProperty("glom.document.directory");
109                 File documentDir = new File(documentDirName);
110                 if (!documentDir.isDirectory()) {
111                         Log.fatal(documentDirName + " is not a directory.");
112                         throw new IOException();
113                 }
114                 if (!documentDir.canRead()) {
115                         Log.fatal("Can't read the files in : " + documentDirName);
116                         throw new IOException();
117                 }
118
119                 // get and check the glom files in the specified directory
120                 File[] glomFiles = documentDir.listFiles(new FilenameFilter() {
121                         @Override
122                         public boolean accept(File dir, String name) {
123                                 return name.endsWith(".glom");
124                         }
125                 });
126                 Glom.libglom_init();
127                 for (File glomFile : glomFiles) {
128                         Document document = new Document();
129                         document.set_file_uri("file://" + glomFile.getAbsolutePath());
130                         int error = 0;
131                         boolean retval = document.load(error);
132                         if (retval == false) {
133                                 String message;
134                                 if (LoadFailureCodes.LOAD_FAILURE_CODE_NOT_FOUND == LoadFailureCodes.swigToEnum(error)) {
135                                         message = "Could not find file: " + glomFile.getAbsolutePath();
136                                 } else {
137                                         message = "An unknown error occurred when trying to load file: " + glomFile.getAbsolutePath();
138                                 }
139                                 Log.error(message);
140                                 // continue with for loop because there may be other documents in the directory
141                                 continue;
142                         }
143
144                         ConfiguredDocument configuredDocument = new ConfiguredDocument(document);
145                         // check if a username and password have been set and work for the current document
146                         String documentTitle = document.get_database_title().trim();
147                         String key = config.getKey(documentTitle);
148                         if (key != null) {
149                                 String[] keyArray = key.split("\\.");
150                                 if (keyArray.length == 3 && "title".equals(keyArray[2])) {
151                                         // username/password could be set, let's check to see if it works
152                                         String usernameKey = key.replaceAll(keyArray[2], "username");
153                                         String passwordKey = key.replaceAll(keyArray[2], "password");
154                                         configuredDocument.setUsernameAndPassword(config.getProperty(usernameKey),
155                                                         config.getProperty(passwordKey));
156                                 }
157                         }
158
159                         // check the if the global username and password have been set and work with this document
160                         if (!configuredDocument.isAuthenticated()) {
161                                 configuredDocument.setUsernameAndPassword(config.getProperty("glom.document.username"),
162                                                 config.getProperty("glom.document.password"));
163                         }
164
165                         // add information to the hash table
166                         documents.put(documentTitle, configuredDocument);
167
168                 }
169         }
170
171         /*
172          * This is called when the servlet is stopped or restarted.
173          * 
174          * @see javax.servlet.GenericServlet#destroy()
175          */
176         @Override
177         public void destroy() {
178                 Glom.libglom_deinit();
179
180                 for (String documenTitle : documents.keySet()) {
181                         ConfiguredDocument configuredDoc = documents.get(documenTitle);
182                         try {
183                                 DataSources.destroy(configuredDoc.getCpds());
184                         } catch (SQLException e) {
185                                 Log.error(documenTitle, "Error cleaning up the ComboPooledDataSource.", e);
186                         }
187                 }
188
189         }
190
191         public GlomDocument getGlomDocument(String documentTitle) {
192
193                 Document document = documents.get(documentTitle).getDocument();
194                 GlomDocument glomDocument = new GlomDocument();
195
196                 // get arrays of table names and titles, and find the default table index
197                 StringVector tablesVec = document.get_table_names();
198
199                 int numTables = safeLongToInt(tablesVec.size());
200                 // we don't know how many tables will be hidden so we'll use half of the number of tables for the default size
201                 // of the ArrayList
202                 ArrayList<String> tableNames = new ArrayList<String>(numTables / 2);
203                 ArrayList<String> tableTitles = new ArrayList<String>(numTables / 2);
204                 boolean foundDefaultTable = false;
205                 int visibleIndex = 0;
206                 for (int i = 0; i < numTables; i++) {
207                         String tableName = tablesVec.get(i);
208                         if (!document.get_table_is_hidden(tableName)) {
209                                 tableNames.add(tableName);
210                                 // JNI is "expensive", the comparison will only be called if we haven't already found the default table
211                                 if (!foundDefaultTable && tableName.equals(document.get_default_table())) {
212                                         glomDocument.setDefaultTableIndex(visibleIndex);
213                                         foundDefaultTable = true;
214                                 }
215                                 tableTitles.add(document.get_table_title(tableName));
216                                 visibleIndex++;
217                         }
218                 }
219
220                 // set everything we need
221                 glomDocument.setTableNames(tableNames);
222                 glomDocument.setTableTitles(tableTitles);
223
224                 return glomDocument;
225         }
226
227         /*
228          * (non-Javadoc)
229          * 
230          * @see org.glom.web.client.OnlineGlomService#getDefaultLayoutListTable(java.lang.String)
231          */
232         @Override
233         public LayoutGroup getDefaultListLayout(String documentTitle) {
234                 GlomDocument glomDocument = getGlomDocument(documentTitle);
235                 String tableName = glomDocument.getTableNames().get(glomDocument.getDefaultTableIndex());
236                 LayoutGroup layoutGroup = getListLayout(documentTitle, tableName);
237                 layoutGroup.setDefaultTableName(tableName);
238                 return layoutGroup;
239         }
240
241         public LayoutGroup getListLayout(String documentTitle, String tableName) {
242                 ConfiguredDocument configuredDoc = documents.get(documentTitle);
243                 Document document = configuredDoc.getDocument();
244
245                 // access the layout list
246                 LayoutGroupVector layoutGroupVec = document.get_data_layout_groups("list", tableName);
247                 int listViewLayoutGroupSize = safeLongToInt(layoutGroupVec.size());
248                 org.glom.libglom.LayoutGroup libglomLayoutGroup = null;
249                 if (listViewLayoutGroupSize > 0) {
250                         // a list layout group is defined; we can use the first group as the list
251                         if (listViewLayoutGroupSize > 1)
252                                 Log.warn(documentTitle, tableName,
253                                                 "The size of the list layout group is greater than 1. Attempting to use the first item for the layout list view.");
254
255                         libglomLayoutGroup = layoutGroupVec.get(0);
256                 } else {
257                         // a list layout group is *not* defined; we are going make a libglom layout group from the list of fields
258                         Log.info(documentTitle, tableName,
259                                         "A list layout is not defined for this table. Displaying a list layout based on the field list.");
260
261                         FieldVector fieldsVec = document.get_table_fields(tableName);
262                         libglomLayoutGroup = new org.glom.libglom.LayoutGroup();
263                         for (int i = 0; i < fieldsVec.size(); i++) {
264                                 Field field = fieldsVec.get(i);
265                                 LayoutItem_Field layoutItemField = new LayoutItem_Field();
266                                 layoutItemField.set_full_field_details(field);
267                                 libglomLayoutGroup.add_item(layoutItemField);
268                         }
269                 }
270
271                 // confirm the libglom LayoutGroup is not null as per the method's precondition
272                 if (libglomLayoutGroup == null) {
273                         Log.error(documentTitle, tableName, "A LayoutGroup was not found. Returning null.");
274                         return null;
275                 }
276
277                 LayoutGroup layoutGroup = getLayoutGroup(documentTitle, tableName, libglomLayoutGroup);
278
279                 // use the same fields list as will be used for the query
280                 LayoutFieldVector fieldsToGet = getFieldsToShowForSQLQuery(document, tableName, "list");
281                 layoutGroup.setExpectedResultSize(getResultSizeOfSQLQuery(documentTitle, tableName, fieldsToGet));
282
283                 // Set the primary key index for the table and add a LayoutItemField for the primary key to the end of the item
284                 // list in the LayoutGroup if it doesn't already contain a primary key.
285                 int primaryKeyIndex = getPrimaryKeyIndex(fieldsToGet);
286                 if (primaryKeyIndex < 0) {
287                         LayoutItem_Field libglomLayoutItemField = getPrimaryKeyLayoutItemFromFields(document, tableName);
288                         layoutGroup.addItem(convertToGWTGlomLayoutItemField(libglomLayoutItemField));
289                         layoutGroup.setPrimaryKeyIndex(layoutGroup.getItems().size() - 1);
290                         layoutGroup.setHiddenPrimaryKey(true);
291
292                 } else {
293                         layoutGroup.setPrimaryKeyIndex(primaryKeyIndex);
294                 }
295                 return layoutGroup;
296         }
297
298         private LayoutItem_Field getPrimaryKeyLayoutItemFromFields(Document document, String tableName) {
299                 Field primaryKey = null;
300                 FieldVector fieldVec = document.get_table_fields(tableName);
301                 for (int i = 0; i < fieldVec.size(); i++) {
302                         Field field = fieldVec.get(i);
303                         if (field != null && field.get_primary_key()) {
304                                 primaryKey = field;
305                                 break;
306                         }
307                 }
308                 if (primaryKey == null) {
309                         Log.fatal(document.get_database_title(), tableName,
310                                         "A primary key was not found in the FieldVector for this table.");
311                         // TODO throw exception
312                 }
313
314                 LayoutItem_Field libglomLayoutItemField = new LayoutItem_Field();
315                 libglomLayoutItemField.set_full_field_details(primaryKey);
316                 return libglomLayoutItemField;
317         }
318
319         /*
320          * Gets the primary key index of the LayoutFieldVector.
321          */
322         private int getPrimaryKeyIndex(LayoutFieldVector layoutFieldVec) {
323                 for (int i = 0; i < layoutFieldVec.size(); i++) {
324                         LayoutItem_Field layoutItemField = layoutFieldVec.get(i);
325                         Field field = layoutItemField.get_full_field_details();
326                         if (field != null && field.get_primary_key())
327                                 return i;
328                 }
329                 return -1;
330         }
331
332         /*
333          * Get the number of rows a query with the table name and layout fields would return. This is needed for the /* list
334          * view pager.
335          */
336         private int getResultSizeOfSQLQuery(String documentTitle, String tableName, LayoutFieldVector fieldsToGet) {
337                 ConfiguredDocument configuredDoc = documents.get(documentTitle);
338                 if (!configuredDoc.isAuthenticated())
339                         return -1;
340                 Connection conn = null;
341                 Statement st = null;
342                 ResultSet rs = null;
343                 try {
344                         // Setup and execute the count query. Special care needs to be take to ensure that the results will be based
345                         // on a cursor so that large amounts of memory are not consumed when the query retrieve a large amount of
346                         // data. Here's the relevant PostgreSQL documentation:
347                         // http://jdbc.postgresql.org/documentation/83/query.html#query-with-cursor
348                         ComboPooledDataSource cpds = configuredDoc.getCpds();
349                         conn = cpds.getConnection();
350                         conn.setAutoCommit(false);
351                         st = conn.createStatement(ResultSet.TYPE_FORWARD_ONLY, ResultSet.CONCUR_READ_ONLY);
352                         String query = Glom.build_sql_select_count_simple(tableName, fieldsToGet);
353                         // TODO Test execution time of this query with when the number of rows in the table is large (say >
354                         // 1,000,000). Test memory usage at the same time (see the todo item in getTableData()).
355                         rs = st.executeQuery(query);
356
357                         // get the number of rows in the query
358                         rs.next();
359                         return rs.getInt(1);
360
361                 } catch (SQLException e) {
362                         Log.error(documentTitle, tableName, "Error calculating number of rows in the query.", e);
363                         return -1;
364                 } finally {
365                         // cleanup everything that has been used
366                         try {
367                                 if (rs != null)
368                                         rs.close();
369                                 if (st != null)
370                                         st.close();
371                                 if (conn != null)
372                                         conn.close();
373                         } catch (Exception e) {
374                                 Log.error(documentTitle, tableName,
375                                                 "Error closing database resources. Subsequent database queries may not work.", e);
376                         }
377                 }
378         }
379
380         // FIXME Check if we can use getFieldsToShowForSQLQuery() in these methods
381         public ArrayList<GlomField[]> getListData(String documentTitle, String tableName, int start, int length) {
382                 return getListData(documentTitle, tableName, start, length, false, 0, false);
383         }
384
385         public ArrayList<GlomField[]> getSortedListData(String documentTitle, String tableName, int start, int length,
386                         int sortColumnIndex, boolean isAscending) {
387                 return getListData(documentTitle, tableName, start, length, true, sortColumnIndex, isAscending);
388         }
389
390         private ArrayList<GlomField[]> getListData(String documentTitle, String tableName, int start, int length,
391                         boolean useSortClause, int sortColumnIndex, boolean isAscending) {
392
393                 ConfiguredDocument configuredDoc = documents.get(documentTitle);
394                 if (!configuredDoc.isAuthenticated())
395                         return new ArrayList<GlomField[]>();
396                 Document document = configuredDoc.getDocument();
397
398                 // access the layout list using the defined layout list or the table fields if there's no layout list
399                 LayoutGroupVector layoutListVec = document.get_data_layout_groups("list", tableName);
400                 LayoutFieldVector layoutFields = new LayoutFieldVector();
401                 SortClause sortClause = new SortClause();
402                 int listViewLayoutGroupSize = safeLongToInt(layoutListVec.size());
403                 if (layoutListVec.size() > 0) {
404                         // a layout list is defined, we can use it to for the LayoutListTable
405                         if (listViewLayoutGroupSize > 1)
406                                 Log.warn(documentTitle, tableName,
407                                                 "The size of the list view layout group for table is greater than 1. "
408                                                                 + "Attempting to use the first item for the layout list view.");
409                         LayoutItemVector layoutItemsVec = layoutListVec.get(0).get_items();
410
411                         // find the defined layout list fields
412                         int numItems = safeLongToInt(layoutItemsVec.size());
413                         for (int i = 0; i < numItems; i++) {
414                                 // TODO add support for other LayoutItems (Text, Image, Button)
415                                 LayoutItem item = layoutItemsVec.get(i);
416                                 LayoutItem_Field layoutItemfield = LayoutItem_Field.cast_dynamic(item);
417                                 if (layoutItemfield != null) {
418                                         // use this field in the layout
419                                         layoutFields.add(layoutItemfield);
420
421                                         // create a sort clause if it's a primary key and we're not asked to sort a specific column
422                                         if (!useSortClause) {
423                                                 Field details = layoutItemfield.get_full_field_details();
424                                                 if (details != null && details.get_primary_key()) {
425                                                         sortClause.addLast(new SortFieldPair(layoutItemfield, true)); // ascending
426                                                 }
427                                         }
428                                 }
429                         }
430                 } else {
431                         // no layout list is defined, use the table fields as the layout list
432                         FieldVector fieldsVec = document.get_table_fields(tableName);
433
434                         // find the fields to display in the layout list
435                         int numItems = safeLongToInt(fieldsVec.size());
436                         for (int i = 0; i < numItems; i++) {
437                                 Field field = fieldsVec.get(i);
438                                 LayoutItem_Field layoutItemField = new LayoutItem_Field();
439                                 layoutItemField.set_full_field_details(field);
440                                 layoutFields.add(layoutItemField);
441
442                                 // create a sort clause if it's a primary key and we're not asked to sort a specific column
443                                 if (!useSortClause) {
444                                         if (field.get_primary_key()) {
445                                                 sortClause.addLast(new SortFieldPair(layoutItemField, true)); // ascending
446                                         }
447                                 }
448                         }
449                 }
450
451                 // create a sort clause for the column we've been asked to sort
452                 if (useSortClause) {
453                         LayoutItem item = layoutFields.get(sortColumnIndex);
454                         LayoutItem_Field field = LayoutItem_Field.cast_dynamic(item);
455                         if (field != null)
456                                 sortClause.addLast(new SortFieldPair(field, isAscending));
457                         else {
458                                 Log.error(documentTitle, tableName, "Error getting LayoutItem_Field for column index "
459                                                 + sortColumnIndex + ". Cannot create a sort clause for this column.");
460                         }
461                 }
462
463                 // Add a LayoutItem_Field for the primary key to the end of the LayoutFieldVector if it doesn't already contain
464                 // a primary key.
465                 // TODO Can we use a cached LayoutGroup object to find out if we need to add a LayoutItem_Field object for the
466                 // primary key field?
467                 if (getPrimaryKeyIndex(layoutFields) < 0) {
468                         layoutFields.add(getPrimaryKeyLayoutItemFromFields(document, tableName));
469                 }
470
471                 ArrayList<GlomField[]> rowsList = new ArrayList<GlomField[]>();
472                 Connection conn = null;
473                 Statement st = null;
474                 ResultSet rs = null;
475                 try {
476                         // Setup the JDBC driver and get the query. Special care needs to be take to ensure that the results will be
477                         // based on a cursor so that large amounts of memory are not consumed when the query retrieve a large amount
478                         // of data. Here's the relevant PostgreSQL documentation:
479                         // http://jdbc.postgresql.org/documentation/83/query.html#query-with-cursor
480                         ComboPooledDataSource cpds = configuredDoc.getCpds();
481                         conn = cpds.getConnection();
482                         conn.setAutoCommit(false);
483                         st = conn.createStatement(ResultSet.TYPE_FORWARD_ONLY, ResultSet.CONCUR_READ_ONLY);
484                         st.setFetchSize(length);
485                         String query = Glom.build_sql_select_simple(tableName, layoutFields, sortClause) + " OFFSET " + start;
486                         // TODO Test memory usage before and after we execute the query that would result in a large ResultSet.
487                         // We need to ensure that the JDBC driver is in fact returning a cursor based result set that has a low
488                         // memory footprint. Check the difference between this value before and after the query:
489                         // Runtime.getRuntime().totalMemory() - Runtime.getRuntime().freeMemory()
490                         // Test the execution time at the same time (see the todo item in getLayoutListTable()).
491                         rs = st.executeQuery(query);
492
493                         // get the results from the ResultSet
494                         rowsList = getData(documentTitle, tableName, length, layoutFields, rs);
495                 } catch (SQLException e) {
496                         Log.error(documentTitle, tableName, "Error executing database query.", e);
497                         // TODO: somehow notify user of problem
498                 } finally {
499                         // cleanup everything that has been used
500                         try {
501                                 if (rs != null)
502                                         rs.close();
503                                 if (st != null)
504                                         st.close();
505                                 if (conn != null)
506                                         conn.close();
507                         } catch (Exception e) {
508                                 Log.error(documentTitle, tableName,
509                                                 "Error closing database resources. Subsequent database queries may not work.", e);
510                         }
511                 }
512                 return rowsList;
513         }
514
515         private ArrayList<GlomField[]> getData(String documentTitle, String tableName, int length,
516                         LayoutFieldVector layoutFields, ResultSet rs) throws SQLException {
517
518                 // get the data we've been asked for
519                 int rowCount = 0;
520                 ArrayList<GlomField[]> rowsList = new ArrayList<GlomField[]>();
521                 while (rs.next() && rowCount <= length) {
522                         int layoutFieldsSize = safeLongToInt(layoutFields.size());
523                         GlomField[] rowArray = new GlomField[layoutFieldsSize];
524                         for (int i = 0; i < layoutFieldsSize; i++) {
525                                 // make a new GlomField to set the text and colours
526                                 rowArray[i] = new GlomField();
527
528                                 // get foreground and background colours
529                                 LayoutItem_Field field = layoutFields.get(i);
530                                 FieldFormatting formatting = field.get_formatting_used();
531                                 String fgcolour = formatting.get_text_format_color_foreground();
532                                 if (!fgcolour.isEmpty())
533                                         rowArray[i].setFGColour(convertGdkColorToHtmlColour(fgcolour));
534                                 String bgcolour = formatting.get_text_format_color_background();
535                                 if (!bgcolour.isEmpty())
536                                         rowArray[i].setBGColour(convertGdkColorToHtmlColour(bgcolour));
537
538                                 // Convert the field value to a string based on the glom type. We're doing the formatting on the
539                                 // server side for now but it might be useful to move this to the client side.
540                                 switch (field.get_glom_type()) {
541                                 case TYPE_TEXT:
542                                         String text = rs.getString(i + 1);
543                                         rowArray[i].setText(text != null ? text : "");
544                                         break;
545                                 case TYPE_BOOLEAN:
546                                         rowArray[i].setBoolean(rs.getBoolean(i + 1));
547                                         break;
548                                 case TYPE_NUMERIC:
549                                         // Take care of the numeric formatting before converting the number to a string.
550                                         NumericFormat numFormatGlom = formatting.getM_numeric_format();
551                                         // There's no isCurrency() method in the glom NumericFormat class so we're assuming that the
552                                         // number should be formatted as a currency if the currency code string is not empty.
553                                         String currencyCode = numFormatGlom.getM_currency_symbol();
554                                         NumberFormat numFormatJava = null;
555                                         boolean useGlomCurrencyCode = false;
556                                         if (currencyCode.length() == 3) {
557                                                 // Try to format the currency using the Java Locales system.
558                                                 try {
559                                                         Currency currency = Currency.getInstance(currencyCode);
560                                                         Log.info(documentTitle, tableName, "A valid ISO 4217 currency code is being used."
561                                                                         + " Overriding the numeric formatting with information from the locale.");
562                                                         int digits = currency.getDefaultFractionDigits();
563                                                         numFormatJava = NumberFormat.getCurrencyInstance(locale);
564                                                         numFormatJava.setCurrency(currency);
565                                                         numFormatJava.setMinimumFractionDigits(digits);
566                                                         numFormatJava.setMaximumFractionDigits(digits);
567                                                 } catch (IllegalArgumentException e) {
568                                                         Log.warn(documentTitle, tableName, currencyCode + " is not a valid ISO 4217 code."
569                                                                         + " Manually setting currency code with this value.");
570                                                         // The currency code is not this is not an ISO 4217 currency code.
571                                                         // We're going to manually set the currency code and use the glom numeric formatting.
572                                                         useGlomCurrencyCode = true;
573                                                         numFormatJava = convertToJavaNumberFormat(numFormatGlom);
574                                                 }
575                                         } else if (currencyCode.length() > 0) {
576                                                 Log.warn(documentTitle, tableName, currencyCode + " is not a valid ISO 4217 code."
577                                                                 + " Manually setting currency code with this value.");
578                                                 // The length of the currency code is > 0 and != 3; this is not an ISO 4217 currency code.
579                                                 // We're going to manually set the currency code and use the glom numeric formatting.
580                                                 useGlomCurrencyCode = true;
581                                                 numFormatJava = convertToJavaNumberFormat(numFormatGlom);
582                                         } else {
583                                                 // The length of the currency code is 0; the number is not a currency.
584                                                 numFormatJava = convertToJavaNumberFormat(numFormatGlom);
585                                         }
586
587                                         // TODO: Do I need to do something with NumericFormat.get_default_precision() from libglom?
588
589                                         double number = rs.getDouble(i + 1);
590                                         if (number < 0) {
591                                                 if (formatting.getM_numeric_format().getM_alt_foreground_color_for_negatives())
592                                                         // overrides the set foreground colour
593                                                         rowArray[i].setFGColour(convertGdkColorToHtmlColour(NumericFormat
594                                                                         .get_alternative_color_for_negatives()));
595                                         }
596
597                                         // Finally convert the number to text using the glom currency string if required.
598                                         if (useGlomCurrencyCode) {
599                                                 rowArray[i].setText(currencyCode + " " + numFormatJava.format(number));
600                                         } else {
601                                                 rowArray[i].setText(numFormatJava.format(number));
602                                         }
603                                         break;
604                                 case TYPE_DATE:
605                                         Date date = rs.getDate(i + 1);
606                                         if (date != null) {
607                                                 DateFormat dateFormat = DateFormat.getDateInstance(DateFormat.MEDIUM, locale);
608                                                 rowArray[i].setText(dateFormat.format(date));
609                                         } else {
610                                                 rowArray[i].setText("");
611                                         }
612                                         break;
613                                 case TYPE_TIME:
614                                         Time time = rs.getTime(i + 1);
615                                         if (time != null) {
616                                                 DateFormat timeFormat = DateFormat.getTimeInstance(DateFormat.MEDIUM, locale);
617                                                 rowArray[i].setText(timeFormat.format(time));
618                                         } else {
619                                                 rowArray[i].setText("");
620                                         }
621                                         break;
622                                 case TYPE_IMAGE:
623                                         byte[] image = rs.getBytes(i + 1);
624                                         if (image != null) {
625                                                 // TODO implement field TYPE_IMAGE
626                                                 rowArray[i].setText("Image (FIXME)");
627                                         } else {
628                                                 rowArray[i].setText("");
629                                         }
630                                         break;
631                                 case TYPE_INVALID:
632                                 default:
633                                         Log.warn(documentTitle, tableName, "Invalid LayoutItem Field type. Using empty string for value.");
634                                         rowArray[i].setText("");
635                                         break;
636                                 }
637                         }
638
639                         // add the row of GlomFields to the ArrayList we're going to return and update the row count
640                         rowsList.add(rowArray);
641                         rowCount++;
642                 }
643
644                 return rowsList;
645         }
646
647         public ArrayList<String> getDocumentTitles() {
648                 ArrayList<String> documentTitles = new ArrayList<String>();
649                 for (String title : documents.keySet()) {
650                         documentTitles.add(title);
651                 }
652                 return documentTitles;
653         }
654
655         /*
656          * This method safely converts longs from libglom into ints. This method was taken from stackoverflow:
657          * 
658          * http://stackoverflow.com/questions/1590831/safely-casting-long-to-int-in-java
659          */
660         private int safeLongToInt(long value) {
661                 if (value < Integer.MIN_VALUE || value > Integer.MAX_VALUE) {
662                         throw new IllegalArgumentException(value + " cannot be cast to int without changing its value.");
663                 }
664                 return (int) value;
665         }
666
667         private NumberFormat convertToJavaNumberFormat(NumericFormat numFormatGlom) {
668                 NumberFormat numFormatJava = NumberFormat.getInstance(locale);
669                 if (numFormatGlom.getM_decimal_places_restricted()) {
670                         int digits = safeLongToInt(numFormatGlom.getM_decimal_places());
671                         numFormatJava.setMinimumFractionDigits(digits);
672                         numFormatJava.setMaximumFractionDigits(digits);
673                 }
674                 numFormatJava.setGroupingUsed(numFormatGlom.getM_use_thousands_separator());
675                 return numFormatJava;
676         }
677
678         /*
679          * Converts a Gdk::Color (16-bits per channel) to an HTML colour (8-bits per channel) by discarding the least
680          * significant 8-bits in each channel.
681          */
682         private String convertGdkColorToHtmlColour(String gdkColor) {
683                 if (gdkColor.length() == 13)
684                         return gdkColor.substring(0, 3) + gdkColor.substring(5, 7) + gdkColor.substring(9, 11);
685                 else if (gdkColor.length() == 7) {
686                         // This shouldn't happen but let's deal with it if it does.
687                         Log.warn("Expected a 13 character string but received a 7 character string. Returning received string.");
688                         return gdkColor;
689                 } else {
690                         Log.error("Did not receive a 13 or 7 character string. Returning black HTML colour code.");
691                         return "#000000";
692                 }
693         }
694
695         /*
696          * This method converts a FieldFormatting.HorizontalAlignment to the equivalent ColumnInfo.HorizontalAlignment. The
697          * need for this comes from the fact that the GWT HorizontalAlignment classes can't be used with RPC and there's no
698          * easy way to use the java-libglom FieldFormatting.HorizontalAlignment enum with RPC. An enum identical to
699          * FieldFormatting.HorizontalAlignment is included in the ColumnInfo class.
700          */
701         private Formatting.HorizontalAlignment convertToGWTGlomHorizonalAlignment(
702                         FieldFormatting.HorizontalAlignment alignment) {
703                 switch (alignment) {
704                 case HORIZONTAL_ALIGNMENT_AUTO:
705                         return Formatting.HorizontalAlignment.HORIZONTAL_ALIGNMENT_AUTO;
706                 case HORIZONTAL_ALIGNMENT_LEFT:
707                         return Formatting.HorizontalAlignment.HORIZONTAL_ALIGNMENT_LEFT;
708                 case HORIZONTAL_ALIGNMENT_RIGHT:
709                         return Formatting.HorizontalAlignment.HORIZONTAL_ALIGNMENT_RIGHT;
710                 default:
711                         Log.error("Recieved an alignment that I don't know about: "
712                                         + FieldFormatting.HorizontalAlignment.class.getName() + "." + alignment.toString() + ". Returning "
713                                         + Formatting.HorizontalAlignment.HORIZONTAL_ALIGNMENT_RIGHT.toString() + ".");
714                         return Formatting.HorizontalAlignment.HORIZONTAL_ALIGNMENT_RIGHT;
715                 }
716         }
717
718         /*
719          * This method converts a Field.glom_field_type to the equivalent ColumnInfo.FieldType. The need for this comes from
720          * the fact that the GWT FieldType classes can't be used with RPC and there's no easy way to use the java-libglom
721          * Field.glom_field_type enum with RPC. An enum identical to FieldFormatting.glom_field_type is included in the
722          * ColumnInfo class.
723          */
724         private LayoutItemField.GlomFieldType convertToGWTGlomFieldType(Field.glom_field_type type) {
725                 switch (type) {
726                 case TYPE_BOOLEAN:
727                         return LayoutItemField.GlomFieldType.TYPE_BOOLEAN;
728                 case TYPE_DATE:
729                         return LayoutItemField.GlomFieldType.TYPE_DATE;
730                 case TYPE_IMAGE:
731                         return LayoutItemField.GlomFieldType.TYPE_IMAGE;
732                 case TYPE_NUMERIC:
733                         return LayoutItemField.GlomFieldType.TYPE_NUMERIC;
734                 case TYPE_TEXT:
735                         return LayoutItemField.GlomFieldType.TYPE_TEXT;
736                 case TYPE_TIME:
737                         return LayoutItemField.GlomFieldType.TYPE_TIME;
738                 case TYPE_INVALID:
739                         Log.info("Returning TYPE_INVALID.");
740                         return LayoutItemField.GlomFieldType.TYPE_INVALID;
741                 default:
742                         Log.error("Recieved a type that I don't know about: " + Field.glom_field_type.class.getName() + "."
743                                         + type.toString() + ". Returning " + LayoutItemField.GlomFieldType.TYPE_INVALID.toString() + ".");
744                         return LayoutItemField.GlomFieldType.TYPE_INVALID;
745                 }
746         }
747
748         /*
749          * (non-Javadoc)
750          * 
751          * @see org.glom.web.client.OnlineGlomService#isAuthenticated(java.lang.String)
752          */
753         public boolean isAuthenticated(String documentTitle) {
754                 return documents.get(documentTitle).isAuthenticated();
755         }
756
757         /*
758          * (non-Javadoc)
759          * 
760          * @see org.glom.web.client.OnlineGlomService#checkAuthentication(java.lang.String, java.lang.String,
761          * java.lang.String)
762          */
763         public boolean checkAuthentication(String documentTitle, String username, String password) {
764                 ConfiguredDocument configuredDoc = documents.get(documentTitle);
765                 try {
766                         return configuredDoc.setUsernameAndPassword(username, password);
767                 } catch (SQLException e) {
768                         Log.error(documentTitle, "Unknown SQL Error checking the database authentication.", e);
769                         return false;
770                 }
771         }
772
773         /*
774          * (non-Javadoc)
775          * 
776          * @see org.glom.web.client.OnlineGlomService#getDefaultDetailsLayoutGroup(java.lang.String)
777          */
778         @Override
779         public ArrayList<LayoutGroup> getDefaultDetailsLayout(String documentTitle) {
780                 GlomDocument glomDocument = getGlomDocument(documentTitle);
781                 String tableName = glomDocument.getTableNames().get(glomDocument.getDefaultTableIndex());
782                 return getDetailsLayout(documentTitle, tableName);
783         }
784
785         /*
786          * (non-Javadoc)
787          * 
788          * @see org.glom.web.client.OnlineGlomService#getDetailsLayoutGroup(java.lang.String, java.lang.String)
789          */
790         public ArrayList<LayoutGroup> getDetailsLayout(String documentTitle, String tableName) {
791                 ConfiguredDocument configuredDoc = documents.get(documentTitle);
792                 Document document = configuredDoc.getDocument();
793                 LayoutGroupVector layoutGroupVec = document.get_data_layout_groups("details", tableName);
794
795                 ArrayList<LayoutGroup> layoutGroups = new ArrayList<LayoutGroup>();
796                 for (int i = 0; i < layoutGroupVec.size(); i++) {
797                         org.glom.libglom.LayoutGroup libglomLayoutGroup = layoutGroupVec.get(i);
798
799                         if (libglomLayoutGroup == null)
800                                 continue;
801
802                         layoutGroups.add(getLayoutGroup(documentTitle, tableName, libglomLayoutGroup));
803                 }
804                 return layoutGroups;
805         }
806
807         /*
808          * Gets a GWT-Glom LayoutGroup object for the specified libglom LayoutGroup object. This is used for getting layout
809          * information for the list and details views.
810          * 
811          * @param documentTitle Glom document title
812          * 
813          * @param tableName table name in the specified Glom document
814          * 
815          * @param libglomLayoutGroup libglom LayoutGroup to convert
816          * 
817          * @precondition libglomLayoutGroup must not be null
818          * 
819          * @return {@link LayoutGroup} object that represents the layout for the specified {@link
820          * org.glom.libglom.LayoutGroup}
821          */
822         private LayoutGroup getLayoutGroup(String documentTitle, String tableName,
823                         org.glom.libglom.LayoutGroup libglomLayoutGroup) {
824                 LayoutGroup layoutGroup = new LayoutGroup();
825                 layoutGroup.setColumnCount(safeLongToInt(libglomLayoutGroup.get_columns_count()));
826
827                 // look at each child item
828                 LayoutItemVector layoutItemsVec = libglomLayoutGroup.get_items();
829                 for (int i = 0; i < layoutItemsVec.size(); i++) {
830                         org.glom.libglom.LayoutItem libglomLayoutItem = layoutItemsVec.get(i);
831
832                         // just a safety check
833                         if (libglomLayoutItem == null)
834                                 continue;
835
836                         org.glom.web.shared.layout.LayoutItem layoutItem = null;
837                         org.glom.libglom.LayoutGroup group = org.glom.libglom.LayoutGroup.cast_dynamic(libglomLayoutItem);
838                         if (group != null) {
839                                 // recurse into child groups
840                                 layoutItem = getLayoutGroup(documentTitle, tableName, group);
841                         } else {
842                                 // create GWT-Glom LayoutItem types based on the the libglom type
843                                 // TODO add support for other LayoutItems (Text, Image, Button etc.)
844                                 LayoutItem_Field libglomLayoutField = LayoutItem_Field.cast_dynamic(libglomLayoutItem);
845                                 if (libglomLayoutField != null) {
846                                         layoutItem = convertToGWTGlomLayoutItemField(libglomLayoutField);
847                                 } else {
848                                         Log.info(documentTitle, tableName,
849                                                         "Ignoring unknown LayoutItem of type " + libglomLayoutItem.get_part_type_name() + ".");
850                                         continue;
851                                 }
852
853                         }
854
855                         layoutItem.setTitle(libglomLayoutItem.get_title_or_name());
856                         layoutGroup.addItem(layoutItem);
857                 }
858
859                 return layoutGroup;
860         }
861
862         private LayoutItemField convertToGWTGlomLayoutItemField(LayoutItem_Field libglomLayoutField) {
863                 LayoutItemField layoutItemField = new LayoutItemField();
864
865                 // set type
866                 layoutItemField.setType(convertToGWTGlomFieldType(libglomLayoutField.get_glom_type()));
867
868                 // set formatting
869                 Formatting formatting = new Formatting();
870                 formatting.setHorizontalAlignment(convertToGWTGlomHorizonalAlignment(libglomLayoutField
871                                 .get_formatting_used_horizontal_alignment()));
872                 layoutItemField.setFormatting(formatting);
873
874                 return layoutItemField;
875         }
876
877         public GlomField[] getDetailsData(String documentTitle, String tableName, String primaryKeyValue) {
878
879                 ConfiguredDocument configuredDoc = documents.get(documentTitle);
880                 Document document = configuredDoc.getDocument();
881
882                 LayoutFieldVector fieldsToGet = getFieldsToShowForSQLQuery(document, tableName, "details");
883
884                 if (fieldsToGet == null || fieldsToGet.size() <= 0) {
885                         Log.warn(documentTitle, tableName, "Didn't find any fields to show. Returning null.");
886                         return null;
887                 }
888
889                 // get primary key for the table to use in the SQL query
890                 Field primaryKey = null;
891                 FieldVector fieldsVec = document.get_table_fields(tableName);
892                 for (int i = 0; i < safeLongToInt(fieldsVec.size()); i++) {
893                         Field field = fieldsVec.get(i);
894                         if (field.get_primary_key()) {
895                                 primaryKey = field;
896                                 break;
897                         }
898                 }
899
900                 if (primaryKey == null) {
901                         Log.error(documentTitle, tableName, "Couldn't find primary key in table. Returning null.");
902                         return null;
903                 }
904
905                 ArrayList<GlomField[]> rowsList = new ArrayList<GlomField[]>();
906                 Connection conn = null;
907                 Statement st = null;
908                 ResultSet rs = null;
909                 try {
910                         // Setup the JDBC driver and get the query.
911                         ComboPooledDataSource cpds = configuredDoc.getCpds();
912                         conn = cpds.getConnection();
913                         st = conn.createStatement(ResultSet.TYPE_FORWARD_ONLY, ResultSet.CONCUR_READ_ONLY);
914                         String query = Glom.build_sql_select_with_key(tableName, fieldsToGet, primaryKey, primaryKeyValue);
915                         rs = st.executeQuery(query);
916
917                         // get the results from the ResultSet
918                         // using 2 as a length parameter so we can log a warning if the result set is greater than one
919                         rowsList = getData(documentTitle, tableName, 2, fieldsToGet, rs);
920                 } catch (SQLException e) {
921                         Log.error(documentTitle, tableName, "Error executing database query.", e);
922                         // TODO: somehow notify user of problem
923                         return null;
924                 } finally {
925                         // cleanup everything that has been used
926                         try {
927                                 if (rs != null)
928                                         rs.close();
929                                 if (st != null)
930                                         st.close();
931                                 if (conn != null)
932                                         conn.close();
933                         } catch (Exception e) {
934                                 Log.error(documentTitle, tableName,
935                                                 "Error closing database resources. Subsequent database queries may not work.", e);
936                         }
937                 }
938
939                 if (rowsList.size() == 0) {
940                         Log.error(documentTitle, tableName, "The query returned an empty ResultSet. Returning null.");
941                         return null;
942                 } else if (rowsList.size() > 1) {
943                         Log.warn(documentTitle, tableName,
944                                         "The query did not return a unique result. Returning the first result in the set.");
945                 }
946
947                 return rowsList.get(0);
948
949         }
950
951         /*
952          * Gets a LayoutFieldVector to use when generating an SQL query.
953          */
954         private LayoutFieldVector getFieldsToShowForSQLQuery(Document document, String tableName, String layoutName) {
955                 LayoutGroupVector layoutGroupVec = document.get_data_layout_groups(layoutName, tableName);
956                 LayoutFieldVector layoutFieldVector = new LayoutFieldVector();
957
958                 // special case for list layouts that don't have a defined layout group
959                 if ("list".equals(layoutName) && layoutGroupVec.size() == 0) {
960                         FieldVector fieldsVec = document.get_table_fields(tableName);
961                         for (int i = 0; i < fieldsVec.size(); i++) {
962                                 Field field = fieldsVec.get(i);
963                                 LayoutItem_Field layoutItemField = new LayoutItem_Field();
964                                 layoutItemField.set_full_field_details(field);
965                                 layoutFieldVector.add(layoutItemField);
966                         }
967                         return layoutFieldVector;
968                 }
969
970                 // We will show the fields that the document says we should:
971                 for (int i = 0; i < layoutGroupVec.size(); i++) {
972                         org.glom.libglom.LayoutGroup layoutGroup = layoutGroupVec.get(i);
973
974                         // Get the fields:
975                         ArrayList<LayoutItem_Field> layoutItemsFields = getFieldsToShowForSQLQueryAddGroup(document, tableName,
976                                         layoutGroup);
977                         for (LayoutItem_Field layoutItem_Field : layoutItemsFields) {
978                                 layoutFieldVector.add(layoutItem_Field);
979                         }
980                 }
981                 return layoutFieldVector;
982         }
983
984         private ArrayList<LayoutItem_Field> getFieldsToShowForSQLQueryAddGroup(Document document, String tableName,
985                         org.glom.libglom.LayoutGroup layoutGroup) {
986
987                 ArrayList<LayoutItem_Field> layoutItemFields = new ArrayList<LayoutItem_Field>();
988                 LayoutItemVector items = layoutGroup.get_items();
989                 for (int i = 0; i < items.size(); i++) {
990                         LayoutItem layoutItem = items.get(i);
991
992                         LayoutItem_Field layoutItemField = LayoutItem_Field.cast_dynamic(layoutItem);
993                         if (layoutItemField != null) {
994                                 // the layoutItem is a LayoutItem_Field
995                                 FieldVector fields;
996                                 if (layoutItemField.get_has_relationship_name()) {
997                                         // layoutItemField is a field in a related table
998                                         fields = document.get_table_fields(layoutItemField.get_table_used(tableName));
999                                 } else {
1000                                         // layoutItemField is a field in this table
1001                                         fields = document.get_table_fields(tableName);
1002                                 }
1003
1004                                 // set the layoutItemFeild with details from its Field in the document and
1005                                 // add it to the list to be returned
1006                                 for (int j = 0; j < fields.size(); j++) {
1007                                         // check the names to see if they're the same
1008                                         // this works because we're using the field list from the related table if necessary
1009                                         if (layoutItemField.get_name().equals(fields.get(j).get_name())) {
1010                                                 Field field = fields.get(j);
1011                                                 if (field != null) {
1012                                                         layoutItemField.set_full_field_details(field);
1013                                                         layoutItemFields.add(layoutItemField);
1014                                                 } else {
1015                                                         Log.warn(document.get_database_title(), tableName,
1016                                                                         "LayoutItem_Field " + layoutItemField.get_layout_display_name()
1017                                                                                         + " not found in document field list.");
1018                                                 }
1019                                                 break;
1020                                         }
1021                                 }
1022
1023                         } else {
1024                                 // the layoutItem is not a LayoutItem_Field
1025                                 org.glom.libglom.LayoutGroup subLayoutGroup = org.glom.libglom.LayoutGroup.cast_dynamic(layoutItem);
1026                                 if (subLayoutGroup != null) {
1027                                         // the layoutItem is a LayoutGroup
1028                                         LayoutItem_Portal layoutItemPortal = LayoutItem_Portal.cast_dynamic(layoutItem);
1029                                         if (layoutItemPortal == null) {
1030                                                 // The subGroup is not a LayoutItem_Portal.
1031                                                 // We're ignoring portals because they are filled by means of a separate SQL query.
1032                                                 layoutItemFields
1033                                                                 .addAll(getFieldsToShowForSQLQueryAddGroup(document, tableName, subLayoutGroup));
1034                                         }
1035                                 }
1036                         }
1037                 }
1038                 return layoutItemFields;
1039         }
1040
1041 }