Use cursor mode in the query that gets data for the list view.
[online-glom:gwt-glom.git] / src / main / java / org / glom / web / server / OnlineGlomServiceImpl.java
1 /*
2  * Copyright (C) 2010, 2011 Openismus GmbH
3  *
4  * This file is part of GWT-Glom.
5  *
6  * GWT-Glom is free software: you can redistribute it and/or modify it
7  * under the terms of the GNU Lesser General Public License as published by the
8  * Free Software Foundation, either version 3 of the License, or (at your
9  * option) any later version.
10  *
11  * GWT-Glom is distributed in the hope that it will be useful, but WITHOUT
12  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
13  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU Lesser General Public License
14  * for more details.
15  *
16  * You should have received a copy of the GNU Lesser General Public License
17  * along with GWT-Glom.  If not, see <http://www.gnu.org/licenses/>.
18  */
19
20 package org.glom.web.server;
21
22 import java.beans.PropertyVetoException;
23 import java.io.File;
24 import java.io.IOException;
25 import java.sql.Connection;
26 import java.sql.Date;
27 import java.sql.ResultSet;
28 import java.sql.SQLException;
29 import java.sql.Statement;
30 import java.sql.Time;
31 import java.text.DateFormat;
32 import java.text.NumberFormat;
33 import java.util.ArrayList;
34 import java.util.Currency;
35 import java.util.Locale;
36 import java.util.Properties;
37
38 import org.glom.libglom.Document;
39 import org.glom.libglom.Field;
40 import org.glom.libglom.FieldFormatting;
41 import org.glom.libglom.FieldVector;
42 import org.glom.libglom.Glom;
43 import org.glom.libglom.LayoutFieldVector;
44 import org.glom.libglom.LayoutGroupVector;
45 import org.glom.libglom.LayoutItem;
46 import org.glom.libglom.LayoutItemVector;
47 import org.glom.libglom.LayoutItem_Field;
48 import org.glom.libglom.NumericFormat;
49 import org.glom.libglom.SortClause;
50 import org.glom.libglom.SortFieldPair;
51 import org.glom.libglom.StringVector;
52 import org.glom.web.client.OnlineGlomService;
53 import org.glom.web.shared.ColumnInfo;
54 import org.glom.web.shared.GlomDocument;
55 import org.glom.web.shared.GlomField;
56 import org.glom.web.shared.LayoutListTable;
57
58 import com.google.gwt.user.server.rpc.RemoteServiceServlet;
59 import com.mchange.v2.c3p0.ComboPooledDataSource;
60 import com.mchange.v2.c3p0.DataSources;
61
62 @SuppressWarnings("serial")
63 public class OnlineGlomServiceImpl extends RemoteServiceServlet implements OnlineGlomService {
64         private Document document = null;
65         private ComboPooledDataSource cpds = null;
66         // TODO implement locale
67         private final Locale locale = Locale.ROOT;
68         private boolean configured = false;
69
70         /*
71          * This is called when the servlet is started or restarted.
72          */
73         public OnlineGlomServiceImpl() {
74                 Glom.libglom_init();
75         }
76
77         /*
78          * The properties file can't be loaded in the constructor because the servlet is not yet initialised and
79          * getServletContext() will return null. The work-around for this problem is to initialise the database and glom
80          * document object when makes its first request.
81          */
82         private void configureServlet() {
83                 document = new Document();
84
85                 Properties props = new Properties();
86                 String propFileName = "/WEB-INF/OnlineGlom.properties";
87                 try {
88                         props.load(getServletContext().getResourceAsStream(propFileName));
89                 } catch (IOException e) {
90                         // TODO log fatal error, notify user of problem
91                         e.printStackTrace();
92                 }
93
94                 File file = new File(props.getProperty("glomfile"));
95                 document.set_file_uri("file://" + file.getAbsolutePath());
96                 int error = 0;
97                 @SuppressWarnings("unused")
98                 boolean retval = document.load(error);
99                 // TODO handle error condition
100
101                 // load the jdbc driver
102                 cpds = new ComboPooledDataSource();
103                 try {
104                         cpds.setDriverClass("org.postgresql.Driver");
105                 } catch (PropertyVetoException e) {
106                         // TODO log error, fatal error can't continue, user can be notified when db access doesn't work
107                         e.printStackTrace();
108                 }
109
110                 cpds.setJdbcUrl("jdbc:postgresql://" + document.get_connection_server() + "/"
111                                 + document.get_connection_database());
112                 cpds.setUser(props.getProperty("dbusername"));
113                 cpds.setPassword(props.getProperty("dbpassword"));
114                 configured = true;
115         }
116
117         /*
118          * This is called when the servlet is stopped or restarted.
119          * 
120          * @see javax.servlet.GenericServlet#destroy()
121          */
122         @Override
123         public void destroy() {
124                 Glom.libglom_deinit();
125                 try {
126                         if (configured)
127                                 DataSources.destroy(cpds);
128                 } catch (SQLException e) {
129                         // TODO log error, don't need to notify user because this is a clean up method
130                         e.printStackTrace();
131                 }
132         }
133
134         /*
135          * FIXME I think Swig is generating long on 64-bit machines and int on 32-bit machines - need to keep this constant
136          * http://stackoverflow.com/questions/1590831/safely-casting-long-to-int-in-java
137          */
138         private static int safeLongToInt(long l) {
139                 if (l < Integer.MIN_VALUE || l > Integer.MAX_VALUE) {
140                         throw new IllegalArgumentException(l + " cannot be cast to int without changing its value.");
141                 }
142                 return (int) l;
143         }
144
145         public GlomDocument getGlomDocument() {
146                 if (!configured)
147                         configureServlet();
148
149                 GlomDocument glomDocument = new GlomDocument();
150
151                 // get arrays of table names and titles, and find the default table index
152                 StringVector tablesVec = document.get_table_names();
153
154                 int numTables = safeLongToInt(tablesVec.size());
155                 // we don't know how many tables will be hidden so we'll use half of the number of tables for the default size
156                 // of the ArrayList
157                 ArrayList<String> tableNames = new ArrayList<String>(numTables / 2);
158                 ArrayList<String> tableTitles = new ArrayList<String>(numTables / 2);
159                 boolean foundDefaultTable = false;
160                 int visibleIndex = 0;
161                 for (int i = 0; i < numTables; i++) {
162                         String tableName = tablesVec.get(i);
163                         if (!document.get_table_is_hidden(tableName)) {
164                                 tableNames.add(tableName);
165                                 // JNI is "expensive", the comparison will only be called if we haven't already found the default table
166                                 if (!foundDefaultTable && tableName.equals(document.get_default_table())) {
167                                         glomDocument.setDefaultTableIndex(visibleIndex);
168                                         foundDefaultTable = true;
169                                 }
170                                 tableTitles.add(document.get_table_title(tableName));
171                                 visibleIndex++;
172                         }
173                 }
174
175                 // set everything we need
176                 glomDocument.setTableNames(tableNames);
177                 glomDocument.setTableTitles(tableTitles);
178                 glomDocument.setTitle(document.get_database_title());
179
180                 return glomDocument;
181         }
182
183         public LayoutListTable getLayoutListTable(String tableName) {
184                 if (!configured)
185                         configureServlet();
186
187                 LayoutListTable tableInfo = new LayoutListTable();
188
189                 // access the layout list
190                 LayoutGroupVector layoutListVec = document.get_data_layout_groups("list", tableName);
191                 ColumnInfo[] columns = null;
192                 LayoutFieldVector layoutFields = new LayoutFieldVector();
193                 if (layoutListVec.size() > 0) {
194                         // a layout list is defined, we can use it to for the LayoutListTable
195                         // TODO log warning when the layoutListVec.size() > 1 but still use the first list
196                         LayoutItemVector layoutItemsVec = layoutListVec.get(0).get_items();
197
198                         // find the defined layout list fields
199                         int numItems = safeLongToInt(layoutItemsVec.size());
200                         columns = new ColumnInfo[numItems];
201                         for (int i = 0; i < numItems; i++) {
202                                 // TODO add support for other LayoutItems (Text, Image, Button)
203                                 LayoutItem item = layoutItemsVec.get(i);
204                                 LayoutItem_Field layoutItemField = LayoutItem_Field.cast_dynamic(item);
205                                 if (layoutItemField != null) {
206                                         layoutFields.add(layoutItemField);
207                                         FieldFormatting.HorizontalAlignment alignment = layoutItemField
208                                                         .get_formatting_used_horizontal_alignment();
209                                         columns[i] = new ColumnInfo(layoutItemField.get_title_or_name(),
210                                                         getColumnInfoHorizontalAlignment(alignment));
211                                 }
212                         }
213                 } else {
214                         // no layout list is defined, use the table fields as the layout list
215                         FieldVector fieldsVec = document.get_table_fields(tableName);
216
217                         // find the fields to display in the layout list
218                         int numItems = safeLongToInt(fieldsVec.size());
219                         columns = new ColumnInfo[numItems];
220                         for (int i = 0; i < numItems; i++) {
221                                 Field field = fieldsVec.get(i);
222                                 LayoutItem_Field layoutItemField = new LayoutItem_Field();
223                                 layoutItemField.set_full_field_details(field);
224                                 layoutFields.add(layoutItemField);
225                                 FieldFormatting.HorizontalAlignment alignment = layoutItemField
226                                                 .get_formatting_used_horizontal_alignment();
227                                 columns[i] = new ColumnInfo(layoutItemField.get_title_or_name(),
228                                                 getColumnInfoHorizontalAlignment(alignment));
229                         }
230                 }
231
232                 tableInfo.setColumns(columns);
233
234                 // get the size of the returned query for the pager
235
236                 // TODO Create a memory and time efficient way to get the row count. Right now the ResultSet is not being
237                 // returned in cursor mode because we're using TYPE_SCROLL_INSENSITIVE to able to seek to the last record of the
238                 // ResultSet. The PostgreSQL JDBC driver doesn't support moving around in the ResultSet when in cursor mode.
239                 // Here's the relevant PostgreSQL documentation:
240                 // http://jdbc.postgresql.org/documentation/83/query.html#query-with-cursor
241                 // An option might be to do a SELECT COUNT query using a method in libglom or a custom method in java-libglom to
242                 // build the query. If we do this, we wouldn't need to think about the two TODOs below.
243                 // See: https://bugzilla.gnome.org/show_bug.cgi?id=645110
244
245                 // TODO since we're executing a query anyway, maybe we should return the rows that will be displayed on the
246                 // first page
247
248                 // TODO this code is really similar to code in getTableData, find a way to not duplicate the code
249                 Connection conn = null;
250                 Statement st = null;
251                 ResultSet rs = null;
252                 try {
253                         // setup and execute the query
254                         conn = cpds.getConnection();
255                         st = conn.createStatement(ResultSet.TYPE_SCROLL_INSENSITIVE, ResultSet.CONCUR_READ_ONLY);
256                         String query = Glom.build_sql_select_simple(tableName, layoutFields);
257                         rs = st.executeQuery(query);
258
259                         // get the number of rows in the query
260                         rs.setFetchDirection(ResultSet.FETCH_FORWARD);
261                         rs.last();
262                         tableInfo.setNumRows(rs.getRow());
263
264                 } catch (SQLException e) {
265                         // TODO log error
266                         // we don't know how many rows are in the query
267                         e.printStackTrace();
268                         tableInfo.setNumRows(0);
269                 } finally {
270                         // cleanup everything that has been used
271                         try {
272                                 rs.close();
273                                 st.close();
274                                 conn.close();
275                         } catch (Exception e) {
276                                 // TODO log error
277                                 e.printStackTrace();
278                         }
279                 }
280
281                 return tableInfo;
282         }
283
284         public ArrayList<GlomField[]> getTableData(String table, int start, int length) {
285                 return getTableData(table, start, length, false, 0, false);
286         }
287
288         public ArrayList<GlomField[]> getSortedTableData(String table, int start, int length, int sortColumnIndex,
289                         boolean isAscending) {
290                 return getTableData(table, start, length, true, sortColumnIndex, isAscending);
291         }
292
293         private ArrayList<GlomField[]> getTableData(String table, int start, int length, boolean useSortClause,
294                         int sortColumnIndex, boolean isAscending) {
295                 if (!configured)
296                         configureServlet();
297
298                 // access the layout list using the defined layout list or the table fields if there's no layout list
299                 LayoutGroupVector layoutListVec = document.get_data_layout_groups("list", table);
300                 LayoutFieldVector layoutFields = new LayoutFieldVector();
301                 SortClause sortClause = new SortClause();
302                 if (layoutListVec.size() > 0) {
303                         // a layout list is defined, we can use it to for the LayoutListTable
304                         // TODO log warning when the layoutListVec.size() > 1 but still use the first list
305                         LayoutItemVector layoutItemsVec = layoutListVec.get(0).get_items();
306
307                         // find the defined layout list fields
308                         int numItems = safeLongToInt(layoutItemsVec.size());
309                         for (int i = 0; i < numItems; i++) {
310                                 // TODO add support for other LayoutItems (Text, Image, Button)
311                                 LayoutItem item = layoutItemsVec.get(i);
312                                 LayoutItem_Field layoutItemfield = LayoutItem_Field.cast_dynamic(item);
313                                 if (layoutItemfield != null) {
314                                         // use this field in the layout
315                                         layoutFields.add(layoutItemfield);
316
317                                         // create a sort clause if it's a primary key and we're not asked to sort a specific column
318                                         if (!useSortClause) {
319                                                 Field details = layoutItemfield.get_full_field_details();
320                                                 if (details != null && details.get_primary_key()) {
321                                                         sortClause.addLast(new SortFieldPair(layoutItemfield, true)); // ascending
322                                                 }
323                                         }
324                                 }
325                         }
326                 } else {
327                         // no layout list is defined, use the table fields as the layout list
328                         FieldVector fieldsVec = document.get_table_fields(table);
329
330                         // find the fields to display in the layout list
331                         int numItems = safeLongToInt(fieldsVec.size());
332                         for (int i = 0; i < numItems; i++) {
333                                 Field field = fieldsVec.get(i);
334                                 LayoutItem_Field layoutItemField = new LayoutItem_Field();
335                                 layoutItemField.set_full_field_details(field);
336                                 layoutFields.add(layoutItemField);
337
338                                 // create a sort clause if it's a primary key and we're not asked to sort a specific column
339                                 if (!useSortClause) {
340                                         if (field.get_primary_key()) {
341                                                 sortClause.addLast(new SortFieldPair(layoutItemField, true)); // ascending
342                                         }
343                                 }
344                         }
345                 }
346
347                 // create a sort clause for the column we've been asked to sort
348                 if (useSortClause) {
349                         LayoutItem item = layoutFields.get(sortColumnIndex);
350                         LayoutItem_Field field = LayoutItem_Field.cast_dynamic(item);
351                         if (field != null)
352                                 sortClause.addLast(new SortFieldPair(field, isAscending));
353                         // TODO: log error in the else condition
354                 }
355
356                 ArrayList<GlomField[]> rowsList = new ArrayList<GlomField[]>();
357                 Connection conn = null;
358                 Statement st = null;
359                 ResultSet rs = null;
360                 try {
361                         // Setup and execute the query. Special care needs to be take to ensure that the results will be based on a
362                         // cursor so that large amounts of memory are not consumed when the query retrieve a large amount of data.
363                         // Here's the relevant PostgreSQL documentation:
364                         // http://jdbc.postgresql.org/documentation/83/query.html#query-with-cursor
365                         conn = cpds.getConnection();
366                         conn.setAutoCommit(false);
367                         st = conn.createStatement(ResultSet.TYPE_FORWARD_ONLY, ResultSet.CONCUR_READ_ONLY);
368                         st.setFetchSize(length);
369                         String query = Glom.build_sql_select_simple(table, layoutFields, sortClause) + " OFFSET " + start;
370                         // TODO Test memory usage before and after we execute the query that would result in a large ResultSet.
371                         // We need to ensure that the JDBC driver is in fact returning a cursor based result set that has a low
372                         // memory footprint. Check the difference between this value before and after the query:
373                         // Runtime.getRuntime().totalMemory() - Runtime.getRuntime().freeMemory()
374                         rs = st.executeQuery(query);
375
376                         // get the data we've been asked for
377                         int rowCount = 0;
378                         while (rs.next() && rowCount <= length) {
379                                 int layoutFieldsSize = safeLongToInt(layoutFields.size());
380                                 GlomField[] rowArray = new GlomField[layoutFieldsSize];
381                                 for (int i = 0; i < layoutFieldsSize; i++) {
382                                         // make a new GlomField to set the text and colours
383                                         rowArray[i] = new GlomField();
384
385                                         // get foreground and background colours
386                                         LayoutItem_Field field = layoutFields.get(i);
387                                         FieldFormatting formatting = field.get_formatting_used();
388                                         String fgcolour = formatting.get_text_format_color_foreground();
389                                         if (!fgcolour.isEmpty())
390                                                 rowArray[i].setFGColour(convertGdkColorToHtmlColour(fgcolour));
391                                         String bgcolour = formatting.get_text_format_color_background();
392                                         if (!bgcolour.isEmpty())
393                                                 rowArray[i].setBGColour(convertGdkColorToHtmlColour(bgcolour));
394
395                                         // convert field values are to strings based on the glom type
396                                         switch (field.get_glom_type()) {
397                                         case TYPE_TEXT:
398                                                 String text = rs.getString(i + 1);
399                                                 rowArray[i].setText(text != null ? text : "");
400                                                 break;
401                                         case TYPE_BOOLEAN:
402                                                 rowArray[i].setText(rs.getBoolean(i + 1) ? "TRUE" : "FALSE");
403                                                 break;
404                                         case TYPE_NUMERIC:
405                                                 // Take care of the numeric formatting before converting the number to a string.
406                                                 NumericFormat numFormatGlom = formatting.getM_numeric_format();
407                                                 // There's no isCurrency() method in the glom NumericFormat class so we're assuming that the
408                                                 // number should be formatted as a currency if the currency code string is not empty.
409                                                 String currencyCode = numFormatGlom.getM_currency_symbol();
410                                                 NumberFormat numFormatJava = null;
411                                                 boolean useGlomCurrencyCode = false;
412                                                 if (currencyCode.length() == 3) {
413                                                         // Try to format the currency using the Java Locales system.
414                                                         try {
415                                                                 Currency currency = Currency.getInstance(currencyCode);
416                                                                 // Ignore the glom numeric formatting when a valid ISO 4217 currency code is being used.
417                                                                 int digits = currency.getDefaultFractionDigits();
418                                                                 numFormatJava = NumberFormat.getCurrencyInstance(locale);
419                                                                 numFormatJava.setCurrency(currency);
420                                                                 numFormatJava.setMinimumFractionDigits(digits);
421                                                                 numFormatJava.setMaximumFractionDigits(digits);
422                                                         } catch (IllegalArgumentException e) {
423                                                                 // TODO: log warning
424                                                                 // The currency code is not this is not an ISO 4217 currency code.
425                                                                 // We're going to manually set the currency code and use the glom numeric formatting.
426                                                                 useGlomCurrencyCode = true;
427                                                                 numFormatJava = getJavaNumberFormat(numFormatGlom);
428                                                         }
429                                                 } else if (currencyCode.length() > 0) {
430                                                         // The length of the currency code is > 0 and != 3; this is not an ISO 4217 currency code.
431                                                         // We're going to manually set the currency code and use the glom numeric formatting.
432                                                         useGlomCurrencyCode = true;
433                                                         numFormatJava = getJavaNumberFormat(numFormatGlom);
434                                                 } else {
435                                                         // The length of the currency code is 0; the number is not a currency.
436                                                         numFormatJava = getJavaNumberFormat(numFormatGlom);
437                                                 }
438
439                                                 // TODO: Do I need to do something with NumericFormat.get_default_precision() from libglom?
440
441                                                 double number = rs.getDouble(i + 1);
442                                                 if (number < 0) {
443                                                         if (formatting.getM_numeric_format().getM_alt_foreground_color_for_negatives())
444                                                                 // overrides the set foreground colour
445                                                                 rowArray[i].setFGColour(convertGdkColorToHtmlColour(NumericFormat
446                                                                                 .get_alternative_color_for_negatives()));
447                                                 }
448
449                                                 // Finally convert the number to text using the glom currency string if required.
450                                                 if (useGlomCurrencyCode) {
451                                                         rowArray[i].setText(currencyCode + " " + numFormatJava.format(number));
452                                                 } else {
453                                                         rowArray[i].setText(numFormatJava.format(number));
454                                                 }
455                                                 break;
456                                         case TYPE_DATE:
457                                                 Date date = rs.getDate(i + 1);
458                                                 if (date != null) {
459                                                         DateFormat dateFormat = DateFormat.getDateInstance(DateFormat.MEDIUM, locale);
460                                                         rowArray[i].setText(dateFormat.format(rs.getDate(i + 1)));
461                                                 } else {
462                                                         rowArray[i].setText("");
463                                                 }
464                                                 break;
465                                         case TYPE_TIME:
466                                                 Time time = rs.getTime(i + 1);
467                                                 if (time != null) {
468                                                         DateFormat timeFormat = DateFormat.getTimeInstance(DateFormat.MEDIUM, locale);
469                                                         rowArray[i].setText(timeFormat.format(time));
470                                                 } else {
471                                                         rowArray[i].setText("");
472                                                 }
473                                                 break;
474                                         case TYPE_IMAGE:
475                                                 byte[] image = rs.getBytes(i + 1);
476                                                 if (image != null) {
477                                                         // TODO implement field TYPE_IMAGE
478                                                         rowArray[i].setText("Image (FIXME)");
479                                                 } else {
480                                                         rowArray[i].setText("");
481                                                 }
482                                                 break;
483                                         case TYPE_INVALID:
484                                         default:
485                                                 // TODO log warning message
486                                                 rowArray[i].setText("");
487                                                 break;
488                                         }
489                                 }
490
491                                 // add the row of GlomFields to the ArrayList we're going to return and update the row count
492                                 rowsList.add(rowArray);
493                                 rowCount++;
494                         }
495                 } catch (SQLException e) {
496                         // TODO: log error, notify user of problem
497                         e.printStackTrace();
498                 } finally {
499                         // cleanup everything that has been used
500                         try {
501                                 rs.close();
502                                 st.close();
503                                 conn.close();
504                         } catch (Exception e) {
505                                 // TODO log error
506                                 e.printStackTrace();
507                         }
508                 }
509                 return rowsList;
510         }
511
512         private NumberFormat getJavaNumberFormat(NumericFormat numFormatGlom) {
513                 NumberFormat numFormatJava = NumberFormat.getInstance(locale);
514                 if (numFormatGlom.getM_decimal_places_restricted()) {
515                         int digits = safeLongToInt(numFormatGlom.getM_decimal_places());
516                         numFormatJava.setMinimumFractionDigits(digits);
517                         numFormatJava.setMaximumFractionDigits(digits);
518                 }
519                 numFormatJava.setGroupingUsed(numFormatGlom.getM_use_thousands_separator());
520                 return numFormatJava;
521         }
522
523         /*
524          * Converts a Gdk::Color (16-bits per channel) to an HTML colour (8-bits per channel) by disgarding the least
525          * significant 8-bits in each channel.
526          */
527         private String convertGdkColorToHtmlColour(String gdkColor) {
528                 if (gdkColor.length() == 13)
529                         return gdkColor.substring(0, 2) + gdkColor.substring(5, 6) + gdkColor.substring(9, 10);
530                 else if (gdkColor.length() == 7)
531                         // TODO: log warning because we're expecting a 13 character string
532                         return gdkColor;
533                 else
534                         // TODO: log error
535                         return "";
536         }
537
538         /*
539          * This method converts a FieldFormatting.HorizontalAlignment to the equivalent ColumnInfo.HorizontalAlignment. The
540          * need for this comes from the fact that the GWT HorizontalAlignment classes can't be used with RPC and there's no
541          * easy way to use the java-libglom FieldFormatting.HorizontalAlignment enum with RPC. An enum indentical to
542          * FieldFormatting.HorizontalAlignment is included in the ColumnInfo class.
543          */
544         private ColumnInfo.HorizontalAlignment getColumnInfoHorizontalAlignment(
545                         FieldFormatting.HorizontalAlignment alignment) {
546                 int value = alignment.swigValue();
547                 ColumnInfo.HorizontalAlignment[] columnInfoValues = ColumnInfo.HorizontalAlignment.class.getEnumConstants();
548                 if (value < columnInfoValues.length && value >= 0)
549                         return columnInfoValues[value];
550                 // TODO: log error: value out of range, returning HORIZONTAL_ALIGNMENT_RIGHT
551                 return columnInfoValues[FieldFormatting.HorizontalAlignment.HORIZONTAL_ALIGNMENT_RIGHT.swigValue()];
552         }
553
554 }