2 * Copyright (C) 2012 Openismus GmbH
4 * This file is part of GWT-Glom.
6 * GWT-Glom is free software: you can redistribute it and/or modify it
7 * under the terms of the GNU Lesser General Public License as published by the
8 * Free Software Foundation, either version 3 of the License, or (at your
9 * option) any later version.
11 * GWT-Glom is distributed in the hope that it will be useful, but WITHOUT
12 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
13 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License
16 * You should have received a copy of the GNU Lesser General Public License
17 * along with GWT-Glom. If not, see <http://www.gnu.org/licenses/>.
20 package org.glom.web.server;
22 import java.beans.PropertyVetoException;
23 import java.sql.Connection;
25 import java.sql.DriverManager;
26 import java.sql.ResultSet;
27 import java.sql.SQLException;
28 import java.sql.Statement;
30 import java.text.DateFormat;
31 import java.util.ArrayList;
32 import java.util.List;
33 import java.util.Locale;
35 import org.apache.commons.lang3.StringUtils;
36 import org.glom.web.server.libglom.Document;
37 import org.glom.web.shared.DataItem;
38 import org.glom.web.shared.TypedDataItem;
39 import org.glom.web.shared.libglom.Field;
40 import org.glom.web.shared.libglom.Relationship;
41 import org.glom.web.shared.libglom.layout.LayoutItemField;
42 import org.glom.web.shared.libglom.layout.SortClause;
43 import org.glom.web.shared.libglom.layout.UsesRelationship;
44 import org.glom.web.shared.libglom.layout.UsesRelationshipImpl;
45 import org.jooq.AggregateFunction;
46 import org.jooq.Condition;
47 import org.jooq.Record;
48 import org.jooq.SQLDialect;
49 import org.jooq.SelectFinalStep;
50 import org.jooq.SelectJoinStep;
51 import org.jooq.SelectSelectStep;
52 import org.jooq.Table;
53 import org.jooq.conf.RenderKeywordStyle;
54 import org.jooq.conf.RenderNameStyle;
55 import org.jooq.conf.Settings;
56 import org.jooq.impl.Factory;
58 import com.mchange.v2.c3p0.ComboPooledDataSource;
61 * @author Murray Cumming <murrayc@openismus.com>
64 public class SqlUtils {
70 private static ComboPooledDataSource createAndSetupDataSource(final Document document) {
71 return createAndSetupDataSource(document.getHostingMode(), document.getConnectionServer(), document.getConnectionPort(), document.getConnectionDatabase());
74 static class JdbcConnectionDetails {
75 String driverClass = null;
76 String jdbcURL = null;
79 public static JdbcConnectionDetails getJdbcConnectionDetails(final Document document) {
80 return getJdbcConnectionDetails(document.getHostingMode(), document.getConnectionServer(), document.getConnectionPort(), document.getConnectionDatabase());
83 public static JdbcConnectionDetails getJdbcConnectionDetails(final Document.HostingMode hostingMode, final String serverHost, int serverPort, final String database) {
84 final JdbcConnectionDetails details = new JdbcConnectionDetails();
86 String defaultDatabase = null;
87 switch (hostingMode) {
88 case HOSTING_MODE_POSTGRES_CENTRAL:
89 case HOSTING_MODE_POSTGRES_SELF: {
90 details.driverClass = "org.postgresql.Driver";
91 details.jdbcURL = "jdbc:postgresql://";
92 defaultDatabase = "template1";
95 case HOSTING_MODE_MYSQL_CENTRAL:
96 case HOSTING_MODE_MYSQL_SELF: {
97 details.driverClass = "com.mysql.jdbc.Driver";
98 details.jdbcURL = "jdbc:mysql://";
99 defaultDatabase = "INFORMATION_SCHEMA";
103 // TODO: We allow self-hosting here, for testing,
104 // but maybe the startup of self-hosting should happen here.
105 Log.fatal("Error configuring the database connection." + " Only PostgreSQL and MYSQL hosting are supported.");
106 // FIXME: Throw exception?
111 // setup the JDBC driver for the current glom document
112 details.jdbcURL += serverHost + ":" + serverPort;
114 String db = database;
115 if (StringUtils.isEmpty(db)) {
116 // Use the default PostgreSQL database, because ComboPooledDataSource.connect() fails otherwise.
117 db = defaultDatabase;
119 details.jdbcURL += "/" + db; // TODO: Quote the database name?
128 private static ComboPooledDataSource createAndSetupDataSource(final Document.HostingMode hostingMode, final String serverHost, int serverPort, final String database) {
129 final ComboPooledDataSource cpds = new ComboPooledDataSource();
131 final JdbcConnectionDetails details = getJdbcConnectionDetails(hostingMode, serverHost, serverPort, database);
132 if (details == null) {
137 cpds.setDriverClass(details.driverClass);
138 } catch (final PropertyVetoException e) {
139 Log.fatal("Error loading the PostgreSQL JDBC driver."
140 + " Is the PostgreSQL JDBC jar available to the servlet?", e);
144 cpds.setJdbcUrl(details.jdbcURL);
150 * Sets the username and password for the database associated with the Glom document.
152 * @return true if the username and password works, false otherwise
154 public static ComboPooledDataSource tryUsernameAndPassword(final Document document, final String username, final String password) throws SQLException {
155 ComboPooledDataSource cpds = createAndSetupDataSource(document);
159 /* Do not bother trying if there are no credentials. */
160 if(StringUtils.isEmpty(username) && StringUtils.isEmpty(password)) {
164 cpds.setUser(username);
165 cpds.setPassword(password);
167 final int acquireRetryAttempts = cpds.getAcquireRetryAttempts();
168 cpds.setAcquireRetryAttempts(1);
169 Connection conn = null;
171 // FIXME find a better way to check authentication
172 // it's possible that the connection could be failing for another reason
174 //Change the timeout, because it otherwise takes ages to fail sometimes when the details are not setup.
175 //This is more than enough.
176 DriverManager.setLoginTimeout(5);
178 conn = cpds.getConnection();
180 } catch (final SQLException e) {
181 Log.info(Utils.getFileName(document.getFileURI()), e.getMessage());
182 Log.info(Utils.getFileName(document.getFileURI()),
183 "Connection Failed. Maybe the username or password is not correct.");
189 cpds.setAcquireRetryAttempts(acquireRetryAttempts);
193 public static ResultSet executeQuery(final ComboPooledDataSource cpds, final String query) throws SQLException {
194 return executeQuery(cpds, query, 0);
197 public static ResultSet executeQuery(final ComboPooledDataSource cpds, final String query, int expectedLength) throws SQLException {
199 Log.error("cpds is null.");
203 // Setup the JDBC driver and run the query.
204 final Connection conn = cpds.getConnection();
206 Log.error("The connection is null.");
210 return executeQuery(conn, query, expectedLength);
213 public static ResultSet executeQuery(final Connection conn, final String query) throws SQLException {
214 return executeQuery(conn, query, 0);
217 public static ResultSet executeQuery(final Connection conn, final String query, int expectedLength) throws SQLException {
218 // Setup and execute the query. Special care needs to be take to ensure that the results will be based
219 // on a cursor so that large amounts of memory are not consumed when the query retrieve a large amount of
220 // data. Here's the relevant PostgreSQL documentation:
221 // http://jdbc.postgresql.org/documentation/83/query.html#query-with-cursor
222 conn.setAutoCommit(false);
223 final Statement st = conn.createStatement(ResultSet.TYPE_FORWARD_ONLY, ResultSet.CONCUR_READ_ONLY);
224 if(expectedLength > 0) {
225 st.setFetchSize(expectedLength);
228 return st.executeQuery(query);
235 public static void executeUpdate(final Connection conn, final String query) throws SQLException {
236 conn.setAutoCommit(false);
237 final Statement st = conn.createStatement(ResultSet.TYPE_FORWARD_ONLY, ResultSet.CONCUR_READ_ONLY);
238 st.executeUpdate(query);
241 // TODO: Change to final ArrayList<LayoutItem_Field> fieldsToGet
242 public static String buildSqlSelectWithKey(final String tableName, final List<LayoutItemField> fieldsToGet,
243 final Field primaryKey, final TypedDataItem primaryKeyValue, final SQLDialect sqlDialect) {
245 Condition whereClause = null; // Note that we ignore quickFind.
246 if (primaryKeyValue != null) {
247 whereClause = buildSimpleWhereExpression(tableName, primaryKey, primaryKeyValue);
250 final SortClause sortClause = null; // Ignored.
251 return buildSqlSelectWithWhereClause(tableName, fieldsToGet, whereClause, sortClause, sqlDialect);
254 public static Condition buildSimpleWhereExpression(final String tableName, final Field primaryKey,
255 final TypedDataItem primaryKeyValue) {
257 Condition result = null;
259 if (primaryKey == null) {
263 final String fieldName = primaryKey.getName();
264 if (StringUtils.isEmpty(fieldName)) {
268 final org.jooq.Field<Object> field = createField(tableName, fieldName);
269 result = field.equal(primaryKeyValue.getValue());
274 * private static String buildSqlSelectWithWhereClause(final String tableName, final LayoutFieldVector fieldsToGet)
275 * { final Condition whereClause = null; return buildSqlSelectWithWhereClause(tableName, fieldsToGet, whereClause);
280 * private static String buildSqlSelectWithWhereClause(final String tableName, final LayoutFieldVector fieldsToGet,
281 * final Condition whereClause) { final SortClause sortClause = null; return
282 * buildSqlSelectWithWhereClause(tableName, fieldsToGet, whereClause, sortClause); }
285 public static String buildSqlSelectWithWhereClause(final String tableName, final List<LayoutItemField> fieldsToGet,
286 final Condition whereClause, final SortClause sortClause, final SQLDialect sqlDialect) {
287 final SelectFinalStep step = buildSqlSelectStepWithWhereClause(tableName, fieldsToGet, whereClause, sortClause, sqlDialect);
292 final String query = step.getQuery().getSQL(true);
293 // Log.info("Query: " + query);
297 private static SelectSelectStep createSelect(final SQLDialect sqlDialect) {
298 final Factory factory = new Factory(sqlDialect);
299 final Settings settings = factory.getSettings();
300 settings.setRenderNameStyle(RenderNameStyle.QUOTED); // TODO: This doesn't seem to have any effect.
301 settings.setRenderKeywordStyle(RenderKeywordStyle.UPPER); // TODO: Just to make debugging nicer.
303 final SelectSelectStep selectStep = factory.select();
307 private static SelectFinalStep buildSqlSelectStepWithWhereClause(final String tableName,
308 final List<LayoutItemField> fieldsToGet, final Condition whereClause, final SortClause sortClause, final SQLDialect sqlDialect) {
310 final SelectSelectStep selectStep = createSelect(sqlDialect);
312 // Add the fields, and any necessary joins:
313 final List<UsesRelationship> listRelationships = buildSqlSelectAddFieldsToGet(selectStep, tableName,
314 fieldsToGet, sortClause, false /* extraJoin */);
316 final Table<Record> table = Factory.tableByName(tableName);
317 final SelectJoinStep joinStep = selectStep.from(table);
319 // LEFT OUTER JOIN will get the field values from the other tables,
320 // and give us our fields for this table even if there is no corresponding value in the other table.
321 for (final UsesRelationship usesRelationship : listRelationships) {
322 builderAddJoin(joinStep, usesRelationship);
325 SelectFinalStep finalStep = joinStep;
326 if (whereClause != null) {
327 finalStep = joinStep.where(whereClause);
333 public static String buildSqlCountSelectWithWhereClause(final String tableName,
334 final List<LayoutItemField> fieldsToGet, final SQLDialect sqlDialect) {
335 final SelectFinalStep selectInner = buildSqlSelectStepWithWhereClause(tableName, fieldsToGet, null, null, sqlDialect);
336 return buildSqlSelectCountRows(selectInner, sqlDialect);
339 public static String buildSqlCountSelectWithWhereClause(final String tableName,
340 final List<LayoutItemField> fieldsToGet, final Condition whereClause, final SQLDialect sqlDialect) {
341 final SelectFinalStep selectInner = buildSqlSelectStepWithWhereClause(tableName, fieldsToGet, whereClause, null, sqlDialect);
342 return buildSqlSelectCountRows(selectInner, sqlDialect);
345 private static String buildSqlSelectCountRows(final SelectFinalStep selectInner, final SQLDialect sqlDialect) {
346 // TODO: Find a way to do this with the jOOQ API:
347 final SelectSelectStep select = createSelect(sqlDialect);
349 final org.jooq.Field<?> field = Factory.field("*");
350 final AggregateFunction<?> count = Factory.count(field);
351 select.select(count).from(selectInner);
352 return select.getQuery().getSQL(true);
353 // return "SELECT COUNT(*) FROM (" + query + ") AS glomarbitraryalias";
356 private static List<UsesRelationship> buildSqlSelectAddFieldsToGet(SelectSelectStep step, final String tableName,
357 final List<LayoutItemField> fieldsToGet, final SortClause sortClause, final boolean extraJoin) {
359 // Get all relationships used in the query:
360 final List<UsesRelationship> listRelationships = new ArrayList<UsesRelationship>();
362 final int layoutFieldsSize = Utils.safeLongToInt(fieldsToGet.size());
363 for (int i = 0; i < layoutFieldsSize; i++) {
364 final UsesRelationship layoutItem = fieldsToGet.get(i);
365 addToRelationshipsList(listRelationships, layoutItem);
368 if (sortClause != null) {
369 final int sortFieldsSize = Utils.safeLongToInt(sortClause.size());
370 for (int i = 0; i < sortFieldsSize; i++) {
371 final SortClause.SortField pair = sortClause.get(i);
372 final UsesRelationship layoutItem = pair.field;
373 addToRelationshipsList(listRelationships, layoutItem);
377 boolean oneAdded = false;
378 for (int i = 0; i < layoutFieldsSize; i++) {
379 final LayoutItemField layoutItem = fieldsToGet.get(i);
381 if (layoutItem == null) {
382 // g_warn_if_reached();
386 // Get the parent, such as the table name, or the alias name for the join:
387 // final String parent = layout_item.get_sql_table_or_join_alias_name(tableName);
390 * TODO: const LayoutItem_FieldSummary* fieldsummary = dynamic_cast<const
391 * LayoutItem_FieldSummary*>(layout_item.obj()); if(fieldsummary) { const Gnome::Gda::SqlBuilder::Id
392 * id_function = builder->add_function( fieldsummary->get_summary_type_sql(),
393 * builder->add_field_id(layout_item->get_name(), tableName)); builder->add_field_value_id(id_function); }
396 final org.jooq.Field<?> field = createField(tableName, layoutItem);
398 step = step.select(field);
400 // Avoid duplicate records with doubly-related fields:
401 // TODO: if(extra_join)
402 // builder->select_group_by(id);
410 // TODO: std::cerr << G_STRFUNC << ": No fields added: fieldsToGet.size()=" << fieldsToGet.size() <<
412 return listRelationships;
415 return listRelationships;
418 private static org.jooq.Field<Object> createField(final String tableName, final String fieldName) {
419 if (StringUtils.isEmpty(tableName)) {
423 if (StringUtils.isEmpty(fieldName)) {
427 return Factory.fieldByName(tableName, fieldName);
430 private static org.jooq.Field<Object> createField(final String tableName, final LayoutItemField layoutField) {
431 if (StringUtils.isEmpty(tableName)) {
435 if (layoutField == null) {
439 return createField(layoutField.getSqlTableOrJoinAliasName(tableName), layoutField.getName());
442 private static void addToRelationshipsList(final List<UsesRelationship> listRelationships,
443 final UsesRelationship layoutItem) {
445 if (layoutItem == null) {
449 if (!layoutItem.getHasRelationshipName()) {
453 // If this is a related relationship, add the first-level relationship too, so that the related relationship can
454 // be defined in terms of it:
455 // TODO: //If the table is not yet in the list:
456 if (layoutItem.getHasRelatedRelationshipName()) {
457 final UsesRelationship usesRel = new UsesRelationshipImpl();
458 usesRel.setRelationship(layoutItem.getRelationship());
460 // Remove any UsesRelationship that has only the same relationship (not related relationship),
461 // to avoid adding that part of the relationship to the SQL twice (two identical JOINS).
462 // listRemoveIfUsesRelationship(listRelationships, usesRel.getRelationship());
464 if (!listRelationships.contains(usesRel)) {
465 // These need to be at the front, so that related relationships can use
466 // them later in the SQL statement.
467 listRelationships.add(usesRel);
472 // Add the relationship to the list:
473 final UsesRelationship usesRel = new UsesRelationshipImpl();
474 usesRel.setRelationship(layoutItem.getRelationship());
475 usesRel.setRelatedRelationship(layoutItem.getRelatedRelationship());
476 if (!listRelationships.contains(usesRel)) {
477 listRelationships.add(usesRel);
483 * @param listRelationships
484 * @param relationship
487 * private static void listRemoveIfUsesRelationship(final List<UsesRelationship> listRelationships, final
488 * Relationship relationship) { if (relationship == null) { return; }
490 * final Iterator<UsesRelationship> i = listRelationships.iterator(); while (i.hasNext()) { final UsesRelationship
491 * eachUsesRel = i.next(); if (eachUsesRel == null) continue;
493 * // Ignore these: if (eachUsesRel.getHasRelatedRelationshipName()) { continue; }
495 * final Relationship eachRel = eachUsesRel.getRelationship(); if (eachRel == null) { continue; }
497 * Log.info("Checking: rel name=" + relationship.get_name() + ", eachRel name=" + eachRel.get_name());
499 * if (UsesRelationship.relationship_equals(relationship, eachRel)) { i.remove(); Log.info(" Removed"); } else {
500 * Log.info(" not equal"); }
505 private static void builderAddJoin(SelectJoinStep step, final UsesRelationship usesRelationship) {
506 final Relationship relationship = usesRelationship.getRelationship();
507 if (!relationship.getHasFields()) { // TODO: Handle related_record has_fields.
508 if (relationship.getHasToTable()) {
509 // It is a relationship that only specifies the table, without specifying linking fields:
511 // Table<Record> toTable = Factory.tableByName(relationship.getToTable());
512 // TODO: stepResult = step.from(toTable);
518 // Define the alias name as returned by getSqlJoinAliasName():
520 // Specify an alias, to avoid ambiguity when using 2 relationships to the same table.
521 final String aliasName = usesRelationship.getSqlJoinAliasName();
524 if (!usesRelationship.getHasRelatedRelationshipName()) {
526 final org.jooq.Field<Object> fieldFrom = createField(relationship.getFromTable(),
527 relationship.getFromField());
528 final org.jooq.Field<Object> fieldTo = createField(aliasName, relationship.getToField());
529 final Condition condition = fieldFrom.equal(fieldTo);
531 // Note that LEFT JOIN (used in libglom/GdaSqlBuilder) is apparently the same as LEFT OUTER JOIN.
532 final Table<Record> toTable = Factory.tableByName(relationship.getToTable());
533 step = step.leftOuterJoin(toTable.as(aliasName)).on(condition);
535 final UsesRelationship parentRelationship = new UsesRelationshipImpl();
536 parentRelationship.setRelationship(relationship);
537 final Relationship relatedRelationship = usesRelationship.getRelatedRelationship();
539 final org.jooq.Field<Object> fieldFrom = createField(parentRelationship.getSqlJoinAliasName(),
540 relatedRelationship.getFromField());
541 final org.jooq.Field<Object> fieldTo = createField(aliasName, relatedRelationship.getToField());
542 final Condition condition = fieldFrom.equal(fieldTo);
544 // Note that LEFT JOIN (used in libglom/GdaSqlBuilder) is apparently the same as LEFT OUTER JOIN.
545 final Table<Record> toTable = Factory.tableByName(relatedRelationship.getToTable());
546 step = step.leftOuterJoin(toTable.as(aliasName)).on(condition);
550 public static Condition getFindWhereClauseQuick(final Document document, final String tableName,
551 final TypedDataItem quickFindValue) {
552 if (StringUtils.isEmpty(tableName)) {
556 // TODO: if(Conversions::value_is_empty(quick_search))
557 // return Gnome::Gda::SqlExpr();
559 Condition condition = null;
561 // TODO: Cache the list of all fields, as well as caching (m_Fields) the list of all visible fields:
562 final List<Field> fields = document.getTableFields(tableName);
564 final int fieldsSize = Utils.safeLongToInt(fields.size());
565 for (int i = 0; i < fieldsSize; i++) {
566 final Field field = fields.get(i);
571 if (field.getGlomType() != Field.GlomFieldType.TYPE_TEXT) {
575 final org.jooq.Field<Object> jooqField = createField(tableName, field.getName());
577 // Do a case-insensitive substring search:
578 // TODO: Use ILIKE: http://sourceforge.net/apps/trac/jooq/ticket/1423
579 // http://groups.google.com/group/jooq-user/browse_thread/thread/203ae5a1a06ae65f
580 final Condition thisCondition = jooqField.lower().contains(quickFindValue.getText().toLowerCase());
582 if (condition == null) {
583 condition = thisCondition;
585 condition = condition.or(thisCondition);
597 * @param primaryKeyValue
598 * @throws SQLException
600 public static void fillDataItemFromResultSet(final DataItem dataItem, final LayoutItemField field, final int rsIndex,
601 final ResultSet rs, final String documentID, final String tableName, final TypedDataItem primaryKeyValue) throws SQLException {
603 switch (field.getGlomType()) {
605 final String text = rs.getString(rsIndex);
606 dataItem.setText(text != null ? text : "");
609 dataItem.setBoolean(rs.getBoolean(rsIndex));
612 dataItem.setNumber(rs.getDouble(rsIndex));
615 final Date date = rs.getDate(rsIndex);
617 // TODO: Pass Date and Time types instead of converting to text here?
618 // TODO: Use a 4-digit-year short form, somehow.
619 final DateFormat dateFormat = DateFormat.getDateInstance(DateFormat.SHORT, Locale.ROOT);
620 dataItem.setText(dateFormat.format(date));
622 dataItem.setText("");
626 final Time time = rs.getTime(rsIndex);
628 final DateFormat timeFormat = DateFormat.getTimeInstance(DateFormat.SHORT, Locale.ROOT);
629 dataItem.setText(timeFormat.format(time));
631 dataItem.setText("");
635 //We don't get the data here.
636 //Instead we provide a way for the client to get the image separately.
638 //This doesn't seem to work,
639 //presumably because the base64 encoding is wrong:
640 //final byte[] imageByteArray = rs.getBytes(rsIndex);
641 //if (imageByteArray != null) {
642 // String base64 = org.apache.commons.codec.binary.Base64.encodeBase64URLSafeString(imageByteArray);
643 // base64 = "data:image/png;base64," + base64;
645 final String url = Utils.buildImageDataUrl(primaryKeyValue, documentID, tableName, field);
646 dataItem.setImageDataUrl(url);
650 Log.warn(documentID, tableName, "Invalid LayoutItem Field type. Using empty string for value.");
651 dataItem.setText("");