Update versions of depdencies.
[online-glom:gwt-glom.git] / src / test / java / org / glom / web / server / SelfHoster.java
1 /*
2  * Copyright (C) 2012 Openismus GmbH
3  *
4  * This file is part of GWT-Glom.
5  *
6  * GWT-Glom is free software: you can redistribute it and/or modify it
7  * under the terms of the GNU Lesser General Public License as published by the
8  * Free Software Foundation, either version 3 of the License, or (at your
9  * option) any later version.
10  *
11  * GWT-Glom is distributed in the hope that it will be useful, but WITHOUT
12  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
13  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU Lesser General Public License
14  * for more details.
15  *
16  * You should have received a copy of the GNU Lesser General Public License
17  * along with GWT-Glom.  If not, see <http://www.gnu.org/licenses/>.
18  */
19
20 package org.glom.web.server;
21
22 import java.io.File;
23 import java.io.FileNotFoundException;
24 import java.io.FileOutputStream;
25 import java.io.IOException;
26 import java.net.ServerSocket;
27 import java.sql.Connection;
28 import java.sql.DriverManager;
29 import java.sql.SQLException;
30 import java.util.ArrayList;
31 import java.util.List;
32 import java.util.Locale;
33 import java.util.Map;
34 import java.util.Map.Entry;
35 import java.util.Properties;
36
37 import org.apache.commons.lang3.StringUtils;
38 import org.glom.web.server.libglom.Document;
39 import org.glom.web.shared.DataItem;
40 import org.glom.web.shared.libglom.Field;
41 import org.jooq.InsertResultStep;
42 import org.jooq.InsertSetStep;
43 import org.jooq.Record;
44 import org.jooq.SQLDialect;
45 import org.jooq.Table;
46 import org.jooq.exception.DataAccessException;
47 import org.jooq.impl.Factory;
48
49 import com.google.common.io.Files;
50 import com.ibm.icu.text.NumberFormat;
51
52 /**
53  * @author Murray Cumming <murrayc@murrayc.com>
54  * 
55  */
56 public class SelfHoster {
57         // private String tempFilepathDir = "";
58         private boolean selfHostingActive = false;
59         private Document document = null;
60         private String username = "";
61         private String password = "";
62
63         SelfHoster(final Document document) {
64                 this.document = document;
65         }
66
67         private static final int PORT_POSTGRESQL_SELF_HOSTED_START = 5433;
68         private static final int PORT_POSTGRESQL_SELF_HOSTED_END = 5500;
69
70         private static final String DEFAULT_CONFIG_PG_HBA_LOCAL_8p4 = "# TYPE  DATABASE    USER        CIDR-ADDRESS          METHOD\n"
71                         + "\n"
72                         + "# local is for Unix domain socket connections only\n"
73                         + "# trust allows connection from the current PC without a password:\n"
74                         + "local   all         all                               trust\n"
75                         + "local   all         all                               ident\n"
76                         + "local   all         all                               md5\n"
77                         + "\n"
78                         + "# TCP connections from the same computer, with a password:\n"
79                         + "host    all         all         127.0.0.1    255.255.255.255    md5\n"
80                         + "# IPv6 local connections:\n"
81                         + "host    all         all         ::1/128               md5\n";
82
83         private static final String DEFAULT_CONFIG_PG_IDENT = "";
84         private static final String FILENAME_DATA = "data";
85
86         public boolean createAndSelfHostFromExample(final Document.HostingMode hostingMode) {
87
88                 if (!createAndSelfHostNewEmpty(hostingMode)) {
89                         // std::cerr << G_STRFUNC << ": test_create_and_selfhost_new_empty() failed." << std::endl;
90                         return false;
91                 }
92
93                 final boolean recreated = recreateDatabaseFromDocument(); /* TODO: Progress callback */
94                 if (!recreated) {
95                         if (!cleanup()) {
96                                 return false;
97                         }
98                 }
99
100                 return recreated;
101         }
102
103         /**
104          * @param document
105          * @param
106          * @param subDirectoryPath
107          * @return
108          */
109         private boolean createAndSelfHostNewEmpty(final Document.HostingMode hostingMode) {
110                 if (hostingMode != Document.HostingMode.HOSTING_MODE_POSTGRES_SELF) {
111                         // TODO: std::cerr << G_STRFUNC << ": This test function does not support the specified hosting_mode: " <<
112                         // hosting_mode << std::endl;
113                         return false;
114                 }
115
116                 // Save a copy, specifying the path to file in a directory:
117                 // For instance, /tmp/testglom/testglom.glom");
118                 final String tempFilename = "testglom";
119                 final File tempFolder = Files.createTempDir();
120                 final File tempDir = new File(tempFolder, tempFilename);
121
122                 final String tempDirPath = tempDir.getPath();
123                 final String tempFilePath = tempDirPath + File.separator + tempFilename;
124                 final File file = new File(tempFilePath);
125
126                 // Make sure that the file does not exist yet:
127                 {
128                         tempDir.delete();
129                 }
130
131                 // Save the example as a real file:
132                 document.setFileURI(file.getPath());
133
134                 document.setHostingMode(hostingMode);
135                 document.setIsExampleFile(false);
136                 final boolean saved = document.save();
137                 if (!saved) {
138                         System.out.println("createAndSelfHostNewEmpty(): Document.save() failed.");
139                         return false; // TODO: Delete the directory.
140                 }
141
142                 // We must specify a default username and password:
143                 final String user = "glom_default_developer_user";
144                 final String password = "glom_default_developer_password";
145
146                 // Create the self-hosting files:
147                 if (!initialize(user, password)) {
148                         System.out.println("createAndSelfHostNewEmpty(): initialize failed.");
149                         // TODO: Delete directory.
150                 }
151
152                 // Check that it really created some files:
153                 if (!tempDir.exists()) {
154                         System.out.println("createAndSelfHostNewEmpty(): tempDir does not exist.");
155                         // TODO: Delete directory.
156                 }
157
158                 return selfHost(user, password);
159         }
160
161         /**
162          * @param document
163          * @param user
164          * @param password
165          * @return
166          */
167         private boolean selfHost(final String user, final String password) {
168                 // TODO: m_network_shared = network_shared;
169
170                 if (getSelfHostingActive()) {
171                         // TODO: std::cerr << G_STRFUNC << ": Already started." << std::endl;
172                         return false; // STARTUPERROR_NONE; //Just do it once.
173                 }
174
175                 final String dbDirData = getSelfHostingDataPath(false);
176                 if (StringUtils.isEmpty(dbDirData) || !fileExists(dbDirData)) {
177                         /*
178                          * final String dbDirBackup = dbDir + File.separator + FILENAME_BACKUP;
179                          * 
180                          * if(fileExists(dbDirBackup)) { //TODO: std::cerr << G_STRFUNC <<
181                          * ": There is no data, but there is backup data." << std::endl; //Let the caller convert the backup to real
182                          * data and then try again: return false; // STARTUPERROR_FAILED_NO_DATA_HAS_BACKUP_DATA; } else {
183                          */
184                         // TODO: std::cerr << "ConnectionPool::create_self_hosting(): The data sub-directory could not be found." <<
185                         // dbdir_data_uri << std::endl;
186                         return false; // STARTUPERROR_FAILED_NO_DATA;
187                         // }
188                 }
189
190                 final int availablePort = discoverFirstFreePort(PORT_POSTGRESQL_SELF_HOSTED_START,
191                                 PORT_POSTGRESQL_SELF_HOSTED_END);
192                 // std::cout << "debug: " << G_STRFUNC << ":() : debug: Available port for self-hosting: " << available_port <<
193                 // std::endl;
194                 if (availablePort == 0) {
195                         // TODO: Use a return enum or exception so we can tell the user about this:
196                         // TODO: std::cerr << G_STRFUNC << ": No port was available between " << PORT_POSTGRESQL_SELF_HOSTED_START
197                         // << " and " << PORT_POSTGRESQL_SELF_HOSTED_END << std::endl;
198                         return false; // STARTUPERROR_FAILED_UNKNOWN_REASON;
199                 }
200
201                 final NumberFormat format = NumberFormat.getInstance(Locale.US);
202                 format.setGroupingUsed(false); // TODO: Does this change it system-wide?
203                 final String portAsText = format.format(availablePort);
204
205                 // -D specifies the data directory.
206                 // -c config_file= specifies the configuration file
207                 // -k specifies a directory to use for the socket. This must be writable by us.
208                 // Make sure to use double quotes for the executable path, because the
209                 // CreateProcess() API used on Windows does not support single quotes.
210                 final String dbDir = getSelfHostingPath("", false);
211                 final String dbDirConfig = getSelfHostingPath("config", false);
212                 final String dbDirHba = dbDirConfig + File.separator + "pg_hba.conf";
213                 final String dbDirIdent = dbDirConfig + File.separator + "pg_ident.conf";
214                 final String dbDirPid = getSelfHostingPath("pid", false);
215
216                 // Note that postgres returns this error if we split the arguments more,
217                 // for instance splitting -D and dbDirData into separate strings:
218                 // too many command-line arguments (first is "(null)")
219                 // Note: If we use "-D " instead of "-D" then the initdb seems to make the space part of the filepath,
220                 // though that does not happen with the normal command line.
221                 // However, we must have a space after -k.
222                 // Also, the c hba_file=path argument must be split after -c, or postgres will get a " hba_file" configuration
223                 // parameter instead of "hba_file".
224                 final String commandPathStart = getPathToPostgresExecutable("postgres");
225                 if (StringUtils.isEmpty(commandPathStart)) {
226                         System.out.println("selfHost(): getPathToPostgresExecutable(postgres) failed.");
227                         return false;
228                 }
229                 final ProcessBuilder commandPostgresStart = new ProcessBuilder(commandPathStart, "-D" + shellQuote(dbDirData),
230                                 "-p", portAsText, "-i", // Equivalent to -h "*", which in turn is equivalent
231                                                                                 // to
232                                 // listen_addresses in postgresql.conf. Listen to all IP addresses,
233                                 // so any client can connect (with a username+password)
234                                 "-c", "hba_file=" + shellQuote(dbDirHba), "-c", "ident_file=" + shellQuote(dbDirIdent), "-k"
235                                                 + shellQuote(dbDir), "--external_pid_file=" + shellQuote(dbDirPid));
236                 // std::cout << G_STRFUNC << ": debug: " << command_postgres_start << std::endl;
237
238                 // Make sure to use double quotes for the executable path, because the
239                 // CreateProcess() API used on Windows does not support single quotes.
240                 //
241                 // Note that postgres returns this error if we split the arguments more,
242                 // for instance splitting -D and dbDirData into separate strings:
243                 // too many command-line arguments (first is "(null)")
244                 // Note: If we use "-D " instead of "-D" then the initdb seems to make the space part of the filepath,
245                 // though that does not happen with the normal command line.
246                 final String commandPathCheck = getPathToPostgresExecutable("pg_ctl");
247                 if (StringUtils.isEmpty(commandPathCheck)) {
248                         System.out.println("selfHost(): getPathToPostgresExecutable(pg_ctl) failed.");
249                         return false;
250                 }
251                 final ProcessBuilder commandCheckPostgresHasStarted = new ProcessBuilder(commandPathCheck, "status", "-D"
252                                 + shellQuote(dbDirData));
253
254                 // For postgres 8.1, this is "postmaster is running".
255                 // For postgres 8.2, this is "server is running".
256                 // This is a big hack that we should avoid. murrayc.
257                 //
258                 // pg_ctl actually seems to return a 0 result code for "is running" and a 1 for not running, at least with
259                 // Postgres 8.2,
260                 // so maybe we can avoid this in future.
261                 // Please do test it with your postgres version, using "echo $?" to see the result code of the last command.
262                 final String secondCommandSuccessText = "is running"; // TODO: This is not a stable API. Also, watch out for
263                                                                                                                                 // localisation.
264
265                 // The first command does not return, but the second command can check whether it succeeded:
266                 // TODO: Progress
267                 final boolean result = executeCommandLineAndWaitUntilSecondCommandReturnsSuccess(commandPostgresStart,
268                                 commandCheckPostgresHasStarted, secondCommandSuccessText);
269                 if (!result) {
270                         System.out.println("selfHost(): Error while attempting to self-host a database.");
271                         return false; // STARTUPERROR_FAILED_UNKNOWN_REASON;
272                 }
273
274                 // Remember the port for later:
275                 document.setConnectionPort(availablePort);
276
277                 // Check that we can really connect:
278                 
279                 //Sleep for a fairly long time initially to avoid distracting error messages when trying to connect,
280                 //while the database server is still starting up.
281                 try {
282                         Thread.sleep(5000);
283                 } catch (InterruptedException e) {
284                         // TODO Auto-generated catch block
285                         e.printStackTrace();
286                 }
287
288                 // pg_ctl sometimes reports success before it is really ready to let us connect,
289                 // so in this case we can just keep trying until it works, for a while:
290                 for (int i = 0; i < 10; i++) {
291
292                         try {
293                                 Thread.sleep(1000);
294                         } catch (InterruptedException e) {
295                                 // TODO Auto-generated catch block
296                                 e.printStackTrace();
297                         }
298
299                         final String dbName = document.getConnectionDatabase();
300                         document.setConnectionDatabase(""); // We have not created the database yet.
301
302                         //Check that we can connect:
303                         final Connection connection = createConnection(false);
304                         document.setConnectionDatabase(dbName);
305                         if (connection != null) {
306                                 //Close the connection:
307                                 try {
308                                         connection.close();
309                                 } catch (SQLException e) {
310                                         // TODO Auto-generated catch block
311                                         e.printStackTrace();
312                                 }
313
314                                 System.out.println("selfHost(): Connection succeeded after retries=" + i);
315                                 return true; // STARTUPERROR_NONE;
316                         }
317
318                         System.out
319                                         .println("selfHost(): Waiting and retrying the connection due to suspected too-early success of pg_ctl. retries="
320                                                         + i);
321                 }
322
323                 System.out.println("selfHost(): Test connection failed after multiple retries.");
324                 return false;
325         }
326
327         /**
328          * @param dbDirData
329          * @return
330          */
331         private String shellQuote(final String str) {
332                 // TODO: If we add the quotes then they seem to be used as part of the path, though that is not a problem with
333                 // the normal command line.
334                 return str;
335
336                 // TODO: Escape.
337                 // return "'" + str + "'";
338         }
339
340         private String getSelfHostingPath(final String subpath, final boolean create) {
341                 final String dbDir = document.getSelfHostedDirectoryPath();
342                 if (StringUtils.isEmpty(subpath)) {
343                         return dbDir;
344                 }
345
346                 final String dbDirData = dbDir + File.separator + subpath;
347                 final File file = new File(dbDirData);
348
349                 // Return the path regardless of whether it exists:
350                 if (!create) {
351                         return dbDirData;
352                 }
353
354                 if (!file.exists()) {
355                         try {
356                                 Files.createParentDirs(file);
357                         } catch (final IOException e) {
358                                 // TODO Auto-generated catch block
359                                 e.printStackTrace();
360                                 return "";
361                         }
362
363                         if (!file.mkdir()) {
364                                 return "";
365                         }
366                 }
367
368                 return dbDirData;
369         }
370
371         private String getSelfHostingDataPath(final boolean create) {
372                 return getSelfHostingPath(FILENAME_DATA, create);
373         }
374
375         private boolean executeCommandLineAndWait(final ProcessBuilder command) {
376
377                 command.redirectErrorStream(true);
378
379                 // Run the first command, and wait for it to return:
380                 Process process = null;
381                 try {
382                         process = command.start();
383                 } catch (final IOException e) {
384                         // TODO Auto-generated catch block
385                         e.printStackTrace();
386                         return false;
387                 }
388
389                 // final InputStream stderr = process.getInputStream();
390                 // final InputStreamReader isr = new InputStreamReader(stderr);
391                 // final BufferedReader br = new BufferedReader(isr);
392                 // String output = "";
393                 // String line;
394                 /*
395                  * try { //TODO: readLine() can hang, waiting for an end of line that never comes. while ((line = br.readLine())
396                  * != null) { output += line + "\n"; } } catch (final IOException e1) { e1.printStackTrace(); return false; }
397                  */
398
399                 int result = 0;
400                 try {
401                         result = process.waitFor();
402                 } catch (final InterruptedException e) {
403                         // TODO Auto-generated catch block
404                         e.printStackTrace();
405                         return false;
406                 }
407
408                 if (result != 0) {
409                         System.out.println("executeCommandLineAndWait(): Command failed: " + command.command().toString());
410                         // System.out.print("  Output: " + output);
411                         return false;
412                 }
413
414                 return true;
415         }
416
417         private boolean executeCommandLineAndWaitUntilSecondCommandReturnsSuccess(final ProcessBuilder command,
418                         final ProcessBuilder commandSecond, final String secondCommandSuccessText) {
419                 command.redirectErrorStream(true);
420
421                 // Run the first command, and do not wait for it to return:
422                 // Process process = null;
423                 try {
424                         // Process process =
425                         command.start();
426                 } catch (final IOException e) {
427                         // TODO Auto-generated catch block
428                         e.printStackTrace();
429                         return false;
430                 }
431
432                 // final InputStream stderr = process.getInputStream();
433                 // final InputStreamReader isr = new InputStreamReader(stderr);
434                 // final BufferedReader br = new BufferedReader(isr);
435
436                 /*
437                  * We do not wait, because this (postgres, for instance), does not return: final int result = process.waitFor();
438                  * if (result != 0) { // TODO: Warn. return false; }
439                  */
440
441                 // Now run the second command, usually to verify that the first command has really done its work:
442                 // We run this repeatedly until it succeeds, to show that the first command has finished.
443                 boolean result = false;
444                 while (true) {
445                         result = executeCommandLineAndWait(commandSecond);
446                         if (result) {
447                                 System.out.println("executeCommandLineAndWait(): second command succeeded.");
448                                 return true;
449                         } else {
450                                 try {
451                                         Thread.sleep(1000);
452                                 } catch (InterruptedException e) {
453                                         // TODO Auto-generated catch block
454                                         e.printStackTrace();
455                                         return false;
456                                 }
457
458                                 System.out.println("executeCommandLineAndWait(): Trying the second command again.");
459                         }
460                 }
461
462                 // Try to get the output:
463                 /*
464                  * if (!result) { String output = ""; /* String line; try { // TODO: readLine() can hang, waiting for an end of
465                  * line that never comes. while ((line = br.readLine()) != null) { output += line + "\n";
466                  * System.out.println(line); } } catch (final IOException e1) { // TODO Auto-generated catch block
467                  * e1.printStackTrace(); return false; }
468                  */
469
470                 // System.out.println("  Output of first command: " + output);
471                 // System.out.println("  first command: " + command.command().toString());
472                 // System.out.println("  second command: " + commandSecond.command().toString());
473                 // }
474         }
475
476         /**
477          * @param string
478          * @return
479          */
480         private static String getPathToPostgresExecutable(final String string) {
481                 final List<String> dirPaths = new ArrayList<String>();
482                 dirPaths.add("/usr/bin");
483                 dirPaths.add("/usr/lib/postgresql/9.1/bin");
484                 dirPaths.add("/usr/lib/postgresql/9.0/bin");
485                 dirPaths.add("/usr/lib/postgresql/8.4/bin");
486
487                 for (String dir : dirPaths) {
488                         final String path = dir + File.separator + string;
489                         if (fileExistsAndIsExecutable(path)) {
490                                 return path;
491                         }
492                 }
493
494                 return "";
495         }
496
497         /**
498          * @param path
499          * @return
500          */
501         private static boolean fileExistsAndIsExecutable(String path) {
502                 final File file = new File(path);
503                 if (!file.exists()) {
504                         return false;
505                 }
506
507                 if (!file.canExecute()) {
508                         return false;
509                 }
510
511                 return true;
512         }
513
514         /**
515          * @param start
516          * @param end
517          * @return
518          */
519         private static int discoverFirstFreePort(final int start, final int end) {
520                 for (int port = start; port <= end; ++port) {
521                         try {
522                                 final ServerSocket socket = new ServerSocket(port);
523
524                                 // If the instantiation succeeded then the port was free:
525                                 final int result = socket.getLocalPort(); // This must equal port.
526                                 socket.close();
527                                 return result;
528                         } catch (final IOException ex) {
529                                 continue; // try next port
530                         }
531                 }
532
533                 return 0;
534         }
535
536         /**
537          * @param dbDir
538          * @return
539          */
540         private static boolean fileExists(final String filePath) {
541                 final File file = new File(filePath);
542                 return file.exists();
543         }
544
545         /**
546          * @return
547          */
548         private boolean getSelfHostingActive() {
549                 return selfHostingActive;
550         }
551
552         /**
553          * @param cpds
554          * @return
555          */
556         private boolean initialize(final String initialUsername, final String initialPassword) {
557                 if (!initializeConfFiles()) {
558                         System.out.println("initialize(): initializeConfFiles() failed.");
559                         return false;
560                 }
561
562                 // initdb creates a new postgres database cluster:
563
564                 // Get file:// URI for the tmp/ directory:
565                 File filePwFile = null;
566                 try {
567                         filePwFile = File.createTempFile("glom_initdb_pwfile", "");
568                 } catch (final IOException e) {
569                         // TODO Auto-generated catch block
570                         e.printStackTrace();
571                 }
572                 final String tempPwFile = filePwFile.getPath();
573
574                 final boolean pwfileCreationSucceeded = createTextFile(tempPwFile, initialPassword);
575                 if (!pwfileCreationSucceeded) {
576                         System.out.println("initialize(): createTextFile() failed.");
577                         return false;
578                 }
579
580                 // Make sure to use double quotes for the executable path, because the
581                 // CreateProcess() API used on Windows does not support single quotes.
582                 final String dbDirData = getSelfHostingDataPath(false /* create */);
583
584                 // Note that initdb returns this error if we split the arguments more,
585                 // for instance splitting -D and dbDirData into separate strings:
586                 // too many command-line arguments (first is "(null)")
587                 // TODO: If we quote tempPwFile then initdb says that it cannot find it.
588                 // Note: If we use "-D " instead of "-D" then the initdb seems to make the space part of the filepath,
589                 // though that does not happen with the normal command line.
590                 boolean result = false;
591                 final String commandPath = getPathToPostgresExecutable("initdb");
592                 if (StringUtils.isEmpty(commandPath)) {
593                         System.out.println("initialize(): getPathToPostgresExecutable(initdb) failed.");
594                 } else {
595                         final ProcessBuilder commandInitdb = new ProcessBuilder(commandPath, "-D" + shellQuote(dbDirData), "-U",
596                                         initialUsername, "--pwfile=" + tempPwFile);
597
598                         // Note that --pwfile takes the password from the first line of a file. It's an alternative to supplying it
599                         // when
600                         // prompted on stdin.
601                         result = executeCommandLineAndWait(commandInitdb);
602                 }
603
604                 // Of course, we don't want this to stay around. It would be a security risk.
605                 final File fileTempPwFile = new File(tempPwFile);
606                 if (!fileTempPwFile.delete()) {
607                         System.out.println("initialize(): Failed to delete the password file.");
608                 }
609
610                 if (!result) {
611                         System.out.println("initialize(): Error while attempting to create self-hosting database.");
612                         return false;
613                 }
614
615                 // Save the username and password for later;
616                 this.username = initialUsername;
617                 this.password = initialPassword;
618
619                 return result; // ? INITERROR_NONE : INITERROR_COULD_NOT_START_SERVER;
620
621         }
622
623         private boolean initializeConfFiles() {
624                 final String dataDirPath = document.getSelfHostedDirectoryPath();
625
626                 final String dbDirConfig = dataDirPath + File.separator + "config";
627                 // String defaultConfContents = "";
628
629                 // Choose the configuration contents based on the postgresql version
630                 // and whether we want to be network-shared:
631                 // final float postgresqlVersion = 9.0f; //TODO: get_postgresql_utils_version_as_number(slot_progress);
632                 // final boolean networkShared = true;
633                 // std::cout << "DEBUG: postgresql_version=" << postgresql_version << std::endl;
634
635                 // TODO: Support the other configurations, as in libglom.
636                 final String defaultConfContents = DEFAULT_CONFIG_PG_HBA_LOCAL_8p4;
637
638                 // std::cout << "DEBUG: default_conf_contents=" << default_conf_contents << std::endl;
639
640                 final boolean hbaConfCreationSucceeded = createTextFile(dbDirConfig + File.separator + "pg_hba.conf",
641                                 defaultConfContents);
642                 if (!hbaConfCreationSucceeded) {
643                         System.out.println("initialize(): createTextFile() failed.");
644                         return false;
645                 }
646
647                 final boolean identConfCreationSucceeded = createTextFile(dbDirConfig + File.separator + "pg_ident.conf",
648                                 DEFAULT_CONFIG_PG_IDENT);
649                 if (!identConfCreationSucceeded) {
650                         System.out.println("initialize(): createTextFile() failed.");
651                         return false;
652                 }
653
654                 return true;
655         }
656
657         /**
658          * @param path
659          * @param contents
660          * @return
661          */
662         private static boolean createTextFile(final String path, final String contents) {
663                 final File file = new File(path);
664                 final File parent = file.getParentFile();
665                 if (parent == null) {
666                         System.out.println("initialize(): getParentFile() failed.");
667                         return false;
668                 }
669
670                 parent.mkdirs();
671                 try {
672                         file.createNewFile();
673                 } catch (final IOException e) {
674                         // TODO Auto-generated catch block
675                         e.printStackTrace();
676                         return false;
677                 }
678
679                 FileOutputStream output = null;
680                 try {
681                         output = new FileOutputStream(file);
682                 } catch (final FileNotFoundException e) {
683                         // TODO Auto-generated catch block
684                         e.printStackTrace();
685                         return false;
686                 }
687
688                 try {
689                         output.write(contents.getBytes());
690                 } catch (final IOException e) {
691                         // TODO Auto-generated catch block
692                         e.printStackTrace();
693
694                         //TODO: Avoid the duplicate close() here.
695                         try {
696                                 output.close();
697                         } catch (IOException e1) {
698                                 // TODO Auto-generated catch block
699                                 e1.printStackTrace();
700                         }
701
702                         return false;
703                 }
704
705                 try {
706                         output.close();
707                 } catch (IOException e) {
708                         // TODO Auto-generated catch block
709                         e.printStackTrace();
710                 }
711
712                 return true;
713         }
714
715         /**
716          * @param document
717          * @return
718          */
719         private boolean recreateDatabaseFromDocument() {
720                 // Check whether the database exists already.
721                 final String dbName = document.getConnectionDatabase();
722                 if (StringUtils.isEmpty(dbName)) {
723                         return false;
724                 }
725
726                 document.setConnectionDatabase(dbName);
727                 Connection connection = createConnection(true);
728                 if (connection != null) {
729                         // Connection to the database succeeded, so the database
730                         // exists already.
731                         try {
732                                 connection.close();
733                         } catch (final SQLException e) {
734                                 // TODO Auto-generated catch block
735                                 e.printStackTrace();
736                         }
737                         return false;
738                 }
739
740                 // Create the database:
741                 progress();
742                 document.setConnectionDatabase("");
743
744                 connection = createConnection(false);
745                 if (connection == null) {
746                         System.out.println("recreatedDatabase(): createConnection() failed, before creating the database.");
747                         return false;
748                 }
749
750                 final boolean dbCreated = createDatabase(connection, dbName);
751
752                 if (!dbCreated) {
753                         return false;
754                 }
755
756                 progress();
757
758                 // Check that we can connect:
759                 try {
760                         connection.close();
761                 } catch (final SQLException e) {
762                         // TODO Auto-generated catch block
763                         e.printStackTrace();
764                 }
765                 connection = null;
766
767                 document.setConnectionDatabase(dbName);
768                 connection = createConnection(false);
769                 if (connection == null) {
770                         System.out.println("recreatedDatabase(): createConnection() failed, after creating the database.");
771                         return false;
772                 }
773
774                 progress();
775
776                 // Create each table:
777                 final List<String> tables = document.getTableNames();
778                 for (final String tableName : tables) {
779
780                         // Create SQL to describe all fields in this table:
781                         final List<Field> fields = document.getTableFields(tableName);
782
783                         progress();
784                         final boolean tableCreationSucceeded = createTable(connection, document, tableName, fields);
785                         progress();
786                         if (!tableCreationSucceeded) {
787                                 // TODO: std::cerr << G_STRFUNC << ": CREATE TABLE failed with the newly-created database." <<
788                                 // std::endl;
789                                 return false;
790                         }
791                 }
792
793                 // Note that create_database() has already called add_standard_tables() and add_standard_groups(document).
794
795                 // Add groups from the document:
796                 progress();
797                 if (!addGroupsFromDocument(document)) {
798                         // TODO: std::cerr << G_STRFUNC << ": add_groups_from_document() failed." << std::endl;
799                         return false;
800                 }
801
802                 // Set table privileges, using the groups we just added:
803                 progress();
804                 if (!setTablePrivilegesGroupsFromDocument(document)) {
805                         // TODO: std::cerr << G_STRFUNC << ": set_table_privileges_groups_from_document() failed." << std::endl;
806                         return false;
807                 }
808
809                 for (final String tableName : tables) {
810                         // Add any example data to the table:
811                         progress();
812
813                         // try
814                         // {
815                         progress();
816                         final boolean tableInsertSucceeded = insertExampleData(connection, document, tableName);
817
818                         if (!tableInsertSucceeded) {
819                                 // TODO: std::cerr << G_STRFUNC << ": INSERT of example data failed with the newly-created database." <<
820                                 // std::endl;
821                                 return false;
822                         }
823                         // }
824                         // catch(final std::exception& ex)
825                         // {
826                         // std::cerr << G_STRFUNC << ": exception: " << ex.what() << std::endl;
827                         // HandleError(ex);
828                         // }
829
830                 } // for(tables)
831
832                 return true; // All tables created successfully.
833         }
834
835         /**
836          */
837         public Connection createConnection(boolean failureExpected) {
838                 final Properties connectionProps = new Properties();
839                 connectionProps.put("user", this.username);
840                 connectionProps.put("password", this.password);
841
842                 String jdbcURL = "jdbc:postgresql://" + document.getConnectionServer() + ":" + document.getConnectionPort();
843                 String db = document.getConnectionDatabase();
844                 if (StringUtils.isEmpty(db)) {
845                         // Use the default PostgreSQL database, because ComboPooledDataSource.connect() fails otherwise.
846                         db = "template1";
847                 }
848                 jdbcURL += "/" + db; // TODO: Quote the database name?
849
850                 Connection conn = null;
851                 try {
852                         //TODO: Remove these debug prints when we figure out why getConnection sometimes hangs. 
853                         //System.out.println("debug: SelfHoster.createConnection(): before createConnection()");
854                         DriverManager.setLoginTimeout(10);
855                         conn = DriverManager.getConnection(jdbcURL + "/", connectionProps);
856                         //System.out.println("debug: SelfHoster.createConnection(): before createConnection()");
857                 } catch (final SQLException e) {
858                         if(!failureExpected) {
859                                 e.printStackTrace();
860                         }
861                         return null;
862                 }
863
864                 return conn;
865         }
866
867         /**
868          *
869          */
870         private void progress() {
871                 // TODO Auto-generated method stub
872
873         }
874
875         /**
876          * @param document
877          * @param tableName
878          * @return
879          */
880         private boolean insertExampleData(final Connection connection, final Document document, final String tableName) {
881
882                 final Factory factory = new Factory(connection, SQLDialect.POSTGRES);
883                 final Table<Record> table = Factory.tableByName(tableName);
884
885                 final List<Map<String, DataItem>> exampleRows = document.getExampleRows(tableName);
886                 for (final Map<String, DataItem> row : exampleRows) {
887                         InsertSetStep<Record> insertStep = factory.insertInto(table);
888
889                         for (final Entry<String, DataItem> entry : row.entrySet()) {
890                                 final String fieldName = entry.getKey();
891                                 final DataItem value = entry.getValue();
892                                 if (value == null) {
893                                         continue;
894                                 }
895
896                                 final Field field = document.getField(tableName, fieldName);
897                                 if (field == null) {
898                                         continue;
899                                 }
900
901                                 final org.jooq.Field<Object> jooqField = Factory.fieldByName(field.getName());
902                                 if (jooqField == null) {
903                                         continue;
904                                 }
905
906                                 final Object fieldValue = value.getValue(field.getGlomType());
907                                 insertStep = insertStep.set(jooqField, fieldValue);
908                         }
909
910                         if (!(insertStep instanceof InsertResultStep<?>)) {
911                                 continue;
912                         }
913
914                         // We suppress the warning because we _do_ check the cast above.
915                         @SuppressWarnings("unchecked")
916                         final InsertResultStep<Record> insertResultStep = (InsertResultStep<Record>) insertStep;
917
918                         try {
919                                 insertResultStep.fetchOne();
920                         } catch (final DataAccessException e) {
921                                 // e.printStackTrace();
922                                 return false;
923                         }
924                         // TODO: Check that it worked.
925                 }
926
927                 return true;
928         }
929
930         /**
931          * @param document2
932          * @return
933          */
934         private boolean setTablePrivilegesGroupsFromDocument(final Document document2) {
935                 // TODO Auto-generated method stub
936                 return true;
937         }
938
939         /**
940          * @param document2
941          * @return
942          */
943         private boolean addGroupsFromDocument(final Document document2) {
944                 // TODO Auto-generated method stub
945                 return true;
946         }
947
948         /**
949          * @param document
950          * @param tableName
951          * @param fields
952          * @return
953          */
954         private boolean createTable(final Connection connection, final Document document, final String tableName,
955                         final List<Field> fields) {
956                 boolean tableCreationSucceeded = false;
957
958                 /*
959                  * TODO: //Create the standard field too: //(We don't actually use this yet) if(std::find_if(fields.begin(),
960                  * fields.end(), predicate_FieldHasName<Field>(GLOM_STANDARD_FIELD_LOCK)) == fields.end()) { sharedptr<Field>
961                  * field = sharedptr<Field>::create(); field->set_name(GLOM_STANDARD_FIELD_LOCK);
962                  * field->set_glom_type(Field::TYPE_TEXT); fields.push_back(field); }
963                  */
964
965                 // Create SQL to describe all fields in this table:
966                 String sqlFields = "";
967                 for (final Field field : fields) {
968                         // Create SQL to describe this field:
969                         String sqlFieldDescription = escapeSqlId(field.getName()) + " " + field.getSqlType();
970
971                         if (field.getPrimaryKey()) {
972                                 sqlFieldDescription += " NOT NULL  PRIMARY KEY";
973                         }
974
975                         // Append it:
976                         if (!StringUtils.isEmpty(sqlFields)) {
977                                 sqlFields += ", ";
978                         }
979
980                         sqlFields += sqlFieldDescription;
981                 }
982
983                 if (StringUtils.isEmpty(sqlFields)) {
984                         // TODO: std::cerr << G_STRFUNC << ": sql_fields is empty." << std::endl;
985                 }
986
987                 // Actually create the table
988                 final String query = "CREATE TABLE " + escapeSqlId(tableName) + " (" + sqlFields + ");";
989                 final Factory factory = new Factory(connection, SQLDialect.POSTGRES);
990                 factory.execute(query);
991                 tableCreationSucceeded = true;
992                 if (!tableCreationSucceeded) {
993                         System.out.println("recreatedDatabase(): CREATE TABLE() failed.");
994                 }
995
996                 return tableCreationSucceeded;
997         }
998
999         /**
1000          * @param name
1001          * @return
1002          */
1003         private String escapeSqlId(final String name) {
1004                 // TODO:
1005                 return "\"" + name + "\"";
1006         }
1007
1008         /**
1009          * @return
1010          */
1011         private static boolean createDatabase(final Connection connection, final String databaseName) {
1012
1013                 final String query = "CREATE DATABASE \"" + databaseName + "\""; // TODO: Escaping.
1014                 final Factory factory = new Factory(connection, SQLDialect.POSTGRES);
1015
1016                 factory.execute(query);
1017
1018                 return true;
1019         }
1020
1021         /**
1022          *
1023          */
1024         public boolean cleanup() {
1025                 boolean result = true;
1026
1027                 // Stop the server:
1028                 if ((document != null) && (document.getConnectionPort() != 0)) {
1029                         final String dbDirData = getSelfHostingDataPath(false);
1030
1031                         // -D specifies the data directory.
1032                         // -c config_file= specifies the configuration file
1033                         // -k specifies a directory to use for the socket. This must be writable by us.
1034                         // We use "-m fast" instead of the default "-m smart" because that waits for clients to disconnect (and
1035                         // sometimes never succeeds).
1036                         // TODO: Warn about connected clients on other computers? Warn those other users?
1037                         // Make sure to use double quotes for the executable path, because the
1038                         // CreateProcess() API used on Windows does not support single quotes.
1039                         final String commandPath = getPathToPostgresExecutable("pg_ctl");
1040                         if (StringUtils.isEmpty(commandPath)) {
1041                                 System.out.println("cleanup(): getPathToPostgresExecutable(pg_ctl) failed.");
1042                         } else {
1043                                 final ProcessBuilder commandPostgresStop = new ProcessBuilder(commandPath,
1044                                                 "-D" + shellQuote(dbDirData), "stop", "-m", "fast");
1045                                 result = executeCommandLineAndWait(commandPostgresStop);
1046                                 if (!result) {
1047                                         System.out.println("cleanup(): Failed to stop the PostgreSQL server.");
1048                                 }
1049                         }
1050
1051                         document.setConnectionPort(0);
1052                 }
1053
1054                 // Delete the files:
1055                 final String selfhostingPath = getSelfHostingPath("", false);
1056                 final File fileSelfHosting = new File(selfhostingPath);
1057                 fileSelfHosting.delete();
1058
1059                 final String docPath = document.getFileURI();
1060                 final File fileDoc = new File(docPath);
1061                 fileDoc.delete();
1062
1063                 return result;
1064         }
1065         
1066         public String getUsername() {
1067                 return username;
1068         }
1069         
1070         public String getPassword() {
1071                 return password;
1072         }
1073 }