soup-cache.c: Replaced the type of some SoupCacheEntry fields.
[libsoup-meego:libsoup-meego.git] / libsoup / soup-cache.c
1 /* -*- Mode: C; tab-width: 8; indent-tabs-mode: t; c-basic-offset: 8 -*- */
2 /*
3  * soup-cache.c
4  *
5  * Copyright (C) 2009, 2010 Igalia S.L.
6  *
7  * This library is free software; you can redistribute it and/or
8  * modify it under the terms of the GNU Library General Public
9  * License as published by the Free Software Foundation; either
10  * version 2 of the License, or (at your option) any later version.
11  *
12  * This library is distributed in the hope that it will be useful,
13  * but WITHOUT ANY WARRANTY; without even the implied warranty of
14  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
15  * Library General Public License for more details.
16  *
17  * You should have received a copy of the GNU Library General Public License
18  * along with this library; see the file COPYING.LIB.  If not, write to
19  * the Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
20  * Boston, MA 02110-1301, USA.
21  */
22
23 /* TODO:
24  * - Need to hook the feature in the sync SoupSession.
25  * - Need more tests.
26  */
27
28 #ifdef HAVE_CONFIG_H
29 #include <config.h>
30 #endif
31
32 #include <stdlib.h>
33 #include <string.h>
34 #include <gio/gio.h>
35
36 #define LIBSOUP_USE_UNSTABLE_REQUEST_API
37
38 #include "soup-cache.h"
39 #include "soup-cache-private.h"
40 #include "soup-date.h"
41 #include "soup-enum-types.h"
42 #include "soup-headers.h"
43 #include "soup-session.h"
44 #include "soup-session-feature.h"
45 #include "soup-uri.h"
46
47 static SoupSessionFeatureInterface *soup_cache_default_feature_interface;
48 static void soup_cache_session_feature_init (SoupSessionFeatureInterface *feature_interface, gpointer interface_data);
49
50 #define DEFAULT_MAX_SIZE 50 * 1024 * 1024
51 #define MAX_ENTRY_DATA_PERCENTAGE 10 /* Percentage of the total size
52                                         of the cache that can be
53                                         filled by a single entry */
54
55 /*
56  * Version 2: cache is now saved in soup.cache2. Added the version
57  * number to the beginning of the file.
58  *
59  * Version 3: added HTTP status code to the cache entries.
60  *
61  * Version 4: replaced several types.
62  *   - freshness_lifetime,corrected_initial_age,response_time: time_t -> guint32
63  *   - status_code: guint -> guint16
64  *   - hits: guint -> guint32
65  */
66 #define SOUP_CACHE_CURRENT_VERSION 4
67
68 typedef struct _SoupCacheEntry {
69         char *key;
70         char *filename;
71         guint32 freshness_lifetime;
72         gboolean must_revalidate;
73         GString *data;
74         gsize pos;
75         gsize length;
76         guint32 corrected_initial_age;
77         guint32 response_time;
78         gboolean writing;
79         gboolean dirty;
80         gboolean got_body;
81         gboolean being_validated;
82         SoupMessageHeaders *headers;
83         GOutputStream *stream;
84         GError *error;
85         guint32 hits;
86         GCancellable *cancellable;
87         guint16 status_code;
88 } SoupCacheEntry;
89
90 struct _SoupCachePrivate {
91         char *cache_dir;
92         GHashTable *cache;
93         guint n_pending;
94         SoupSession *session;
95         SoupCacheType cache_type;
96         guint size;
97         guint max_size;
98         guint max_entry_data_size; /* Computed value. Here for performance reasons */
99         GList *lru_start;
100 };
101
102 typedef struct {
103         SoupCache *cache;
104         SoupCacheEntry *entry;
105         SoupMessage *msg;
106         gulong got_chunk_handler;
107         gulong got_body_handler;
108         gulong restarted_handler;
109 } SoupCacheWritingFixture;
110
111 enum {
112         PROP_0,
113         PROP_CACHE_DIR,
114         PROP_CACHE_TYPE
115 };
116
117 #define SOUP_CACHE_GET_PRIVATE(o) (G_TYPE_INSTANCE_GET_PRIVATE ((o), SOUP_TYPE_CACHE, SoupCachePrivate))
118
119 G_DEFINE_TYPE_WITH_CODE (SoupCache, soup_cache, G_TYPE_OBJECT,
120                          G_IMPLEMENT_INTERFACE (SOUP_TYPE_SESSION_FEATURE,
121                                                 soup_cache_session_feature_init))
122
123 static gboolean soup_cache_entry_remove (SoupCache *cache, SoupCacheEntry *entry);
124 static void make_room_for_new_entry (SoupCache *cache, guint length_to_add);
125 static gboolean cache_accepts_entries_of_size (SoupCache *cache, guint length_to_add);
126
127 static SoupCacheability
128 get_cacheability (SoupCache *cache, SoupMessage *msg)
129 {
130         SoupCacheability cacheability;
131         const char *cache_control, *content_type;
132
133         /* 1. The request method must be cacheable */
134         if (msg->method == SOUP_METHOD_GET)
135                 cacheability = SOUP_CACHE_CACHEABLE;
136         else if (msg->method == SOUP_METHOD_HEAD ||
137                  msg->method == SOUP_METHOD_TRACE ||
138                  msg->method == SOUP_METHOD_CONNECT)
139                 return SOUP_CACHE_UNCACHEABLE;
140         else
141                 return (SOUP_CACHE_UNCACHEABLE | SOUP_CACHE_INVALIDATES);
142
143         content_type = soup_message_headers_get_content_type (msg->response_headers, NULL);
144         if (content_type && !g_ascii_strcasecmp (content_type, "multipart/x-mixed-replace"))
145                 return SOUP_CACHE_UNCACHEABLE;
146
147         cache_control = soup_message_headers_get (msg->response_headers, "Cache-Control");
148         if (cache_control) {
149                 GHashTable *hash;
150                 SoupCachePrivate *priv = SOUP_CACHE_GET_PRIVATE (cache);
151
152                 hash = soup_header_parse_param_list (cache_control);
153
154                 /* Shared caches MUST NOT store private resources */
155                 if (priv->cache_type == SOUP_CACHE_SHARED) {
156                         if (g_hash_table_lookup_extended (hash, "private", NULL, NULL)) {
157                                 soup_header_free_param_list (hash);
158                                 return SOUP_CACHE_UNCACHEABLE;
159                         }
160                 }
161
162                 /* 2. The 'no-store' cache directive does not appear in the
163                  * headers
164                  */
165                 if (g_hash_table_lookup_extended (hash, "no-store", NULL, NULL)) {
166                         soup_header_free_param_list (hash);
167                         return SOUP_CACHE_UNCACHEABLE;
168                 }
169
170                 /* This does not appear in section 2.1, but I think it makes
171                  * sense to check it too?
172                  */
173                 if (g_hash_table_lookup_extended (hash, "no-cache", NULL, NULL)) {
174                         soup_header_free_param_list (hash);
175                         return SOUP_CACHE_UNCACHEABLE;
176                 }
177
178                 soup_header_free_param_list (hash);
179         }
180
181         switch (msg->status_code) {
182         case SOUP_STATUS_PARTIAL_CONTENT:
183                 /* We don't cache partial responses, but they only
184                  * invalidate cached full responses if the headers
185                  * don't match.
186                  */
187                 cacheability = SOUP_CACHE_UNCACHEABLE;
188                 break;
189
190         case SOUP_STATUS_NOT_MODIFIED:
191                 /* A 304 response validates an existing cache entry */
192                 cacheability = SOUP_CACHE_VALIDATES;
193                 break;
194
195         case SOUP_STATUS_MULTIPLE_CHOICES:
196         case SOUP_STATUS_MOVED_PERMANENTLY:
197         case SOUP_STATUS_GONE:
198                 /* FIXME: cacheable unless indicated otherwise */
199                 cacheability = SOUP_CACHE_UNCACHEABLE;
200                 break;
201
202         case SOUP_STATUS_FOUND:
203         case SOUP_STATUS_TEMPORARY_REDIRECT:
204                 /* FIXME: cacheable if explicitly indicated */
205                 cacheability = SOUP_CACHE_UNCACHEABLE;
206                 break;
207
208         case SOUP_STATUS_SEE_OTHER:
209         case SOUP_STATUS_FORBIDDEN:
210         case SOUP_STATUS_NOT_FOUND:
211         case SOUP_STATUS_METHOD_NOT_ALLOWED:
212                 return (SOUP_CACHE_UNCACHEABLE | SOUP_CACHE_INVALIDATES);
213
214         default:
215                 /* Any 5xx status or any 4xx status not handled above
216                  * is uncacheable but doesn't break the cache.
217                  */
218                 if ((msg->status_code >= SOUP_STATUS_BAD_REQUEST &&
219                      msg->status_code <= SOUP_STATUS_FAILED_DEPENDENCY) ||
220                     msg->status_code >= SOUP_STATUS_INTERNAL_SERVER_ERROR)
221                         return SOUP_CACHE_UNCACHEABLE;
222
223                 /* An unrecognized 2xx, 3xx, or 4xx response breaks
224                  * the cache.
225                  */
226                 if ((msg->status_code > SOUP_STATUS_PARTIAL_CONTENT &&
227                      msg->status_code < SOUP_STATUS_MULTIPLE_CHOICES) ||
228                     (msg->status_code > SOUP_STATUS_TEMPORARY_REDIRECT &&
229                      msg->status_code < SOUP_STATUS_INTERNAL_SERVER_ERROR))
230                         return (SOUP_CACHE_UNCACHEABLE | SOUP_CACHE_INVALIDATES);
231                 break;
232         }
233
234         return cacheability;
235 }
236
237 static void
238 soup_cache_entry_free (SoupCacheEntry *entry, gboolean purge)
239 {
240         if (purge) {
241                 GFile *file = g_file_new_for_path (entry->filename);
242                 g_file_delete (file, NULL, NULL);
243                 g_object_unref (file);
244         }
245
246         g_free (entry->filename);
247         entry->filename = NULL;
248         g_free (entry->key);
249         entry->key = NULL;
250
251         if (entry->headers) {
252                 soup_message_headers_free (entry->headers);
253                 entry->headers = NULL;
254         }
255
256         if (entry->data) {
257                 g_string_free (entry->data, TRUE);
258                 entry->data = NULL;
259         }
260         if (entry->error) {
261                 g_error_free (entry->error);
262                 entry->error = NULL;
263         }
264         if (entry->cancellable) {
265                 g_object_unref (entry->cancellable);
266                 entry->cancellable = NULL;
267         }
268
269         g_slice_free (SoupCacheEntry, entry);
270 }
271
272 static void
273 copy_headers (const char *name, const char *value, SoupMessageHeaders *headers)
274 {
275         soup_message_headers_append (headers, name, value);
276 }
277
278 static char *hop_by_hop_headers[] = {"Connection", "Keep-Alive", "Proxy-Authenticate", "Proxy-Authorization", "TE", "Trailer", "Transfer-Encoding", "Upgrade"};
279
280 static void
281 copy_end_to_end_headers (SoupMessageHeaders *source, SoupMessageHeaders *destination)
282 {
283         int i;
284
285         soup_message_headers_foreach (source, (SoupMessageHeadersForeachFunc) copy_headers, destination);
286         for (i = 0; i < G_N_ELEMENTS (hop_by_hop_headers); i++)
287                 soup_message_headers_remove (destination, hop_by_hop_headers[i]);
288         soup_message_headers_clean_connection_headers (destination);
289 }
290
291 static guint
292 soup_cache_entry_get_current_age (SoupCacheEntry *entry)
293 {
294         time_t now = time (NULL);
295         time_t resident_time;
296
297         resident_time = now - entry->response_time;
298         return entry->corrected_initial_age + resident_time;
299 }
300
301 static gboolean
302 soup_cache_entry_is_fresh_enough (SoupCacheEntry *entry, gint min_fresh)
303 {
304         guint limit = (min_fresh == -1) ? soup_cache_entry_get_current_age (entry) : (guint) min_fresh;
305         return entry->freshness_lifetime > limit;
306 }
307
308 static char *
309 soup_message_get_cache_key (SoupMessage *msg)
310 {
311         SoupURI *uri = soup_message_get_uri (msg);
312         return soup_uri_to_string (uri, FALSE);
313 }
314
315 static void
316 soup_cache_entry_set_freshness (SoupCacheEntry *entry, SoupMessage *msg, SoupCache *cache)
317 {
318         const char *cache_control;
319         const char *expires, *date, *last_modified;
320
321         cache_control = soup_message_headers_get (entry->headers, "Cache-Control");
322         if (cache_control) {
323                 const char *max_age, *s_maxage;
324                 gint64 freshness_lifetime = 0;
325                 GHashTable *hash;
326                 SoupCachePrivate *priv = SOUP_CACHE_GET_PRIVATE (cache);
327
328                 hash = soup_header_parse_param_list (cache_control);
329
330                 /* Should we re-validate the entry when it goes stale */
331                 entry->must_revalidate = g_hash_table_lookup_extended (hash, "must-revalidate", NULL, NULL);
332
333                 /* Section 2.3.1 */
334                 if (priv->cache_type == SOUP_CACHE_SHARED) {
335                         s_maxage = g_hash_table_lookup (hash, "s-maxage");
336                         if (s_maxage) {
337                                 freshness_lifetime = g_ascii_strtoll (s_maxage, NULL, 10);
338                                 if (freshness_lifetime) {
339                                         /* Implies proxy-revalidate. TODO: is it true? */
340                                         entry->must_revalidate = TRUE;
341                                         soup_header_free_param_list (hash);
342                                         return;
343                                 }
344                         }
345                 }
346
347                 /* If 'max-age' cache directive is present, use that */
348                 max_age = g_hash_table_lookup (hash, "max-age");
349                 if (max_age)
350                         freshness_lifetime = g_ascii_strtoll (max_age, NULL, 10);
351
352                 if (freshness_lifetime) {
353                         entry->freshness_lifetime = (guint32) MIN (freshness_lifetime, G_MAXUINT32);
354                         soup_header_free_param_list (hash);
355                         return;
356                 }
357
358                 soup_header_free_param_list (hash);
359         }
360
361         /* If the 'Expires' response header is present, use its value
362          * minus the value of the 'Date' response header
363          */
364         expires = soup_message_headers_get (entry->headers, "Expires");
365         date = soup_message_headers_get (entry->headers, "Date");
366         if (expires && date) {
367                 SoupDate *expires_d, *date_d;
368                 time_t expires_t, date_t;
369
370                 expires_d = soup_date_new_from_string (expires);
371                 if (expires_d) {
372                         date_d = soup_date_new_from_string (date);
373
374                         expires_t = soup_date_to_time_t (expires_d);
375                         date_t = soup_date_to_time_t (date_d);
376
377                         soup_date_free (expires_d);
378                         soup_date_free (date_d);
379
380                         if (expires_t && date_t) {
381                                 entry->freshness_lifetime = (guint32) MAX (expires_t - date_t, 0);
382                                 return;
383                         }
384                 } else {
385                         /* If Expires is not a valid date we should
386                            treat it as already expired, see section
387                            3.3 */
388                         entry->freshness_lifetime = 0;
389                         return;
390                 }
391         }
392
393         /* Otherwise an heuristic may be used */
394
395         /* Heuristics MUST NOT be used with these status codes
396            (section 2.3.1.1) */
397         if (msg->status_code != SOUP_STATUS_OK &&
398             msg->status_code != SOUP_STATUS_NON_AUTHORITATIVE &&
399             msg->status_code != SOUP_STATUS_PARTIAL_CONTENT &&
400             msg->status_code != SOUP_STATUS_MULTIPLE_CHOICES &&
401             msg->status_code != SOUP_STATUS_MOVED_PERMANENTLY &&
402             msg->status_code != SOUP_STATUS_GONE)
403                 goto expire;
404
405         /* TODO: attach warning 113 if response's current_age is more
406            than 24h (section 2.3.1.1) when using heuristics */
407
408         /* Last-Modified based heuristic */
409         last_modified = soup_message_headers_get (entry->headers, "Last-Modified");
410         if (last_modified) {
411                 SoupDate *soup_date;
412                 time_t now, last_modified_t;
413
414                 soup_date = soup_date_new_from_string (last_modified);
415                 last_modified_t = soup_date_to_time_t (soup_date);
416                 now = time (NULL);
417
418 #define HEURISTIC_FACTOR 0.1 /* From Section 2.3.1.1 */
419
420                 entry->freshness_lifetime = MAX (0, (now - last_modified_t) * HEURISTIC_FACTOR);
421                 soup_date_free (soup_date);
422         }
423
424         return;
425
426  expire:
427         /* If all else fails, make the entry expire immediately */
428         entry->freshness_lifetime = 0;
429 }
430
431 static SoupCacheEntry *
432 soup_cache_entry_new (SoupCache *cache, SoupMessage *msg, time_t request_time, time_t response_time)
433 {
434         SoupCacheEntry *entry;
435         const char *date;
436         char *md5;
437
438         entry = g_slice_new0 (SoupCacheEntry);
439         entry->dirty = FALSE;
440         entry->writing = FALSE;
441         entry->got_body = FALSE;
442         entry->being_validated = FALSE;
443         entry->data = g_string_new (NULL);
444         entry->pos = 0;
445         entry->error = NULL;
446         entry->status_code = msg->status_code;
447
448         /* key & filename */
449         entry->key = soup_message_get_cache_key (msg);
450         md5 = g_compute_checksum_for_string (G_CHECKSUM_MD5, entry->key, -1);
451         entry->filename = g_build_filename (cache->priv->cache_dir, md5, NULL);
452         g_free (md5);
453
454         /* Headers */
455         entry->headers = soup_message_headers_new (SOUP_MESSAGE_HEADERS_RESPONSE);
456         copy_end_to_end_headers (msg->response_headers, entry->headers);
457
458         /* LRU list */
459         entry->hits = 0;
460
461         /* Section 2.3.1, Freshness Lifetime */
462         soup_cache_entry_set_freshness (entry, msg, cache);
463
464         /* Section 2.3.2, Calculating Age */
465         date = soup_message_headers_get (entry->headers, "Date");
466
467         if (date) {
468                 SoupDate *soup_date;
469                 const char *age;
470                 time_t date_value, apparent_age, corrected_received_age, response_delay, age_value = 0;
471
472                 soup_date = soup_date_new_from_string (date);
473                 date_value = soup_date_to_time_t (soup_date);
474                 soup_date_free (soup_date);
475
476                 age = soup_message_headers_get (entry->headers, "Age");
477                 if (age)
478                         age_value = g_ascii_strtoll (age, NULL, 10);
479
480                 entry->response_time = response_time;
481                 apparent_age = MAX (0, entry->response_time - date_value);
482                 corrected_received_age = MAX (apparent_age, age_value);
483                 response_delay = entry->response_time - request_time;
484                 entry->corrected_initial_age = corrected_received_age + response_delay;
485         } else {
486                 /* Is this correct ? */
487                 entry->corrected_initial_age = time (NULL);
488         }
489
490         return entry;
491 }
492
493 static void
494 soup_cache_writing_fixture_free (SoupCacheWritingFixture *fixture)
495 {
496         /* Free fixture. And disconnect signals, we don't want to
497            listen to more SoupMessage events as we're finished with
498            this resource */
499         if (g_signal_handler_is_connected (fixture->msg, fixture->got_chunk_handler))
500                 g_signal_handler_disconnect (fixture->msg, fixture->got_chunk_handler);
501         if (g_signal_handler_is_connected (fixture->msg, fixture->got_body_handler))
502                 g_signal_handler_disconnect (fixture->msg, fixture->got_body_handler);
503         if (g_signal_handler_is_connected (fixture->msg, fixture->restarted_handler))
504                 g_signal_handler_disconnect (fixture->msg, fixture->restarted_handler);
505         g_object_unref (fixture->msg);
506         g_object_unref (fixture->cache);
507         g_slice_free (SoupCacheWritingFixture, fixture);
508 }
509
510 static void
511 close_ready_cb (GObject *source, GAsyncResult *result, SoupCacheWritingFixture *fixture)
512 {
513         SoupCacheEntry *entry = fixture->entry;
514         SoupCache *cache = fixture->cache;
515         GOutputStream *stream = G_OUTPUT_STREAM (source);
516         goffset content_length;
517
518         g_warn_if_fail (entry->error == NULL);
519
520         /* FIXME: what do we do on error ? */
521
522         if (stream) {
523                 g_output_stream_close_finish (stream, result, NULL);
524                 g_object_unref (stream);
525         }
526         entry->stream = NULL;
527
528         content_length = soup_message_headers_get_content_length (entry->headers);
529
530         /* If the process was cancelled, then delete the entry from
531            the cache. Do it also if the size of a chunked resource is
532            too much for the cache */
533         if (g_cancellable_is_cancelled (entry->cancellable)) {
534                 entry->dirty = FALSE;
535                 soup_cache_entry_remove (cache, entry);
536                 soup_cache_entry_free (entry, TRUE);
537                 entry = NULL;
538         } else if ((soup_message_headers_get_encoding (entry->headers) == SOUP_ENCODING_CHUNKED) ||
539                    entry->length != (gsize) content_length) {
540                 /** Two options here:
541                  *
542                  * 1. "chunked" data, entry was temporarily added to
543                  * cache (as content-length is 0) and now that we have
544                  * the actual size we have to evaluate if we want it
545                  * in the cache or not
546                  *
547                  * 2. Content-Length has a different value than actual
548                  * length, means that the content was encoded for
549                  * transmission (typically compressed) and thus we
550                  * have to substract the content-length value that was
551                  * added to the cache and add the unencoded length
552                  **/
553                 gint length_to_add = entry->length - content_length;
554
555                 /* Make room in cache if needed */
556                 if (cache_accepts_entries_of_size (cache, length_to_add)) {
557                         make_room_for_new_entry (cache, length_to_add);
558
559                         cache->priv->size += length_to_add;
560                 } else {
561                         entry->dirty = FALSE;
562                         soup_cache_entry_remove (cache, entry);
563                         soup_cache_entry_free (entry, TRUE);
564                         entry = NULL;
565                 }
566         }
567
568         if (entry) {
569                 /* Get rid of the GString in memory for the resource now */
570                 if (entry->data) {
571                         g_string_free (entry->data, TRUE);
572                         entry->data = NULL;
573                 }
574
575                 entry->dirty = FALSE;
576                 entry->writing = FALSE;
577                 entry->got_body = FALSE;
578                 entry->pos = 0;
579
580                 g_object_unref (entry->cancellable);
581                 entry->cancellable = NULL;
582         }
583
584         cache->priv->n_pending--;
585
586         /* Frees */
587         soup_cache_writing_fixture_free (fixture);
588 }
589
590 static void
591 write_ready_cb (GObject *source, GAsyncResult *result, SoupCacheWritingFixture *fixture)
592 {
593         GOutputStream *stream = G_OUTPUT_STREAM (source);
594         GError *error = NULL;
595         gssize write_size;
596         SoupCacheEntry *entry = fixture->entry;
597
598         if (g_cancellable_is_cancelled (entry->cancellable)) {
599                 g_output_stream_close_async (stream,
600                                              G_PRIORITY_LOW,
601                                              entry->cancellable,
602                                              (GAsyncReadyCallback)close_ready_cb,
603                                              fixture);
604                 return;
605         }
606
607         write_size = g_output_stream_write_finish (stream, result, &error);
608         if (write_size <= 0 || error) {
609                 if (error)
610                         entry->error = error;
611                 g_output_stream_close_async (stream,
612                                              G_PRIORITY_LOW,
613                                              entry->cancellable,
614                                              (GAsyncReadyCallback)close_ready_cb,
615                                              fixture);
616                 /* FIXME: We should completely stop caching the
617                    resource at this point */
618         } else {
619                 entry->pos += write_size;
620
621                 /* Are we still writing and is there new data to write
622                    already ? */
623                 if (entry->data && entry->pos < entry->data->len) {
624                         g_output_stream_write_async (entry->stream,
625                                                      entry->data->str + entry->pos,
626                                                      entry->data->len - entry->pos,
627                                                      G_PRIORITY_LOW,
628                                                      entry->cancellable,
629                                                      (GAsyncReadyCallback)write_ready_cb,
630                                                      fixture);
631                 } else {
632                         entry->writing = FALSE;
633
634                         if (entry->got_body) {
635                                 /* If we already received 'got-body'
636                                    and we have written all the data,
637                                    we can close the stream */
638                                 g_output_stream_close_async (entry->stream,
639                                                              G_PRIORITY_LOW,
640                                                              entry->cancellable,
641                                                              (GAsyncReadyCallback)close_ready_cb,
642                                                              fixture);
643                         }
644                 }
645         }
646 }
647
648 static void
649 msg_got_chunk_cb (SoupMessage *msg, SoupBuffer *chunk, SoupCacheWritingFixture *fixture)
650 {
651         SoupCacheEntry *entry = fixture->entry;
652
653         g_return_if_fail (chunk->data && chunk->length);
654         g_return_if_fail (entry);
655
656         /* Ignore this if the writing or appending was cancelled */
657         if (!g_cancellable_is_cancelled (entry->cancellable)) {
658                 g_string_append_len (entry->data, chunk->data, chunk->length);
659                 entry->length = entry->data->len;
660
661                 if (!cache_accepts_entries_of_size (fixture->cache, entry->length)) {
662                         /* Quickly cancel the caching of the resource */
663                         g_cancellable_cancel (entry->cancellable);
664                 }
665         }
666
667         /* FIXME: remove the error check when we cancel the caching at
668            the first write error */
669         /* Only write if the entry stream is ready */
670         if (entry->writing == FALSE && entry->error == NULL && entry->stream) {
671                 GString *data = entry->data;
672                 entry->writing = TRUE;
673                 g_output_stream_write_async (entry->stream,
674                                              data->str + entry->pos,
675                                              data->len - entry->pos,
676                                              G_PRIORITY_LOW,
677                                              entry->cancellable,
678                                              (GAsyncReadyCallback)write_ready_cb,
679                                              fixture);
680         }
681 }
682
683 static void
684 msg_got_body_cb (SoupMessage *msg, SoupCacheWritingFixture *fixture)
685 {
686         SoupCacheEntry *entry = fixture->entry;
687         g_return_if_fail (entry);
688
689         entry->got_body = TRUE;
690
691         if (!entry->stream && entry->pos != entry->length)
692                 /* The stream is not ready to be written but we still
693                    have data to write, we'll write it when the stream
694                    is opened for writing */
695                 return;
696
697
698         if (entry->pos != entry->length) {
699                 /* If we still have data to write, write it,
700                    write_ready_cb will close the stream */
701                 if (entry->writing == FALSE && entry->error == NULL && entry->stream) {
702                         g_output_stream_write_async (entry->stream,
703                                                      entry->data->str + entry->pos,
704                                                      entry->data->len - entry->pos,
705                                                      G_PRIORITY_LOW,
706                                                      entry->cancellable,
707                                                      (GAsyncReadyCallback)write_ready_cb,
708                                                      fixture);
709                 }
710                 return;
711         }
712
713         if (entry->stream && !entry->writing)
714                 g_output_stream_close_async (entry->stream,
715                                              G_PRIORITY_LOW,
716                                              entry->cancellable,
717                                              (GAsyncReadyCallback)close_ready_cb,
718                                              fixture);
719 }
720
721 static gboolean
722 soup_cache_entry_remove (SoupCache *cache, SoupCacheEntry *entry)
723 {
724         GList *lru_item;
725
726         /* if (entry->dirty && !g_cancellable_is_cancelled (entry->cancellable)) { */
727         if (entry->dirty) {
728                 g_cancellable_cancel (entry->cancellable);
729                 return FALSE;
730         }
731
732         g_assert (!entry->dirty);
733         g_assert (g_list_length (cache->priv->lru_start) == g_hash_table_size (cache->priv->cache));
734
735         /* Remove from cache */
736         if (!g_hash_table_remove (cache->priv->cache, entry->key))
737                 return FALSE;
738
739         /* Remove from LRU */
740         lru_item = g_list_find (cache->priv->lru_start, entry);
741         cache->priv->lru_start = g_list_delete_link (cache->priv->lru_start, lru_item);
742
743         /* Adjust cache size */
744         cache->priv->size -= entry->length;
745
746         g_assert (g_list_length (cache->priv->lru_start) == g_hash_table_size (cache->priv->cache));
747
748         return TRUE;
749 }
750
751 static gint
752 lru_compare_func (gconstpointer a, gconstpointer b)
753 {
754         SoupCacheEntry *entry_a = (SoupCacheEntry *)a;
755         SoupCacheEntry *entry_b = (SoupCacheEntry *)b;
756
757         /** The rationale of this sorting func is
758          *
759          * 1. sort by hits -> LRU algorithm, then
760          *
761          * 2. sort by freshness lifetime, we better discard first
762          * entries that are close to expire
763          *
764          * 3. sort by size, replace first small size resources as they
765          * are cheaper to download
766          **/
767
768         /* Sort by hits */
769         if (entry_a->hits != entry_b->hits)
770                 return entry_a->hits - entry_b->hits;
771
772         /* Sort by freshness_lifetime */
773         if (entry_a->freshness_lifetime != entry_b->freshness_lifetime)
774                 return entry_a->freshness_lifetime - entry_b->freshness_lifetime;
775
776         /* Sort by size */
777         return entry_a->length - entry_b->length;
778 }
779
780 static gboolean
781 cache_accepts_entries_of_size (SoupCache *cache, guint length_to_add)
782 {
783         /* We could add here some more heuristics. TODO: review how
784            this is done by other HTTP caches */
785
786         return length_to_add <= cache->priv->max_entry_data_size;
787 }
788
789 static void
790 make_room_for_new_entry (SoupCache *cache, guint length_to_add)
791 {
792         GList *lru_entry = cache->priv->lru_start;
793
794         /* Check that there is enough room for the new entry. This is
795            an approximation as we're not working out the size of the
796            cache file or the size of the headers for performance
797            reasons. TODO: check if that would be really that expensive */
798
799         while (lru_entry &&
800                (length_to_add + cache->priv->size > cache->priv->max_size)) {
801                 SoupCacheEntry *old_entry = (SoupCacheEntry *)lru_entry->data;
802
803                 /* Discard entries. Once cancelled resources will be
804                  * freed in close_ready_cb
805                  */
806                 if (soup_cache_entry_remove (cache, old_entry)) {
807                         soup_cache_entry_free (old_entry, TRUE);
808                         lru_entry = cache->priv->lru_start;
809                 } else
810                         lru_entry = g_list_next (lru_entry);
811         }
812 }
813
814 static gboolean
815 soup_cache_entry_insert_by_key (SoupCache *cache,
816                                 const char *key,
817                                 SoupCacheEntry *entry,
818                                 gboolean sort)
819 {
820         guint length_to_add = 0;
821
822         if (soup_message_headers_get_encoding (entry->headers) != SOUP_ENCODING_CHUNKED)
823                 length_to_add = soup_message_headers_get_content_length (entry->headers);
824
825         /* Check if we are going to store the resource depending on its size */
826         if (length_to_add) {
827                 if (!cache_accepts_entries_of_size (cache, length_to_add))
828                         return FALSE;
829
830                 /* Make room for new entry if needed */
831                 make_room_for_new_entry (cache, length_to_add);
832         }
833
834         g_hash_table_insert (cache->priv->cache, g_strdup (key), entry);
835
836         /* Compute new cache size */
837         cache->priv->size += length_to_add;
838
839         /* Update LRU */
840         if (sort)
841                 cache->priv->lru_start = g_list_insert_sorted (cache->priv->lru_start, entry, lru_compare_func);
842         else
843                 cache->priv->lru_start = g_list_prepend (cache->priv->lru_start, entry);
844
845         g_assert (g_list_length (cache->priv->lru_start) == g_hash_table_size (cache->priv->cache));
846
847         return TRUE;
848 }
849
850 static void
851 msg_restarted_cb (SoupMessage *msg, SoupCacheEntry *entry)
852 {
853         /* FIXME: What should we do here exactly? */
854 }
855
856 static void
857 replace_cb (GObject *source, GAsyncResult *result, SoupCacheWritingFixture *fixture)
858 {
859         SoupCacheEntry *entry = fixture->entry;
860         GOutputStream *stream = (GOutputStream *) g_file_replace_finish (G_FILE (source),
861                                                                          result, &entry->error);
862
863         if (g_cancellable_is_cancelled (entry->cancellable) || entry->error) {
864                 if (stream)
865                         g_object_unref (stream);
866                 fixture->cache->priv->n_pending--;
867                 entry->dirty = FALSE;
868                 soup_cache_entry_remove (fixture->cache, entry);
869                 soup_cache_entry_free (entry, TRUE);
870                 soup_cache_writing_fixture_free (fixture);
871                 return;
872         }
873
874         entry->stream = stream;
875
876         /* If we already got all the data we have to initiate the
877          * writing here, since we won't get more 'got-chunk'
878          * signals
879          */
880         if (!entry->got_body)
881                 return;
882
883         /* It could happen that reading the data from server
884          * was completed before this happens. In that case
885          * there is no data
886          */
887         if (entry->data) {
888                 entry->writing = TRUE;
889                 g_output_stream_write_async (entry->stream,
890                                              entry->data->str + entry->pos,
891                                              entry->data->len - entry->pos,
892                                              G_PRIORITY_LOW,
893                                              entry->cancellable,
894                                              (GAsyncReadyCallback)write_ready_cb,
895                                              fixture);
896         }
897 }
898
899 typedef struct {
900         time_t request_time;
901         SoupSessionFeature *feature;
902         gulong got_headers_handler;
903 } RequestHelper;
904
905 static void
906 msg_got_headers_cb (SoupMessage *msg, gpointer user_data)
907 {
908         SoupCache *cache;
909         SoupCacheability cacheable;
910         RequestHelper *helper;
911         time_t request_time, response_time;
912
913         response_time = time (NULL);
914
915         helper = (RequestHelper *)user_data;
916         cache = SOUP_CACHE (helper->feature);
917         request_time = helper->request_time;
918         g_signal_handlers_disconnect_by_func (msg, msg_got_headers_cb, user_data);
919         g_slice_free (RequestHelper, helper);
920
921         cacheable = soup_cache_get_cacheability (cache, msg);
922
923         if (cacheable & SOUP_CACHE_CACHEABLE) {
924                 SoupCacheEntry *entry;
925                 char *key;
926                 GFile *file;
927                 SoupCacheWritingFixture *fixture;
928
929                 /* Check if we are already caching this resource */
930                 key = soup_message_get_cache_key (msg);
931                 entry = g_hash_table_lookup (cache->priv->cache, key);
932                 g_free (key);
933
934                 if (entry && entry->dirty)
935                         return;
936
937                 /* Create a new entry, deleting any old one if present */
938                 if (entry) {
939                         soup_cache_entry_remove (cache, entry);
940                         soup_cache_entry_free (entry, TRUE);
941                 }
942
943                 entry = soup_cache_entry_new (cache, msg, request_time, response_time);
944                 entry->hits = 1;
945
946                 /* Do not continue if it can not be stored */
947                 if (!soup_cache_entry_insert_by_key (cache, (const gchar *)entry->key, entry, TRUE)) {
948                         soup_cache_entry_free (entry, TRUE);
949                         return;
950                 }
951
952                 fixture = g_slice_new0 (SoupCacheWritingFixture);
953                 fixture->cache = g_object_ref (cache);
954                 fixture->entry = entry;
955                 fixture->msg = g_object_ref (msg);
956
957                 /* We connect now to these signals and buffer the data
958                    if it comes before the file is ready for writing */
959                 fixture->got_chunk_handler =
960                         g_signal_connect (msg, "got-chunk", G_CALLBACK (msg_got_chunk_cb), fixture);
961                 fixture->got_body_handler =
962                         g_signal_connect (msg, "got-body", G_CALLBACK (msg_got_body_cb), fixture);
963                 fixture->restarted_handler =
964                         g_signal_connect (msg, "restarted", G_CALLBACK (msg_restarted_cb), entry);
965
966                 /* Prepare entry */
967                 file = g_file_new_for_path (entry->filename);
968                 cache->priv->n_pending++;
969
970                 entry->dirty = TRUE;
971                 entry->cancellable = g_cancellable_new ();
972                 g_file_replace_async (file, NULL, FALSE,
973                                       G_FILE_CREATE_PRIVATE | G_FILE_CREATE_REPLACE_DESTINATION,
974                                       G_PRIORITY_LOW, entry->cancellable,
975                                       (GAsyncReadyCallback) replace_cb, fixture);
976                 g_object_unref (file);
977         } else if (cacheable & SOUP_CACHE_INVALIDATES) {
978                 char *key;
979                 SoupCacheEntry *entry;
980
981                 key = soup_message_get_cache_key (msg);
982                 entry = g_hash_table_lookup (cache->priv->cache, key);
983                 g_free (key);
984
985                 if (entry) {
986                         if (soup_cache_entry_remove (cache, entry))
987                                 soup_cache_entry_free (entry, TRUE);
988                 }
989         } else if (cacheable & SOUP_CACHE_VALIDATES) {
990                 char *key;
991                 SoupCacheEntry *entry;
992
993                 key = soup_message_get_cache_key (msg);
994                 entry = g_hash_table_lookup (cache->priv->cache, key);
995                 g_free (key);
996
997                 /* It's possible to get a CACHE_VALIDATES with no
998                  * entry in the hash table. This could happen if for
999                  * example the soup client is the one creating the
1000                  * conditional request.
1001                  */
1002                 if (entry) {
1003                         entry->being_validated = FALSE;
1004                         copy_end_to_end_headers (msg->response_headers, entry->headers);
1005                         soup_cache_entry_set_freshness (entry, msg, cache);
1006                 }
1007         }
1008 }
1009
1010 GInputStream *
1011 soup_cache_send_response (SoupCache *cache, SoupMessage *msg)
1012 {
1013         char *key;
1014         SoupCacheEntry *entry;
1015         char *current_age;
1016         GInputStream *stream = NULL;
1017         GFile *file;
1018
1019         g_return_val_if_fail (SOUP_IS_CACHE (cache), NULL);
1020         g_return_val_if_fail (SOUP_IS_MESSAGE (msg), NULL);
1021
1022         key = soup_message_get_cache_key (msg);
1023         entry = g_hash_table_lookup (cache->priv->cache, key);
1024         g_free (key);
1025         g_return_val_if_fail (entry, NULL);
1026
1027         /* TODO: the original idea was to save reads, but current code
1028            assumes that a stream is always returned. Need to reach
1029            some agreement here. Also we have to handle the situation
1030            were the file was no longer there (for example files
1031            removed without notifying the cache */
1032         file = g_file_new_for_path (entry->filename);
1033         stream = G_INPUT_STREAM (g_file_read (file, NULL, NULL));
1034         g_object_unref (file);
1035
1036         /* Do not change the original message if there is no resource */
1037         if (stream == NULL)
1038                 return stream;
1039
1040         /* If we are told to send a response from cache any validation
1041            in course is over by now */
1042         entry->being_validated = FALSE;
1043
1044         /* Status */
1045         soup_message_set_status (msg, entry->status_code);
1046
1047         /* Headers */
1048         copy_end_to_end_headers (entry->headers, msg->response_headers);
1049
1050         /* Add 'Age' header with the current age */
1051         current_age = g_strdup_printf ("%d", soup_cache_entry_get_current_age (entry));
1052         soup_message_headers_replace (msg->response_headers,
1053                                       "Age",
1054                                       current_age);
1055         g_free (current_age);
1056
1057         return stream;
1058 }
1059
1060 static void
1061 request_started (SoupSessionFeature *feature, SoupSession *session,
1062                  SoupMessage *msg, SoupSocket *socket)
1063 {
1064         RequestHelper *helper = g_slice_new0 (RequestHelper);
1065         helper->request_time = time (NULL);
1066         helper->feature = feature;
1067         helper->got_headers_handler = g_signal_connect (msg, "got-headers",
1068                                                         G_CALLBACK (msg_got_headers_cb),
1069                                                         helper);
1070 }
1071
1072 static void
1073 attach (SoupSessionFeature *feature, SoupSession *session)
1074 {
1075         SoupCache *cache = SOUP_CACHE (feature);
1076         cache->priv->session = session;
1077
1078         soup_cache_default_feature_interface->attach (feature, session);
1079 }
1080
1081 static void
1082 soup_cache_session_feature_init (SoupSessionFeatureInterface *feature_interface,
1083                                         gpointer interface_data)
1084 {
1085         soup_cache_default_feature_interface =
1086                 g_type_default_interface_peek (SOUP_TYPE_SESSION_FEATURE);
1087
1088         feature_interface->attach = attach;
1089         feature_interface->request_started = request_started;
1090 }
1091
1092 static void
1093 soup_cache_init (SoupCache *cache)
1094 {
1095         SoupCachePrivate *priv;
1096
1097         priv = cache->priv = SOUP_CACHE_GET_PRIVATE (cache);
1098
1099         priv->cache = g_hash_table_new_full (g_str_hash,
1100                                              g_str_equal,
1101                                              (GDestroyNotify)g_free,
1102                                              NULL);
1103
1104         /* LRU */
1105         priv->lru_start = NULL;
1106
1107         /* */
1108         priv->n_pending = 0;
1109
1110         /* Cache size */
1111         priv->max_size = DEFAULT_MAX_SIZE;
1112         priv->max_entry_data_size = priv->max_size / MAX_ENTRY_DATA_PERCENTAGE;
1113         priv->size = 0;
1114 }
1115
1116 static void
1117 remove_cache_item (gpointer data,
1118                    gpointer user_data)
1119 {
1120         SoupCache *cache = (SoupCache *) user_data;
1121         SoupCacheEntry *entry = (SoupCacheEntry *) data;
1122
1123         if (soup_cache_entry_remove (cache, entry))
1124                 soup_cache_entry_free (entry, FALSE);
1125 }
1126
1127 static void
1128 soup_cache_finalize (GObject *object)
1129 {
1130         SoupCachePrivate *priv;
1131         GList *entries;
1132
1133         priv = SOUP_CACHE (object)->priv;
1134
1135         // Cannot use g_hash_table_foreach as callbacks must not modify the hash table
1136         entries = g_hash_table_get_values (priv->cache);
1137         g_list_foreach (entries, remove_cache_item, object);
1138         g_list_free (entries);
1139
1140         g_hash_table_destroy (priv->cache);
1141         g_free (priv->cache_dir);
1142
1143         g_list_free (priv->lru_start);
1144         priv->lru_start = NULL;
1145
1146         G_OBJECT_CLASS (soup_cache_parent_class)->finalize (object);
1147 }
1148
1149 static void
1150 soup_cache_set_property (GObject *object, guint prop_id,
1151                                 const GValue *value, GParamSpec *pspec)
1152 {
1153         SoupCachePrivate *priv = SOUP_CACHE (object)->priv;
1154
1155         switch (prop_id) {
1156         case PROP_CACHE_DIR:
1157                 priv->cache_dir = g_value_dup_string (value);
1158                 /* Create directory if it does not exist (FIXME: should we?) */
1159                 if (!g_file_test (priv->cache_dir, G_FILE_TEST_EXISTS | G_FILE_TEST_IS_DIR))
1160                         g_mkdir_with_parents (priv->cache_dir, 0700);
1161                 break;
1162         case PROP_CACHE_TYPE:
1163                 priv->cache_type = g_value_get_enum (value);
1164                 /* TODO: clear private entries and issue a warning if moving to shared? */
1165                 break;
1166         default:
1167                 G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec);
1168                 break;
1169         }
1170 }
1171
1172 static void
1173 soup_cache_get_property (GObject *object, guint prop_id,
1174                          GValue *value, GParamSpec *pspec)
1175 {
1176         SoupCachePrivate *priv = SOUP_CACHE (object)->priv;
1177
1178         switch (prop_id) {
1179         case PROP_CACHE_DIR:
1180                 g_value_set_string (value, priv->cache_dir);
1181                 break;
1182         case PROP_CACHE_TYPE:
1183                 g_value_set_enum (value, priv->cache_type);
1184                 break;
1185         default:
1186                 G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec);
1187                 break;
1188         }
1189 }
1190
1191 static void
1192 soup_cache_constructed (GObject *object)
1193 {
1194         SoupCachePrivate *priv;
1195
1196         priv = SOUP_CACHE (object)->priv;
1197
1198         if (!priv->cache_dir) {
1199                 /* Set a default cache dir, different for each user */
1200                 priv->cache_dir = g_build_filename (g_get_user_cache_dir (),
1201                                                     "httpcache",
1202                                                     NULL);
1203                 if (!g_file_test (priv->cache_dir, G_FILE_TEST_EXISTS | G_FILE_TEST_IS_DIR))
1204                         g_mkdir_with_parents (priv->cache_dir, 0700);
1205         }
1206
1207         if (G_OBJECT_CLASS (soup_cache_parent_class)->constructed)
1208                 G_OBJECT_CLASS (soup_cache_parent_class)->constructed (object);
1209 }
1210
1211 static void
1212 soup_cache_class_init (SoupCacheClass *cache_class)
1213 {
1214         GObjectClass *gobject_class = (GObjectClass *)cache_class;
1215
1216         gobject_class->finalize = soup_cache_finalize;
1217         gobject_class->constructed = soup_cache_constructed;
1218         gobject_class->set_property = soup_cache_set_property;
1219         gobject_class->get_property = soup_cache_get_property;
1220
1221         cache_class->get_cacheability = get_cacheability;
1222
1223         g_object_class_install_property (gobject_class, PROP_CACHE_DIR,
1224                                          g_param_spec_string ("cache-dir",
1225                                                               "Cache directory",
1226                                                               "The directory to store the cache files",
1227                                                               NULL,
1228                                                               G_PARAM_READWRITE | G_PARAM_CONSTRUCT_ONLY));
1229
1230         g_object_class_install_property (gobject_class, PROP_CACHE_TYPE,
1231                                          g_param_spec_enum ("cache-type",
1232                                                             "Cache type",
1233                                                             "Whether the cache is private or shared",
1234                                                             SOUP_TYPE_CACHE_TYPE,
1235                                                             SOUP_CACHE_SINGLE_USER,
1236                                                             G_PARAM_READWRITE | G_PARAM_CONSTRUCT_ONLY));
1237
1238         g_type_class_add_private (cache_class, sizeof (SoupCachePrivate));
1239 }
1240
1241 /**
1242  * soup_cache_new:
1243  * @cache_dir: the directory to store the cached data, or %NULL to use the default one
1244  * @cache_type: the #SoupCacheType of the cache
1245  *
1246  * Creates a new #SoupCache.
1247  *
1248  * Returns: a new #SoupCache
1249  *
1250  * Since: 2.28
1251  **/
1252 SoupCache *
1253 soup_cache_new (const char *cache_dir, SoupCacheType cache_type)
1254 {
1255         return g_object_new (SOUP_TYPE_CACHE,
1256                              "cache-dir", cache_dir,
1257                              "cache-type", cache_type,
1258                              NULL);
1259 }
1260
1261 /**
1262  * soup_cache_has_response:
1263  * @cache: a #SoupCache
1264  * @msg: a #SoupMessage
1265  *
1266  * This function calculates whether the @cache object has a proper
1267  * response for the request @msg given the flags both in the request
1268  * and the cached reply and the time ellapsed since it was cached.
1269  *
1270  * Returns: whether or not the @cache has a valid response for @msg
1271  **/
1272 SoupCacheResponse
1273 soup_cache_has_response (SoupCache *cache, SoupMessage *msg)
1274 {
1275         char *key;
1276         SoupCacheEntry *entry;
1277         const char *cache_control, *pragma;
1278         gpointer value;
1279         int max_age, max_stale, min_fresh;
1280         GList *lru_item, *item;
1281
1282         key = soup_message_get_cache_key (msg);
1283         entry = g_hash_table_lookup (cache->priv->cache, key);
1284         g_free (key);
1285
1286         /* 1. The presented Request-URI and that of stored response
1287          * match
1288          */
1289         if (!entry)
1290                 return SOUP_CACHE_RESPONSE_STALE;
1291
1292         /* Increase hit count. Take sorting into account */
1293         entry->hits++;
1294         lru_item = g_list_find (cache->priv->lru_start, entry);
1295         item = lru_item;
1296         while (item->next && lru_compare_func (item->data, item->next->data) > 0)
1297                 item = g_list_next (item);
1298
1299         if (item != lru_item) {
1300                 cache->priv->lru_start = g_list_remove_link (cache->priv->lru_start, lru_item);
1301                 item = g_list_insert_sorted (item, lru_item->data, lru_compare_func);
1302                 g_list_free (lru_item);
1303         }
1304
1305         if (entry->dirty || entry->being_validated)
1306                 return SOUP_CACHE_RESPONSE_STALE;
1307
1308         /* 2. The request method associated with the stored response
1309          *  allows it to be used for the presented request
1310          */
1311
1312         /* In practice this means we only return our resource for GET,
1313          * cacheability for other methods is a TODO in the RFC
1314          * (TODO: although we could return the headers for HEAD
1315          * probably).
1316          */
1317         if (msg->method != SOUP_METHOD_GET)
1318                 return SOUP_CACHE_RESPONSE_STALE;
1319
1320         /* 3. Selecting request-headers nominated by the stored
1321          * response (if any) match those presented.
1322          */
1323
1324         /* TODO */
1325
1326         /* 4. The request is a conditional request issued by the client.
1327          */
1328         if (soup_message_headers_get (msg->request_headers, "If-Modified-Since") ||
1329             soup_message_headers_get (msg->request_headers, "If-None-Match"))
1330                 return SOUP_CACHE_RESPONSE_STALE;
1331
1332         /* 5. The presented request and stored response are free from
1333          * directives that would prevent its use.
1334          */
1335
1336         max_age = max_stale = min_fresh = -1;
1337
1338         /* For HTTP 1.0 compatibility. RFC2616 section 14.9.4
1339          */
1340         pragma = soup_message_headers_get (msg->request_headers, "Pragma");
1341         if (pragma && soup_header_contains (pragma, "no-cache"))
1342                 return SOUP_CACHE_RESPONSE_STALE;
1343
1344         cache_control = soup_message_headers_get (msg->request_headers, "Cache-Control");
1345         if (cache_control) {
1346                 GHashTable *hash = soup_header_parse_param_list (cache_control);
1347
1348                 if (g_hash_table_lookup_extended (hash, "no-store", NULL, NULL)) {
1349                         soup_header_free_param_list (hash);
1350                         return SOUP_CACHE_RESPONSE_STALE;
1351                 }
1352
1353                 if (g_hash_table_lookup_extended (hash, "no-cache", NULL, NULL)) {
1354                         soup_header_free_param_list (hash);
1355                         return SOUP_CACHE_RESPONSE_STALE;
1356                 }
1357
1358                 if (g_hash_table_lookup_extended (hash, "max-age", NULL, &value)) {
1359                         max_age = (int)MIN (g_ascii_strtoll (value, NULL, 10), G_MAXINT32);
1360                         /* Forcing cache revalidaton
1361                          */
1362                         if (!max_age) {
1363                                 soup_header_free_param_list (hash);
1364                                 return SOUP_CACHE_RESPONSE_NEEDS_VALIDATION;
1365                         }
1366                 }
1367
1368                 /* max-stale can have no value set, we need to use _extended */
1369                 if (g_hash_table_lookup_extended (hash, "max-stale", NULL, &value)) {
1370                         if (value)
1371                                 max_stale = (int)MIN (g_ascii_strtoll (value, NULL, 10), G_MAXINT32);
1372                         else
1373                                 max_stale = G_MAXINT32;
1374                 }
1375
1376                 value = g_hash_table_lookup (hash, "min-fresh");
1377                 if (value)
1378                         min_fresh = (int)MIN (g_ascii_strtoll (value, NULL, 10), G_MAXINT32);
1379
1380                 soup_header_free_param_list (hash);
1381
1382                 if (max_age > 0) {
1383                         guint current_age = soup_cache_entry_get_current_age (entry);
1384
1385                         /* If we are over max-age and max-stale is not
1386                            set, do not use the value from the cache
1387                            without validation */
1388                         if ((guint) max_age <= current_age && max_stale == -1)
1389                                 return SOUP_CACHE_RESPONSE_NEEDS_VALIDATION;
1390                 }
1391         }
1392
1393         /* 6. The stored response is either: fresh, allowed to be
1394          * served stale or succesfully validated
1395          */
1396         /* TODO consider also proxy-revalidate & s-maxage */
1397         if (entry->must_revalidate)
1398                 return SOUP_CACHE_RESPONSE_NEEDS_VALIDATION;
1399
1400         if (!soup_cache_entry_is_fresh_enough (entry, min_fresh)) {
1401                 /* Not fresh, can it be served stale? */
1402                 if (max_stale != -1) {
1403                         /* G_MAXINT32 means we accept any staleness */
1404                         if (max_stale == G_MAXINT32)
1405                                 return SOUP_CACHE_RESPONSE_FRESH;
1406
1407                         if ((soup_cache_entry_get_current_age (entry) - entry->freshness_lifetime) <= (guint) max_stale)
1408                                 return SOUP_CACHE_RESPONSE_FRESH;
1409                 }
1410
1411                 return SOUP_CACHE_RESPONSE_NEEDS_VALIDATION;
1412         }
1413
1414         return SOUP_CACHE_RESPONSE_FRESH;
1415 }
1416
1417 /**
1418  * soup_cache_get_cacheability:
1419  * @cache: a #SoupCache
1420  * @msg: a #SoupMessage
1421  *
1422  * Calculates whether the @msg can be cached or not.
1423  *
1424  * Returns: a #SoupCacheability value indicating whether the @msg can be cached or not.
1425  **/
1426 SoupCacheability
1427 soup_cache_get_cacheability (SoupCache *cache, SoupMessage *msg)
1428 {
1429         g_return_val_if_fail (SOUP_IS_CACHE (cache), SOUP_CACHE_UNCACHEABLE);
1430         g_return_val_if_fail (SOUP_IS_MESSAGE (msg), SOUP_CACHE_UNCACHEABLE);
1431
1432         return SOUP_CACHE_GET_CLASS (cache)->get_cacheability (cache, msg);
1433 }
1434
1435 static gboolean
1436 force_flush_timeout (gpointer data)
1437 {
1438         gboolean *forced = (gboolean *)data;
1439         *forced = TRUE;
1440
1441         return FALSE;
1442 }
1443
1444 /**
1445  * soup_cache_flush:
1446  * @cache: a #SoupCache
1447  * @session: the #SoupSession associated with the @cache
1448  *
1449  * This function will force all pending writes in the @cache to be
1450  * committed to disk. For doing so it will iterate the #GMainContext
1451  * associated with the @session (which can be the default one) as long
1452  * as needed.
1453  **/
1454 void
1455 soup_cache_flush (SoupCache *cache)
1456 {
1457         GMainContext *async_context;
1458         SoupSession *session;
1459         guint timeout_id;
1460         gboolean forced = FALSE;
1461
1462         g_return_if_fail (SOUP_IS_CACHE (cache));
1463
1464         session = cache->priv->session;
1465         g_return_if_fail (SOUP_IS_SESSION (session));
1466         async_context = soup_session_get_async_context (session);
1467
1468         /* We give cache 10 secs to finish */
1469         timeout_id = g_timeout_add (10000, force_flush_timeout, &forced);
1470
1471         while (!forced && cache->priv->n_pending > 0)
1472                 g_main_context_iteration (async_context, FALSE);
1473
1474         if (!forced)
1475                 g_source_remove (timeout_id);
1476         else
1477                 g_warning ("Cache flush finished despite %d pending requests", cache->priv->n_pending);
1478 }
1479
1480 static void
1481 clear_cache_item (gpointer data,
1482                   gpointer user_data)
1483 {
1484         SoupCache *cache = (SoupCache *) user_data;
1485         SoupCacheEntry *entry = (SoupCacheEntry *) data;
1486
1487         if (soup_cache_entry_remove (cache, entry))
1488                 soup_cache_entry_free (entry, TRUE);
1489 }
1490
1491 /**
1492  * soup_cache_clear:
1493  * @cache: a #SoupCache
1494  *
1495  * Will remove all entries in the @cache plus all the cache files
1496  * associated with them.
1497  **/
1498 void
1499 soup_cache_clear (SoupCache *cache)
1500 {
1501         GHashTable *hash;
1502         GList *entries;
1503
1504         g_return_if_fail (SOUP_IS_CACHE (cache));
1505
1506         hash = cache->priv->cache;
1507         g_return_if_fail (hash);
1508
1509         // Cannot use g_hash_table_foreach as callbacks must not modify the hash table
1510         entries = g_hash_table_get_values (hash);
1511         g_list_foreach (entries, clear_cache_item, cache);
1512         g_list_free (entries);
1513 }
1514
1515 SoupMessage *
1516 soup_cache_generate_conditional_request (SoupCache *cache, SoupMessage *original)
1517 {
1518         SoupMessage *msg;
1519         SoupURI *uri;
1520         SoupCacheEntry *entry;
1521         char *key;
1522         const char *value;
1523
1524         g_return_val_if_fail (SOUP_IS_CACHE (cache), NULL);
1525         g_return_val_if_fail (SOUP_IS_MESSAGE (original), NULL);
1526
1527         /* First copy the data we need from the original message */
1528         uri = soup_message_get_uri (original);
1529         msg = soup_message_new_from_uri (original->method, uri);
1530
1531         soup_message_headers_foreach (original->request_headers,
1532                                       (SoupMessageHeadersForeachFunc)copy_headers,
1533                                       msg->request_headers);
1534
1535         /* Now add the validator entries in the header from the cached
1536            data */
1537         key = soup_message_get_cache_key (original);
1538         entry = g_hash_table_lookup (cache->priv->cache, key);
1539         g_free (key);
1540
1541         g_return_val_if_fail (entry, NULL);
1542
1543         entry->being_validated = TRUE;
1544
1545         value = soup_message_headers_get (entry->headers, "Last-Modified");
1546         if (value)
1547                 soup_message_headers_append (msg->request_headers,
1548                                              "If-Modified-Since",
1549                                              value);
1550         value = soup_message_headers_get (entry->headers, "ETag");
1551         if (value)
1552                 soup_message_headers_append (msg->request_headers,
1553                                              "If-None-Match",
1554                                              value);
1555         return msg;
1556 }
1557
1558 #define OLD_SOUP_CACHE_FILE "soup.cache"
1559 #define SOUP_CACHE_FILE "soup.cache2"
1560
1561 #define SOUP_CACHE_HEADERS_FORMAT "{ss}"
1562 #define SOUP_CACHE_PHEADERS_FORMAT "(ssbuuuuuqa" SOUP_CACHE_HEADERS_FORMAT ")"
1563 #define SOUP_CACHE_ENTRIES_FORMAT "(qa" SOUP_CACHE_PHEADERS_FORMAT ")"
1564
1565 /* Basically the same format than above except that some strings are
1566    prepended with &. This way the GVariant returns a pointer to the
1567    data instead of duplicating the string */
1568 #define SOUP_CACHE_DECODE_HEADERS_FORMAT "{&s&s}"
1569
1570 static void
1571 pack_entry (gpointer data,
1572             gpointer user_data)
1573 {
1574         SoupCacheEntry *entry = (SoupCacheEntry *) data;
1575         SoupMessageHeadersIter iter;
1576         const gchar *header_key, *header_value;
1577         GVariantBuilder *entries_builder = (GVariantBuilder *)user_data;
1578
1579         /* Do not store non-consolidated entries */
1580         if (entry->dirty || entry->writing || !entry->key)
1581                 return;
1582
1583         g_variant_builder_open (entries_builder, G_VARIANT_TYPE (SOUP_CACHE_PHEADERS_FORMAT));
1584         g_variant_builder_add (entries_builder, "s", entry->key);
1585         g_variant_builder_add (entries_builder, "s", entry->filename);
1586         g_variant_builder_add (entries_builder, "b", entry->must_revalidate);
1587         g_variant_builder_add (entries_builder, "u", entry->freshness_lifetime);
1588         g_variant_builder_add (entries_builder, "u", entry->corrected_initial_age);
1589         g_variant_builder_add (entries_builder, "u", entry->response_time);
1590         g_variant_builder_add (entries_builder, "u", entry->hits);
1591         g_variant_builder_add (entries_builder, "u", entry->length);
1592         g_variant_builder_add (entries_builder, "q", entry->status_code);
1593
1594         /* Pack headers */
1595         g_variant_builder_open (entries_builder, G_VARIANT_TYPE ("a" SOUP_CACHE_HEADERS_FORMAT));
1596         soup_message_headers_iter_init (&iter, entry->headers);
1597         while (soup_message_headers_iter_next (&iter, &header_key, &header_value)) {
1598                 if (g_utf8_validate (header_value, -1, NULL))
1599                         g_variant_builder_add (entries_builder, SOUP_CACHE_HEADERS_FORMAT,
1600                                                header_key, header_value);
1601         }
1602         g_variant_builder_close (entries_builder); /* "a" SOUP_CACHE_HEADERS_FORMAT */
1603         g_variant_builder_close (entries_builder); /* SOUP_CACHE_PHEADERS_FORMAT */
1604 }
1605
1606 void
1607 soup_cache_dump (SoupCache *cache)
1608 {
1609         SoupCachePrivate *priv = SOUP_CACHE_GET_PRIVATE (cache);
1610         gchar *filename;
1611         GVariantBuilder entries_builder;
1612         GVariant *cache_variant;
1613
1614         if (!g_list_length (cache->priv->lru_start))
1615                 return;
1616
1617         /* Create the builder and iterate over all entries */
1618         g_variant_builder_init (&entries_builder, G_VARIANT_TYPE (SOUP_CACHE_ENTRIES_FORMAT));
1619         g_variant_builder_add (&entries_builder, "q", SOUP_CACHE_CURRENT_VERSION);
1620         g_variant_builder_open (&entries_builder, G_VARIANT_TYPE ("a" SOUP_CACHE_PHEADERS_FORMAT));
1621         g_list_foreach (cache->priv->lru_start, pack_entry, &entries_builder);
1622         g_variant_builder_close (&entries_builder);
1623
1624         /* Serialize and dump */
1625         cache_variant = g_variant_builder_end (&entries_builder);
1626         g_variant_ref_sink (cache_variant);
1627         filename = g_build_filename (priv->cache_dir, SOUP_CACHE_FILE, NULL);
1628         g_file_set_contents (filename, (const gchar *)g_variant_get_data (cache_variant),
1629                              g_variant_get_size (cache_variant), NULL);
1630         g_free (filename);
1631         g_variant_unref (cache_variant);
1632 }
1633
1634 static void
1635 clear_cache_files (SoupCache *cache)
1636 {
1637         GFileInfo *file_info;
1638         GFileEnumerator *file_enumerator;
1639         GFile *cache_dir_file = g_file_new_for_path (cache->priv->cache_dir);
1640
1641         file_enumerator = g_file_enumerate_children (cache_dir_file, G_FILE_ATTRIBUTE_STANDARD_NAME,
1642                                                      G_FILE_QUERY_INFO_NONE, NULL, NULL);
1643         if (file_enumerator) {
1644                 while ((file_info = g_file_enumerator_next_file (file_enumerator, NULL, NULL)) != NULL) {
1645                         const gchar *filename = g_file_info_get_name (file_info);
1646
1647                         if (strcmp (filename, SOUP_CACHE_FILE) != 0) {
1648                                 GFile *cache_file = g_file_get_child (cache_dir_file, filename);
1649                                 g_file_delete (cache_file, NULL, NULL);
1650                                 g_object_unref (cache_file);
1651                         }
1652                 }
1653                 g_object_unref (file_enumerator);
1654         }
1655         g_object_unref (cache_dir_file);
1656 }
1657
1658 void
1659 soup_cache_load (SoupCache *cache)
1660 {
1661         gboolean must_revalidate;
1662         guint32 freshness_lifetime, hits;
1663         guint32 corrected_initial_age, response_time;
1664         char *key, *filename = NULL, *contents = NULL;
1665         GVariant *cache_variant;
1666         GVariantIter *entries_iter = NULL, *headers_iter = NULL;
1667         gsize length, items;
1668         SoupCacheEntry *entry;
1669         SoupCachePrivate *priv = cache->priv;
1670         guint16 version, status_code;
1671
1672         filename = g_build_filename (priv->cache_dir, SOUP_CACHE_FILE, NULL);
1673         if (!g_file_get_contents (filename, &contents, &length, NULL)) {
1674                 g_free (filename);
1675                 g_free (contents);
1676                 clear_cache_files (cache);
1677                 return;
1678         }
1679         g_free (filename);
1680
1681         cache_variant = g_variant_new_from_data (G_VARIANT_TYPE (SOUP_CACHE_ENTRIES_FORMAT),
1682                                                  (const gchar *) contents, length, FALSE, g_free, contents);
1683         g_variant_get (cache_variant, SOUP_CACHE_ENTRIES_FORMAT, &version, &entries_iter);
1684         if (version != SOUP_CACHE_CURRENT_VERSION) {
1685                 g_variant_iter_free (entries_iter);
1686                 g_variant_unref (cache_variant);
1687                 clear_cache_files (cache);
1688                 return;
1689         }
1690
1691         items = g_variant_iter_n_children (entries_iter);
1692
1693         while (g_variant_iter_loop (entries_iter, SOUP_CACHE_PHEADERS_FORMAT,
1694                                     &key, &filename, &must_revalidate,
1695                                     &freshness_lifetime, &corrected_initial_age,
1696                                     &response_time, &hits, &length, &status_code,
1697                                     &headers_iter)) {
1698                 const gchar *header_key, *header_value;
1699                 SoupMessageHeaders *headers;
1700                 SoupMessageHeadersIter soup_headers_iter;
1701
1702                 /* SoupMessage Headers */
1703                 headers = soup_message_headers_new (SOUP_MESSAGE_HEADERS_RESPONSE);
1704                 while (g_variant_iter_loop (headers_iter, SOUP_CACHE_HEADERS_FORMAT, &header_key, &header_value))
1705                         if (*header_key && *header_value)
1706                                 soup_message_headers_append (headers, header_key, header_value);
1707
1708                 /* Check that we have headers */
1709                 soup_message_headers_iter_init (&soup_headers_iter, headers);
1710                 if (!soup_message_headers_iter_next (&soup_headers_iter, &header_key, &header_value)) {
1711                         soup_message_headers_free (headers);
1712                         continue;
1713                 }
1714
1715                 /* Insert in cache */
1716                 entry = g_slice_new0 (SoupCacheEntry);
1717                 entry->key = g_strdup (key);
1718                 entry->filename = g_strdup (filename);
1719                 entry->must_revalidate = must_revalidate;
1720                 entry->freshness_lifetime = freshness_lifetime;
1721                 entry->corrected_initial_age = corrected_initial_age;
1722                 entry->response_time = response_time;
1723                 entry->hits = hits;
1724                 entry->length = length;
1725                 entry->headers = headers;
1726                 entry->status_code = status_code;
1727
1728                 if (!soup_cache_entry_insert_by_key (cache, entry->key, entry, FALSE))
1729                         soup_cache_entry_free (entry, TRUE);
1730         }
1731
1732         cache->priv->lru_start = g_list_reverse (cache->priv->lru_start);
1733
1734         /* frees */
1735         g_variant_unref (cache_variant);
1736 }
1737
1738 void
1739 soup_cache_set_max_size (SoupCache *cache,
1740                          guint      max_size)
1741 {
1742         cache->priv->max_size = max_size;
1743         cache->priv->max_entry_data_size = cache->priv->max_size / MAX_ENTRY_DATA_PERCENTAGE;
1744 }
1745
1746 guint
1747 soup_cache_get_max_size (SoupCache *cache)
1748 {
1749         return cache->priv->max_size;
1750 }