Merge lp:~midori/midori/hstsWebExtension into lp:midori

[Cris Dywan]

Implement HSTS as a web extension

This branch enables building of separate web extensions.

Note: At this point we probably want to go for libpeas rather than the direct WebKit2 API. That would save us an additional porting step later if we do it in one go. And give us control over extensions that we can't have otherwise.

Unmerged revisions

6965. By Cris Dywan on 2015-06-13

Implement HSTS as a web extension

=== modified file 'CMakeLists.txt'
--- CMakeLists.txt	2015-03-23 11:33:24 +0000
+++ CMakeLists.txt	2015-06-13 13:38:06 +0000
@@ -268,9 +268,9 @@
 install(FILES AUTHORS COPYING ChangeLog EXPAT README DESTINATION ${CMAKE_INSTALL_DOCDIR})
 
 add_subdirectory (midori)
-add_subdirectory (extensions)
 enable_testing()
 add_subdirectory (tests)
+add_subdirectory (extensions)
 add_subdirectory (po)
 add_subdirectory (icons)
 add_subdirectory (data)

=== modified file 'extensions/CMakeLists.txt'
--- extensions/CMakeLists.txt	2015-04-19 14:06:12 +0000
+++ extensions/CMakeLists.txt	2015-06-13 13:38:06 +0000
@@ -23,6 +23,10 @@
          "nsplugin-manager.vala"
          "tabs2one.c"
          )
+else ()
+    list(REMOVE_ITEM EXTENSIONS
+         "hsts.web.vala"
+         )
 endif ()
 
 # FIXME: re-enable webmedia extension
@@ -36,99 +40,105 @@
     list(REMOVE_ITEM EXTENSIONS "tabs2one.c")
 endif()
 
-foreach(UNIT_SRC ${EXTENSIONS})
-    string(FIND ${UNIT_SRC} ".c" UNIT_EXTENSION)
-    if (UNIT_EXTENSION GREATER -1)
-        string(REPLACE ".c" "" UNIT ${UNIT_SRC})
-        add_library(${UNIT} MODULE ${UNIT_SRC})
-        target_link_libraries(${UNIT}
-                              ${LIBMIDORI}
-                              )
-        set_target_properties(${UNIT} PROPERTIES
-                              COMPILE_FLAGS ${CFLAGS}
-                              )
-        install(TARGETS ${UNIT}
-            LIBRARY DESTINATION ${EXTENSIONDIR}
-                )
-    endif ()
-endforeach ()
+include(ValaPrecompile)
 
-foreach(UNIT_SRC ${EXTENSIONS})
-    string(FIND ${UNIT_SRC} "." UNIT_EXTENSION)
-    if (UNIT_EXTENSION EQUAL -1)
-        file(GLOB UNIT_FILES RELATIVE ${CMAKE_CURRENT_SOURCE_DIR} "${UNIT_SRC}/*.c")
-        file(GLOB UNIT_FILES_VALA RELATIVE ${CMAKE_CURRENT_SOURCE_DIR} "${UNIT_SRC}/*.vala")
-        if (UNIT_FILES_VALA)
-            include(ValaPrecompile)
-            vala_precompile(UNIT_SRC_C ${UNIT_SRC}
-                ${UNIT_FILES_VALA}
-            PACKAGES
+macro(build_extension name)
+    set(source "${ARGN}")
+    set(SOURCE_VALA "")
+    foreach(FILE ${source})
+        string(FIND ${FILE} ".web." WEB_EXTENSION)
+        string(FIND ${FILE} ".vala" VALA_EXTENSION)
+        if (WEB_EXTENSION GREATER -1)
+            string(REPLACE ".vala" "" WEB_NAME "${name}_${FILE}")
+            string(REPLACE "." "_" WEB_NAME "${WEB_NAME}")
+            vala_precompile(WEB_EXTENSION_C "${WEB_NAME}"
+                ${FILE}
+             PACKAGES
                 ${PKGS}
-            OPTIONS
-                ${VALAFLAGS}
-                --use-header="${CMAKE_PROJECT_NAME}-core.h"
-            GENERATE_HEADER
-                "${UNIT_SRC}"
-            GENERATE_HEADER
-                ${UNIT}
-            CUSTOM_VAPIS
-                ${EXTRA_VAPIS}
-                "${CMAKE_SOURCE_DIR}/midori/midori.vapi"
-                "${CMAKE_BINARY_DIR}/midori/${LIBMIDORI}.vapi"
-            )
-            set(UNIT_FILES ${UNIT_FILES} ${UNIT_SRC_C})
-        endif ()
-        if (UNIT_FILES)
-            add_library(${UNIT_SRC} MODULE ${UNIT_FILES})
-            target_link_libraries(${UNIT_SRC}
-                                  ${LIBMIDORI}
+             OPTIONS
+                 ${VALAFLAGS}
+             GENERATE_HEADER
+                ${FILE}
+             CUSTOM_VAPIS
+                "${CMAKE_SOURCE_DIR}/midori/webkit2gtk-web-extension-4.0.vapi"
+             )
+            add_library(${WEB_NAME} MODULE ${WEB_EXTENSION_C})
+            target_link_libraries(${WEB_NAME}
+                                  ${LIBMIDORI2}
+                                   )
+            install(TARGETS ${WEB_NAME}
+                     LIBRARY DESTINATION ${EXTENSIONDIR}
+                     )
+            # Extensions with Vala code get the lenient VALA_CFLAGS
+            set_target_properties(${WEB_NAME} PROPERTIES
+                                  COMPILE_FLAGS ${VALA_CFLAGS}
                                   )
-            install(TARGETS ${UNIT_SRC}
-                    LIBRARY DESTINATION ${EXTENSIONDIR}
-                    )
-            # extensions with vala code get the lenient VALA_CFLAGS
-            # others get the usual CFLAGS with -Wall and -Werror
-	    if (UNIT_FILES_VALA)
-		set_target_properties(${UNIT_SRC} PROPERTIES
-				      COMPILE_FLAGS ${VALA_CFLAGS}
-				      )
-	    else ()
-		set_target_properties(${UNIT_SRC} PROPERTIES
-				      COMPILE_FLAGS ${CFLAGS}
-				      )
-	    endif ()
-        endif ()
-    endif ()
-endforeach ()
+            list(REMOVE_ITEM source ${FILE})
 
-foreach(UNIT_SRC ${EXTENSIONS})
-    string(FIND ${UNIT_SRC} ".vala" UNIT_EXTENSION)
-    if (UNIT_EXTENSION GREATER -1)
-        string(REPLACE ".vala" "" UNIT ${UNIT_SRC})
-        include(ValaPrecompile)
-        vala_precompile(UNIT_SRC_C ${UNIT}
-            ${UNIT_SRC}
+            # Mandatory unit testing
+            add_test(NAME "test-${WEB_NAME}" COMMAND $<TARGET_FILE:midori> -t $<TARGET_FILE:${WEB_NAME}>)
+            contain_test("test-${WEB_NAME}" $<TARGET_FILE:midori> -t $<TARGET_FILE:${WEB_NAME}>)
+        elseif (VALA_EXTENSION GREATER -1)
+            list(APPEND SOURCE_VALA ${FILE})
+        endif ()
+    endforeach()
+    if (SOURCE_VALA)
+        vala_precompile(SOURCE_C ${name}
+            ${SOURCE_VALA}
         PACKAGES
             ${PKGS}
         OPTIONS
-            ${VALAFLAGS}
-            --use-header="${CMAKE_PROJECT_NAME}-core.h"
+             ${VALAFLAGS}
         GENERATE_HEADER
-            ${UNIT}
+            ${name}
         CUSTOM_VAPIS
-            ${EXTRA_VAPIS}
+             ${EXTRA_VAPIS}
             "${CMAKE_SOURCE_DIR}/midori/midori.vapi"
+            # "${CMAKE_SOURCE_DIR}/katze/katze.vapi"
             "${CMAKE_BINARY_DIR}/midori/${LIBMIDORI}.vapi"
         )
-        add_library(${UNIT} MODULE ${UNIT_SRC_C})
-        target_link_libraries(${UNIT}
-                              ${LIBMIDORI}
-                              )
-        set_target_properties(${UNIT} PROPERTIES
-                              COMPILE_FLAGS "${VALA_CFLAGS}"
-                              )
-        install(TARGETS ${UNIT}
-                LIBRARY DESTINATION ${EXTENSIONDIR}
-                )
+       set(source ${SOURCE_C})
+    endif ()
+    if (source)
+        add_library(${name} MODULE ${source})
+        target_link_libraries(${name}
+                               ${LIBMIDORI}
+                               )
+        install(TARGETS ${name}
+                 LIBRARY DESTINATION ${EXTENSIONDIR}
+                 )
+        if (SOURCE_VALA)
+            # Extensions with Vala code get the lenient VALA_CFLAGS
+            set_target_properties(${name} PROPERTIES
+                                  COMPILE_FLAGS ${VALA_CFLAGS}
+                                  )
+        else ()
+            set_target_properties(${name} PROPERTIES
+                                  COMPILE_FLAGS ${CFLAGS}
+                                  )
+        endif ()
+        # Optional unit test
+        add_test(NAME "test-${name}" COMMAND $<TARGET_FILE:midori> -t $<TARGET_FILE:${name}>)
+        contain_test("test-${name}" $<TARGET_FILE:midori> -t $<TARGET_FILE:${name}>)
+    endif ()
+endmacro(build_extension)
+
+foreach(UNIT_SRC ${EXTENSIONS})
+    string(FIND ${UNIT_SRC} ".c" UNIT_EXTENSION)
+    if (UNIT_EXTENSION GREATER -1)
+        string(REPLACE ".c" "" UNIT ${UNIT_SRC})
+        build_extension(${UNIT} ${UNIT_SRC})
+    endif ()
+
+    string(FIND ${UNIT_SRC} "." UNIT_EXTENSION)
+    if (UNIT_EXTENSION GREATER -1)
+        file(GLOB UNIT_FILES RELATIVE ${CMAKE_CURRENT_SOURCE_DIR} "${UNIT_SRC}/*.c" "${UNIT_SRC}/*.vala")
+        build_extension(${UNIT_SRC} ${UNIT_FILES})
+    endif ()
+
+    string(FIND ${UNIT_SRC} ".vala" UNIT_EXTENSION)
+    if (UNIT_EXTENSION GREATER -1)
+        string(REPLACE ".vala" "" UNIT ${UNIT_SRC})
+        build_extension(${UNIT} ${UNIT_SRC})
     endif ()
 endforeach ()

=== added file 'extensions/hsts.web.vala'
--- extensions/hsts.web.vala	1970-01-01 00:00:00 +0000
+++ extensions/hsts.web.vala	2015-06-13 13:38:06 +0000
@@ -0,0 +1,177 @@
+/*
+ Copyright (C) 2012-2014 Christian Dywan <[e-mail]>
+
+ This library is free software; you can redistribute it and/or
+ modify it under the terms of the GNU Lesser General Public
+ License as published by the Free Software Foundation; either
+ version 2.1 of the License, or (at your option) any later version.
+
+ See the file COPYING for the full license text.
+*/
+
+namespace HSTS {
+    public class Directive {
+        public Soup.Date? expires = null;
+        public bool sub_domains = false;
+
+        public Directive (bool include_sub_domains) {
+            expires = new Soup.Date.from_now (int.MAX);
+            sub_domains = include_sub_domains;
+        }
+
+        public Directive.from_header (string header) {
+            var param_list = Soup.header_parse_param_list (header);
+            if (param_list == null)
+                return;
+
+            string? max_age = param_list.lookup ("max-age");
+            if (max_age != null)
+                expires = new Soup.Date.from_now (max_age.to_int ());
+            if ("includeSubDomains" in header)
+                sub_domains = true;
+            Soup.header_free_param_list (param_list);
+        }
+
+        public bool is_valid () {
+            return expires != null && !expires.is_past ();
+        }
+    }
+
+    public class Whitelist : Object {
+        HashTable<string, Directive> whitelist;
+        string preset = "/etc/xdg/midori/hsts";
+        File config;
+
+        public Whitelist () {
+            whitelist = new HashTable<string, Directive> (str_hash, str_equal);
+            read_cache.begin (File.new_for_path (preset));
+            // FIXME: config = File.new_for_path (Midori.Paths.get_config_filename_for_writing ("hsts"));
+            string config_dir = Path.build_path (Path.DIR_SEPARATOR_S,
+                Environment.get_user_config_dir (), "midori");
+            config = File.new_for_path (Path.build_path (Path.DIR_SEPARATOR_S, config_dir, "hsts"));
+            read_cache.begin (config);
+        }
+
+        async void read_cache (File file) {
+            try {
+                var stream = new DataInputStream (yield file.read_async ());
+                do {
+                    string? line = yield stream.read_line_async ();
+                    if (line == null)
+                        break;
+                    string[] parts = line.split (" ", 2);
+                    if (parts[0] == null || parts[1] == null)
+                        break;
+                    var directive = new Directive.from_header (parts[1]);
+                    if (directive.is_valid ())
+                        append_to_whitelist (parts[0], directive);
+                } while (true);
+            } catch (IOError.NOT_FOUND exist_error) {
+                /* It's no error if no cache file exists */
+            } catch (Error error) {
+                warning ("Failed to read cache %s: %s", file.get_path (), error.message);
+            }
+        }
+
+        public bool should_secure_host (string host) {
+            Directive? directive = whitelist.lookup (host);
+            if (directive == null)
+                directive = whitelist.lookup ("*." + host);
+            return directive != null && directive.is_valid ();
+        }
+
+        void append_to_whitelist (string host, Directive directive) {
+            whitelist.insert (host, directive);
+            if (directive.sub_domains)
+                whitelist.insert ("*." + host, directive);
+        }
+
+        async void append_to_cache (string host, string header) {
+            /* FIXME: Don't write in private browsing */
+
+            try {
+                var stream = yield config.append_to_async (FileCreateFlags.NONE);
+                yield stream.write_async ((host + " " + header + "\n").data);
+                yield stream.flush_async ();
+            }
+            catch (Error error) {
+                critical ("Failed to update %s: %s", config.get_path (), error.message);
+            }
+        }
+
+        public Directive? strict_transport_security_handled (string host, Soup.MessageHeaders headers) {
+            unowned string? hsts = headers.get_one ("Strict-Transport-Security");
+            if (hsts == null)
+                return null;
+
+            var directive = new Directive.from_header (hsts);
+            if (directive.is_valid ()) {
+                append_to_whitelist (host, directive);
+                append_to_cache.begin (host, hsts);
+            }
+            return directive;
+        }
+
+    }
+
+    public class Extension : Object {
+        Whitelist whitelist;
+        bool debug = false;
+
+        public Extension (WebKit.WebExtension extension) {
+            if (strcmp (Environment.get_variable ("MIDORI_DEBUG"), "hsts") == 0)
+                debug = true;
+            whitelist = new Whitelist ();
+            extension.page_created.connect (page_created);
+        }
+
+        void page_created (WebKit.WebPage page) {
+            page.send_request.connect (send_request);
+        }
+
+        bool send_request (WebKit.URIRequest request, WebKit.URIResponse? redirect) {
+            Soup.MessageHeaders? headers = request.get_http_headers ();
+            if (headers == null || !request.uri.contains ("/"))
+                return false;
+
+            string host = request.uri.split ("/")[2];
+            if (whitelist.should_secure_host (host)) {
+                request.uri = request.uri.replace ("http://", "https://");
+                if (debug)
+                    stdout.printf ("HSTS: Enforce %s\n", host);
+            } else if (request.uri.has_prefix ("http://")) {
+                var directive = whitelist.strict_transport_security_handled (host, headers);
+                if (debug)
+                    stdout.printf ("HSTS: %s valid? %s\n",
+                        host, directive != null ? directive.is_valid ().to_string () : "/");
+            }
+            return false;
+        }
+    }
+}
+
+HSTS.Extension? hsts;
+// public void webkit_web_extension_initialize_with_user_data (WebKit.WebExtension extension, Variant user_data) {
+public void webkit_web_extension_initialize (WebKit.WebExtension extension) {
+    // FIXME: Midori.Paths.init_with_user_data (user_data);
+    hsts = new HSTS.Extension (extension);
+}
+
+void hsts_directive () {
+    HSTS.Directive d;
+    d = new HSTS.Directive.from_header ("max-age=31536000");
+    assert (d.is_valid () && !d.sub_domains);
+    d = new HSTS.Directive.from_header ("max-age=15768000 ; includeSubDomains");
+    assert (d.is_valid () && d.sub_domains);
+
+    /* Invalid */
+    d = new HSTS.Directive.from_header ("");
+    assert (!d.is_valid () && !d.sub_domains);
+    d = new HSTS.Directive.from_header ("includeSubDomains");
+    assert (!d.is_valid () && d.sub_domains);
+}
+
+public void extension_test () {
+    Test.add_func ("/hsts/directive", hsts_directive);
+}
+

=== added directory 'ipc'
=== modified file 'katze/midori-paths.vala'
--- katze/midori-paths.vala	2015-06-11 22:33:48 +0000
+++ katze/midori-paths.vala	2015-06-13 13:38:06 +0000
@@ -99,6 +99,7 @@
 
         public static void init (RuntimeMode new_mode, string? config) {
             assert (mode == RuntimeMode.UNDEFINED);
+            assert (exec_path != null);
             assert (new_mode != RuntimeMode.UNDEFINED);
             mode = new_mode;
             if (mode == RuntimeMode.PORTABLE || mode == RuntimeMode.PRIVATE)
@@ -148,16 +149,19 @@
                 tmp_dir = get_runtime_dir ();
             }
 #if HAVE_WEBKIT2
+            var context = WebKit.WebContext.get_default ();
+            context.set_web_extensions_directory (get_lib_path (PACKAGE_NAME));
+            context.initialize_web_extensions.connect (() => {
+                /* TODO: Pass user data */
+                context.set_web_extensions_initialization_user_data ("");
+            });
             if (cache_dir != null) {
-                /* Cache and extension dir MUST be set no later than here to work */
-                WebKit.WebContext.get_default ().set_web_extensions_directory (
-                    Path.build_path (Path.DIR_SEPARATOR_S, cache_dir, "wk2ext"));
-                WebKit.WebContext.get_default ().set_disk_cache_directory (
+                context.set_disk_cache_directory (
                     Path.build_path (Path.DIR_SEPARATOR_S, cache_dir, "web"));
             }
 
             if (config_dir != null) {
-                var cookie_manager = WebKit.WebContext.get_default ().get_cookie_manager ();
+                var cookie_manager = context.get_cookie_manager ();
                 cookie_manager.set_persistent_storage (Path.build_filename (config_dir, "cookies.db"),
                     WebKit.CookiePersistentStorage.SQLITE);
             }
@@ -165,7 +169,7 @@
             if (user_data_dir != null) {
                 string folder = Path.build_filename (user_data_dir, "webkit", "icondatabase");
 #if HAVE_WEBKIT2
-                WebKit.WebContext.get_default ().set_favicon_database_directory (folder);
+                context.set_favicon_database_directory (folder);
 #else
                 WebKit.get_favicon_database ().set_path (folder);
 #endif

=== modified file 'midori/main.c'
--- midori/main.c	2014-04-23 03:34:23 +0000
+++ midori/main.c	2015-06-13 13:38:06 +0000
@@ -62,6 +62,7 @@
     gchar* config;
     gboolean private;
     gboolean portable;
+    gchar* test;
     gboolean plain;
     gboolean diagnostic_dialog = FALSE;
     gboolean debug = FALSE;
@@ -85,6 +86,8 @@
        { "portable", 'P', 0, G_OPTION_ARG_NONE, &portable,
        N_("Portable mode, all runtime files are stored in one place"), NULL },
        #endif
+       { "test", 't', 0, G_OPTION_ARG_STRING, &test,
+       N_("Run unit tests for the specified extension"), NULL },
        { "plain", '\0', 0, G_OPTION_ARG_NONE, &plain,
        N_("Plain GTK+ window with WebKit, akin to GtkLauncher"), NULL },
        { "diagnostic-dialog", 'd', 0, G_OPTION_ARG_NONE, &diagnostic_dialog,
@@ -118,6 +121,7 @@
     config = NULL;
     private = FALSE;
     portable = FALSE;
+    test = NULL;
     plain = FALSE;
     run = FALSE;
     snapshot = NULL;
@@ -287,6 +291,52 @@
         return 0;
     }
 
+    if (test != NULL)
+    {
+        g_assert (g_module_supported ());
+
+        GModule* module = g_module_open (test, G_MODULE_BIND_LOCAL);
+        if (module == NULL)
+            g_error (_("Failed to load %s."), test);
+
+        midori_test_init (&argc, &argv);
+
+        typedef void (*extension_test_func)(void);
+        extension_test_func extension_test;
+        /* Midori.Extension */
+        typedef GObject* (*extension_init_func)(void);
+        extension_init_func extension_init;
+        /* WebKit.WebExtension */
+        typedef void (*webkit_web_extension_initialize_func)(GObject* extension);
+        webkit_web_extension_initialize_func web_extension_init;
+
+        if (g_module_symbol (module, "extension_init",
+                             (gpointer) &extension_init))
+        {
+            /* It's fine to conditionally return no extension */
+            if (!(extension_init ()))
+                return 0;
+            midori_paths_init (MIDORI_RUNTIME_MODE_NORMAL, NULL);
+
+            /* Not all extensions have unit tests :-( */
+            if (!g_module_symbol (module, "extension_test", (gpointer) &extension_test))
+                return 0;
+        }
+        else if (g_module_symbol (module, "webkit_web_extension_initialize",
+                                  (gpointer) &web_extension_init)
+              || g_module_symbol (module, "webkit_web_extension_initialize_with_user_data",
+                                  (gpointer) &web_extension_init))
+        {
+            if (!g_module_symbol (module, "extension_test", (gpointer) &extension_test))
+                g_error (_("%s doesn't provide unit tests."), test);
+        }
+        else
+            g_error (_("%s doesn't look like a Midori extension."), test);
+
+        extension_test ();
+        return g_test_run ();
+    }
+
     if (plain)
     {
         GtkWidget* window = gtk_window_new (GTK_WINDOW_TOPLEVEL);