Merge lp:~zorba-coders/zorba/fixing_issues_FOTS_driver into lp:zorba

[Sorin Marian Nasoi]

- implemented support for <assert-xml ignore-prefixes='true'>; as a result 2 more test cases pass.
- fixed 'qischema016'
- added bug number for test cases 'qischema040' and 'qischema040a'
- added correct validation type for the source documents
- in 'assert-xml' we now first try to see if deep-equal is true, then we try to canonicalize (fixed 4 test cases)
- document URL resolving is now done using URL resolver. Fixed 6 test cases.

- implemented support for <assert-xml ignore-prefixes='true'>; as a result 2 more test cases pass.
- fixed 'qischema016'
- added bug number for test cases 'qischema040' and 'qischema040a'
- added correct validation type for the source documents
- in 'assert-xml' we now first try to see if deep-equal is true, then we try to canonicalize (fixed 4 test cases)
- document URL resolving is now done using URL resolver. Fixed 6 test cases.

[Sorin Marian Nasoi (sorin.marian.nasoi)]

review: Approve

[Zorba Build Bot (zorba-buildbot)]

Validation queue starting for merge proposal.
Log at: http://zorbatest.lambda.nu:8080/remotequeue/fixing_issues_FOTS_driver-2013-03-25T17-38-55.092Z/log.html

[Zorba Build Bot (zorba-buildbot)]

Validation queue job fixing_issues_FOTS_driver-2013-03-25T17-38-55.092Z is finished. The final status was:

All tests succeeded!

[Zorba Build Bot (zorba-buildbot)]

Voting does not meet specified criteria. Required: Approve > 1, Disapprove < 1, Needs Fixing < 1, Pending < 1, Needs Information < 1, Resubmit < 1. Got: 1 Approve.

[Sorin Marian Nasoi (sorin.marian.nasoi)]

review: Approve

[Zorba Build Bot (zorba-buildbot)]

Validation queue starting for merge proposal.
Log at: http://zorbatest.lambda.nu:8080/remotequeue/fixing_issues_FOTS_driver-2013-03-26T16-29-45.841Z/log.html

[Zorba Build Bot (zorba-buildbot)]

Validation queue job fixing_issues_FOTS_driver-2013-03-26T16-29-45.841Z is finished. The final status was:

All tests succeeded!

[Zorba Build Bot (zorba-buildbot)]

Voting does not meet specified criteria. Required: Approve > 1, Disapprove < 1, Needs Fixing < 1, Pending < 1, Needs Information < 1, Resubmit < 1. Got: 1 Approve.

[Chris Hillery (ceejatec)]

I don't think it's correct to try deep-equal() for <assert-xml> test cases. Certainly this doesn't match with the description of that assertion at http://dev.w3.org/2011/QT3-test-suite/catalog-schema.html#elem_assert-xml .

Can you tell me why you did that? I'm also confused how this could have corrected 4 test cases.

The remainder of the changes seem fine.

review: Needs Information

[Sorin Marian Nasoi (sorin.marian.nasoi)]

> I don't think it's correct to try deep-equal() for <assert-xml> test cases.
> Certainly this doesn't match with the description of that assertion at
> http://dev.w3.org/2011/QT3-test-suite/catalog-schema.html#elem_assert-xml.
"As an alternative to canonicalizing, the results may be compared using the fn:deep-equal() function."

> Can you tell me why you did that? I'm also confused how this could have
> corrected 4 test cases.
These fixed 4 test cases where the parsing of the serialized XML content with libxml2 raised errors.
Although parsing the same exact thing with parse-xml did not. This is how, for these 4 cases,
fn:deep-equal(parse-xml($x1), parse-xml($x2)) returned correct result while
$x1 eq $x2 raised errors where $x1, $x2 are xml:canonicalize(....)

Take for example this example "namespaceDecl-24" from "prod-NamespaceDecl":

This works:
import module namespace zorba-xml = "http://www.zorba-xquery.com/modules/xml#2.1";
import schema namespace output = "http://www.w3.org/2010/xslt-xquery-serialization";

 declare variable $serParamXml :=
  <output:serialization-parameters>
    <output:method value="xml" />
    <output:indent value="no" />
    <output:omit-xml-declaration value="yes" />
  </output:serialization-parameters>;

let $content := <abc:someElement xmlns:abc="mailto:<email address hidden>">some content</abc:someElement>
return
parse-xml(fn:serialize($content, $serParamXml))

while this fails:
import module namespace zorba-xml =
  "http://www.zorba-xquery.com/modules/xml#2.1";

import schema namespace output =
  "http://www.w3.org/2010/xslt-xquery-serialization";

 declare variable $serParamXml :=
  <output:serialization-parameters>
    <output:method value="xml" />
    <output:indent value="no" />
    <output:omit-xml-declaration value="yes" />
  </output:serialization-parameters>;

let $content := <abc:someElement xmlns:abc="mailto:<email address hidden>">some content</abc:someElement>
return
zorba-xml:canonicalize(fn:serialize($content, $serParamXml))

Error:

C14N error : Relative namespace UR is invalid here : mailto
C14N error : Internal error : checking for relative namespaces
C14N error : Internal error : processing docs children list
C14N error : Internal error : saving doc to output buffer
</home/spungi/work/zorba/repo/tmp/bump_FOTS_snapshot/build/URI_PATH/com/zorba-xquery/www/modules/xml_2.xq>:444,3: dynamic error [err:FOCZ0001]: invalid content passed to x:canonicalize(): "<abc:someElement xmlns:abc="mailto:<email address hidden>">some content</abc:someElement>"

[Chris Hillery (ceejatec)]

review: Approve

[Zorba Build Bot (zorba-buildbot)]

Validation queue starting for merge proposal.
Log at: http://zorbatest.lambda.nu:8080/remotequeue/fixing_issues_FOTS_driver-2013-03-27T09-03-54.253Z/log.html

[Zorba Build Bot (zorba-buildbot)]

Validation queue job fixing_issues_FOTS_driver-2013-03-27T09-03-54.253Z is finished. The final status was:

All tests succeeded!

=== modified file 'test/fots/CMakeLists.txt'
--- test/fots/CMakeLists.txt	2013-03-23 15:23:57 +0000
+++ test/fots/CMakeLists.txt	2013-03-26 16:30:37 +0000
@@ -441,11 +441,7 @@
 EXPECTED_FOTS_FAILURE (prod-CastableExpr cbcl-castable-duration-002 0)
 EXPECTED_FOTS_FAILURE (prod-CastableExpr cbcl-castable-yearMonthDuration-001 0)
 EXPECTED_FOTS_FAILURE (prod-CastableExpr cbcl-castable-yearMonthDuration-002 0)
-EXPECTED_FOTS_FAILURE (prod-CompAttrConstructor K2-ComputeConAttr-51 0)
-EXPECTED_FOTS_FAILURE (prod-CompAttrConstructor K2-ComputeConAttr-53 0)
-EXPECTED_FOTS_FAILURE (prod-CompAttrConstructor K2-ComputeConAttr-58 0)
 EXPECTED_FOTS_FAILURE (prod-CompDocConstructor K2-ConDocNode-2 0)
-EXPECTED_FOTS_FAILURE (prod-CompNamespaceConstructor nscons-039 0)
 EXPECTED_FOTS_FAILURE (prod-ContextItemDecl contextDecl-016 0)
 EXPECTED_FOTS_FAILURE (prod-ContextItemDecl contextDecl-017 0)
 EXPECTED_FOTS_FAILURE (prod-ContextItemDecl contextDecl-047 0)
@@ -515,7 +511,6 @@
 EXPECTED_FOTS_FAILURE (prod-ModuleImport modules-pub-priv-30 0)
 EXPECTED_FOTS_FAILURE (prod-ModuleImport modules-pub-priv-34 0)
 EXPECTED_FOTS_FAILURE (prod-ModuleImport cbcl-module-004 0)
-EXPECTED_FOTS_FAILURE (prod-NamespaceDecl namespaceDecl-24 0)
 EXPECTED_FOTS_FAILURE (prod-OptionDecl K-OptionDeclarationProlog-1b 0)
 EXPECTED_FOTS_FAILURE (prod-OptionDecl.serialization Serialization-003 0)
 EXPECTED_FOTS_FAILURE (prod-OptionDecl.serialization Serialization-004 0)
@@ -527,8 +522,8 @@
 EXPECTED_FOTS_FAILURE (prod-OptionDecl.serialization Serialization-030 0)
 EXPECTED_FOTS_FAILURE (prod-OptionDecl.serialization Serialization-031 0)
 EXPECTED_FOTS_FAILURE (prod-OptionDecl.serialization Serialization-032 0)
-EXPECTED_FOTS_FAILURE (prod-OrderByClause orderBy20 0)
-EXPECTED_FOTS_FAILURE (prod-OrderByClause orderBy21 0)
+EXPECTED_FOTS_FAILURE (prod-OrderByClause orderBy20 1156249)
+EXPECTED_FOTS_FAILURE (prod-OrderByClause orderBy21 1156249)
 EXPECTED_FOTS_FAILURE (prod-OrderByClause orderBy40 0)
 EXPECTED_FOTS_FAILURE (prod-OrderByClause orderBy41 0)
 EXPECTED_FOTS_FAILURE (prod-OrderByClause orderBy49 0)
@@ -564,11 +559,9 @@
 EXPECTED_FOTS_FAILURE (prod-RequireProhibitFeature require-all-optional-features-2-ns2 0)
 EXPECTED_FOTS_FAILURE (prod-RequireProhibitFeature prohibit-all-optional-features-1 0)
 EXPECTED_FOTS_FAILURE (prod-RequireProhibitFeature require-all-optional-features-7-s 0)
-EXPECTED_FOTS_FAILURE (prod-SchemaImport qischema016 0)
 EXPECTED_FOTS_FAILURE (prod-SchemaImport qischema031 0)
-EXPECTED_FOTS_FAILURE (prod-SchemaImport qischema032c 0)
-EXPECTED_FOTS_FAILURE (prod-SchemaImport qischema040 0)
-EXPECTED_FOTS_FAILURE (prod-SchemaImport qischema040a 0)
+EXPECTED_FOTS_FAILURE (prod-SchemaImport qischema040 21339)
+EXPECTED_FOTS_FAILURE (prod-SchemaImport qischema040a 21339)
 EXPECTED_FOTS_FAILURE (prod-SchemaImport substitution-001 0)
 EXPECTED_FOTS_FAILURE (prod-SchemaImport substitution-002 0)
 EXPECTED_FOTS_FAILURE (prod-SchemaImport substitution-003 0)
@@ -631,10 +624,7 @@
 EXPECTED_FOTS_FAILURE (fn-fold-left fold-left-009 0)
 EXPECTED_FOTS_FAILURE (fn-function-lookup fn-function-lookup-018 0)
 EXPECTED_FOTS_FAILURE (fn-function-lookup fn-function-lookup-020 0)
-EXPECTED_FOTS_FAILURE (fn-function-lookup fn-function-lookup-022 0)
-EXPECTED_FOTS_FAILURE (fn-function-lookup fn-function-lookup-024 0)
 EXPECTED_FOTS_FAILURE (fn-function-lookup fn-function-lookup-062 0)
-EXPECTED_FOTS_FAILURE (fn-function-lookup fn-function-lookup-366 0)
 EXPECTED_FOTS_FAILURE (fn-function-lookup fn-function-lookup-370 0)
 EXPECTED_FOTS_FAILURE (fn-function-lookup fn-function-lookup-372 0)
 EXPECTED_FOTS_FAILURE (fn-function-lookup fn-function-lookup-374 0)
@@ -671,10 +661,7 @@
 EXPECTED_FOTS_FAILURE (prod-InstanceofExpr instanceof140 0)
 EXPECTED_FOTS_FAILURE (prod-NamedFunctionRef function-literal-018 0)
 EXPECTED_FOTS_FAILURE (prod-NamedFunctionRef function-literal-020 0)
-EXPECTED_FOTS_FAILURE (prod-NamedFunctionRef function-literal-022 0)
-EXPECTED_FOTS_FAILURE (prod-NamedFunctionRef function-literal-024 0)
 EXPECTED_FOTS_FAILURE (prod-NamedFunctionRef function-literal-062 0)
-EXPECTED_FOTS_FAILURE (prod-NamedFunctionRef function-literal-366 0)
 EXPECTED_FOTS_FAILURE (prod-NamedFunctionRef function-literal-370 0)
 EXPECTED_FOTS_FAILURE (prod-NamedFunctionRef function-literal-372 0)
 EXPECTED_FOTS_FAILURE (prod-NamedFunctionRef function-literal-374 0)

=== modified file 'test/fots_driver/environment.xq'
--- test/fots_driver/environment.xq	2013-03-21 18:43:40 +0000
+++ test/fots_driver/environment.xq	2013-03-26 16:30:37 +0000
@@ -424,28 +424,32 @@
   $needsDTDValidation as xs:boolean
 ) as xs:string
 {
-  let $ciURI := resolve-uri($env/fots:source[@role = "."]/@file, $envBaseURI)
+  let $ciURI := if(exists($env/fots:source[@role = "."]/@uri))
+                then xs:string($env/fots:source[@role = "."]/@uri)
+                else resolve-uri($env/fots:source[@role = "."]/@file, $envBaseURI)
+  let $needsSchemaValidation := exists($env/fots:source/@validation)
   return
-  if (empty($env/fots:source[@validation = "strict"]))
-  then
-  {
-    if($needsDTDValidation)
+  if($needsDTDValidation)
     then concat('variable $contextItem := zorba-xml:parse(fn:unparsed-text("',
                 $ciURI,
                 '"),<opt:options><opt:DTD-validate/></opt:options> );')
-    else concat('variable $contextItem := doc("', $ciURI, '");')
-  }
-  else 
-    string-join
-    (
+  else if(empty($env/fots:source[@role = "."]/@uri) and
+          not($needsSchemaValidation))
+    then concat('variable $contextItem := doc("', $ciURI, '");')
+  else
+  {
+    string-join(
     (
     "&#xA;",
-
     "variable $contextItemQuery := xqxq:prepare-main-module",
     "(",
     "'",
-    env:get-schema-import($env),
-    concat('validate { doc("', $ciURI, '")', " }"),
+    if ($needsSchemaValidation) then env:get-schema-import($env) else (),
+    if ($needsSchemaValidation)
+    then concat('validate ', xs:string($env/fots:source/@validation),' { ',
+                concat(' doc("', $ciURI, '")'),
+                " }")
+    else concat(' doc("', $ciURI, '")'),
     "',",
     "(), mapper:uri-mapper#2",
     ");",
@@ -453,8 +457,8 @@
     "variable $contextItem := xqxq:evaluate($contextItemQuery);"
     )
     ,
-    "&#xA;"
-    )
+    "&#xA;")
+  }
 };
 
 
@@ -653,11 +657,23 @@
   $testSetBaseURI as xs:anyURI
 ) as xs:string?
 {
-  let $envSchema := $env/fots:schema,
-      $tcSchema := $case/fots:environment/fots:schema,
-      $schemas := ($envSchema, $tcSchema)
+  let $envSchema := $env/fots:schema
+  let $tcSchema := $case/fots:environment/fots:schema
+  (:
+  Schema documents are identified in the environment in a similar way to source
+  documents. The role attribute indicates whether the schema is imported into
+  the query, or used for source document validation.
+  :)
+  let $schemas := ($envSchema, $tcSchema)
+  let $envSource := for $s in $env/fots:source
+                    where exists($s/@uri)
+                    return $s
+  let $tcSource := for $s in $case/fots:environment/fots:source
+                   where exists($s/@uri)
+                   return $s
+  let $sources := ($envSource, $tcSource)
   return
-    if (empty($schemas))
+    if (empty($schemas) and empty($sources))
     then ()
     else string-join(
             ("declare function mapper:uri-mapper($namespace as xs:string, $entity as xs:string) {",
@@ -680,6 +696,24 @@
                       "    default return ()")),
                       "&#xA;")
     else (),
+    if (exists($sources))
+    then string-join(("case ''",
+                     "  return switch($namespace)",
+                    (for $source in $envSource
+                     return concat("    case '",
+                                   data($source/@uri),
+                                   "' return '",
+                                   resolve-uri($source/@file, $envBaseURI),
+                                   "'"),
+                      for $source in $tcSource
+                      return concat("    case '",
+                                   data($source/@uri),
+                                   "' return '",
+                                   resolve-uri($source/@file, $testSetBaseURI),
+                                   "'"),
+                      "    default return ()")),
+                      "&#xA;")
+    else (),
     "default return ()","};"),
     "&#xA;")
 };

=== modified file 'test/fots_driver/evaluate.xq'
--- test/fots_driver/evaluate.xq	2013-03-21 18:31:19 +0000
+++ test/fots_driver/evaluate.xq	2013-03-26 16:30:37 +0000
@@ -486,13 +486,25 @@
 )
 {
   try {
-    let $actualResult   := zorba-xml:canonicalize(concat('<root>', fn:serialize($result, $util:serParamXml), '</root>'))
-    let $expectedResult := zorba-xml:canonicalize(concat('<root>', util:get-value($expResult, $baseURI, "assert-xml"), '</root>'))
-    
+    let $actualResult   := concat('<root>', fn:serialize($result, $util:serParamXml), '</root>')
+    let $expectedResult := concat('<root>', util:get-value($expResult, $baseURI, "assert-xml"), '</root>')
     return
-      if ($actualResult eq $expectedResult)
-      then ()
-      else concat("'assert-xml' returned: result &#xA;'", $actualResult, "'&#xA; is different from the expected result &#xA;'", $expectedResult,"'&#xA;")
+    (: first try to see if deep-equal is true:)
+    if(deep-equal(parse-xml($actualResult), parse-xml($expectedResult)))
+    then ()
+    else (: second try to canonicalize :)
+    {
+      let $canActualResult   := zorba-xml:canonicalize($actualResult)
+      let $canExpectedResult := zorba-xml:canonicalize($expectedResult)
+      return
+        if(empty($expResult[@ignore-prefixes='true']) and
+           ($canActualResult eq $canExpectedResult))
+        then ()
+        else if(exists($expResult[@ignore-prefixes='true']) and
+              fn:parse-xml($canActualResult) eq fn:parse-xml($canExpectedResult)) (: the namespace prefixes are ignored in this comparison :)
+        then ()
+        else concat("'assert-xml' returned: result &#xA;'", $canActualResult, "'&#xA; is different from the expected result &#xA;'", $canExpectedResult,"'&#xA;")
+    }
   } catch * {
     concat("'assert-xml' returned: fail with error ",
             $err:code, " : ", $err:description)