Merge lp:~henrik-lochmann/goobi-presentation/bug-878893 into lp:~slub.team/goobi-presentation/old-bzr-trunk

[Henrik Lochmann]

Unmerged revisions

137. By Henrik Lochmann on 2012-11-14

- pageview plugin got a template with ###TEXT### marker and detects whether linked files are images or OCR files

136. By Henrik Lochmann on 2012-11-14

- indexes plain text for document pages
ATTENTION: this requires the new lucene field in schema.xml of solr: '<field name="text" type="standard" indexed="true" stored="true" multiValued="false" />'; provided on demand

135. By Henrik Lochmann on 2012-11-14

- new method getText(..) and new field $text, to retrieve and cache plain text information from linked OCR files (currently supported: ALTO and plain text)

134. By Henrik Lochmann on 2012-11-14

- configuration of OCR file groups in extension settings

=== modified file 'dlf/common/class.tx_dlf_document.php'
--- dlf/common/class.tx_dlf_document.php	2012-10-02 12:30:10 +0000
+++ dlf/common/class.tx_dlf_document.php	2012-11-19 15:34:20 +0000
@@ -265,6 +265,14 @@
 	protected $tableOfContentsLoaded = FALSE;
 
 	/**
+	 * This holds plain text, derived from linked OCR files.
+	 *
+	 * @var	array
+	 * @access protected
+	 */
+	protected $text = array();
+	
+	/**
 	 * This holds the document's thumbnail location.
 	 *
 	 * @var	string
@@ -1879,7 +1887,150 @@
 		return $this->smLinks;
 
 	}
-
+	
+	/**
+	 * This returns the passed page's plain text, in case any OCR information is stored in the underlying document.
+	 *
+	 * @access	protected
+	 * 
+	 * @param	integer		$page: The number of the page, plain text is requested for
+	 *
+	 * @return	string		The passed page's plain text
+	 */
+	public function getText($page) {
+	
+		// Validate passed value.
+		if (!isset($page) || !t3lib_div::testInt($page)) {
+
+			return '';
+		
+		}
+
+		if ($this->text[$page] === NULL) {
+			
+			$extConf = unserialize($GLOBALS['TYPO3_CONF_VARS']['EXT']['extConf'][$this->extKey]);
+			
+			$ocrGrps = t3lib_div::trimExplode(',', $extConf['fileGrpsOcr']);
+			
+			foreach ($ocrGrps as $ocrGrp) {
+
+				// Get file ID for current group.
+				$textFileId = $this->physicalPagesInfo[$this->physicalPages[$page]]['files'][strtolower($ocrGrp)];
+
+				if (empty($textFileId)) {
+					
+					if (TYPO3_DLOG) {
+					
+						t3lib_div::devLog('[tx_dlf_document->getText('.$page.')] No text information found for METS fileGrp "'.$ocrGrp.'"', $this->extKey, t3lib_div::SYSLOG_SEVERITY_INFO);
+					
+					}
+					
+					continue;
+					
+				}
+
+				// Get file URL for extracted file ID.
+				$textFile = $this->getFileLocation($textFileId);
+				
+				// Extract content depending on text file format.
+				$content = NULL;
+				
+				switch ($ocrGrp) {
+					
+					// For ALTO groups, we load the ALTO XML file, get string nodes and append their CONTENT attributes to the resulting text.
+					case 'ALTO':
+
+						// Turn off libxml's error logging.
+						$libxmlErrors = libxml_use_internal_errors(TRUE);
+						
+						// Load XML from file.
+						$xml = @simplexml_load_file($textFile);
+						
+						// Reset libxml's error logging.
+						libxml_use_internal_errors($libxmlErrors);
+						
+						if ($xml !== FALSE) {
+						
+							$xml->registerXPathNamespace('alto', 'http://www.loc.gov/standards/alto/');
+						
+							$altoContentNodes = $xml->xpath('//String/@CONTENT');
+							
+							// If we did not get a result w/o namespaces prefix, we try again with namespace prefix. 
+							if (empty($altoContentNodes)) {
+								
+								$altoContentNodes = $xml->xpath('//alto:String/@alto:CONTENT');
+							
+							}
+
+							// Iterate content nodes and concatenate content to result.
+							if (!empty($altoContentNodes)) {
+						
+								$content = '';
+								
+								foreach($altoContentNodes as $string) {
+										
+									$content .= ' '.(string) $string;
+										
+								}
+
+							}
+						
+						} else {
+						
+							if (TYPO3_DLOG) {
+						
+								t3lib_div::devLog('[tx_dlf_document->getText('.$page.')] Could not load XML file from "'.$textFile.'"', $this->extKey, t3lib_div::SYSLOG_SEVERITY_ERROR);
+						
+							}
+						
+						}
+						
+						break;
+						
+					case 'TXT':
+						
+						$content = file_get_contents($textFile);
+						
+						if (!$content) {
+							
+							$content = NULL;
+							
+							if (TYPO3_DLOG) {
+								
+								t3lib_div::devLog('[tx_dlf_document->getText('.$page.')] Could not load plain text file from "'.$textFile.'"', $this->extKey, t3lib_div::SYSLOG_SEVERITY_ERROR);
+								
+							}
+							
+						}
+					
+						break;
+				
+				}
+				
+				// If current OCR group brought a result, end iterating OCR groups and set result.
+				if ($content !== NULL) {
+					
+					$this->text[$page] = $content;
+					
+					break;
+					
+				}
+
+			}
+			
+		}
+		
+		// Remark text of page loaded. 
+		if ($this->text[$page] === NULL) {
+			
+			$this->text[$page] = '';
+			
+		}
+		
+		return $this->text[$page];
+
+	}
+	
 	/**
 	 * This returns the document's thumbnail location
 	 *

=== modified file 'dlf/common/class.tx_dlf_indexing.php'
--- dlf/common/class.tx_dlf_indexing.php	2012-10-02 07:02:46 +0000
+++ dlf/common/class.tx_dlf_indexing.php	2012-11-19 15:34:20 +0000
@@ -547,6 +547,9 @@
 
 				$solrDoc->setField('page', $logicalUnit['points']);
 
+				// Extract full text for entire document and add to solrDoc.
+				$solrDoc->setField('text', $doc->getText(intval($logicalUnit['points'])));
+
 			}
 
 			if ($logicalUnit['id'] == $doc->toplevelId) {

=== modified file 'dlf/ext_conf_template.txt'
--- dlf/ext_conf_template.txt	2012-09-13 17:53:13 +0000
+++ dlf/ext_conf_template.txt	2012-11-19 15:34:20 +0000
@@ -22,6 +22,9 @@
 # cat=Files; type=string; label=LLL:EXT:dlf/locallang.xml:config.fileGrpThumbs
 fileGrpThumbs = THUMBS
 
+# cat=Files; type=string; label=LLL:EXT:dlf/locallang.xml:config.fileGrpsOcr
+fileGrpsOcr = ALTO,TXT
+
 # cat=Solr; type=user[EXT:dlf/hooks/class.tx_dlf_em.php:tx_dlf_em->checkSolrConnection]; label=LLL:EXT:dlf/locallang.xml:config.solrConnect
 solrConnect = 0
 

=== modified file 'dlf/locallang.xml'
--- dlf/locallang.xml	2012-10-02 07:02:46 +0000
+++ dlf/locallang.xml	2012-11-19 15:34:20 +0000
@@ -130,6 +130,7 @@
 			<label index="config.unhideOnIndex">Unhide indexed documents?: Should hidden documents be unhidden when re-indexing them? (default is "FALSE")</label>
 			<label index="config.fileGrps">METS fileGrps: comma-separated list of all @USE attribute values (default is "THUMBS,MIN,DEFAULT,MAX")</label>
 			<label index="config.fileGrpThumbs">Thumbnail fileGrp: @USE attribute value (default is "THUMBS")</label>
+			<label index="config.fileGrpsOcr">OCR fileGrp: comma-separated list of all @USE attribute values for OCR processing, prioritized in order (default is "ALTO,TXT")</label>
 			<label index="config.solrConnect">Solr Connection</label>
 			<label index="config.solrHost">Solr Server Host: (default is "localhost")</label>
 			<label index="config.solrPort">Solr Server Port: (default is "8180")</label>
@@ -292,6 +293,7 @@
 			<label index="config.unhideOnIndex">Indexierte Dokumente einblenden?: Sollen ausgeblendete Dokumente bei der erneuten Indexierung wieder eingeblendet werden? (Standard ist "FALSE")</label>
 			<label index="config.fileGrps">METS fileGrps: Komma-getrennte Liste aller @USE Attributwerte (Standard ist "THUMBS,MIN,DEFAULT,MAX")</label>
 			<label index="config.fileGrpThumbs">Vorschau fileGrp: @USE Attributwert der Vorschaubilder (Standard ist "THUMBS")</label>
+			<label index="config.fileGrpsOcr">OCR fileGrp: Komma-getrennte Liste aller @USE Attributwerte für die OCR Verarbeitung, priorisiert nach Reihenfolge (Standard ist "ALTO,TXT")</label>
 			<label index="config.solrConnect">Solr Verbindung</label>
 			<label index="config.solrHost">Solr Server Host: (Standard ist "localhost")</label>
 			<label index="config.solrPort">Solr Server Port: (Standard ist "8180")</label>

=== modified file 'dlf/plugins/pageview/class.tx_dlf_pageview.php'
--- dlf/plugins/pageview/class.tx_dlf_pageview.php	2012-09-14 16:54:56 +0000
+++ dlf/plugins/pageview/class.tx_dlf_pageview.php	2012-11-19 15:34:20 +0000
@@ -108,10 +108,17 @@
 				$fileGrpUrl = $this->doc->getFileLocation($this->doc->physicalPagesInfo[$this->doc->physicalPages[$page]]['files'][$fileGrp]);
 
 				// Check file's existence.
-				$headers = @get_headers($fileGrpUrl);
+				$headers = @get_headers($fileGrpUrl, 1);
 
 				if (is_array($headers) && preg_match('/^HTTP\\/\\d+\\.\\d+\\s+2\\d\\d\\s+.*$/', $headers[0])) {
 
+					// skip non-image files 
+					if (!strstr($headers['Content-Type'], 'image/')) {
+
+						continue;
+
+					}
+					
 					$fileGrpSize = getimagesize($fileGrpUrl);
 
 					$imageData[] = array (
@@ -170,8 +177,21 @@
 
 		}
 
+		// Load template file.
+		if (!empty($this->conf['templateFile'])) {
+		
+			$this->template = $this->cObj->getSubpart($this->cObj->fileResource($this->conf['templateFile']), '###TEMPLATE###');
+		
+		} else {
+		
+			$this->template = $this->cObj->getSubpart($this->cObj->fileResource('EXT:dlf/plugins/pageview/template.tmpl'), '###TEMPLATE###');
+		
+		}
+
+		$content = $this->cObj->substituteMarkerArray($this->template, array( '###TEXT###' => $this->doc->getText($this->piVars['page']) ));
+
 		$content .= $this->showViewer();
-
+		
 		return $this->pi_wrapInBaseClass($content);
 
 	}

=== modified file 'dlf/plugins/pageview/flexform.xml'
--- dlf/plugins/pageview/flexform.xml	2011-03-09 15:36:27 +0000
+++ dlf/plugins/pageview/flexform.xml	2012-11-19 15:34:20 +0000
@@ -35,6 +35,21 @@
 							</config>
 						</TCEforms>
 					</excludeOther>
+					<templateFile>
+						<TCEforms>
+							<exclude>1</exclude>
+							<label>LLL:EXT:dlf/plugins/pageview/locallang.xml:tt_content.pi_flexform.templateFile</label>
+							<config>
+								<type>group</type>
+								<internal_type>file_reference</internal_type>
+								<allowed>tmpl,tpl,html,htm,txt</allowed>
+								<size>1</size>
+								<maxitems>1</maxitems>
+								<minitems>0</minitems>
+								<disable_controls>upload</disable_controls>
+							</config>
+						</TCEforms>
+					</templateFile>
 				</el>
 			</ROOT>
 		</sDEF>

=== modified file 'dlf/plugins/pageview/locallang.xml'
--- dlf/plugins/pageview/locallang.xml	2011-03-09 15:36:27 +0000
+++ dlf/plugins/pageview/locallang.xml	2012-11-19 15:34:20 +0000
@@ -8,10 +8,12 @@
 		<languageKey index="default" type="array">
 			<label index="tt_content.pi_flexform.sheet_general">Options</label>
 			<label index="tt_content.pi_flexform.excludeOther">Show only documents from the selected page</label>
+			<label index="tt_content.pi_flexform.templateFile">Template file</label>
 		</languageKey>
 		<languageKey index="de" type="array">
 			<label index="tt_content.pi_flexform.sheet_general">Einstellungen</label>
 			<label index="tt_content.pi_flexform.excludeOther">Nur Dokumente der ausgewählten Seite anzeigen</label>
+			<label index="tt_content.pi_flexform.templateFile">HTML-Template</label>
 		</languageKey>
 	</data>
 </T3locallang>
\ No newline at end of file

=== added file 'dlf/plugins/pageview/template.tmpl'
--- dlf/plugins/pageview/template.tmpl	1970-01-01 00:00:00 +0000
+++ dlf/plugins/pageview/template.tmpl	2012-11-19 15:34:20 +0000
@@ -0,0 +1,3 @@
+<!-- ###TEMPLATE### -->
+<p>###TEXT###</p>
+<!-- ###TEMPLATE### -->
\ No newline at end of file